d0144f3586
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
93 lines
2.7 KiB
Go
93 lines
2.7 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// wikiLogEntryRepository implements interfaces.WikiLogEntryRepository.
|
|
type wikiLogEntryRepository struct {
|
|
db *gorm.DB
|
|
}
|
|
|
|
// NewWikiLogEntryRepository constructs a GORM-backed WikiLogEntryRepository.
|
|
func NewWikiLogEntryRepository(db *gorm.DB) interfaces.WikiLogEntryRepository {
|
|
return &wikiLogEntryRepository{db: db}
|
|
}
|
|
|
|
// AppendBatch inserts every entry in one statement. Empty batches are a
|
|
// no-op so callers can invoke unconditionally at the end of a wiki ingest
|
|
// batch without guarding against the "no events this round" case.
|
|
func (r *wikiLogEntryRepository) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error {
|
|
if len(entries) == 0 {
|
|
return nil
|
|
}
|
|
return r.db.WithContext(ctx).Create(&entries).Error
|
|
}
|
|
|
|
// List returns up to `limit` entries strictly older than `cursor`, newest
|
|
// first. `cursor` is the stringified ID of the oldest entry from the
|
|
// previous page; an empty string starts from the newest. Callers get back
|
|
// a nextCursor string to pass on the next request — empty means no more.
|
|
//
|
|
// `limit` is clamped to [1, 200]. Values outside that range are coerced
|
|
// silently to keep the handler simple.
|
|
func (r *wikiLogEntryRepository) List(
|
|
ctx context.Context,
|
|
kbID string,
|
|
cursor string,
|
|
limit int,
|
|
) ([]*types.WikiLogEntry, string, error) {
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
|
|
q := r.db.WithContext(ctx).
|
|
Where("knowledge_base_id = ?", kbID).
|
|
Order("id DESC").
|
|
Limit(limit)
|
|
|
|
if cursor != "" {
|
|
cursorID, err := strconv.ParseUint(cursor, 10, 64)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("invalid cursor %q: %w", cursor, err)
|
|
}
|
|
q = q.Where("id < ?", cursorID)
|
|
}
|
|
|
|
var entries []*types.WikiLogEntry
|
|
if err := q.Find(&entries).Error; err != nil {
|
|
return nil, "", err
|
|
}
|
|
|
|
nextCursor := ""
|
|
// Only emit a cursor when we actually filled the page — a short page
|
|
// is guaranteed to be the tail, so returning a cursor would just cause
|
|
// the frontend to fire a final empty request.
|
|
if len(entries) == limit {
|
|
nextCursor = strconv.FormatUint(entries[len(entries)-1].ID, 10)
|
|
}
|
|
return entries, nextCursor, nil
|
|
}
|
|
|
|
// DeleteByKB drops every log entry for a KB. No "affected rows" signal is
|
|
// surfaced — missing rows are a legitimate state (e.g., a KB that was
|
|
// created before wiki_log_entries existed) and not a failure condition.
|
|
func (r *wikiLogEntryRepository) DeleteByKB(ctx context.Context, kbID string) error {
|
|
if kbID == "" {
|
|
return errors.New("wiki log entries: empty kb id")
|
|
}
|
|
return r.db.WithContext(ctx).
|
|
Where("knowledge_base_id = ?", kbID).
|
|
Delete(&types.WikiLogEntry{}).Error
|
|
}
|