Files
Enterprise-Knowledge-Base/internal/application/repository/wiki_log_entry_test.go
T
wizardchen d0144f3586 perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.

Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.

Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.

Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.

Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.

- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
  renderIndexOverviewForAgent output, isTransientLLMError classifier

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 21:48:52 +08:00

220 lines
8.1 KiB
Go

package repository
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// wikiLogEntriesTestDDL mirrors migrations/versioned/000040_wiki_log_entries.up.sql
// but uses SQLite-compatible types so we can exercise the repo layer without
// spinning up Postgres. INTEGER PRIMARY KEY AUTOINCREMENT gives us the same
// monotonically-increasing ID semantics the production schema relies on for
// cursor pagination.
const wikiLogEntriesTestDDL = `
CREATE TABLE IF NOT EXISTS wiki_log_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id INTEGER NOT NULL,
knowledge_base_id VARCHAR(36) NOT NULL,
action VARCHAR(32) NOT NULL,
knowledge_id VARCHAR(36) NOT NULL DEFAULT '',
doc_title TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
pages_affected TEXT NOT NULL DEFAULT '[]',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
`
func setupWikiLogTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(wikiLogEntriesTestDDL).Error)
return db
}
func makeLogEntry(kbID, action, knowledgeID, title string, pages ...string) *types.WikiLogEntry {
refs := make(types.WikiLogPageRefs, 0, len(pages))
for _, slug := range pages {
// Tests only care about round-tripping; title is derived from
// the slug so assertions don't have to spell out both fields.
refs = append(refs, types.WikiLogPageRef{Slug: slug, Title: "title-of-" + slug})
}
return &types.WikiLogEntry{
TenantID: 1,
KnowledgeBaseID: kbID,
Action: action,
KnowledgeID: knowledgeID,
DocTitle: title,
Summary: "summary for " + title,
PagesAffected: refs,
}
}
// TestWikiLogEntryRepository_AppendBatch_WritesAllRows verifies a batch
// insert persists every entry with a monotonic ID and respects the caller-
// provided column values.
func TestWikiLogEntryRepository_AppendBatch_WritesAllRows(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
entries := []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1", "entity/a", "summary/k1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-a", "retract", "k3", "Doc 3", "concept/c"),
}
require.NoError(t, repo.AppendBatch(ctx, entries))
// Every row should have a non-zero ID assigned by the DB, and IDs
// should increase in insertion order so the cursor pagination logic
// can rely on them.
for i := range entries {
assert.NotZero(t, entries[i].ID, "row %d should have a DB-assigned ID", i)
if i > 0 {
assert.Greater(t, entries[i].ID, entries[i-1].ID,
"AppendBatch should preserve insertion order")
}
}
// Round-trip sanity: select back and confirm payload was preserved.
var got []types.WikiLogEntry
require.NoError(t, db.Order("id ASC").Find(&got).Error)
require.Len(t, got, len(entries))
assert.Equal(t, "ingest", got[0].Action)
assert.Equal(t, "Doc 1", got[0].DocTitle)
assert.Equal(t, types.WikiLogPageRefs{
{Slug: "entity/a", Title: "title-of-entity/a"},
{Slug: "summary/k1", Title: "title-of-summary/k1"},
}, got[0].PagesAffected)
assert.Equal(t, "retract", got[2].Action)
}
// TestWikiLogEntryRepository_AppendBatch_Empty asserts an empty batch is a
// cheap no-op — callers at the end of a wiki ingest pass can invoke it
// unconditionally without a DB round trip.
func TestWikiLogEntryRepository_AppendBatch_Empty(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
require.NoError(t, repo.AppendBatch(ctx, nil))
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{}))
var count int64
require.NoError(t, db.Model(&types.WikiLogEntry{}).Count(&count).Error)
assert.Equal(t, int64(0), count)
}
// TestWikiLogEntryRepository_List_CursorPagination walks the feed with a
// limit smaller than the population size and confirms:
// - newest entries surface first (id DESC order),
// - passing next_cursor returns the next batch strictly older than it,
// - end-of-feed is signalled by an empty cursor.
func TestWikiLogEntryRepository_List_CursorPagination(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
// 5 entries in one KB + a sibling row in another KB to ensure the
// WHERE filter actually narrows by KB.
entries := []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-a", "ingest", "k3", "Doc 3"),
makeLogEntry("kb-a", "ingest", "k4", "Doc 4"),
makeLogEntry("kb-a", "retract", "k5", "Doc 5"),
}
require.NoError(t, repo.AppendBatch(ctx, entries))
other := makeLogEntry("kb-other", "ingest", "kx", "Other doc")
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{other}))
// First page: latest 2 of kb-a, newest first.
page1, cursor1, err := repo.List(ctx, "kb-a", "", 2)
require.NoError(t, err)
require.Len(t, page1, 2)
assert.Equal(t, "Doc 5", page1[0].DocTitle)
assert.Equal(t, "Doc 4", page1[1].DocTitle)
assert.NotEmpty(t, cursor1, "should emit next_cursor when the page is full")
// Second page continues strictly older than cursor1.
page2, cursor2, err := repo.List(ctx, "kb-a", cursor1, 2)
require.NoError(t, err)
require.Len(t, page2, 2)
assert.Equal(t, "Doc 3", page2[0].DocTitle)
assert.Equal(t, "Doc 2", page2[1].DocTitle)
assert.NotEmpty(t, cursor2)
// Third (final) page: one row remains, so cursor is empty — the repo
// must NOT emit a cursor for a short page, otherwise the frontend
// would fire a doomed follow-up request.
page3, cursor3, err := repo.List(ctx, "kb-a", cursor2, 2)
require.NoError(t, err)
require.Len(t, page3, 1)
assert.Equal(t, "Doc 1", page3[0].DocTitle)
assert.Empty(t, cursor3, "short page should signal end-of-feed with an empty cursor")
// Sibling KB row must NOT leak into kb-a's feed.
for _, p := range append(append(page1, page2...), page3...) {
assert.Equal(t, "kb-a", p.KnowledgeBaseID)
}
}
// TestWikiLogEntryRepository_List_InvalidCursor rejects a non-numeric
// cursor rather than silently returning the newest page, so clients
// notice bugs before they turn into phantom paginations.
func TestWikiLogEntryRepository_List_InvalidCursor(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
_, _, err := repo.List(ctx, "kb-a", "not-a-number", 10)
assert.Error(t, err)
}
// TestWikiLogEntryRepository_DeleteByKB purges one KB's rows without
// touching others. Used during KB deletion so the log table does not
// accumulate orphans after the parent KB is gone.
func TestWikiLogEntryRepository_DeleteByKB(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-b", "ingest", "k3", "Doc 3"),
}))
require.NoError(t, repo.DeleteByKB(ctx, "kb-a"))
aEntries, _, err := repo.List(ctx, "kb-a", "", 100)
require.NoError(t, err)
assert.Empty(t, aEntries)
bEntries, _, err := repo.List(ctx, "kb-b", "", 100)
require.NoError(t, err)
assert.Len(t, bEntries, 1)
}
// TestWikiLogPageRefs_ScanAcceptsLegacyStringArray covers the backward-
// compat path in the Scan method. Any rows the ingest pipeline wrote
// before pages_affected gained a title field are stored as a plain JSON
// string array; the repo must decode them as slug-only refs so the log
// feed renders without a data migration.
func TestWikiLogPageRefs_ScanAcceptsLegacyStringArray(t *testing.T) {
var refs types.WikiLogPageRefs
require.NoError(t, refs.Scan([]byte(`["entity/acme","concept/rag"]`)))
require.Len(t, refs, 2)
assert.Equal(t, "entity/acme", refs[0].Slug)
assert.Empty(t, refs[0].Title, "legacy rows have no title until re-ingest overwrites them")
assert.Equal(t, "concept/rag", refs[1].Slug)
}