d0144f3586
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
194 lines
7.3 KiB
Go
194 lines
7.3 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// wikiPagesTestDDL is a minimal SQLite-compatible subset of the
|
|
// production wiki_pages DDL (migrations/versioned/000037_wiki_and_indexing.up.sql).
|
|
// JSONB is stored as TEXT in SQLite; the StringArray Scan/Value pair
|
|
// handles the JSON round-trip unchanged.
|
|
const wikiPagesTestDDL = `
|
|
CREATE TABLE IF NOT EXISTS wiki_pages (
|
|
id VARCHAR(36) PRIMARY KEY,
|
|
tenant_id INTEGER NOT NULL,
|
|
knowledge_base_id VARCHAR(36) NOT NULL,
|
|
slug VARCHAR(255) NOT NULL,
|
|
title VARCHAR(512) NOT NULL DEFAULT '',
|
|
page_type VARCHAR(32) NOT NULL DEFAULT 'summary',
|
|
status VARCHAR(32) NOT NULL DEFAULT 'published',
|
|
content TEXT NOT NULL DEFAULT '',
|
|
summary TEXT NOT NULL DEFAULT '',
|
|
source_refs TEXT DEFAULT '[]',
|
|
chunk_refs TEXT DEFAULT '[]',
|
|
in_links TEXT DEFAULT '[]',
|
|
out_links TEXT DEFAULT '[]',
|
|
page_metadata TEXT DEFAULT '{}',
|
|
aliases TEXT DEFAULT '[]',
|
|
version INTEGER NOT NULL DEFAULT 1,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
deleted_at DATETIME
|
|
);
|
|
`
|
|
|
|
func setupWikiPagesTestDB(t *testing.T) *gorm.DB {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
|
require.NoError(t, err)
|
|
require.NoError(t, db.Exec(wikiPagesTestDDL).Error)
|
|
return db
|
|
}
|
|
|
|
// makeWikiPage builds a minimal WikiPage suitable for insert. Title is
|
|
// derived from the slug so ORDER BY title ASC yields a predictable
|
|
// test ordering without callers having to spell out both fields.
|
|
func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage {
|
|
title := slug
|
|
if idx := strings.LastIndex(slug, "/"); idx >= 0 {
|
|
title = slug[idx+1:]
|
|
}
|
|
return &types.WikiPage{
|
|
ID: uuid.New().String(),
|
|
TenantID: 1,
|
|
KnowledgeBaseID: kbID,
|
|
Slug: slug,
|
|
Title: title,
|
|
PageType: pageType,
|
|
Status: status,
|
|
Content: "body of " + slug,
|
|
Summary: "summary of " + slug,
|
|
Version: 1,
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
|
|
// that the index-view projection only emits the slug/title/summary
|
|
// triples and respects the archived-status filter. This is the whole
|
|
// point of splitting the method off ListByType — the index reader must
|
|
// not pay for TEXT content transport.
|
|
func TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived(t *testing.T) {
|
|
db := setupWikiPagesTestDB(t)
|
|
repo := NewWikiPageRepository(db)
|
|
ctx := context.Background()
|
|
|
|
pages := []*types.WikiPage{
|
|
makeWikiPage("kb-a", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
|
|
makeWikiPage("kb-a", "entity/beta", types.WikiPageTypeEntity, types.WikiPageStatusDraft),
|
|
makeWikiPage("kb-a", "entity/gamma", types.WikiPageTypeEntity, types.WikiPageStatusArchived),
|
|
makeWikiPage("kb-a", "concept/delta", types.WikiPageTypeConcept, types.WikiPageStatusPublished),
|
|
makeWikiPage("kb-other", "entity/leaked", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
|
|
}
|
|
for _, p := range pages {
|
|
require.NoError(t, repo.Create(ctx, p))
|
|
}
|
|
|
|
entries, total, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 50, 0)
|
|
require.NoError(t, err)
|
|
|
|
// Archived is excluded; sibling KB is excluded; everything else
|
|
// surfaces regardless of draft/published status (the index shows
|
|
// both so admins notice newly-drafted pages).
|
|
assert.Equal(t, int64(2), total)
|
|
require.Len(t, entries, 2)
|
|
|
|
// ORDER BY title ASC => alpha, beta.
|
|
assert.Equal(t, "entity/alpha", entries[0].Slug)
|
|
assert.Equal(t, "alpha", entries[0].Title)
|
|
assert.Equal(t, "summary of entity/alpha", entries[0].Summary)
|
|
assert.Equal(t, "entity/beta", entries[1].Slug)
|
|
}
|
|
|
|
// TestListByTypeLight_Pagination walks the type list using offsets and
|
|
// asserts the count stays stable regardless of where in the list we
|
|
// are — the index handler uses total to render "showing N of M".
|
|
func TestListByTypeLight_Pagination(t *testing.T) {
|
|
db := setupWikiPagesTestDB(t)
|
|
repo := NewWikiPageRepository(db)
|
|
ctx := context.Background()
|
|
|
|
for _, s := range []string{"entity/a", "entity/b", "entity/c", "entity/d", "entity/e"} {
|
|
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-a", s, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
|
|
}
|
|
|
|
page1, total1, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 0)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(5), total1)
|
|
require.Len(t, page1, 2)
|
|
assert.Equal(t, "entity/a", page1[0].Slug)
|
|
assert.Equal(t, "entity/b", page1[1].Slug)
|
|
|
|
page2, total2, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 2)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(5), total2, "total should be stable across pages")
|
|
require.Len(t, page2, 2)
|
|
assert.Equal(t, "entity/c", page2[0].Slug)
|
|
assert.Equal(t, "entity/d", page2[1].Slug)
|
|
|
|
page3, total3, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 4)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(5), total3)
|
|
require.Len(t, page3, 1)
|
|
assert.Equal(t, "entity/e", page3[0].Slug)
|
|
|
|
// Offset past the end yields an empty list, not an error — the
|
|
// handler relies on this to short-circuit pagination tails.
|
|
page4, total4, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 10)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(5), total4)
|
|
assert.Empty(t, page4)
|
|
}
|
|
|
|
// TestListByTypeLight_EmptyType_ReturnsZero exercises the "no rows"
|
|
// short-circuit. We skip the SELECT entirely when count is zero, so
|
|
// a KB with no pages of a type shouldn't burn a pointless query.
|
|
func TestListByTypeLight_EmptyType_ReturnsZero(t *testing.T) {
|
|
db := setupWikiPagesTestDB(t)
|
|
repo := NewWikiPageRepository(db)
|
|
ctx := context.Background()
|
|
|
|
entries, total, err := repo.ListByTypeLight(ctx, "kb-empty", types.WikiPageTypeSynthesis, 50, 0)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(0), total)
|
|
assert.Empty(t, entries)
|
|
}
|
|
|
|
// TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't
|
|
// want a client passing limit=100000 and forcing the DB to return a
|
|
// multi-MB response.
|
|
func TestListByTypeLight_ClampsLimit(t *testing.T) {
|
|
db := setupWikiPagesTestDB(t)
|
|
repo := NewWikiPageRepository(db)
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 250; i++ {
|
|
slug := "entity/bulk-"
|
|
// Use a stable, zero-padded suffix so title ordering is
|
|
// deterministic for the cap assertion below.
|
|
slug += string(rune('a'+(i/26)%26)) + string(rune('a'+i%26))
|
|
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-cap", slug, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
|
|
}
|
|
|
|
// limit=0 falls back to the default of 50.
|
|
defaultEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 0, 0)
|
|
require.NoError(t, err)
|
|
assert.Len(t, defaultEntries, 50)
|
|
|
|
// limit=5000 clamps to 200.
|
|
clampedEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 5000, 0)
|
|
require.NoError(t, err)
|
|
assert.LessOrEqual(t, len(clampedEntries), 200)
|
|
}
|