Files
Enterprise-Knowledge-Base/internal/application/repository/wiki_page_test.go
T
wizardchen d0144f3586 perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.

Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.

Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.

Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.

Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.

- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
  renderIndexOverviewForAgent output, isTransientLLMError classifier

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 21:48:52 +08:00

194 lines
7.3 KiB
Go

package repository
import (
"context"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// wikiPagesTestDDL is a minimal SQLite-compatible subset of the
// production wiki_pages DDL (migrations/versioned/000037_wiki_and_indexing.up.sql).
// JSONB is stored as TEXT in SQLite; the StringArray Scan/Value pair
// handles the JSON round-trip unchanged.
const wikiPagesTestDDL = `
CREATE TABLE IF NOT EXISTS wiki_pages (
id VARCHAR(36) PRIMARY KEY,
tenant_id INTEGER NOT NULL,
knowledge_base_id VARCHAR(36) NOT NULL,
slug VARCHAR(255) NOT NULL,
title VARCHAR(512) NOT NULL DEFAULT '',
page_type VARCHAR(32) NOT NULL DEFAULT 'summary',
status VARCHAR(32) NOT NULL DEFAULT 'published',
content TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
source_refs TEXT DEFAULT '[]',
chunk_refs TEXT DEFAULT '[]',
in_links TEXT DEFAULT '[]',
out_links TEXT DEFAULT '[]',
page_metadata TEXT DEFAULT '{}',
aliases TEXT DEFAULT '[]',
version INTEGER NOT NULL DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
`
func setupWikiPagesTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(wikiPagesTestDDL).Error)
return db
}
// makeWikiPage builds a minimal WikiPage suitable for insert. Title is
// derived from the slug so ORDER BY title ASC yields a predictable
// test ordering without callers having to spell out both fields.
func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage {
title := slug
if idx := strings.LastIndex(slug, "/"); idx >= 0 {
title = slug[idx+1:]
}
return &types.WikiPage{
ID: uuid.New().String(),
TenantID: 1,
KnowledgeBaseID: kbID,
Slug: slug,
Title: title,
PageType: pageType,
Status: status,
Content: "body of " + slug,
Summary: "summary of " + slug,
Version: 1,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
}
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
// that the index-view projection only emits the slug/title/summary
// triples and respects the archived-status filter. This is the whole
// point of splitting the method off ListByType — the index reader must
// not pay for TEXT content transport.
func TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
pages := []*types.WikiPage{
makeWikiPage("kb-a", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
makeWikiPage("kb-a", "entity/beta", types.WikiPageTypeEntity, types.WikiPageStatusDraft),
makeWikiPage("kb-a", "entity/gamma", types.WikiPageTypeEntity, types.WikiPageStatusArchived),
makeWikiPage("kb-a", "concept/delta", types.WikiPageTypeConcept, types.WikiPageStatusPublished),
makeWikiPage("kb-other", "entity/leaked", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
}
for _, p := range pages {
require.NoError(t, repo.Create(ctx, p))
}
entries, total, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 50, 0)
require.NoError(t, err)
// Archived is excluded; sibling KB is excluded; everything else
// surfaces regardless of draft/published status (the index shows
// both so admins notice newly-drafted pages).
assert.Equal(t, int64(2), total)
require.Len(t, entries, 2)
// ORDER BY title ASC => alpha, beta.
assert.Equal(t, "entity/alpha", entries[0].Slug)
assert.Equal(t, "alpha", entries[0].Title)
assert.Equal(t, "summary of entity/alpha", entries[0].Summary)
assert.Equal(t, "entity/beta", entries[1].Slug)
}
// TestListByTypeLight_Pagination walks the type list using offsets and
// asserts the count stays stable regardless of where in the list we
// are — the index handler uses total to render "showing N of M".
func TestListByTypeLight_Pagination(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
for _, s := range []string{"entity/a", "entity/b", "entity/c", "entity/d", "entity/e"} {
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-a", s, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
}
page1, total1, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 0)
require.NoError(t, err)
assert.Equal(t, int64(5), total1)
require.Len(t, page1, 2)
assert.Equal(t, "entity/a", page1[0].Slug)
assert.Equal(t, "entity/b", page1[1].Slug)
page2, total2, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 2)
require.NoError(t, err)
assert.Equal(t, int64(5), total2, "total should be stable across pages")
require.Len(t, page2, 2)
assert.Equal(t, "entity/c", page2[0].Slug)
assert.Equal(t, "entity/d", page2[1].Slug)
page3, total3, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 4)
require.NoError(t, err)
assert.Equal(t, int64(5), total3)
require.Len(t, page3, 1)
assert.Equal(t, "entity/e", page3[0].Slug)
// Offset past the end yields an empty list, not an error — the
// handler relies on this to short-circuit pagination tails.
page4, total4, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 10)
require.NoError(t, err)
assert.Equal(t, int64(5), total4)
assert.Empty(t, page4)
}
// TestListByTypeLight_EmptyType_ReturnsZero exercises the "no rows"
// short-circuit. We skip the SELECT entirely when count is zero, so
// a KB with no pages of a type shouldn't burn a pointless query.
func TestListByTypeLight_EmptyType_ReturnsZero(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
entries, total, err := repo.ListByTypeLight(ctx, "kb-empty", types.WikiPageTypeSynthesis, 50, 0)
require.NoError(t, err)
assert.Equal(t, int64(0), total)
assert.Empty(t, entries)
}
// TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't
// want a client passing limit=100000 and forcing the DB to return a
// multi-MB response.
func TestListByTypeLight_ClampsLimit(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
for i := 0; i < 250; i++ {
slug := "entity/bulk-"
// Use a stable, zero-padded suffix so title ordering is
// deterministic for the cap assertion below.
slug += string(rune('a'+(i/26)%26)) + string(rune('a'+i%26))
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-cap", slug, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
}
// limit=0 falls back to the default of 50.
defaultEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 0, 0)
require.NoError(t, err)
assert.Len(t, defaultEntries, 50)
// limit=5000 clamps to 200.
clampedEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 5000, 0)
require.NoError(t, err)
assert.LessOrEqual(t, len(clampedEntries), 200)
}