Files
Enterprise-Knowledge-Base/internal/application/service/wiki_ingest_retry_test.go
T
wizardchen d0144f3586 perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.

Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.

Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.

Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.

Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.

- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
  renderIndexOverviewForAgent output, isTransientLLMError classifier

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 21:48:52 +08:00

100 lines
4.0 KiB
Go

package service
import (
"context"
"errors"
"testing"
)
// TestIsTransientLLMError_HTTPStatuses covers the status codes we know
// the generic REST chat provider bubbles up as "API request failed with
// status NNN: ...". The status-code arm of the classifier is the one
// that actually fires for the 504 gateway timeouts the upstream LLM
// vendors throw most often.
func TestIsTransientLLMError_HTTPStatuses(t *testing.T) {
ctx := context.Background()
cases := []struct {
msg string
want bool
}{
{"API request failed with status 504: Remote error, timeout with 60", true},
{"API request failed with status 503: service unavailable", true},
{"API request failed with status 502: bad gateway", true},
{"API request failed with status 500: internal server error", true},
{"API request failed with status 429: rate limited", true},
{"API request failed with status 408: request timeout", true},
{"API request failed with status 520: Web server returned unknown", true},
{"API request failed with status 524: timeout (Cloudflare)", true},
// 4xx permanent: a retry will just fail the same way.
{"API request failed with status 400: bad request", false},
{"API request failed with status 401: unauthorized", false},
{"API request failed with status 403: forbidden (quota exhausted)", false},
{"API request failed with status 404: model not found", false},
}
for _, tc := range cases {
got := isTransientLLMError(ctx, errors.New(tc.msg))
if got != tc.want {
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
}
}
}
// TestIsTransientLLMError_TransportSubstrings covers providers (and Go
// stdlib errors) that surface transient conditions as free-form text
// without a numeric status code — DNS blips, TLS handshake failures,
// broken pipes during an in-flight body read, etc.
func TestIsTransientLLMError_TransportSubstrings(t *testing.T) {
ctx := context.Background()
cases := []struct {
msg string
want bool
}{
{"send request: dial tcp: lookup api.example.com: no such host", true},
{"send request: read tcp 10.0.0.1: i/o timeout", true},
{"send request: Post \"...\": context deadline exceeded", true},
{"send request: connection reset by peer", true},
{"send request: connection refused", true},
{"send request: broken pipe", true},
{"send request: tls handshake timeout", true},
{"send request: unexpected EOF", true},
// Not transient: the error is actionable (missing model id,
// malformed tool call, etc.) — retry is just wasted budget.
{"model not configured for tool use", false},
{"invalid JSON in tool arguments", false},
{"parse template: syntax error", false},
}
for _, tc := range cases {
got := isTransientLLMError(ctx, errors.New(tc.msg))
if got != tc.want {
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
}
}
}
// TestIsTransientLLMError_AbortsWhenParentCtxDone ensures a cancelled
// outer context short-circuits the retry decision. Without this guard
// an ingest task that was interrupted (asynq shutdown, deletion race,
// parent timeout) would keep retrying the LLM during its teardown
// window and waste the remainder of the backoff budget.
func TestIsTransientLLMError_AbortsWhenParentCtxDone(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
// This exact error would normally be classified transient (see
// TestIsTransientLLMError_HTTPStatuses), but a cancelled ctx must
// override that so the task can unwind promptly.
err := errors.New("API request failed with status 504: Remote error")
if isTransientLLMError(ctx, err) {
t.Fatal("cancelled ctx should short-circuit to non-transient")
}
}
// TestIsTransientLLMError_NilError is a sanity check — nil should never
// be treated as retryable, otherwise callers that forget to handle the
// happy path would spin forever.
func TestIsTransientLLMError_NilError(t *testing.T) {
if isTransientLLMError(context.Background(), nil) {
t.Fatal("nil error should not be transient")
}
}