d0144f3586
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
100 lines
4.0 KiB
Go
100 lines
4.0 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
)
|
|
|
|
// TestIsTransientLLMError_HTTPStatuses covers the status codes we know
|
|
// the generic REST chat provider bubbles up as "API request failed with
|
|
// status NNN: ...". The status-code arm of the classifier is the one
|
|
// that actually fires for the 504 gateway timeouts the upstream LLM
|
|
// vendors throw most often.
|
|
func TestIsTransientLLMError_HTTPStatuses(t *testing.T) {
|
|
ctx := context.Background()
|
|
cases := []struct {
|
|
msg string
|
|
want bool
|
|
}{
|
|
{"API request failed with status 504: Remote error, timeout with 60", true},
|
|
{"API request failed with status 503: service unavailable", true},
|
|
{"API request failed with status 502: bad gateway", true},
|
|
{"API request failed with status 500: internal server error", true},
|
|
{"API request failed with status 429: rate limited", true},
|
|
{"API request failed with status 408: request timeout", true},
|
|
{"API request failed with status 520: Web server returned unknown", true},
|
|
{"API request failed with status 524: timeout (Cloudflare)", true},
|
|
// 4xx permanent: a retry will just fail the same way.
|
|
{"API request failed with status 400: bad request", false},
|
|
{"API request failed with status 401: unauthorized", false},
|
|
{"API request failed with status 403: forbidden (quota exhausted)", false},
|
|
{"API request failed with status 404: model not found", false},
|
|
}
|
|
for _, tc := range cases {
|
|
got := isTransientLLMError(ctx, errors.New(tc.msg))
|
|
if got != tc.want {
|
|
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestIsTransientLLMError_TransportSubstrings covers providers (and Go
|
|
// stdlib errors) that surface transient conditions as free-form text
|
|
// without a numeric status code — DNS blips, TLS handshake failures,
|
|
// broken pipes during an in-flight body read, etc.
|
|
func TestIsTransientLLMError_TransportSubstrings(t *testing.T) {
|
|
ctx := context.Background()
|
|
cases := []struct {
|
|
msg string
|
|
want bool
|
|
}{
|
|
{"send request: dial tcp: lookup api.example.com: no such host", true},
|
|
{"send request: read tcp 10.0.0.1: i/o timeout", true},
|
|
{"send request: Post \"...\": context deadline exceeded", true},
|
|
{"send request: connection reset by peer", true},
|
|
{"send request: connection refused", true},
|
|
{"send request: broken pipe", true},
|
|
{"send request: tls handshake timeout", true},
|
|
{"send request: unexpected EOF", true},
|
|
// Not transient: the error is actionable (missing model id,
|
|
// malformed tool call, etc.) — retry is just wasted budget.
|
|
{"model not configured for tool use", false},
|
|
{"invalid JSON in tool arguments", false},
|
|
{"parse template: syntax error", false},
|
|
}
|
|
for _, tc := range cases {
|
|
got := isTransientLLMError(ctx, errors.New(tc.msg))
|
|
if got != tc.want {
|
|
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestIsTransientLLMError_AbortsWhenParentCtxDone ensures a cancelled
|
|
// outer context short-circuits the retry decision. Without this guard
|
|
// an ingest task that was interrupted (asynq shutdown, deletion race,
|
|
// parent timeout) would keep retrying the LLM during its teardown
|
|
// window and waste the remainder of the backoff budget.
|
|
func TestIsTransientLLMError_AbortsWhenParentCtxDone(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
// This exact error would normally be classified transient (see
|
|
// TestIsTransientLLMError_HTTPStatuses), but a cancelled ctx must
|
|
// override that so the task can unwind promptly.
|
|
err := errors.New("API request failed with status 504: Remote error")
|
|
if isTransientLLMError(ctx, err) {
|
|
t.Fatal("cancelled ctx should short-circuit to non-transient")
|
|
}
|
|
}
|
|
|
|
// TestIsTransientLLMError_NilError is a sanity check — nil should never
|
|
// be treated as retryable, otherwise callers that forget to handle the
|
|
// happy path would spin forever.
|
|
func TestIsTransientLLMError_NilError(t *testing.T) {
|
|
if isTransientLLMError(context.Background(), nil) {
|
|
t.Fatal("nil error should not be transient")
|
|
}
|
|
}
|