d0144f3586
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
251 lines
7.6 KiB
TypeScript
251 lines
7.6 KiB
TypeScript
import { get, post, put, del } from "../../utils/request";
|
|
|
|
// encodeSlugPath encodes each segment of a hierarchical wiki slug (e.g.
|
|
// "foo/bar baz?") so the URL is safe while preserving the "/" separators
|
|
// between segments. Using encodeURIComponent on the whole slug would also
|
|
// escape the "/" and break hierarchical routing on the backend.
|
|
function encodeSlugPath(slug: string): string {
|
|
return slug.split("/").map(encodeURIComponent).join("/");
|
|
}
|
|
|
|
// Wiki Page Types
|
|
export interface WikiPage {
|
|
id: string;
|
|
tenant_id: number;
|
|
knowledge_base_id: string;
|
|
slug: string;
|
|
title: string;
|
|
page_type: string;
|
|
status: string;
|
|
content: string;
|
|
summary: string;
|
|
aliases: string[];
|
|
source_refs: string[];
|
|
in_links: string[];
|
|
out_links: string[];
|
|
page_metadata: Record<string, any>;
|
|
version: number;
|
|
created_at: string;
|
|
updated_at: string;
|
|
}
|
|
|
|
export interface WikiPageListResponse {
|
|
pages: WikiPage[];
|
|
total: number;
|
|
page: number;
|
|
page_size: number;
|
|
total_pages: number;
|
|
}
|
|
|
|
export interface WikiGraphMeta {
|
|
mode: 'overview' | 'ego' | string;
|
|
total: number;
|
|
returned: number;
|
|
truncated: boolean;
|
|
center?: string;
|
|
depth?: number;
|
|
}
|
|
|
|
export interface WikiGraphData {
|
|
nodes: { slug: string; title: string; page_type: string; link_count: number }[];
|
|
edges: { source: string; target: string }[];
|
|
meta: WikiGraphMeta;
|
|
}
|
|
|
|
export interface WikiStats {
|
|
total_pages: number;
|
|
pages_by_type: Record<string, number>;
|
|
total_links: number;
|
|
orphan_count: number;
|
|
recent_updates: WikiPage[];
|
|
pending_tasks: number;
|
|
pending_issues: number;
|
|
is_active: boolean;
|
|
}
|
|
|
|
export interface WikiPageIssue {
|
|
id: string;
|
|
tenant_id: number;
|
|
knowledge_base_id: string;
|
|
slug: string;
|
|
issue_type: string;
|
|
description: string;
|
|
suspected_knowledge_ids: string[];
|
|
status: string;
|
|
reported_by: string;
|
|
created_at: string;
|
|
updated_at: string;
|
|
}
|
|
|
|
// Wiki API Functions
|
|
export function listWikiPages(kbId: string, params?: {
|
|
page_type?: string;
|
|
status?: string;
|
|
query?: string;
|
|
page?: number;
|
|
page_size?: number;
|
|
sort_by?: string;
|
|
sort_order?: string;
|
|
}) {
|
|
const query = new URLSearchParams();
|
|
if (params) {
|
|
Object.entries(params).forEach(([key, value]) => {
|
|
if (value !== undefined && value !== '') {
|
|
query.set(key, String(value));
|
|
}
|
|
});
|
|
}
|
|
const qs = query.toString();
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages${qs ? '?' + qs : ''}`);
|
|
}
|
|
|
|
export function createWikiPage(kbId: string, data: Partial<WikiPage>) {
|
|
return post(`/api/v1/knowledgebase/${kbId}/wiki/pages`, data);
|
|
}
|
|
|
|
export function getWikiPage(kbId: string, slug: string) {
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
|
|
}
|
|
|
|
export function updateWikiPage(kbId: string, slug: string, data: Partial<WikiPage>) {
|
|
return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`, data);
|
|
}
|
|
|
|
export function deleteWikiPage(kbId: string, slug: string) {
|
|
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
|
|
}
|
|
|
|
export interface WikiIndexEntryDTO {
|
|
slug: string;
|
|
title: string;
|
|
summary: string;
|
|
}
|
|
|
|
export interface WikiIndexGroup {
|
|
type: string;
|
|
total: number;
|
|
items: WikiIndexEntryDTO[];
|
|
next_cursor?: string;
|
|
}
|
|
|
|
export interface WikiIndexResponse {
|
|
intro: string;
|
|
version: number;
|
|
groups: WikiIndexGroup[];
|
|
}
|
|
|
|
// getWikiIndex fetches the structured index view for a wiki KB. The
|
|
// backend replaced the legacy "markdown blob of intro + directory" with
|
|
// { intro, groups } so a 40k-page wiki no longer round-trips multiple
|
|
// megabytes on every index open. Pass `types` to restrict which
|
|
// page_type buckets come back; `limit` bounds the per-group window;
|
|
// `cursor` resumes from a previous response.
|
|
export function getWikiIndex(
|
|
kbId: string,
|
|
params?: { types?: string[]; limit?: number; cursor?: string },
|
|
) {
|
|
const query = new URLSearchParams();
|
|
if (params) {
|
|
if (params.types && params.types.length > 0) query.set('types', params.types.join(','));
|
|
if (params.limit !== undefined) query.set('limit', String(params.limit));
|
|
if (params.cursor) query.set('cursor', params.cursor);
|
|
}
|
|
const qs = query.toString();
|
|
const suffix = qs ? `?${qs}` : '';
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/index${suffix}`);
|
|
}
|
|
|
|
export interface WikiLogPageRef {
|
|
slug: string;
|
|
title?: string;
|
|
}
|
|
|
|
export interface WikiLogEntry {
|
|
id: number;
|
|
tenant_id: number;
|
|
knowledge_base_id: string;
|
|
action: string;
|
|
knowledge_id: string;
|
|
doc_title: string;
|
|
summary: string;
|
|
// Each ref carries both slug (for navigation) and title (captured at
|
|
// ingest time for display). Legacy rows written before the title
|
|
// column was added surface as refs with an empty title; render falls
|
|
// back to the slug in that case.
|
|
pages_affected: WikiLogPageRef[];
|
|
created_at: string;
|
|
}
|
|
|
|
export interface WikiLogListResponse {
|
|
entries: WikiLogEntry[];
|
|
next_cursor?: string;
|
|
}
|
|
|
|
// getWikiLog fetches a page of wiki operation events (newest first). Pass the
|
|
// `next_cursor` from the previous response back as `cursor` to load more;
|
|
// an empty / missing `next_cursor` signals end-of-feed. `limit` is clamped
|
|
// server-side to [1, 200] and defaults to 50.
|
|
export function getWikiLog(kbId: string, params?: { cursor?: string; limit?: number }) {
|
|
const query = new URLSearchParams();
|
|
if (params) {
|
|
if (params.cursor) query.set('cursor', params.cursor);
|
|
if (params.limit !== undefined) query.set('limit', String(params.limit));
|
|
}
|
|
const qs = query.toString();
|
|
const suffix = qs ? `?${qs}` : '';
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/log${suffix}`);
|
|
}
|
|
|
|
export interface WikiGraphQueryParams {
|
|
mode?: 'overview' | 'ego';
|
|
center?: string;
|
|
depth?: number;
|
|
types?: string[];
|
|
limit?: number;
|
|
}
|
|
|
|
// getWikiGraph fetches a slice of the wiki link graph. Without params the
|
|
// backend returns the top-500 most-connected pages (overview mode). Pass
|
|
// `mode: 'ego', center: <slug>` to drill into a specific page's neighborhood.
|
|
// For knowledge bases with tens of thousands of pages the overview cap is
|
|
// what prevents the browser from choking on a 30MB payload / 100k SVG nodes.
|
|
export function getWikiGraph(kbId: string, params?: WikiGraphQueryParams) {
|
|
const query = new URLSearchParams();
|
|
if (params) {
|
|
if (params.mode) query.set('mode', params.mode);
|
|
if (params.center) query.set('center', params.center);
|
|
if (params.depth !== undefined) query.set('depth', String(params.depth));
|
|
if (params.limit !== undefined) query.set('limit', String(params.limit));
|
|
if (params.types && params.types.length > 0) {
|
|
query.set('types', params.types.join(','));
|
|
}
|
|
}
|
|
const qs = query.toString();
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/graph${qs ? '?' + qs : ''}`);
|
|
}
|
|
|
|
export function getWikiStats(kbId: string) {
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/stats`);
|
|
}
|
|
|
|
export function searchWikiPages(kbId: string, q: string, limit?: number) {
|
|
const params = new URLSearchParams({ q });
|
|
if (limit) params.set('limit', String(limit));
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/search?${params.toString()}`);
|
|
}
|
|
|
|
export function listWikiIssues(kbId: string, slug?: string, status?: string) {
|
|
const params = new URLSearchParams();
|
|
if (slug) params.set('slug', slug);
|
|
if (status) params.set('status', status);
|
|
return get(`/api/v1/knowledgebase/${kbId}/wiki/issues?${params.toString()}`);
|
|
}
|
|
|
|
export function updateWikiIssueStatus(kbId: string, issueId: string, status: string) {
|
|
return put(`/api/v1/knowledgebase/${kbId}/wiki/issues/${issueId}/status`, { status });
|
|
}
|
|
|
|
export function rebuildWikiLinks(kbId: string) {
|
|
return post(`/api/v1/knowledgebase/${kbId}/wiki/rebuild-links`, {});
|
|
}
|