perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -19,9 +19,10 @@ import (
|
||||
|
||||
// WikiPageHandler handles HTTP requests for wiki page operations
|
||||
type WikiPageHandler struct {
|
||||
wikiService interfaces.WikiPageService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
lintService *service.WikiLintService
|
||||
wikiService interfaces.WikiPageService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
lintService *service.WikiLintService
|
||||
logEntryService interfaces.WikiLogEntryService
|
||||
}
|
||||
|
||||
// NewWikiPageHandler creates a new wiki page handler
|
||||
@@ -29,11 +30,13 @@ func NewWikiPageHandler(
|
||||
wikiService interfaces.WikiPageService,
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
lintService *service.WikiLintService,
|
||||
logEntryService interfaces.WikiLogEntryService,
|
||||
) *WikiPageHandler {
|
||||
return &WikiPageHandler{
|
||||
wikiService: wikiService,
|
||||
kbService: kbService,
|
||||
lintService: lintService,
|
||||
wikiService: wikiService,
|
||||
kbService: kbService,
|
||||
lintService: lintService,
|
||||
logEntryService: logEntryService,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,12 +277,19 @@ func (h *WikiPageHandler) DeletePage(c *gin.Context) {
|
||||
}
|
||||
|
||||
// GetIndex godoc
|
||||
// @Summary Get wiki index page
|
||||
// @Description Returns the wiki index page (creates default if not exists)
|
||||
// @Summary Get wiki index view
|
||||
// @Description Returns the wiki index as intro text plus per-type paginated
|
||||
// @Description directory groups. The heavy directory markdown that used to
|
||||
// @Description live in wiki_pages.content was replaced with this structured
|
||||
// @Description response so a KB with tens of thousands of pages no longer
|
||||
// @Description materializes megabytes of TEXT on every index open.
|
||||
// @Tags Wiki
|
||||
// @Produce json
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Success 200 {object} types.WikiPage
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Param types query string false "Comma-separated page types (default: all content types)"
|
||||
// @Param limit query int false "Per-group window size, 1-200 (default 50)"
|
||||
// @Param cursor query string false "Opaque offset cursor from previous response"
|
||||
// @Success 200 {object} types.WikiIndexResponse
|
||||
// @Security Bearer
|
||||
// @Router /api/v1/knowledgebase/{kb_id}/wiki/index [get]
|
||||
func (h *WikiPageHandler) GetIndex(c *gin.Context) {
|
||||
@@ -289,22 +299,43 @@ func (h *WikiPageHandler) GetIndex(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
page, err := h.wikiService.GetIndex(c.Request.Context(), kbID)
|
||||
var pageTypes []string
|
||||
if raw := c.Query("types"); raw != "" {
|
||||
for _, t := range strings.Split(raw, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
if t != "" {
|
||||
pageTypes = append(pageTypes, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
limit := 50
|
||||
if raw := c.Query("limit"); raw != "" {
|
||||
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
|
||||
limit = v
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := h.wikiService.GetIndexView(c.Request.Context(), kbID, pageTypes, limit, c.Query("cursor"))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, page)
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// GetLog godoc
|
||||
// @Summary Get wiki operation log
|
||||
// @Description Returns the wiki operation log page (creates default if not exists)
|
||||
// @Description Returns a paginated feed of wiki operation events (ingest, retract, ...)
|
||||
// @Description newest-first. Pagination is cursor-based: pass `next_cursor` from the
|
||||
// @Description previous response back as `cursor` to fetch the next page.
|
||||
// @Tags Wiki
|
||||
// @Produce json
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Success 200 {object} types.WikiPage
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Param cursor query string false "Opaque cursor from the previous page (empty = newest)"
|
||||
// @Param limit query int false "Page size, 1-200 (default 50)"
|
||||
// @Success 200 {object} types.WikiLogEntryListResponse
|
||||
// @Security Bearer
|
||||
// @Router /api/v1/knowledgebase/{kb_id}/wiki/log [get]
|
||||
func (h *WikiPageHandler) GetLog(c *gin.Context) {
|
||||
@@ -314,13 +345,21 @@ func (h *WikiPageHandler) GetLog(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
page, err := h.wikiService.GetLog(c.Request.Context(), kbID)
|
||||
cursor := c.Query("cursor")
|
||||
limit := 50
|
||||
if raw := c.Query("limit"); raw != "" {
|
||||
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
|
||||
limit = v
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := h.logEntryService.List(c.Request.Context(), kbID, cursor, limit)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, page)
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Graph query parameter bounds. The defaults cap an `overview` request at
|
||||
|
||||
Reference in New Issue
Block a user