perf(wiki): move ingest log to event table and index to on-demand API

Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.

Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.

Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.

Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.

Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.

- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
  renderIndexOverviewForAgent output, isTransientLLMError classifier

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
wizardchen
2026-05-08 21:43:53 +08:00
committed by lyingbug
parent bf12868923
commit d0144f3586
25 changed files with 2353 additions and 272 deletions
+77 -4
View File
@@ -115,12 +115,85 @@ export function deleteWikiPage(kbId: string, slug: string) {
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
}
export function getWikiIndex(kbId: string) {
return get(`/api/v1/knowledgebase/${kbId}/wiki/index`);
export interface WikiIndexEntryDTO {
slug: string;
title: string;
summary: string;
}
export function getWikiLog(kbId: string) {
return get(`/api/v1/knowledgebase/${kbId}/wiki/log`);
export interface WikiIndexGroup {
type: string;
total: number;
items: WikiIndexEntryDTO[];
next_cursor?: string;
}
export interface WikiIndexResponse {
intro: string;
version: number;
groups: WikiIndexGroup[];
}
// getWikiIndex fetches the structured index view for a wiki KB. The
// backend replaced the legacy "markdown blob of intro + directory" with
// { intro, groups } so a 40k-page wiki no longer round-trips multiple
// megabytes on every index open. Pass `types` to restrict which
// page_type buckets come back; `limit` bounds the per-group window;
// `cursor` resumes from a previous response.
export function getWikiIndex(
kbId: string,
params?: { types?: string[]; limit?: number; cursor?: string },
) {
const query = new URLSearchParams();
if (params) {
if (params.types && params.types.length > 0) query.set('types', params.types.join(','));
if (params.limit !== undefined) query.set('limit', String(params.limit));
if (params.cursor) query.set('cursor', params.cursor);
}
const qs = query.toString();
const suffix = qs ? `?${qs}` : '';
return get(`/api/v1/knowledgebase/${kbId}/wiki/index${suffix}`);
}
export interface WikiLogPageRef {
slug: string;
title?: string;
}
export interface WikiLogEntry {
id: number;
tenant_id: number;
knowledge_base_id: string;
action: string;
knowledge_id: string;
doc_title: string;
summary: string;
// Each ref carries both slug (for navigation) and title (captured at
// ingest time for display). Legacy rows written before the title
// column was added surface as refs with an empty title; render falls
// back to the slug in that case.
pages_affected: WikiLogPageRef[];
created_at: string;
}
export interface WikiLogListResponse {
entries: WikiLogEntry[];
next_cursor?: string;
}
// getWikiLog fetches a page of wiki operation events (newest first). Pass the
// `next_cursor` from the previous response back as `cursor` to load more;
// an empty / missing `next_cursor` signals end-of-feed. `limit` is clamped
// server-side to [1, 200] and defaults to 50.
export function getWikiLog(kbId: string, params?: { cursor?: string; limit?: number }) {
const query = new URLSearchParams();
if (params) {
if (params.cursor) query.set('cursor', params.cursor);
if (params.limit !== undefined) query.set('limit', String(params.limit));
}
const qs = query.toString();
const suffix = qs ? `?${qs}` : '';
return get(`/api/v1/knowledgebase/${kbId}/wiki/log${suffix}`);
}
export interface WikiGraphQueryParams {
+6
View File
@@ -1708,6 +1708,12 @@ export default {
fitView: 'Fit to View',
logTitle: 'Activity Log',
indexTitle: 'Index',
logFeedTag: 'Event feed',
logEmpty: 'No log entries yet',
logLoading: 'Loading…',
logLoadMore: 'Load more',
indexOverviewTag: 'Directory',
indexEmpty: 'No wiki pages yet. Upload documents first.',
graphNoData: 'No graph data yet. Upload documents first.',
showArrows: 'Show arrows',
hideArrows: 'Hide arrows',
+6
View File
@@ -2320,6 +2320,12 @@ export default {
fitView: "화면에 맞춤",
logTitle: '활동 로그',
indexTitle: '인덱스',
logFeedTag: '이벤트 피드',
logEmpty: '로그 기록이 없습니다',
logLoading: '로딩 중…',
logLoadMore: '더 보기',
indexOverviewTag: '분류 목록',
indexEmpty: 'Wiki 페이지가 없습니다. 먼저 문서를 업로드하세요.',
graphNoData: '그래프 데이터가 없습니다. 먼저 문서를 업로드하세요.',
showArrows: '화살표 표시',
hideArrows: '화살표 숨기기',
+6
View File
@@ -1930,6 +1930,12 @@ export default {
fitView: 'По размеру экрана',
logTitle: 'Журнал активности',
indexTitle: 'Индекс',
logFeedTag: 'Лента событий',
logEmpty: 'Записей журнала пока нет',
logLoading: 'Загрузка…',
logLoadMore: 'Загрузить ещё',
indexOverviewTag: 'Каталог',
indexEmpty: 'Пока нет wiki-страниц. Сначала загрузите документы.',
graphNoData: 'Нет данных графа. Сначала загрузите документы.',
showArrows: 'Показать стрелки',
hideArrows: 'Скрыть стрелки',
+6
View File
@@ -2291,6 +2291,12 @@ export default {
fitView: "适应屏幕",
logTitle: '日志',
indexTitle: '索引',
logFeedTag: '事件流',
logEmpty: '暂无日志记录',
logLoading: '加载中…',
logLoadMore: '加载更多',
indexOverviewTag: '分类目录',
indexEmpty: '暂无 Wiki 页面,请先上传文档',
graphNoData: '暂无图谱数据,请先上传文档',
showArrows: '显示箭头',
hideArrows: '隐藏箭头',
+575 -28
View File
@@ -241,27 +241,30 @@
</template>
<template v-else>
<!-- Index page (pinned at top) -->
<!-- Index overview (pinned at top). Rendered lazily from a
structured API response — never loads the full directory
as markdown. -->
<div
v-if="indexPage"
:class="['wiki-nav-item', { active: selectedPage?.id === indexPage.id }]"
@click="selectPage(indexPage)"
v-if="indexAvailable"
:class="['wiki-nav-item', { active: activeSystemView === 'index' }]"
@click="openIndexView"
>
<t-icon name="catalog" class="wiki-nav-icon" />
<span class="wiki-nav-text">{{ $t('knowledgeEditor.wikiBrowser.indexTitle') }}</span>
</div>
<!-- Log page (pinned) -->
<!-- Log feed (pinned). Events live in wiki_log_entries and
are loaded lazily when the user clicks this entry. -->
<div
v-if="logPage"
:class="['wiki-nav-item', { active: selectedPage?.id === logPage.id }]"
@click="selectPage(logPage)"
v-if="logAvailable"
:class="['wiki-nav-item', { active: activeSystemView === 'log' }]"
@click="openLogView"
>
<t-icon name="history" class="wiki-nav-icon" />
<span class="wiki-nav-text">{{ $t('knowledgeEditor.wikiBrowser.logTitle') }}</span>
</div>
<div class="wiki-sidebar-divider" v-if="indexPage || logPage"></div>
<div class="wiki-sidebar-divider" v-if="indexAvailable || logAvailable"></div>
<!-- Horizontal tab bar: one per non-empty page_type. Clicking a
tab swaps the visible list to that bucket. Parallel tabs are
@@ -337,10 +340,10 @@
<div class="wiki-reader-inner">
<template v-if="selectedPage">
<!-- Navigation -->
<div v-if="navHistory.length" class="wiki-nav-bar">
<div v-if="navHistory.length || navFromSystemView" class="wiki-nav-bar">
<a href="#" class="wiki-nav-back" @click.prevent="goBack">
<t-icon name="arrow-left" size="14px" />
<span>{{ navHistory[navHistory.length - 1].title }}</span>
<span>{{ backLabel }}</span>
</a>
</div>
@@ -453,6 +456,91 @@
</div>
</template>
<!-- System view: index overview rendered as markdown. Starts
with intro only; an IntersectionObserver-driven sentinel
at the bottom auto-appends the next directory section
(Summary → Entity → Concept → …) as the user scrolls
near the end. [[wiki-link]] clicks inside the rendered
body are handled by handleContentClick just like a
regular wiki page. -->
<template v-else-if="activeSystemView === 'index'">
<div class="wiki-reader-header">
<h2 class="wiki-reader-title">{{ $t('knowledgeEditor.wikiBrowser.indexTitle') }}</h2>
<div class="wiki-reader-meta">
<t-tag size="small" theme="default" variant="light-outline">
{{ $t('knowledgeEditor.wikiBrowser.indexOverviewTag') }}
</t-tag>
</div>
</div>
<div v-if="indexLoading && !indexMarkdown" class="wiki-reader-empty">
<p class="wiki-empty-title">{{ $t('knowledgeEditor.wikiBrowser.logLoading') }}</p>
</div>
<template v-else-if="indexMarkdown">
<div
ref="indexBodyRef"
class="wiki-reader-body wiki-index-body"
v-html="renderedIndexMarkdown"
@click="handleContentClick"
></div>
<div v-if="indexHasMore" ref="indexSentinelRef" class="wiki-index-sentinel">
<span v-if="indexLoading" class="wiki-index-loading">
{{ $t('knowledgeEditor.wikiBrowser.logLoading') }}
</span>
</div>
</template>
<div v-else-if="!indexLoading" class="wiki-reader-empty">
<p class="wiki-empty-title">{{ $t('knowledgeEditor.wikiBrowser.indexEmpty') }}</p>
</div>
</template>
<!-- System view: log feed. Mutually exclusive with selectedPage. -->
<template v-else-if="activeSystemView === 'log'">
<div class="wiki-reader-header">
<h2 class="wiki-reader-title">{{ $t('knowledgeEditor.wikiBrowser.logTitle') }}</h2>
<div class="wiki-reader-meta">
<t-tag size="small" theme="default" variant="light-outline">
{{ $t('knowledgeEditor.wikiBrowser.logFeedTag') }}
</t-tag>
</div>
</div>
<div class="wiki-log-feed">
<div v-if="logEntries.length === 0 && logInitialized" class="wiki-log-empty">
{{ $t('knowledgeEditor.wikiBrowser.logEmpty') }}
</div>
<div v-for="entry in logEntries" :key="entry.id" class="wiki-log-entry">
<div class="wiki-log-entry-header">
<t-tag size="small" :theme="entry.action === 'retract' ? 'danger' : 'primary'" variant="light">
{{ entry.action }}
</t-tag>
<span class="wiki-log-entry-title">{{ entry.doc_title || entry.knowledge_id || '—' }}</span>
<span class="wiki-log-entry-time">{{ formatDate(entry.created_at) }}</span>
</div>
<div v-if="entry.summary" class="wiki-log-entry-summary">{{ entry.summary }}</div>
<div v-if="entry.pages_affected && entry.pages_affected.length" class="wiki-log-entry-pages">
<a
v-for="ref in entry.pages_affected"
:key="entry.id + ':' + ref.slug"
href="#"
class="wiki-log-entry-page"
:title="ref.slug"
@click.prevent="navigateToSlug(ref.slug)"
>{{ ref.title || ref.slug }}</a>
</div>
</div>
<div v-if="logNextCursor || !logInitialized" class="wiki-log-load-more">
<t-button
size="small"
variant="outline"
theme="default"
:loading="logLoading"
@click="loadMoreLog"
>
{{ logInitialized ? $t('knowledgeEditor.wikiBrowser.logLoadMore') : $t('knowledgeEditor.wikiBrowser.logLoading') }}
</t-button>
</div>
</div>
</template>
<!-- No page selected -->
<div v-else class="wiki-reader-empty">
<div class="wiki-empty-icon">
@@ -566,6 +654,9 @@ import {
type WikiGraphData,
type WikiStats,
type WikiPageIssue,
type WikiLogEntry,
type WikiIndexGroup,
type WikiIndexEntryDTO,
} from '@/api/wiki'
const router = useRouter()
@@ -610,8 +701,60 @@ interface PageTypeBucket {
initialized: boolean // true once the first page has been fetched
}
const pagesByType = ref<Record<string, PageTypeBucket>>({})
const indexPage = ref<WikiPage | null>(null)
const logPage = ref<WikiPage | null>(null)
// Index view state. The reader renders an incrementally-built markdown
// string rather than a structured list — opening the view loads intro
// only, and "Load more" appends one directory section at a time in a
// fixed order (Summary → Entity → Concept → Synthesis → Comparison).
// Once the last section exhausts its pages, indexHasMore flips off.
//
// We deliberately avoid keeping a parallel structured list + a parallel
// markdown buffer; the markdown is the single source of truth the reader
// renders, and [[wiki-link]] clicks flow through the same
// handleContentClick as regular page bodies.
const indexMarkdown = ref('')
const indexLoading = ref(false)
const indexAvailable = ref(false)
// Per-section pagination cursor. Empty string = not yet loaded; empty
// cursor AFTER a load = that section is exhausted. `indexSectionIdx`
// tracks which section in INDEX_SECTION_ORDER is "next to load" — we
// advance to the following section only when the current one runs out.
const indexSections = ref<Record<string, { loaded: boolean; cursor: string; total: number }>>({})
const indexSectionIdx = ref(0)
const indexBodyRef = ref<HTMLElement | null>(null)
const indexSentinelRef = ref<HTMLElement | null>(null)
let indexObserver: IntersectionObserver | null = null
// Order matters: Summary first (these are the document-level pages the
// user most often wants to see), then the LLM-derived ones. Matches the
// plan's "intro then Summary → Entity → Concept → …" progression.
const INDEX_SECTION_ORDER = [
'summary',
'entity',
'concept',
'synthesis',
'comparison',
] as const
// logAvailable is a flag: the sidebar "Log" entry is always shown once a
// KB exists, because the backing wiki_log_entries table is KB-independent
// and `GET /wiki/log` returns an empty entries list when nothing has been
// logged yet. We don't need a full WikiPage object anymore — selecting
// Log swaps the reader into a dedicated feed view below.
const logAvailable = ref(true)
// activeSystemView lets the reader toggle between a regular wiki page
// (selectedPage) and a "virtual" system view — index overview and log
// feed. These modes are mutually exclusive: entering a system view
// clears selectedPage, and picking a page clears the system view flag.
const activeSystemView = ref<'' | 'index' | 'log'>('')
// Paginated state for the log view. `entries` grows as the user scrolls;
// `nextCursor` is the opaque cursor returned by the backend and empty
// signals end-of-feed. `loading` is the guard that prevents overlapping
// loadMore calls while a request is in flight.
const logEntries = ref<WikiLogEntry[]>([])
const logNextCursor = ref('')
const logLoading = ref(false)
const logInitialized = ref(false)
// When the user types into the search box we leave pagination mode and
// show a flat result list instead. Bucketed state is preserved behind
@@ -780,6 +923,13 @@ function fitGraphToView() {
const graphDrawerVisible = ref(false)
const graphDrawerPage = ref<WikiPage | null>(null)
const navHistory = ref<WikiPage[]>([])
// navFromSystemView remembers which system view (Index / Log) the user
// was viewing when they clicked into a slug, so goBack can restore it
// once the page-level history stack is empty. We keep this parallel to
// navHistory rather than widening its element type — navHistory is
// consumed everywhere as `WikiPage[]` and that contract stays cleaner
// if the system-view sentinel lives in its own ref.
const navFromSystemView = ref<'' | 'index' | 'log'>('')
// Index and log pages are now state refs (loaded by their own endpoints
// at startup) rather than computed over the full page list. The old
// computed implementation required pulling every page into memory just
@@ -1242,6 +1392,39 @@ const renderedContent = computed(() => {
return renderMarkdown(selectedPage.value.content)
})
// Label shown next to the back arrow on page headers. Prefers the
// nearest page-history entry when available so the user sees where
// they'll land; falls back to the Index/Log label when the current
// page was opened directly from a system view.
const backLabel = computed(() => {
if (navHistory.value.length > 0) {
return navHistory.value[navHistory.value.length - 1].title
}
if (navFromSystemView.value === 'index') {
return t('knowledgeEditor.wikiBrowser.indexTitle')
}
if (navFromSystemView.value === 'log') {
return t('knowledgeEditor.wikiBrowser.logTitle')
}
return ''
})
// Rendered markdown for the incremental index view. Re-runs every time
// indexMarkdown grows (initial intro load or a loadMore section append).
const renderedIndexMarkdown = computed(() => {
if (!indexMarkdown.value) return ''
return renderMarkdown(indexMarkdown.value)
})
// True while another section or page is available to load. Starts true
// after the first fetch (intro only loaded, sections untouched), becomes
// false once the last section in INDEX_SECTION_ORDER is exhausted.
const indexHasMore = computed(() => {
if (!indexAvailable.value) return false
if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return false
return true
})
watch(renderedContent, async () => {
await nextTick()
if (readerBodyRef.value) {
@@ -1249,6 +1432,16 @@ watch(renderedContent, async () => {
}
})
// Index body may contain image markdown from an LLM-generated intro;
// hydrate the same way as regular page content so protected URLs
// resolve. Also re-applied after every loadMore append.
watch(renderedIndexMarkdown, async () => {
await nextTick()
if (indexBodyRef.value) {
await hydrateProtectedFileImages(indexBodyRef.value)
}
})
function handleContentClick(e: MouseEvent) {
const target = e.target as HTMLElement
if (target.classList.contains('wiki-content-link')) {
@@ -1341,20 +1534,250 @@ async function loadPagesForType(type: string, opts: { reset?: boolean } = {}) {
groupScrollerRef.value?.updateVisibleItems?.(true)
}
// loadIndexAndLog fetches the two pinned system pages directly by slug
// instead of scanning the full page list for them. The endpoints return
// auto-created defaults if the wiki hasn't produced them yet, so we can
// always bind the refs and the template's v-if gates decide visibility.
// loadIndexAndLog probes the wiki index so the sidebar knows to show
// the pinned Index/Log entries. We ask the backend for intro only (zero
// group types) — a bounded response regardless of KB size. Sections are
// fetched lazily after the user actually opens the Index view; see
// loadMoreIndexSection.
//
// The log "page" is no longer stored in wiki_pages — it lives in the
// dedicated wiki_log_entries table. We don't need to pre-fetch anything
// here to decide whether to render the sidebar Log entry; the flag is
// always on, and the actual feed is fetched lazily when the user clicks
// the entry (see openLogView / loadMoreLog).
// stripLegacyIndexDirectory removes the inline "## Summary (N)\n[[...]]
// ..." directory listing from a legacy index row. Old wiki_pages rows
// stored "intro + directory markdown" in content; after the refactor
// intro is the whole payload, but pre-existing KBs still carry the
// directory until the next ingest batch rewrites it (see
// wikiIngestService.rebuildIndexPage). We don't want the stale
// directory to show up in the reader alongside the new live-fetched
// sections, so we clip everything from the first `\n## ` heading on.
function stripLegacyIndexDirectory(intro: string): string {
if (!intro) return ''
const idx = intro.indexOf('\n## ')
if (idx < 0) return intro.trim()
return intro.slice(0, idx).trim()
}
async function loadIndexAndLog() {
try {
const [idxRes, logRes] = await Promise.all([
getWikiIndex(props.knowledgeBaseId),
getWikiLog(props.knowledgeBaseId),
])
indexPage.value = (idxRes as any).data || (idxRes as any)
logPage.value = (logRes as any).data || (logRes as any)
// We only need intro on the initial probe — the directory groups
// are fetched lazily once the user opens the Index view. Passing
// an unknown type filter yields a cheap single count(*) + 0 rows
// on the backend instead of scanning every directory group, and
// the frontend discards the resulting empty group unconditionally.
const idxRes = await getWikiIndex(props.knowledgeBaseId, { types: ['__intro_only__'], limit: 1 })
const body: any = (idxRes as any).data || (idxRes as any)
const intro: string = body?.intro || ''
const cleanIntro = stripLegacyIndexDirectory(intro)
indexMarkdown.value = cleanIntro ? cleanIntro + '\n' : ''
indexAvailable.value = true
indexSections.value = {}
indexSectionIdx.value = 0
logAvailable.value = true
} catch (e) {
console.error('Failed to load wiki index/log:', e)
console.error('Failed to load wiki index:', e)
}
}
// openIndexView switches the reader into the markdown-rendered index
// overview. Re-uses the intro already fetched during loadPages(); only
// re-fetches on first ever open or if a prior attempt failed.
async function openIndexView() {
selectedPage.value = null
activeSystemView.value = 'index'
if (!indexMarkdown.value) {
indexLoading.value = true
try {
await loadIndexAndLog()
} finally {
indexLoading.value = false
}
}
// Observer is mounted/unmounted from a watch on activeSystemView
// + indexSentinelRef below, so nothing else to do here — entering
// the view is a render-time concern.
}
// loadMoreIndexSection advances the directory one step forward. The
// order is fixed (Summary → Entity → Concept → …); within a section we
// paginate with the backend's cursor, and only move to the next section
// when the current one is exhausted. Each call produces one network
// round trip that appends a markdown block to indexMarkdown.
//
// Rendering is append-only markdown rather than a structured list so
// the viewer feels like a regular wiki page — [[wiki-link]] clicks flow
// through handleContentClick just like every other page body. Entries
// are rendered as plain lines (not list items) so the reader doesn't
// carry list bullets next to every link.
async function loadMoreIndexSection() {
if (indexLoading.value) return
if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return
const type = INDEX_SECTION_ORDER[indexSectionIdx.value]
const state = indexSections.value[type] || { loaded: false, cursor: '', total: 0 }
const isFirstChunkOfSection = !state.loaded
indexLoading.value = true
try {
const res = await getWikiIndex(props.knowledgeBaseId, {
types: [type],
limit: 50,
cursor: isFirstChunkOfSection ? undefined : state.cursor || undefined,
})
const body: any = (res as any).data || (res as any)
const group = (body?.groups || []).find((g: WikiIndexGroup) => g.type === type)
const items: WikiIndexEntryDTO[] = group?.items || []
const total: number = group?.total || 0
const nextCursor: string = group?.next_cursor || ''
// Only emit a section heading the first time we see entries for a
// type. An empty section is skipped entirely so the reader doesn't
// see "## Entity (0)" for a KB with no entities.
let appended = ''
if (isFirstChunkOfSection && items.length > 0) {
const label = getTypeLabel(type)
appended += `\n## ${label} (${total})\n\n`
}
for (const entry of items) {
// Plain lines rather than `- [[slug]]`: marked renders the latter
// as <ul><li>, which adds a disc bullet before every slug. Each
// entry occupies one line thanks to `breaks: true` in renderMarkdown.
//
// Use the `[[slug|display]]` form so the anchor text shows the
// human-readable title (e.g. "东城区") instead of the URL-safe slug
// ("entity/dongcheng-qu"). The [[ ]] preprocessor in renderMarkdown
// splits on the pipe and uses the right-hand side for display text
// while the left-hand side drives navigation via handleContentClick.
//
// Fall back to the slug when the page has no title — the backend
// guarantees title is non-empty for published pages, but drafts
// or partially-indexed pages can slip through.
const display = entry.title || entry.slug
if (entry.summary) {
appended += `[[${entry.slug}|${display}]] — ${entry.summary}\n`
} else {
appended += `[[${entry.slug}|${display}]]\n`
}
}
if (appended) {
indexMarkdown.value = indexMarkdown.value + appended
}
indexSections.value[type] = {
loaded: true,
cursor: nextCursor,
total,
}
// Advance to the next section when this one has no more pages.
// When the section is flat-out empty (total === 0), skip the
// heading entirely and move on without emitting any markdown.
if (!nextCursor) {
indexSectionIdx.value += 1
}
} catch (e) {
console.error(`Failed to load more index entries for ${INDEX_SECTION_ORDER[indexSectionIdx.value]}:`, e)
} finally {
indexLoading.value = false
}
// IntersectionObserver does NOT re-fire while the target stays
// continuously intersecting. On small KBs (say a wiki with only
// 3 summary pages and no entities / concepts) the sentinel sits
// inside the viewport from the moment we finish the first section,
// so without this nudge the remaining sections would never load.
//
// After every append we yield a tick (so the DOM reflows and the
// sentinel's new rect is valid) then re-check: if it's still in
// view and we have more to load, recurse. The recursion bottoms
// out when either hasMore turns off or the sentinel is pushed
// below the fold by the accumulated entries.
await nextTick()
if (indexHasMore.value && sentinelInView()) {
loadMoreIndexSection()
}
}
// sentinelInView reports whether the load sentinel's rect currently
// overlaps the viewport (± the same 200px cushion the observer uses),
// so after loading a section we know whether to drain another round
// even though the observer itself won't fire again while the target
// stays visible.
function sentinelInView(): boolean {
const el = indexSentinelRef.value
if (!el) return false
const rect = el.getBoundingClientRect()
const vh = window.innerHeight || document.documentElement.clientHeight
// 200px margin matches the observer's rootMargin so the drain
// threshold and the scroll-triggered threshold stay consistent.
return rect.top < vh + 200 && rect.bottom > -200
}
// Mount/unmount the IntersectionObserver around the Index sentinel.
// We use rootMargin to pre-load when the user scrolls within ~200px
// of the sentinel, which hides the network round-trip behind the
// scroll motion.
watch([indexSentinelRef, () => activeSystemView.value], async ([el, view]) => {
if (indexObserver) {
indexObserver.disconnect()
indexObserver = null
}
if (view !== 'index' || !el) return
indexObserver = new IntersectionObserver((entries) => {
for (const entry of entries) {
if (entry.isIntersecting && indexHasMore.value && !indexLoading.value) {
loadMoreIndexSection()
}
}
}, { rootMargin: '200px' })
indexObserver.observe(el)
})
onUnmounted(() => {
if (indexObserver) {
indexObserver.disconnect()
indexObserver = null
}
})
// openLogView switches the reader into the log feed and (re)loads the
// first page. Called when the user clicks the sidebar Log entry.
async function openLogView() {
selectedPage.value = null
activeSystemView.value = 'log'
logEntries.value = []
logNextCursor.value = ''
logInitialized.value = false
await loadMoreLog()
}
// loadMoreLog appends the next page of log entries using the cursor from
// the previous response. Guarded so overlapping scroll events don't fire
// multiple requests and double-append entries.
async function loadMoreLog() {
if (logLoading.value) return
// Once a previous request reported end-of-feed (empty next_cursor), we
// stop — but only after the first fetch, so a fresh KB still runs the
// initial empty request to populate logInitialized.
if (logInitialized.value && !logNextCursor.value) return
logLoading.value = true
try {
const res = await getWikiLog(props.knowledgeBaseId, {
cursor: logNextCursor.value || undefined,
limit: 50,
})
const body: any = (res as any).data || res
const entries: WikiLogEntry[] = body?.entries || []
logEntries.value.push(...entries)
logNextCursor.value = body?.next_cursor || ''
logInitialized.value = true
} catch (e) {
console.error('Failed to load wiki log:', e)
} finally {
logLoading.value = false
}
}
@@ -1390,12 +1813,14 @@ async function loadPages() {
}
}
// Auto-select based on query or index page
if (!selectedPage.value) {
// Auto-select based on query string or default to the index
// overview. The index is the natural landing view — it shows
// intro + a paginated directory of every page type.
if (!selectedPage.value && activeSystemView.value === '') {
if (route.query.slug && typeof route.query.slug === 'string') {
navigateToSlug(route.query.slug)
} else if (indexPage.value) {
selectPage(indexPage.value)
} else if (indexAvailable.value) {
openIndexView()
}
}
} finally {
@@ -1855,7 +2280,14 @@ async function selectPage(page: WikiPage) {
try {
if (selectedPage.value && selectedPage.value.id !== page.id) {
navHistory.value.push(selectedPage.value)
} else if (!selectedPage.value && activeSystemView.value) {
// Jumping out of a system view (Index / Log) onto a page.
// navHistory only holds WikiPages, so we stash the origin
// system view separately; goBack restores it when the history
// stack is empty.
navFromSystemView.value = activeSystemView.value
}
activeSystemView.value = ''
const res = await getWikiPage(props.knowledgeBaseId, page.slug)
selectedPage.value = (res as any).data || res as any
await loadPageIssues(page.slug)
@@ -1868,7 +2300,13 @@ async function navigateToSlug(slug: string) {
try {
if (selectedPage.value && selectedPage.value.slug !== slug) {
navHistory.value.push(selectedPage.value)
} else if (!selectedPage.value && activeSystemView.value) {
// Clicking a [[slug]] from inside Index / Log — same rationale
// as selectPage above: record the system-view origin so the
// reader's back arrow can return to it.
navFromSystemView.value = activeSystemView.value
}
activeSystemView.value = ''
const res = await getWikiPage(props.knowledgeBaseId, slug)
selectedPage.value = (res as any).data || res as any
await loadPageIssues(slug)
@@ -1882,6 +2320,19 @@ function goBack() {
if (prev) {
selectedPage.value = prev
loadPageIssues(prev.slug)
return
}
// History stack is empty but we remember the page was opened from
// a system view — restore that instead of leaving the reader empty.
if (navFromSystemView.value) {
const view = navFromSystemView.value
navFromSystemView.value = ''
selectedPage.value = null
if (view === 'index') {
openIndexView()
} else if (view === 'log') {
openLogView()
}
}
}
@@ -3736,6 +4187,102 @@ onUnmounted(() => {
text-align: center;
}
// ── Log feed (system view) ──
// Rendered when activeSystemView === 'log'. Sits where the markdown body
// would be for a regular wiki page — so the header/meta rules above
// already apply. We just style the feed list itself.
.wiki-log-feed {
display: flex;
flex-direction: column;
gap: 12px;
padding-top: 12px;
}
// ── Index overview (system view) ──
// The index view renders as markdown through the same pipeline as a
// normal wiki page, so it inherits .wiki-reader-body styling automatically.
// We use a sentinel below the body to drive auto-pagination via
// IntersectionObserver — the user never sees a "Load more" button.
.wiki-index-sentinel {
display: flex;
align-items: center;
justify-content: center;
min-height: 32px;
padding: 16px 0 24px;
color: var(--td-text-color-placeholder);
font-size: 13px;
}
.wiki-index-loading {
opacity: 0.7;
}
.wiki-log-empty {
color: var(--td-text-color-placeholder);
text-align: center;
padding: 40px 0;
font-size: 13px;
}
.wiki-log-entry {
border: 1px solid var(--td-border-level-1-color);
border-radius: 8px;
padding: 10px 12px;
background: var(--td-bg-color-container);
}
.wiki-log-entry-header {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 4px;
}
.wiki-log-entry-title {
font-weight: 500;
color: var(--td-text-color-primary);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
flex: 1;
min-width: 0;
}
.wiki-log-entry-time {
color: var(--td-text-color-placeholder);
font-size: 12px;
white-space: nowrap;
}
.wiki-log-entry-summary {
color: var(--td-text-color-secondary);
font-size: 13px;
margin: 4px 0;
}
.wiki-log-entry-pages {
display: flex;
flex-wrap: wrap;
gap: 4px 10px;
margin-top: 4px;
}
.wiki-log-entry-page {
color: var(--td-brand-color);
font-size: 12px;
text-decoration: none;
}
.wiki-log-entry-page:hover {
text-decoration: underline;
}
.wiki-log-load-more {
display: flex;
justify-content: center;
padding: 12px 0;
}
.wiki-empty-icon {
width: 64px;
height: 64px;
@@ -0,0 +1,121 @@
package tools
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint verifies
// that when a group's total exceeds its item count, the synthesized
// markdown surfaces both numbers and the "showing top N" banner so the
// model knows further exploration requires a separate tool call.
func TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint(t *testing.T) {
resp := &types.WikiIndexResponse{
Intro: "# Wiki Index\n\nWelcome to the wiki.",
Groups: []types.WikiIndexGroup{
{
Type: types.WikiPageTypeEntity,
Total: 4500,
Items: []types.WikiIndexEntry{
{Slug: "entity/acme", Title: "Acme", Summary: "A company"},
{Slug: "entity/beta", Title: "Beta", Summary: "Another company"},
},
},
{
Type: types.WikiPageTypeConcept,
Total: 2,
Items: []types.WikiIndexEntry{
{Slug: "concept/rag", Title: "RAG", Summary: "Retrieval-augmented generation"},
{Slug: "concept/emb", Title: "Embedding", Summary: ""},
},
},
},
}
got := renderIndexOverviewForAgent(resp)
assertContains(t, got, "# Wiki Index")
// Truncation banner for the big group: Total != len(Items).
assertContains(t, got, "4500 total, showing top 2")
// Non-truncated group uses the simple "(N)" form.
assertContains(t, got, "## Concept (2)")
// Entry formatting — with and without summary. Entries emit the
// [[slug|title]] form so the LLM sees the human-readable name.
assertContains(t, got, "[[entity/acme|Acme]] — A company")
// When the entry has a title but no summary, slug|title still
// renders; the dash + summary portion is elided.
assertContains(t, got, "[[concept/emb|Embedding]]")
// Exploration hint.
assertContains(t, got, "wiki_search")
}
// TestRenderIndexOverviewForAgent_EmptyKB explains to the model that the
// wiki has no pages yet — otherwise it might hallucinate slugs to read.
func TestRenderIndexOverviewForAgent_EmptyKB(t *testing.T) {
resp := &types.WikiIndexResponse{
Intro: "# Empty\n",
Groups: []types.WikiIndexGroup{
{Type: types.WikiPageTypeEntity, Total: 0, Items: nil},
},
}
got := renderIndexOverviewForAgent(resp)
assertContains(t, got, "No wiki pages yet")
if strings.Contains(got, "## Entity") {
t.Fatalf("empty groups should not surface a heading: %q", got)
}
}
// TestRenderIndexOverviewForAgent_UnknownType falls through to the raw
// type string — covers forward compatibility for LLM-created page types
// the frontend label map does not yet know about.
func TestRenderIndexOverviewForAgent_UnknownType(t *testing.T) {
resp := &types.WikiIndexResponse{
Intro: "# Wiki\n",
Groups: []types.WikiIndexGroup{
{
Type: "custom-type",
Total: 1,
Items: []types.WikiIndexEntry{{Slug: "custom-type/x", Title: "X"}},
},
},
}
got := renderIndexOverviewForAgent(resp)
assertContains(t, got, "## custom-type (1)")
}
// TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory covers
// the forward-compat path: KBs that haven't been re-ingested since the
// index refactor still carry an "intro + ## Summary..." payload on the
// index row. The agent synthesizer must clip the legacy directory so
// the model doesn't see it next to the live top-K block the group
// loop emits below.
func TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory(t *testing.T) {
resp := &types.WikiIndexResponse{
Intro: "# Wiki Index\n\nWelcome.\n\n## Summary (3)\n\n[[summary/legacy]] — old",
Groups: []types.WikiIndexGroup{
{
Type: types.WikiPageTypeEntity,
Total: 1,
Items: []types.WikiIndexEntry{{Slug: "entity/live", Title: "Live", Summary: "fresh"}},
},
},
}
got := renderIndexOverviewForAgent(resp)
if strings.Contains(got, "summary/legacy") {
t.Fatalf("legacy inline directory should have been stripped, got:\n%s", got)
}
assertContains(t, got, "Welcome.")
assertContains(t, got, "[[entity/live|Live]] — fresh")
}
func assertContains(t *testing.T, haystack, needle string) {
t.Helper()
if !strings.Contains(haystack, needle) {
t.Fatalf("expected output to contain %q, got:\n%s", needle, haystack)
}
}
+93 -1
View File
@@ -12,6 +12,83 @@ import (
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// wikiIndexAgentTopK is the per-type cap applied when synthesizing the
// index overview for wiki_read_page('index'). The agent uses the overview
// to get a sense of the wiki's shape; any deeper exploration should go
// through wiki_search. Keeping the cap small bounds the content envelope
// served to the LLM, which is the main reason this synthesis exists.
const wikiIndexAgentTopK = 20
// renderIndexOverviewForAgent formats a WikiIndexResponse as a compact
// markdown block suitable for embedding inside the <content> tag that
// wiki_read_page returns. We deliberately keep the structure almost
// identical to the legacy "intro + ## Type (N)\n[[slug]] — summary"
// markdown so existing agent prompts (prompts_wiki.go) that reason
// about index layouts do not need retraining.
func renderIndexOverviewForAgent(resp *types.WikiIndexResponse) string {
var sb strings.Builder
// Intro may still carry a legacy inline directory on KBs that
// haven't been re-ingested since the index refactor — clip
// everything from the first "\n## " heading onwards so the model
// doesn't see the old directory alongside the live top-K below.
intro := strings.TrimSpace(resp.Intro)
if idx := strings.Index(intro, "\n## "); idx >= 0 {
intro = strings.TrimSpace(intro[:idx])
}
if intro != "" {
sb.WriteString(intro)
sb.WriteString("\n")
}
typeLabels := map[string]string{
types.WikiPageTypeSummary: "Summary",
types.WikiPageTypeEntity: "Entity",
types.WikiPageTypeConcept: "Concept",
types.WikiPageTypeSynthesis: "Synthesis",
types.WikiPageTypeComparison: "Comparison",
}
nonEmpty := 0
for _, g := range resp.Groups {
if g.Total == 0 {
continue
}
label := typeLabels[g.Type]
if label == "" {
label = g.Type
}
if int64(len(g.Items)) < g.Total {
fmt.Fprintf(&sb, "\n## %s (%d total, showing top %d)\n\n", label, g.Total, len(g.Items))
} else {
fmt.Fprintf(&sb, "\n## %s (%d)\n\n", label, g.Total)
}
for _, item := range g.Items {
// Emit [[slug|title]] so the LLM sees the human-readable
// name next to the slug it needs for downstream tool calls
// (wiki_read_page, wiki_search by title, etc.). Falling back
// to the slug when a page has no title keeps the wiki-link
// syntactically valid either way.
display := item.Title
if display == "" {
display = item.Slug
}
if item.Summary != "" {
fmt.Fprintf(&sb, "[[%s|%s]] — %s\n", item.Slug, display, item.Summary)
} else {
fmt.Fprintf(&sb, "[[%s|%s]]\n", item.Slug, display)
}
}
nonEmpty++
}
if nonEmpty == 0 {
sb.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
} else {
sb.WriteString("\n_To explore more pages under any category, use wiki_search with a query, or read a specific slug directly._\n")
}
return sb.String()
}
// ---- wiki_read_page ----
// WikiScope describes the effective retrieval scope for a single wiki
@@ -292,6 +369,21 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (*
}
}
// Index page special-case. wiki_pages.content used to hold
// "intro + full directory" markdown; on big KBs that was a
// multi-MB blob the LLM had no hope of reading end-to-end.
// After the index refactor, content holds only the intro, and
// the directory is assembled on demand. Surface a bounded
// top-K overview so the agent still sees the wiki's shape
// without overflowing its context window — the explicit hint
// steers the model to wiki_search for deeper exploration.
contentBody := page.Content
if page.PageType == types.WikiPageTypeIndex {
if overview, err := t.wikiService.GetIndexView(ctx, kbID, nil, wikiIndexAgentTopK, ""); err == nil && overview != nil {
contentBody = renderIndexOverviewForAgent(overview)
}
}
return fmt.Sprintf(`<wiki_page>
<metadata>
<knowledge_base_id>%s</knowledge_base_id>
@@ -320,7 +412,7 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (*
strings.Join(inLinksDesc, ", "),
strings.Join(sourcesDesc, "\n"),
page.Summary,
page.Content,
contentBody,
)
}
@@ -0,0 +1,92 @@
package repository
import (
"context"
"errors"
"fmt"
"strconv"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
)
// wikiLogEntryRepository implements interfaces.WikiLogEntryRepository.
type wikiLogEntryRepository struct {
db *gorm.DB
}
// NewWikiLogEntryRepository constructs a GORM-backed WikiLogEntryRepository.
func NewWikiLogEntryRepository(db *gorm.DB) interfaces.WikiLogEntryRepository {
return &wikiLogEntryRepository{db: db}
}
// AppendBatch inserts every entry in one statement. Empty batches are a
// no-op so callers can invoke unconditionally at the end of a wiki ingest
// batch without guarding against the "no events this round" case.
func (r *wikiLogEntryRepository) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error {
if len(entries) == 0 {
return nil
}
return r.db.WithContext(ctx).Create(&entries).Error
}
// List returns up to `limit` entries strictly older than `cursor`, newest
// first. `cursor` is the stringified ID of the oldest entry from the
// previous page; an empty string starts from the newest. Callers get back
// a nextCursor string to pass on the next request — empty means no more.
//
// `limit` is clamped to [1, 200]. Values outside that range are coerced
// silently to keep the handler simple.
func (r *wikiLogEntryRepository) List(
ctx context.Context,
kbID string,
cursor string,
limit int,
) ([]*types.WikiLogEntry, string, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
q := r.db.WithContext(ctx).
Where("knowledge_base_id = ?", kbID).
Order("id DESC").
Limit(limit)
if cursor != "" {
cursorID, err := strconv.ParseUint(cursor, 10, 64)
if err != nil {
return nil, "", fmt.Errorf("invalid cursor %q: %w", cursor, err)
}
q = q.Where("id < ?", cursorID)
}
var entries []*types.WikiLogEntry
if err := q.Find(&entries).Error; err != nil {
return nil, "", err
}
nextCursor := ""
// Only emit a cursor when we actually filled the page — a short page
// is guaranteed to be the tail, so returning a cursor would just cause
// the frontend to fire a final empty request.
if len(entries) == limit {
nextCursor = strconv.FormatUint(entries[len(entries)-1].ID, 10)
}
return entries, nextCursor, nil
}
// DeleteByKB drops every log entry for a KB. No "affected rows" signal is
// surfaced — missing rows are a legitimate state (e.g., a KB that was
// created before wiki_log_entries existed) and not a failure condition.
func (r *wikiLogEntryRepository) DeleteByKB(ctx context.Context, kbID string) error {
if kbID == "" {
return errors.New("wiki log entries: empty kb id")
}
return r.db.WithContext(ctx).
Where("knowledge_base_id = ?", kbID).
Delete(&types.WikiLogEntry{}).Error
}
@@ -0,0 +1,219 @@
package repository
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// wikiLogEntriesTestDDL mirrors migrations/versioned/000040_wiki_log_entries.up.sql
// but uses SQLite-compatible types so we can exercise the repo layer without
// spinning up Postgres. INTEGER PRIMARY KEY AUTOINCREMENT gives us the same
// monotonically-increasing ID semantics the production schema relies on for
// cursor pagination.
const wikiLogEntriesTestDDL = `
CREATE TABLE IF NOT EXISTS wiki_log_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id INTEGER NOT NULL,
knowledge_base_id VARCHAR(36) NOT NULL,
action VARCHAR(32) NOT NULL,
knowledge_id VARCHAR(36) NOT NULL DEFAULT '',
doc_title TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
pages_affected TEXT NOT NULL DEFAULT '[]',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
`
func setupWikiLogTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(wikiLogEntriesTestDDL).Error)
return db
}
func makeLogEntry(kbID, action, knowledgeID, title string, pages ...string) *types.WikiLogEntry {
refs := make(types.WikiLogPageRefs, 0, len(pages))
for _, slug := range pages {
// Tests only care about round-tripping; title is derived from
// the slug so assertions don't have to spell out both fields.
refs = append(refs, types.WikiLogPageRef{Slug: slug, Title: "title-of-" + slug})
}
return &types.WikiLogEntry{
TenantID: 1,
KnowledgeBaseID: kbID,
Action: action,
KnowledgeID: knowledgeID,
DocTitle: title,
Summary: "summary for " + title,
PagesAffected: refs,
}
}
// TestWikiLogEntryRepository_AppendBatch_WritesAllRows verifies a batch
// insert persists every entry with a monotonic ID and respects the caller-
// provided column values.
func TestWikiLogEntryRepository_AppendBatch_WritesAllRows(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
entries := []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1", "entity/a", "summary/k1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-a", "retract", "k3", "Doc 3", "concept/c"),
}
require.NoError(t, repo.AppendBatch(ctx, entries))
// Every row should have a non-zero ID assigned by the DB, and IDs
// should increase in insertion order so the cursor pagination logic
// can rely on them.
for i := range entries {
assert.NotZero(t, entries[i].ID, "row %d should have a DB-assigned ID", i)
if i > 0 {
assert.Greater(t, entries[i].ID, entries[i-1].ID,
"AppendBatch should preserve insertion order")
}
}
// Round-trip sanity: select back and confirm payload was preserved.
var got []types.WikiLogEntry
require.NoError(t, db.Order("id ASC").Find(&got).Error)
require.Len(t, got, len(entries))
assert.Equal(t, "ingest", got[0].Action)
assert.Equal(t, "Doc 1", got[0].DocTitle)
assert.Equal(t, types.WikiLogPageRefs{
{Slug: "entity/a", Title: "title-of-entity/a"},
{Slug: "summary/k1", Title: "title-of-summary/k1"},
}, got[0].PagesAffected)
assert.Equal(t, "retract", got[2].Action)
}
// TestWikiLogEntryRepository_AppendBatch_Empty asserts an empty batch is a
// cheap no-op — callers at the end of a wiki ingest pass can invoke it
// unconditionally without a DB round trip.
func TestWikiLogEntryRepository_AppendBatch_Empty(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
require.NoError(t, repo.AppendBatch(ctx, nil))
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{}))
var count int64
require.NoError(t, db.Model(&types.WikiLogEntry{}).Count(&count).Error)
assert.Equal(t, int64(0), count)
}
// TestWikiLogEntryRepository_List_CursorPagination walks the feed with a
// limit smaller than the population size and confirms:
// - newest entries surface first (id DESC order),
// - passing next_cursor returns the next batch strictly older than it,
// - end-of-feed is signalled by an empty cursor.
func TestWikiLogEntryRepository_List_CursorPagination(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
// 5 entries in one KB + a sibling row in another KB to ensure the
// WHERE filter actually narrows by KB.
entries := []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-a", "ingest", "k3", "Doc 3"),
makeLogEntry("kb-a", "ingest", "k4", "Doc 4"),
makeLogEntry("kb-a", "retract", "k5", "Doc 5"),
}
require.NoError(t, repo.AppendBatch(ctx, entries))
other := makeLogEntry("kb-other", "ingest", "kx", "Other doc")
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{other}))
// First page: latest 2 of kb-a, newest first.
page1, cursor1, err := repo.List(ctx, "kb-a", "", 2)
require.NoError(t, err)
require.Len(t, page1, 2)
assert.Equal(t, "Doc 5", page1[0].DocTitle)
assert.Equal(t, "Doc 4", page1[1].DocTitle)
assert.NotEmpty(t, cursor1, "should emit next_cursor when the page is full")
// Second page continues strictly older than cursor1.
page2, cursor2, err := repo.List(ctx, "kb-a", cursor1, 2)
require.NoError(t, err)
require.Len(t, page2, 2)
assert.Equal(t, "Doc 3", page2[0].DocTitle)
assert.Equal(t, "Doc 2", page2[1].DocTitle)
assert.NotEmpty(t, cursor2)
// Third (final) page: one row remains, so cursor is empty — the repo
// must NOT emit a cursor for a short page, otherwise the frontend
// would fire a doomed follow-up request.
page3, cursor3, err := repo.List(ctx, "kb-a", cursor2, 2)
require.NoError(t, err)
require.Len(t, page3, 1)
assert.Equal(t, "Doc 1", page3[0].DocTitle)
assert.Empty(t, cursor3, "short page should signal end-of-feed with an empty cursor")
// Sibling KB row must NOT leak into kb-a's feed.
for _, p := range append(append(page1, page2...), page3...) {
assert.Equal(t, "kb-a", p.KnowledgeBaseID)
}
}
// TestWikiLogEntryRepository_List_InvalidCursor rejects a non-numeric
// cursor rather than silently returning the newest page, so clients
// notice bugs before they turn into phantom paginations.
func TestWikiLogEntryRepository_List_InvalidCursor(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
_, _, err := repo.List(ctx, "kb-a", "not-a-number", 10)
assert.Error(t, err)
}
// TestWikiLogEntryRepository_DeleteByKB purges one KB's rows without
// touching others. Used during KB deletion so the log table does not
// accumulate orphans after the parent KB is gone.
func TestWikiLogEntryRepository_DeleteByKB(t *testing.T) {
db := setupWikiLogTestDB(t)
repo := NewWikiLogEntryRepository(db)
ctx := context.Background()
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
makeLogEntry("kb-b", "ingest", "k3", "Doc 3"),
}))
require.NoError(t, repo.DeleteByKB(ctx, "kb-a"))
aEntries, _, err := repo.List(ctx, "kb-a", "", 100)
require.NoError(t, err)
assert.Empty(t, aEntries)
bEntries, _, err := repo.List(ctx, "kb-b", "", 100)
require.NoError(t, err)
assert.Len(t, bEntries, 1)
}
// TestWikiLogPageRefs_ScanAcceptsLegacyStringArray covers the backward-
// compat path in the Scan method. Any rows the ingest pipeline wrote
// before pages_affected gained a title field are stored as a plain JSON
// string array; the repo must decode them as slug-only refs so the log
// feed renders without a data migration.
func TestWikiLogPageRefs_ScanAcceptsLegacyStringArray(t *testing.T) {
var refs types.WikiLogPageRefs
require.NoError(t, refs.Scan([]byte(`["entity/acme","concept/rag"]`)))
require.Len(t, refs, 2)
assert.Equal(t, "entity/acme", refs[0].Slug)
assert.Empty(t, refs[0].Title, "legacy rows have no title until re-ingest overwrites them")
assert.Equal(t, "concept/rag", refs[1].Slug)
}
@@ -208,6 +208,57 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy
return pages, nil
}
// ListByTypeLight projects only the columns needed to render an index
// directory entry (slug, title, summary) and paginates by title ASC.
// This keeps the GET /wiki/index response cheap on KBs with tens of
// thousands of pages — the old path loaded every row including its TEXT
// content just to throw the content away on the way out.
//
// Archived pages are excluded. `limit` clamps to [1, 200]; `offset` is
// honored as-is. Returns the total non-archived count for the type
// alongside the page so the caller can render "showing N of M".
func (r *wikiPageRepository) ListByTypeLight(
ctx context.Context,
kbID string,
pageType string,
limit int,
offset int,
) ([]types.WikiIndexEntry, int64, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
if offset < 0 {
offset = 0
}
base := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Where("knowledge_base_id = ? AND page_type = ? AND status <> ?",
kbID, pageType, types.WikiPageStatusArchived)
var total int64
if err := base.Count(&total).Error; err != nil {
return nil, 0, err
}
if total == 0 {
return nil, 0, nil
}
var entries []types.WikiIndexEntry
if err := base.
Select("slug", "title", "summary").
Order("title ASC").
Limit(limit).
Offset(offset).
Scan(&entries).Error; err != nil {
return nil, 0, err
}
return entries, total, nil
}
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array.
func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) {
@@ -0,0 +1,193 @@
package repository
import (
"context"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// wikiPagesTestDDL is a minimal SQLite-compatible subset of the
// production wiki_pages DDL (migrations/versioned/000037_wiki_and_indexing.up.sql).
// JSONB is stored as TEXT in SQLite; the StringArray Scan/Value pair
// handles the JSON round-trip unchanged.
const wikiPagesTestDDL = `
CREATE TABLE IF NOT EXISTS wiki_pages (
id VARCHAR(36) PRIMARY KEY,
tenant_id INTEGER NOT NULL,
knowledge_base_id VARCHAR(36) NOT NULL,
slug VARCHAR(255) NOT NULL,
title VARCHAR(512) NOT NULL DEFAULT '',
page_type VARCHAR(32) NOT NULL DEFAULT 'summary',
status VARCHAR(32) NOT NULL DEFAULT 'published',
content TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
source_refs TEXT DEFAULT '[]',
chunk_refs TEXT DEFAULT '[]',
in_links TEXT DEFAULT '[]',
out_links TEXT DEFAULT '[]',
page_metadata TEXT DEFAULT '{}',
aliases TEXT DEFAULT '[]',
version INTEGER NOT NULL DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
`
func setupWikiPagesTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(wikiPagesTestDDL).Error)
return db
}
// makeWikiPage builds a minimal WikiPage suitable for insert. Title is
// derived from the slug so ORDER BY title ASC yields a predictable
// test ordering without callers having to spell out both fields.
func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage {
title := slug
if idx := strings.LastIndex(slug, "/"); idx >= 0 {
title = slug[idx+1:]
}
return &types.WikiPage{
ID: uuid.New().String(),
TenantID: 1,
KnowledgeBaseID: kbID,
Slug: slug,
Title: title,
PageType: pageType,
Status: status,
Content: "body of " + slug,
Summary: "summary of " + slug,
Version: 1,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
}
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
// that the index-view projection only emits the slug/title/summary
// triples and respects the archived-status filter. This is the whole
// point of splitting the method off ListByType — the index reader must
// not pay for TEXT content transport.
func TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
pages := []*types.WikiPage{
makeWikiPage("kb-a", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
makeWikiPage("kb-a", "entity/beta", types.WikiPageTypeEntity, types.WikiPageStatusDraft),
makeWikiPage("kb-a", "entity/gamma", types.WikiPageTypeEntity, types.WikiPageStatusArchived),
makeWikiPage("kb-a", "concept/delta", types.WikiPageTypeConcept, types.WikiPageStatusPublished),
makeWikiPage("kb-other", "entity/leaked", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
}
for _, p := range pages {
require.NoError(t, repo.Create(ctx, p))
}
entries, total, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 50, 0)
require.NoError(t, err)
// Archived is excluded; sibling KB is excluded; everything else
// surfaces regardless of draft/published status (the index shows
// both so admins notice newly-drafted pages).
assert.Equal(t, int64(2), total)
require.Len(t, entries, 2)
// ORDER BY title ASC => alpha, beta.
assert.Equal(t, "entity/alpha", entries[0].Slug)
assert.Equal(t, "alpha", entries[0].Title)
assert.Equal(t, "summary of entity/alpha", entries[0].Summary)
assert.Equal(t, "entity/beta", entries[1].Slug)
}
// TestListByTypeLight_Pagination walks the type list using offsets and
// asserts the count stays stable regardless of where in the list we
// are — the index handler uses total to render "showing N of M".
func TestListByTypeLight_Pagination(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
for _, s := range []string{"entity/a", "entity/b", "entity/c", "entity/d", "entity/e"} {
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-a", s, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
}
page1, total1, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 0)
require.NoError(t, err)
assert.Equal(t, int64(5), total1)
require.Len(t, page1, 2)
assert.Equal(t, "entity/a", page1[0].Slug)
assert.Equal(t, "entity/b", page1[1].Slug)
page2, total2, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 2)
require.NoError(t, err)
assert.Equal(t, int64(5), total2, "total should be stable across pages")
require.Len(t, page2, 2)
assert.Equal(t, "entity/c", page2[0].Slug)
assert.Equal(t, "entity/d", page2[1].Slug)
page3, total3, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 4)
require.NoError(t, err)
assert.Equal(t, int64(5), total3)
require.Len(t, page3, 1)
assert.Equal(t, "entity/e", page3[0].Slug)
// Offset past the end yields an empty list, not an error — the
// handler relies on this to short-circuit pagination tails.
page4, total4, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 10)
require.NoError(t, err)
assert.Equal(t, int64(5), total4)
assert.Empty(t, page4)
}
// TestListByTypeLight_EmptyType_ReturnsZero exercises the "no rows"
// short-circuit. We skip the SELECT entirely when count is zero, so
// a KB with no pages of a type shouldn't burn a pointless query.
func TestListByTypeLight_EmptyType_ReturnsZero(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
entries, total, err := repo.ListByTypeLight(ctx, "kb-empty", types.WikiPageTypeSynthesis, 50, 0)
require.NoError(t, err)
assert.Equal(t, int64(0), total)
assert.Empty(t, entries)
}
// TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't
// want a client passing limit=100000 and forcing the DB to return a
// multi-MB response.
func TestListByTypeLight_ClampsLimit(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
for i := 0; i < 250; i++ {
slug := "entity/bulk-"
// Use a stable, zero-padded suffix so title ordering is
// deterministic for the cap assertion below.
slug += string(rune('a'+(i/26)%26)) + string(rune('a'+i%26))
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-cap", slug, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
}
// limit=0 falls back to the default of 50.
defaultEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 0, 0)
require.NoError(t, err)
assert.Len(t, defaultEntries, 50)
// limit=5000 clamps to 200.
clampedEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 5000, 0)
require.NoError(t, err)
assert.LessOrEqual(t, len(clampedEntries), 200)
}
+211 -112
View File
@@ -97,6 +97,17 @@ const (
// tick (GC pause, Redis blip) doesn't let the lock slip out from under a
// live handler.
wikiActiveLockRenew = 20 * time.Second
// wikiLLMMaxAttempts is the total attempt count (initial + retries) for
// every LLM call routed through generateWithTemplate. 3 was chosen to
// absorb transient 504/timeouts from upstream gateways without
// materially prolonging task runtime when the remote is genuinely down.
wikiLLMMaxAttempts = 3
// wikiLLMBackoffBase is the base delay for the exponential backoff
// between retry attempts. The nth retry waits base << (n-1) — so with
// a 2s base we wait 2s, 4s, 8s between attempts.
wikiLLMBackoffBase = 2 * time.Second
)
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
@@ -156,6 +167,7 @@ type wikiIngestService struct {
chunkRepo interfaces.ChunkRepository
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
logEntrySvc interfaces.WikiLogEntryService
redisClient *redis.Client // nil in Lite mode (no Redis)
// liteLocks provides per-KB mutual exclusion in Lite mode (no Redis).
// Keys are kbID strings; values are unused (presence = locked).
@@ -170,6 +182,7 @@ func NewWikiIngestService(
chunkRepo interfaces.ChunkRepository,
modelService interfaces.ModelService,
task interfaces.TaskEnqueuer,
logEntrySvc interfaces.WikiLogEntryService,
redisClient *redis.Client,
) interfaces.TaskHandler {
svc := &wikiIngestService{
@@ -179,6 +192,7 @@ func NewWikiIngestService(
chunkRepo: chunkRepo,
modelService: modelService,
task: task,
logEntrySvc: logEntrySvc,
redisClient: redisClient,
}
return svc
@@ -427,8 +441,11 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
type docIngestResult struct {
KnowledgeID string
DocTitle string
Summary string // one-line summary of the document (from summary page)
Pages []string // affected page slugs
Summary string // one-line summary of the document (from summary page)
// Pages records the wiki pages this document touched, carrying both
// the slug (for navigation / retract lookups) and the human-readable
// title captured at ingest time (for the log feed's display layer).
Pages []types.WikiLogPageRef
}
// WikiBatchContext holds shared data across Map and Reduce phases
@@ -695,62 +712,72 @@ type combinedExtraction struct {
Concepts []extractedItem `json:"concepts"`
}
// rebuildIndexPage regenerates the index page.
// rebuildIndexPage refreshes the LLM-generated intro that sits on the
// index wiki_pages row.
//
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
// Subsequent: incrementally updated with changeDescription.
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
// History: the index page used to store "intro + full directory listing" as
// a single multi-MB markdown blob in content. Every ingest batch rewrote
// the whole column, which on KBs with tens of thousands of pages caused
// O(N) TOAST writes per batch. The directory was lifted out into the
// structured GET /wiki/index endpoint (see wikiPageService.GetIndexView),
// and this method now only maintains the intro.
//
// Intro lifecycle:
// - First time (empty or legacy placeholder): generate from all document
// summaries via WikiIndexIntroPrompt.
// - Subsequent calls with a change description: incremental update via
// WikiIndexIntroUpdatePrompt so the intro reflects what just landed.
// - No change description: keep the existing intro untouched.
//
// The new intro is written to both Content and Summary so readers that
// still fall back to Summary (older clients, legacy migrations) stay in
// sync with the column the view actually renders.
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
if indexPage == nil {
return nil
}
// List all live pages
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
// Gather just the Summary-type pages the LLM needs for intro framing.
// We do NOT load or iterate every wiki page here — the directory is
// now assembled on demand by GetIndexView, which is the only reader
// that needs the full enumeration.
summaries, err := s.wikiService.ListByType(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary)
if err != nil {
return err
}
typeOrder := []string{
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
}
typeLabels := map[string]string{
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
types.WikiPageTypeComparison: "Comparison",
}
grouped := make(map[string][]*types.WikiPage)
totalPages := 0
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
if p.Status == types.WikiPageStatusArchived {
continue
}
grouped[p.PageType] = append(grouped[p.PageType], p)
totalPages++
}
// Build document summaries listing (only summary-type pages — they represent documents)
var docSummaries strings.Builder
for _, p := range grouped[types.WikiPageTypeSummary] {
for _, p := range summaries {
if p == nil || p.Status == types.WikiPageStatusArchived {
continue
}
fmt.Fprintf(&docSummaries, "<document>\n<title>%s</title>\n<summary>%s</summary>\n</document>\n\n", p.Title, p.Summary)
}
if docSummaries.Len() == 0 {
docSummaries.WriteString("(no documents yet)")
}
// Generate or update intro
existingIntro := indexPage.Summary
var intro string
// The intro lives on both Content and Summary. Prefer Content since
// that's what the new index view returns; fall back to Summary for
// rows written before this refactor so the incremental-update prompt
// has something to work with.
existingIntro := strings.TrimSpace(indexPage.Content)
if existingIntro == "" {
existingIntro = strings.TrimSpace(indexPage.Summary)
}
// Detect the legacy "intro + directory" payload. Such rows embed the
// fence-separated "## Summary" sections right after the intro, so we
// clip everything from the first directory heading onward to keep the
// intro length bounded when we feed it back into the update prompt.
if idx := strings.Index(existingIntro, "\n## "); idx >= 0 {
existingIntro = strings.TrimSpace(existingIntro[:idx])
}
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
// First time — generate intro from scratch
var intro string
switch {
case existingIntro == "" || existingIntro == "Wiki index - table of contents":
// First time — generate intro from scratch.
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
"DocumentSummaries": docSummaries.String(),
"Language": lang,
@@ -760,8 +787,8 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
} else {
intro = strings.TrimSpace(generatedIntro)
}
} else if changeDesc != "" {
// Incremental update — tell LLM what changed
case changeDesc != "":
// Incremental update — tell the LLM what changed.
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
"ExistingIntro": existingIntro,
"ChangeDescription": changeDesc,
@@ -773,45 +800,23 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
} else {
intro = strings.TrimSpace(updatedIntro)
}
} else {
intro = existingIntro // no change description, keep as-is
default:
// No change description and an existing intro: leave it as-is so
// we don't bump the version for a no-op.
intro = existingIntro
}
// Build directory (pure code, no LLM)
var dir strings.Builder
for _, pt := range typeOrder {
pages := grouped[pt]
if len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
for _, p := range pages {
summary := p.Summary
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
}
}
for pt, pages := range grouped {
inOrder := false
for _, o := range typeOrder {
if o == pt {
inOrder = true
break
}
}
if inOrder || len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
for _, p := range pages {
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
}
}
if totalPages == 0 {
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
// Defensive: some LLM outputs occasionally bleed into a directory-
// like section even when the intro prompt doesn't ask for one. If
// the freshly-generated intro starts to look like a legacy payload,
// clip it at the first "\n## " just like we did on the read path
// above. This keeps indexPage.Content a bounded intro-only blob.
if idx := strings.Index(intro, "\n## "); idx >= 0 {
intro = strings.TrimSpace(intro[:idx])
}
indexPage.Content = intro + "\n" + dir.String()
indexPage.Summary = intro // persist intro for next incremental update
indexPage.Content = intro
indexPage.Summary = intro
_, err = s.wikiService.UpdatePage(ctx, indexPage)
return err
}
@@ -834,34 +839,31 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// appendLogEntry appends a structured, grep-parseable entry to the log page.
// Format: ## [2026-04-07 19:50:02] action | title
// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean.
func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) {
logPage, _ := s.wikiService.GetLog(ctx, kbID)
if logPage == nil {
return
}
var sb strings.Builder
fmt.Fprintf(&sb, "\n## [%s] %s | %s\n",
time.Now().Format("2006-01-02 15:04:05"),
action,
docTitle,
)
if knowledgeID != "" {
fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID)
}
if summary != "" {
fmt.Fprintf(&sb, "- **Summary**: %s\n", summary)
}
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
// pure (no DB access) so callers can accumulate entries cheaply under their
// lock and flush them in a single AppendBatch call at the end of the batch.
//
// Historically this was a per-event `GetLog + UpdatePage` round trip, which
// rewrote the entire log page's TEXT column on every ingest/retract op —
// O(n^2) write amplification as the log grew. The batch writer now uses
// wikiLogEntryService.AppendBatch instead; see ProcessWikiIngest.
func (s *wikiIngestService) buildLogEntry(tenantID uint64, kbID, action, knowledgeID, docTitle, summary string, pagesAffected []types.WikiLogPageRef) *types.WikiLogEntry {
// Copy pagesAffected so the entry does not alias caller-owned slices.
// The batch accumulates SlugUpdate results that may be reused downstream.
var pages types.WikiLogPageRefs
if len(pagesAffected) > 0 {
fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", "))
pages = make(types.WikiLogPageRefs, len(pagesAffected))
copy(pages, pagesAffected)
}
logPage.Content = logPage.Content + sb.String()
if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to update log page: %v", err)
return &types.WikiLogEntry{
TenantID: tenantID,
KnowledgeBaseID: kbID,
Action: action,
KnowledgeID: knowledgeID,
DocTitle: docTitle,
Summary: summary,
PagesAffected: pages,
CreatedAt: time.Now(),
}
}
@@ -1017,7 +1019,23 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
return entities, concepts
}
// generateWithTemplate executes a prompt template and calls the LLM
// generateWithTemplate executes a prompt template and calls the LLM with
// bounded exponential-backoff retries for transient infrastructure errors.
//
// Retry policy:
// - Up to wikiLLMMaxAttempts total attempts (initial + retries).
// - Only retry errors classified as transient by isTransientLLMError:
// HTTP 408/429/5xx, context deadline exceeded (when the parent ctx is
// still alive), or generic "timeout"/"connection reset" wording.
// 4xx (except 408/429) is a caller-side fault and fails fast.
// - Backoff is exponential base 2s: 2s, 4s, 8s — roughly wikiLLMBackoffBase
// * 2^(attempt-1). Honors ctx cancellation so the task can abort.
//
// This exists because wiki ingest makes several independent LLM calls per
// document (extraction, summary, dedup, citations, intro) and a single
// transient 504 from the upstream gateway used to drop the document's
// summary page permanently. Retries plus failedOps requeuing (see
// mapOneDocument) turn those events into at-most-a-few-minute hiccups.
func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) {
tmpl, err := template.New("wiki").Parse(promptTpl)
if err != nil {
@@ -1031,17 +1049,98 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel
prompt := buf.String()
thinking := false
response, err := chatModel.Chat(ctx, []chat.Message{
{Role: "user", Content: prompt},
}, &chat.ChatOptions{
Temperature: 0.3,
Thinking: &thinking,
})
if err != nil {
return "", fmt.Errorf("LLM call failed: %w", err)
var lastErr error
for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ {
response, err := chatModel.Chat(ctx, []chat.Message{
{Role: "user", Content: prompt},
}, &chat.ChatOptions{
Temperature: 0.3,
Thinking: &thinking,
})
if err == nil {
return response.Content, nil
}
lastErr = err
// Abort immediately on non-retryable errors (4xx except 408/429,
// parse/marshal failures, tool-side bugs, etc.). Retrying a
// hard "invalid arguments" error just wastes the model's budget.
if !isTransientLLMError(ctx, err) {
return "", fmt.Errorf("LLM call failed: %w", err)
}
if attempt == wikiLLMMaxAttempts {
break
}
backoff := wikiLLMBackoffBase << (attempt - 1)
logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v",
attempt, wikiLLMMaxAttempts, backoff, err)
select {
case <-ctx.Done():
return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err())
case <-time.After(backoff):
}
}
return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr)
}
// isTransientLLMError reports whether an error from the chat provider
// looks like an infrastructure hiccup worth retrying. Classification is
// intentionally conservative: the truthful "could not tell, assume
// permanent" choice keeps retries cheap and avoids masking real bugs.
//
// We treat the following as transient:
// - HTTP 408 (client request timeout — upstream usually didn't process),
// 429 (rate-limited — retry after backoff may succeed), 5xx (any
// server-side fault, including the 504 "Remote error, timeout with
// 60" we see from the gateway in front of several LLM providers).
// - Wrapped context.DeadlineExceeded when the parent ctx is still alive
// (nested per-call timeouts).
// - Substring matches on the error text for common transport failures
// ("timeout", "connection reset", "EOF") that providers surface
// without a structured status code.
func isTransientLLMError(ctx context.Context, err error) bool {
if err == nil {
return false
}
// Never retry after the parent ctx itself expired — the task is
// being cancelled and the next attempt would just fail again.
if ctx.Err() != nil {
return false
}
return response.Content, nil
msg := err.Error()
// Providers that bubble HTTP status up formatted as
// "API request failed with status NNN: ..." — match that first.
for _, s := range []string{
"status 408", "status 429",
"status 500", "status 501", "status 502", "status 503", "status 504",
"status 520", "status 521", "status 522", "status 523", "status 524",
} {
if strings.Contains(msg, s) {
return true
}
}
lower := strings.ToLower(msg)
for _, s := range []string{
"timeout",
"timed out",
"connection reset",
"connection refused",
"broken pipe",
"no such host", // DNS hiccup
"i/o timeout",
"unexpected eof",
"tls handshake",
"context deadline exceeded", // nested per-call deadline
} {
if strings.Contains(lower, s) {
return true
}
}
return false
}
// --- Helpers ---
@@ -392,14 +392,42 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
totalPagesAffected = len(allPagesAffected)
// Append log entries — one per operation for chronological traceability
// Collect log entries for this batch and flush them in a single INSERT.
// Historically each op triggered its own `GetLog + UpdatePage` round
// trip, which rewrote the entire log page TEXT column and caused O(n^2)
// write amplification as the log grew. AppendBatch writes one row per
// event into wiki_log_entries instead.
//
// slugsToRefs resolves each retract slug against the batch-start
// snapshot (batchCtx.SlugTitleMap) so the log feed carries titles for
// pages that existed when the batch began. Pages created or renamed
// during this batch fall through the map lookup and log as slug-only
// refs, which the frontend renders as the slug itself — a sensible
// fallback given retracts only touch pre-existing pages.
slugsToRefs := func(slugs []string) []types.WikiLogPageRef {
if len(slugs) == 0 {
return nil
}
out := make([]types.WikiLogPageRef, 0, len(slugs))
for _, slug := range slugs {
title := batchCtx.SlugTitleMap[slug]
out = append(out, types.WikiLogPageRef{Slug: slug, Title: title})
}
return out
}
logEntries := make([]*types.WikiLogEntry, 0, len(pendingOps)+len(docResults))
for _, op := range pendingOps {
if op.Op == WikiOpRetract {
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, op.PageSlugs)
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, slugsToRefs(op.PageSlugs)))
}
}
for _, r := range docResults {
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages)
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages))
}
if len(logEntries) > 0 && s.logEntrySvc != nil {
if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err)
}
}
// Build change description for the Index Intro LLM prompt
@@ -625,9 +653,18 @@ func (s *wikiIngestService) mapOneDocument(
}
}
extractedPages := make([]string, 0, len(slugItems)+1)
for slug := range slugItems {
extractedPages = append(extractedPages, slug)
// extractedPages records every wiki page this document materialized
// (entities, concepts, plus the summary page appended below). The
// slug is used for link/retract bookkeeping; the title is captured
// for the log feed so the user sees "提供本学位在线验证报告查询…"
// rather than "entity/xue-xin-wang".
extractedPages := make([]types.WikiLogPageRef, 0, len(slugItems)+1)
for slug, item := range slugItems {
title := item.Name
if title == "" {
title = slug
}
extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: slug, Title: title})
}
// Count total distinct chunks cited across all slugs for logging.
@@ -647,32 +684,44 @@ func (s *wikiIngestService) mapOneDocument(
var docSummary string
if summaryErr != nil {
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, summaryErr)
} else {
sumLine, sumBody := splitSummaryLine(summaryContent)
if sumBody == "" {
sumBody = summaryContent
}
if sumLine == "" {
sumLine = docTitle
}
docSummaryLine = sumLine
docSummary = sumBody
if strings.TrimSpace(docSummary) == "" {
docSummary = sumLine
}
updates = append(updates, SlugUpdate{
Slug: summarySlug,
Type: types.WikiPageTypeSummary,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
SummaryLine: sumLine,
SummaryBody: sumBody,
})
extractedPages = append(extractedPages, summarySlug)
// Summary is the headline artifact of an ingested document — a
// document with no summary page is half-ingested and leaves the
// entity/concept updates hanging without a root to link back to
// from the index. Historically we just logged and moved on,
// which meant a single transient 504 permanently dropped the
// summary page for that document.
//
// Returning an error here sends the op to failedOps (see the
// map-phase loop in ProcessWikiIngest), which requeueFailedOps
// appends back onto the pending list so the next batch retries.
// The internal retries in generateWithTemplate already exhaust
// the LLM's own transient-error budget before we give up here.
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s, will requeue: %v", knowledgeID, summaryErr)
return nil, nil, fmt.Errorf("generate summary: %w", summaryErr)
}
sumLine, sumBody := splitSummaryLine(summaryContent)
if sumBody == "" {
sumBody = summaryContent
}
if sumLine == "" {
sumLine = docTitle
}
docSummaryLine = sumLine
docSummary = sumBody
if strings.TrimSpace(docSummary) == "" {
docSummary = sumLine
}
updates = append(updates, SlugUpdate{
Slug: summarySlug,
Type: types.WikiPageTypeSummary,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
SummaryLine: sumLine,
SummaryBody: sumBody,
})
extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: summarySlug, Title: docTitle})
// Entities
for _, item := range extractedEntities {
@@ -740,7 +789,7 @@ func (s *wikiIngestService) mapOneDocument(
newSlugSet := make(map[string]bool, len(extractedPages))
for _, ns := range extractedPages {
newSlugSet[ns] = true
newSlugSet[ns.Slug] = true
}
var reparseOverlap, staleCount int
@@ -0,0 +1,99 @@
package service
import (
"context"
"errors"
"testing"
)
// TestIsTransientLLMError_HTTPStatuses covers the status codes we know
// the generic REST chat provider bubbles up as "API request failed with
// status NNN: ...". The status-code arm of the classifier is the one
// that actually fires for the 504 gateway timeouts the upstream LLM
// vendors throw most often.
func TestIsTransientLLMError_HTTPStatuses(t *testing.T) {
ctx := context.Background()
cases := []struct {
msg string
want bool
}{
{"API request failed with status 504: Remote error, timeout with 60", true},
{"API request failed with status 503: service unavailable", true},
{"API request failed with status 502: bad gateway", true},
{"API request failed with status 500: internal server error", true},
{"API request failed with status 429: rate limited", true},
{"API request failed with status 408: request timeout", true},
{"API request failed with status 520: Web server returned unknown", true},
{"API request failed with status 524: timeout (Cloudflare)", true},
// 4xx permanent: a retry will just fail the same way.
{"API request failed with status 400: bad request", false},
{"API request failed with status 401: unauthorized", false},
{"API request failed with status 403: forbidden (quota exhausted)", false},
{"API request failed with status 404: model not found", false},
}
for _, tc := range cases {
got := isTransientLLMError(ctx, errors.New(tc.msg))
if got != tc.want {
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
}
}
}
// TestIsTransientLLMError_TransportSubstrings covers providers (and Go
// stdlib errors) that surface transient conditions as free-form text
// without a numeric status code — DNS blips, TLS handshake failures,
// broken pipes during an in-flight body read, etc.
func TestIsTransientLLMError_TransportSubstrings(t *testing.T) {
ctx := context.Background()
cases := []struct {
msg string
want bool
}{
{"send request: dial tcp: lookup api.example.com: no such host", true},
{"send request: read tcp 10.0.0.1: i/o timeout", true},
{"send request: Post \"...\": context deadline exceeded", true},
{"send request: connection reset by peer", true},
{"send request: connection refused", true},
{"send request: broken pipe", true},
{"send request: tls handshake timeout", true},
{"send request: unexpected EOF", true},
// Not transient: the error is actionable (missing model id,
// malformed tool call, etc.) — retry is just wasted budget.
{"model not configured for tool use", false},
{"invalid JSON in tool arguments", false},
{"parse template: syntax error", false},
}
for _, tc := range cases {
got := isTransientLLMError(ctx, errors.New(tc.msg))
if got != tc.want {
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
}
}
}
// TestIsTransientLLMError_AbortsWhenParentCtxDone ensures a cancelled
// outer context short-circuits the retry decision. Without this guard
// an ingest task that was interrupted (asynq shutdown, deletion race,
// parent timeout) would keep retrying the LLM during its teardown
// window and waste the remainder of the backoff budget.
func TestIsTransientLLMError_AbortsWhenParentCtxDone(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
// This exact error would normally be classified transient (see
// TestIsTransientLLMError_HTTPStatuses), but a cancelled ctx must
// override that so the task can unwind promptly.
err := errors.New("API request failed with status 504: Remote error")
if isTransientLLMError(ctx, err) {
t.Fatal("cancelled ctx should short-circuit to non-transient")
}
}
// TestIsTransientLLMError_NilError is a sanity check — nil should never
// be treated as retryable, otherwise callers that forget to handle the
// happy path would spin forever.
func TestIsTransientLLMError_NilError(t *testing.T) {
if isTransientLLMError(context.Background(), nil) {
t.Fatal("nil error should not be transient")
}
}
@@ -0,0 +1,51 @@
package service
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// wikiLogEntryService is a thin wrapper over WikiLogEntryRepository. The
// event layer carries no business rules today, so the service just
// forwards to the repo — the indirection exists so handlers / ingest
// batch code depend on an interface rather than a concrete repo type.
type wikiLogEntryService struct {
repo interfaces.WikiLogEntryRepository
}
// NewWikiLogEntryService constructs a WikiLogEntryService backed by the
// given repository.
func NewWikiLogEntryService(repo interfaces.WikiLogEntryRepository) interfaces.WikiLogEntryService {
return &wikiLogEntryService{repo: repo}
}
// AppendBatch records the given events in one database round trip. Empty
// batches are a no-op (the repo handles that).
func (s *wikiLogEntryService) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error {
return s.repo.AppendBatch(ctx, entries)
}
// List paginates the per-KB event feed. See repo.List for cursor semantics.
func (s *wikiLogEntryService) List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error) {
entries, nextCursor, err := s.repo.List(ctx, kbID, cursor, limit)
if err != nil {
return nil, err
}
if entries == nil {
// Normalise to an empty slice so clients don't need to
// distinguish `null` from `[]`.
entries = []*types.WikiLogEntry{}
}
return &types.WikiLogEntryListResponse{
Entries: entries,
NextCursor: nextCursor,
}, nil
}
// DeleteByKB removes the log feed for a KB. Called when the KB itself is
// being deleted, so no further reads happen.
func (s *wikiLogEntryService) DeleteByKB(ctx context.Context, kbID string) error {
return s.repo.DeleteByKB(ctx, kbID)
}
+141 -77
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -246,13 +247,125 @@ func (s *wikiPageService) GetIndex(ctx context.Context, kbID string) (*types.Wik
return page, nil
}
// GetLog returns the log page for a knowledge base
// wikiIndexContentPageTypes enumerates the page types that make up a wiki's
// user-visible directory. System pages (index/log) are excluded; any
// LLM-created type we do not recognize surfaces under a generic "other"
// bucket.
var wikiIndexContentPageTypes = []string{
types.WikiPageTypeSummary,
types.WikiPageTypeEntity,
types.WikiPageTypeConcept,
types.WikiPageTypeSynthesis,
types.WikiPageTypeComparison,
}
// GetIndexView builds the structured index response without ever
// materializing a multi-MB directory markdown string. Intro is read from
// the index wiki_page row (which now carries only intro text — see
// rebuildIndexPage). Each requested page_type is paginated independently
// with ListByTypeLight so reads stay O(page_size) rather than O(total
// pages in the KB).
//
// `pageTypes` narrows which groups to include; empty = all content types.
// `limit` is the per-group window size (defaults to 50, capped at 200).
// `cursor` is an opaque offset string; currently we use the stringified
// offset so clients can resume where they left off. Because different
// page_types paginate independently, `cursor` applies uniformly to every
// group — if the caller wants per-group cursors it should request one
// type at a time via `pageTypes`. That simplifies the wire format and
// matches the frontend's tabbed UX.
func (s *wikiPageService) GetIndexView(
ctx context.Context,
kbID string,
pageTypes []string,
limit int,
cursor string,
) (*types.WikiIndexResponse, error) {
indexPage, err := s.GetIndex(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("load index page: %w", err)
}
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
offset := 0
if cursor != "" {
v, parseErr := strconv.Atoi(cursor)
if parseErr != nil || v < 0 {
return nil, fmt.Errorf("invalid cursor %q", cursor)
}
offset = v
}
// Default to every known content type when the caller passes no
// filter. Any unknown request-time type is passed through verbatim so
// future page types (declared in types/wiki_page.go) start showing
// up in the index the moment the LLM starts creating them, without a
// handler change.
selected := pageTypes
if len(selected) == 0 {
selected = append([]string{}, wikiIndexContentPageTypes...)
}
groups := make([]types.WikiIndexGroup, 0, len(selected))
for _, pt := range selected {
entries, total, listErr := s.repo.ListByTypeLight(ctx, kbID, pt, limit, offset)
if listErr != nil {
return nil, fmt.Errorf("list %s pages: %w", pt, listErr)
}
if entries == nil {
entries = []types.WikiIndexEntry{}
}
next := ""
// Only emit a cursor when a full page was returned AND more rows
// remain past `offset + limit`. A short page or one that exactly
// consumed the remainder should signal end-of-feed.
if len(entries) == limit && int64(offset+len(entries)) < total {
next = strconv.Itoa(offset + limit)
}
groups = append(groups, types.WikiIndexGroup{
Type: pt,
Total: total,
Items: entries,
NextCursor: next,
})
}
// The intro used to be stored on indexPage.Summary while
// indexPage.Content held intro + directory markdown. After the
// directory was lifted out of wiki_pages the content column holds
// only the intro. Fall back to Summary for KBs that haven't been
// re-ingested since the change so the response is never blank.
intro := indexPage.Content
if strings.TrimSpace(intro) == "" {
intro = indexPage.Summary
}
return &types.WikiIndexResponse{
Intro: intro,
Version: indexPage.Version,
Groups: groups,
}, nil
}
// GetLog returns the wiki_pages row for slug='log' if it exists.
//
// Log events are now stored in the dedicated `wiki_log_entries` table and
// paginated via wikiLogEntryService — the per-KB log is no longer a single
// TEXT column on a wiki_pages row (that model caused O(n^2) write
// amplification as logs grew). This method is retained for callers that
// still probe the legacy row (wiki_lint, knowledge delete, etc.), but it
// no longer auto-creates the placeholder page on miss; a missing row is a
// normal state and the helper returns `nil, nil`.
func (s *wikiPageService) GetLog(ctx context.Context, kbID string) (*types.WikiPage, error) {
page, err := s.repo.GetBySlug(ctx, kbID, "log")
if err != nil {
if errors.Is(err, repository.ErrWikiPageNotFound) {
return s.createDefaultPage(ctx, kbID, "log", "Log", types.WikiPageTypeLog,
"# Wiki Operation Log\n\nChronological record of wiki operations.\n")
return nil, nil
}
return nil, err
}
@@ -619,6 +732,13 @@ func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*typ
return s.repo.ListAll(ctx, kbID)
}
// ListByType returns every wiki page of a given type for a KB. Exposed so
// callers like intro regeneration can load only the page type they need
// (summaries) instead of paying for the full ListAll scan.
func (s *wikiPageService) ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) {
return s.repo.ListByType(ctx, kbID, pageType)
}
// ListPagesBySourceRef exposes the repository's source-ref lookup so higher
// layers (delete flow, retract reconciliation) can re-query the current wiki
// state without depending on a stale caller-captured slug list.
@@ -821,79 +941,23 @@ func (s *wikiPageService) InjectCrossLinks(ctx context.Context, kbID string, aff
}
}
// RebuildIndexPage regenerates the index page directory.
// RebuildIndexPage was historically called by agent write/rename tools to
// refresh the index page's directory listing after a page mutation.
//
// The directory is no longer persisted in wiki_pages.content — it is
// assembled on demand by GetIndexView from the lightweight ListByTypeLight
// projection, so individual page writes don't need to redo O(N) string
// concatenation and rewrite a multi-MB TEXT column anymore. Keeping the
// method name lets existing agent tool call sites (wiki_write_page,
// wiki_rename_page) compile unchanged; the body is now intentionally a
// no-op.
//
// The intro that still lives on the index row is managed separately by
// the ingest pipeline (see wikiIngestService.rebuildIndexPage) on batch
// completion, which is where we actually have the LLM + change description
// context needed to rewrite it.
func (s *wikiPageService) RebuildIndexPage(ctx context.Context, kbID string) error {
indexPage, err := s.GetIndex(ctx, kbID)
if err != nil {
return err
}
allPages, err := s.ListAllPages(ctx, kbID)
if err != nil {
return err
}
typeOrder := []string{
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
}
typeLabels := map[string]string{
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
types.WikiPageTypeComparison: "Comparison",
}
grouped := make(map[string][]*types.WikiPage)
totalPages := 0
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
if p.Status == types.WikiPageStatusArchived {
continue
}
grouped[p.PageType] = append(grouped[p.PageType], p)
totalPages++
}
var dir strings.Builder
for _, pt := range typeOrder {
pages := grouped[pt]
if len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
for _, p := range pages {
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
}
}
for pt, pages := range grouped {
inOrder := false
for _, o := range typeOrder {
if o == pt {
inOrder = true
break
}
}
if inOrder || len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
for _, p := range pages {
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
}
}
if totalPages == 0 {
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
}
intro := indexPage.Summary
if intro == "" {
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
indexPage.Summary = intro
}
indexPage.Content = intro + "\n" + dir.String()
_, err = s.UpdatePage(ctx, indexPage)
return err
_ = ctx
_ = kbID
return nil
}
+2
View File
@@ -153,6 +153,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(repository.NewDataSourceRepository))
must(container.Provide(repository.NewSyncLogRepository))
must(container.Provide(repository.NewWikiPageRepository))
must(container.Provide(repository.NewWikiLogEntryRepository))
// MCP manager for managing MCP client connections
logger.Debugf(ctx, "[Container] Registering MCP manager...")
@@ -186,6 +187,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(service.NewCustomAgentService))
must(container.Provide(memoryService.NewMemoryService))
must(container.Provide(service.NewWikiPageService))
must(container.Provide(service.NewWikiLogEntryService))
must(container.Provide(service.NewWikiIngestService, dig.Name("wikiIngest")))
must(container.Provide(service.NewWikiLintService))
+56 -17
View File
@@ -19,9 +19,10 @@ import (
// WikiPageHandler handles HTTP requests for wiki page operations
type WikiPageHandler struct {
wikiService interfaces.WikiPageService
kbService interfaces.KnowledgeBaseService
lintService *service.WikiLintService
wikiService interfaces.WikiPageService
kbService interfaces.KnowledgeBaseService
lintService *service.WikiLintService
logEntryService interfaces.WikiLogEntryService
}
// NewWikiPageHandler creates a new wiki page handler
@@ -29,11 +30,13 @@ func NewWikiPageHandler(
wikiService interfaces.WikiPageService,
kbService interfaces.KnowledgeBaseService,
lintService *service.WikiLintService,
logEntryService interfaces.WikiLogEntryService,
) *WikiPageHandler {
return &WikiPageHandler{
wikiService: wikiService,
kbService: kbService,
lintService: lintService,
wikiService: wikiService,
kbService: kbService,
lintService: lintService,
logEntryService: logEntryService,
}
}
@@ -274,12 +277,19 @@ func (h *WikiPageHandler) DeletePage(c *gin.Context) {
}
// GetIndex godoc
// @Summary Get wiki index page
// @Description Returns the wiki index page (creates default if not exists)
// @Summary Get wiki index view
// @Description Returns the wiki index as intro text plus per-type paginated
// @Description directory groups. The heavy directory markdown that used to
// @Description live in wiki_pages.content was replaced with this structured
// @Description response so a KB with tens of thousands of pages no longer
// @Description materializes megabytes of TEXT on every index open.
// @Tags Wiki
// @Produce json
// @Param kb_id path string true "Knowledge base ID"
// @Success 200 {object} types.WikiPage
// @Param kb_id path string true "Knowledge base ID"
// @Param types query string false "Comma-separated page types (default: all content types)"
// @Param limit query int false "Per-group window size, 1-200 (default 50)"
// @Param cursor query string false "Opaque offset cursor from previous response"
// @Success 200 {object} types.WikiIndexResponse
// @Security Bearer
// @Router /api/v1/knowledgebase/{kb_id}/wiki/index [get]
func (h *WikiPageHandler) GetIndex(c *gin.Context) {
@@ -289,22 +299,43 @@ func (h *WikiPageHandler) GetIndex(c *gin.Context) {
return
}
page, err := h.wikiService.GetIndex(c.Request.Context(), kbID)
var pageTypes []string
if raw := c.Query("types"); raw != "" {
for _, t := range strings.Split(raw, ",") {
t = strings.TrimSpace(t)
if t != "" {
pageTypes = append(pageTypes, t)
}
}
}
limit := 50
if raw := c.Query("limit"); raw != "" {
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
limit = v
}
}
resp, err := h.wikiService.GetIndexView(c.Request.Context(), kbID, pageTypes, limit, c.Query("cursor"))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, page)
c.JSON(http.StatusOK, resp)
}
// GetLog godoc
// @Summary Get wiki operation log
// @Description Returns the wiki operation log page (creates default if not exists)
// @Description Returns a paginated feed of wiki operation events (ingest, retract, ...)
// @Description newest-first. Pagination is cursor-based: pass `next_cursor` from the
// @Description previous response back as `cursor` to fetch the next page.
// @Tags Wiki
// @Produce json
// @Param kb_id path string true "Knowledge base ID"
// @Success 200 {object} types.WikiPage
// @Param kb_id path string true "Knowledge base ID"
// @Param cursor query string false "Opaque cursor from the previous page (empty = newest)"
// @Param limit query int false "Page size, 1-200 (default 50)"
// @Success 200 {object} types.WikiLogEntryListResponse
// @Security Bearer
// @Router /api/v1/knowledgebase/{kb_id}/wiki/log [get]
func (h *WikiPageHandler) GetLog(c *gin.Context) {
@@ -314,13 +345,21 @@ func (h *WikiPageHandler) GetLog(c *gin.Context) {
return
}
page, err := h.wikiService.GetLog(c.Request.Context(), kbID)
cursor := c.Query("cursor")
limit := 50
if raw := c.Query("limit"); raw != "" {
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
limit = v
}
}
resp, err := h.logEntryService.List(c.Request.Context(), kbID, cursor, limit)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, page)
c.JSON(http.StatusOK, resp)
}
// Graph query parameter bounds. The defaults cap an `overview` request at
@@ -0,0 +1,41 @@
package interfaces
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
)
// WikiLogEntryRepository persists per-KB wiki operation events. The table
// is append-only; updates are never issued. Reads paginate newest-first
// with a stable numeric cursor.
type WikiLogEntryRepository interface {
// AppendBatch inserts a batch of events in a single INSERT...VALUES
// statement. All entries in a batch must belong to the same KB in
// practice, but the repo does not enforce that — the caller owns the
// grouping.
AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error
// List returns up to `limit` entries in (knowledge_base_id, id DESC)
// order. `cursor` is the ID of the last entry from the previous page
// ("return entries strictly older than this"); empty string means
// "from the newest".
//
// The returned nextCursor is empty when no further rows remain; the
// caller can treat that as end-of-stream.
List(ctx context.Context, kbID string, cursor string, limit int) (entries []*types.WikiLogEntry, nextCursor string, err error)
// DeleteByKB removes every log entry belonging to the given KB. Used
// when a wiki KB is deleted so the log table does not accumulate
// orphans. Safe to call even if no rows exist.
DeleteByKB(ctx context.Context, kbID string) error
}
// WikiLogEntryService is the service layer over WikiLogEntryRepository.
// Kept intentionally thin — it mirrors the repo today because the events
// are pure bookkeeping and carry no business rules.
type WikiLogEntryService interface {
AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error
List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error)
DeleteByKB(ctx context.Context, kbID string) error
}
+36 -1
View File
@@ -50,10 +50,33 @@ type WikiPageService interface {
// GetIndex returns the index page for a knowledge base.
// Creates a default one if it doesn't exist.
//
// The index row now carries only the LLM-generated intro in its content
// column — the heavy directory listing was moved out of wiki_pages.
// Callers that want a renderable directory should use GetIndexView,
// which assembles {intro, groups} structured output on demand.
GetIndex(ctx context.Context, kbID string) (*types.WikiPage, error)
// GetIndexView returns the structured index response — intro (from the
// index row) plus a paginated window of directory entries per requested
// page_type. `types` restricts which page types to include (empty =
// all), `limit` bounds the per-type window size, `cursor` is an opaque
// offset string resumed from a previous response.
//
// Keeping the directory as JSON rather than a multi-MB markdown blob
// means a 40k-page KB no longer pays O(N) transport + rendering cost
// on every index open. See /wiki/index handler + WikiBrowser.vue for
// the consumer.
GetIndexView(ctx context.Context, kbID string, pageTypes []string, limit int, cursor string) (*types.WikiIndexResponse, error)
// GetLog returns the log page for a knowledge base.
// Creates a default one if it doesn't exist.
//
// Wiki operation events now live in the dedicated wiki_log_entries
// table, so this method no longer auto-creates a placeholder row on
// miss and may legitimately return (nil, nil) for KBs that never had
// the legacy row written. Retained for back-compat with callers that
// still probe the row (lint, knowledge delete); new code should use
// WikiLogEntryService.List for the event feed instead.
GetLog(ctx context.Context, kbID string) (*types.WikiPage, error)
// GetGraph returns the link graph data for visualization. The caller
@@ -81,6 +104,12 @@ type WikiPageService interface {
// Used for index rebuild, graph generation, cross-link injection, etc.
ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error)
// ListByType returns every wiki page of the given type. Used by
// callers that need the full set for a specific type (e.g. intro
// regeneration needs summary pages). For directory rendering, use
// GetIndexView instead — it paginates via ListByTypeLight.
ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error)
// ListPagesBySourceRef retrieves all wiki pages whose source_refs reference
// the given knowledge ID. Used by delete/ingest reconciliation paths that
// need to find pages touched by a specific document at read time (rather
@@ -136,6 +165,12 @@ type WikiPageRepository interface {
// ListByType retrieves all wiki pages of a given type within a knowledge base.
ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error)
// ListByTypeLight returns a paginated window of lightweight entries
// (slug/title/summary only) for the given page_type plus the total
// non-archived count. Used by the structured index API so reads do not
// have to materialize TEXT content for every wiki_pages row.
ListByTypeLight(ctx context.Context, kbID string, pageType string, limit int, offset int) ([]types.WikiIndexEntry, int64, error)
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error)
+117
View File
@@ -0,0 +1,117 @@
package types
import (
"database/sql/driver"
"encoding/json"
"fmt"
"time"
)
// WikiLogPageRef identifies a wiki page that was affected by a log event,
// paired with the human-readable title at the time of the event. Title is
// captured at write time so the log feed stays readable even if the page
// is later renamed or deleted — the previously-stored title is the one
// the user remembers seeing in the event stream.
type WikiLogPageRef struct {
Slug string `json:"slug"`
Title string `json:"title,omitempty"`
}
// WikiLogPageRefs is a JSON-marshalled list of page references. The
// column is TEXT/JSONB depending on driver; the Value/Scan pair keeps
// GORM happy on both Postgres and the SQLite tests.
type WikiLogPageRefs []WikiLogPageRef
// Value implements the driver.Valuer interface.
func (r WikiLogPageRefs) Value() (driver.Value, error) {
if r == nil {
return []byte("[]"), nil
}
return json.Marshal(r)
}
// Scan implements the sql.Scanner interface. Also tolerates the legacy
// []string shape so we can roll forward without a data migration — any
// rows the ingest pipeline wrote before this commit surface as refs
// with a slug and an empty title, which the frontend renders identically.
func (r *WikiLogPageRefs) Scan(value interface{}) error {
if value == nil {
*r = nil
return nil
}
var raw []byte
switch v := value.(type) {
case []byte:
raw = v
case string:
raw = []byte(v)
default:
return fmt.Errorf("unsupported type for WikiLogPageRefs: %T", value)
}
// Try the new [{slug,title}] shape first.
var refs []WikiLogPageRef
if err := json.Unmarshal(raw, &refs); err == nil {
*r = refs
return nil
}
// Fall back to the legacy []string shape.
var slugs []string
if err := json.Unmarshal(raw, &slugs); err != nil {
return err
}
out := make([]WikiLogPageRef, 0, len(slugs))
for _, s := range slugs {
out = append(out, WikiLogPageRef{Slug: s})
}
*r = out
return nil
}
// WikiLogEntry is a single ingest/retract/edit event appended to the
// per-KB operation log. It replaces the legacy "single giant TEXT column
// on slug='log' wiki_pages row" model, which rewrote the whole column on
// every event and caused O(n^2) write amplification as the KB grew.
//
// Each event is one INSERT; reads paginate by (knowledge_base_id, id DESC).
type WikiLogEntry struct {
// Auto-increment identifier. Monotonic within a single database, so
// frontend pagination uses it as a stable cursor without needing to
// disambiguate identical created_at values.
ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"`
// Tenant scope, mirrored from the enclosing knowledge base.
TenantID uint64 `json:"tenant_id" gorm:"index"`
// Knowledge base this event belongs to.
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
// Short operation tag: "ingest", "retract", etc. Matches the `action`
// argument historically passed to appendLogEntry.
Action string `json:"action" gorm:"type:varchar(32)"`
// Knowledge ID the event was about (may be empty for KB-level events).
KnowledgeID string `json:"knowledge_id" gorm:"type:varchar(36);default:''"`
// Document title at the time of the event. Stored verbatim rather than
// joined at read time so deleted knowledge still has a human-readable
// label in the log.
DocTitle string `json:"doc_title" gorm:"type:text"`
// One-line summary of the change, as it was when the event was logged.
Summary string `json:"summary" gorm:"type:text"`
// Wiki pages affected by this event. Each ref carries both slug (for
// navigation) and title (for display) so the log renders human-
// readable text without a post-hoc slug→title lookup that might fail
// for now-deleted pages.
PagesAffected WikiLogPageRefs `json:"pages_affected" gorm:"type:jsonb;default:'[]'"`
// Server-side timestamp (UTC).
CreatedAt time.Time `json:"created_at"`
}
// TableName specifies the database table name.
func (WikiLogEntry) TableName() string {
return "wiki_log_entries"
}
// WikiLogEntryListResponse is the paginated response for GET /wiki/log.
// NextCursor is the stringified ID of the oldest entry in `Entries`; pass
// it back as `cursor` to fetch the next page. Empty string means "no more".
type WikiLogEntryListResponse struct {
Entries []*WikiLogEntry `json:"entries"`
NextCursor string `json:"next_cursor,omitempty"`
}
+32
View File
@@ -292,3 +292,35 @@ type WikiPageIssue struct {
func (WikiPageIssue) TableName() string {
return "wiki_page_issues"
}
// WikiIndexEntry is a single row in the structured wiki index response.
// Only the columns needed to render a clickable directory entry are
// carried — the backend projects SELECT slug, title, summary so a 40k-
// page KB does not pay for TEXT content transport on every index open.
type WikiIndexEntry struct {
Slug string `json:"slug"`
Title string `json:"title"`
Summary string `json:"summary"`
}
// WikiIndexGroup bundles the entries for one page_type into a page-sized
// slice. `Total` is the full count across the KB for the type; `Items`
// holds the current paginated window starting at `NextOffset - len(Items)`.
// An empty NextCursor means the window is already at the end of the type.
type WikiIndexGroup struct {
Type string `json:"type"`
Total int64 `json:"total"`
Items []WikiIndexEntry `json:"items"`
NextCursor string `json:"next_cursor,omitempty"`
}
// WikiIndexResponse is what GET /wiki/index returns. The heavy directory
// markdown that used to sit in wiki_pages.content is gone — only the LLM-
// generated intro survives there. Everything else is assembled on demand
// from the index repo's light-column projection, keeping index reads
// O(page_size) regardless of KB size.
type WikiIndexResponse struct {
Intro string `json:"intro"`
Version int `json:"version"`
Groups []WikiIndexGroup `json:"groups"`
}
@@ -0,0 +1,9 @@
-- Migration: 000040_wiki_log_entries (rollback)
-- Description: Drop the wiki_log_entries table. No data migration was
-- performed on the way up, so rollback is a plain DROP.
DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] Reverting wiki_log_entries schema'; END $$;
DROP TABLE IF EXISTS wiki_log_entries;
DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] wiki_log_entries schema reverted successfully'; END $$;
@@ -0,0 +1,32 @@
-- Migration: 000040_wiki_log_entries
-- Description: Dedicated append-only event table for wiki operation log entries.
-- Replaces the "single giant TEXT column on slug='log' wiki_pages
-- row" model that caused O(n^2) write amplification as KBs grew:
-- every ingest/retract op previously did GetLog + UpdatePage
-- rewriting the entire (potentially multi-MB) TEXT column.
-- Here each event is one INSERT; reads paginate by (kb_id, id DESC).
DO $$ BEGIN RAISE NOTICE '[Migration 000040] Applying wiki_log_entries schema'; END $$;
CREATE TABLE IF NOT EXISTS wiki_log_entries (
id BIGSERIAL PRIMARY KEY,
tenant_id BIGINT NOT NULL,
knowledge_base_id VARCHAR(36) NOT NULL,
action VARCHAR(32) NOT NULL,
knowledge_id VARCHAR(36) NOT NULL DEFAULT '',
doc_title TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
pages_affected JSONB NOT NULL DEFAULT '[]'::JSONB,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
-- Primary list query: cursor-paginated feed per KB, newest first.
-- Using id (BIGSERIAL, monotonic) as the cursor sidesteps duplicate-timestamp
-- tie-breaking that a created_at cursor would require.
CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_kb_id_desc
ON wiki_log_entries (knowledge_base_id, id DESC);
CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_tenant_id
ON wiki_log_entries (tenant_id);
DO $$ BEGIN RAISE NOTICE '[Migration 000040] wiki_log_entries schema applied successfully'; END $$;