perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.
Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.
Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.
Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.
Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.
- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
renderIndexOverviewForAgent output, isTransientLLMError classifier
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -115,12 +115,85 @@ export function deleteWikiPage(kbId: string, slug: string) {
|
||||
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
|
||||
}
|
||||
|
||||
export function getWikiIndex(kbId: string) {
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/index`);
|
||||
export interface WikiIndexEntryDTO {
|
||||
slug: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export function getWikiLog(kbId: string) {
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/log`);
|
||||
export interface WikiIndexGroup {
|
||||
type: string;
|
||||
total: number;
|
||||
items: WikiIndexEntryDTO[];
|
||||
next_cursor?: string;
|
||||
}
|
||||
|
||||
export interface WikiIndexResponse {
|
||||
intro: string;
|
||||
version: number;
|
||||
groups: WikiIndexGroup[];
|
||||
}
|
||||
|
||||
// getWikiIndex fetches the structured index view for a wiki KB. The
|
||||
// backend replaced the legacy "markdown blob of intro + directory" with
|
||||
// { intro, groups } so a 40k-page wiki no longer round-trips multiple
|
||||
// megabytes on every index open. Pass `types` to restrict which
|
||||
// page_type buckets come back; `limit` bounds the per-group window;
|
||||
// `cursor` resumes from a previous response.
|
||||
export function getWikiIndex(
|
||||
kbId: string,
|
||||
params?: { types?: string[]; limit?: number; cursor?: string },
|
||||
) {
|
||||
const query = new URLSearchParams();
|
||||
if (params) {
|
||||
if (params.types && params.types.length > 0) query.set('types', params.types.join(','));
|
||||
if (params.limit !== undefined) query.set('limit', String(params.limit));
|
||||
if (params.cursor) query.set('cursor', params.cursor);
|
||||
}
|
||||
const qs = query.toString();
|
||||
const suffix = qs ? `?${qs}` : '';
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/index${suffix}`);
|
||||
}
|
||||
|
||||
export interface WikiLogPageRef {
|
||||
slug: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface WikiLogEntry {
|
||||
id: number;
|
||||
tenant_id: number;
|
||||
knowledge_base_id: string;
|
||||
action: string;
|
||||
knowledge_id: string;
|
||||
doc_title: string;
|
||||
summary: string;
|
||||
// Each ref carries both slug (for navigation) and title (captured at
|
||||
// ingest time for display). Legacy rows written before the title
|
||||
// column was added surface as refs with an empty title; render falls
|
||||
// back to the slug in that case.
|
||||
pages_affected: WikiLogPageRef[];
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface WikiLogListResponse {
|
||||
entries: WikiLogEntry[];
|
||||
next_cursor?: string;
|
||||
}
|
||||
|
||||
// getWikiLog fetches a page of wiki operation events (newest first). Pass the
|
||||
// `next_cursor` from the previous response back as `cursor` to load more;
|
||||
// an empty / missing `next_cursor` signals end-of-feed. `limit` is clamped
|
||||
// server-side to [1, 200] and defaults to 50.
|
||||
export function getWikiLog(kbId: string, params?: { cursor?: string; limit?: number }) {
|
||||
const query = new URLSearchParams();
|
||||
if (params) {
|
||||
if (params.cursor) query.set('cursor', params.cursor);
|
||||
if (params.limit !== undefined) query.set('limit', String(params.limit));
|
||||
}
|
||||
const qs = query.toString();
|
||||
const suffix = qs ? `?${qs}` : '';
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/log${suffix}`);
|
||||
}
|
||||
|
||||
export interface WikiGraphQueryParams {
|
||||
|
||||
@@ -1708,6 +1708,12 @@ export default {
|
||||
fitView: 'Fit to View',
|
||||
logTitle: 'Activity Log',
|
||||
indexTitle: 'Index',
|
||||
logFeedTag: 'Event feed',
|
||||
logEmpty: 'No log entries yet',
|
||||
logLoading: 'Loading…',
|
||||
logLoadMore: 'Load more',
|
||||
indexOverviewTag: 'Directory',
|
||||
indexEmpty: 'No wiki pages yet. Upload documents first.',
|
||||
graphNoData: 'No graph data yet. Upload documents first.',
|
||||
showArrows: 'Show arrows',
|
||||
hideArrows: 'Hide arrows',
|
||||
|
||||
@@ -2320,6 +2320,12 @@ export default {
|
||||
fitView: "화면에 맞춤",
|
||||
logTitle: '활동 로그',
|
||||
indexTitle: '인덱스',
|
||||
logFeedTag: '이벤트 피드',
|
||||
logEmpty: '로그 기록이 없습니다',
|
||||
logLoading: '로딩 중…',
|
||||
logLoadMore: '더 보기',
|
||||
indexOverviewTag: '분류 목록',
|
||||
indexEmpty: 'Wiki 페이지가 없습니다. 먼저 문서를 업로드하세요.',
|
||||
graphNoData: '그래프 데이터가 없습니다. 먼저 문서를 업로드하세요.',
|
||||
showArrows: '화살표 표시',
|
||||
hideArrows: '화살표 숨기기',
|
||||
|
||||
@@ -1930,6 +1930,12 @@ export default {
|
||||
fitView: 'По размеру экрана',
|
||||
logTitle: 'Журнал активности',
|
||||
indexTitle: 'Индекс',
|
||||
logFeedTag: 'Лента событий',
|
||||
logEmpty: 'Записей журнала пока нет',
|
||||
logLoading: 'Загрузка…',
|
||||
logLoadMore: 'Загрузить ещё',
|
||||
indexOverviewTag: 'Каталог',
|
||||
indexEmpty: 'Пока нет wiki-страниц. Сначала загрузите документы.',
|
||||
graphNoData: 'Нет данных графа. Сначала загрузите документы.',
|
||||
showArrows: 'Показать стрелки',
|
||||
hideArrows: 'Скрыть стрелки',
|
||||
|
||||
@@ -2291,6 +2291,12 @@ export default {
|
||||
fitView: "适应屏幕",
|
||||
logTitle: '日志',
|
||||
indexTitle: '索引',
|
||||
logFeedTag: '事件流',
|
||||
logEmpty: '暂无日志记录',
|
||||
logLoading: '加载中…',
|
||||
logLoadMore: '加载更多',
|
||||
indexOverviewTag: '分类目录',
|
||||
indexEmpty: '暂无 Wiki 页面,请先上传文档',
|
||||
graphNoData: '暂无图谱数据,请先上传文档',
|
||||
showArrows: '显示箭头',
|
||||
hideArrows: '隐藏箭头',
|
||||
|
||||
@@ -241,27 +241,30 @@
|
||||
</template>
|
||||
|
||||
<template v-else>
|
||||
<!-- Index page (pinned at top) -->
|
||||
<!-- Index overview (pinned at top). Rendered lazily from a
|
||||
structured API response — never loads the full directory
|
||||
as markdown. -->
|
||||
<div
|
||||
v-if="indexPage"
|
||||
:class="['wiki-nav-item', { active: selectedPage?.id === indexPage.id }]"
|
||||
@click="selectPage(indexPage)"
|
||||
v-if="indexAvailable"
|
||||
:class="['wiki-nav-item', { active: activeSystemView === 'index' }]"
|
||||
@click="openIndexView"
|
||||
>
|
||||
<t-icon name="catalog" class="wiki-nav-icon" />
|
||||
<span class="wiki-nav-text">{{ $t('knowledgeEditor.wikiBrowser.indexTitle') }}</span>
|
||||
</div>
|
||||
|
||||
<!-- Log page (pinned) -->
|
||||
<!-- Log feed (pinned). Events live in wiki_log_entries and
|
||||
are loaded lazily when the user clicks this entry. -->
|
||||
<div
|
||||
v-if="logPage"
|
||||
:class="['wiki-nav-item', { active: selectedPage?.id === logPage.id }]"
|
||||
@click="selectPage(logPage)"
|
||||
v-if="logAvailable"
|
||||
:class="['wiki-nav-item', { active: activeSystemView === 'log' }]"
|
||||
@click="openLogView"
|
||||
>
|
||||
<t-icon name="history" class="wiki-nav-icon" />
|
||||
<span class="wiki-nav-text">{{ $t('knowledgeEditor.wikiBrowser.logTitle') }}</span>
|
||||
</div>
|
||||
|
||||
<div class="wiki-sidebar-divider" v-if="indexPage || logPage"></div>
|
||||
<div class="wiki-sidebar-divider" v-if="indexAvailable || logAvailable"></div>
|
||||
|
||||
<!-- Horizontal tab bar: one per non-empty page_type. Clicking a
|
||||
tab swaps the visible list to that bucket. Parallel tabs are
|
||||
@@ -337,10 +340,10 @@
|
||||
<div class="wiki-reader-inner">
|
||||
<template v-if="selectedPage">
|
||||
<!-- Navigation -->
|
||||
<div v-if="navHistory.length" class="wiki-nav-bar">
|
||||
<div v-if="navHistory.length || navFromSystemView" class="wiki-nav-bar">
|
||||
<a href="#" class="wiki-nav-back" @click.prevent="goBack">
|
||||
<t-icon name="arrow-left" size="14px" />
|
||||
<span>{{ navHistory[navHistory.length - 1].title }}</span>
|
||||
<span>{{ backLabel }}</span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -453,6 +456,91 @@
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- System view: index overview rendered as markdown. Starts
|
||||
with intro only; an IntersectionObserver-driven sentinel
|
||||
at the bottom auto-appends the next directory section
|
||||
(Summary → Entity → Concept → …) as the user scrolls
|
||||
near the end. [[wiki-link]] clicks inside the rendered
|
||||
body are handled by handleContentClick just like a
|
||||
regular wiki page. -->
|
||||
<template v-else-if="activeSystemView === 'index'">
|
||||
<div class="wiki-reader-header">
|
||||
<h2 class="wiki-reader-title">{{ $t('knowledgeEditor.wikiBrowser.indexTitle') }}</h2>
|
||||
<div class="wiki-reader-meta">
|
||||
<t-tag size="small" theme="default" variant="light-outline">
|
||||
{{ $t('knowledgeEditor.wikiBrowser.indexOverviewTag') }}
|
||||
</t-tag>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="indexLoading && !indexMarkdown" class="wiki-reader-empty">
|
||||
<p class="wiki-empty-title">{{ $t('knowledgeEditor.wikiBrowser.logLoading') }}</p>
|
||||
</div>
|
||||
<template v-else-if="indexMarkdown">
|
||||
<div
|
||||
ref="indexBodyRef"
|
||||
class="wiki-reader-body wiki-index-body"
|
||||
v-html="renderedIndexMarkdown"
|
||||
@click="handleContentClick"
|
||||
></div>
|
||||
<div v-if="indexHasMore" ref="indexSentinelRef" class="wiki-index-sentinel">
|
||||
<span v-if="indexLoading" class="wiki-index-loading">
|
||||
{{ $t('knowledgeEditor.wikiBrowser.logLoading') }}
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
<div v-else-if="!indexLoading" class="wiki-reader-empty">
|
||||
<p class="wiki-empty-title">{{ $t('knowledgeEditor.wikiBrowser.indexEmpty') }}</p>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- System view: log feed. Mutually exclusive with selectedPage. -->
|
||||
<template v-else-if="activeSystemView === 'log'">
|
||||
<div class="wiki-reader-header">
|
||||
<h2 class="wiki-reader-title">{{ $t('knowledgeEditor.wikiBrowser.logTitle') }}</h2>
|
||||
<div class="wiki-reader-meta">
|
||||
<t-tag size="small" theme="default" variant="light-outline">
|
||||
{{ $t('knowledgeEditor.wikiBrowser.logFeedTag') }}
|
||||
</t-tag>
|
||||
</div>
|
||||
</div>
|
||||
<div class="wiki-log-feed">
|
||||
<div v-if="logEntries.length === 0 && logInitialized" class="wiki-log-empty">
|
||||
{{ $t('knowledgeEditor.wikiBrowser.logEmpty') }}
|
||||
</div>
|
||||
<div v-for="entry in logEntries" :key="entry.id" class="wiki-log-entry">
|
||||
<div class="wiki-log-entry-header">
|
||||
<t-tag size="small" :theme="entry.action === 'retract' ? 'danger' : 'primary'" variant="light">
|
||||
{{ entry.action }}
|
||||
</t-tag>
|
||||
<span class="wiki-log-entry-title">{{ entry.doc_title || entry.knowledge_id || '—' }}</span>
|
||||
<span class="wiki-log-entry-time">{{ formatDate(entry.created_at) }}</span>
|
||||
</div>
|
||||
<div v-if="entry.summary" class="wiki-log-entry-summary">{{ entry.summary }}</div>
|
||||
<div v-if="entry.pages_affected && entry.pages_affected.length" class="wiki-log-entry-pages">
|
||||
<a
|
||||
v-for="ref in entry.pages_affected"
|
||||
:key="entry.id + ':' + ref.slug"
|
||||
href="#"
|
||||
class="wiki-log-entry-page"
|
||||
:title="ref.slug"
|
||||
@click.prevent="navigateToSlug(ref.slug)"
|
||||
>{{ ref.title || ref.slug }}</a>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="logNextCursor || !logInitialized" class="wiki-log-load-more">
|
||||
<t-button
|
||||
size="small"
|
||||
variant="outline"
|
||||
theme="default"
|
||||
:loading="logLoading"
|
||||
@click="loadMoreLog"
|
||||
>
|
||||
{{ logInitialized ? $t('knowledgeEditor.wikiBrowser.logLoadMore') : $t('knowledgeEditor.wikiBrowser.logLoading') }}
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- No page selected -->
|
||||
<div v-else class="wiki-reader-empty">
|
||||
<div class="wiki-empty-icon">
|
||||
@@ -566,6 +654,9 @@ import {
|
||||
type WikiGraphData,
|
||||
type WikiStats,
|
||||
type WikiPageIssue,
|
||||
type WikiLogEntry,
|
||||
type WikiIndexGroup,
|
||||
type WikiIndexEntryDTO,
|
||||
} from '@/api/wiki'
|
||||
|
||||
const router = useRouter()
|
||||
@@ -610,8 +701,60 @@ interface PageTypeBucket {
|
||||
initialized: boolean // true once the first page has been fetched
|
||||
}
|
||||
const pagesByType = ref<Record<string, PageTypeBucket>>({})
|
||||
const indexPage = ref<WikiPage | null>(null)
|
||||
const logPage = ref<WikiPage | null>(null)
|
||||
// Index view state. The reader renders an incrementally-built markdown
|
||||
// string rather than a structured list — opening the view loads intro
|
||||
// only, and "Load more" appends one directory section at a time in a
|
||||
// fixed order (Summary → Entity → Concept → Synthesis → Comparison).
|
||||
// Once the last section exhausts its pages, indexHasMore flips off.
|
||||
//
|
||||
// We deliberately avoid keeping a parallel structured list + a parallel
|
||||
// markdown buffer; the markdown is the single source of truth the reader
|
||||
// renders, and [[wiki-link]] clicks flow through the same
|
||||
// handleContentClick as regular page bodies.
|
||||
const indexMarkdown = ref('')
|
||||
const indexLoading = ref(false)
|
||||
const indexAvailable = ref(false)
|
||||
// Per-section pagination cursor. Empty string = not yet loaded; empty
|
||||
// cursor AFTER a load = that section is exhausted. `indexSectionIdx`
|
||||
// tracks which section in INDEX_SECTION_ORDER is "next to load" — we
|
||||
// advance to the following section only when the current one runs out.
|
||||
const indexSections = ref<Record<string, { loaded: boolean; cursor: string; total: number }>>({})
|
||||
const indexSectionIdx = ref(0)
|
||||
const indexBodyRef = ref<HTMLElement | null>(null)
|
||||
const indexSentinelRef = ref<HTMLElement | null>(null)
|
||||
let indexObserver: IntersectionObserver | null = null
|
||||
|
||||
// Order matters: Summary first (these are the document-level pages the
|
||||
// user most often wants to see), then the LLM-derived ones. Matches the
|
||||
// plan's "intro then Summary → Entity → Concept → …" progression.
|
||||
const INDEX_SECTION_ORDER = [
|
||||
'summary',
|
||||
'entity',
|
||||
'concept',
|
||||
'synthesis',
|
||||
'comparison',
|
||||
] as const
|
||||
// logAvailable is a flag: the sidebar "Log" entry is always shown once a
|
||||
// KB exists, because the backing wiki_log_entries table is KB-independent
|
||||
// and `GET /wiki/log` returns an empty entries list when nothing has been
|
||||
// logged yet. We don't need a full WikiPage object anymore — selecting
|
||||
// Log swaps the reader into a dedicated feed view below.
|
||||
const logAvailable = ref(true)
|
||||
|
||||
// activeSystemView lets the reader toggle between a regular wiki page
|
||||
// (selectedPage) and a "virtual" system view — index overview and log
|
||||
// feed. These modes are mutually exclusive: entering a system view
|
||||
// clears selectedPage, and picking a page clears the system view flag.
|
||||
const activeSystemView = ref<'' | 'index' | 'log'>('')
|
||||
|
||||
// Paginated state for the log view. `entries` grows as the user scrolls;
|
||||
// `nextCursor` is the opaque cursor returned by the backend and empty
|
||||
// signals end-of-feed. `loading` is the guard that prevents overlapping
|
||||
// loadMore calls while a request is in flight.
|
||||
const logEntries = ref<WikiLogEntry[]>([])
|
||||
const logNextCursor = ref('')
|
||||
const logLoading = ref(false)
|
||||
const logInitialized = ref(false)
|
||||
|
||||
// When the user types into the search box we leave pagination mode and
|
||||
// show a flat result list instead. Bucketed state is preserved behind
|
||||
@@ -780,6 +923,13 @@ function fitGraphToView() {
|
||||
const graphDrawerVisible = ref(false)
|
||||
const graphDrawerPage = ref<WikiPage | null>(null)
|
||||
const navHistory = ref<WikiPage[]>([])
|
||||
// navFromSystemView remembers which system view (Index / Log) the user
|
||||
// was viewing when they clicked into a slug, so goBack can restore it
|
||||
// once the page-level history stack is empty. We keep this parallel to
|
||||
// navHistory rather than widening its element type — navHistory is
|
||||
// consumed everywhere as `WikiPage[]` and that contract stays cleaner
|
||||
// if the system-view sentinel lives in its own ref.
|
||||
const navFromSystemView = ref<'' | 'index' | 'log'>('')
|
||||
// Index and log pages are now state refs (loaded by their own endpoints
|
||||
// at startup) rather than computed over the full page list. The old
|
||||
// computed implementation required pulling every page into memory just
|
||||
@@ -1242,6 +1392,39 @@ const renderedContent = computed(() => {
|
||||
return renderMarkdown(selectedPage.value.content)
|
||||
})
|
||||
|
||||
// Label shown next to the back arrow on page headers. Prefers the
|
||||
// nearest page-history entry when available so the user sees where
|
||||
// they'll land; falls back to the Index/Log label when the current
|
||||
// page was opened directly from a system view.
|
||||
const backLabel = computed(() => {
|
||||
if (navHistory.value.length > 0) {
|
||||
return navHistory.value[navHistory.value.length - 1].title
|
||||
}
|
||||
if (navFromSystemView.value === 'index') {
|
||||
return t('knowledgeEditor.wikiBrowser.indexTitle')
|
||||
}
|
||||
if (navFromSystemView.value === 'log') {
|
||||
return t('knowledgeEditor.wikiBrowser.logTitle')
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
// Rendered markdown for the incremental index view. Re-runs every time
|
||||
// indexMarkdown grows (initial intro load or a loadMore section append).
|
||||
const renderedIndexMarkdown = computed(() => {
|
||||
if (!indexMarkdown.value) return ''
|
||||
return renderMarkdown(indexMarkdown.value)
|
||||
})
|
||||
|
||||
// True while another section or page is available to load. Starts true
|
||||
// after the first fetch (intro only loaded, sections untouched), becomes
|
||||
// false once the last section in INDEX_SECTION_ORDER is exhausted.
|
||||
const indexHasMore = computed(() => {
|
||||
if (!indexAvailable.value) return false
|
||||
if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return false
|
||||
return true
|
||||
})
|
||||
|
||||
watch(renderedContent, async () => {
|
||||
await nextTick()
|
||||
if (readerBodyRef.value) {
|
||||
@@ -1249,6 +1432,16 @@ watch(renderedContent, async () => {
|
||||
}
|
||||
})
|
||||
|
||||
// Index body may contain image markdown from an LLM-generated intro;
|
||||
// hydrate the same way as regular page content so protected URLs
|
||||
// resolve. Also re-applied after every loadMore append.
|
||||
watch(renderedIndexMarkdown, async () => {
|
||||
await nextTick()
|
||||
if (indexBodyRef.value) {
|
||||
await hydrateProtectedFileImages(indexBodyRef.value)
|
||||
}
|
||||
})
|
||||
|
||||
function handleContentClick(e: MouseEvent) {
|
||||
const target = e.target as HTMLElement
|
||||
if (target.classList.contains('wiki-content-link')) {
|
||||
@@ -1341,20 +1534,250 @@ async function loadPagesForType(type: string, opts: { reset?: boolean } = {}) {
|
||||
groupScrollerRef.value?.updateVisibleItems?.(true)
|
||||
}
|
||||
|
||||
// loadIndexAndLog fetches the two pinned system pages directly by slug
|
||||
// instead of scanning the full page list for them. The endpoints return
|
||||
// auto-created defaults if the wiki hasn't produced them yet, so we can
|
||||
// always bind the refs and the template's v-if gates decide visibility.
|
||||
// loadIndexAndLog probes the wiki index so the sidebar knows to show
|
||||
// the pinned Index/Log entries. We ask the backend for intro only (zero
|
||||
// group types) — a bounded response regardless of KB size. Sections are
|
||||
// fetched lazily after the user actually opens the Index view; see
|
||||
// loadMoreIndexSection.
|
||||
//
|
||||
// The log "page" is no longer stored in wiki_pages — it lives in the
|
||||
// dedicated wiki_log_entries table. We don't need to pre-fetch anything
|
||||
// here to decide whether to render the sidebar Log entry; the flag is
|
||||
// always on, and the actual feed is fetched lazily when the user clicks
|
||||
// the entry (see openLogView / loadMoreLog).
|
||||
// stripLegacyIndexDirectory removes the inline "## Summary (N)\n[[...]]
|
||||
// ..." directory listing from a legacy index row. Old wiki_pages rows
|
||||
// stored "intro + directory markdown" in content; after the refactor
|
||||
// intro is the whole payload, but pre-existing KBs still carry the
|
||||
// directory until the next ingest batch rewrites it (see
|
||||
// wikiIngestService.rebuildIndexPage). We don't want the stale
|
||||
// directory to show up in the reader alongside the new live-fetched
|
||||
// sections, so we clip everything from the first `\n## ` heading on.
|
||||
function stripLegacyIndexDirectory(intro: string): string {
|
||||
if (!intro) return ''
|
||||
const idx = intro.indexOf('\n## ')
|
||||
if (idx < 0) return intro.trim()
|
||||
return intro.slice(0, idx).trim()
|
||||
}
|
||||
|
||||
async function loadIndexAndLog() {
|
||||
try {
|
||||
const [idxRes, logRes] = await Promise.all([
|
||||
getWikiIndex(props.knowledgeBaseId),
|
||||
getWikiLog(props.knowledgeBaseId),
|
||||
])
|
||||
indexPage.value = (idxRes as any).data || (idxRes as any)
|
||||
logPage.value = (logRes as any).data || (logRes as any)
|
||||
// We only need intro on the initial probe — the directory groups
|
||||
// are fetched lazily once the user opens the Index view. Passing
|
||||
// an unknown type filter yields a cheap single count(*) + 0 rows
|
||||
// on the backend instead of scanning every directory group, and
|
||||
// the frontend discards the resulting empty group unconditionally.
|
||||
const idxRes = await getWikiIndex(props.knowledgeBaseId, { types: ['__intro_only__'], limit: 1 })
|
||||
const body: any = (idxRes as any).data || (idxRes as any)
|
||||
const intro: string = body?.intro || ''
|
||||
const cleanIntro = stripLegacyIndexDirectory(intro)
|
||||
indexMarkdown.value = cleanIntro ? cleanIntro + '\n' : ''
|
||||
indexAvailable.value = true
|
||||
indexSections.value = {}
|
||||
indexSectionIdx.value = 0
|
||||
logAvailable.value = true
|
||||
} catch (e) {
|
||||
console.error('Failed to load wiki index/log:', e)
|
||||
console.error('Failed to load wiki index:', e)
|
||||
}
|
||||
}
|
||||
|
||||
// openIndexView switches the reader into the markdown-rendered index
|
||||
// overview. Re-uses the intro already fetched during loadPages(); only
|
||||
// re-fetches on first ever open or if a prior attempt failed.
|
||||
async function openIndexView() {
|
||||
selectedPage.value = null
|
||||
activeSystemView.value = 'index'
|
||||
if (!indexMarkdown.value) {
|
||||
indexLoading.value = true
|
||||
try {
|
||||
await loadIndexAndLog()
|
||||
} finally {
|
||||
indexLoading.value = false
|
||||
}
|
||||
}
|
||||
// Observer is mounted/unmounted from a watch on activeSystemView
|
||||
// + indexSentinelRef below, so nothing else to do here — entering
|
||||
// the view is a render-time concern.
|
||||
}
|
||||
|
||||
// loadMoreIndexSection advances the directory one step forward. The
|
||||
// order is fixed (Summary → Entity → Concept → …); within a section we
|
||||
// paginate with the backend's cursor, and only move to the next section
|
||||
// when the current one is exhausted. Each call produces one network
|
||||
// round trip that appends a markdown block to indexMarkdown.
|
||||
//
|
||||
// Rendering is append-only markdown rather than a structured list so
|
||||
// the viewer feels like a regular wiki page — [[wiki-link]] clicks flow
|
||||
// through handleContentClick just like every other page body. Entries
|
||||
// are rendered as plain lines (not list items) so the reader doesn't
|
||||
// carry list bullets next to every link.
|
||||
async function loadMoreIndexSection() {
|
||||
if (indexLoading.value) return
|
||||
if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return
|
||||
|
||||
const type = INDEX_SECTION_ORDER[indexSectionIdx.value]
|
||||
const state = indexSections.value[type] || { loaded: false, cursor: '', total: 0 }
|
||||
const isFirstChunkOfSection = !state.loaded
|
||||
|
||||
indexLoading.value = true
|
||||
try {
|
||||
const res = await getWikiIndex(props.knowledgeBaseId, {
|
||||
types: [type],
|
||||
limit: 50,
|
||||
cursor: isFirstChunkOfSection ? undefined : state.cursor || undefined,
|
||||
})
|
||||
const body: any = (res as any).data || (res as any)
|
||||
const group = (body?.groups || []).find((g: WikiIndexGroup) => g.type === type)
|
||||
|
||||
const items: WikiIndexEntryDTO[] = group?.items || []
|
||||
const total: number = group?.total || 0
|
||||
const nextCursor: string = group?.next_cursor || ''
|
||||
|
||||
// Only emit a section heading the first time we see entries for a
|
||||
// type. An empty section is skipped entirely so the reader doesn't
|
||||
// see "## Entity (0)" for a KB with no entities.
|
||||
let appended = ''
|
||||
if (isFirstChunkOfSection && items.length > 0) {
|
||||
const label = getTypeLabel(type)
|
||||
appended += `\n## ${label} (${total})\n\n`
|
||||
}
|
||||
for (const entry of items) {
|
||||
// Plain lines rather than `- [[slug]]`: marked renders the latter
|
||||
// as <ul><li>, which adds a disc bullet before every slug. Each
|
||||
// entry occupies one line thanks to `breaks: true` in renderMarkdown.
|
||||
//
|
||||
// Use the `[[slug|display]]` form so the anchor text shows the
|
||||
// human-readable title (e.g. "东城区") instead of the URL-safe slug
|
||||
// ("entity/dongcheng-qu"). The [[ ]] preprocessor in renderMarkdown
|
||||
// splits on the pipe and uses the right-hand side for display text
|
||||
// while the left-hand side drives navigation via handleContentClick.
|
||||
//
|
||||
// Fall back to the slug when the page has no title — the backend
|
||||
// guarantees title is non-empty for published pages, but drafts
|
||||
// or partially-indexed pages can slip through.
|
||||
const display = entry.title || entry.slug
|
||||
if (entry.summary) {
|
||||
appended += `[[${entry.slug}|${display}]] — ${entry.summary}\n`
|
||||
} else {
|
||||
appended += `[[${entry.slug}|${display}]]\n`
|
||||
}
|
||||
}
|
||||
if (appended) {
|
||||
indexMarkdown.value = indexMarkdown.value + appended
|
||||
}
|
||||
|
||||
indexSections.value[type] = {
|
||||
loaded: true,
|
||||
cursor: nextCursor,
|
||||
total,
|
||||
}
|
||||
|
||||
// Advance to the next section when this one has no more pages.
|
||||
// When the section is flat-out empty (total === 0), skip the
|
||||
// heading entirely and move on without emitting any markdown.
|
||||
if (!nextCursor) {
|
||||
indexSectionIdx.value += 1
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Failed to load more index entries for ${INDEX_SECTION_ORDER[indexSectionIdx.value]}:`, e)
|
||||
} finally {
|
||||
indexLoading.value = false
|
||||
}
|
||||
|
||||
// IntersectionObserver does NOT re-fire while the target stays
|
||||
// continuously intersecting. On small KBs (say a wiki with only
|
||||
// 3 summary pages and no entities / concepts) the sentinel sits
|
||||
// inside the viewport from the moment we finish the first section,
|
||||
// so without this nudge the remaining sections would never load.
|
||||
//
|
||||
// After every append we yield a tick (so the DOM reflows and the
|
||||
// sentinel's new rect is valid) then re-check: if it's still in
|
||||
// view and we have more to load, recurse. The recursion bottoms
|
||||
// out when either hasMore turns off or the sentinel is pushed
|
||||
// below the fold by the accumulated entries.
|
||||
await nextTick()
|
||||
if (indexHasMore.value && sentinelInView()) {
|
||||
loadMoreIndexSection()
|
||||
}
|
||||
}
|
||||
|
||||
// sentinelInView reports whether the load sentinel's rect currently
|
||||
// overlaps the viewport (± the same 200px cushion the observer uses),
|
||||
// so after loading a section we know whether to drain another round
|
||||
// even though the observer itself won't fire again while the target
|
||||
// stays visible.
|
||||
function sentinelInView(): boolean {
|
||||
const el = indexSentinelRef.value
|
||||
if (!el) return false
|
||||
const rect = el.getBoundingClientRect()
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight
|
||||
// 200px margin matches the observer's rootMargin so the drain
|
||||
// threshold and the scroll-triggered threshold stay consistent.
|
||||
return rect.top < vh + 200 && rect.bottom > -200
|
||||
}
|
||||
|
||||
// Mount/unmount the IntersectionObserver around the Index sentinel.
|
||||
// We use rootMargin to pre-load when the user scrolls within ~200px
|
||||
// of the sentinel, which hides the network round-trip behind the
|
||||
// scroll motion.
|
||||
watch([indexSentinelRef, () => activeSystemView.value], async ([el, view]) => {
|
||||
if (indexObserver) {
|
||||
indexObserver.disconnect()
|
||||
indexObserver = null
|
||||
}
|
||||
if (view !== 'index' || !el) return
|
||||
indexObserver = new IntersectionObserver((entries) => {
|
||||
for (const entry of entries) {
|
||||
if (entry.isIntersecting && indexHasMore.value && !indexLoading.value) {
|
||||
loadMoreIndexSection()
|
||||
}
|
||||
}
|
||||
}, { rootMargin: '200px' })
|
||||
indexObserver.observe(el)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
if (indexObserver) {
|
||||
indexObserver.disconnect()
|
||||
indexObserver = null
|
||||
}
|
||||
})
|
||||
|
||||
// openLogView switches the reader into the log feed and (re)loads the
|
||||
// first page. Called when the user clicks the sidebar Log entry.
|
||||
async function openLogView() {
|
||||
selectedPage.value = null
|
||||
activeSystemView.value = 'log'
|
||||
logEntries.value = []
|
||||
logNextCursor.value = ''
|
||||
logInitialized.value = false
|
||||
await loadMoreLog()
|
||||
}
|
||||
|
||||
// loadMoreLog appends the next page of log entries using the cursor from
|
||||
// the previous response. Guarded so overlapping scroll events don't fire
|
||||
// multiple requests and double-append entries.
|
||||
async function loadMoreLog() {
|
||||
if (logLoading.value) return
|
||||
// Once a previous request reported end-of-feed (empty next_cursor), we
|
||||
// stop — but only after the first fetch, so a fresh KB still runs the
|
||||
// initial empty request to populate logInitialized.
|
||||
if (logInitialized.value && !logNextCursor.value) return
|
||||
logLoading.value = true
|
||||
try {
|
||||
const res = await getWikiLog(props.knowledgeBaseId, {
|
||||
cursor: logNextCursor.value || undefined,
|
||||
limit: 50,
|
||||
})
|
||||
const body: any = (res as any).data || res
|
||||
const entries: WikiLogEntry[] = body?.entries || []
|
||||
logEntries.value.push(...entries)
|
||||
logNextCursor.value = body?.next_cursor || ''
|
||||
logInitialized.value = true
|
||||
} catch (e) {
|
||||
console.error('Failed to load wiki log:', e)
|
||||
} finally {
|
||||
logLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1390,12 +1813,14 @@ async function loadPages() {
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-select based on query or index page
|
||||
if (!selectedPage.value) {
|
||||
// Auto-select based on query string or default to the index
|
||||
// overview. The index is the natural landing view — it shows
|
||||
// intro + a paginated directory of every page type.
|
||||
if (!selectedPage.value && activeSystemView.value === '') {
|
||||
if (route.query.slug && typeof route.query.slug === 'string') {
|
||||
navigateToSlug(route.query.slug)
|
||||
} else if (indexPage.value) {
|
||||
selectPage(indexPage.value)
|
||||
} else if (indexAvailable.value) {
|
||||
openIndexView()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
@@ -1855,7 +2280,14 @@ async function selectPage(page: WikiPage) {
|
||||
try {
|
||||
if (selectedPage.value && selectedPage.value.id !== page.id) {
|
||||
navHistory.value.push(selectedPage.value)
|
||||
} else if (!selectedPage.value && activeSystemView.value) {
|
||||
// Jumping out of a system view (Index / Log) onto a page.
|
||||
// navHistory only holds WikiPages, so we stash the origin
|
||||
// system view separately; goBack restores it when the history
|
||||
// stack is empty.
|
||||
navFromSystemView.value = activeSystemView.value
|
||||
}
|
||||
activeSystemView.value = ''
|
||||
const res = await getWikiPage(props.knowledgeBaseId, page.slug)
|
||||
selectedPage.value = (res as any).data || res as any
|
||||
await loadPageIssues(page.slug)
|
||||
@@ -1868,7 +2300,13 @@ async function navigateToSlug(slug: string) {
|
||||
try {
|
||||
if (selectedPage.value && selectedPage.value.slug !== slug) {
|
||||
navHistory.value.push(selectedPage.value)
|
||||
} else if (!selectedPage.value && activeSystemView.value) {
|
||||
// Clicking a [[slug]] from inside Index / Log — same rationale
|
||||
// as selectPage above: record the system-view origin so the
|
||||
// reader's back arrow can return to it.
|
||||
navFromSystemView.value = activeSystemView.value
|
||||
}
|
||||
activeSystemView.value = ''
|
||||
const res = await getWikiPage(props.knowledgeBaseId, slug)
|
||||
selectedPage.value = (res as any).data || res as any
|
||||
await loadPageIssues(slug)
|
||||
@@ -1882,6 +2320,19 @@ function goBack() {
|
||||
if (prev) {
|
||||
selectedPage.value = prev
|
||||
loadPageIssues(prev.slug)
|
||||
return
|
||||
}
|
||||
// History stack is empty but we remember the page was opened from
|
||||
// a system view — restore that instead of leaving the reader empty.
|
||||
if (navFromSystemView.value) {
|
||||
const view = navFromSystemView.value
|
||||
navFromSystemView.value = ''
|
||||
selectedPage.value = null
|
||||
if (view === 'index') {
|
||||
openIndexView()
|
||||
} else if (view === 'log') {
|
||||
openLogView()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3736,6 +4187,102 @@ onUnmounted(() => {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
// ── Log feed (system view) ──
|
||||
// Rendered when activeSystemView === 'log'. Sits where the markdown body
|
||||
// would be for a regular wiki page — so the header/meta rules above
|
||||
// already apply. We just style the feed list itself.
|
||||
.wiki-log-feed {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
padding-top: 12px;
|
||||
}
|
||||
|
||||
// ── Index overview (system view) ──
|
||||
// The index view renders as markdown through the same pipeline as a
|
||||
// normal wiki page, so it inherits .wiki-reader-body styling automatically.
|
||||
// We use a sentinel below the body to drive auto-pagination via
|
||||
// IntersectionObserver — the user never sees a "Load more" button.
|
||||
.wiki-index-sentinel {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 32px;
|
||||
padding: 16px 0 24px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.wiki-index-loading {
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.wiki-log-empty {
|
||||
color: var(--td-text-color-placeholder);
|
||||
text-align: center;
|
||||
padding: 40px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.wiki-log-entry {
|
||||
border: 1px solid var(--td-border-level-1-color);
|
||||
border-radius: 8px;
|
||||
padding: 10px 12px;
|
||||
background: var(--td-bg-color-container);
|
||||
}
|
||||
|
||||
.wiki-log-entry-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.wiki-log-entry-title {
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-primary);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.wiki-log-entry-time {
|
||||
color: var(--td-text-color-placeholder);
|
||||
font-size: 12px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.wiki-log-entry-summary {
|
||||
color: var(--td-text-color-secondary);
|
||||
font-size: 13px;
|
||||
margin: 4px 0;
|
||||
}
|
||||
|
||||
.wiki-log-entry-pages {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 10px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.wiki-log-entry-page {
|
||||
color: var(--td-brand-color);
|
||||
font-size: 12px;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.wiki-log-entry-page:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.wiki-log-load-more {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 12px 0;
|
||||
}
|
||||
|
||||
.wiki-empty-icon {
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint verifies
|
||||
// that when a group's total exceeds its item count, the synthesized
|
||||
// markdown surfaces both numbers and the "showing top N" banner so the
|
||||
// model knows further exploration requires a separate tool call.
|
||||
func TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint(t *testing.T) {
|
||||
resp := &types.WikiIndexResponse{
|
||||
Intro: "# Wiki Index\n\nWelcome to the wiki.",
|
||||
Groups: []types.WikiIndexGroup{
|
||||
{
|
||||
Type: types.WikiPageTypeEntity,
|
||||
Total: 4500,
|
||||
Items: []types.WikiIndexEntry{
|
||||
{Slug: "entity/acme", Title: "Acme", Summary: "A company"},
|
||||
{Slug: "entity/beta", Title: "Beta", Summary: "Another company"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: types.WikiPageTypeConcept,
|
||||
Total: 2,
|
||||
Items: []types.WikiIndexEntry{
|
||||
{Slug: "concept/rag", Title: "RAG", Summary: "Retrieval-augmented generation"},
|
||||
{Slug: "concept/emb", Title: "Embedding", Summary: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got := renderIndexOverviewForAgent(resp)
|
||||
|
||||
assertContains(t, got, "# Wiki Index")
|
||||
// Truncation banner for the big group: Total != len(Items).
|
||||
assertContains(t, got, "4500 total, showing top 2")
|
||||
// Non-truncated group uses the simple "(N)" form.
|
||||
assertContains(t, got, "## Concept (2)")
|
||||
// Entry formatting — with and without summary. Entries emit the
|
||||
// [[slug|title]] form so the LLM sees the human-readable name.
|
||||
assertContains(t, got, "[[entity/acme|Acme]] — A company")
|
||||
// When the entry has a title but no summary, slug|title still
|
||||
// renders; the dash + summary portion is elided.
|
||||
assertContains(t, got, "[[concept/emb|Embedding]]")
|
||||
// Exploration hint.
|
||||
assertContains(t, got, "wiki_search")
|
||||
}
|
||||
|
||||
// TestRenderIndexOverviewForAgent_EmptyKB explains to the model that the
|
||||
// wiki has no pages yet — otherwise it might hallucinate slugs to read.
|
||||
func TestRenderIndexOverviewForAgent_EmptyKB(t *testing.T) {
|
||||
resp := &types.WikiIndexResponse{
|
||||
Intro: "# Empty\n",
|
||||
Groups: []types.WikiIndexGroup{
|
||||
{Type: types.WikiPageTypeEntity, Total: 0, Items: nil},
|
||||
},
|
||||
}
|
||||
|
||||
got := renderIndexOverviewForAgent(resp)
|
||||
assertContains(t, got, "No wiki pages yet")
|
||||
if strings.Contains(got, "## Entity") {
|
||||
t.Fatalf("empty groups should not surface a heading: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderIndexOverviewForAgent_UnknownType falls through to the raw
|
||||
// type string — covers forward compatibility for LLM-created page types
|
||||
// the frontend label map does not yet know about.
|
||||
func TestRenderIndexOverviewForAgent_UnknownType(t *testing.T) {
|
||||
resp := &types.WikiIndexResponse{
|
||||
Intro: "# Wiki\n",
|
||||
Groups: []types.WikiIndexGroup{
|
||||
{
|
||||
Type: "custom-type",
|
||||
Total: 1,
|
||||
Items: []types.WikiIndexEntry{{Slug: "custom-type/x", Title: "X"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got := renderIndexOverviewForAgent(resp)
|
||||
assertContains(t, got, "## custom-type (1)")
|
||||
}
|
||||
|
||||
// TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory covers
|
||||
// the forward-compat path: KBs that haven't been re-ingested since the
|
||||
// index refactor still carry an "intro + ## Summary..." payload on the
|
||||
// index row. The agent synthesizer must clip the legacy directory so
|
||||
// the model doesn't see it next to the live top-K block the group
|
||||
// loop emits below.
|
||||
func TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory(t *testing.T) {
|
||||
resp := &types.WikiIndexResponse{
|
||||
Intro: "# Wiki Index\n\nWelcome.\n\n## Summary (3)\n\n[[summary/legacy]] — old",
|
||||
Groups: []types.WikiIndexGroup{
|
||||
{
|
||||
Type: types.WikiPageTypeEntity,
|
||||
Total: 1,
|
||||
Items: []types.WikiIndexEntry{{Slug: "entity/live", Title: "Live", Summary: "fresh"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got := renderIndexOverviewForAgent(resp)
|
||||
if strings.Contains(got, "summary/legacy") {
|
||||
t.Fatalf("legacy inline directory should have been stripped, got:\n%s", got)
|
||||
}
|
||||
assertContains(t, got, "Welcome.")
|
||||
assertContains(t, got, "[[entity/live|Live]] — fresh")
|
||||
}
|
||||
|
||||
func assertContains(t *testing.T, haystack, needle string) {
|
||||
t.Helper()
|
||||
if !strings.Contains(haystack, needle) {
|
||||
t.Fatalf("expected output to contain %q, got:\n%s", needle, haystack)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,83 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// wikiIndexAgentTopK is the per-type cap applied when synthesizing the
|
||||
// index overview for wiki_read_page('index'). The agent uses the overview
|
||||
// to get a sense of the wiki's shape; any deeper exploration should go
|
||||
// through wiki_search. Keeping the cap small bounds the content envelope
|
||||
// served to the LLM, which is the main reason this synthesis exists.
|
||||
const wikiIndexAgentTopK = 20
|
||||
|
||||
// renderIndexOverviewForAgent formats a WikiIndexResponse as a compact
|
||||
// markdown block suitable for embedding inside the <content> tag that
|
||||
// wiki_read_page returns. We deliberately keep the structure almost
|
||||
// identical to the legacy "intro + ## Type (N)\n[[slug]] — summary"
|
||||
// markdown so existing agent prompts (prompts_wiki.go) that reason
|
||||
// about index layouts do not need retraining.
|
||||
func renderIndexOverviewForAgent(resp *types.WikiIndexResponse) string {
|
||||
var sb strings.Builder
|
||||
// Intro may still carry a legacy inline directory on KBs that
|
||||
// haven't been re-ingested since the index refactor — clip
|
||||
// everything from the first "\n## " heading onwards so the model
|
||||
// doesn't see the old directory alongside the live top-K below.
|
||||
intro := strings.TrimSpace(resp.Intro)
|
||||
if idx := strings.Index(intro, "\n## "); idx >= 0 {
|
||||
intro = strings.TrimSpace(intro[:idx])
|
||||
}
|
||||
if intro != "" {
|
||||
sb.WriteString(intro)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
typeLabels := map[string]string{
|
||||
types.WikiPageTypeSummary: "Summary",
|
||||
types.WikiPageTypeEntity: "Entity",
|
||||
types.WikiPageTypeConcept: "Concept",
|
||||
types.WikiPageTypeSynthesis: "Synthesis",
|
||||
types.WikiPageTypeComparison: "Comparison",
|
||||
}
|
||||
|
||||
nonEmpty := 0
|
||||
for _, g := range resp.Groups {
|
||||
if g.Total == 0 {
|
||||
continue
|
||||
}
|
||||
label := typeLabels[g.Type]
|
||||
if label == "" {
|
||||
label = g.Type
|
||||
}
|
||||
if int64(len(g.Items)) < g.Total {
|
||||
fmt.Fprintf(&sb, "\n## %s (%d total, showing top %d)\n\n", label, g.Total, len(g.Items))
|
||||
} else {
|
||||
fmt.Fprintf(&sb, "\n## %s (%d)\n\n", label, g.Total)
|
||||
}
|
||||
for _, item := range g.Items {
|
||||
// Emit [[slug|title]] so the LLM sees the human-readable
|
||||
// name next to the slug it needs for downstream tool calls
|
||||
// (wiki_read_page, wiki_search by title, etc.). Falling back
|
||||
// to the slug when a page has no title keeps the wiki-link
|
||||
// syntactically valid either way.
|
||||
display := item.Title
|
||||
if display == "" {
|
||||
display = item.Slug
|
||||
}
|
||||
if item.Summary != "" {
|
||||
fmt.Fprintf(&sb, "[[%s|%s]] — %s\n", item.Slug, display, item.Summary)
|
||||
} else {
|
||||
fmt.Fprintf(&sb, "[[%s|%s]]\n", item.Slug, display)
|
||||
}
|
||||
}
|
||||
nonEmpty++
|
||||
}
|
||||
|
||||
if nonEmpty == 0 {
|
||||
sb.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
|
||||
} else {
|
||||
sb.WriteString("\n_To explore more pages under any category, use wiki_search with a query, or read a specific slug directly._\n")
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// ---- wiki_read_page ----
|
||||
|
||||
// WikiScope describes the effective retrieval scope for a single wiki
|
||||
@@ -292,6 +369,21 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (*
|
||||
}
|
||||
}
|
||||
|
||||
// Index page special-case. wiki_pages.content used to hold
|
||||
// "intro + full directory" markdown; on big KBs that was a
|
||||
// multi-MB blob the LLM had no hope of reading end-to-end.
|
||||
// After the index refactor, content holds only the intro, and
|
||||
// the directory is assembled on demand. Surface a bounded
|
||||
// top-K overview so the agent still sees the wiki's shape
|
||||
// without overflowing its context window — the explicit hint
|
||||
// steers the model to wiki_search for deeper exploration.
|
||||
contentBody := page.Content
|
||||
if page.PageType == types.WikiPageTypeIndex {
|
||||
if overview, err := t.wikiService.GetIndexView(ctx, kbID, nil, wikiIndexAgentTopK, ""); err == nil && overview != nil {
|
||||
contentBody = renderIndexOverviewForAgent(overview)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`<wiki_page>
|
||||
<metadata>
|
||||
<knowledge_base_id>%s</knowledge_base_id>
|
||||
@@ -320,7 +412,7 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (*
|
||||
strings.Join(inLinksDesc, ", "),
|
||||
strings.Join(sourcesDesc, "\n"),
|
||||
page.Summary,
|
||||
page.Content,
|
||||
contentBody,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// wikiLogEntryRepository implements interfaces.WikiLogEntryRepository.
|
||||
type wikiLogEntryRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewWikiLogEntryRepository constructs a GORM-backed WikiLogEntryRepository.
|
||||
func NewWikiLogEntryRepository(db *gorm.DB) interfaces.WikiLogEntryRepository {
|
||||
return &wikiLogEntryRepository{db: db}
|
||||
}
|
||||
|
||||
// AppendBatch inserts every entry in one statement. Empty batches are a
|
||||
// no-op so callers can invoke unconditionally at the end of a wiki ingest
|
||||
// batch without guarding against the "no events this round" case.
|
||||
func (r *wikiLogEntryRepository) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(&entries).Error
|
||||
}
|
||||
|
||||
// List returns up to `limit` entries strictly older than `cursor`, newest
|
||||
// first. `cursor` is the stringified ID of the oldest entry from the
|
||||
// previous page; an empty string starts from the newest. Callers get back
|
||||
// a nextCursor string to pass on the next request — empty means no more.
|
||||
//
|
||||
// `limit` is clamped to [1, 200]. Values outside that range are coerced
|
||||
// silently to keep the handler simple.
|
||||
func (r *wikiLogEntryRepository) List(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
cursor string,
|
||||
limit int,
|
||||
) ([]*types.WikiLogEntry, string, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
|
||||
q := r.db.WithContext(ctx).
|
||||
Where("knowledge_base_id = ?", kbID).
|
||||
Order("id DESC").
|
||||
Limit(limit)
|
||||
|
||||
if cursor != "" {
|
||||
cursorID, err := strconv.ParseUint(cursor, 10, 64)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("invalid cursor %q: %w", cursor, err)
|
||||
}
|
||||
q = q.Where("id < ?", cursorID)
|
||||
}
|
||||
|
||||
var entries []*types.WikiLogEntry
|
||||
if err := q.Find(&entries).Error; err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
nextCursor := ""
|
||||
// Only emit a cursor when we actually filled the page — a short page
|
||||
// is guaranteed to be the tail, so returning a cursor would just cause
|
||||
// the frontend to fire a final empty request.
|
||||
if len(entries) == limit {
|
||||
nextCursor = strconv.FormatUint(entries[len(entries)-1].ID, 10)
|
||||
}
|
||||
return entries, nextCursor, nil
|
||||
}
|
||||
|
||||
// DeleteByKB drops every log entry for a KB. No "affected rows" signal is
|
||||
// surfaced — missing rows are a legitimate state (e.g., a KB that was
|
||||
// created before wiki_log_entries existed) and not a failure condition.
|
||||
func (r *wikiLogEntryRepository) DeleteByKB(ctx context.Context, kbID string) error {
|
||||
if kbID == "" {
|
||||
return errors.New("wiki log entries: empty kb id")
|
||||
}
|
||||
return r.db.WithContext(ctx).
|
||||
Where("knowledge_base_id = ?", kbID).
|
||||
Delete(&types.WikiLogEntry{}).Error
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// wikiLogEntriesTestDDL mirrors migrations/versioned/000040_wiki_log_entries.up.sql
|
||||
// but uses SQLite-compatible types so we can exercise the repo layer without
|
||||
// spinning up Postgres. INTEGER PRIMARY KEY AUTOINCREMENT gives us the same
|
||||
// monotonically-increasing ID semantics the production schema relies on for
|
||||
// cursor pagination.
|
||||
const wikiLogEntriesTestDDL = `
|
||||
CREATE TABLE IF NOT EXISTS wiki_log_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
knowledge_base_id VARCHAR(36) NOT NULL,
|
||||
action VARCHAR(32) NOT NULL,
|
||||
knowledge_id VARCHAR(36) NOT NULL DEFAULT '',
|
||||
doc_title TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
pages_affected TEXT NOT NULL DEFAULT '[]',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`
|
||||
|
||||
func setupWikiLogTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Exec(wikiLogEntriesTestDDL).Error)
|
||||
return db
|
||||
}
|
||||
|
||||
func makeLogEntry(kbID, action, knowledgeID, title string, pages ...string) *types.WikiLogEntry {
|
||||
refs := make(types.WikiLogPageRefs, 0, len(pages))
|
||||
for _, slug := range pages {
|
||||
// Tests only care about round-tripping; title is derived from
|
||||
// the slug so assertions don't have to spell out both fields.
|
||||
refs = append(refs, types.WikiLogPageRef{Slug: slug, Title: "title-of-" + slug})
|
||||
}
|
||||
return &types.WikiLogEntry{
|
||||
TenantID: 1,
|
||||
KnowledgeBaseID: kbID,
|
||||
Action: action,
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: title,
|
||||
Summary: "summary for " + title,
|
||||
PagesAffected: refs,
|
||||
}
|
||||
}
|
||||
|
||||
// TestWikiLogEntryRepository_AppendBatch_WritesAllRows verifies a batch
|
||||
// insert persists every entry with a monotonic ID and respects the caller-
|
||||
// provided column values.
|
||||
func TestWikiLogEntryRepository_AppendBatch_WritesAllRows(t *testing.T) {
|
||||
db := setupWikiLogTestDB(t)
|
||||
repo := NewWikiLogEntryRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
entries := []*types.WikiLogEntry{
|
||||
makeLogEntry("kb-a", "ingest", "k1", "Doc 1", "entity/a", "summary/k1"),
|
||||
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
|
||||
makeLogEntry("kb-a", "retract", "k3", "Doc 3", "concept/c"),
|
||||
}
|
||||
|
||||
require.NoError(t, repo.AppendBatch(ctx, entries))
|
||||
|
||||
// Every row should have a non-zero ID assigned by the DB, and IDs
|
||||
// should increase in insertion order so the cursor pagination logic
|
||||
// can rely on them.
|
||||
for i := range entries {
|
||||
assert.NotZero(t, entries[i].ID, "row %d should have a DB-assigned ID", i)
|
||||
if i > 0 {
|
||||
assert.Greater(t, entries[i].ID, entries[i-1].ID,
|
||||
"AppendBatch should preserve insertion order")
|
||||
}
|
||||
}
|
||||
|
||||
// Round-trip sanity: select back and confirm payload was preserved.
|
||||
var got []types.WikiLogEntry
|
||||
require.NoError(t, db.Order("id ASC").Find(&got).Error)
|
||||
require.Len(t, got, len(entries))
|
||||
assert.Equal(t, "ingest", got[0].Action)
|
||||
assert.Equal(t, "Doc 1", got[0].DocTitle)
|
||||
assert.Equal(t, types.WikiLogPageRefs{
|
||||
{Slug: "entity/a", Title: "title-of-entity/a"},
|
||||
{Slug: "summary/k1", Title: "title-of-summary/k1"},
|
||||
}, got[0].PagesAffected)
|
||||
assert.Equal(t, "retract", got[2].Action)
|
||||
}
|
||||
|
||||
// TestWikiLogEntryRepository_AppendBatch_Empty asserts an empty batch is a
|
||||
// cheap no-op — callers at the end of a wiki ingest pass can invoke it
|
||||
// unconditionally without a DB round trip.
|
||||
func TestWikiLogEntryRepository_AppendBatch_Empty(t *testing.T) {
|
||||
db := setupWikiLogTestDB(t)
|
||||
repo := NewWikiLogEntryRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repo.AppendBatch(ctx, nil))
|
||||
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{}))
|
||||
|
||||
var count int64
|
||||
require.NoError(t, db.Model(&types.WikiLogEntry{}).Count(&count).Error)
|
||||
assert.Equal(t, int64(0), count)
|
||||
}
|
||||
|
||||
// TestWikiLogEntryRepository_List_CursorPagination walks the feed with a
|
||||
// limit smaller than the population size and confirms:
|
||||
// - newest entries surface first (id DESC order),
|
||||
// - passing next_cursor returns the next batch strictly older than it,
|
||||
// - end-of-feed is signalled by an empty cursor.
|
||||
func TestWikiLogEntryRepository_List_CursorPagination(t *testing.T) {
|
||||
db := setupWikiLogTestDB(t)
|
||||
repo := NewWikiLogEntryRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
// 5 entries in one KB + a sibling row in another KB to ensure the
|
||||
// WHERE filter actually narrows by KB.
|
||||
entries := []*types.WikiLogEntry{
|
||||
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
|
||||
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
|
||||
makeLogEntry("kb-a", "ingest", "k3", "Doc 3"),
|
||||
makeLogEntry("kb-a", "ingest", "k4", "Doc 4"),
|
||||
makeLogEntry("kb-a", "retract", "k5", "Doc 5"),
|
||||
}
|
||||
require.NoError(t, repo.AppendBatch(ctx, entries))
|
||||
|
||||
other := makeLogEntry("kb-other", "ingest", "kx", "Other doc")
|
||||
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{other}))
|
||||
|
||||
// First page: latest 2 of kb-a, newest first.
|
||||
page1, cursor1, err := repo.List(ctx, "kb-a", "", 2)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page1, 2)
|
||||
assert.Equal(t, "Doc 5", page1[0].DocTitle)
|
||||
assert.Equal(t, "Doc 4", page1[1].DocTitle)
|
||||
assert.NotEmpty(t, cursor1, "should emit next_cursor when the page is full")
|
||||
|
||||
// Second page continues strictly older than cursor1.
|
||||
page2, cursor2, err := repo.List(ctx, "kb-a", cursor1, 2)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page2, 2)
|
||||
assert.Equal(t, "Doc 3", page2[0].DocTitle)
|
||||
assert.Equal(t, "Doc 2", page2[1].DocTitle)
|
||||
assert.NotEmpty(t, cursor2)
|
||||
|
||||
// Third (final) page: one row remains, so cursor is empty — the repo
|
||||
// must NOT emit a cursor for a short page, otherwise the frontend
|
||||
// would fire a doomed follow-up request.
|
||||
page3, cursor3, err := repo.List(ctx, "kb-a", cursor2, 2)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page3, 1)
|
||||
assert.Equal(t, "Doc 1", page3[0].DocTitle)
|
||||
assert.Empty(t, cursor3, "short page should signal end-of-feed with an empty cursor")
|
||||
|
||||
// Sibling KB row must NOT leak into kb-a's feed.
|
||||
for _, p := range append(append(page1, page2...), page3...) {
|
||||
assert.Equal(t, "kb-a", p.KnowledgeBaseID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWikiLogEntryRepository_List_InvalidCursor rejects a non-numeric
|
||||
// cursor rather than silently returning the newest page, so clients
|
||||
// notice bugs before they turn into phantom paginations.
|
||||
func TestWikiLogEntryRepository_List_InvalidCursor(t *testing.T) {
|
||||
db := setupWikiLogTestDB(t)
|
||||
repo := NewWikiLogEntryRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
_, _, err := repo.List(ctx, "kb-a", "not-a-number", 10)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestWikiLogEntryRepository_DeleteByKB purges one KB's rows without
|
||||
// touching others. Used during KB deletion so the log table does not
|
||||
// accumulate orphans after the parent KB is gone.
|
||||
func TestWikiLogEntryRepository_DeleteByKB(t *testing.T) {
|
||||
db := setupWikiLogTestDB(t)
|
||||
repo := NewWikiLogEntryRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{
|
||||
makeLogEntry("kb-a", "ingest", "k1", "Doc 1"),
|
||||
makeLogEntry("kb-a", "ingest", "k2", "Doc 2"),
|
||||
makeLogEntry("kb-b", "ingest", "k3", "Doc 3"),
|
||||
}))
|
||||
|
||||
require.NoError(t, repo.DeleteByKB(ctx, "kb-a"))
|
||||
|
||||
aEntries, _, err := repo.List(ctx, "kb-a", "", 100)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, aEntries)
|
||||
|
||||
bEntries, _, err := repo.List(ctx, "kb-b", "", 100)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, bEntries, 1)
|
||||
}
|
||||
|
||||
// TestWikiLogPageRefs_ScanAcceptsLegacyStringArray covers the backward-
|
||||
// compat path in the Scan method. Any rows the ingest pipeline wrote
|
||||
// before pages_affected gained a title field are stored as a plain JSON
|
||||
// string array; the repo must decode them as slug-only refs so the log
|
||||
// feed renders without a data migration.
|
||||
func TestWikiLogPageRefs_ScanAcceptsLegacyStringArray(t *testing.T) {
|
||||
var refs types.WikiLogPageRefs
|
||||
require.NoError(t, refs.Scan([]byte(`["entity/acme","concept/rag"]`)))
|
||||
require.Len(t, refs, 2)
|
||||
assert.Equal(t, "entity/acme", refs[0].Slug)
|
||||
assert.Empty(t, refs[0].Title, "legacy rows have no title until re-ingest overwrites them")
|
||||
assert.Equal(t, "concept/rag", refs[1].Slug)
|
||||
}
|
||||
@@ -208,6 +208,57 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy
|
||||
return pages, nil
|
||||
}
|
||||
|
||||
// ListByTypeLight projects only the columns needed to render an index
|
||||
// directory entry (slug, title, summary) and paginates by title ASC.
|
||||
// This keeps the GET /wiki/index response cheap on KBs with tens of
|
||||
// thousands of pages — the old path loaded every row including its TEXT
|
||||
// content just to throw the content away on the way out.
|
||||
//
|
||||
// Archived pages are excluded. `limit` clamps to [1, 200]; `offset` is
|
||||
// honored as-is. Returns the total non-archived count for the type
|
||||
// alongside the page so the caller can render "showing N of M".
|
||||
func (r *wikiPageRepository) ListByTypeLight(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
pageType string,
|
||||
limit int,
|
||||
offset int,
|
||||
) ([]types.WikiIndexEntry, int64, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
base := r.db.WithContext(ctx).
|
||||
Model(&types.WikiPage{}).
|
||||
Where("knowledge_base_id = ? AND page_type = ? AND status <> ?",
|
||||
kbID, pageType, types.WikiPageStatusArchived)
|
||||
|
||||
var total int64
|
||||
if err := base.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
var entries []types.WikiIndexEntry
|
||||
if err := base.
|
||||
Select("slug", "title", "summary").
|
||||
Order("title ASC").
|
||||
Limit(limit).
|
||||
Offset(offset).
|
||||
Scan(&entries).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return entries, total, nil
|
||||
}
|
||||
|
||||
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
|
||||
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array.
|
||||
func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) {
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// wikiPagesTestDDL is a minimal SQLite-compatible subset of the
|
||||
// production wiki_pages DDL (migrations/versioned/000037_wiki_and_indexing.up.sql).
|
||||
// JSONB is stored as TEXT in SQLite; the StringArray Scan/Value pair
|
||||
// handles the JSON round-trip unchanged.
|
||||
const wikiPagesTestDDL = `
|
||||
CREATE TABLE IF NOT EXISTS wiki_pages (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
knowledge_base_id VARCHAR(36) NOT NULL,
|
||||
slug VARCHAR(255) NOT NULL,
|
||||
title VARCHAR(512) NOT NULL DEFAULT '',
|
||||
page_type VARCHAR(32) NOT NULL DEFAULT 'summary',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'published',
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
source_refs TEXT DEFAULT '[]',
|
||||
chunk_refs TEXT DEFAULT '[]',
|
||||
in_links TEXT DEFAULT '[]',
|
||||
out_links TEXT DEFAULT '[]',
|
||||
page_metadata TEXT DEFAULT '{}',
|
||||
aliases TEXT DEFAULT '[]',
|
||||
version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at DATETIME
|
||||
);
|
||||
`
|
||||
|
||||
func setupWikiPagesTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Exec(wikiPagesTestDDL).Error)
|
||||
return db
|
||||
}
|
||||
|
||||
// makeWikiPage builds a minimal WikiPage suitable for insert. Title is
|
||||
// derived from the slug so ORDER BY title ASC yields a predictable
|
||||
// test ordering without callers having to spell out both fields.
|
||||
func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage {
|
||||
title := slug
|
||||
if idx := strings.LastIndex(slug, "/"); idx >= 0 {
|
||||
title = slug[idx+1:]
|
||||
}
|
||||
return &types.WikiPage{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: 1,
|
||||
KnowledgeBaseID: kbID,
|
||||
Slug: slug,
|
||||
Title: title,
|
||||
PageType: pageType,
|
||||
Status: status,
|
||||
Content: "body of " + slug,
|
||||
Summary: "summary of " + slug,
|
||||
Version: 1,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
|
||||
// that the index-view projection only emits the slug/title/summary
|
||||
// triples and respects the archived-status filter. This is the whole
|
||||
// point of splitting the method off ListByType — the index reader must
|
||||
// not pay for TEXT content transport.
|
||||
func TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived(t *testing.T) {
|
||||
db := setupWikiPagesTestDB(t)
|
||||
repo := NewWikiPageRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
pages := []*types.WikiPage{
|
||||
makeWikiPage("kb-a", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
|
||||
makeWikiPage("kb-a", "entity/beta", types.WikiPageTypeEntity, types.WikiPageStatusDraft),
|
||||
makeWikiPage("kb-a", "entity/gamma", types.WikiPageTypeEntity, types.WikiPageStatusArchived),
|
||||
makeWikiPage("kb-a", "concept/delta", types.WikiPageTypeConcept, types.WikiPageStatusPublished),
|
||||
makeWikiPage("kb-other", "entity/leaked", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
|
||||
}
|
||||
for _, p := range pages {
|
||||
require.NoError(t, repo.Create(ctx, p))
|
||||
}
|
||||
|
||||
entries, total, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 50, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Archived is excluded; sibling KB is excluded; everything else
|
||||
// surfaces regardless of draft/published status (the index shows
|
||||
// both so admins notice newly-drafted pages).
|
||||
assert.Equal(t, int64(2), total)
|
||||
require.Len(t, entries, 2)
|
||||
|
||||
// ORDER BY title ASC => alpha, beta.
|
||||
assert.Equal(t, "entity/alpha", entries[0].Slug)
|
||||
assert.Equal(t, "alpha", entries[0].Title)
|
||||
assert.Equal(t, "summary of entity/alpha", entries[0].Summary)
|
||||
assert.Equal(t, "entity/beta", entries[1].Slug)
|
||||
}
|
||||
|
||||
// TestListByTypeLight_Pagination walks the type list using offsets and
|
||||
// asserts the count stays stable regardless of where in the list we
|
||||
// are — the index handler uses total to render "showing N of M".
|
||||
func TestListByTypeLight_Pagination(t *testing.T) {
|
||||
db := setupWikiPagesTestDB(t)
|
||||
repo := NewWikiPageRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, s := range []string{"entity/a", "entity/b", "entity/c", "entity/d", "entity/e"} {
|
||||
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-a", s, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
|
||||
}
|
||||
|
||||
page1, total1, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), total1)
|
||||
require.Len(t, page1, 2)
|
||||
assert.Equal(t, "entity/a", page1[0].Slug)
|
||||
assert.Equal(t, "entity/b", page1[1].Slug)
|
||||
|
||||
page2, total2, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 2)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), total2, "total should be stable across pages")
|
||||
require.Len(t, page2, 2)
|
||||
assert.Equal(t, "entity/c", page2[0].Slug)
|
||||
assert.Equal(t, "entity/d", page2[1].Slug)
|
||||
|
||||
page3, total3, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 4)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), total3)
|
||||
require.Len(t, page3, 1)
|
||||
assert.Equal(t, "entity/e", page3[0].Slug)
|
||||
|
||||
// Offset past the end yields an empty list, not an error — the
|
||||
// handler relies on this to short-circuit pagination tails.
|
||||
page4, total4, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), total4)
|
||||
assert.Empty(t, page4)
|
||||
}
|
||||
|
||||
// TestListByTypeLight_EmptyType_ReturnsZero exercises the "no rows"
|
||||
// short-circuit. We skip the SELECT entirely when count is zero, so
|
||||
// a KB with no pages of a type shouldn't burn a pointless query.
|
||||
func TestListByTypeLight_EmptyType_ReturnsZero(t *testing.T) {
|
||||
db := setupWikiPagesTestDB(t)
|
||||
repo := NewWikiPageRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
entries, total, err := repo.ListByTypeLight(ctx, "kb-empty", types.WikiPageTypeSynthesis, 50, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), total)
|
||||
assert.Empty(t, entries)
|
||||
}
|
||||
|
||||
// TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't
|
||||
// want a client passing limit=100000 and forcing the DB to return a
|
||||
// multi-MB response.
|
||||
func TestListByTypeLight_ClampsLimit(t *testing.T) {
|
||||
db := setupWikiPagesTestDB(t)
|
||||
repo := NewWikiPageRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 250; i++ {
|
||||
slug := "entity/bulk-"
|
||||
// Use a stable, zero-padded suffix so title ordering is
|
||||
// deterministic for the cap assertion below.
|
||||
slug += string(rune('a'+(i/26)%26)) + string(rune('a'+i%26))
|
||||
require.NoError(t, repo.Create(ctx, makeWikiPage("kb-cap", slug, types.WikiPageTypeEntity, types.WikiPageStatusPublished)))
|
||||
}
|
||||
|
||||
// limit=0 falls back to the default of 50.
|
||||
defaultEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 0, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, defaultEntries, 50)
|
||||
|
||||
// limit=5000 clamps to 200.
|
||||
clampedEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 5000, 0)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(clampedEntries), 200)
|
||||
}
|
||||
@@ -97,6 +97,17 @@ const (
|
||||
// tick (GC pause, Redis blip) doesn't let the lock slip out from under a
|
||||
// live handler.
|
||||
wikiActiveLockRenew = 20 * time.Second
|
||||
|
||||
// wikiLLMMaxAttempts is the total attempt count (initial + retries) for
|
||||
// every LLM call routed through generateWithTemplate. 3 was chosen to
|
||||
// absorb transient 504/timeouts from upstream gateways without
|
||||
// materially prolonging task runtime when the remote is genuinely down.
|
||||
wikiLLMMaxAttempts = 3
|
||||
|
||||
// wikiLLMBackoffBase is the base delay for the exponential backoff
|
||||
// between retry attempts. The nth retry waits base << (n-1) — so with
|
||||
// a 2s base we wait 2s, 4s, 8s between attempts.
|
||||
wikiLLMBackoffBase = 2 * time.Second
|
||||
)
|
||||
|
||||
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
|
||||
@@ -156,6 +167,7 @@ type wikiIngestService struct {
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
modelService interfaces.ModelService
|
||||
task interfaces.TaskEnqueuer
|
||||
logEntrySvc interfaces.WikiLogEntryService
|
||||
redisClient *redis.Client // nil in Lite mode (no Redis)
|
||||
// liteLocks provides per-KB mutual exclusion in Lite mode (no Redis).
|
||||
// Keys are kbID strings; values are unused (presence = locked).
|
||||
@@ -170,6 +182,7 @@ func NewWikiIngestService(
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
modelService interfaces.ModelService,
|
||||
task interfaces.TaskEnqueuer,
|
||||
logEntrySvc interfaces.WikiLogEntryService,
|
||||
redisClient *redis.Client,
|
||||
) interfaces.TaskHandler {
|
||||
svc := &wikiIngestService{
|
||||
@@ -179,6 +192,7 @@ func NewWikiIngestService(
|
||||
chunkRepo: chunkRepo,
|
||||
modelService: modelService,
|
||||
task: task,
|
||||
logEntrySvc: logEntrySvc,
|
||||
redisClient: redisClient,
|
||||
}
|
||||
return svc
|
||||
@@ -427,8 +441,11 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
|
||||
type docIngestResult struct {
|
||||
KnowledgeID string
|
||||
DocTitle string
|
||||
Summary string // one-line summary of the document (from summary page)
|
||||
Pages []string // affected page slugs
|
||||
Summary string // one-line summary of the document (from summary page)
|
||||
// Pages records the wiki pages this document touched, carrying both
|
||||
// the slug (for navigation / retract lookups) and the human-readable
|
||||
// title captured at ingest time (for the log feed's display layer).
|
||||
Pages []types.WikiLogPageRef
|
||||
}
|
||||
|
||||
// WikiBatchContext holds shared data across Map and Reduce phases
|
||||
@@ -695,62 +712,72 @@ type combinedExtraction struct {
|
||||
Concepts []extractedItem `json:"concepts"`
|
||||
}
|
||||
|
||||
// rebuildIndexPage regenerates the index page.
|
||||
// rebuildIndexPage refreshes the LLM-generated intro that sits on the
|
||||
// index wiki_pages row.
|
||||
//
|
||||
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
|
||||
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
|
||||
// Subsequent: incrementally updated with changeDescription.
|
||||
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
|
||||
// History: the index page used to store "intro + full directory listing" as
|
||||
// a single multi-MB markdown blob in content. Every ingest batch rewrote
|
||||
// the whole column, which on KBs with tens of thousands of pages caused
|
||||
// O(N) TOAST writes per batch. The directory was lifted out into the
|
||||
// structured GET /wiki/index endpoint (see wikiPageService.GetIndexView),
|
||||
// and this method now only maintains the intro.
|
||||
//
|
||||
// Intro lifecycle:
|
||||
// - First time (empty or legacy placeholder): generate from all document
|
||||
// summaries via WikiIndexIntroPrompt.
|
||||
// - Subsequent calls with a change description: incremental update via
|
||||
// WikiIndexIntroUpdatePrompt so the intro reflects what just landed.
|
||||
// - No change description: keep the existing intro untouched.
|
||||
//
|
||||
// The new intro is written to both Content and Summary so readers that
|
||||
// still fall back to Summary (older clients, legacy migrations) stay in
|
||||
// sync with the column the view actually renders.
|
||||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||||
if indexPage == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// List all live pages
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
|
||||
// Gather just the Summary-type pages the LLM needs for intro framing.
|
||||
// We do NOT load or iterate every wiki page here — the directory is
|
||||
// now assembled on demand by GetIndexView, which is the only reader
|
||||
// that needs the full enumeration.
|
||||
summaries, err := s.wikiService.ListByType(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
typeOrder := []string{
|
||||
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
|
||||
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
|
||||
}
|
||||
typeLabels := map[string]string{
|
||||
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
|
||||
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
|
||||
types.WikiPageTypeComparison: "Comparison",
|
||||
}
|
||||
|
||||
grouped := make(map[string][]*types.WikiPage)
|
||||
totalPages := 0
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
if p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
grouped[p.PageType] = append(grouped[p.PageType], p)
|
||||
totalPages++
|
||||
}
|
||||
|
||||
// Build document summaries listing (only summary-type pages — they represent documents)
|
||||
var docSummaries strings.Builder
|
||||
for _, p := range grouped[types.WikiPageTypeSummary] {
|
||||
for _, p := range summaries {
|
||||
if p == nil || p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&docSummaries, "<document>\n<title>%s</title>\n<summary>%s</summary>\n</document>\n\n", p.Title, p.Summary)
|
||||
}
|
||||
if docSummaries.Len() == 0 {
|
||||
docSummaries.WriteString("(no documents yet)")
|
||||
}
|
||||
|
||||
// Generate or update intro
|
||||
existingIntro := indexPage.Summary
|
||||
var intro string
|
||||
// The intro lives on both Content and Summary. Prefer Content since
|
||||
// that's what the new index view returns; fall back to Summary for
|
||||
// rows written before this refactor so the incremental-update prompt
|
||||
// has something to work with.
|
||||
existingIntro := strings.TrimSpace(indexPage.Content)
|
||||
if existingIntro == "" {
|
||||
existingIntro = strings.TrimSpace(indexPage.Summary)
|
||||
}
|
||||
// Detect the legacy "intro + directory" payload. Such rows embed the
|
||||
// fence-separated "## Summary" sections right after the intro, so we
|
||||
// clip everything from the first directory heading onward to keep the
|
||||
// intro length bounded when we feed it back into the update prompt.
|
||||
if idx := strings.Index(existingIntro, "\n## "); idx >= 0 {
|
||||
existingIntro = strings.TrimSpace(existingIntro[:idx])
|
||||
}
|
||||
|
||||
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
|
||||
// First time — generate intro from scratch
|
||||
var intro string
|
||||
switch {
|
||||
case existingIntro == "" || existingIntro == "Wiki index - table of contents":
|
||||
// First time — generate intro from scratch.
|
||||
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
|
||||
"DocumentSummaries": docSummaries.String(),
|
||||
"Language": lang,
|
||||
@@ -760,8 +787,8 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
|
||||
} else {
|
||||
intro = strings.TrimSpace(generatedIntro)
|
||||
}
|
||||
} else if changeDesc != "" {
|
||||
// Incremental update — tell LLM what changed
|
||||
case changeDesc != "":
|
||||
// Incremental update — tell the LLM what changed.
|
||||
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
|
||||
"ExistingIntro": existingIntro,
|
||||
"ChangeDescription": changeDesc,
|
||||
@@ -773,45 +800,23 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
|
||||
} else {
|
||||
intro = strings.TrimSpace(updatedIntro)
|
||||
}
|
||||
} else {
|
||||
intro = existingIntro // no change description, keep as-is
|
||||
default:
|
||||
// No change description and an existing intro: leave it as-is so
|
||||
// we don't bump the version for a no-op.
|
||||
intro = existingIntro
|
||||
}
|
||||
|
||||
// Build directory (pure code, no LLM)
|
||||
var dir strings.Builder
|
||||
for _, pt := range typeOrder {
|
||||
pages := grouped[pt]
|
||||
if len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
|
||||
for _, p := range pages {
|
||||
summary := p.Summary
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
|
||||
}
|
||||
}
|
||||
for pt, pages := range grouped {
|
||||
inOrder := false
|
||||
for _, o := range typeOrder {
|
||||
if o == pt {
|
||||
inOrder = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if inOrder || len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
|
||||
for _, p := range pages {
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
|
||||
}
|
||||
}
|
||||
if totalPages == 0 {
|
||||
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
|
||||
// Defensive: some LLM outputs occasionally bleed into a directory-
|
||||
// like section even when the intro prompt doesn't ask for one. If
|
||||
// the freshly-generated intro starts to look like a legacy payload,
|
||||
// clip it at the first "\n## " just like we did on the read path
|
||||
// above. This keeps indexPage.Content a bounded intro-only blob.
|
||||
if idx := strings.Index(intro, "\n## "); idx >= 0 {
|
||||
intro = strings.TrimSpace(intro[:idx])
|
||||
}
|
||||
|
||||
indexPage.Content = intro + "\n" + dir.String()
|
||||
indexPage.Summary = intro // persist intro for next incremental update
|
||||
indexPage.Content = intro
|
||||
indexPage.Summary = intro
|
||||
_, err = s.wikiService.UpdatePage(ctx, indexPage)
|
||||
return err
|
||||
}
|
||||
@@ -834,34 +839,31 @@ func splitSummaryLine(raw string) (summary string, content string) {
|
||||
return "", raw
|
||||
}
|
||||
|
||||
// appendLogEntry appends a structured, grep-parseable entry to the log page.
|
||||
// Format: ## [2026-04-07 19:50:02] action | title
|
||||
// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean.
|
||||
func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) {
|
||||
logPage, _ := s.wikiService.GetLog(ctx, kbID)
|
||||
if logPage == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "\n## [%s] %s | %s\n",
|
||||
time.Now().Format("2006-01-02 15:04:05"),
|
||||
action,
|
||||
docTitle,
|
||||
)
|
||||
if knowledgeID != "" {
|
||||
fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID)
|
||||
}
|
||||
if summary != "" {
|
||||
fmt.Fprintf(&sb, "- **Summary**: %s\n", summary)
|
||||
}
|
||||
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
|
||||
// pure (no DB access) so callers can accumulate entries cheaply under their
|
||||
// lock and flush them in a single AppendBatch call at the end of the batch.
|
||||
//
|
||||
// Historically this was a per-event `GetLog + UpdatePage` round trip, which
|
||||
// rewrote the entire log page's TEXT column on every ingest/retract op —
|
||||
// O(n^2) write amplification as the log grew. The batch writer now uses
|
||||
// wikiLogEntryService.AppendBatch instead; see ProcessWikiIngest.
|
||||
func (s *wikiIngestService) buildLogEntry(tenantID uint64, kbID, action, knowledgeID, docTitle, summary string, pagesAffected []types.WikiLogPageRef) *types.WikiLogEntry {
|
||||
// Copy pagesAffected so the entry does not alias caller-owned slices.
|
||||
// The batch accumulates SlugUpdate results that may be reused downstream.
|
||||
var pages types.WikiLogPageRefs
|
||||
if len(pagesAffected) > 0 {
|
||||
fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", "))
|
||||
pages = make(types.WikiLogPageRefs, len(pagesAffected))
|
||||
copy(pages, pagesAffected)
|
||||
}
|
||||
|
||||
logPage.Content = logPage.Content + sb.String()
|
||||
if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to update log page: %v", err)
|
||||
return &types.WikiLogEntry{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Action: action,
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: docTitle,
|
||||
Summary: summary,
|
||||
PagesAffected: pages,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1017,7 +1019,23 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// generateWithTemplate executes a prompt template and calls the LLM
|
||||
// generateWithTemplate executes a prompt template and calls the LLM with
|
||||
// bounded exponential-backoff retries for transient infrastructure errors.
|
||||
//
|
||||
// Retry policy:
|
||||
// - Up to wikiLLMMaxAttempts total attempts (initial + retries).
|
||||
// - Only retry errors classified as transient by isTransientLLMError:
|
||||
// HTTP 408/429/5xx, context deadline exceeded (when the parent ctx is
|
||||
// still alive), or generic "timeout"/"connection reset" wording.
|
||||
// 4xx (except 408/429) is a caller-side fault and fails fast.
|
||||
// - Backoff is exponential base 2s: 2s, 4s, 8s — roughly wikiLLMBackoffBase
|
||||
// * 2^(attempt-1). Honors ctx cancellation so the task can abort.
|
||||
//
|
||||
// This exists because wiki ingest makes several independent LLM calls per
|
||||
// document (extraction, summary, dedup, citations, intro) and a single
|
||||
// transient 504 from the upstream gateway used to drop the document's
|
||||
// summary page permanently. Retries plus failedOps requeuing (see
|
||||
// mapOneDocument) turn those events into at-most-a-few-minute hiccups.
|
||||
func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) {
|
||||
tmpl, err := template.New("wiki").Parse(promptTpl)
|
||||
if err != nil {
|
||||
@@ -1031,17 +1049,98 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel
|
||||
|
||||
prompt := buf.String()
|
||||
thinking := false
|
||||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||||
{Role: "user", Content: prompt},
|
||||
}, &chat.ChatOptions{
|
||||
Temperature: 0.3,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("LLM call failed: %w", err)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ {
|
||||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||||
{Role: "user", Content: prompt},
|
||||
}, &chat.ChatOptions{
|
||||
Temperature: 0.3,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err == nil {
|
||||
return response.Content, nil
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
// Abort immediately on non-retryable errors (4xx except 408/429,
|
||||
// parse/marshal failures, tool-side bugs, etc.). Retrying a
|
||||
// hard "invalid arguments" error just wastes the model's budget.
|
||||
if !isTransientLLMError(ctx, err) {
|
||||
return "", fmt.Errorf("LLM call failed: %w", err)
|
||||
}
|
||||
if attempt == wikiLLMMaxAttempts {
|
||||
break
|
||||
}
|
||||
|
||||
backoff := wikiLLMBackoffBase << (attempt - 1)
|
||||
logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v",
|
||||
attempt, wikiLLMMaxAttempts, backoff, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err())
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr)
|
||||
}
|
||||
|
||||
// isTransientLLMError reports whether an error from the chat provider
|
||||
// looks like an infrastructure hiccup worth retrying. Classification is
|
||||
// intentionally conservative: the truthful "could not tell, assume
|
||||
// permanent" choice keeps retries cheap and avoids masking real bugs.
|
||||
//
|
||||
// We treat the following as transient:
|
||||
// - HTTP 408 (client request timeout — upstream usually didn't process),
|
||||
// 429 (rate-limited — retry after backoff may succeed), 5xx (any
|
||||
// server-side fault, including the 504 "Remote error, timeout with
|
||||
// 60" we see from the gateway in front of several LLM providers).
|
||||
// - Wrapped context.DeadlineExceeded when the parent ctx is still alive
|
||||
// (nested per-call timeouts).
|
||||
// - Substring matches on the error text for common transport failures
|
||||
// ("timeout", "connection reset", "EOF") that providers surface
|
||||
// without a structured status code.
|
||||
func isTransientLLMError(ctx context.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Never retry after the parent ctx itself expired — the task is
|
||||
// being cancelled and the next attempt would just fail again.
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return response.Content, nil
|
||||
msg := err.Error()
|
||||
// Providers that bubble HTTP status up formatted as
|
||||
// "API request failed with status NNN: ..." — match that first.
|
||||
for _, s := range []string{
|
||||
"status 408", "status 429",
|
||||
"status 500", "status 501", "status 502", "status 503", "status 504",
|
||||
"status 520", "status 521", "status 522", "status 523", "status 524",
|
||||
} {
|
||||
if strings.Contains(msg, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
lower := strings.ToLower(msg)
|
||||
for _, s := range []string{
|
||||
"timeout",
|
||||
"timed out",
|
||||
"connection reset",
|
||||
"connection refused",
|
||||
"broken pipe",
|
||||
"no such host", // DNS hiccup
|
||||
"i/o timeout",
|
||||
"unexpected eof",
|
||||
"tls handshake",
|
||||
"context deadline exceeded", // nested per-call deadline
|
||||
} {
|
||||
if strings.Contains(lower, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
@@ -392,14 +392,42 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
|
||||
totalPagesAffected = len(allPagesAffected)
|
||||
|
||||
// Append log entries — one per operation for chronological traceability
|
||||
// Collect log entries for this batch and flush them in a single INSERT.
|
||||
// Historically each op triggered its own `GetLog + UpdatePage` round
|
||||
// trip, which rewrote the entire log page TEXT column and caused O(n^2)
|
||||
// write amplification as the log grew. AppendBatch writes one row per
|
||||
// event into wiki_log_entries instead.
|
||||
//
|
||||
// slugsToRefs resolves each retract slug against the batch-start
|
||||
// snapshot (batchCtx.SlugTitleMap) so the log feed carries titles for
|
||||
// pages that existed when the batch began. Pages created or renamed
|
||||
// during this batch fall through the map lookup and log as slug-only
|
||||
// refs, which the frontend renders as the slug itself — a sensible
|
||||
// fallback given retracts only touch pre-existing pages.
|
||||
slugsToRefs := func(slugs []string) []types.WikiLogPageRef {
|
||||
if len(slugs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]types.WikiLogPageRef, 0, len(slugs))
|
||||
for _, slug := range slugs {
|
||||
title := batchCtx.SlugTitleMap[slug]
|
||||
out = append(out, types.WikiLogPageRef{Slug: slug, Title: title})
|
||||
}
|
||||
return out
|
||||
}
|
||||
logEntries := make([]*types.WikiLogEntry, 0, len(pendingOps)+len(docResults))
|
||||
for _, op := range pendingOps {
|
||||
if op.Op == WikiOpRetract {
|
||||
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, op.PageSlugs)
|
||||
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, slugsToRefs(op.PageSlugs)))
|
||||
}
|
||||
}
|
||||
for _, r := range docResults {
|
||||
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages)
|
||||
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages))
|
||||
}
|
||||
if len(logEntries) > 0 && s.logEntrySvc != nil {
|
||||
if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Build change description for the Index Intro LLM prompt
|
||||
@@ -625,9 +653,18 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
}
|
||||
}
|
||||
|
||||
extractedPages := make([]string, 0, len(slugItems)+1)
|
||||
for slug := range slugItems {
|
||||
extractedPages = append(extractedPages, slug)
|
||||
// extractedPages records every wiki page this document materialized
|
||||
// (entities, concepts, plus the summary page appended below). The
|
||||
// slug is used for link/retract bookkeeping; the title is captured
|
||||
// for the log feed so the user sees "提供本学位在线验证报告查询…"
|
||||
// rather than "entity/xue-xin-wang".
|
||||
extractedPages := make([]types.WikiLogPageRef, 0, len(slugItems)+1)
|
||||
for slug, item := range slugItems {
|
||||
title := item.Name
|
||||
if title == "" {
|
||||
title = slug
|
||||
}
|
||||
extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: slug, Title: title})
|
||||
}
|
||||
|
||||
// Count total distinct chunks cited across all slugs for logging.
|
||||
@@ -647,32 +684,44 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
var docSummary string
|
||||
|
||||
if summaryErr != nil {
|
||||
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, summaryErr)
|
||||
} else {
|
||||
sumLine, sumBody := splitSummaryLine(summaryContent)
|
||||
if sumBody == "" {
|
||||
sumBody = summaryContent
|
||||
}
|
||||
if sumLine == "" {
|
||||
sumLine = docTitle
|
||||
}
|
||||
docSummaryLine = sumLine
|
||||
docSummary = sumBody
|
||||
if strings.TrimSpace(docSummary) == "" {
|
||||
docSummary = sumLine
|
||||
}
|
||||
updates = append(updates, SlugUpdate{
|
||||
Slug: summarySlug,
|
||||
Type: types.WikiPageTypeSummary,
|
||||
DocTitle: docTitle,
|
||||
KnowledgeID: knowledgeID,
|
||||
SourceRef: sourceRef,
|
||||
Language: lang,
|
||||
SummaryLine: sumLine,
|
||||
SummaryBody: sumBody,
|
||||
})
|
||||
extractedPages = append(extractedPages, summarySlug)
|
||||
// Summary is the headline artifact of an ingested document — a
|
||||
// document with no summary page is half-ingested and leaves the
|
||||
// entity/concept updates hanging without a root to link back to
|
||||
// from the index. Historically we just logged and moved on,
|
||||
// which meant a single transient 504 permanently dropped the
|
||||
// summary page for that document.
|
||||
//
|
||||
// Returning an error here sends the op to failedOps (see the
|
||||
// map-phase loop in ProcessWikiIngest), which requeueFailedOps
|
||||
// appends back onto the pending list so the next batch retries.
|
||||
// The internal retries in generateWithTemplate already exhaust
|
||||
// the LLM's own transient-error budget before we give up here.
|
||||
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s, will requeue: %v", knowledgeID, summaryErr)
|
||||
return nil, nil, fmt.Errorf("generate summary: %w", summaryErr)
|
||||
}
|
||||
sumLine, sumBody := splitSummaryLine(summaryContent)
|
||||
if sumBody == "" {
|
||||
sumBody = summaryContent
|
||||
}
|
||||
if sumLine == "" {
|
||||
sumLine = docTitle
|
||||
}
|
||||
docSummaryLine = sumLine
|
||||
docSummary = sumBody
|
||||
if strings.TrimSpace(docSummary) == "" {
|
||||
docSummary = sumLine
|
||||
}
|
||||
updates = append(updates, SlugUpdate{
|
||||
Slug: summarySlug,
|
||||
Type: types.WikiPageTypeSummary,
|
||||
DocTitle: docTitle,
|
||||
KnowledgeID: knowledgeID,
|
||||
SourceRef: sourceRef,
|
||||
Language: lang,
|
||||
SummaryLine: sumLine,
|
||||
SummaryBody: sumBody,
|
||||
})
|
||||
extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: summarySlug, Title: docTitle})
|
||||
|
||||
// Entities
|
||||
for _, item := range extractedEntities {
|
||||
@@ -740,7 +789,7 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
|
||||
newSlugSet := make(map[string]bool, len(extractedPages))
|
||||
for _, ns := range extractedPages {
|
||||
newSlugSet[ns] = true
|
||||
newSlugSet[ns.Slug] = true
|
||||
}
|
||||
|
||||
var reparseOverlap, staleCount int
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestIsTransientLLMError_HTTPStatuses covers the status codes we know
|
||||
// the generic REST chat provider bubbles up as "API request failed with
|
||||
// status NNN: ...". The status-code arm of the classifier is the one
|
||||
// that actually fires for the 504 gateway timeouts the upstream LLM
|
||||
// vendors throw most often.
|
||||
func TestIsTransientLLMError_HTTPStatuses(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := []struct {
|
||||
msg string
|
||||
want bool
|
||||
}{
|
||||
{"API request failed with status 504: Remote error, timeout with 60", true},
|
||||
{"API request failed with status 503: service unavailable", true},
|
||||
{"API request failed with status 502: bad gateway", true},
|
||||
{"API request failed with status 500: internal server error", true},
|
||||
{"API request failed with status 429: rate limited", true},
|
||||
{"API request failed with status 408: request timeout", true},
|
||||
{"API request failed with status 520: Web server returned unknown", true},
|
||||
{"API request failed with status 524: timeout (Cloudflare)", true},
|
||||
// 4xx permanent: a retry will just fail the same way.
|
||||
{"API request failed with status 400: bad request", false},
|
||||
{"API request failed with status 401: unauthorized", false},
|
||||
{"API request failed with status 403: forbidden (quota exhausted)", false},
|
||||
{"API request failed with status 404: model not found", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := isTransientLLMError(ctx, errors.New(tc.msg))
|
||||
if got != tc.want {
|
||||
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsTransientLLMError_TransportSubstrings covers providers (and Go
|
||||
// stdlib errors) that surface transient conditions as free-form text
|
||||
// without a numeric status code — DNS blips, TLS handshake failures,
|
||||
// broken pipes during an in-flight body read, etc.
|
||||
func TestIsTransientLLMError_TransportSubstrings(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := []struct {
|
||||
msg string
|
||||
want bool
|
||||
}{
|
||||
{"send request: dial tcp: lookup api.example.com: no such host", true},
|
||||
{"send request: read tcp 10.0.0.1: i/o timeout", true},
|
||||
{"send request: Post \"...\": context deadline exceeded", true},
|
||||
{"send request: connection reset by peer", true},
|
||||
{"send request: connection refused", true},
|
||||
{"send request: broken pipe", true},
|
||||
{"send request: tls handshake timeout", true},
|
||||
{"send request: unexpected EOF", true},
|
||||
// Not transient: the error is actionable (missing model id,
|
||||
// malformed tool call, etc.) — retry is just wasted budget.
|
||||
{"model not configured for tool use", false},
|
||||
{"invalid JSON in tool arguments", false},
|
||||
{"parse template: syntax error", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := isTransientLLMError(ctx, errors.New(tc.msg))
|
||||
if got != tc.want {
|
||||
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsTransientLLMError_AbortsWhenParentCtxDone ensures a cancelled
|
||||
// outer context short-circuits the retry decision. Without this guard
|
||||
// an ingest task that was interrupted (asynq shutdown, deletion race,
|
||||
// parent timeout) would keep retrying the LLM during its teardown
|
||||
// window and waste the remainder of the backoff budget.
|
||||
func TestIsTransientLLMError_AbortsWhenParentCtxDone(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// This exact error would normally be classified transient (see
|
||||
// TestIsTransientLLMError_HTTPStatuses), but a cancelled ctx must
|
||||
// override that so the task can unwind promptly.
|
||||
err := errors.New("API request failed with status 504: Remote error")
|
||||
if isTransientLLMError(ctx, err) {
|
||||
t.Fatal("cancelled ctx should short-circuit to non-transient")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsTransientLLMError_NilError is a sanity check — nil should never
|
||||
// be treated as retryable, otherwise callers that forget to handle the
|
||||
// happy path would spin forever.
|
||||
func TestIsTransientLLMError_NilError(t *testing.T) {
|
||||
if isTransientLLMError(context.Background(), nil) {
|
||||
t.Fatal("nil error should not be transient")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// wikiLogEntryService is a thin wrapper over WikiLogEntryRepository. The
|
||||
// event layer carries no business rules today, so the service just
|
||||
// forwards to the repo — the indirection exists so handlers / ingest
|
||||
// batch code depend on an interface rather than a concrete repo type.
|
||||
type wikiLogEntryService struct {
|
||||
repo interfaces.WikiLogEntryRepository
|
||||
}
|
||||
|
||||
// NewWikiLogEntryService constructs a WikiLogEntryService backed by the
|
||||
// given repository.
|
||||
func NewWikiLogEntryService(repo interfaces.WikiLogEntryRepository) interfaces.WikiLogEntryService {
|
||||
return &wikiLogEntryService{repo: repo}
|
||||
}
|
||||
|
||||
// AppendBatch records the given events in one database round trip. Empty
|
||||
// batches are a no-op (the repo handles that).
|
||||
func (s *wikiLogEntryService) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error {
|
||||
return s.repo.AppendBatch(ctx, entries)
|
||||
}
|
||||
|
||||
// List paginates the per-KB event feed. See repo.List for cursor semantics.
|
||||
func (s *wikiLogEntryService) List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error) {
|
||||
entries, nextCursor, err := s.repo.List(ctx, kbID, cursor, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entries == nil {
|
||||
// Normalise to an empty slice so clients don't need to
|
||||
// distinguish `null` from `[]`.
|
||||
entries = []*types.WikiLogEntry{}
|
||||
}
|
||||
return &types.WikiLogEntryListResponse{
|
||||
Entries: entries,
|
||||
NextCursor: nextCursor,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteByKB removes the log feed for a KB. Called when the KB itself is
|
||||
// being deleted, so no further reads happen.
|
||||
func (s *wikiLogEntryService) DeleteByKB(ctx context.Context, kbID string) error {
|
||||
return s.repo.DeleteByKB(ctx, kbID)
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -246,13 +247,125 @@ func (s *wikiPageService) GetIndex(ctx context.Context, kbID string) (*types.Wik
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// GetLog returns the log page for a knowledge base
|
||||
// wikiIndexContentPageTypes enumerates the page types that make up a wiki's
|
||||
// user-visible directory. System pages (index/log) are excluded; any
|
||||
// LLM-created type we do not recognize surfaces under a generic "other"
|
||||
// bucket.
|
||||
var wikiIndexContentPageTypes = []string{
|
||||
types.WikiPageTypeSummary,
|
||||
types.WikiPageTypeEntity,
|
||||
types.WikiPageTypeConcept,
|
||||
types.WikiPageTypeSynthesis,
|
||||
types.WikiPageTypeComparison,
|
||||
}
|
||||
|
||||
// GetIndexView builds the structured index response without ever
|
||||
// materializing a multi-MB directory markdown string. Intro is read from
|
||||
// the index wiki_page row (which now carries only intro text — see
|
||||
// rebuildIndexPage). Each requested page_type is paginated independently
|
||||
// with ListByTypeLight so reads stay O(page_size) rather than O(total
|
||||
// pages in the KB).
|
||||
//
|
||||
// `pageTypes` narrows which groups to include; empty = all content types.
|
||||
// `limit` is the per-group window size (defaults to 50, capped at 200).
|
||||
// `cursor` is an opaque offset string; currently we use the stringified
|
||||
// offset so clients can resume where they left off. Because different
|
||||
// page_types paginate independently, `cursor` applies uniformly to every
|
||||
// group — if the caller wants per-group cursors it should request one
|
||||
// type at a time via `pageTypes`. That simplifies the wire format and
|
||||
// matches the frontend's tabbed UX.
|
||||
func (s *wikiPageService) GetIndexView(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
pageTypes []string,
|
||||
limit int,
|
||||
cursor string,
|
||||
) (*types.WikiIndexResponse, error) {
|
||||
indexPage, err := s.GetIndex(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load index page: %w", err)
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
offset := 0
|
||||
if cursor != "" {
|
||||
v, parseErr := strconv.Atoi(cursor)
|
||||
if parseErr != nil || v < 0 {
|
||||
return nil, fmt.Errorf("invalid cursor %q", cursor)
|
||||
}
|
||||
offset = v
|
||||
}
|
||||
|
||||
// Default to every known content type when the caller passes no
|
||||
// filter. Any unknown request-time type is passed through verbatim so
|
||||
// future page types (declared in types/wiki_page.go) start showing
|
||||
// up in the index the moment the LLM starts creating them, without a
|
||||
// handler change.
|
||||
selected := pageTypes
|
||||
if len(selected) == 0 {
|
||||
selected = append([]string{}, wikiIndexContentPageTypes...)
|
||||
}
|
||||
|
||||
groups := make([]types.WikiIndexGroup, 0, len(selected))
|
||||
for _, pt := range selected {
|
||||
entries, total, listErr := s.repo.ListByTypeLight(ctx, kbID, pt, limit, offset)
|
||||
if listErr != nil {
|
||||
return nil, fmt.Errorf("list %s pages: %w", pt, listErr)
|
||||
}
|
||||
if entries == nil {
|
||||
entries = []types.WikiIndexEntry{}
|
||||
}
|
||||
next := ""
|
||||
// Only emit a cursor when a full page was returned AND more rows
|
||||
// remain past `offset + limit`. A short page or one that exactly
|
||||
// consumed the remainder should signal end-of-feed.
|
||||
if len(entries) == limit && int64(offset+len(entries)) < total {
|
||||
next = strconv.Itoa(offset + limit)
|
||||
}
|
||||
groups = append(groups, types.WikiIndexGroup{
|
||||
Type: pt,
|
||||
Total: total,
|
||||
Items: entries,
|
||||
NextCursor: next,
|
||||
})
|
||||
}
|
||||
|
||||
// The intro used to be stored on indexPage.Summary while
|
||||
// indexPage.Content held intro + directory markdown. After the
|
||||
// directory was lifted out of wiki_pages the content column holds
|
||||
// only the intro. Fall back to Summary for KBs that haven't been
|
||||
// re-ingested since the change so the response is never blank.
|
||||
intro := indexPage.Content
|
||||
if strings.TrimSpace(intro) == "" {
|
||||
intro = indexPage.Summary
|
||||
}
|
||||
|
||||
return &types.WikiIndexResponse{
|
||||
Intro: intro,
|
||||
Version: indexPage.Version,
|
||||
Groups: groups,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetLog returns the wiki_pages row for slug='log' if it exists.
|
||||
//
|
||||
// Log events are now stored in the dedicated `wiki_log_entries` table and
|
||||
// paginated via wikiLogEntryService — the per-KB log is no longer a single
|
||||
// TEXT column on a wiki_pages row (that model caused O(n^2) write
|
||||
// amplification as logs grew). This method is retained for callers that
|
||||
// still probe the legacy row (wiki_lint, knowledge delete, etc.), but it
|
||||
// no longer auto-creates the placeholder page on miss; a missing row is a
|
||||
// normal state and the helper returns `nil, nil`.
|
||||
func (s *wikiPageService) GetLog(ctx context.Context, kbID string) (*types.WikiPage, error) {
|
||||
page, err := s.repo.GetBySlug(ctx, kbID, "log")
|
||||
if err != nil {
|
||||
if errors.Is(err, repository.ErrWikiPageNotFound) {
|
||||
return s.createDefaultPage(ctx, kbID, "log", "Log", types.WikiPageTypeLog,
|
||||
"# Wiki Operation Log\n\nChronological record of wiki operations.\n")
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
@@ -619,6 +732,13 @@ func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*typ
|
||||
return s.repo.ListAll(ctx, kbID)
|
||||
}
|
||||
|
||||
// ListByType returns every wiki page of a given type for a KB. Exposed so
|
||||
// callers like intro regeneration can load only the page type they need
|
||||
// (summaries) instead of paying for the full ListAll scan.
|
||||
func (s *wikiPageService) ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) {
|
||||
return s.repo.ListByType(ctx, kbID, pageType)
|
||||
}
|
||||
|
||||
// ListPagesBySourceRef exposes the repository's source-ref lookup so higher
|
||||
// layers (delete flow, retract reconciliation) can re-query the current wiki
|
||||
// state without depending on a stale caller-captured slug list.
|
||||
@@ -821,79 +941,23 @@ func (s *wikiPageService) InjectCrossLinks(ctx context.Context, kbID string, aff
|
||||
}
|
||||
}
|
||||
|
||||
// RebuildIndexPage regenerates the index page directory.
|
||||
// RebuildIndexPage was historically called by agent write/rename tools to
|
||||
// refresh the index page's directory listing after a page mutation.
|
||||
//
|
||||
// The directory is no longer persisted in wiki_pages.content — it is
|
||||
// assembled on demand by GetIndexView from the lightweight ListByTypeLight
|
||||
// projection, so individual page writes don't need to redo O(N) string
|
||||
// concatenation and rewrite a multi-MB TEXT column anymore. Keeping the
|
||||
// method name lets existing agent tool call sites (wiki_write_page,
|
||||
// wiki_rename_page) compile unchanged; the body is now intentionally a
|
||||
// no-op.
|
||||
//
|
||||
// The intro that still lives on the index row is managed separately by
|
||||
// the ingest pipeline (see wikiIngestService.rebuildIndexPage) on batch
|
||||
// completion, which is where we actually have the LLM + change description
|
||||
// context needed to rewrite it.
|
||||
func (s *wikiPageService) RebuildIndexPage(ctx context.Context, kbID string) error {
|
||||
indexPage, err := s.GetIndex(ctx, kbID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
allPages, err := s.ListAllPages(ctx, kbID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
typeOrder := []string{
|
||||
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
|
||||
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
|
||||
}
|
||||
typeLabels := map[string]string{
|
||||
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
|
||||
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
|
||||
types.WikiPageTypeComparison: "Comparison",
|
||||
}
|
||||
|
||||
grouped := make(map[string][]*types.WikiPage)
|
||||
totalPages := 0
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
if p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
grouped[p.PageType] = append(grouped[p.PageType], p)
|
||||
totalPages++
|
||||
}
|
||||
|
||||
var dir strings.Builder
|
||||
for _, pt := range typeOrder {
|
||||
pages := grouped[pt]
|
||||
if len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
|
||||
for _, p := range pages {
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
|
||||
}
|
||||
}
|
||||
for pt, pages := range grouped {
|
||||
inOrder := false
|
||||
for _, o := range typeOrder {
|
||||
if o == pt {
|
||||
inOrder = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if inOrder || len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
|
||||
for _, p := range pages {
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
|
||||
}
|
||||
}
|
||||
if totalPages == 0 {
|
||||
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
|
||||
}
|
||||
|
||||
intro := indexPage.Summary
|
||||
if intro == "" {
|
||||
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
|
||||
indexPage.Summary = intro
|
||||
}
|
||||
|
||||
indexPage.Content = intro + "\n" + dir.String()
|
||||
_, err = s.UpdatePage(ctx, indexPage)
|
||||
return err
|
||||
_ = ctx
|
||||
_ = kbID
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -153,6 +153,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(repository.NewDataSourceRepository))
|
||||
must(container.Provide(repository.NewSyncLogRepository))
|
||||
must(container.Provide(repository.NewWikiPageRepository))
|
||||
must(container.Provide(repository.NewWikiLogEntryRepository))
|
||||
|
||||
// MCP manager for managing MCP client connections
|
||||
logger.Debugf(ctx, "[Container] Registering MCP manager...")
|
||||
@@ -186,6 +187,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(service.NewCustomAgentService))
|
||||
must(container.Provide(memoryService.NewMemoryService))
|
||||
must(container.Provide(service.NewWikiPageService))
|
||||
must(container.Provide(service.NewWikiLogEntryService))
|
||||
must(container.Provide(service.NewWikiIngestService, dig.Name("wikiIngest")))
|
||||
must(container.Provide(service.NewWikiLintService))
|
||||
|
||||
|
||||
@@ -19,9 +19,10 @@ import (
|
||||
|
||||
// WikiPageHandler handles HTTP requests for wiki page operations
|
||||
type WikiPageHandler struct {
|
||||
wikiService interfaces.WikiPageService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
lintService *service.WikiLintService
|
||||
wikiService interfaces.WikiPageService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
lintService *service.WikiLintService
|
||||
logEntryService interfaces.WikiLogEntryService
|
||||
}
|
||||
|
||||
// NewWikiPageHandler creates a new wiki page handler
|
||||
@@ -29,11 +30,13 @@ func NewWikiPageHandler(
|
||||
wikiService interfaces.WikiPageService,
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
lintService *service.WikiLintService,
|
||||
logEntryService interfaces.WikiLogEntryService,
|
||||
) *WikiPageHandler {
|
||||
return &WikiPageHandler{
|
||||
wikiService: wikiService,
|
||||
kbService: kbService,
|
||||
lintService: lintService,
|
||||
wikiService: wikiService,
|
||||
kbService: kbService,
|
||||
lintService: lintService,
|
||||
logEntryService: logEntryService,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,12 +277,19 @@ func (h *WikiPageHandler) DeletePage(c *gin.Context) {
|
||||
}
|
||||
|
||||
// GetIndex godoc
|
||||
// @Summary Get wiki index page
|
||||
// @Description Returns the wiki index page (creates default if not exists)
|
||||
// @Summary Get wiki index view
|
||||
// @Description Returns the wiki index as intro text plus per-type paginated
|
||||
// @Description directory groups. The heavy directory markdown that used to
|
||||
// @Description live in wiki_pages.content was replaced with this structured
|
||||
// @Description response so a KB with tens of thousands of pages no longer
|
||||
// @Description materializes megabytes of TEXT on every index open.
|
||||
// @Tags Wiki
|
||||
// @Produce json
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Success 200 {object} types.WikiPage
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Param types query string false "Comma-separated page types (default: all content types)"
|
||||
// @Param limit query int false "Per-group window size, 1-200 (default 50)"
|
||||
// @Param cursor query string false "Opaque offset cursor from previous response"
|
||||
// @Success 200 {object} types.WikiIndexResponse
|
||||
// @Security Bearer
|
||||
// @Router /api/v1/knowledgebase/{kb_id}/wiki/index [get]
|
||||
func (h *WikiPageHandler) GetIndex(c *gin.Context) {
|
||||
@@ -289,22 +299,43 @@ func (h *WikiPageHandler) GetIndex(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
page, err := h.wikiService.GetIndex(c.Request.Context(), kbID)
|
||||
var pageTypes []string
|
||||
if raw := c.Query("types"); raw != "" {
|
||||
for _, t := range strings.Split(raw, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
if t != "" {
|
||||
pageTypes = append(pageTypes, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
limit := 50
|
||||
if raw := c.Query("limit"); raw != "" {
|
||||
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
|
||||
limit = v
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := h.wikiService.GetIndexView(c.Request.Context(), kbID, pageTypes, limit, c.Query("cursor"))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, page)
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// GetLog godoc
|
||||
// @Summary Get wiki operation log
|
||||
// @Description Returns the wiki operation log page (creates default if not exists)
|
||||
// @Description Returns a paginated feed of wiki operation events (ingest, retract, ...)
|
||||
// @Description newest-first. Pagination is cursor-based: pass `next_cursor` from the
|
||||
// @Description previous response back as `cursor` to fetch the next page.
|
||||
// @Tags Wiki
|
||||
// @Produce json
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Success 200 {object} types.WikiPage
|
||||
// @Param kb_id path string true "Knowledge base ID"
|
||||
// @Param cursor query string false "Opaque cursor from the previous page (empty = newest)"
|
||||
// @Param limit query int false "Page size, 1-200 (default 50)"
|
||||
// @Success 200 {object} types.WikiLogEntryListResponse
|
||||
// @Security Bearer
|
||||
// @Router /api/v1/knowledgebase/{kb_id}/wiki/log [get]
|
||||
func (h *WikiPageHandler) GetLog(c *gin.Context) {
|
||||
@@ -314,13 +345,21 @@ func (h *WikiPageHandler) GetLog(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
page, err := h.wikiService.GetLog(c.Request.Context(), kbID)
|
||||
cursor := c.Query("cursor")
|
||||
limit := 50
|
||||
if raw := c.Query("limit"); raw != "" {
|
||||
if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 {
|
||||
limit = v
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := h.logEntryService.List(c.Request.Context(), kbID, cursor, limit)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, page)
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Graph query parameter bounds. The defaults cap an `overview` request at
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// WikiLogEntryRepository persists per-KB wiki operation events. The table
|
||||
// is append-only; updates are never issued. Reads paginate newest-first
|
||||
// with a stable numeric cursor.
|
||||
type WikiLogEntryRepository interface {
|
||||
// AppendBatch inserts a batch of events in a single INSERT...VALUES
|
||||
// statement. All entries in a batch must belong to the same KB in
|
||||
// practice, but the repo does not enforce that — the caller owns the
|
||||
// grouping.
|
||||
AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error
|
||||
|
||||
// List returns up to `limit` entries in (knowledge_base_id, id DESC)
|
||||
// order. `cursor` is the ID of the last entry from the previous page
|
||||
// ("return entries strictly older than this"); empty string means
|
||||
// "from the newest".
|
||||
//
|
||||
// The returned nextCursor is empty when no further rows remain; the
|
||||
// caller can treat that as end-of-stream.
|
||||
List(ctx context.Context, kbID string, cursor string, limit int) (entries []*types.WikiLogEntry, nextCursor string, err error)
|
||||
|
||||
// DeleteByKB removes every log entry belonging to the given KB. Used
|
||||
// when a wiki KB is deleted so the log table does not accumulate
|
||||
// orphans. Safe to call even if no rows exist.
|
||||
DeleteByKB(ctx context.Context, kbID string) error
|
||||
}
|
||||
|
||||
// WikiLogEntryService is the service layer over WikiLogEntryRepository.
|
||||
// Kept intentionally thin — it mirrors the repo today because the events
|
||||
// are pure bookkeeping and carry no business rules.
|
||||
type WikiLogEntryService interface {
|
||||
AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error
|
||||
List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error)
|
||||
DeleteByKB(ctx context.Context, kbID string) error
|
||||
}
|
||||
@@ -50,10 +50,33 @@ type WikiPageService interface {
|
||||
|
||||
// GetIndex returns the index page for a knowledge base.
|
||||
// Creates a default one if it doesn't exist.
|
||||
//
|
||||
// The index row now carries only the LLM-generated intro in its content
|
||||
// column — the heavy directory listing was moved out of wiki_pages.
|
||||
// Callers that want a renderable directory should use GetIndexView,
|
||||
// which assembles {intro, groups} structured output on demand.
|
||||
GetIndex(ctx context.Context, kbID string) (*types.WikiPage, error)
|
||||
|
||||
// GetIndexView returns the structured index response — intro (from the
|
||||
// index row) plus a paginated window of directory entries per requested
|
||||
// page_type. `types` restricts which page types to include (empty =
|
||||
// all), `limit` bounds the per-type window size, `cursor` is an opaque
|
||||
// offset string resumed from a previous response.
|
||||
//
|
||||
// Keeping the directory as JSON rather than a multi-MB markdown blob
|
||||
// means a 40k-page KB no longer pays O(N) transport + rendering cost
|
||||
// on every index open. See /wiki/index handler + WikiBrowser.vue for
|
||||
// the consumer.
|
||||
GetIndexView(ctx context.Context, kbID string, pageTypes []string, limit int, cursor string) (*types.WikiIndexResponse, error)
|
||||
|
||||
// GetLog returns the log page for a knowledge base.
|
||||
// Creates a default one if it doesn't exist.
|
||||
//
|
||||
// Wiki operation events now live in the dedicated wiki_log_entries
|
||||
// table, so this method no longer auto-creates a placeholder row on
|
||||
// miss and may legitimately return (nil, nil) for KBs that never had
|
||||
// the legacy row written. Retained for back-compat with callers that
|
||||
// still probe the row (lint, knowledge delete); new code should use
|
||||
// WikiLogEntryService.List for the event feed instead.
|
||||
GetLog(ctx context.Context, kbID string) (*types.WikiPage, error)
|
||||
|
||||
// GetGraph returns the link graph data for visualization. The caller
|
||||
@@ -81,6 +104,12 @@ type WikiPageService interface {
|
||||
// Used for index rebuild, graph generation, cross-link injection, etc.
|
||||
ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error)
|
||||
|
||||
// ListByType returns every wiki page of the given type. Used by
|
||||
// callers that need the full set for a specific type (e.g. intro
|
||||
// regeneration needs summary pages). For directory rendering, use
|
||||
// GetIndexView instead — it paginates via ListByTypeLight.
|
||||
ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error)
|
||||
|
||||
// ListPagesBySourceRef retrieves all wiki pages whose source_refs reference
|
||||
// the given knowledge ID. Used by delete/ingest reconciliation paths that
|
||||
// need to find pages touched by a specific document at read time (rather
|
||||
@@ -136,6 +165,12 @@ type WikiPageRepository interface {
|
||||
// ListByType retrieves all wiki pages of a given type within a knowledge base.
|
||||
ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error)
|
||||
|
||||
// ListByTypeLight returns a paginated window of lightweight entries
|
||||
// (slug/title/summary only) for the given page_type plus the total
|
||||
// non-archived count. Used by the structured index API so reads do not
|
||||
// have to materialize TEXT content for every wiki_pages row.
|
||||
ListByTypeLight(ctx context.Context, kbID string, pageType string, limit int, offset int) ([]types.WikiIndexEntry, int64, error)
|
||||
|
||||
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
|
||||
ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error)
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WikiLogPageRef identifies a wiki page that was affected by a log event,
|
||||
// paired with the human-readable title at the time of the event. Title is
|
||||
// captured at write time so the log feed stays readable even if the page
|
||||
// is later renamed or deleted — the previously-stored title is the one
|
||||
// the user remembers seeing in the event stream.
|
||||
type WikiLogPageRef struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title,omitempty"`
|
||||
}
|
||||
|
||||
// WikiLogPageRefs is a JSON-marshalled list of page references. The
|
||||
// column is TEXT/JSONB depending on driver; the Value/Scan pair keeps
|
||||
// GORM happy on both Postgres and the SQLite tests.
|
||||
type WikiLogPageRefs []WikiLogPageRef
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (r WikiLogPageRefs) Value() (driver.Value, error) {
|
||||
if r == nil {
|
||||
return []byte("[]"), nil
|
||||
}
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface. Also tolerates the legacy
|
||||
// []string shape so we can roll forward without a data migration — any
|
||||
// rows the ingest pipeline wrote before this commit surface as refs
|
||||
// with a slug and an empty title, which the frontend renders identically.
|
||||
func (r *WikiLogPageRefs) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
*r = nil
|
||||
return nil
|
||||
}
|
||||
var raw []byte
|
||||
switch v := value.(type) {
|
||||
case []byte:
|
||||
raw = v
|
||||
case string:
|
||||
raw = []byte(v)
|
||||
default:
|
||||
return fmt.Errorf("unsupported type for WikiLogPageRefs: %T", value)
|
||||
}
|
||||
|
||||
// Try the new [{slug,title}] shape first.
|
||||
var refs []WikiLogPageRef
|
||||
if err := json.Unmarshal(raw, &refs); err == nil {
|
||||
*r = refs
|
||||
return nil
|
||||
}
|
||||
// Fall back to the legacy []string shape.
|
||||
var slugs []string
|
||||
if err := json.Unmarshal(raw, &slugs); err != nil {
|
||||
return err
|
||||
}
|
||||
out := make([]WikiLogPageRef, 0, len(slugs))
|
||||
for _, s := range slugs {
|
||||
out = append(out, WikiLogPageRef{Slug: s})
|
||||
}
|
||||
*r = out
|
||||
return nil
|
||||
}
|
||||
|
||||
// WikiLogEntry is a single ingest/retract/edit event appended to the
|
||||
// per-KB operation log. It replaces the legacy "single giant TEXT column
|
||||
// on slug='log' wiki_pages row" model, which rewrote the whole column on
|
||||
// every event and caused O(n^2) write amplification as the KB grew.
|
||||
//
|
||||
// Each event is one INSERT; reads paginate by (knowledge_base_id, id DESC).
|
||||
type WikiLogEntry struct {
|
||||
// Auto-increment identifier. Monotonic within a single database, so
|
||||
// frontend pagination uses it as a stable cursor without needing to
|
||||
// disambiguate identical created_at values.
|
||||
ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
// Tenant scope, mirrored from the enclosing knowledge base.
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
// Knowledge base this event belongs to.
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
|
||||
// Short operation tag: "ingest", "retract", etc. Matches the `action`
|
||||
// argument historically passed to appendLogEntry.
|
||||
Action string `json:"action" gorm:"type:varchar(32)"`
|
||||
// Knowledge ID the event was about (may be empty for KB-level events).
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"type:varchar(36);default:''"`
|
||||
// Document title at the time of the event. Stored verbatim rather than
|
||||
// joined at read time so deleted knowledge still has a human-readable
|
||||
// label in the log.
|
||||
DocTitle string `json:"doc_title" gorm:"type:text"`
|
||||
// One-line summary of the change, as it was when the event was logged.
|
||||
Summary string `json:"summary" gorm:"type:text"`
|
||||
// Wiki pages affected by this event. Each ref carries both slug (for
|
||||
// navigation) and title (for display) so the log renders human-
|
||||
// readable text without a post-hoc slug→title lookup that might fail
|
||||
// for now-deleted pages.
|
||||
PagesAffected WikiLogPageRefs `json:"pages_affected" gorm:"type:jsonb;default:'[]'"`
|
||||
// Server-side timestamp (UTC).
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TableName specifies the database table name.
|
||||
func (WikiLogEntry) TableName() string {
|
||||
return "wiki_log_entries"
|
||||
}
|
||||
|
||||
// WikiLogEntryListResponse is the paginated response for GET /wiki/log.
|
||||
// NextCursor is the stringified ID of the oldest entry in `Entries`; pass
|
||||
// it back as `cursor` to fetch the next page. Empty string means "no more".
|
||||
type WikiLogEntryListResponse struct {
|
||||
Entries []*WikiLogEntry `json:"entries"`
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
}
|
||||
@@ -292,3 +292,35 @@ type WikiPageIssue struct {
|
||||
func (WikiPageIssue) TableName() string {
|
||||
return "wiki_page_issues"
|
||||
}
|
||||
|
||||
// WikiIndexEntry is a single row in the structured wiki index response.
|
||||
// Only the columns needed to render a clickable directory entry are
|
||||
// carried — the backend projects SELECT slug, title, summary so a 40k-
|
||||
// page KB does not pay for TEXT content transport on every index open.
|
||||
type WikiIndexEntry struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
// WikiIndexGroup bundles the entries for one page_type into a page-sized
|
||||
// slice. `Total` is the full count across the KB for the type; `Items`
|
||||
// holds the current paginated window starting at `NextOffset - len(Items)`.
|
||||
// An empty NextCursor means the window is already at the end of the type.
|
||||
type WikiIndexGroup struct {
|
||||
Type string `json:"type"`
|
||||
Total int64 `json:"total"`
|
||||
Items []WikiIndexEntry `json:"items"`
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
}
|
||||
|
||||
// WikiIndexResponse is what GET /wiki/index returns. The heavy directory
|
||||
// markdown that used to sit in wiki_pages.content is gone — only the LLM-
|
||||
// generated intro survives there. Everything else is assembled on demand
|
||||
// from the index repo's light-column projection, keeping index reads
|
||||
// O(page_size) regardless of KB size.
|
||||
type WikiIndexResponse struct {
|
||||
Intro string `json:"intro"`
|
||||
Version int `json:"version"`
|
||||
Groups []WikiIndexGroup `json:"groups"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration: 000040_wiki_log_entries (rollback)
|
||||
-- Description: Drop the wiki_log_entries table. No data migration was
|
||||
-- performed on the way up, so rollback is a plain DROP.
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] Reverting wiki_log_entries schema'; END $$;
|
||||
|
||||
DROP TABLE IF EXISTS wiki_log_entries;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] wiki_log_entries schema reverted successfully'; END $$;
|
||||
@@ -0,0 +1,32 @@
|
||||
-- Migration: 000040_wiki_log_entries
|
||||
-- Description: Dedicated append-only event table for wiki operation log entries.
|
||||
-- Replaces the "single giant TEXT column on slug='log' wiki_pages
|
||||
-- row" model that caused O(n^2) write amplification as KBs grew:
|
||||
-- every ingest/retract op previously did GetLog + UpdatePage
|
||||
-- rewriting the entire (potentially multi-MB) TEXT column.
|
||||
-- Here each event is one INSERT; reads paginate by (kb_id, id DESC).
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000040] Applying wiki_log_entries schema'; END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS wiki_log_entries (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
knowledge_base_id VARCHAR(36) NOT NULL,
|
||||
action VARCHAR(32) NOT NULL,
|
||||
knowledge_id VARCHAR(36) NOT NULL DEFAULT '',
|
||||
doc_title TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
pages_affected JSONB NOT NULL DEFAULT '[]'::JSONB,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Primary list query: cursor-paginated feed per KB, newest first.
|
||||
-- Using id (BIGSERIAL, monotonic) as the cursor sidesteps duplicate-timestamp
|
||||
-- tie-breaking that a created_at cursor would require.
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_kb_id_desc
|
||||
ON wiki_log_entries (knowledge_base_id, id DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_tenant_id
|
||||
ON wiki_log_entries (tenant_id);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000040] wiki_log_entries schema applied successfully'; END $$;
|
||||
Reference in New Issue
Block a user