diff --git a/frontend/src/api/wiki/index.ts b/frontend/src/api/wiki/index.ts index bb62d8fe..954f5085 100644 --- a/frontend/src/api/wiki/index.ts +++ b/frontend/src/api/wiki/index.ts @@ -115,12 +115,85 @@ export function deleteWikiPage(kbId: string, slug: string) { return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`); } -export function getWikiIndex(kbId: string) { - return get(`/api/v1/knowledgebase/${kbId}/wiki/index`); +export interface WikiIndexEntryDTO { + slug: string; + title: string; + summary: string; } -export function getWikiLog(kbId: string) { - return get(`/api/v1/knowledgebase/${kbId}/wiki/log`); +export interface WikiIndexGroup { + type: string; + total: number; + items: WikiIndexEntryDTO[]; + next_cursor?: string; +} + +export interface WikiIndexResponse { + intro: string; + version: number; + groups: WikiIndexGroup[]; +} + +// getWikiIndex fetches the structured index view for a wiki KB. The +// backend replaced the legacy "markdown blob of intro + directory" with +// { intro, groups } so a 40k-page wiki no longer round-trips multiple +// megabytes on every index open. Pass `types` to restrict which +// page_type buckets come back; `limit` bounds the per-group window; +// `cursor` resumes from a previous response. +export function getWikiIndex( + kbId: string, + params?: { types?: string[]; limit?: number; cursor?: string }, +) { + const query = new URLSearchParams(); + if (params) { + if (params.types && params.types.length > 0) query.set('types', params.types.join(',')); + if (params.limit !== undefined) query.set('limit', String(params.limit)); + if (params.cursor) query.set('cursor', params.cursor); + } + const qs = query.toString(); + const suffix = qs ? `?${qs}` : ''; + return get(`/api/v1/knowledgebase/${kbId}/wiki/index${suffix}`); +} + +export interface WikiLogPageRef { + slug: string; + title?: string; +} + +export interface WikiLogEntry { + id: number; + tenant_id: number; + knowledge_base_id: string; + action: string; + knowledge_id: string; + doc_title: string; + summary: string; + // Each ref carries both slug (for navigation) and title (captured at + // ingest time for display). Legacy rows written before the title + // column was added surface as refs with an empty title; render falls + // back to the slug in that case. + pages_affected: WikiLogPageRef[]; + created_at: string; +} + +export interface WikiLogListResponse { + entries: WikiLogEntry[]; + next_cursor?: string; +} + +// getWikiLog fetches a page of wiki operation events (newest first). Pass the +// `next_cursor` from the previous response back as `cursor` to load more; +// an empty / missing `next_cursor` signals end-of-feed. `limit` is clamped +// server-side to [1, 200] and defaults to 50. +export function getWikiLog(kbId: string, params?: { cursor?: string; limit?: number }) { + const query = new URLSearchParams(); + if (params) { + if (params.cursor) query.set('cursor', params.cursor); + if (params.limit !== undefined) query.set('limit', String(params.limit)); + } + const qs = query.toString(); + const suffix = qs ? `?${qs}` : ''; + return get(`/api/v1/knowledgebase/${kbId}/wiki/log${suffix}`); } export interface WikiGraphQueryParams { diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 9e531151..ef323dff 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1708,6 +1708,12 @@ export default { fitView: 'Fit to View', logTitle: 'Activity Log', indexTitle: 'Index', + logFeedTag: 'Event feed', + logEmpty: 'No log entries yet', + logLoading: 'Loading…', + logLoadMore: 'Load more', + indexOverviewTag: 'Directory', + indexEmpty: 'No wiki pages yet. Upload documents first.', graphNoData: 'No graph data yet. Upload documents first.', showArrows: 'Show arrows', hideArrows: 'Hide arrows', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 1bac023d..3624f0f0 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2320,6 +2320,12 @@ export default { fitView: "화면에 맞춤", logTitle: '활동 로그', indexTitle: '인덱스', + logFeedTag: '이벤트 피드', + logEmpty: '로그 기록이 없습니다', + logLoading: '로딩 중…', + logLoadMore: '더 보기', + indexOverviewTag: '분류 목록', + indexEmpty: 'Wiki 페이지가 없습니다. 먼저 문서를 업로드하세요.', graphNoData: '그래프 데이터가 없습니다. 먼저 문서를 업로드하세요.', showArrows: '화살표 표시', hideArrows: '화살표 숨기기', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 41b27e8e..02e74dd2 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1930,6 +1930,12 @@ export default { fitView: 'По размеру экрана', logTitle: 'Журнал активности', indexTitle: 'Индекс', + logFeedTag: 'Лента событий', + logEmpty: 'Записей журнала пока нет', + logLoading: 'Загрузка…', + logLoadMore: 'Загрузить ещё', + indexOverviewTag: 'Каталог', + indexEmpty: 'Пока нет wiki-страниц. Сначала загрузите документы.', graphNoData: 'Нет данных графа. Сначала загрузите документы.', showArrows: 'Показать стрелки', hideArrows: 'Скрыть стрелки', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 306ae160..43e04f36 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2291,6 +2291,12 @@ export default { fitView: "适应屏幕", logTitle: '日志', indexTitle: '索引', + logFeedTag: '事件流', + logEmpty: '暂无日志记录', + logLoading: '加载中…', + logLoadMore: '加载更多', + indexOverviewTag: '分类目录', + indexEmpty: '暂无 Wiki 页面,请先上传文档', graphNoData: '暂无图谱数据,请先上传文档', showArrows: '显示箭头', hideArrows: '隐藏箭头', diff --git a/frontend/src/views/knowledge/wiki/WikiBrowser.vue b/frontend/src/views/knowledge/wiki/WikiBrowser.vue index 6eb13951..d382c74a 100644 --- a/frontend/src/views/knowledge/wiki/WikiBrowser.vue +++ b/frontend/src/views/knowledge/wiki/WikiBrowser.vue @@ -241,27 +241,30 @@ + + + + + +
@@ -566,6 +654,9 @@ import { type WikiGraphData, type WikiStats, type WikiPageIssue, + type WikiLogEntry, + type WikiIndexGroup, + type WikiIndexEntryDTO, } from '@/api/wiki' const router = useRouter() @@ -610,8 +701,60 @@ interface PageTypeBucket { initialized: boolean // true once the first page has been fetched } const pagesByType = ref>({}) -const indexPage = ref(null) -const logPage = ref(null) +// Index view state. The reader renders an incrementally-built markdown +// string rather than a structured list — opening the view loads intro +// only, and "Load more" appends one directory section at a time in a +// fixed order (Summary → Entity → Concept → Synthesis → Comparison). +// Once the last section exhausts its pages, indexHasMore flips off. +// +// We deliberately avoid keeping a parallel structured list + a parallel +// markdown buffer; the markdown is the single source of truth the reader +// renders, and [[wiki-link]] clicks flow through the same +// handleContentClick as regular page bodies. +const indexMarkdown = ref('') +const indexLoading = ref(false) +const indexAvailable = ref(false) +// Per-section pagination cursor. Empty string = not yet loaded; empty +// cursor AFTER a load = that section is exhausted. `indexSectionIdx` +// tracks which section in INDEX_SECTION_ORDER is "next to load" — we +// advance to the following section only when the current one runs out. +const indexSections = ref>({}) +const indexSectionIdx = ref(0) +const indexBodyRef = ref(null) +const indexSentinelRef = ref(null) +let indexObserver: IntersectionObserver | null = null + +// Order matters: Summary first (these are the document-level pages the +// user most often wants to see), then the LLM-derived ones. Matches the +// plan's "intro then Summary → Entity → Concept → …" progression. +const INDEX_SECTION_ORDER = [ + 'summary', + 'entity', + 'concept', + 'synthesis', + 'comparison', +] as const +// logAvailable is a flag: the sidebar "Log" entry is always shown once a +// KB exists, because the backing wiki_log_entries table is KB-independent +// and `GET /wiki/log` returns an empty entries list when nothing has been +// logged yet. We don't need a full WikiPage object anymore — selecting +// Log swaps the reader into a dedicated feed view below. +const logAvailable = ref(true) + +// activeSystemView lets the reader toggle between a regular wiki page +// (selectedPage) and a "virtual" system view — index overview and log +// feed. These modes are mutually exclusive: entering a system view +// clears selectedPage, and picking a page clears the system view flag. +const activeSystemView = ref<'' | 'index' | 'log'>('') + +// Paginated state for the log view. `entries` grows as the user scrolls; +// `nextCursor` is the opaque cursor returned by the backend and empty +// signals end-of-feed. `loading` is the guard that prevents overlapping +// loadMore calls while a request is in flight. +const logEntries = ref([]) +const logNextCursor = ref('') +const logLoading = ref(false) +const logInitialized = ref(false) // When the user types into the search box we leave pagination mode and // show a flat result list instead. Bucketed state is preserved behind @@ -780,6 +923,13 @@ function fitGraphToView() { const graphDrawerVisible = ref(false) const graphDrawerPage = ref(null) const navHistory = ref([]) +// navFromSystemView remembers which system view (Index / Log) the user +// was viewing when they clicked into a slug, so goBack can restore it +// once the page-level history stack is empty. We keep this parallel to +// navHistory rather than widening its element type — navHistory is +// consumed everywhere as `WikiPage[]` and that contract stays cleaner +// if the system-view sentinel lives in its own ref. +const navFromSystemView = ref<'' | 'index' | 'log'>('') // Index and log pages are now state refs (loaded by their own endpoints // at startup) rather than computed over the full page list. The old // computed implementation required pulling every page into memory just @@ -1242,6 +1392,39 @@ const renderedContent = computed(() => { return renderMarkdown(selectedPage.value.content) }) +// Label shown next to the back arrow on page headers. Prefers the +// nearest page-history entry when available so the user sees where +// they'll land; falls back to the Index/Log label when the current +// page was opened directly from a system view. +const backLabel = computed(() => { + if (navHistory.value.length > 0) { + return navHistory.value[navHistory.value.length - 1].title + } + if (navFromSystemView.value === 'index') { + return t('knowledgeEditor.wikiBrowser.indexTitle') + } + if (navFromSystemView.value === 'log') { + return t('knowledgeEditor.wikiBrowser.logTitle') + } + return '' +}) + +// Rendered markdown for the incremental index view. Re-runs every time +// indexMarkdown grows (initial intro load or a loadMore section append). +const renderedIndexMarkdown = computed(() => { + if (!indexMarkdown.value) return '' + return renderMarkdown(indexMarkdown.value) +}) + +// True while another section or page is available to load. Starts true +// after the first fetch (intro only loaded, sections untouched), becomes +// false once the last section in INDEX_SECTION_ORDER is exhausted. +const indexHasMore = computed(() => { + if (!indexAvailable.value) return false + if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return false + return true +}) + watch(renderedContent, async () => { await nextTick() if (readerBodyRef.value) { @@ -1249,6 +1432,16 @@ watch(renderedContent, async () => { } }) +// Index body may contain image markdown from an LLM-generated intro; +// hydrate the same way as regular page content so protected URLs +// resolve. Also re-applied after every loadMore append. +watch(renderedIndexMarkdown, async () => { + await nextTick() + if (indexBodyRef.value) { + await hydrateProtectedFileImages(indexBodyRef.value) + } +}) + function handleContentClick(e: MouseEvent) { const target = e.target as HTMLElement if (target.classList.contains('wiki-content-link')) { @@ -1341,20 +1534,250 @@ async function loadPagesForType(type: string, opts: { reset?: boolean } = {}) { groupScrollerRef.value?.updateVisibleItems?.(true) } -// loadIndexAndLog fetches the two pinned system pages directly by slug -// instead of scanning the full page list for them. The endpoints return -// auto-created defaults if the wiki hasn't produced them yet, so we can -// always bind the refs and the template's v-if gates decide visibility. +// loadIndexAndLog probes the wiki index so the sidebar knows to show +// the pinned Index/Log entries. We ask the backend for intro only (zero +// group types) — a bounded response regardless of KB size. Sections are +// fetched lazily after the user actually opens the Index view; see +// loadMoreIndexSection. +// +// The log "page" is no longer stored in wiki_pages — it lives in the +// dedicated wiki_log_entries table. We don't need to pre-fetch anything +// here to decide whether to render the sidebar Log entry; the flag is +// always on, and the actual feed is fetched lazily when the user clicks +// the entry (see openLogView / loadMoreLog). +// stripLegacyIndexDirectory removes the inline "## Summary (N)\n[[...]] +// ..." directory listing from a legacy index row. Old wiki_pages rows +// stored "intro + directory markdown" in content; after the refactor +// intro is the whole payload, but pre-existing KBs still carry the +// directory until the next ingest batch rewrites it (see +// wikiIngestService.rebuildIndexPage). We don't want the stale +// directory to show up in the reader alongside the new live-fetched +// sections, so we clip everything from the first `\n## ` heading on. +function stripLegacyIndexDirectory(intro: string): string { + if (!intro) return '' + const idx = intro.indexOf('\n## ') + if (idx < 0) return intro.trim() + return intro.slice(0, idx).trim() +} + async function loadIndexAndLog() { try { - const [idxRes, logRes] = await Promise.all([ - getWikiIndex(props.knowledgeBaseId), - getWikiLog(props.knowledgeBaseId), - ]) - indexPage.value = (idxRes as any).data || (idxRes as any) - logPage.value = (logRes as any).data || (logRes as any) + // We only need intro on the initial probe — the directory groups + // are fetched lazily once the user opens the Index view. Passing + // an unknown type filter yields a cheap single count(*) + 0 rows + // on the backend instead of scanning every directory group, and + // the frontend discards the resulting empty group unconditionally. + const idxRes = await getWikiIndex(props.knowledgeBaseId, { types: ['__intro_only__'], limit: 1 }) + const body: any = (idxRes as any).data || (idxRes as any) + const intro: string = body?.intro || '' + const cleanIntro = stripLegacyIndexDirectory(intro) + indexMarkdown.value = cleanIntro ? cleanIntro + '\n' : '' + indexAvailable.value = true + indexSections.value = {} + indexSectionIdx.value = 0 + logAvailable.value = true } catch (e) { - console.error('Failed to load wiki index/log:', e) + console.error('Failed to load wiki index:', e) + } +} + +// openIndexView switches the reader into the markdown-rendered index +// overview. Re-uses the intro already fetched during loadPages(); only +// re-fetches on first ever open or if a prior attempt failed. +async function openIndexView() { + selectedPage.value = null + activeSystemView.value = 'index' + if (!indexMarkdown.value) { + indexLoading.value = true + try { + await loadIndexAndLog() + } finally { + indexLoading.value = false + } + } + // Observer is mounted/unmounted from a watch on activeSystemView + // + indexSentinelRef below, so nothing else to do here — entering + // the view is a render-time concern. +} + +// loadMoreIndexSection advances the directory one step forward. The +// order is fixed (Summary → Entity → Concept → …); within a section we +// paginate with the backend's cursor, and only move to the next section +// when the current one is exhausted. Each call produces one network +// round trip that appends a markdown block to indexMarkdown. +// +// Rendering is append-only markdown rather than a structured list so +// the viewer feels like a regular wiki page — [[wiki-link]] clicks flow +// through handleContentClick just like every other page body. Entries +// are rendered as plain lines (not list items) so the reader doesn't +// carry list bullets next to every link. +async function loadMoreIndexSection() { + if (indexLoading.value) return + if (indexSectionIdx.value >= INDEX_SECTION_ORDER.length) return + + const type = INDEX_SECTION_ORDER[indexSectionIdx.value] + const state = indexSections.value[type] || { loaded: false, cursor: '', total: 0 } + const isFirstChunkOfSection = !state.loaded + + indexLoading.value = true + try { + const res = await getWikiIndex(props.knowledgeBaseId, { + types: [type], + limit: 50, + cursor: isFirstChunkOfSection ? undefined : state.cursor || undefined, + }) + const body: any = (res as any).data || (res as any) + const group = (body?.groups || []).find((g: WikiIndexGroup) => g.type === type) + + const items: WikiIndexEntryDTO[] = group?.items || [] + const total: number = group?.total || 0 + const nextCursor: string = group?.next_cursor || '' + + // Only emit a section heading the first time we see entries for a + // type. An empty section is skipped entirely so the reader doesn't + // see "## Entity (0)" for a KB with no entities. + let appended = '' + if (isFirstChunkOfSection && items.length > 0) { + const label = getTypeLabel(type) + appended += `\n## ${label} (${total})\n\n` + } + for (const entry of items) { + // Plain lines rather than `- [[slug]]`: marked renders the latter + // as
  • , which adds a disc bullet before every slug. Each + // entry occupies one line thanks to `breaks: true` in renderMarkdown. + // + // Use the `[[slug|display]]` form so the anchor text shows the + // human-readable title (e.g. "东城区") instead of the URL-safe slug + // ("entity/dongcheng-qu"). The [[ ]] preprocessor in renderMarkdown + // splits on the pipe and uses the right-hand side for display text + // while the left-hand side drives navigation via handleContentClick. + // + // Fall back to the slug when the page has no title — the backend + // guarantees title is non-empty for published pages, but drafts + // or partially-indexed pages can slip through. + const display = entry.title || entry.slug + if (entry.summary) { + appended += `[[${entry.slug}|${display}]] — ${entry.summary}\n` + } else { + appended += `[[${entry.slug}|${display}]]\n` + } + } + if (appended) { + indexMarkdown.value = indexMarkdown.value + appended + } + + indexSections.value[type] = { + loaded: true, + cursor: nextCursor, + total, + } + + // Advance to the next section when this one has no more pages. + // When the section is flat-out empty (total === 0), skip the + // heading entirely and move on without emitting any markdown. + if (!nextCursor) { + indexSectionIdx.value += 1 + } + } catch (e) { + console.error(`Failed to load more index entries for ${INDEX_SECTION_ORDER[indexSectionIdx.value]}:`, e) + } finally { + indexLoading.value = false + } + + // IntersectionObserver does NOT re-fire while the target stays + // continuously intersecting. On small KBs (say a wiki with only + // 3 summary pages and no entities / concepts) the sentinel sits + // inside the viewport from the moment we finish the first section, + // so without this nudge the remaining sections would never load. + // + // After every append we yield a tick (so the DOM reflows and the + // sentinel's new rect is valid) then re-check: if it's still in + // view and we have more to load, recurse. The recursion bottoms + // out when either hasMore turns off or the sentinel is pushed + // below the fold by the accumulated entries. + await nextTick() + if (indexHasMore.value && sentinelInView()) { + loadMoreIndexSection() + } +} + +// sentinelInView reports whether the load sentinel's rect currently +// overlaps the viewport (± the same 200px cushion the observer uses), +// so after loading a section we know whether to drain another round +// even though the observer itself won't fire again while the target +// stays visible. +function sentinelInView(): boolean { + const el = indexSentinelRef.value + if (!el) return false + const rect = el.getBoundingClientRect() + const vh = window.innerHeight || document.documentElement.clientHeight + // 200px margin matches the observer's rootMargin so the drain + // threshold and the scroll-triggered threshold stay consistent. + return rect.top < vh + 200 && rect.bottom > -200 +} + +// Mount/unmount the IntersectionObserver around the Index sentinel. +// We use rootMargin to pre-load when the user scrolls within ~200px +// of the sentinel, which hides the network round-trip behind the +// scroll motion. +watch([indexSentinelRef, () => activeSystemView.value], async ([el, view]) => { + if (indexObserver) { + indexObserver.disconnect() + indexObserver = null + } + if (view !== 'index' || !el) return + indexObserver = new IntersectionObserver((entries) => { + for (const entry of entries) { + if (entry.isIntersecting && indexHasMore.value && !indexLoading.value) { + loadMoreIndexSection() + } + } + }, { rootMargin: '200px' }) + indexObserver.observe(el) +}) + +onUnmounted(() => { + if (indexObserver) { + indexObserver.disconnect() + indexObserver = null + } +}) + +// openLogView switches the reader into the log feed and (re)loads the +// first page. Called when the user clicks the sidebar Log entry. +async function openLogView() { + selectedPage.value = null + activeSystemView.value = 'log' + logEntries.value = [] + logNextCursor.value = '' + logInitialized.value = false + await loadMoreLog() +} + +// loadMoreLog appends the next page of log entries using the cursor from +// the previous response. Guarded so overlapping scroll events don't fire +// multiple requests and double-append entries. +async function loadMoreLog() { + if (logLoading.value) return + // Once a previous request reported end-of-feed (empty next_cursor), we + // stop — but only after the first fetch, so a fresh KB still runs the + // initial empty request to populate logInitialized. + if (logInitialized.value && !logNextCursor.value) return + logLoading.value = true + try { + const res = await getWikiLog(props.knowledgeBaseId, { + cursor: logNextCursor.value || undefined, + limit: 50, + }) + const body: any = (res as any).data || res + const entries: WikiLogEntry[] = body?.entries || [] + logEntries.value.push(...entries) + logNextCursor.value = body?.next_cursor || '' + logInitialized.value = true + } catch (e) { + console.error('Failed to load wiki log:', e) + } finally { + logLoading.value = false } } @@ -1390,12 +1813,14 @@ async function loadPages() { } } - // Auto-select based on query or index page - if (!selectedPage.value) { + // Auto-select based on query string or default to the index + // overview. The index is the natural landing view — it shows + // intro + a paginated directory of every page type. + if (!selectedPage.value && activeSystemView.value === '') { if (route.query.slug && typeof route.query.slug === 'string') { navigateToSlug(route.query.slug) - } else if (indexPage.value) { - selectPage(indexPage.value) + } else if (indexAvailable.value) { + openIndexView() } } } finally { @@ -1855,7 +2280,14 @@ async function selectPage(page: WikiPage) { try { if (selectedPage.value && selectedPage.value.id !== page.id) { navHistory.value.push(selectedPage.value) + } else if (!selectedPage.value && activeSystemView.value) { + // Jumping out of a system view (Index / Log) onto a page. + // navHistory only holds WikiPages, so we stash the origin + // system view separately; goBack restores it when the history + // stack is empty. + navFromSystemView.value = activeSystemView.value } + activeSystemView.value = '' const res = await getWikiPage(props.knowledgeBaseId, page.slug) selectedPage.value = (res as any).data || res as any await loadPageIssues(page.slug) @@ -1868,7 +2300,13 @@ async function navigateToSlug(slug: string) { try { if (selectedPage.value && selectedPage.value.slug !== slug) { navHistory.value.push(selectedPage.value) + } else if (!selectedPage.value && activeSystemView.value) { + // Clicking a [[slug]] from inside Index / Log — same rationale + // as selectPage above: record the system-view origin so the + // reader's back arrow can return to it. + navFromSystemView.value = activeSystemView.value } + activeSystemView.value = '' const res = await getWikiPage(props.knowledgeBaseId, slug) selectedPage.value = (res as any).data || res as any await loadPageIssues(slug) @@ -1882,6 +2320,19 @@ function goBack() { if (prev) { selectedPage.value = prev loadPageIssues(prev.slug) + return + } + // History stack is empty but we remember the page was opened from + // a system view — restore that instead of leaving the reader empty. + if (navFromSystemView.value) { + const view = navFromSystemView.value + navFromSystemView.value = '' + selectedPage.value = null + if (view === 'index') { + openIndexView() + } else if (view === 'log') { + openLogView() + } } } @@ -3736,6 +4187,102 @@ onUnmounted(() => { text-align: center; } +// ── Log feed (system view) ── +// Rendered when activeSystemView === 'log'. Sits where the markdown body +// would be for a regular wiki page — so the header/meta rules above +// already apply. We just style the feed list itself. +.wiki-log-feed { + display: flex; + flex-direction: column; + gap: 12px; + padding-top: 12px; +} + +// ── Index overview (system view) ── +// The index view renders as markdown through the same pipeline as a +// normal wiki page, so it inherits .wiki-reader-body styling automatically. +// We use a sentinel below the body to drive auto-pagination via +// IntersectionObserver — the user never sees a "Load more" button. +.wiki-index-sentinel { + display: flex; + align-items: center; + justify-content: center; + min-height: 32px; + padding: 16px 0 24px; + color: var(--td-text-color-placeholder); + font-size: 13px; +} + +.wiki-index-loading { + opacity: 0.7; +} + +.wiki-log-empty { + color: var(--td-text-color-placeholder); + text-align: center; + padding: 40px 0; + font-size: 13px; +} + +.wiki-log-entry { + border: 1px solid var(--td-border-level-1-color); + border-radius: 8px; + padding: 10px 12px; + background: var(--td-bg-color-container); +} + +.wiki-log-entry-header { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 4px; +} + +.wiki-log-entry-title { + font-weight: 500; + color: var(--td-text-color-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + flex: 1; + min-width: 0; +} + +.wiki-log-entry-time { + color: var(--td-text-color-placeholder); + font-size: 12px; + white-space: nowrap; +} + +.wiki-log-entry-summary { + color: var(--td-text-color-secondary); + font-size: 13px; + margin: 4px 0; +} + +.wiki-log-entry-pages { + display: flex; + flex-wrap: wrap; + gap: 4px 10px; + margin-top: 4px; +} + +.wiki-log-entry-page { + color: var(--td-brand-color); + font-size: 12px; + text-decoration: none; +} + +.wiki-log-entry-page:hover { + text-decoration: underline; +} + +.wiki-log-load-more { + display: flex; + justify-content: center; + padding: 12px 0; +} + .wiki-empty-icon { width: 64px; height: 64px; diff --git a/internal/agent/tools/wiki_index_overview_test.go b/internal/agent/tools/wiki_index_overview_test.go new file mode 100644 index 00000000..0c247d18 --- /dev/null +++ b/internal/agent/tools/wiki_index_overview_test.go @@ -0,0 +1,121 @@ +package tools + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint verifies +// that when a group's total exceeds its item count, the synthesized +// markdown surfaces both numbers and the "showing top N" banner so the +// model knows further exploration requires a separate tool call. +func TestRenderIndexOverviewForAgent_IncludesIntroAndTruncationHint(t *testing.T) { + resp := &types.WikiIndexResponse{ + Intro: "# Wiki Index\n\nWelcome to the wiki.", + Groups: []types.WikiIndexGroup{ + { + Type: types.WikiPageTypeEntity, + Total: 4500, + Items: []types.WikiIndexEntry{ + {Slug: "entity/acme", Title: "Acme", Summary: "A company"}, + {Slug: "entity/beta", Title: "Beta", Summary: "Another company"}, + }, + }, + { + Type: types.WikiPageTypeConcept, + Total: 2, + Items: []types.WikiIndexEntry{ + {Slug: "concept/rag", Title: "RAG", Summary: "Retrieval-augmented generation"}, + {Slug: "concept/emb", Title: "Embedding", Summary: ""}, + }, + }, + }, + } + + got := renderIndexOverviewForAgent(resp) + + assertContains(t, got, "# Wiki Index") + // Truncation banner for the big group: Total != len(Items). + assertContains(t, got, "4500 total, showing top 2") + // Non-truncated group uses the simple "(N)" form. + assertContains(t, got, "## Concept (2)") + // Entry formatting — with and without summary. Entries emit the + // [[slug|title]] form so the LLM sees the human-readable name. + assertContains(t, got, "[[entity/acme|Acme]] — A company") + // When the entry has a title but no summary, slug|title still + // renders; the dash + summary portion is elided. + assertContains(t, got, "[[concept/emb|Embedding]]") + // Exploration hint. + assertContains(t, got, "wiki_search") +} + +// TestRenderIndexOverviewForAgent_EmptyKB explains to the model that the +// wiki has no pages yet — otherwise it might hallucinate slugs to read. +func TestRenderIndexOverviewForAgent_EmptyKB(t *testing.T) { + resp := &types.WikiIndexResponse{ + Intro: "# Empty\n", + Groups: []types.WikiIndexGroup{ + {Type: types.WikiPageTypeEntity, Total: 0, Items: nil}, + }, + } + + got := renderIndexOverviewForAgent(resp) + assertContains(t, got, "No wiki pages yet") + if strings.Contains(got, "## Entity") { + t.Fatalf("empty groups should not surface a heading: %q", got) + } +} + +// TestRenderIndexOverviewForAgent_UnknownType falls through to the raw +// type string — covers forward compatibility for LLM-created page types +// the frontend label map does not yet know about. +func TestRenderIndexOverviewForAgent_UnknownType(t *testing.T) { + resp := &types.WikiIndexResponse{ + Intro: "# Wiki\n", + Groups: []types.WikiIndexGroup{ + { + Type: "custom-type", + Total: 1, + Items: []types.WikiIndexEntry{{Slug: "custom-type/x", Title: "X"}}, + }, + }, + } + + got := renderIndexOverviewForAgent(resp) + assertContains(t, got, "## custom-type (1)") +} + +// TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory covers +// the forward-compat path: KBs that haven't been re-ingested since the +// index refactor still carry an "intro + ## Summary..." payload on the +// index row. The agent synthesizer must clip the legacy directory so +// the model doesn't see it next to the live top-K block the group +// loop emits below. +func TestRenderIndexOverviewForAgent_StripsLegacyInlineDirectory(t *testing.T) { + resp := &types.WikiIndexResponse{ + Intro: "# Wiki Index\n\nWelcome.\n\n## Summary (3)\n\n[[summary/legacy]] — old", + Groups: []types.WikiIndexGroup{ + { + Type: types.WikiPageTypeEntity, + Total: 1, + Items: []types.WikiIndexEntry{{Slug: "entity/live", Title: "Live", Summary: "fresh"}}, + }, + }, + } + + got := renderIndexOverviewForAgent(resp) + if strings.Contains(got, "summary/legacy") { + t.Fatalf("legacy inline directory should have been stripped, got:\n%s", got) + } + assertContains(t, got, "Welcome.") + assertContains(t, got, "[[entity/live|Live]] — fresh") +} + +func assertContains(t *testing.T, haystack, needle string) { + t.Helper() + if !strings.Contains(haystack, needle) { + t.Fatalf("expected output to contain %q, got:\n%s", needle, haystack) + } +} diff --git a/internal/agent/tools/wiki_tools.go b/internal/agent/tools/wiki_tools.go index 3212e131..d7a988f9 100644 --- a/internal/agent/tools/wiki_tools.go +++ b/internal/agent/tools/wiki_tools.go @@ -12,6 +12,83 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" ) +// wikiIndexAgentTopK is the per-type cap applied when synthesizing the +// index overview for wiki_read_page('index'). The agent uses the overview +// to get a sense of the wiki's shape; any deeper exploration should go +// through wiki_search. Keeping the cap small bounds the content envelope +// served to the LLM, which is the main reason this synthesis exists. +const wikiIndexAgentTopK = 20 + +// renderIndexOverviewForAgent formats a WikiIndexResponse as a compact +// markdown block suitable for embedding inside the tag that +// wiki_read_page returns. We deliberately keep the structure almost +// identical to the legacy "intro + ## Type (N)\n[[slug]] — summary" +// markdown so existing agent prompts (prompts_wiki.go) that reason +// about index layouts do not need retraining. +func renderIndexOverviewForAgent(resp *types.WikiIndexResponse) string { + var sb strings.Builder + // Intro may still carry a legacy inline directory on KBs that + // haven't been re-ingested since the index refactor — clip + // everything from the first "\n## " heading onwards so the model + // doesn't see the old directory alongside the live top-K below. + intro := strings.TrimSpace(resp.Intro) + if idx := strings.Index(intro, "\n## "); idx >= 0 { + intro = strings.TrimSpace(intro[:idx]) + } + if intro != "" { + sb.WriteString(intro) + sb.WriteString("\n") + } + + typeLabels := map[string]string{ + types.WikiPageTypeSummary: "Summary", + types.WikiPageTypeEntity: "Entity", + types.WikiPageTypeConcept: "Concept", + types.WikiPageTypeSynthesis: "Synthesis", + types.WikiPageTypeComparison: "Comparison", + } + + nonEmpty := 0 + for _, g := range resp.Groups { + if g.Total == 0 { + continue + } + label := typeLabels[g.Type] + if label == "" { + label = g.Type + } + if int64(len(g.Items)) < g.Total { + fmt.Fprintf(&sb, "\n## %s (%d total, showing top %d)\n\n", label, g.Total, len(g.Items)) + } else { + fmt.Fprintf(&sb, "\n## %s (%d)\n\n", label, g.Total) + } + for _, item := range g.Items { + // Emit [[slug|title]] so the LLM sees the human-readable + // name next to the slug it needs for downstream tool calls + // (wiki_read_page, wiki_search by title, etc.). Falling back + // to the slug when a page has no title keeps the wiki-link + // syntactically valid either way. + display := item.Title + if display == "" { + display = item.Slug + } + if item.Summary != "" { + fmt.Fprintf(&sb, "[[%s|%s]] — %s\n", item.Slug, display, item.Summary) + } else { + fmt.Fprintf(&sb, "[[%s|%s]]\n", item.Slug, display) + } + } + nonEmpty++ + } + + if nonEmpty == 0 { + sb.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n") + } else { + sb.WriteString("\n_To explore more pages under any category, use wiki_search with a query, or read a specific slug directly._\n") + } + return sb.String() +} + // ---- wiki_read_page ---- // WikiScope describes the effective retrieval scope for a single wiki @@ -292,6 +369,21 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (* } } + // Index page special-case. wiki_pages.content used to hold + // "intro + full directory" markdown; on big KBs that was a + // multi-MB blob the LLM had no hope of reading end-to-end. + // After the index refactor, content holds only the intro, and + // the directory is assembled on demand. Surface a bounded + // top-K overview so the agent still sees the wiki's shape + // without overflowing its context window — the explicit hint + // steers the model to wiki_search for deeper exploration. + contentBody := page.Content + if page.PageType == types.WikiPageTypeIndex { + if overview, err := t.wikiService.GetIndexView(ctx, kbID, nil, wikiIndexAgentTopK, ""); err == nil && overview != nil { + contentBody = renderIndexOverviewForAgent(overview) + } + } + return fmt.Sprintf(` %s @@ -320,7 +412,7 @@ func (t *wikiReadPageTool) Execute(ctx context.Context, args json.RawMessage) (* strings.Join(inLinksDesc, ", "), strings.Join(sourcesDesc, "\n"), page.Summary, - page.Content, + contentBody, ) } diff --git a/internal/application/repository/wiki_log_entry.go b/internal/application/repository/wiki_log_entry.go new file mode 100644 index 00000000..73d91458 --- /dev/null +++ b/internal/application/repository/wiki_log_entry.go @@ -0,0 +1,92 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// wikiLogEntryRepository implements interfaces.WikiLogEntryRepository. +type wikiLogEntryRepository struct { + db *gorm.DB +} + +// NewWikiLogEntryRepository constructs a GORM-backed WikiLogEntryRepository. +func NewWikiLogEntryRepository(db *gorm.DB) interfaces.WikiLogEntryRepository { + return &wikiLogEntryRepository{db: db} +} + +// AppendBatch inserts every entry in one statement. Empty batches are a +// no-op so callers can invoke unconditionally at the end of a wiki ingest +// batch without guarding against the "no events this round" case. +func (r *wikiLogEntryRepository) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error { + if len(entries) == 0 { + return nil + } + return r.db.WithContext(ctx).Create(&entries).Error +} + +// List returns up to `limit` entries strictly older than `cursor`, newest +// first. `cursor` is the stringified ID of the oldest entry from the +// previous page; an empty string starts from the newest. Callers get back +// a nextCursor string to pass on the next request — empty means no more. +// +// `limit` is clamped to [1, 200]. Values outside that range are coerced +// silently to keep the handler simple. +func (r *wikiLogEntryRepository) List( + ctx context.Context, + kbID string, + cursor string, + limit int, +) ([]*types.WikiLogEntry, string, error) { + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + + q := r.db.WithContext(ctx). + Where("knowledge_base_id = ?", kbID). + Order("id DESC"). + Limit(limit) + + if cursor != "" { + cursorID, err := strconv.ParseUint(cursor, 10, 64) + if err != nil { + return nil, "", fmt.Errorf("invalid cursor %q: %w", cursor, err) + } + q = q.Where("id < ?", cursorID) + } + + var entries []*types.WikiLogEntry + if err := q.Find(&entries).Error; err != nil { + return nil, "", err + } + + nextCursor := "" + // Only emit a cursor when we actually filled the page — a short page + // is guaranteed to be the tail, so returning a cursor would just cause + // the frontend to fire a final empty request. + if len(entries) == limit { + nextCursor = strconv.FormatUint(entries[len(entries)-1].ID, 10) + } + return entries, nextCursor, nil +} + +// DeleteByKB drops every log entry for a KB. No "affected rows" signal is +// surfaced — missing rows are a legitimate state (e.g., a KB that was +// created before wiki_log_entries existed) and not a failure condition. +func (r *wikiLogEntryRepository) DeleteByKB(ctx context.Context, kbID string) error { + if kbID == "" { + return errors.New("wiki log entries: empty kb id") + } + return r.db.WithContext(ctx). + Where("knowledge_base_id = ?", kbID). + Delete(&types.WikiLogEntry{}).Error +} diff --git a/internal/application/repository/wiki_log_entry_test.go b/internal/application/repository/wiki_log_entry_test.go new file mode 100644 index 00000000..649da960 --- /dev/null +++ b/internal/application/repository/wiki_log_entry_test.go @@ -0,0 +1,219 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// wikiLogEntriesTestDDL mirrors migrations/versioned/000040_wiki_log_entries.up.sql +// but uses SQLite-compatible types so we can exercise the repo layer without +// spinning up Postgres. INTEGER PRIMARY KEY AUTOINCREMENT gives us the same +// monotonically-increasing ID semantics the production schema relies on for +// cursor pagination. +const wikiLogEntriesTestDDL = ` +CREATE TABLE IF NOT EXISTS wiki_log_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + action VARCHAR(32) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL DEFAULT '', + doc_title TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + pages_affected TEXT NOT NULL DEFAULT '[]', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +` + +func setupWikiLogTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.Exec(wikiLogEntriesTestDDL).Error) + return db +} + +func makeLogEntry(kbID, action, knowledgeID, title string, pages ...string) *types.WikiLogEntry { + refs := make(types.WikiLogPageRefs, 0, len(pages)) + for _, slug := range pages { + // Tests only care about round-tripping; title is derived from + // the slug so assertions don't have to spell out both fields. + refs = append(refs, types.WikiLogPageRef{Slug: slug, Title: "title-of-" + slug}) + } + return &types.WikiLogEntry{ + TenantID: 1, + KnowledgeBaseID: kbID, + Action: action, + KnowledgeID: knowledgeID, + DocTitle: title, + Summary: "summary for " + title, + PagesAffected: refs, + } +} + +// TestWikiLogEntryRepository_AppendBatch_WritesAllRows verifies a batch +// insert persists every entry with a monotonic ID and respects the caller- +// provided column values. +func TestWikiLogEntryRepository_AppendBatch_WritesAllRows(t *testing.T) { + db := setupWikiLogTestDB(t) + repo := NewWikiLogEntryRepository(db) + ctx := context.Background() + + entries := []*types.WikiLogEntry{ + makeLogEntry("kb-a", "ingest", "k1", "Doc 1", "entity/a", "summary/k1"), + makeLogEntry("kb-a", "ingest", "k2", "Doc 2"), + makeLogEntry("kb-a", "retract", "k3", "Doc 3", "concept/c"), + } + + require.NoError(t, repo.AppendBatch(ctx, entries)) + + // Every row should have a non-zero ID assigned by the DB, and IDs + // should increase in insertion order so the cursor pagination logic + // can rely on them. + for i := range entries { + assert.NotZero(t, entries[i].ID, "row %d should have a DB-assigned ID", i) + if i > 0 { + assert.Greater(t, entries[i].ID, entries[i-1].ID, + "AppendBatch should preserve insertion order") + } + } + + // Round-trip sanity: select back and confirm payload was preserved. + var got []types.WikiLogEntry + require.NoError(t, db.Order("id ASC").Find(&got).Error) + require.Len(t, got, len(entries)) + assert.Equal(t, "ingest", got[0].Action) + assert.Equal(t, "Doc 1", got[0].DocTitle) + assert.Equal(t, types.WikiLogPageRefs{ + {Slug: "entity/a", Title: "title-of-entity/a"}, + {Slug: "summary/k1", Title: "title-of-summary/k1"}, + }, got[0].PagesAffected) + assert.Equal(t, "retract", got[2].Action) +} + +// TestWikiLogEntryRepository_AppendBatch_Empty asserts an empty batch is a +// cheap no-op — callers at the end of a wiki ingest pass can invoke it +// unconditionally without a DB round trip. +func TestWikiLogEntryRepository_AppendBatch_Empty(t *testing.T) { + db := setupWikiLogTestDB(t) + repo := NewWikiLogEntryRepository(db) + ctx := context.Background() + + require.NoError(t, repo.AppendBatch(ctx, nil)) + require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{})) + + var count int64 + require.NoError(t, db.Model(&types.WikiLogEntry{}).Count(&count).Error) + assert.Equal(t, int64(0), count) +} + +// TestWikiLogEntryRepository_List_CursorPagination walks the feed with a +// limit smaller than the population size and confirms: +// - newest entries surface first (id DESC order), +// - passing next_cursor returns the next batch strictly older than it, +// - end-of-feed is signalled by an empty cursor. +func TestWikiLogEntryRepository_List_CursorPagination(t *testing.T) { + db := setupWikiLogTestDB(t) + repo := NewWikiLogEntryRepository(db) + ctx := context.Background() + + // 5 entries in one KB + a sibling row in another KB to ensure the + // WHERE filter actually narrows by KB. + entries := []*types.WikiLogEntry{ + makeLogEntry("kb-a", "ingest", "k1", "Doc 1"), + makeLogEntry("kb-a", "ingest", "k2", "Doc 2"), + makeLogEntry("kb-a", "ingest", "k3", "Doc 3"), + makeLogEntry("kb-a", "ingest", "k4", "Doc 4"), + makeLogEntry("kb-a", "retract", "k5", "Doc 5"), + } + require.NoError(t, repo.AppendBatch(ctx, entries)) + + other := makeLogEntry("kb-other", "ingest", "kx", "Other doc") + require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{other})) + + // First page: latest 2 of kb-a, newest first. + page1, cursor1, err := repo.List(ctx, "kb-a", "", 2) + require.NoError(t, err) + require.Len(t, page1, 2) + assert.Equal(t, "Doc 5", page1[0].DocTitle) + assert.Equal(t, "Doc 4", page1[1].DocTitle) + assert.NotEmpty(t, cursor1, "should emit next_cursor when the page is full") + + // Second page continues strictly older than cursor1. + page2, cursor2, err := repo.List(ctx, "kb-a", cursor1, 2) + require.NoError(t, err) + require.Len(t, page2, 2) + assert.Equal(t, "Doc 3", page2[0].DocTitle) + assert.Equal(t, "Doc 2", page2[1].DocTitle) + assert.NotEmpty(t, cursor2) + + // Third (final) page: one row remains, so cursor is empty — the repo + // must NOT emit a cursor for a short page, otherwise the frontend + // would fire a doomed follow-up request. + page3, cursor3, err := repo.List(ctx, "kb-a", cursor2, 2) + require.NoError(t, err) + require.Len(t, page3, 1) + assert.Equal(t, "Doc 1", page3[0].DocTitle) + assert.Empty(t, cursor3, "short page should signal end-of-feed with an empty cursor") + + // Sibling KB row must NOT leak into kb-a's feed. + for _, p := range append(append(page1, page2...), page3...) { + assert.Equal(t, "kb-a", p.KnowledgeBaseID) + } +} + +// TestWikiLogEntryRepository_List_InvalidCursor rejects a non-numeric +// cursor rather than silently returning the newest page, so clients +// notice bugs before they turn into phantom paginations. +func TestWikiLogEntryRepository_List_InvalidCursor(t *testing.T) { + db := setupWikiLogTestDB(t) + repo := NewWikiLogEntryRepository(db) + ctx := context.Background() + + _, _, err := repo.List(ctx, "kb-a", "not-a-number", 10) + assert.Error(t, err) +} + +// TestWikiLogEntryRepository_DeleteByKB purges one KB's rows without +// touching others. Used during KB deletion so the log table does not +// accumulate orphans after the parent KB is gone. +func TestWikiLogEntryRepository_DeleteByKB(t *testing.T) { + db := setupWikiLogTestDB(t) + repo := NewWikiLogEntryRepository(db) + ctx := context.Background() + + require.NoError(t, repo.AppendBatch(ctx, []*types.WikiLogEntry{ + makeLogEntry("kb-a", "ingest", "k1", "Doc 1"), + makeLogEntry("kb-a", "ingest", "k2", "Doc 2"), + makeLogEntry("kb-b", "ingest", "k3", "Doc 3"), + })) + + require.NoError(t, repo.DeleteByKB(ctx, "kb-a")) + + aEntries, _, err := repo.List(ctx, "kb-a", "", 100) + require.NoError(t, err) + assert.Empty(t, aEntries) + + bEntries, _, err := repo.List(ctx, "kb-b", "", 100) + require.NoError(t, err) + assert.Len(t, bEntries, 1) +} + +// TestWikiLogPageRefs_ScanAcceptsLegacyStringArray covers the backward- +// compat path in the Scan method. Any rows the ingest pipeline wrote +// before pages_affected gained a title field are stored as a plain JSON +// string array; the repo must decode them as slug-only refs so the log +// feed renders without a data migration. +func TestWikiLogPageRefs_ScanAcceptsLegacyStringArray(t *testing.T) { + var refs types.WikiLogPageRefs + require.NoError(t, refs.Scan([]byte(`["entity/acme","concept/rag"]`))) + require.Len(t, refs, 2) + assert.Equal(t, "entity/acme", refs[0].Slug) + assert.Empty(t, refs[0].Title, "legacy rows have no title until re-ingest overwrites them") + assert.Equal(t, "concept/rag", refs[1].Slug) +} diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index 506fd55a..e7e486c7 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -208,6 +208,57 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy return pages, nil } +// ListByTypeLight projects only the columns needed to render an index +// directory entry (slug, title, summary) and paginates by title ASC. +// This keeps the GET /wiki/index response cheap on KBs with tens of +// thousands of pages — the old path loaded every row including its TEXT +// content just to throw the content away on the way out. +// +// Archived pages are excluded. `limit` clamps to [1, 200]; `offset` is +// honored as-is. Returns the total non-archived count for the type +// alongside the page so the caller can render "showing N of M". +func (r *wikiPageRepository) ListByTypeLight( + ctx context.Context, + kbID string, + pageType string, + limit int, + offset int, +) ([]types.WikiIndexEntry, int64, error) { + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + if offset < 0 { + offset = 0 + } + + base := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Where("knowledge_base_id = ? AND page_type = ? AND status <> ?", + kbID, pageType, types.WikiPageStatusArchived) + + var total int64 + if err := base.Count(&total).Error; err != nil { + return nil, 0, err + } + if total == 0 { + return nil, 0, nil + } + + var entries []types.WikiIndexEntry + if err := base. + Select("slug", "title", "summary"). + Order("title ASC"). + Limit(limit). + Offset(offset). + Scan(&entries).Error; err != nil { + return nil, 0, err + } + return entries, total, nil +} + // ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID. // Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array. func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) { diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go new file mode 100644 index 00000000..4f3d5430 --- /dev/null +++ b/internal/application/repository/wiki_page_test.go @@ -0,0 +1,193 @@ +package repository + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// wikiPagesTestDDL is a minimal SQLite-compatible subset of the +// production wiki_pages DDL (migrations/versioned/000037_wiki_and_indexing.up.sql). +// JSONB is stored as TEXT in SQLite; the StringArray Scan/Value pair +// handles the JSON round-trip unchanged. +const wikiPagesTestDDL = ` +CREATE TABLE IF NOT EXISTS wiki_pages ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + slug VARCHAR(255) NOT NULL, + title VARCHAR(512) NOT NULL DEFAULT '', + page_type VARCHAR(32) NOT NULL DEFAULT 'summary', + status VARCHAR(32) NOT NULL DEFAULT 'published', + content TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + source_refs TEXT DEFAULT '[]', + chunk_refs TEXT DEFAULT '[]', + in_links TEXT DEFAULT '[]', + out_links TEXT DEFAULT '[]', + page_metadata TEXT DEFAULT '{}', + aliases TEXT DEFAULT '[]', + version INTEGER NOT NULL DEFAULT 1, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + deleted_at DATETIME +); +` + +func setupWikiPagesTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.Exec(wikiPagesTestDDL).Error) + return db +} + +// makeWikiPage builds a minimal WikiPage suitable for insert. Title is +// derived from the slug so ORDER BY title ASC yields a predictable +// test ordering without callers having to spell out both fields. +func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage { + title := slug + if idx := strings.LastIndex(slug, "/"); idx >= 0 { + title = slug[idx+1:] + } + return &types.WikiPage{ + ID: uuid.New().String(), + TenantID: 1, + KnowledgeBaseID: kbID, + Slug: slug, + Title: title, + PageType: pageType, + Status: status, + Content: "body of " + slug, + Summary: "summary of " + slug, + Version: 1, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } +} + +// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies +// that the index-view projection only emits the slug/title/summary +// triples and respects the archived-status filter. This is the whole +// point of splitting the method off ListByType — the index reader must +// not pay for TEXT content transport. +func TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + pages := []*types.WikiPage{ + makeWikiPage("kb-a", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + makeWikiPage("kb-a", "entity/beta", types.WikiPageTypeEntity, types.WikiPageStatusDraft), + makeWikiPage("kb-a", "entity/gamma", types.WikiPageTypeEntity, types.WikiPageStatusArchived), + makeWikiPage("kb-a", "concept/delta", types.WikiPageTypeConcept, types.WikiPageStatusPublished), + makeWikiPage("kb-other", "entity/leaked", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + } + for _, p := range pages { + require.NoError(t, repo.Create(ctx, p)) + } + + entries, total, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 50, 0) + require.NoError(t, err) + + // Archived is excluded; sibling KB is excluded; everything else + // surfaces regardless of draft/published status (the index shows + // both so admins notice newly-drafted pages). + assert.Equal(t, int64(2), total) + require.Len(t, entries, 2) + + // ORDER BY title ASC => alpha, beta. + assert.Equal(t, "entity/alpha", entries[0].Slug) + assert.Equal(t, "alpha", entries[0].Title) + assert.Equal(t, "summary of entity/alpha", entries[0].Summary) + assert.Equal(t, "entity/beta", entries[1].Slug) +} + +// TestListByTypeLight_Pagination walks the type list using offsets and +// asserts the count stays stable regardless of where in the list we +// are — the index handler uses total to render "showing N of M". +func TestListByTypeLight_Pagination(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + for _, s := range []string{"entity/a", "entity/b", "entity/c", "entity/d", "entity/e"} { + require.NoError(t, repo.Create(ctx, makeWikiPage("kb-a", s, types.WikiPageTypeEntity, types.WikiPageStatusPublished))) + } + + page1, total1, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 0) + require.NoError(t, err) + assert.Equal(t, int64(5), total1) + require.Len(t, page1, 2) + assert.Equal(t, "entity/a", page1[0].Slug) + assert.Equal(t, "entity/b", page1[1].Slug) + + page2, total2, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 2) + require.NoError(t, err) + assert.Equal(t, int64(5), total2, "total should be stable across pages") + require.Len(t, page2, 2) + assert.Equal(t, "entity/c", page2[0].Slug) + assert.Equal(t, "entity/d", page2[1].Slug) + + page3, total3, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 4) + require.NoError(t, err) + assert.Equal(t, int64(5), total3) + require.Len(t, page3, 1) + assert.Equal(t, "entity/e", page3[0].Slug) + + // Offset past the end yields an empty list, not an error — the + // handler relies on this to short-circuit pagination tails. + page4, total4, err := repo.ListByTypeLight(ctx, "kb-a", types.WikiPageTypeEntity, 2, 10) + require.NoError(t, err) + assert.Equal(t, int64(5), total4) + assert.Empty(t, page4) +} + +// TestListByTypeLight_EmptyType_ReturnsZero exercises the "no rows" +// short-circuit. We skip the SELECT entirely when count is zero, so +// a KB with no pages of a type shouldn't burn a pointless query. +func TestListByTypeLight_EmptyType_ReturnsZero(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + entries, total, err := repo.ListByTypeLight(ctx, "kb-empty", types.WikiPageTypeSynthesis, 50, 0) + require.NoError(t, err) + assert.Equal(t, int64(0), total) + assert.Empty(t, entries) +} + +// TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't +// want a client passing limit=100000 and forcing the DB to return a +// multi-MB response. +func TestListByTypeLight_ClampsLimit(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + for i := 0; i < 250; i++ { + slug := "entity/bulk-" + // Use a stable, zero-padded suffix so title ordering is + // deterministic for the cap assertion below. + slug += string(rune('a'+(i/26)%26)) + string(rune('a'+i%26)) + require.NoError(t, repo.Create(ctx, makeWikiPage("kb-cap", slug, types.WikiPageTypeEntity, types.WikiPageStatusPublished))) + } + + // limit=0 falls back to the default of 50. + defaultEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 0, 0) + require.NoError(t, err) + assert.Len(t, defaultEntries, 50) + + // limit=5000 clamps to 200. + clampedEntries, _, err := repo.ListByTypeLight(ctx, "kb-cap", types.WikiPageTypeEntity, 5000, 0) + require.NoError(t, err) + assert.LessOrEqual(t, len(clampedEntries), 200) +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index a55f587a..2fd7cf39 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -97,6 +97,17 @@ const ( // tick (GC pause, Redis blip) doesn't let the lock slip out from under a // live handler. wikiActiveLockRenew = 20 * time.Second + + // wikiLLMMaxAttempts is the total attempt count (initial + retries) for + // every LLM call routed through generateWithTemplate. 3 was chosen to + // absorb transient 504/timeouts from upstream gateways without + // materially prolonging task runtime when the remote is genuinely down. + wikiLLMMaxAttempts = 3 + + // wikiLLMBackoffBase is the base delay for the exponential backoff + // between retry attempts. The nth retry waits base << (n-1) — so with + // a 2s base we wait 2s, 4s, 8s between attempts. + wikiLLMBackoffBase = 2 * time.Second ) // WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as @@ -156,6 +167,7 @@ type wikiIngestService struct { chunkRepo interfaces.ChunkRepository modelService interfaces.ModelService task interfaces.TaskEnqueuer + logEntrySvc interfaces.WikiLogEntryService redisClient *redis.Client // nil in Lite mode (no Redis) // liteLocks provides per-KB mutual exclusion in Lite mode (no Redis). // Keys are kbID strings; values are unused (presence = locked). @@ -170,6 +182,7 @@ func NewWikiIngestService( chunkRepo interfaces.ChunkRepository, modelService interfaces.ModelService, task interfaces.TaskEnqueuer, + logEntrySvc interfaces.WikiLogEntryService, redisClient *redis.Client, ) interfaces.TaskHandler { svc := &wikiIngestService{ @@ -179,6 +192,7 @@ func NewWikiIngestService( chunkRepo: chunkRepo, modelService: modelService, task: task, + logEntrySvc: logEntrySvc, redisClient: redisClient, } return svc @@ -427,8 +441,11 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn type docIngestResult struct { KnowledgeID string DocTitle string - Summary string // one-line summary of the document (from summary page) - Pages []string // affected page slugs + Summary string // one-line summary of the document (from summary page) + // Pages records the wiki pages this document touched, carrying both + // the slug (for navigation / retract lookups) and the human-readable + // title captured at ingest time (for the log feed's display layer). + Pages []types.WikiLogPageRef } // WikiBatchContext holds shared data across Map and Reduce phases @@ -695,62 +712,72 @@ type combinedExtraction struct { Concepts []extractedItem `json:"concepts"` } -// rebuildIndexPage regenerates the index page. +// rebuildIndexPage refreshes the LLM-generated intro that sits on the +// index wiki_pages row. // -// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory. -// - Intro: stored in indexPage.Summary. First time: generated from document summaries. -// Subsequent: incrementally updated with changeDescription. -// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM. +// History: the index page used to store "intro + full directory listing" as +// a single multi-MB markdown blob in content. Every ingest batch rewrote +// the whole column, which on KBs with tens of thousands of pages caused +// O(N) TOAST writes per batch. The directory was lifted out into the +// structured GET /wiki/index endpoint (see wikiPageService.GetIndexView), +// and this method now only maintains the intro. +// +// Intro lifecycle: +// - First time (empty or legacy placeholder): generate from all document +// summaries via WikiIndexIntroPrompt. +// - Subsequent calls with a change description: incremental update via +// WikiIndexIntroUpdatePrompt so the intro reflects what just landed. +// - No change description: keep the existing intro untouched. +// +// The new intro is written to both Content and Summary so readers that +// still fall back to Summary (older clients, legacy migrations) stay in +// sync with the column the view actually renders. func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error { indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID) if indexPage == nil { return nil } - // List all live pages - allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID) + // Gather just the Summary-type pages the LLM needs for intro framing. + // We do NOT load or iterate every wiki page here — the directory is + // now assembled on demand by GetIndexView, which is the only reader + // that needs the full enumeration. + summaries, err := s.wikiService.ListByType(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary) if err != nil { return err } - typeOrder := []string{ - types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, - types.WikiPageTypeSynthesis, types.WikiPageTypeComparison, - } - typeLabels := map[string]string{ - types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity", - types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis", - types.WikiPageTypeComparison: "Comparison", - } - - grouped := make(map[string][]*types.WikiPage) - totalPages := 0 - for _, p := range allPages { - if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { - continue - } - if p.Status == types.WikiPageStatusArchived { - continue - } - grouped[p.PageType] = append(grouped[p.PageType], p) - totalPages++ - } - - // Build document summaries listing (only summary-type pages — they represent documents) var docSummaries strings.Builder - for _, p := range grouped[types.WikiPageTypeSummary] { + for _, p := range summaries { + if p == nil || p.Status == types.WikiPageStatusArchived { + continue + } fmt.Fprintf(&docSummaries, "\n%s\n%s\n\n\n", p.Title, p.Summary) } if docSummaries.Len() == 0 { docSummaries.WriteString("(no documents yet)") } - // Generate or update intro - existingIntro := indexPage.Summary - var intro string + // The intro lives on both Content and Summary. Prefer Content since + // that's what the new index view returns; fall back to Summary for + // rows written before this refactor so the incremental-update prompt + // has something to work with. + existingIntro := strings.TrimSpace(indexPage.Content) + if existingIntro == "" { + existingIntro = strings.TrimSpace(indexPage.Summary) + } + // Detect the legacy "intro + directory" payload. Such rows embed the + // fence-separated "## Summary" sections right after the intro, so we + // clip everything from the first directory heading onward to keep the + // intro length bounded when we feed it back into the update prompt. + if idx := strings.Index(existingIntro, "\n## "); idx >= 0 { + existingIntro = strings.TrimSpace(existingIntro[:idx]) + } - if existingIntro == "" || existingIntro == "Wiki index - table of contents" { - // First time — generate intro from scratch + var intro string + switch { + case existingIntro == "" || existingIntro == "Wiki index - table of contents": + // First time — generate intro from scratch. generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{ "DocumentSummaries": docSummaries.String(), "Language": lang, @@ -760,8 +787,8 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat } else { intro = strings.TrimSpace(generatedIntro) } - } else if changeDesc != "" { - // Incremental update — tell LLM what changed + case changeDesc != "": + // Incremental update — tell the LLM what changed. updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{ "ExistingIntro": existingIntro, "ChangeDescription": changeDesc, @@ -773,45 +800,23 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat } else { intro = strings.TrimSpace(updatedIntro) } - } else { - intro = existingIntro // no change description, keep as-is + default: + // No change description and an existing intro: leave it as-is so + // we don't bump the version for a no-op. + intro = existingIntro } - // Build directory (pure code, no LLM) - var dir strings.Builder - for _, pt := range typeOrder { - pages := grouped[pt] - if len(pages) == 0 { - continue - } - fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages)) - for _, p := range pages { - summary := p.Summary - fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary) - } - } - for pt, pages := range grouped { - inOrder := false - for _, o := range typeOrder { - if o == pt { - inOrder = true - break - } - } - if inOrder || len(pages) == 0 { - continue - } - fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages)) - for _, p := range pages { - fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary) - } - } - if totalPages == 0 { - dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n") + // Defensive: some LLM outputs occasionally bleed into a directory- + // like section even when the intro prompt doesn't ask for one. If + // the freshly-generated intro starts to look like a legacy payload, + // clip it at the first "\n## " just like we did on the read path + // above. This keeps indexPage.Content a bounded intro-only blob. + if idx := strings.Index(intro, "\n## "); idx >= 0 { + intro = strings.TrimSpace(intro[:idx]) } - indexPage.Content = intro + "\n" + dir.String() - indexPage.Summary = intro // persist intro for next incremental update + indexPage.Content = intro + indexPage.Summary = intro _, err = s.wikiService.UpdatePage(ctx, indexPage) return err } @@ -834,34 +839,31 @@ func splitSummaryLine(raw string) (summary string, content string) { return "", raw } -// appendLogEntry appends a structured, grep-parseable entry to the log page. -// Format: ## [2026-04-07 19:50:02] action | title -// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean. -func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) { - logPage, _ := s.wikiService.GetLog(ctx, kbID) - if logPage == nil { - return - } - - var sb strings.Builder - fmt.Fprintf(&sb, "\n## [%s] %s | %s\n", - time.Now().Format("2006-01-02 15:04:05"), - action, - docTitle, - ) - if knowledgeID != "" { - fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID) - } - if summary != "" { - fmt.Fprintf(&sb, "- **Summary**: %s\n", summary) - } +// buildLogEntry builds a WikiLogEntry struct for the current batch. It is +// pure (no DB access) so callers can accumulate entries cheaply under their +// lock and flush them in a single AppendBatch call at the end of the batch. +// +// Historically this was a per-event `GetLog + UpdatePage` round trip, which +// rewrote the entire log page's TEXT column on every ingest/retract op — +// O(n^2) write amplification as the log grew. The batch writer now uses +// wikiLogEntryService.AppendBatch instead; see ProcessWikiIngest. +func (s *wikiIngestService) buildLogEntry(tenantID uint64, kbID, action, knowledgeID, docTitle, summary string, pagesAffected []types.WikiLogPageRef) *types.WikiLogEntry { + // Copy pagesAffected so the entry does not alias caller-owned slices. + // The batch accumulates SlugUpdate results that may be reused downstream. + var pages types.WikiLogPageRefs if len(pagesAffected) > 0 { - fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", ")) + pages = make(types.WikiLogPageRefs, len(pagesAffected)) + copy(pages, pagesAffected) } - - logPage.Content = logPage.Content + sb.String() - if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to update log page: %v", err) + return &types.WikiLogEntry{ + TenantID: tenantID, + KnowledgeBaseID: kbID, + Action: action, + KnowledgeID: knowledgeID, + DocTitle: docTitle, + Summary: summary, + PagesAffected: pages, + CreatedAt: time.Now(), } } @@ -1017,7 +1019,23 @@ func (s *wikiIngestService) deduplicateExtractedBatch( return entities, concepts } -// generateWithTemplate executes a prompt template and calls the LLM +// generateWithTemplate executes a prompt template and calls the LLM with +// bounded exponential-backoff retries for transient infrastructure errors. +// +// Retry policy: +// - Up to wikiLLMMaxAttempts total attempts (initial + retries). +// - Only retry errors classified as transient by isTransientLLMError: +// HTTP 408/429/5xx, context deadline exceeded (when the parent ctx is +// still alive), or generic "timeout"/"connection reset" wording. +// 4xx (except 408/429) is a caller-side fault and fails fast. +// - Backoff is exponential base 2s: 2s, 4s, 8s — roughly wikiLLMBackoffBase +// * 2^(attempt-1). Honors ctx cancellation so the task can abort. +// +// This exists because wiki ingest makes several independent LLM calls per +// document (extraction, summary, dedup, citations, intro) and a single +// transient 504 from the upstream gateway used to drop the document's +// summary page permanently. Retries plus failedOps requeuing (see +// mapOneDocument) turn those events into at-most-a-few-minute hiccups. func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) { tmpl, err := template.New("wiki").Parse(promptTpl) if err != nil { @@ -1031,17 +1049,98 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel prompt := buf.String() thinking := false - response, err := chatModel.Chat(ctx, []chat.Message{ - {Role: "user", Content: prompt}, - }, &chat.ChatOptions{ - Temperature: 0.3, - Thinking: &thinking, - }) - if err != nil { - return "", fmt.Errorf("LLM call failed: %w", err) + + var lastErr error + for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ { + response, err := chatModel.Chat(ctx, []chat.Message{ + {Role: "user", Content: prompt}, + }, &chat.ChatOptions{ + Temperature: 0.3, + Thinking: &thinking, + }) + if err == nil { + return response.Content, nil + } + lastErr = err + + // Abort immediately on non-retryable errors (4xx except 408/429, + // parse/marshal failures, tool-side bugs, etc.). Retrying a + // hard "invalid arguments" error just wastes the model's budget. + if !isTransientLLMError(ctx, err) { + return "", fmt.Errorf("LLM call failed: %w", err) + } + if attempt == wikiLLMMaxAttempts { + break + } + + backoff := wikiLLMBackoffBase << (attempt - 1) + logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v", + attempt, wikiLLMMaxAttempts, backoff, err) + select { + case <-ctx.Done(): + return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err()) + case <-time.After(backoff): + } + } + return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr) +} + +// isTransientLLMError reports whether an error from the chat provider +// looks like an infrastructure hiccup worth retrying. Classification is +// intentionally conservative: the truthful "could not tell, assume +// permanent" choice keeps retries cheap and avoids masking real bugs. +// +// We treat the following as transient: +// - HTTP 408 (client request timeout — upstream usually didn't process), +// 429 (rate-limited — retry after backoff may succeed), 5xx (any +// server-side fault, including the 504 "Remote error, timeout with +// 60" we see from the gateway in front of several LLM providers). +// - Wrapped context.DeadlineExceeded when the parent ctx is still alive +// (nested per-call timeouts). +// - Substring matches on the error text for common transport failures +// ("timeout", "connection reset", "EOF") that providers surface +// without a structured status code. +func isTransientLLMError(ctx context.Context, err error) bool { + if err == nil { + return false + } + // Never retry after the parent ctx itself expired — the task is + // being cancelled and the next attempt would just fail again. + if ctx.Err() != nil { + return false } - return response.Content, nil + msg := err.Error() + // Providers that bubble HTTP status up formatted as + // "API request failed with status NNN: ..." — match that first. + for _, s := range []string{ + "status 408", "status 429", + "status 500", "status 501", "status 502", "status 503", "status 504", + "status 520", "status 521", "status 522", "status 523", "status 524", + } { + if strings.Contains(msg, s) { + return true + } + } + + lower := strings.ToLower(msg) + for _, s := range []string{ + "timeout", + "timed out", + "connection reset", + "connection refused", + "broken pipe", + "no such host", // DNS hiccup + "i/o timeout", + "unexpected eof", + "tls handshake", + "context deadline exceeded", // nested per-call deadline + } { + if strings.Contains(lower, s) { + return true + } + } + return false } // --- Helpers --- diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 7036d3f6..33ef7c28 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -392,14 +392,42 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task totalPagesAffected = len(allPagesAffected) - // Append log entries — one per operation for chronological traceability + // Collect log entries for this batch and flush them in a single INSERT. + // Historically each op triggered its own `GetLog + UpdatePage` round + // trip, which rewrote the entire log page TEXT column and caused O(n^2) + // write amplification as the log grew. AppendBatch writes one row per + // event into wiki_log_entries instead. + // + // slugsToRefs resolves each retract slug against the batch-start + // snapshot (batchCtx.SlugTitleMap) so the log feed carries titles for + // pages that existed when the batch began. Pages created or renamed + // during this batch fall through the map lookup and log as slug-only + // refs, which the frontend renders as the slug itself — a sensible + // fallback given retracts only touch pre-existing pages. + slugsToRefs := func(slugs []string) []types.WikiLogPageRef { + if len(slugs) == 0 { + return nil + } + out := make([]types.WikiLogPageRef, 0, len(slugs)) + for _, slug := range slugs { + title := batchCtx.SlugTitleMap[slug] + out = append(out, types.WikiLogPageRef{Slug: slug, Title: title}) + } + return out + } + logEntries := make([]*types.WikiLogEntry, 0, len(pendingOps)+len(docResults)) for _, op := range pendingOps { if op.Op == WikiOpRetract { - s.appendLogEntry(ctx, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, op.PageSlugs) + logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, slugsToRefs(op.PageSlugs))) } } for _, r := range docResults { - s.appendLogEntry(ctx, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages) + logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages)) + } + if len(logEntries) > 0 && s.logEntrySvc != nil { + if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err) + } } // Build change description for the Index Intro LLM prompt @@ -625,9 +653,18 @@ func (s *wikiIngestService) mapOneDocument( } } - extractedPages := make([]string, 0, len(slugItems)+1) - for slug := range slugItems { - extractedPages = append(extractedPages, slug) + // extractedPages records every wiki page this document materialized + // (entities, concepts, plus the summary page appended below). The + // slug is used for link/retract bookkeeping; the title is captured + // for the log feed so the user sees "提供本学位在线验证报告查询…" + // rather than "entity/xue-xin-wang". + extractedPages := make([]types.WikiLogPageRef, 0, len(slugItems)+1) + for slug, item := range slugItems { + title := item.Name + if title == "" { + title = slug + } + extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: slug, Title: title}) } // Count total distinct chunks cited across all slugs for logging. @@ -647,32 +684,44 @@ func (s *wikiIngestService) mapOneDocument( var docSummary string if summaryErr != nil { - logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, summaryErr) - } else { - sumLine, sumBody := splitSummaryLine(summaryContent) - if sumBody == "" { - sumBody = summaryContent - } - if sumLine == "" { - sumLine = docTitle - } - docSummaryLine = sumLine - docSummary = sumBody - if strings.TrimSpace(docSummary) == "" { - docSummary = sumLine - } - updates = append(updates, SlugUpdate{ - Slug: summarySlug, - Type: types.WikiPageTypeSummary, - DocTitle: docTitle, - KnowledgeID: knowledgeID, - SourceRef: sourceRef, - Language: lang, - SummaryLine: sumLine, - SummaryBody: sumBody, - }) - extractedPages = append(extractedPages, summarySlug) + // Summary is the headline artifact of an ingested document — a + // document with no summary page is half-ingested and leaves the + // entity/concept updates hanging without a root to link back to + // from the index. Historically we just logged and moved on, + // which meant a single transient 504 permanently dropped the + // summary page for that document. + // + // Returning an error here sends the op to failedOps (see the + // map-phase loop in ProcessWikiIngest), which requeueFailedOps + // appends back onto the pending list so the next batch retries. + // The internal retries in generateWithTemplate already exhaust + // the LLM's own transient-error budget before we give up here. + logger.Errorf(ctx, "wiki ingest: generate summary failed for %s, will requeue: %v", knowledgeID, summaryErr) + return nil, nil, fmt.Errorf("generate summary: %w", summaryErr) } + sumLine, sumBody := splitSummaryLine(summaryContent) + if sumBody == "" { + sumBody = summaryContent + } + if sumLine == "" { + sumLine = docTitle + } + docSummaryLine = sumLine + docSummary = sumBody + if strings.TrimSpace(docSummary) == "" { + docSummary = sumLine + } + updates = append(updates, SlugUpdate{ + Slug: summarySlug, + Type: types.WikiPageTypeSummary, + DocTitle: docTitle, + KnowledgeID: knowledgeID, + SourceRef: sourceRef, + Language: lang, + SummaryLine: sumLine, + SummaryBody: sumBody, + }) + extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: summarySlug, Title: docTitle}) // Entities for _, item := range extractedEntities { @@ -740,7 +789,7 @@ func (s *wikiIngestService) mapOneDocument( newSlugSet := make(map[string]bool, len(extractedPages)) for _, ns := range extractedPages { - newSlugSet[ns] = true + newSlugSet[ns.Slug] = true } var reparseOverlap, staleCount int diff --git a/internal/application/service/wiki_ingest_retry_test.go b/internal/application/service/wiki_ingest_retry_test.go new file mode 100644 index 00000000..ebe01566 --- /dev/null +++ b/internal/application/service/wiki_ingest_retry_test.go @@ -0,0 +1,99 @@ +package service + +import ( + "context" + "errors" + "testing" +) + +// TestIsTransientLLMError_HTTPStatuses covers the status codes we know +// the generic REST chat provider bubbles up as "API request failed with +// status NNN: ...". The status-code arm of the classifier is the one +// that actually fires for the 504 gateway timeouts the upstream LLM +// vendors throw most often. +func TestIsTransientLLMError_HTTPStatuses(t *testing.T) { + ctx := context.Background() + cases := []struct { + msg string + want bool + }{ + {"API request failed with status 504: Remote error, timeout with 60", true}, + {"API request failed with status 503: service unavailable", true}, + {"API request failed with status 502: bad gateway", true}, + {"API request failed with status 500: internal server error", true}, + {"API request failed with status 429: rate limited", true}, + {"API request failed with status 408: request timeout", true}, + {"API request failed with status 520: Web server returned unknown", true}, + {"API request failed with status 524: timeout (Cloudflare)", true}, + // 4xx permanent: a retry will just fail the same way. + {"API request failed with status 400: bad request", false}, + {"API request failed with status 401: unauthorized", false}, + {"API request failed with status 403: forbidden (quota exhausted)", false}, + {"API request failed with status 404: model not found", false}, + } + for _, tc := range cases { + got := isTransientLLMError(ctx, errors.New(tc.msg)) + if got != tc.want { + t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want) + } + } +} + +// TestIsTransientLLMError_TransportSubstrings covers providers (and Go +// stdlib errors) that surface transient conditions as free-form text +// without a numeric status code — DNS blips, TLS handshake failures, +// broken pipes during an in-flight body read, etc. +func TestIsTransientLLMError_TransportSubstrings(t *testing.T) { + ctx := context.Background() + cases := []struct { + msg string + want bool + }{ + {"send request: dial tcp: lookup api.example.com: no such host", true}, + {"send request: read tcp 10.0.0.1: i/o timeout", true}, + {"send request: Post \"...\": context deadline exceeded", true}, + {"send request: connection reset by peer", true}, + {"send request: connection refused", true}, + {"send request: broken pipe", true}, + {"send request: tls handshake timeout", true}, + {"send request: unexpected EOF", true}, + // Not transient: the error is actionable (missing model id, + // malformed tool call, etc.) — retry is just wasted budget. + {"model not configured for tool use", false}, + {"invalid JSON in tool arguments", false}, + {"parse template: syntax error", false}, + } + for _, tc := range cases { + got := isTransientLLMError(ctx, errors.New(tc.msg)) + if got != tc.want { + t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want) + } + } +} + +// TestIsTransientLLMError_AbortsWhenParentCtxDone ensures a cancelled +// outer context short-circuits the retry decision. Without this guard +// an ingest task that was interrupted (asynq shutdown, deletion race, +// parent timeout) would keep retrying the LLM during its teardown +// window and waste the remainder of the backoff budget. +func TestIsTransientLLMError_AbortsWhenParentCtxDone(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // This exact error would normally be classified transient (see + // TestIsTransientLLMError_HTTPStatuses), but a cancelled ctx must + // override that so the task can unwind promptly. + err := errors.New("API request failed with status 504: Remote error") + if isTransientLLMError(ctx, err) { + t.Fatal("cancelled ctx should short-circuit to non-transient") + } +} + +// TestIsTransientLLMError_NilError is a sanity check — nil should never +// be treated as retryable, otherwise callers that forget to handle the +// happy path would spin forever. +func TestIsTransientLLMError_NilError(t *testing.T) { + if isTransientLLMError(context.Background(), nil) { + t.Fatal("nil error should not be transient") + } +} diff --git a/internal/application/service/wiki_log_entry.go b/internal/application/service/wiki_log_entry.go new file mode 100644 index 00000000..f2d18a35 --- /dev/null +++ b/internal/application/service/wiki_log_entry.go @@ -0,0 +1,51 @@ +package service + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// wikiLogEntryService is a thin wrapper over WikiLogEntryRepository. The +// event layer carries no business rules today, so the service just +// forwards to the repo — the indirection exists so handlers / ingest +// batch code depend on an interface rather than a concrete repo type. +type wikiLogEntryService struct { + repo interfaces.WikiLogEntryRepository +} + +// NewWikiLogEntryService constructs a WikiLogEntryService backed by the +// given repository. +func NewWikiLogEntryService(repo interfaces.WikiLogEntryRepository) interfaces.WikiLogEntryService { + return &wikiLogEntryService{repo: repo} +} + +// AppendBatch records the given events in one database round trip. Empty +// batches are a no-op (the repo handles that). +func (s *wikiLogEntryService) AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error { + return s.repo.AppendBatch(ctx, entries) +} + +// List paginates the per-KB event feed. See repo.List for cursor semantics. +func (s *wikiLogEntryService) List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error) { + entries, nextCursor, err := s.repo.List(ctx, kbID, cursor, limit) + if err != nil { + return nil, err + } + if entries == nil { + // Normalise to an empty slice so clients don't need to + // distinguish `null` from `[]`. + entries = []*types.WikiLogEntry{} + } + return &types.WikiLogEntryListResponse{ + Entries: entries, + NextCursor: nextCursor, + }, nil +} + +// DeleteByKB removes the log feed for a KB. Called when the KB itself is +// being deleted, so no further reads happen. +func (s *wikiLogEntryService) DeleteByKB(ctx context.Context, kbID string) error { + return s.repo.DeleteByKB(ctx, kbID) +} diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index c7e4176f..09900280 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -6,6 +6,7 @@ import ( "fmt" "regexp" "sort" + "strconv" "strings" "time" @@ -246,13 +247,125 @@ func (s *wikiPageService) GetIndex(ctx context.Context, kbID string) (*types.Wik return page, nil } -// GetLog returns the log page for a knowledge base +// wikiIndexContentPageTypes enumerates the page types that make up a wiki's +// user-visible directory. System pages (index/log) are excluded; any +// LLM-created type we do not recognize surfaces under a generic "other" +// bucket. +var wikiIndexContentPageTypes = []string{ + types.WikiPageTypeSummary, + types.WikiPageTypeEntity, + types.WikiPageTypeConcept, + types.WikiPageTypeSynthesis, + types.WikiPageTypeComparison, +} + +// GetIndexView builds the structured index response without ever +// materializing a multi-MB directory markdown string. Intro is read from +// the index wiki_page row (which now carries only intro text — see +// rebuildIndexPage). Each requested page_type is paginated independently +// with ListByTypeLight so reads stay O(page_size) rather than O(total +// pages in the KB). +// +// `pageTypes` narrows which groups to include; empty = all content types. +// `limit` is the per-group window size (defaults to 50, capped at 200). +// `cursor` is an opaque offset string; currently we use the stringified +// offset so clients can resume where they left off. Because different +// page_types paginate independently, `cursor` applies uniformly to every +// group — if the caller wants per-group cursors it should request one +// type at a time via `pageTypes`. That simplifies the wire format and +// matches the frontend's tabbed UX. +func (s *wikiPageService) GetIndexView( + ctx context.Context, + kbID string, + pageTypes []string, + limit int, + cursor string, +) (*types.WikiIndexResponse, error) { + indexPage, err := s.GetIndex(ctx, kbID) + if err != nil { + return nil, fmt.Errorf("load index page: %w", err) + } + + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + offset := 0 + if cursor != "" { + v, parseErr := strconv.Atoi(cursor) + if parseErr != nil || v < 0 { + return nil, fmt.Errorf("invalid cursor %q", cursor) + } + offset = v + } + + // Default to every known content type when the caller passes no + // filter. Any unknown request-time type is passed through verbatim so + // future page types (declared in types/wiki_page.go) start showing + // up in the index the moment the LLM starts creating them, without a + // handler change. + selected := pageTypes + if len(selected) == 0 { + selected = append([]string{}, wikiIndexContentPageTypes...) + } + + groups := make([]types.WikiIndexGroup, 0, len(selected)) + for _, pt := range selected { + entries, total, listErr := s.repo.ListByTypeLight(ctx, kbID, pt, limit, offset) + if listErr != nil { + return nil, fmt.Errorf("list %s pages: %w", pt, listErr) + } + if entries == nil { + entries = []types.WikiIndexEntry{} + } + next := "" + // Only emit a cursor when a full page was returned AND more rows + // remain past `offset + limit`. A short page or one that exactly + // consumed the remainder should signal end-of-feed. + if len(entries) == limit && int64(offset+len(entries)) < total { + next = strconv.Itoa(offset + limit) + } + groups = append(groups, types.WikiIndexGroup{ + Type: pt, + Total: total, + Items: entries, + NextCursor: next, + }) + } + + // The intro used to be stored on indexPage.Summary while + // indexPage.Content held intro + directory markdown. After the + // directory was lifted out of wiki_pages the content column holds + // only the intro. Fall back to Summary for KBs that haven't been + // re-ingested since the change so the response is never blank. + intro := indexPage.Content + if strings.TrimSpace(intro) == "" { + intro = indexPage.Summary + } + + return &types.WikiIndexResponse{ + Intro: intro, + Version: indexPage.Version, + Groups: groups, + }, nil +} + +// GetLog returns the wiki_pages row for slug='log' if it exists. +// +// Log events are now stored in the dedicated `wiki_log_entries` table and +// paginated via wikiLogEntryService — the per-KB log is no longer a single +// TEXT column on a wiki_pages row (that model caused O(n^2) write +// amplification as logs grew). This method is retained for callers that +// still probe the legacy row (wiki_lint, knowledge delete, etc.), but it +// no longer auto-creates the placeholder page on miss; a missing row is a +// normal state and the helper returns `nil, nil`. func (s *wikiPageService) GetLog(ctx context.Context, kbID string) (*types.WikiPage, error) { page, err := s.repo.GetBySlug(ctx, kbID, "log") if err != nil { if errors.Is(err, repository.ErrWikiPageNotFound) { - return s.createDefaultPage(ctx, kbID, "log", "Log", types.WikiPageTypeLog, - "# Wiki Operation Log\n\nChronological record of wiki operations.\n") + return nil, nil } return nil, err } @@ -619,6 +732,13 @@ func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*typ return s.repo.ListAll(ctx, kbID) } +// ListByType returns every wiki page of a given type for a KB. Exposed so +// callers like intro regeneration can load only the page type they need +// (summaries) instead of paying for the full ListAll scan. +func (s *wikiPageService) ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) { + return s.repo.ListByType(ctx, kbID, pageType) +} + // ListPagesBySourceRef exposes the repository's source-ref lookup so higher // layers (delete flow, retract reconciliation) can re-query the current wiki // state without depending on a stale caller-captured slug list. @@ -821,79 +941,23 @@ func (s *wikiPageService) InjectCrossLinks(ctx context.Context, kbID string, aff } } -// RebuildIndexPage regenerates the index page directory. +// RebuildIndexPage was historically called by agent write/rename tools to +// refresh the index page's directory listing after a page mutation. +// +// The directory is no longer persisted in wiki_pages.content — it is +// assembled on demand by GetIndexView from the lightweight ListByTypeLight +// projection, so individual page writes don't need to redo O(N) string +// concatenation and rewrite a multi-MB TEXT column anymore. Keeping the +// method name lets existing agent tool call sites (wiki_write_page, +// wiki_rename_page) compile unchanged; the body is now intentionally a +// no-op. +// +// The intro that still lives on the index row is managed separately by +// the ingest pipeline (see wikiIngestService.rebuildIndexPage) on batch +// completion, which is where we actually have the LLM + change description +// context needed to rewrite it. func (s *wikiPageService) RebuildIndexPage(ctx context.Context, kbID string) error { - indexPage, err := s.GetIndex(ctx, kbID) - if err != nil { - return err - } - - allPages, err := s.ListAllPages(ctx, kbID) - if err != nil { - return err - } - - typeOrder := []string{ - types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, - types.WikiPageTypeSynthesis, types.WikiPageTypeComparison, - } - typeLabels := map[string]string{ - types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity", - types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis", - types.WikiPageTypeComparison: "Comparison", - } - - grouped := make(map[string][]*types.WikiPage) - totalPages := 0 - for _, p := range allPages { - if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { - continue - } - if p.Status == types.WikiPageStatusArchived { - continue - } - grouped[p.PageType] = append(grouped[p.PageType], p) - totalPages++ - } - - var dir strings.Builder - for _, pt := range typeOrder { - pages := grouped[pt] - if len(pages) == 0 { - continue - } - fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages)) - for _, p := range pages { - fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary) - } - } - for pt, pages := range grouped { - inOrder := false - for _, o := range typeOrder { - if o == pt { - inOrder = true - break - } - } - if inOrder || len(pages) == 0 { - continue - } - fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages)) - for _, p := range pages { - fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary) - } - } - if totalPages == 0 { - dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n") - } - - intro := indexPage.Summary - if intro == "" { - intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n" - indexPage.Summary = intro - } - - indexPage.Content = intro + "\n" + dir.String() - _, err = s.UpdatePage(ctx, indexPage) - return err + _ = ctx + _ = kbID + return nil } diff --git a/internal/container/container.go b/internal/container/container.go index 622d7b72..57bdbac7 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -153,6 +153,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(repository.NewDataSourceRepository)) must(container.Provide(repository.NewSyncLogRepository)) must(container.Provide(repository.NewWikiPageRepository)) + must(container.Provide(repository.NewWikiLogEntryRepository)) // MCP manager for managing MCP client connections logger.Debugf(ctx, "[Container] Registering MCP manager...") @@ -186,6 +187,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(service.NewCustomAgentService)) must(container.Provide(memoryService.NewMemoryService)) must(container.Provide(service.NewWikiPageService)) + must(container.Provide(service.NewWikiLogEntryService)) must(container.Provide(service.NewWikiIngestService, dig.Name("wikiIngest"))) must(container.Provide(service.NewWikiLintService)) diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index 90cf9418..4a354c3b 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -19,9 +19,10 @@ import ( // WikiPageHandler handles HTTP requests for wiki page operations type WikiPageHandler struct { - wikiService interfaces.WikiPageService - kbService interfaces.KnowledgeBaseService - lintService *service.WikiLintService + wikiService interfaces.WikiPageService + kbService interfaces.KnowledgeBaseService + lintService *service.WikiLintService + logEntryService interfaces.WikiLogEntryService } // NewWikiPageHandler creates a new wiki page handler @@ -29,11 +30,13 @@ func NewWikiPageHandler( wikiService interfaces.WikiPageService, kbService interfaces.KnowledgeBaseService, lintService *service.WikiLintService, + logEntryService interfaces.WikiLogEntryService, ) *WikiPageHandler { return &WikiPageHandler{ - wikiService: wikiService, - kbService: kbService, - lintService: lintService, + wikiService: wikiService, + kbService: kbService, + lintService: lintService, + logEntryService: logEntryService, } } @@ -274,12 +277,19 @@ func (h *WikiPageHandler) DeletePage(c *gin.Context) { } // GetIndex godoc -// @Summary Get wiki index page -// @Description Returns the wiki index page (creates default if not exists) +// @Summary Get wiki index view +// @Description Returns the wiki index as intro text plus per-type paginated +// @Description directory groups. The heavy directory markdown that used to +// @Description live in wiki_pages.content was replaced with this structured +// @Description response so a KB with tens of thousands of pages no longer +// @Description materializes megabytes of TEXT on every index open. // @Tags Wiki // @Produce json -// @Param kb_id path string true "Knowledge base ID" -// @Success 200 {object} types.WikiPage +// @Param kb_id path string true "Knowledge base ID" +// @Param types query string false "Comma-separated page types (default: all content types)" +// @Param limit query int false "Per-group window size, 1-200 (default 50)" +// @Param cursor query string false "Opaque offset cursor from previous response" +// @Success 200 {object} types.WikiIndexResponse // @Security Bearer // @Router /api/v1/knowledgebase/{kb_id}/wiki/index [get] func (h *WikiPageHandler) GetIndex(c *gin.Context) { @@ -289,22 +299,43 @@ func (h *WikiPageHandler) GetIndex(c *gin.Context) { return } - page, err := h.wikiService.GetIndex(c.Request.Context(), kbID) + var pageTypes []string + if raw := c.Query("types"); raw != "" { + for _, t := range strings.Split(raw, ",") { + t = strings.TrimSpace(t) + if t != "" { + pageTypes = append(pageTypes, t) + } + } + } + + limit := 50 + if raw := c.Query("limit"); raw != "" { + if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 { + limit = v + } + } + + resp, err := h.wikiService.GetIndexView(c.Request.Context(), kbID, pageTypes, limit, c.Query("cursor")) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, page) + c.JSON(http.StatusOK, resp) } // GetLog godoc // @Summary Get wiki operation log -// @Description Returns the wiki operation log page (creates default if not exists) +// @Description Returns a paginated feed of wiki operation events (ingest, retract, ...) +// @Description newest-first. Pagination is cursor-based: pass `next_cursor` from the +// @Description previous response back as `cursor` to fetch the next page. // @Tags Wiki // @Produce json -// @Param kb_id path string true "Knowledge base ID" -// @Success 200 {object} types.WikiPage +// @Param kb_id path string true "Knowledge base ID" +// @Param cursor query string false "Opaque cursor from the previous page (empty = newest)" +// @Param limit query int false "Page size, 1-200 (default 50)" +// @Success 200 {object} types.WikiLogEntryListResponse // @Security Bearer // @Router /api/v1/knowledgebase/{kb_id}/wiki/log [get] func (h *WikiPageHandler) GetLog(c *gin.Context) { @@ -314,13 +345,21 @@ func (h *WikiPageHandler) GetLog(c *gin.Context) { return } - page, err := h.wikiService.GetLog(c.Request.Context(), kbID) + cursor := c.Query("cursor") + limit := 50 + if raw := c.Query("limit"); raw != "" { + if v, convErr := strconv.Atoi(raw); convErr == nil && v > 0 { + limit = v + } + } + + resp, err := h.logEntryService.List(c.Request.Context(), kbID, cursor, limit) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, page) + c.JSON(http.StatusOK, resp) } // Graph query parameter bounds. The defaults cap an `overview` request at diff --git a/internal/types/interfaces/wiki_log_entry.go b/internal/types/interfaces/wiki_log_entry.go new file mode 100644 index 00000000..61f10f36 --- /dev/null +++ b/internal/types/interfaces/wiki_log_entry.go @@ -0,0 +1,41 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// WikiLogEntryRepository persists per-KB wiki operation events. The table +// is append-only; updates are never issued. Reads paginate newest-first +// with a stable numeric cursor. +type WikiLogEntryRepository interface { + // AppendBatch inserts a batch of events in a single INSERT...VALUES + // statement. All entries in a batch must belong to the same KB in + // practice, but the repo does not enforce that — the caller owns the + // grouping. + AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error + + // List returns up to `limit` entries in (knowledge_base_id, id DESC) + // order. `cursor` is the ID of the last entry from the previous page + // ("return entries strictly older than this"); empty string means + // "from the newest". + // + // The returned nextCursor is empty when no further rows remain; the + // caller can treat that as end-of-stream. + List(ctx context.Context, kbID string, cursor string, limit int) (entries []*types.WikiLogEntry, nextCursor string, err error) + + // DeleteByKB removes every log entry belonging to the given KB. Used + // when a wiki KB is deleted so the log table does not accumulate + // orphans. Safe to call even if no rows exist. + DeleteByKB(ctx context.Context, kbID string) error +} + +// WikiLogEntryService is the service layer over WikiLogEntryRepository. +// Kept intentionally thin — it mirrors the repo today because the events +// are pure bookkeeping and carry no business rules. +type WikiLogEntryService interface { + AppendBatch(ctx context.Context, entries []*types.WikiLogEntry) error + List(ctx context.Context, kbID string, cursor string, limit int) (*types.WikiLogEntryListResponse, error) + DeleteByKB(ctx context.Context, kbID string) error +} diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index d878f258..2229e6c3 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -50,10 +50,33 @@ type WikiPageService interface { // GetIndex returns the index page for a knowledge base. // Creates a default one if it doesn't exist. + // + // The index row now carries only the LLM-generated intro in its content + // column — the heavy directory listing was moved out of wiki_pages. + // Callers that want a renderable directory should use GetIndexView, + // which assembles {intro, groups} structured output on demand. GetIndex(ctx context.Context, kbID string) (*types.WikiPage, error) + // GetIndexView returns the structured index response — intro (from the + // index row) plus a paginated window of directory entries per requested + // page_type. `types` restricts which page types to include (empty = + // all), `limit` bounds the per-type window size, `cursor` is an opaque + // offset string resumed from a previous response. + // + // Keeping the directory as JSON rather than a multi-MB markdown blob + // means a 40k-page KB no longer pays O(N) transport + rendering cost + // on every index open. See /wiki/index handler + WikiBrowser.vue for + // the consumer. + GetIndexView(ctx context.Context, kbID string, pageTypes []string, limit int, cursor string) (*types.WikiIndexResponse, error) + // GetLog returns the log page for a knowledge base. - // Creates a default one if it doesn't exist. + // + // Wiki operation events now live in the dedicated wiki_log_entries + // table, so this method no longer auto-creates a placeholder row on + // miss and may legitimately return (nil, nil) for KBs that never had + // the legacy row written. Retained for back-compat with callers that + // still probe the row (lint, knowledge delete); new code should use + // WikiLogEntryService.List for the event feed instead. GetLog(ctx context.Context, kbID string) (*types.WikiPage, error) // GetGraph returns the link graph data for visualization. The caller @@ -81,6 +104,12 @@ type WikiPageService interface { // Used for index rebuild, graph generation, cross-link injection, etc. ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error) + // ListByType returns every wiki page of the given type. Used by + // callers that need the full set for a specific type (e.g. intro + // regeneration needs summary pages). For directory rendering, use + // GetIndexView instead — it paginates via ListByTypeLight. + ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) + // ListPagesBySourceRef retrieves all wiki pages whose source_refs reference // the given knowledge ID. Used by delete/ingest reconciliation paths that // need to find pages touched by a specific document at read time (rather @@ -136,6 +165,12 @@ type WikiPageRepository interface { // ListByType retrieves all wiki pages of a given type within a knowledge base. ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) + // ListByTypeLight returns a paginated window of lightweight entries + // (slug/title/summary only) for the given page_type plus the total + // non-archived count. Used by the structured index API so reads do not + // have to materialize TEXT content for every wiki_pages row. + ListByTypeLight(ctx context.Context, kbID string, pageType string, limit int, offset int) ([]types.WikiIndexEntry, int64, error) + // ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID. ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) diff --git a/internal/types/wiki_log_entry.go b/internal/types/wiki_log_entry.go new file mode 100644 index 00000000..d45f060d --- /dev/null +++ b/internal/types/wiki_log_entry.go @@ -0,0 +1,117 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "fmt" + "time" +) + +// WikiLogPageRef identifies a wiki page that was affected by a log event, +// paired with the human-readable title at the time of the event. Title is +// captured at write time so the log feed stays readable even if the page +// is later renamed or deleted — the previously-stored title is the one +// the user remembers seeing in the event stream. +type WikiLogPageRef struct { + Slug string `json:"slug"` + Title string `json:"title,omitempty"` +} + +// WikiLogPageRefs is a JSON-marshalled list of page references. The +// column is TEXT/JSONB depending on driver; the Value/Scan pair keeps +// GORM happy on both Postgres and the SQLite tests. +type WikiLogPageRefs []WikiLogPageRef + +// Value implements the driver.Valuer interface. +func (r WikiLogPageRefs) Value() (driver.Value, error) { + if r == nil { + return []byte("[]"), nil + } + return json.Marshal(r) +} + +// Scan implements the sql.Scanner interface. Also tolerates the legacy +// []string shape so we can roll forward without a data migration — any +// rows the ingest pipeline wrote before this commit surface as refs +// with a slug and an empty title, which the frontend renders identically. +func (r *WikiLogPageRefs) Scan(value interface{}) error { + if value == nil { + *r = nil + return nil + } + var raw []byte + switch v := value.(type) { + case []byte: + raw = v + case string: + raw = []byte(v) + default: + return fmt.Errorf("unsupported type for WikiLogPageRefs: %T", value) + } + + // Try the new [{slug,title}] shape first. + var refs []WikiLogPageRef + if err := json.Unmarshal(raw, &refs); err == nil { + *r = refs + return nil + } + // Fall back to the legacy []string shape. + var slugs []string + if err := json.Unmarshal(raw, &slugs); err != nil { + return err + } + out := make([]WikiLogPageRef, 0, len(slugs)) + for _, s := range slugs { + out = append(out, WikiLogPageRef{Slug: s}) + } + *r = out + return nil +} + +// WikiLogEntry is a single ingest/retract/edit event appended to the +// per-KB operation log. It replaces the legacy "single giant TEXT column +// on slug='log' wiki_pages row" model, which rewrote the whole column on +// every event and caused O(n^2) write amplification as the KB grew. +// +// Each event is one INSERT; reads paginate by (knowledge_base_id, id DESC). +type WikiLogEntry struct { + // Auto-increment identifier. Monotonic within a single database, so + // frontend pagination uses it as a stable cursor without needing to + // disambiguate identical created_at values. + ID uint64 `json:"id" gorm:"primaryKey;autoIncrement"` + // Tenant scope, mirrored from the enclosing knowledge base. + TenantID uint64 `json:"tenant_id" gorm:"index"` + // Knowledge base this event belongs to. + KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"` + // Short operation tag: "ingest", "retract", etc. Matches the `action` + // argument historically passed to appendLogEntry. + Action string `json:"action" gorm:"type:varchar(32)"` + // Knowledge ID the event was about (may be empty for KB-level events). + KnowledgeID string `json:"knowledge_id" gorm:"type:varchar(36);default:''"` + // Document title at the time of the event. Stored verbatim rather than + // joined at read time so deleted knowledge still has a human-readable + // label in the log. + DocTitle string `json:"doc_title" gorm:"type:text"` + // One-line summary of the change, as it was when the event was logged. + Summary string `json:"summary" gorm:"type:text"` + // Wiki pages affected by this event. Each ref carries both slug (for + // navigation) and title (for display) so the log renders human- + // readable text without a post-hoc slug→title lookup that might fail + // for now-deleted pages. + PagesAffected WikiLogPageRefs `json:"pages_affected" gorm:"type:jsonb;default:'[]'"` + // Server-side timestamp (UTC). + CreatedAt time.Time `json:"created_at"` +} + +// TableName specifies the database table name. +func (WikiLogEntry) TableName() string { + return "wiki_log_entries" +} + +// WikiLogEntryListResponse is the paginated response for GET /wiki/log. +// NextCursor is the stringified ID of the oldest entry in `Entries`; pass +// it back as `cursor` to fetch the next page. Empty string means "no more". +type WikiLogEntryListResponse struct { + Entries []*WikiLogEntry `json:"entries"` + NextCursor string `json:"next_cursor,omitempty"` +} diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index 2f4de58e..cc24fe8d 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -292,3 +292,35 @@ type WikiPageIssue struct { func (WikiPageIssue) TableName() string { return "wiki_page_issues" } + +// WikiIndexEntry is a single row in the structured wiki index response. +// Only the columns needed to render a clickable directory entry are +// carried — the backend projects SELECT slug, title, summary so a 40k- +// page KB does not pay for TEXT content transport on every index open. +type WikiIndexEntry struct { + Slug string `json:"slug"` + Title string `json:"title"` + Summary string `json:"summary"` +} + +// WikiIndexGroup bundles the entries for one page_type into a page-sized +// slice. `Total` is the full count across the KB for the type; `Items` +// holds the current paginated window starting at `NextOffset - len(Items)`. +// An empty NextCursor means the window is already at the end of the type. +type WikiIndexGroup struct { + Type string `json:"type"` + Total int64 `json:"total"` + Items []WikiIndexEntry `json:"items"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// WikiIndexResponse is what GET /wiki/index returns. The heavy directory +// markdown that used to sit in wiki_pages.content is gone — only the LLM- +// generated intro survives there. Everything else is assembled on demand +// from the index repo's light-column projection, keeping index reads +// O(page_size) regardless of KB size. +type WikiIndexResponse struct { + Intro string `json:"intro"` + Version int `json:"version"` + Groups []WikiIndexGroup `json:"groups"` +} diff --git a/migrations/versioned/000040_wiki_log_entries.down.sql b/migrations/versioned/000040_wiki_log_entries.down.sql new file mode 100644 index 00000000..e32cd497 --- /dev/null +++ b/migrations/versioned/000040_wiki_log_entries.down.sql @@ -0,0 +1,9 @@ +-- Migration: 000040_wiki_log_entries (rollback) +-- Description: Drop the wiki_log_entries table. No data migration was +-- performed on the way up, so rollback is a plain DROP. + +DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] Reverting wiki_log_entries schema'; END $$; + +DROP TABLE IF EXISTS wiki_log_entries; + +DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] wiki_log_entries schema reverted successfully'; END $$; diff --git a/migrations/versioned/000040_wiki_log_entries.up.sql b/migrations/versioned/000040_wiki_log_entries.up.sql new file mode 100644 index 00000000..d4cd709b --- /dev/null +++ b/migrations/versioned/000040_wiki_log_entries.up.sql @@ -0,0 +1,32 @@ +-- Migration: 000040_wiki_log_entries +-- Description: Dedicated append-only event table for wiki operation log entries. +-- Replaces the "single giant TEXT column on slug='log' wiki_pages +-- row" model that caused O(n^2) write amplification as KBs grew: +-- every ingest/retract op previously did GetLog + UpdatePage +-- rewriting the entire (potentially multi-MB) TEXT column. +-- Here each event is one INSERT; reads paginate by (kb_id, id DESC). + +DO $$ BEGIN RAISE NOTICE '[Migration 000040] Applying wiki_log_entries schema'; END $$; + +CREATE TABLE IF NOT EXISTS wiki_log_entries ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + action VARCHAR(32) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL DEFAULT '', + doc_title TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + pages_affected JSONB NOT NULL DEFAULT '[]'::JSONB, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Primary list query: cursor-paginated feed per KB, newest first. +-- Using id (BIGSERIAL, monotonic) as the cursor sidesteps duplicate-timestamp +-- tie-breaking that a created_at cursor would require. +CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_kb_id_desc + ON wiki_log_entries (knowledge_base_id, id DESC); + +CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_tenant_id + ON wiki_log_entries (tenant_id); + +DO $$ BEGIN RAISE NOTICE '[Migration 000040] wiki_log_entries schema applied successfully'; END $$;