Files
Enterprise-Knowledge-Base/internal/application/repository/wiki_page.go
T
wizardchen d0144f3586 perf(wiki): move ingest log to event table and index to on-demand API
Fixes O(n²) write amplification during wiki ingest on large KBs. Previously
every ingest/retract op re-wrote the single `slug='log'` row end-to-end and
every batch re-wrote the entire `slug='index'` directory markdown. On a 40k-
doc KB the log row grew to tens of MB and the index row to several MB, so
each batch triggered giant TOAST updates that dominated ingest wall time.

Log: new `wiki_log_entries` event table (`id DESC` indexed per KB) replaces
the single TEXT row. Batch ingest now collects entries and flushes them
once per batch via `AppendBatch`. Each entry stores `pages_affected` as
JSONB `[{slug,title}]` so the UI can render real titles; custom Scan falls
back to legacy `[]string` so older rows still deserialize.

Index: `wiki_pages[slug=index].content` keeps only the LLM-generated intro
(a few KB). The directory is now served by a structured paginated API
(`GetIndexView`) that reads `slug/title/summary` per type with cursor
pagination, so the agent and the frontend only pull the slice they need.
`RebuildIndexPage` degrades to a no-op; agent `wiki_read_page('index')`
synthesizes a small top-K overview and points callers at `wiki_search`.

Ingest resilience: LLM calls wrap with 3-attempt exponential backoff on
transient errors (5xx/408/429, transport resets/timeouts). Summary/extract
failures now bubble up so the batch's failed-op requeue path runs instead
of silently dropping the doc.

Frontend: sidebar Index/Log entries switch to dedicated views. Index view
streams intro → Summary → Entity → Concept → Synthesis → Comparison via
IntersectionObserver (with a nextTick re-check so small KBs still load
every section). Log view uses cursor pagination; pages_affected renders
titles with slug tooltip.

- new migration: migrations/versioned/000040_wiki_log_entries.{up,down}.sql
- tests: log repo pagination + legacy Scan, ListByTypeLight windowing,
  renderIndexOverviewForAgent output, isTransientLLMError classifier

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 21:48:52 +08:00

492 lines
16 KiB
Go

package repository
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
)
// ErrWikiPageNotFound is returned when a wiki page is not found
var ErrWikiPageNotFound = errors.New("wiki page not found")
// ErrWikiPageConflict is returned when an optimistic lock conflict is detected
var ErrWikiPageConflict = errors.New("wiki page version conflict")
// wikiPageRepository implements the WikiPageRepository interface
type wikiPageRepository struct {
db *gorm.DB
}
// NewWikiPageRepository creates a new wiki page repository
func NewWikiPageRepository(db *gorm.DB) interfaces.WikiPageRepository {
return &wikiPageRepository{db: db}
}
// Create inserts a new wiki page record
func (r *wikiPageRepository) Create(ctx context.Context, page *types.WikiPage) error {
return r.db.WithContext(ctx).Create(page).Error
}
// Update updates an existing wiki page record with optimistic locking.
// Increments version — use only for content changes visible to the user.
// The caller must set page.Version to the expected current version.
func (r *wikiPageRepository) Update(ctx context.Context, page *types.WikiPage) error {
expectedVersion := page.Version
page.Version = expectedVersion + 1
result := r.db.WithContext(ctx).
Model(page).
Where("id = ? AND version = ?", page.ID, expectedVersion).
Updates(page)
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
// Could be not found or version conflict — check which
var count int64
r.db.WithContext(ctx).Model(&types.WikiPage{}).Where("id = ?", page.ID).Count(&count)
if count == 0 {
return ErrWikiPageNotFound
}
return ErrWikiPageConflict
}
return nil
}
// UpdateAutoLinkedContent persists content changes produced by the automatic
// link decorators (cross-link injection, dead-link cleanup) without bumping
// `version`. These passes rewrite the same revision with wiki-link markup
// added or removed; treating them as real edits would make newly-ingested
// pages appear as v2 on first view and confuse users who expect `version` to
// correspond to the number of intentional revisions.
func (r *wikiPageRepository) UpdateAutoLinkedContent(ctx context.Context, page *types.WikiPage) error {
result := r.db.WithContext(ctx).
Model(page).
Where("id = ?", page.ID).
Updates(map[string]interface{}{
"content": page.Content,
"out_links": page.OutLinks,
"updated_at": page.UpdatedAt,
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return ErrWikiPageNotFound
}
return nil
}
// UpdateMeta updates bookkeeping / provenance fields WITHOUT incrementing the
// version number. "Content" for versioning purposes is the user-visible page
// body (title/content/summary/page_type/status); everything else — links,
// source refs, chunk refs, page_metadata — is considered bookkeeping and is
// refreshed here so the version counter only advances on real edits.
//
// Used by link maintenance, re-ingest (same-content case), and status changes.
func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPage) error {
result := r.db.WithContext(ctx).
Model(page).
Where("id = ?", page.ID).
Updates(map[string]interface{}{
"in_links": page.InLinks,
"out_links": page.OutLinks,
"status": page.Status,
"source_refs": page.SourceRefs,
"chunk_refs": page.ChunkRefs,
"page_metadata": page.PageMetadata,
"updated_at": page.UpdatedAt,
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return ErrWikiPageNotFound
}
return nil
}
// GetByID retrieves a wiki page by its unique ID
func (r *wikiPageRepository) GetByID(ctx context.Context, id string) (*types.WikiPage, error) {
var page types.WikiPage
if err := r.db.WithContext(ctx).Where("id = ?", id).First(&page).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, ErrWikiPageNotFound
}
return nil, err
}
return &page, nil
}
// GetBySlug retrieves a wiki page by slug within a knowledge base
func (r *wikiPageRepository) GetBySlug(ctx context.Context, kbID string, slug string) (*types.WikiPage, error) {
var page types.WikiPage
if err := r.db.WithContext(ctx).
Where("knowledge_base_id = ? AND slug = ?", kbID, slug).
First(&page).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, ErrWikiPageNotFound
}
return nil, err
}
return &page, nil
}
// List retrieves wiki pages with filtering and pagination
func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRequest) ([]*types.WikiPage, int64, error) {
query := r.db.WithContext(ctx).Model(&types.WikiPage{}).
Where("knowledge_base_id = ?", req.KnowledgeBaseID)
if req.PageType != "" {
query = query.Where("page_type = ?", req.PageType)
}
if req.Status != "" {
query = query.Where("status = ?", req.Status)
}
if req.Query != "" {
// Use PostgreSQL full-text search + ILIKE for aliases
query = query.Where(
"(to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')) @@ plainto_tsquery('simple', ?) OR aliases::text ILIKE ?)",
req.Query,
"%"+req.Query+"%",
)
}
var total int64
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
// Sort
sortBy := "updated_at"
if req.SortBy != "" {
switch req.SortBy {
case "title", "created_at", "updated_at", "page_type":
sortBy = req.SortBy
}
}
sortOrder := "DESC"
if req.SortOrder == "asc" {
sortOrder = "ASC"
}
query = query.Order(fmt.Sprintf("%s %s", sortBy, sortOrder))
// Pagination
page := req.Page
if page < 1 {
page = 1
}
pageSize := req.PageSize
if pageSize < 1 {
pageSize = 20
}
offset := (page - 1) * pageSize
query = query.Offset(offset).Limit(pageSize)
var pages []*types.WikiPage
if err := query.Find(&pages).Error; err != nil {
return nil, 0, err
}
return pages, total, nil
}
// ListByType retrieves all wiki pages of a given type within a knowledge base
func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageType string) ([]*types.WikiPage, error) {
var pages []*types.WikiPage
if err := r.db.WithContext(ctx).
Where("knowledge_base_id = ? AND page_type = ?", kbID, pageType).
Order("updated_at DESC").
Find(&pages).Error; err != nil {
return nil, err
}
return pages, nil
}
// ListByTypeLight projects only the columns needed to render an index
// directory entry (slug, title, summary) and paginates by title ASC.
// This keeps the GET /wiki/index response cheap on KBs with tens of
// thousands of pages — the old path loaded every row including its TEXT
// content just to throw the content away on the way out.
//
// Archived pages are excluded. `limit` clamps to [1, 200]; `offset` is
// honored as-is. Returns the total non-archived count for the type
// alongside the page so the caller can render "showing N of M".
func (r *wikiPageRepository) ListByTypeLight(
ctx context.Context,
kbID string,
pageType string,
limit int,
offset int,
) ([]types.WikiIndexEntry, int64, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
if offset < 0 {
offset = 0
}
base := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Where("knowledge_base_id = ? AND page_type = ? AND status <> ?",
kbID, pageType, types.WikiPageStatusArchived)
var total int64
if err := base.Count(&total).Error; err != nil {
return nil, 0, err
}
if total == 0 {
return nil, 0, nil
}
var entries []types.WikiIndexEntry
if err := base.
Select("slug", "title", "summary").
Order("title ASC").
Limit(limit).
Offset(offset).
Scan(&entries).Error; err != nil {
return nil, 0, err
}
return entries, total, nil
}
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array.
func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) {
// Build the JSON needle safely so arbitrary IDs cannot break out of the
// quoted string (e.g. ids containing quotes or backslashes).
needle, err := json.Marshal([]string{sourceKnowledgeID})
if err != nil {
return nil, fmt.Errorf("marshal source ref needle: %w", err)
}
// For the "knowledgeID|title" prefix form, match against the JSON-encoded
// value: json.Marshal escapes special chars so the LIKE pattern is safe.
prefix, err := json.Marshal(sourceKnowledgeID + "|")
if err != nil {
return nil, fmt.Errorf("marshal source ref prefix: %w", err)
}
// prefix is a JSON string including the surrounding quotes; e.g. "abc|".
// We strip the trailing quote so LIKE can continue into the title portion.
prefixStr := string(prefix)
if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' {
prefixStr = prefixStr[:len(prefixStr)-1]
}
// Escape LIKE metacharacters in the already-JSON-escaped prefix, then wrap
// with %…% to match anywhere in the serialized JSON array.
likePattern := "%" + escapeLikePattern(prefixStr) + "%"
var pages []*types.WikiPage
if err := r.db.WithContext(ctx).
Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)",
kbID,
string(needle),
likePattern,
).
Find(&pages).Error; err != nil {
return nil, err
}
return pages, nil
}
// ListAll retrieves all wiki pages in a knowledge base
func (r *wikiPageRepository) ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error) {
var pages []*types.WikiPage
if err := r.db.WithContext(ctx).
Where("knowledge_base_id = ?", kbID).
Order("page_type ASC, title ASC").
Find(&pages).Error; err != nil {
return nil, err
}
return pages, nil
}
// ListRecentForSuggestions returns recent user-visible wiki pages across the given
// knowledge bases, used as a fallback source for agent suggested questions when
// the KB has no FAQ entries or AI-generated document questions (typical for
// Wiki-only KBs). Excludes index/log pages and archived pages.
func (r *wikiPageRepository) ListRecentForSuggestions(
ctx context.Context,
tenantID uint64,
kbIDs []string,
limit int,
) ([]*types.WikiPage, error) {
if len(kbIDs) == 0 || limit <= 0 {
return nil, nil
}
var pages []*types.WikiPage
if err := r.db.WithContext(ctx).
Where("tenant_id = ?", tenantID).
Where("knowledge_base_id IN ?", kbIDs).
Where("page_type NOT IN ?", []string{types.WikiPageTypeIndex, types.WikiPageTypeLog}).
Where("status = ?", types.WikiPageStatusPublished).
Where("title <> ''").
Order("updated_at DESC").
Limit(limit).
Find(&pages).Error; err != nil {
return nil, err
}
return pages, nil
}
// Delete soft-deletes a wiki page by knowledge base ID and slug
func (r *wikiPageRepository) Delete(ctx context.Context, kbID string, slug string) error {
result := r.db.WithContext(ctx).
Where("knowledge_base_id = ? AND slug = ?", kbID, slug).
Delete(&types.WikiPage{})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return ErrWikiPageNotFound
}
return nil
}
// DeleteByID soft-deletes a wiki page by ID
func (r *wikiPageRepository) DeleteByID(ctx context.Context, id string) error {
result := r.db.WithContext(ctx).
Where("id = ?", id).
Delete(&types.WikiPage{})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return ErrWikiPageNotFound
}
return nil
}
// escapeLikePattern escapes LIKE / ILIKE metacharacters so the returned string
// can be safely concatenated with % wildcards without unintended matches.
// Order matters: escape the backslash first, then the wildcards.
func escapeLikePattern(s string) string {
replacer := strings.NewReplacer(
`\`, `\\`,
`%`, `\%`,
`_`, `\_`,
)
return replacer.Replace(s)
}
// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base.
// The query is interpreted as a PostgreSQL regular expression (via ~*).
//
// Results are ranked by where the query hit, highest-relevance first:
//
// title hit → rank 4 (most obvious intent: user typed what the page is called)
// slug hit → rank 3 (url-like identifiers, direct jump)
// summary hit → rank 2 (short authored abstract)
// content hit → rank 1 (body mention — often surfaces unrelated pages whose
// prose merely mentions the query as trivia)
//
// Without this ranking, a user searching for "王新" on a 4万-page wiki will
// see pages like "华为" or "Index" ahead of the actual 王新 page just
// because they mention 王新 in their body and were updated more recently.
// updated_at stays as the tiebreaker so same-rank ties stay deterministic.
func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) {
if limit <= 0 {
limit = 10
}
if limit > 50 {
limit = 50
}
// CASE expression is evaluated per-row during SELECT; we order by the
// alias so the DB only computes the rank once. Parameterized four
// times with the same regex to avoid coupling to GORM's positional
// arg rewriting quirks.
rankExpr := "CASE " +
"WHEN title ~* ? THEN 4 " +
"WHEN slug ~* ? THEN 3 " +
"WHEN summary ~* ? THEN 2 " +
"WHEN content ~* ? THEN 1 " +
"ELSE 0 END AS match_rank"
var pages []*types.WikiPage
if err := r.db.WithContext(ctx).
Select("*, "+rankExpr, query, query, query, query).
Where("knowledge_base_id = ? AND (title ~* ? OR content ~* ? OR summary ~* ? OR slug ~* ?)",
kbID, query, query, query, query).
Where("status != ?", "archived").
Order("match_rank DESC, updated_at DESC").
Limit(limit).
Find(&pages).Error; err != nil {
return nil, err
}
return pages, nil
}
// CountByType returns page counts grouped by type for a knowledge base
func (r *wikiPageRepository) CountByType(ctx context.Context, kbID string) (map[string]int64, error) {
type result struct {
PageType string
Count int64
}
var results []result
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Select("page_type, count(*) as count").
Where("knowledge_base_id = ?", kbID).
Group("page_type").
Scan(&results).Error; err != nil {
return nil, err
}
counts := make(map[string]int64)
for _, r := range results {
counts[r.PageType] = r.Count
}
return counts, nil
}
// CountOrphans returns the number of pages with no inbound links
func (r *wikiPageRepository) CountOrphans(ctx context.Context, kbID string) (int64, error) {
var count int64
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Where("knowledge_base_id = ?", kbID).
Where("(in_links IS NULL OR in_links = '[]'::JSONB)").
// Exclude index and log pages as they are naturally root pages
Where("page_type NOT IN ?", []string{types.WikiPageTypeIndex, types.WikiPageTypeLog}).
Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
func (r *wikiPageRepository) CreateIssue(ctx context.Context, issue *types.WikiPageIssue) error {
return r.db.WithContext(ctx).Create(issue).Error
}
func (r *wikiPageRepository) ListIssues(ctx context.Context, kbID string, slug string, status string) ([]*types.WikiPageIssue, error) {
query := r.db.WithContext(ctx).Where("knowledge_base_id = ?", kbID)
if slug != "" {
query = query.Where("slug = ?", slug)
}
if status != "" {
query = query.Where("status = ?", status)
}
var issues []*types.WikiPageIssue
if err := query.Order("created_at DESC").Find(&issues).Error; err != nil {
return nil, err
}
return issues, nil
}
func (r *wikiPageRepository) UpdateIssueStatus(ctx context.Context, issueID string, status string) error {
return r.db.WithContext(ctx).Model(&types.WikiPageIssue{}).
Where("id = ?", issueID).
Update("status", status).Error
}