From 88c10da0b03b86c3cdd96d3a20bcc6047002f569 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 16 Jun 2026 17:38:31 +0800 Subject: [PATCH] refactor(wiki): unify category cleaning and push directory listing to SQL Centralize the wiki category-path normalization (separator/quote cleanup, type-label stripping, dedup, depth cap) into a single types helper shared by the service, repository, and taxonomy layers so stored and queried paths are always cleaned identically and cannot drift. Flatten category_path into fixed category_l1/l2/l3 columns (kept in sync on write) and rewrite ListDistinctCategoryPathsByType to group, count, and paginate directly in cross-dialect SQL, returning the page of child folders plus the total distinct count. This removes the prior full-column scan and in-memory pagination (whose maxPaths cap was effectively dead) and makes Total and later pages correct. Add the level columns + composite index to the wiki migrations and the repo test schema, and cover the SQL grouping/counting/pagination with a new test. --- internal/application/repository/wiki_page.go | 192 ++++++------------ .../application/repository/wiki_page_test.go | 59 ++++++ .../service/wiki_ingest_taxonomy.go | 11 +- internal/application/service/wiki_page.go | 106 +++------- internal/handler/wiki_page.go | 16 +- internal/types/interfaces/wiki_page.go | 17 +- internal/types/wiki_page.go | 97 +++++++++ .../versioned/000037_wiki_and_indexing.up.sql | 6 + .../000061_wiki_page_hierarchy.down.sql | 4 + .../000061_wiki_page_hierarchy.up.sql | 16 ++ 10 files changed, 289 insertions(+), 235 deletions(-) diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index ec536518..44e22cfe 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -110,6 +110,9 @@ func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPag "page_metadata": page.PageMetadata, "parent_slug": page.ParentSlug, "category_path": page.CategoryPath, + "category_l1": page.CategoryL1, + "category_l2": page.CategoryL2, + "category_l3": page.CategoryL3, "wiki_path": page.WikiPath, "depth": page.Depth, "sort_order": page.SortOrder, @@ -203,7 +206,7 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe if req.CategoryDepth != nil { query = query.Where("depth = ?", *req.CategoryDepth) } - if wantPath := cleanWikiRepositoryCategoryPath(req.CategoryPath); len(wantPath) > 0 { + if wantPath := types.CleanWikiCategoryPath(req.CategoryPath); len(wantPath) > 0 { if encoded, err := json.Marshal([]string(wantPath)); err == nil { if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" { query = query.Where("category_path::jsonb = ?::jsonb", string(encoded)) @@ -470,7 +473,7 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths( if len(row.CategoryPath) == 0 { continue } - clean := cleanWikiRepositoryCategoryPath(row.CategoryPath) + clean := types.CleanWikiCategoryPath(row.CategoryPath) if len(clean) == 0 { continue } @@ -487,144 +490,83 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths( return out, nil } -// ListDistinctCategoryPathsByType scans one page type and returns unique -// non-empty category_path values. Unlike ListDistinctCategoryPaths, this is -// used by the UI and must work for every visible page_type bucket. +// wikiCategoryLevelColumns maps a parent depth (0/1/2) to the flat level column +// holding that depth's direct child. Depth >= WikiCategoryMaxDepth has no child. +var wikiCategoryLevelColumns = []string{"category_l1", "category_l2", "category_l3"} + +// ListDistinctCategoryPathsByType returns the DIRECT child folders of +// parentPath for the given page types, with the page count under each child, +// paginated. Grouping and pagination are pushed entirely to SQL over the flat +// category_l1/l2/l3 columns (kept in sync by normalizeWikiHierarchy), so this +// is cheap, indexable, and identical across Postgres and SQLite — no JSON +// parsing and no full in-memory scan. The second return value is the total +// number of distinct child folders (before pagination). func (r *wikiPageRepository) ListDistinctCategoryPathsByType( ctx context.Context, kbID string, pageTypes []string, parentPath []string, - maxPaths int, -) ([]types.WikiCategoryPath, error) { - if maxPaths <= 0 { - maxPaths = 2000 - } + page, pageSize int, +) ([]types.WikiCategoryPath, int, error) { if len(pageTypes) == 0 { - return nil, nil + return nil, 0, nil + } + parent := types.CleanWikiCategoryPath(parentPath) + if len(parent) >= len(wikiCategoryLevelColumns) { + return nil, 0, nil // no level deeper than the cap can exist + } + childCol := wikiCategoryLevelColumns[len(parent)] + + // Build the shared WHERE as a closure so the count and the page query each + // start from a fresh statement (GORM chains mutate in place otherwise). + scoped := func() *gorm.DB { + q := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?", + kbID, pageTypes, types.WikiPageStatusArchived). + Where(childCol+" <> ?", "") + for i, label := range parent { + q = q.Where(wikiCategoryLevelColumns[i]+" = ?", label) + } + return q + } + + var total int64 + if err := scoped().Distinct(childCol).Count(&total).Error; err != nil { + return nil, 0, err + } + if total == 0 { + return nil, 0, nil + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 2000 } type row struct { - CategoryPath types.StringArray `gorm:"column:category_path"` + Label string `gorm:"column:label"` + Cnt int64 `gorm:"column:cnt"` } var rows []row - if err := r.db.WithContext(ctx). - Model(&types.WikiPage{}). - Select("category_path"). - Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?", - kbID, - pageTypes, - types.WikiPageStatusArchived, - ). - Order(r.wikiCategoryRankOrder()). - Order("wiki_path ASC"). + if err := scoped(). + Select(childCol + " AS label, COUNT(*) AS cnt"). + Group(childCol). + Order("label ASC"). + Offset((page - 1) * pageSize). + Limit(pageSize). Scan(&rows).Error; err != nil { - return nil, err + return nil, 0, err } - parent := cleanWikiRepositoryCategoryPath(parentPath) - - counts := make(map[string]int64, len(rows)) - pathsByKey := make(map[string][]string, len(rows)) - order := make([]string, 0, len(rows)) - for _, row := range rows { - if len(row.CategoryPath) == 0 { - continue - } - clean := cleanWikiRepositoryCategoryPath(row.CategoryPath) - if len(clean) == 0 { - continue - } - if len(clean) <= len(parent) || !wikiPathHasPrefix(clean, parent) { - continue - } - path := clean[:len(parent)+1] - key := strings.Join(path, "\x00") - if _, ok := pathsByKey[key]; !ok { - cp := append([]string(nil), path...) - pathsByKey[key] = cp - order = append(order, key) - } - counts[key]++ + out := make([]types.WikiCategoryPath, 0, len(rows)) + for _, rw := range rows { + full := append(append([]string(nil), parent...), rw.Label) + out = append(out, types.WikiCategoryPath{Path: full, Count: rw.Cnt}) } - - out := make([]types.WikiCategoryPath, 0, len(order)) - for _, key := range order { - out = append(out, types.WikiCategoryPath{ - Path: pathsByKey[key], - Count: counts[key], - }) - } - return out, nil -} - -func cleanWikiRepositoryCategoryPath(parts []string) []string { - cleaned := make([]string, 0, len(parts)) - for _, part := range parts { - for _, label := range cleanWikiRepositoryCategoryPart(part) { - if containsWikiRepositoryString(cleaned, label) { - continue - } - cleaned = append(cleaned, label) - } - } - return cleaned -} - -func cleanWikiRepositoryCategoryPart(part string) []string { - part = strings.TrimSpace(part) - if part == "" { - return nil - } - - part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part) - rawParts := strings.Split(part, "/") - cleaned := make([]string, 0, len(rawParts)) - for _, raw := range rawParts { - label := strings.TrimSpace(raw) - label = strings.Trim(label, `"'“”‘’[]()()`) - label = strings.TrimSpace(label) - if label == "" || isWikiRepositoryTypeCategoryLabel(label) { - continue - } - cleaned = append(cleaned, label) - } - return cleaned -} - -func isWikiRepositoryTypeCategoryLabel(label string) bool { - normalized := strings.ToLower(strings.TrimSpace(label)) - normalized = strings.TrimSuffix(normalized, "s") - switch normalized { - case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面": - return true - default: - return false - } -} - -func containsWikiRepositoryString(slice []string, s string) bool { - for _, v := range slice { - if v == s { - return true - } - } - return false -} - -func wikiPathHasPrefix(path []string, prefix []string) bool { - if len(prefix) == 0 { - return true - } - if len(path) < len(prefix) { - return false - } - for i, part := range prefix { - if path[i] != part { - return false - } - } - return true + return out, int(total), nil } // ListSummariesByKnowledgeIDs returns summary-page content keyed by the diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go index 1efc7615..bde52fb8 100644 --- a/internal/application/repository/wiki_page_test.go +++ b/internal/application/repository/wiki_page_test.go @@ -34,6 +34,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages ( wiki_path VARCHAR(1024) NOT NULL DEFAULT '', depth INTEGER NOT NULL DEFAULT 0, sort_order INTEGER NOT NULL DEFAULT 0, + category_l1 VARCHAR(255) NOT NULL DEFAULT '', + category_l2 VARCHAR(255) NOT NULL DEFAULT '', + category_l3 VARCHAR(255) NOT NULL DEFAULT '', source_refs TEXT DEFAULT '[]', chunk_refs TEXT DEFAULT '[]', in_links TEXT DEFAULT '[]', @@ -86,6 +89,9 @@ func makeCategorizedWikiPage(kbID, slug, pageType, status string, categoryPath . if len(categoryPath) > 0 { page.WikiPath = pageType + "/" + strings.Join(categoryPath, "/") + "/" + page.Title page.Depth = len(categoryPath) + // Mirror the flat level columns the directory query groups on, the same + // way normalizeWikiHierarchy would on a real write. + page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(categoryPath) } return page } @@ -124,6 +130,59 @@ func TestList_WikiPathSortReturnsCategorizedPagesFirst(t *testing.T) { assert.Equal(t, "entity/001-root", got[2].Slug) } +// TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL verifies the +// directory listing groups direct child folders, counts pages under each +// (including deeper pages), and paginates — all pushed to SQL over the flat +// level columns, so it must behave identically on SQLite. +func TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + pages := []*types.WikiPage{ + makeCategorizedWikiPage("kb-c", "entity/a1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"), + makeCategorizedWikiPage("kb-c", "entity/a2", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"), + makeCategorizedWikiPage("kb-c", "entity/a3", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI", "LLM"), + makeCategorizedWikiPage("kb-c", "entity/p1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "人物"), + makeWikiPage("kb-c", "entity/root", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + makeCategorizedWikiPage("kb-c", "entity/arch", types.WikiPageTypeEntity, types.WikiPageStatusArchived, "AI"), + } + for _, p := range pages { + require.NoError(t, repo.Create(ctx, p)) + } + + types4 := []string{types.WikiPageTypeEntity} + + // Root level: two folders (AI, 人物); the loose root page and archived page + // are excluded. AI counts all 3 non-archived pages whose first level is AI. + roots, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 100) + require.NoError(t, err) + assert.Equal(t, 2, total) + require.Len(t, roots, 2) + byLabel := map[string]int64{} + for _, p := range roots { + require.Len(t, p.Path, 1) + byLabel[p.Path[0]] = p.Count + } + assert.Equal(t, int64(3), byLabel["AI"]) + assert.Equal(t, int64(1), byLabel["人物"]) + + // Child of [AI]: only the [AI, LLM] page. + children, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, []string{"AI"}, 1, 100) + require.NoError(t, err) + assert.Equal(t, 1, total) + require.Len(t, children, 1) + assert.Equal(t, []string{"AI", "LLM"}, children[0].Path) + assert.Equal(t, int64(1), children[0].Count) + + // Pagination: page size 1 over the two root folders returns one row but the + // total still reflects both distinct folders. + firstPage, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 1) + require.NoError(t, err) + assert.Equal(t, 2, total) + require.Len(t, firstPage, 1) +} + // TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies // that the index-view projection only emits the slug/title/summary // triples and respects the archived-status filter. This is the whole diff --git a/internal/application/service/wiki_ingest_taxonomy.go b/internal/application/service/wiki_ingest_taxonomy.go index b53cf634..eb78369f 100644 --- a/internal/application/service/wiki_ingest_taxonomy.go +++ b/internal/application/service/wiki_ingest_taxonomy.go @@ -241,16 +241,7 @@ func remapCategoryPath(current []string, remap map[string][]string) ([]string, b // hierarchy normalizer does (separator/quote cleanup, type-label stripping, // depth cap) so remap keys match what is stored on pages. func cleanCategoryPathParts(parts []string) []string { - out := make([]string, 0, len(parts)) - for _, part := range parts { - for _, c := range cleanWikiCategoryPart(part) { - out = append(out, c) - if len(out) >= 3 { - return out - } - } - } - return out + return types.CleanWikiCategoryPath(parts) } // categoryPathKey builds a stable comparison key for a cleaned category path. diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index a6750ae1..b2b57904 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -837,12 +837,13 @@ func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID st return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths) } -// ListDistinctCategoryPathsByType returns unique category_path folders across -// the given page types. The browser uses this to render directory nodes before -// article pages are paged in; passing several types lets one tab (e.g. the -// merged knowledge tab) build a single directory skeleton server-side. -func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) { - return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths) +// ListDistinctCategoryPathsByType returns the direct child folders of +// parentPath across the given page types, paginated, plus the total distinct +// child count. The browser uses this to expand the directory tree level by +// level; passing several types lets one tab (e.g. the merged knowledge tab) +// build a single directory skeleton server-side. +func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error) { + return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, page, pageSize) } // CountByType is a service-layer pass-through over the repo. Used by @@ -965,38 +966,16 @@ func normalizeWikiHierarchy(page *types.WikiPage) { } page.ParentSlug = strings.TrimSpace(page.ParentSlug) - cleanPath := make(types.StringArray, 0, len(page.CategoryPath)) - for _, part := range page.CategoryPath { - for _, cleanPart := range cleanWikiCategoryPart(part) { - if !containsString(cleanPath, cleanPart) { - cleanPath = append(cleanPath, cleanPart) - } - if len(cleanPath) >= 3 { - break - } - } - if len(cleanPath) >= 3 { - break - } - } + cleanPath := types.StringArray(types.CleanWikiCategoryPath(page.CategoryPath)) page.CategoryPath = cleanPath page.Depth = len(cleanPath) + page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(cleanPath) - parts := make([]string, 0, len(cleanPath)+2) - if page.PageType != "" { - parts = append(parts, page.PageType) - } - for _, part := range cleanPath { - parts = append(parts, part) - } display := strings.TrimSpace(page.Title) if display == "" { display = strings.TrimSpace(page.Slug) } - if display != "" { - parts = append(parts, display) - } - page.WikiPath = strings.Join(parts, "/") + page.WikiPath = buildWikiPath(page.PageType, cleanPath, display) } func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) { @@ -1004,68 +983,29 @@ func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType stri return } - cleanPath := make(types.StringArray, 0, len(entry.CategoryPath)) - for _, part := range entry.CategoryPath { - for _, cleanPart := range cleanWikiCategoryPart(part) { - if !containsString(cleanPath, cleanPart) { - cleanPath = append(cleanPath, cleanPart) - } - if len(cleanPath) >= 3 { - break - } - } - if len(cleanPath) >= 3 { - break - } - } + cleanPath := types.StringArray(types.CleanWikiCategoryPath(entry.CategoryPath)) entry.CategoryPath = cleanPath entry.Depth = len(cleanPath) - parts := make([]string, 0, len(cleanPath)+2) - if strings.TrimSpace(pageType) != "" { - parts = append(parts, strings.TrimSpace(pageType)) - } - parts = append(parts, cleanPath...) display := strings.TrimSpace(entry.Title) if display == "" { display = strings.TrimSpace(entry.Slug) } + entry.WikiPath = buildWikiPath(pageType, cleanPath, display) +} + +// buildWikiPath assembles the normalized, sortable "page_type/cat.../title" +// breadcrumb used for directory ordering. Empty segments are skipped. +func buildWikiPath(pageType string, categoryPath []string, display string) string { + parts := make([]string, 0, len(categoryPath)+2) + if pt := strings.TrimSpace(pageType); pt != "" { + parts = append(parts, pt) + } + parts = append(parts, categoryPath...) if display != "" { parts = append(parts, display) } - entry.WikiPath = strings.Join(parts, "/") -} - -func cleanWikiCategoryPart(part string) []string { - part = strings.TrimSpace(part) - if part == "" { - return nil - } - - part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part) - rawParts := strings.Split(part, "/") - cleaned := make([]string, 0, len(rawParts)) - for _, raw := range rawParts { - label := strings.TrimSpace(raw) - label = strings.Trim(label, `"'“”‘’[]()()`) - label = strings.TrimSpace(label) - if label == "" || isWikiTypeCategoryLabel(label) { - continue - } - cleaned = append(cleaned, label) - } - return cleaned -} - -func isWikiTypeCategoryLabel(label string) bool { - normalized := strings.ToLower(strings.TrimSpace(label)) - normalized = strings.TrimSuffix(normalized, "s") - switch normalized { - case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面": - return true - default: - return false - } + return strings.Join(parts, "/") } // containsString checks if a string slice contains a given string diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index 065f0f07..a392e8a0 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -162,19 +162,15 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) { pageSize = 2000 } parentPath := parseWikiCategoryPath(c.Query("parent_path")) - paths, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, pageSize) + // Grouping and pagination are pushed to SQL; the service returns just the + // requested page of child folders plus the total distinct count. + paths, total, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, page, pageSize) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - total := len(paths) - offset := (page - 1) * pageSize - if offset > total { - offset = total - } - end := offset + pageSize - if end > total { - end = total + if paths == nil { + paths = []types.WikiCategoryPath{} } totalPages := 0 if pageSize > 0 { @@ -182,7 +178,7 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) { } c.JSON(http.StatusOK, types.WikiCategoryPathListResponse{ - Paths: paths[offset:end], + Paths: paths, Total: total, Page: page, PageSize: pageSize, diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 47b9af83..d2600b27 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -161,10 +161,12 @@ type WikiPageService interface { // established folder taxonomy into extraction prompts. ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) - // ListDistinctCategoryPathsByType returns unique non-empty category_path - // values across the given page types. Used by the wiki browser to render a - // stable directory skeleton independently of page pagination. - ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) + // ListDistinctCategoryPathsByType returns the direct child folders of + // parentPath for the given page types (with per-child page counts), + // paginated. The second return value is the total number of distinct + // child folders before pagination. Used by the wiki browser to render the + // directory tree level by level. + ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error) // CountByType returns page counts grouped by type for a knowledge // base. Re-exposed at the service layer so the index intro @@ -278,9 +280,10 @@ type WikiPageRepository interface { // unique non-empty category_path values for wiki ingest prompts. ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) - // ListDistinctCategoryPathsByType returns unique non-empty category_path - // values across the given page types. - ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) + // ListDistinctCategoryPathsByType returns the direct child folders of + // parentPath for the given page types (with per-child page counts), + // paginated, plus the total number of distinct child folders. + ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error) // ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation). ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error) diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index a2386ab5..c7f16ee2 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -3,11 +3,101 @@ package types import ( "database/sql/driver" "encoding/json" + "strings" "time" "gorm.io/gorm" ) +// WikiCategoryMaxDepth is the hard cap on how many folder levels a wiki page's +// category_path may keep. The ingest prompts intentionally ask the model for at +// most 2 levels to keep the directory tree shallow; this storage cap is one +// level deeper as a defensive bound so a slightly over-eager model (or a +// reconcile remap) cannot create an unbounded breadcrumb. It is the single +// source of truth shared by the service, repository, and taxonomy layers so +// stored paths and queried paths are always cleaned identically. +const WikiCategoryMaxDepth = 3 + +var wikiCategorySeparatorReplacer = strings.NewReplacer("/", "/", "|", "/", "|", "/") + +// CleanWikiCategoryPart normalizes a single raw category label that may itself +// carry embedded separators, wrapping quotes/brackets, or page-type noise, and +// returns the cleaned sub-labels (type labels such as "entity"/"实体" dropped). +func CleanWikiCategoryPart(part string) []string { + part = strings.TrimSpace(part) + if part == "" { + return nil + } + part = wikiCategorySeparatorReplacer.Replace(part) + rawParts := strings.Split(part, "/") + cleaned := make([]string, 0, len(rawParts)) + for _, raw := range rawParts { + label := strings.TrimSpace(raw) + label = strings.Trim(label, `"'“”‘’[]()()`) + label = strings.TrimSpace(label) + if label == "" || isWikiTypeCategoryLabel(label) { + continue + } + cleaned = append(cleaned, label) + } + return cleaned +} + +// CleanWikiCategoryPath cleans, deduplicates, and caps a full category path at +// WikiCategoryMaxDepth. Centralizing this guarantees that the path a page is +// stored with and the path a list/filter query is matched against go through +// the exact same normalization, so directory filters cannot silently drift. +func CleanWikiCategoryPath(parts []string) []string { + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + for _, label := range CleanWikiCategoryPart(part) { + if containsWikiString(cleaned, label) { + continue + } + cleaned = append(cleaned, label) + if len(cleaned) >= WikiCategoryMaxDepth { + return cleaned + } + } + } + return cleaned +} + +// WikiCategoryLevels flattens a (already cleaned) category path into the three +// fixed level columns used for SQL grouping. Missing levels yield "". +func WikiCategoryLevels(path []string) (l1, l2, l3 string) { + if len(path) > 0 { + l1 = path[0] + } + if len(path) > 1 { + l2 = path[1] + } + if len(path) > 2 { + l3 = path[2] + } + return l1, l2, l3 +} + +func isWikiTypeCategoryLabel(label string) bool { + normalized := strings.ToLower(strings.TrimSpace(label)) + normalized = strings.TrimSuffix(normalized, "s") + switch normalized { + case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面": + return true + default: + return false + } +} + +func containsWikiString(slice []string, s string) bool { + for _, v := range slice { + if v == s { + return true + } + } + return false +} + // WikiPageType constants define the types of wiki pages const ( // WikiPageTypeSummary represents a document summary page @@ -81,6 +171,13 @@ type WikiPage struct { // SortOrder allows generated or manually edited pages to control sibling // ordering before falling back to title. SortOrder int `json:"sort_order,omitempty" gorm:"default:0;index"` + // CategoryL1/L2/L3 mirror CategoryPath[0..2] as flat columns. They exist + // solely so the directory-listing query can GROUP BY / paginate folders in + // plain, cross-dialect SQL (no JSON parsing). They are NOT part of the API + // surface (hence json:"-") and are kept in sync by normalizeWikiHierarchy. + CategoryL1 string `json:"-" gorm:"column:category_l1;type:varchar(255);default:''"` + CategoryL2 string `json:"-" gorm:"column:category_l2;type:varchar(255);default:''"` + CategoryL3 string `json:"-" gorm:"column:category_l3;type:varchar(255);default:''"` // References to source knowledge IDs that contributed to this page. // Format matches the legacy "|" convention used // across the ingest pipeline, so retract / display code can split on `|` diff --git a/migrations/versioned/000037_wiki_and_indexing.up.sql b/migrations/versioned/000037_wiki_and_indexing.up.sql index 36bae6ac..aa28fb4c 100644 --- a/migrations/versioned/000037_wiki_and_indexing.up.sql +++ b/migrations/versioned/000037_wiki_and_indexing.up.sql @@ -27,6 +27,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages ( wiki_path VARCHAR(1024) NOT NULL DEFAULT '', depth INT NOT NULL DEFAULT 0, sort_order INT NOT NULL DEFAULT 0, + category_l1 VARCHAR(255) NOT NULL DEFAULT '', + category_l2 VARCHAR(255) NOT NULL DEFAULT '', + category_l3 VARCHAR(255) NOT NULL DEFAULT '', source_refs JSONB DEFAULT '[]'::JSONB, chunk_refs JSONB DEFAULT '[]'::JSONB, in_links JSONB DEFAULT '[]'::JSONB, @@ -56,6 +59,9 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title); +CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels + ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3); + CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id ON wiki_pages (tenant_id); diff --git a/migrations/versioned/000061_wiki_page_hierarchy.down.sql b/migrations/versioned/000061_wiki_page_hierarchy.down.sql index ee9168aa..eab1b605 100644 --- a/migrations/versioned/000061_wiki_page_hierarchy.down.sql +++ b/migrations/versioned/000061_wiki_page_hierarchy.down.sql @@ -1,8 +1,12 @@ -- Migration: 000061_wiki_page_hierarchy (rollback) +DROP INDEX IF EXISTS idx_wiki_pages_category_levels; DROP INDEX IF EXISTS idx_wiki_pages_tree; DROP INDEX IF EXISTS idx_wiki_pages_parent_slug; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l3; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l2; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l1; ALTER TABLE wiki_pages DROP COLUMN IF EXISTS sort_order; ALTER TABLE wiki_pages DROP COLUMN IF EXISTS depth; ALTER TABLE wiki_pages DROP COLUMN IF EXISTS wiki_path; diff --git a/migrations/versioned/000061_wiki_page_hierarchy.up.sql b/migrations/versioned/000061_wiki_page_hierarchy.up.sql index b10f2979..d22d44ca 100644 --- a/migrations/versioned/000061_wiki_page_hierarchy.up.sql +++ b/migrations/versioned/000061_wiki_page_hierarchy.up.sql @@ -8,6 +8,11 @@ ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_path JSONB DEFAULT '[]' ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS wiki_path VARCHAR(1024) NOT NULL DEFAULT ''; ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS depth INT NOT NULL DEFAULT 0; ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS sort_order INT NOT NULL DEFAULT 0; +-- Flat mirrors of category_path[0..2] for cheap, cross-dialect directory +-- grouping/pagination (kept in sync by the application on write). +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l1 VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l2 VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l3 VARCHAR(255) NOT NULL DEFAULT ''; UPDATE wiki_pages SET @@ -20,10 +25,21 @@ SET END WHERE wiki_path = '' OR wiki_path IS NULL OR category_path IS NULL OR depth IS NULL; +-- Backfill the flat level columns from the JSON category_path. +UPDATE wiki_pages +SET + category_l1 = COALESCE(category_path->>0, ''), + category_l2 = COALESCE(category_path->>1, ''), + category_l3 = COALESCE(category_path->>2, '') +WHERE category_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug ON wiki_pages (knowledge_base_id, parent_slug); CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title); +CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels + ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3); + DO $$ BEGIN RAISE NOTICE '[Migration 000061] wiki page hierarchy schema applied successfully'; END $$;