refactor(wiki): unify category cleaning and push directory listing to SQL

Centralize the wiki category-path normalization (separator/quote cleanup,
type-label stripping, dedup, depth cap) into a single types helper shared by
the service, repository, and taxonomy layers so stored and queried paths are
always cleaned identically and cannot drift.

Flatten category_path into fixed category_l1/l2/l3 columns (kept in sync on
write) and rewrite ListDistinctCategoryPathsByType to group, count, and
paginate directly in cross-dialect SQL, returning the page of child folders
plus the total distinct count. This removes the prior full-column scan and
in-memory pagination (whose maxPaths cap was effectively dead) and makes Total
and later pages correct.

Add the level columns + composite index to the wiki migrations and the repo
test schema, and cover the SQL grouping/counting/pagination with a new test.
This commit is contained in:
wizardchen
2026-06-16 17:38:31 +08:00
committed by lyingbug
parent fe33db9493
commit 88c10da0b0
10 changed files with 289 additions and 235 deletions
+67 -125
View File
@@ -110,6 +110,9 @@ func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPag
"page_metadata": page.PageMetadata,
"parent_slug": page.ParentSlug,
"category_path": page.CategoryPath,
"category_l1": page.CategoryL1,
"category_l2": page.CategoryL2,
"category_l3": page.CategoryL3,
"wiki_path": page.WikiPath,
"depth": page.Depth,
"sort_order": page.SortOrder,
@@ -203,7 +206,7 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
if req.CategoryDepth != nil {
query = query.Where("depth = ?", *req.CategoryDepth)
}
if wantPath := cleanWikiRepositoryCategoryPath(req.CategoryPath); len(wantPath) > 0 {
if wantPath := types.CleanWikiCategoryPath(req.CategoryPath); len(wantPath) > 0 {
if encoded, err := json.Marshal([]string(wantPath)); err == nil {
if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" {
query = query.Where("category_path::jsonb = ?::jsonb", string(encoded))
@@ -470,7 +473,7 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths(
if len(row.CategoryPath) == 0 {
continue
}
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
clean := types.CleanWikiCategoryPath(row.CategoryPath)
if len(clean) == 0 {
continue
}
@@ -487,144 +490,83 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths(
return out, nil
}
// ListDistinctCategoryPathsByType scans one page type and returns unique
// non-empty category_path values. Unlike ListDistinctCategoryPaths, this is
// used by the UI and must work for every visible page_type bucket.
// wikiCategoryLevelColumns maps a parent depth (0/1/2) to the flat level column
// holding that depth's direct child. Depth >= WikiCategoryMaxDepth has no child.
var wikiCategoryLevelColumns = []string{"category_l1", "category_l2", "category_l3"}
// ListDistinctCategoryPathsByType returns the DIRECT child folders of
// parentPath for the given page types, with the page count under each child,
// paginated. Grouping and pagination are pushed entirely to SQL over the flat
// category_l1/l2/l3 columns (kept in sync by normalizeWikiHierarchy), so this
// is cheap, indexable, and identical across Postgres and SQLite — no JSON
// parsing and no full in-memory scan. The second return value is the total
// number of distinct child folders (before pagination).
func (r *wikiPageRepository) ListDistinctCategoryPathsByType(
ctx context.Context,
kbID string,
pageTypes []string,
parentPath []string,
maxPaths int,
) ([]types.WikiCategoryPath, error) {
if maxPaths <= 0 {
maxPaths = 2000
}
page, pageSize int,
) ([]types.WikiCategoryPath, int, error) {
if len(pageTypes) == 0 {
return nil, nil
return nil, 0, nil
}
parent := types.CleanWikiCategoryPath(parentPath)
if len(parent) >= len(wikiCategoryLevelColumns) {
return nil, 0, nil // no level deeper than the cap can exist
}
childCol := wikiCategoryLevelColumns[len(parent)]
// Build the shared WHERE as a closure so the count and the page query each
// start from a fresh statement (GORM chains mutate in place otherwise).
scoped := func() *gorm.DB {
q := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
kbID, pageTypes, types.WikiPageStatusArchived).
Where(childCol+" <> ?", "")
for i, label := range parent {
q = q.Where(wikiCategoryLevelColumns[i]+" = ?", label)
}
return q
}
var total int64
if err := scoped().Distinct(childCol).Count(&total).Error; err != nil {
return nil, 0, err
}
if total == 0 {
return nil, 0, nil
}
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 2000
}
type row struct {
CategoryPath types.StringArray `gorm:"column:category_path"`
Label string `gorm:"column:label"`
Cnt int64 `gorm:"column:cnt"`
}
var rows []row
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Select("category_path").
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
kbID,
pageTypes,
types.WikiPageStatusArchived,
).
Order(r.wikiCategoryRankOrder()).
Order("wiki_path ASC").
if err := scoped().
Select(childCol + " AS label, COUNT(*) AS cnt").
Group(childCol).
Order("label ASC").
Offset((page - 1) * pageSize).
Limit(pageSize).
Scan(&rows).Error; err != nil {
return nil, err
return nil, 0, err
}
parent := cleanWikiRepositoryCategoryPath(parentPath)
counts := make(map[string]int64, len(rows))
pathsByKey := make(map[string][]string, len(rows))
order := make([]string, 0, len(rows))
for _, row := range rows {
if len(row.CategoryPath) == 0 {
continue
}
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
if len(clean) == 0 {
continue
}
if len(clean) <= len(parent) || !wikiPathHasPrefix(clean, parent) {
continue
}
path := clean[:len(parent)+1]
key := strings.Join(path, "\x00")
if _, ok := pathsByKey[key]; !ok {
cp := append([]string(nil), path...)
pathsByKey[key] = cp
order = append(order, key)
}
counts[key]++
out := make([]types.WikiCategoryPath, 0, len(rows))
for _, rw := range rows {
full := append(append([]string(nil), parent...), rw.Label)
out = append(out, types.WikiCategoryPath{Path: full, Count: rw.Cnt})
}
out := make([]types.WikiCategoryPath, 0, len(order))
for _, key := range order {
out = append(out, types.WikiCategoryPath{
Path: pathsByKey[key],
Count: counts[key],
})
}
return out, nil
}
func cleanWikiRepositoryCategoryPath(parts []string) []string {
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
for _, label := range cleanWikiRepositoryCategoryPart(part) {
if containsWikiRepositoryString(cleaned, label) {
continue
}
cleaned = append(cleaned, label)
}
}
return cleaned
}
func cleanWikiRepositoryCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiRepositoryTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
func isWikiRepositoryTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
}
func containsWikiRepositoryString(slice []string, s string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}
func wikiPathHasPrefix(path []string, prefix []string) bool {
if len(prefix) == 0 {
return true
}
if len(path) < len(prefix) {
return false
}
for i, part := range prefix {
if path[i] != part {
return false
}
}
return true
return out, int(total), nil
}
// ListSummariesByKnowledgeIDs returns summary-page content keyed by the
@@ -34,6 +34,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
depth INTEGER NOT NULL DEFAULT 0,
sort_order INTEGER NOT NULL DEFAULT 0,
category_l1 VARCHAR(255) NOT NULL DEFAULT '',
category_l2 VARCHAR(255) NOT NULL DEFAULT '',
category_l3 VARCHAR(255) NOT NULL DEFAULT '',
source_refs TEXT DEFAULT '[]',
chunk_refs TEXT DEFAULT '[]',
in_links TEXT DEFAULT '[]',
@@ -86,6 +89,9 @@ func makeCategorizedWikiPage(kbID, slug, pageType, status string, categoryPath .
if len(categoryPath) > 0 {
page.WikiPath = pageType + "/" + strings.Join(categoryPath, "/") + "/" + page.Title
page.Depth = len(categoryPath)
// Mirror the flat level columns the directory query groups on, the same
// way normalizeWikiHierarchy would on a real write.
page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(categoryPath)
}
return page
}
@@ -124,6 +130,59 @@ func TestList_WikiPathSortReturnsCategorizedPagesFirst(t *testing.T) {
assert.Equal(t, "entity/001-root", got[2].Slug)
}
// TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL verifies the
// directory listing groups direct child folders, counts pages under each
// (including deeper pages), and paginates — all pushed to SQL over the flat
// level columns, so it must behave identically on SQLite.
func TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
pages := []*types.WikiPage{
makeCategorizedWikiPage("kb-c", "entity/a1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"),
makeCategorizedWikiPage("kb-c", "entity/a2", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"),
makeCategorizedWikiPage("kb-c", "entity/a3", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI", "LLM"),
makeCategorizedWikiPage("kb-c", "entity/p1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "人物"),
makeWikiPage("kb-c", "entity/root", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
makeCategorizedWikiPage("kb-c", "entity/arch", types.WikiPageTypeEntity, types.WikiPageStatusArchived, "AI"),
}
for _, p := range pages {
require.NoError(t, repo.Create(ctx, p))
}
types4 := []string{types.WikiPageTypeEntity}
// Root level: two folders (AI, 人物); the loose root page and archived page
// are excluded. AI counts all 3 non-archived pages whose first level is AI.
roots, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 100)
require.NoError(t, err)
assert.Equal(t, 2, total)
require.Len(t, roots, 2)
byLabel := map[string]int64{}
for _, p := range roots {
require.Len(t, p.Path, 1)
byLabel[p.Path[0]] = p.Count
}
assert.Equal(t, int64(3), byLabel["AI"])
assert.Equal(t, int64(1), byLabel["人物"])
// Child of [AI]: only the [AI, LLM] page.
children, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, []string{"AI"}, 1, 100)
require.NoError(t, err)
assert.Equal(t, 1, total)
require.Len(t, children, 1)
assert.Equal(t, []string{"AI", "LLM"}, children[0].Path)
assert.Equal(t, int64(1), children[0].Count)
// Pagination: page size 1 over the two root folders returns one row but the
// total still reflects both distinct folders.
firstPage, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 1)
require.NoError(t, err)
assert.Equal(t, 2, total)
require.Len(t, firstPage, 1)
}
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
// that the index-view projection only emits the slug/title/summary
// triples and respects the archived-status filter. This is the whole
@@ -241,16 +241,7 @@ func remapCategoryPath(current []string, remap map[string][]string) ([]string, b
// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
// depth cap) so remap keys match what is stored on pages.
func cleanCategoryPathParts(parts []string) []string {
out := make([]string, 0, len(parts))
for _, part := range parts {
for _, c := range cleanWikiCategoryPart(part) {
out = append(out, c)
if len(out) >= 3 {
return out
}
}
}
return out
return types.CleanWikiCategoryPath(parts)
}
// categoryPathKey builds a stable comparison key for a cleaned category path.
+23 -83
View File
@@ -837,12 +837,13 @@ func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID st
return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths)
}
// ListDistinctCategoryPathsByType returns unique category_path folders across
// the given page types. The browser uses this to render directory nodes before
// article pages are paged in; passing several types lets one tab (e.g. the
// merged knowledge tab) build a single directory skeleton server-side.
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) {
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths)
// ListDistinctCategoryPathsByType returns the direct child folders of
// parentPath across the given page types, paginated, plus the total distinct
// child count. The browser uses this to expand the directory tree level by
// level; passing several types lets one tab (e.g. the merged knowledge tab)
// build a single directory skeleton server-side.
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error) {
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, page, pageSize)
}
// CountByType is a service-layer pass-through over the repo. Used by
@@ -965,38 +966,16 @@ func normalizeWikiHierarchy(page *types.WikiPage) {
}
page.ParentSlug = strings.TrimSpace(page.ParentSlug)
cleanPath := make(types.StringArray, 0, len(page.CategoryPath))
for _, part := range page.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
cleanPath := types.StringArray(types.CleanWikiCategoryPath(page.CategoryPath))
page.CategoryPath = cleanPath
page.Depth = len(cleanPath)
page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if page.PageType != "" {
parts = append(parts, page.PageType)
}
for _, part := range cleanPath {
parts = append(parts, part)
}
display := strings.TrimSpace(page.Title)
if display == "" {
display = strings.TrimSpace(page.Slug)
}
if display != "" {
parts = append(parts, display)
}
page.WikiPath = strings.Join(parts, "/")
page.WikiPath = buildWikiPath(page.PageType, cleanPath, display)
}
func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) {
@@ -1004,68 +983,29 @@ func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType stri
return
}
cleanPath := make(types.StringArray, 0, len(entry.CategoryPath))
for _, part := range entry.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
cleanPath := types.StringArray(types.CleanWikiCategoryPath(entry.CategoryPath))
entry.CategoryPath = cleanPath
entry.Depth = len(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if strings.TrimSpace(pageType) != "" {
parts = append(parts, strings.TrimSpace(pageType))
}
parts = append(parts, cleanPath...)
display := strings.TrimSpace(entry.Title)
if display == "" {
display = strings.TrimSpace(entry.Slug)
}
entry.WikiPath = buildWikiPath(pageType, cleanPath, display)
}
// buildWikiPath assembles the normalized, sortable "page_type/cat.../title"
// breadcrumb used for directory ordering. Empty segments are skipped.
func buildWikiPath(pageType string, categoryPath []string, display string) string {
parts := make([]string, 0, len(categoryPath)+2)
if pt := strings.TrimSpace(pageType); pt != "" {
parts = append(parts, pt)
}
parts = append(parts, categoryPath...)
if display != "" {
parts = append(parts, display)
}
entry.WikiPath = strings.Join(parts, "/")
}
func cleanWikiCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
func isWikiTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
return strings.Join(parts, "/")
}
// containsString checks if a string slice contains a given string
+6 -10
View File
@@ -162,19 +162,15 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
pageSize = 2000
}
parentPath := parseWikiCategoryPath(c.Query("parent_path"))
paths, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, pageSize)
// Grouping and pagination are pushed to SQL; the service returns just the
// requested page of child folders plus the total distinct count.
paths, total, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, page, pageSize)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
total := len(paths)
offset := (page - 1) * pageSize
if offset > total {
offset = total
}
end := offset + pageSize
if end > total {
end = total
if paths == nil {
paths = []types.WikiCategoryPath{}
}
totalPages := 0
if pageSize > 0 {
@@ -182,7 +178,7 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
}
c.JSON(http.StatusOK, types.WikiCategoryPathListResponse{
Paths: paths[offset:end],
Paths: paths,
Total: total,
Page: page,
PageSize: pageSize,
+10 -7
View File
@@ -161,10 +161,12 @@ type WikiPageService interface {
// established folder taxonomy into extraction prompts.
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
// ListDistinctCategoryPathsByType returns unique non-empty category_path
// values across the given page types. Used by the wiki browser to render a
// stable directory skeleton independently of page pagination.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
// ListDistinctCategoryPathsByType returns the direct child folders of
// parentPath for the given page types (with per-child page counts),
// paginated. The second return value is the total number of distinct
// child folders before pagination. Used by the wiki browser to render the
// directory tree level by level.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error)
// CountByType returns page counts grouped by type for a knowledge
// base. Re-exposed at the service layer so the index intro
@@ -278,9 +280,10 @@ type WikiPageRepository interface {
// unique non-empty category_path values for wiki ingest prompts.
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
// ListDistinctCategoryPathsByType returns unique non-empty category_path
// values across the given page types.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
// ListDistinctCategoryPathsByType returns the direct child folders of
// parentPath for the given page types (with per-child page counts),
// paginated, plus the total number of distinct child folders.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error)
// ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation).
ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error)
+97
View File
@@ -3,11 +3,101 @@ package types
import (
"database/sql/driver"
"encoding/json"
"strings"
"time"
"gorm.io/gorm"
)
// WikiCategoryMaxDepth is the hard cap on how many folder levels a wiki page's
// category_path may keep. The ingest prompts intentionally ask the model for at
// most 2 levels to keep the directory tree shallow; this storage cap is one
// level deeper as a defensive bound so a slightly over-eager model (or a
// reconcile remap) cannot create an unbounded breadcrumb. It is the single
// source of truth shared by the service, repository, and taxonomy layers so
// stored paths and queried paths are always cleaned identically.
const WikiCategoryMaxDepth = 3
var wikiCategorySeparatorReplacer = strings.NewReplacer("/", "/", "|", "/", "|", "/")
// CleanWikiCategoryPart normalizes a single raw category label that may itself
// carry embedded separators, wrapping quotes/brackets, or page-type noise, and
// returns the cleaned sub-labels (type labels such as "entity"/"实体" dropped).
func CleanWikiCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = wikiCategorySeparatorReplacer.Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
// CleanWikiCategoryPath cleans, deduplicates, and caps a full category path at
// WikiCategoryMaxDepth. Centralizing this guarantees that the path a page is
// stored with and the path a list/filter query is matched against go through
// the exact same normalization, so directory filters cannot silently drift.
func CleanWikiCategoryPath(parts []string) []string {
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
for _, label := range CleanWikiCategoryPart(part) {
if containsWikiString(cleaned, label) {
continue
}
cleaned = append(cleaned, label)
if len(cleaned) >= WikiCategoryMaxDepth {
return cleaned
}
}
}
return cleaned
}
// WikiCategoryLevels flattens a (already cleaned) category path into the three
// fixed level columns used for SQL grouping. Missing levels yield "".
func WikiCategoryLevels(path []string) (l1, l2, l3 string) {
if len(path) > 0 {
l1 = path[0]
}
if len(path) > 1 {
l2 = path[1]
}
if len(path) > 2 {
l3 = path[2]
}
return l1, l2, l3
}
func isWikiTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
}
func containsWikiString(slice []string, s string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}
// WikiPageType constants define the types of wiki pages
const (
// WikiPageTypeSummary represents a document summary page
@@ -81,6 +171,13 @@ type WikiPage struct {
// SortOrder allows generated or manually edited pages to control sibling
// ordering before falling back to title.
SortOrder int `json:"sort_order,omitempty" gorm:"default:0;index"`
// CategoryL1/L2/L3 mirror CategoryPath[0..2] as flat columns. They exist
// solely so the directory-listing query can GROUP BY / paginate folders in
// plain, cross-dialect SQL (no JSON parsing). They are NOT part of the API
// surface (hence json:"-") and are kept in sync by normalizeWikiHierarchy.
CategoryL1 string `json:"-" gorm:"column:category_l1;type:varchar(255);default:''"`
CategoryL2 string `json:"-" gorm:"column:category_l2;type:varchar(255);default:''"`
CategoryL3 string `json:"-" gorm:"column:category_l3;type:varchar(255);default:''"`
// References to source knowledge IDs that contributed to this page.
// Format matches the legacy "<knowledge_id>|<doc_title>" convention used
// across the ingest pipeline, so retract / display code can split on `|`
@@ -27,6 +27,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
depth INT NOT NULL DEFAULT 0,
sort_order INT NOT NULL DEFAULT 0,
category_l1 VARCHAR(255) NOT NULL DEFAULT '',
category_l2 VARCHAR(255) NOT NULL DEFAULT '',
category_l3 VARCHAR(255) NOT NULL DEFAULT '',
source_refs JSONB DEFAULT '[]'::JSONB,
chunk_refs JSONB DEFAULT '[]'::JSONB,
in_links JSONB DEFAULT '[]'::JSONB,
@@ -56,6 +59,9 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels
ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id
ON wiki_pages (tenant_id);
@@ -1,8 +1,12 @@
-- Migration: 000061_wiki_page_hierarchy (rollback)
DROP INDEX IF EXISTS idx_wiki_pages_category_levels;
DROP INDEX IF EXISTS idx_wiki_pages_tree;
DROP INDEX IF EXISTS idx_wiki_pages_parent_slug;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l3;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l2;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l1;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS sort_order;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS depth;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS wiki_path;
@@ -8,6 +8,11 @@ ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_path JSONB DEFAULT '[]'
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS wiki_path VARCHAR(1024) NOT NULL DEFAULT '';
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS depth INT NOT NULL DEFAULT 0;
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS sort_order INT NOT NULL DEFAULT 0;
-- Flat mirrors of category_path[0..2] for cheap, cross-dialect directory
-- grouping/pagination (kept in sync by the application on write).
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l1 VARCHAR(255) NOT NULL DEFAULT '';
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l2 VARCHAR(255) NOT NULL DEFAULT '';
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l3 VARCHAR(255) NOT NULL DEFAULT '';
UPDATE wiki_pages
SET
@@ -20,10 +25,21 @@ SET
END
WHERE wiki_path = '' OR wiki_path IS NULL OR category_path IS NULL OR depth IS NULL;
-- Backfill the flat level columns from the JSON category_path.
UPDATE wiki_pages
SET
category_l1 = COALESCE(category_path->>0, ''),
category_l2 = COALESCE(category_path->>1, ''),
category_l3 = COALESCE(category_path->>2, '')
WHERE category_path IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
ON wiki_pages (knowledge_base_id, parent_slug);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels
ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3);
DO $$ BEGIN RAISE NOTICE '[Migration 000061] wiki page hierarchy schema applied successfully'; END $$;