refactor(wiki): unify category cleaning and push directory listing to SQL
Centralize the wiki category-path normalization (separator/quote cleanup, type-label stripping, dedup, depth cap) into a single types helper shared by the service, repository, and taxonomy layers so stored and queried paths are always cleaned identically and cannot drift. Flatten category_path into fixed category_l1/l2/l3 columns (kept in sync on write) and rewrite ListDistinctCategoryPathsByType to group, count, and paginate directly in cross-dialect SQL, returning the page of child folders plus the total distinct count. This removes the prior full-column scan and in-memory pagination (whose maxPaths cap was effectively dead) and makes Total and later pages correct. Add the level columns + composite index to the wiki migrations and the repo test schema, and cover the SQL grouping/counting/pagination with a new test.
This commit is contained in:
@@ -110,6 +110,9 @@ func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPag
|
||||
"page_metadata": page.PageMetadata,
|
||||
"parent_slug": page.ParentSlug,
|
||||
"category_path": page.CategoryPath,
|
||||
"category_l1": page.CategoryL1,
|
||||
"category_l2": page.CategoryL2,
|
||||
"category_l3": page.CategoryL3,
|
||||
"wiki_path": page.WikiPath,
|
||||
"depth": page.Depth,
|
||||
"sort_order": page.SortOrder,
|
||||
@@ -203,7 +206,7 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
|
||||
if req.CategoryDepth != nil {
|
||||
query = query.Where("depth = ?", *req.CategoryDepth)
|
||||
}
|
||||
if wantPath := cleanWikiRepositoryCategoryPath(req.CategoryPath); len(wantPath) > 0 {
|
||||
if wantPath := types.CleanWikiCategoryPath(req.CategoryPath); len(wantPath) > 0 {
|
||||
if encoded, err := json.Marshal([]string(wantPath)); err == nil {
|
||||
if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" {
|
||||
query = query.Where("category_path::jsonb = ?::jsonb", string(encoded))
|
||||
@@ -470,7 +473,7 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths(
|
||||
if len(row.CategoryPath) == 0 {
|
||||
continue
|
||||
}
|
||||
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
|
||||
clean := types.CleanWikiCategoryPath(row.CategoryPath)
|
||||
if len(clean) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -487,144 +490,83 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths(
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListDistinctCategoryPathsByType scans one page type and returns unique
|
||||
// non-empty category_path values. Unlike ListDistinctCategoryPaths, this is
|
||||
// used by the UI and must work for every visible page_type bucket.
|
||||
// wikiCategoryLevelColumns maps a parent depth (0/1/2) to the flat level column
|
||||
// holding that depth's direct child. Depth >= WikiCategoryMaxDepth has no child.
|
||||
var wikiCategoryLevelColumns = []string{"category_l1", "category_l2", "category_l3"}
|
||||
|
||||
// ListDistinctCategoryPathsByType returns the DIRECT child folders of
|
||||
// parentPath for the given page types, with the page count under each child,
|
||||
// paginated. Grouping and pagination are pushed entirely to SQL over the flat
|
||||
// category_l1/l2/l3 columns (kept in sync by normalizeWikiHierarchy), so this
|
||||
// is cheap, indexable, and identical across Postgres and SQLite — no JSON
|
||||
// parsing and no full in-memory scan. The second return value is the total
|
||||
// number of distinct child folders (before pagination).
|
||||
func (r *wikiPageRepository) ListDistinctCategoryPathsByType(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
pageTypes []string,
|
||||
parentPath []string,
|
||||
maxPaths int,
|
||||
) ([]types.WikiCategoryPath, error) {
|
||||
if maxPaths <= 0 {
|
||||
maxPaths = 2000
|
||||
}
|
||||
page, pageSize int,
|
||||
) ([]types.WikiCategoryPath, int, error) {
|
||||
if len(pageTypes) == 0 {
|
||||
return nil, nil
|
||||
return nil, 0, nil
|
||||
}
|
||||
parent := types.CleanWikiCategoryPath(parentPath)
|
||||
if len(parent) >= len(wikiCategoryLevelColumns) {
|
||||
return nil, 0, nil // no level deeper than the cap can exist
|
||||
}
|
||||
childCol := wikiCategoryLevelColumns[len(parent)]
|
||||
|
||||
// Build the shared WHERE as a closure so the count and the page query each
|
||||
// start from a fresh statement (GORM chains mutate in place otherwise).
|
||||
scoped := func() *gorm.DB {
|
||||
q := r.db.WithContext(ctx).
|
||||
Model(&types.WikiPage{}).
|
||||
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
|
||||
kbID, pageTypes, types.WikiPageStatusArchived).
|
||||
Where(childCol+" <> ?", "")
|
||||
for i, label := range parent {
|
||||
q = q.Where(wikiCategoryLevelColumns[i]+" = ?", label)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := scoped().Distinct(childCol).Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 2000
|
||||
}
|
||||
|
||||
type row struct {
|
||||
CategoryPath types.StringArray `gorm:"column:category_path"`
|
||||
Label string `gorm:"column:label"`
|
||||
Cnt int64 `gorm:"column:cnt"`
|
||||
}
|
||||
var rows []row
|
||||
if err := r.db.WithContext(ctx).
|
||||
Model(&types.WikiPage{}).
|
||||
Select("category_path").
|
||||
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
|
||||
kbID,
|
||||
pageTypes,
|
||||
types.WikiPageStatusArchived,
|
||||
).
|
||||
Order(r.wikiCategoryRankOrder()).
|
||||
Order("wiki_path ASC").
|
||||
if err := scoped().
|
||||
Select(childCol + " AS label, COUNT(*) AS cnt").
|
||||
Group(childCol).
|
||||
Order("label ASC").
|
||||
Offset((page - 1) * pageSize).
|
||||
Limit(pageSize).
|
||||
Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
parent := cleanWikiRepositoryCategoryPath(parentPath)
|
||||
|
||||
counts := make(map[string]int64, len(rows))
|
||||
pathsByKey := make(map[string][]string, len(rows))
|
||||
order := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row.CategoryPath) == 0 {
|
||||
continue
|
||||
}
|
||||
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
|
||||
if len(clean) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(clean) <= len(parent) || !wikiPathHasPrefix(clean, parent) {
|
||||
continue
|
||||
}
|
||||
path := clean[:len(parent)+1]
|
||||
key := strings.Join(path, "\x00")
|
||||
if _, ok := pathsByKey[key]; !ok {
|
||||
cp := append([]string(nil), path...)
|
||||
pathsByKey[key] = cp
|
||||
order = append(order, key)
|
||||
}
|
||||
counts[key]++
|
||||
out := make([]types.WikiCategoryPath, 0, len(rows))
|
||||
for _, rw := range rows {
|
||||
full := append(append([]string(nil), parent...), rw.Label)
|
||||
out = append(out, types.WikiCategoryPath{Path: full, Count: rw.Cnt})
|
||||
}
|
||||
|
||||
out := make([]types.WikiCategoryPath, 0, len(order))
|
||||
for _, key := range order {
|
||||
out = append(out, types.WikiCategoryPath{
|
||||
Path: pathsByKey[key],
|
||||
Count: counts[key],
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cleanWikiRepositoryCategoryPath(parts []string) []string {
|
||||
cleaned := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
for _, label := range cleanWikiRepositoryCategoryPart(part) {
|
||||
if containsWikiRepositoryString(cleaned, label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanWikiRepositoryCategoryPart(part string) []string {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
|
||||
rawParts := strings.Split(part, "/")
|
||||
cleaned := make([]string, 0, len(rawParts))
|
||||
for _, raw := range rawParts {
|
||||
label := strings.TrimSpace(raw)
|
||||
label = strings.Trim(label, `"'“”‘’[]()()`)
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" || isWikiRepositoryTypeCategoryLabel(label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func isWikiRepositoryTypeCategoryLabel(label string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(label))
|
||||
normalized = strings.TrimSuffix(normalized, "s")
|
||||
switch normalized {
|
||||
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func containsWikiRepositoryString(slice []string, s string) bool {
|
||||
for _, v := range slice {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func wikiPathHasPrefix(path []string, prefix []string) bool {
|
||||
if len(prefix) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(path) < len(prefix) {
|
||||
return false
|
||||
}
|
||||
for i, part := range prefix {
|
||||
if path[i] != part {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
return out, int(total), nil
|
||||
}
|
||||
|
||||
// ListSummariesByKnowledgeIDs returns summary-page content keyed by the
|
||||
|
||||
@@ -34,6 +34,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
|
||||
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
depth INTEGER NOT NULL DEFAULT 0,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
category_l1 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
category_l2 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
category_l3 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
source_refs TEXT DEFAULT '[]',
|
||||
chunk_refs TEXT DEFAULT '[]',
|
||||
in_links TEXT DEFAULT '[]',
|
||||
@@ -86,6 +89,9 @@ func makeCategorizedWikiPage(kbID, slug, pageType, status string, categoryPath .
|
||||
if len(categoryPath) > 0 {
|
||||
page.WikiPath = pageType + "/" + strings.Join(categoryPath, "/") + "/" + page.Title
|
||||
page.Depth = len(categoryPath)
|
||||
// Mirror the flat level columns the directory query groups on, the same
|
||||
// way normalizeWikiHierarchy would on a real write.
|
||||
page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(categoryPath)
|
||||
}
|
||||
return page
|
||||
}
|
||||
@@ -124,6 +130,59 @@ func TestList_WikiPathSortReturnsCategorizedPagesFirst(t *testing.T) {
|
||||
assert.Equal(t, "entity/001-root", got[2].Slug)
|
||||
}
|
||||
|
||||
// TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL verifies the
|
||||
// directory listing groups direct child folders, counts pages under each
|
||||
// (including deeper pages), and paginates — all pushed to SQL over the flat
|
||||
// level columns, so it must behave identically on SQLite.
|
||||
func TestListDistinctCategoryPathsByType_GroupsAndPaginatesInSQL(t *testing.T) {
|
||||
db := setupWikiPagesTestDB(t)
|
||||
repo := NewWikiPageRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
pages := []*types.WikiPage{
|
||||
makeCategorizedWikiPage("kb-c", "entity/a1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"),
|
||||
makeCategorizedWikiPage("kb-c", "entity/a2", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI"),
|
||||
makeCategorizedWikiPage("kb-c", "entity/a3", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "AI", "LLM"),
|
||||
makeCategorizedWikiPage("kb-c", "entity/p1", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "人物"),
|
||||
makeWikiPage("kb-c", "entity/root", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
|
||||
makeCategorizedWikiPage("kb-c", "entity/arch", types.WikiPageTypeEntity, types.WikiPageStatusArchived, "AI"),
|
||||
}
|
||||
for _, p := range pages {
|
||||
require.NoError(t, repo.Create(ctx, p))
|
||||
}
|
||||
|
||||
types4 := []string{types.WikiPageTypeEntity}
|
||||
|
||||
// Root level: two folders (AI, 人物); the loose root page and archived page
|
||||
// are excluded. AI counts all 3 non-archived pages whose first level is AI.
|
||||
roots, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, total)
|
||||
require.Len(t, roots, 2)
|
||||
byLabel := map[string]int64{}
|
||||
for _, p := range roots {
|
||||
require.Len(t, p.Path, 1)
|
||||
byLabel[p.Path[0]] = p.Count
|
||||
}
|
||||
assert.Equal(t, int64(3), byLabel["AI"])
|
||||
assert.Equal(t, int64(1), byLabel["人物"])
|
||||
|
||||
// Child of [AI]: only the [AI, LLM] page.
|
||||
children, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, []string{"AI"}, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, total)
|
||||
require.Len(t, children, 1)
|
||||
assert.Equal(t, []string{"AI", "LLM"}, children[0].Path)
|
||||
assert.Equal(t, int64(1), children[0].Count)
|
||||
|
||||
// Pagination: page size 1 over the two root folders returns one row but the
|
||||
// total still reflects both distinct folders.
|
||||
firstPage, total, err := repo.ListDistinctCategoryPathsByType(ctx, "kb-c", types4, nil, 1, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, total)
|
||||
require.Len(t, firstPage, 1)
|
||||
}
|
||||
|
||||
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
|
||||
// that the index-view projection only emits the slug/title/summary
|
||||
// triples and respects the archived-status filter. This is the whole
|
||||
|
||||
@@ -241,16 +241,7 @@ func remapCategoryPath(current []string, remap map[string][]string) ([]string, b
|
||||
// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
|
||||
// depth cap) so remap keys match what is stored on pages.
|
||||
func cleanCategoryPathParts(parts []string) []string {
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
for _, c := range cleanWikiCategoryPart(part) {
|
||||
out = append(out, c)
|
||||
if len(out) >= 3 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
return types.CleanWikiCategoryPath(parts)
|
||||
}
|
||||
|
||||
// categoryPathKey builds a stable comparison key for a cleaned category path.
|
||||
|
||||
@@ -837,12 +837,13 @@ func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID st
|
||||
return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths)
|
||||
}
|
||||
|
||||
// ListDistinctCategoryPathsByType returns unique category_path folders across
|
||||
// the given page types. The browser uses this to render directory nodes before
|
||||
// article pages are paged in; passing several types lets one tab (e.g. the
|
||||
// merged knowledge tab) build a single directory skeleton server-side.
|
||||
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) {
|
||||
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths)
|
||||
// ListDistinctCategoryPathsByType returns the direct child folders of
|
||||
// parentPath across the given page types, paginated, plus the total distinct
|
||||
// child count. The browser uses this to expand the directory tree level by
|
||||
// level; passing several types lets one tab (e.g. the merged knowledge tab)
|
||||
// build a single directory skeleton server-side.
|
||||
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error) {
|
||||
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, page, pageSize)
|
||||
}
|
||||
|
||||
// CountByType is a service-layer pass-through over the repo. Used by
|
||||
@@ -965,38 +966,16 @@ func normalizeWikiHierarchy(page *types.WikiPage) {
|
||||
}
|
||||
page.ParentSlug = strings.TrimSpace(page.ParentSlug)
|
||||
|
||||
cleanPath := make(types.StringArray, 0, len(page.CategoryPath))
|
||||
for _, part := range page.CategoryPath {
|
||||
for _, cleanPart := range cleanWikiCategoryPart(part) {
|
||||
if !containsString(cleanPath, cleanPart) {
|
||||
cleanPath = append(cleanPath, cleanPart)
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
cleanPath := types.StringArray(types.CleanWikiCategoryPath(page.CategoryPath))
|
||||
page.CategoryPath = cleanPath
|
||||
page.Depth = len(cleanPath)
|
||||
page.CategoryL1, page.CategoryL2, page.CategoryL3 = types.WikiCategoryLevels(cleanPath)
|
||||
|
||||
parts := make([]string, 0, len(cleanPath)+2)
|
||||
if page.PageType != "" {
|
||||
parts = append(parts, page.PageType)
|
||||
}
|
||||
for _, part := range cleanPath {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
display := strings.TrimSpace(page.Title)
|
||||
if display == "" {
|
||||
display = strings.TrimSpace(page.Slug)
|
||||
}
|
||||
if display != "" {
|
||||
parts = append(parts, display)
|
||||
}
|
||||
page.WikiPath = strings.Join(parts, "/")
|
||||
page.WikiPath = buildWikiPath(page.PageType, cleanPath, display)
|
||||
}
|
||||
|
||||
func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) {
|
||||
@@ -1004,68 +983,29 @@ func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType stri
|
||||
return
|
||||
}
|
||||
|
||||
cleanPath := make(types.StringArray, 0, len(entry.CategoryPath))
|
||||
for _, part := range entry.CategoryPath {
|
||||
for _, cleanPart := range cleanWikiCategoryPart(part) {
|
||||
if !containsString(cleanPath, cleanPart) {
|
||||
cleanPath = append(cleanPath, cleanPart)
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
cleanPath := types.StringArray(types.CleanWikiCategoryPath(entry.CategoryPath))
|
||||
entry.CategoryPath = cleanPath
|
||||
entry.Depth = len(cleanPath)
|
||||
|
||||
parts := make([]string, 0, len(cleanPath)+2)
|
||||
if strings.TrimSpace(pageType) != "" {
|
||||
parts = append(parts, strings.TrimSpace(pageType))
|
||||
}
|
||||
parts = append(parts, cleanPath...)
|
||||
display := strings.TrimSpace(entry.Title)
|
||||
if display == "" {
|
||||
display = strings.TrimSpace(entry.Slug)
|
||||
}
|
||||
entry.WikiPath = buildWikiPath(pageType, cleanPath, display)
|
||||
}
|
||||
|
||||
// buildWikiPath assembles the normalized, sortable "page_type/cat.../title"
|
||||
// breadcrumb used for directory ordering. Empty segments are skipped.
|
||||
func buildWikiPath(pageType string, categoryPath []string, display string) string {
|
||||
parts := make([]string, 0, len(categoryPath)+2)
|
||||
if pt := strings.TrimSpace(pageType); pt != "" {
|
||||
parts = append(parts, pt)
|
||||
}
|
||||
parts = append(parts, categoryPath...)
|
||||
if display != "" {
|
||||
parts = append(parts, display)
|
||||
}
|
||||
entry.WikiPath = strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func cleanWikiCategoryPart(part string) []string {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
|
||||
rawParts := strings.Split(part, "/")
|
||||
cleaned := make([]string, 0, len(rawParts))
|
||||
for _, raw := range rawParts {
|
||||
label := strings.TrimSpace(raw)
|
||||
label = strings.Trim(label, `"'“”‘’[]()()`)
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" || isWikiTypeCategoryLabel(label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func isWikiTypeCategoryLabel(label string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(label))
|
||||
normalized = strings.TrimSuffix(normalized, "s")
|
||||
switch normalized {
|
||||
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
// containsString checks if a string slice contains a given string
|
||||
|
||||
@@ -162,19 +162,15 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
|
||||
pageSize = 2000
|
||||
}
|
||||
parentPath := parseWikiCategoryPath(c.Query("parent_path"))
|
||||
paths, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, pageSize)
|
||||
// Grouping and pagination are pushed to SQL; the service returns just the
|
||||
// requested page of child folders plus the total distinct count.
|
||||
paths, total, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, page, pageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
total := len(paths)
|
||||
offset := (page - 1) * pageSize
|
||||
if offset > total {
|
||||
offset = total
|
||||
}
|
||||
end := offset + pageSize
|
||||
if end > total {
|
||||
end = total
|
||||
if paths == nil {
|
||||
paths = []types.WikiCategoryPath{}
|
||||
}
|
||||
totalPages := 0
|
||||
if pageSize > 0 {
|
||||
@@ -182,7 +178,7 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, types.WikiCategoryPathListResponse{
|
||||
Paths: paths[offset:end],
|
||||
Paths: paths,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
|
||||
@@ -161,10 +161,12 @@ type WikiPageService interface {
|
||||
// established folder taxonomy into extraction prompts.
|
||||
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
|
||||
|
||||
// ListDistinctCategoryPathsByType returns unique non-empty category_path
|
||||
// values across the given page types. Used by the wiki browser to render a
|
||||
// stable directory skeleton independently of page pagination.
|
||||
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
|
||||
// ListDistinctCategoryPathsByType returns the direct child folders of
|
||||
// parentPath for the given page types (with per-child page counts),
|
||||
// paginated. The second return value is the total number of distinct
|
||||
// child folders before pagination. Used by the wiki browser to render the
|
||||
// directory tree level by level.
|
||||
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error)
|
||||
|
||||
// CountByType returns page counts grouped by type for a knowledge
|
||||
// base. Re-exposed at the service layer so the index intro
|
||||
@@ -278,9 +280,10 @@ type WikiPageRepository interface {
|
||||
// unique non-empty category_path values for wiki ingest prompts.
|
||||
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
|
||||
|
||||
// ListDistinctCategoryPathsByType returns unique non-empty category_path
|
||||
// values across the given page types.
|
||||
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
|
||||
// ListDistinctCategoryPathsByType returns the direct child folders of
|
||||
// parentPath for the given page types (with per-child page counts),
|
||||
// paginated, plus the total number of distinct child folders.
|
||||
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, page, pageSize int) ([]types.WikiCategoryPath, int, error)
|
||||
|
||||
// ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation).
|
||||
ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error)
|
||||
|
||||
@@ -3,11 +3,101 @@ package types
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WikiCategoryMaxDepth is the hard cap on how many folder levels a wiki page's
|
||||
// category_path may keep. The ingest prompts intentionally ask the model for at
|
||||
// most 2 levels to keep the directory tree shallow; this storage cap is one
|
||||
// level deeper as a defensive bound so a slightly over-eager model (or a
|
||||
// reconcile remap) cannot create an unbounded breadcrumb. It is the single
|
||||
// source of truth shared by the service, repository, and taxonomy layers so
|
||||
// stored paths and queried paths are always cleaned identically.
|
||||
const WikiCategoryMaxDepth = 3
|
||||
|
||||
var wikiCategorySeparatorReplacer = strings.NewReplacer("/", "/", "|", "/", "|", "/")
|
||||
|
||||
// CleanWikiCategoryPart normalizes a single raw category label that may itself
|
||||
// carry embedded separators, wrapping quotes/brackets, or page-type noise, and
|
||||
// returns the cleaned sub-labels (type labels such as "entity"/"实体" dropped).
|
||||
func CleanWikiCategoryPart(part string) []string {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil
|
||||
}
|
||||
part = wikiCategorySeparatorReplacer.Replace(part)
|
||||
rawParts := strings.Split(part, "/")
|
||||
cleaned := make([]string, 0, len(rawParts))
|
||||
for _, raw := range rawParts {
|
||||
label := strings.TrimSpace(raw)
|
||||
label = strings.Trim(label, `"'“”‘’[]()()`)
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" || isWikiTypeCategoryLabel(label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
// CleanWikiCategoryPath cleans, deduplicates, and caps a full category path at
|
||||
// WikiCategoryMaxDepth. Centralizing this guarantees that the path a page is
|
||||
// stored with and the path a list/filter query is matched against go through
|
||||
// the exact same normalization, so directory filters cannot silently drift.
|
||||
func CleanWikiCategoryPath(parts []string) []string {
|
||||
cleaned := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
for _, label := range CleanWikiCategoryPart(part) {
|
||||
if containsWikiString(cleaned, label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
if len(cleaned) >= WikiCategoryMaxDepth {
|
||||
return cleaned
|
||||
}
|
||||
}
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
// WikiCategoryLevels flattens a (already cleaned) category path into the three
|
||||
// fixed level columns used for SQL grouping. Missing levels yield "".
|
||||
func WikiCategoryLevels(path []string) (l1, l2, l3 string) {
|
||||
if len(path) > 0 {
|
||||
l1 = path[0]
|
||||
}
|
||||
if len(path) > 1 {
|
||||
l2 = path[1]
|
||||
}
|
||||
if len(path) > 2 {
|
||||
l3 = path[2]
|
||||
}
|
||||
return l1, l2, l3
|
||||
}
|
||||
|
||||
func isWikiTypeCategoryLabel(label string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(label))
|
||||
normalized = strings.TrimSuffix(normalized, "s")
|
||||
switch normalized {
|
||||
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func containsWikiString(slice []string, s string) bool {
|
||||
for _, v := range slice {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// WikiPageType constants define the types of wiki pages
|
||||
const (
|
||||
// WikiPageTypeSummary represents a document summary page
|
||||
@@ -81,6 +171,13 @@ type WikiPage struct {
|
||||
// SortOrder allows generated or manually edited pages to control sibling
|
||||
// ordering before falling back to title.
|
||||
SortOrder int `json:"sort_order,omitempty" gorm:"default:0;index"`
|
||||
// CategoryL1/L2/L3 mirror CategoryPath[0..2] as flat columns. They exist
|
||||
// solely so the directory-listing query can GROUP BY / paginate folders in
|
||||
// plain, cross-dialect SQL (no JSON parsing). They are NOT part of the API
|
||||
// surface (hence json:"-") and are kept in sync by normalizeWikiHierarchy.
|
||||
CategoryL1 string `json:"-" gorm:"column:category_l1;type:varchar(255);default:''"`
|
||||
CategoryL2 string `json:"-" gorm:"column:category_l2;type:varchar(255);default:''"`
|
||||
CategoryL3 string `json:"-" gorm:"column:category_l3;type:varchar(255);default:''"`
|
||||
// References to source knowledge IDs that contributed to this page.
|
||||
// Format matches the legacy "<knowledge_id>|<doc_title>" convention used
|
||||
// across the ingest pipeline, so retract / display code can split on `|`
|
||||
|
||||
@@ -27,6 +27,9 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
|
||||
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
depth INT NOT NULL DEFAULT 0,
|
||||
sort_order INT NOT NULL DEFAULT 0,
|
||||
category_l1 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
category_l2 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
category_l3 VARCHAR(255) NOT NULL DEFAULT '',
|
||||
source_refs JSONB DEFAULT '[]'::JSONB,
|
||||
chunk_refs JSONB DEFAULT '[]'::JSONB,
|
||||
in_links JSONB DEFAULT '[]'::JSONB,
|
||||
@@ -56,6 +59,9 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
|
||||
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels
|
||||
ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id
|
||||
ON wiki_pages (tenant_id);
|
||||
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
-- Migration: 000061_wiki_page_hierarchy (rollback)
|
||||
|
||||
DROP INDEX IF EXISTS idx_wiki_pages_category_levels;
|
||||
DROP INDEX IF EXISTS idx_wiki_pages_tree;
|
||||
DROP INDEX IF EXISTS idx_wiki_pages_parent_slug;
|
||||
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l3;
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l2;
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_l1;
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS sort_order;
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS depth;
|
||||
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS wiki_path;
|
||||
|
||||
@@ -8,6 +8,11 @@ ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_path JSONB DEFAULT '[]'
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS wiki_path VARCHAR(1024) NOT NULL DEFAULT '';
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS depth INT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS sort_order INT NOT NULL DEFAULT 0;
|
||||
-- Flat mirrors of category_path[0..2] for cheap, cross-dialect directory
|
||||
-- grouping/pagination (kept in sync by the application on write).
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l1 VARCHAR(255) NOT NULL DEFAULT '';
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l2 VARCHAR(255) NOT NULL DEFAULT '';
|
||||
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_l3 VARCHAR(255) NOT NULL DEFAULT '';
|
||||
|
||||
UPDATE wiki_pages
|
||||
SET
|
||||
@@ -20,10 +25,21 @@ SET
|
||||
END
|
||||
WHERE wiki_path = '' OR wiki_path IS NULL OR category_path IS NULL OR depth IS NULL;
|
||||
|
||||
-- Backfill the flat level columns from the JSON category_path.
|
||||
UPDATE wiki_pages
|
||||
SET
|
||||
category_l1 = COALESCE(category_path->>0, ''),
|
||||
category_l2 = COALESCE(category_path->>1, ''),
|
||||
category_l3 = COALESCE(category_path->>2, '')
|
||||
WHERE category_path IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
|
||||
ON wiki_pages (knowledge_base_id, parent_slug);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
|
||||
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_category_levels
|
||||
ON wiki_pages (knowledge_base_id, page_type, category_l1, category_l2, category_l3);
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000061] wiki page hierarchy schema applied successfully'; END $$;
|
||||
|
||||
Reference in New Issue
Block a user