feat: Enhance wiki API and repository with slug encoding and improved search functionality

- Introduced `encodeSlugPath` function to safely encode wiki slugs for API requests, preserving hierarchical routing.
- Updated API functions to utilize the new slug encoding for `getWikiPage`, `updateWikiPage`, and `deleteWikiPage`.
- Enhanced `ListBySourceRef` method in the repository to safely handle source knowledge IDs with JSON marshalling, improving query security.
- Added `escapeLikePattern` function to ensure safe concatenation of LIKE patterns in SQL queries, preventing unintended matches.
- Updated `KnowledgePostProcessService` to include Redis client for task management, improving performance in handling wiki ingest tasks.
- Enhanced `WikiLintIssue` structure to include `TargetSlug` for better issue tracking and auto-fixing capabilities.

These changes significantly improve the robustness and security of the wiki management system, enhancing user experience and data integrity.
This commit is contained in:
wizardchen
2026-04-17 21:16:35 +08:00
parent 84edfa5439
commit 99cc721140
8 changed files with 104 additions and 43 deletions
+11 -3
View File
@@ -1,5 +1,13 @@
import { get, post, put, del } from "../../utils/request";
// encodeSlugPath encodes each segment of a hierarchical wiki slug (e.g.
// "foo/bar baz?") so the URL is safe while preserving the "/" separators
// between segments. Using encodeURIComponent on the whole slug would also
// escape the "/" and break hierarchical routing on the backend.
function encodeSlugPath(slug: string): string {
return slug.split("/").map(encodeURIComponent).join("/");
}
// Wiki Page Types
export interface WikiPage {
id: string;
@@ -86,15 +94,15 @@ export function createWikiPage(kbId: string, data: Partial<WikiPage>) {
}
export function getWikiPage(kbId: string, slug: string) {
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`);
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
}
export function updateWikiPage(kbId: string, slug: string, data: Partial<WikiPage>) {
return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`, data);
return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`, data);
}
export function deleteWikiPage(kbId: string, slug: string) {
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`);
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
}
export function getWikiIndex(kbId: string) {
+42 -5
View File
@@ -2,8 +2,10 @@ package repository
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -179,13 +181,35 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array.
func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) {
// Build the JSON needle safely so arbitrary IDs cannot break out of the
// quoted string (e.g. ids containing quotes or backslashes).
needle, err := json.Marshal([]string{sourceKnowledgeID})
if err != nil {
return nil, fmt.Errorf("marshal source ref needle: %w", err)
}
// For the "knowledgeID|title" prefix form, match against the JSON-encoded
// value: json.Marshal escapes special chars so the LIKE pattern is safe.
prefix, err := json.Marshal(sourceKnowledgeID + "|")
if err != nil {
return nil, fmt.Errorf("marshal source ref prefix: %w", err)
}
// prefix is a JSON string including the surrounding quotes; e.g. "abc|".
// We strip the trailing quote so LIKE can continue into the title portion.
prefixStr := string(prefix)
if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' {
prefixStr = prefixStr[:len(prefixStr)-1]
}
// Escape LIKE metacharacters in the already-JSON-escaped prefix, then wrap
// with %…% to match anywhere in the serialized JSON array.
likePattern := "%" + escapeLikePattern(prefixStr) + "%"
var pages []*types.WikiPage
// Match either exact "knowledgeID" or "knowledgeID|..." prefix in the JSON array
if err := r.db.WithContext(ctx).
Where("knowledge_base_id = ? AND (source_refs @> ? OR source_refs::text LIKE ?)",
Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)",
kbID,
fmt.Sprintf(`["%s"]`, sourceKnowledgeID),
fmt.Sprintf(`%%"%s|%%`, sourceKnowledgeID),
string(needle),
likePattern,
).
Find(&pages).Error; err != nil {
return nil, err
@@ -233,7 +257,20 @@ func (r *wikiPageRepository) DeleteByID(ctx context.Context, id string) error {
return nil
}
// Search performs full-text search on wiki pages within a knowledge base
// escapeLikePattern escapes LIKE / ILIKE metacharacters so the returned string
// can be safely concatenated with % wildcards without unintended matches.
// Order matters: escape the backslash first, then the wildcards.
func escapeLikePattern(s string) string {
replacer := strings.NewReplacer(
`\`, `\\`,
`%`, `\%`,
`_`, `\_`,
)
return replacer.Replace(s)
}
// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base.
// The query is interpreted as a PostgreSQL regular expression (via ~*).
func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) {
if limit <= 0 {
limit = 10
@@ -2,6 +2,7 @@ package chatpipeline
import (
"context"
"sort"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
@@ -49,16 +50,28 @@ func (p *PluginWikiBoost) OnEvent(
return err
}
// Check if any search target is a wiki KB
// Fast path: skip all work if there are no wiki_page chunks in the result set.
// This avoids hitting the KB service on every chat turn for non-wiki queries.
hasWikiChunk := false
for i := range chatManage.RerankResult {
if chatManage.RerankResult[i].ChunkType == types.ChunkTypeWikiPage {
hasWikiChunk = true
break
}
}
if !hasWikiChunk {
return nil
}
// Confirm at least one search target is actually a wiki KB.
hasWikiKB := false
for _, target := range chatManage.SearchTargets {
kb, err := p.kbService.GetKnowledgeBaseByIDOnly(ctx, target.KnowledgeBaseID)
if err == nil && kb.IsWikiEnabled() {
if err == nil && kb != nil && kb.IsWikiEnabled() {
hasWikiKB = true
break
}
}
if !hasWikiKB {
return nil
}
@@ -74,15 +87,10 @@ func (p *PluginWikiBoost) OnEvent(
if boostedCount > 0 {
logger.Infof(ctx, "WikiBoost: boosted %d wiki page chunks by %.1fx", boostedCount, wikiBoostFactor)
// Re-sort by score after boosting
for i := 0; i < len(chatManage.RerankResult); i++ {
for j := i + 1; j < len(chatManage.RerankResult); j++ {
if chatManage.RerankResult[j].Score > chatManage.RerankResult[i].Score {
chatManage.RerankResult[i], chatManage.RerankResult[j] = chatManage.RerankResult[j], chatManage.RerankResult[i]
}
}
}
// Re-sort by score after boosting; stable sort preserves ordering for ties.
sort.SliceStable(chatManage.RerankResult, func(i, j int) bool {
return chatManage.RerankResult[i].Score > chatManage.RerankResult[j].Score
})
}
return nil
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
// KnowledgePostProcessService acts as an orchestrator for all post-processing tasks
@@ -19,6 +20,7 @@ type KnowledgePostProcessService struct {
kbService interfaces.KnowledgeBaseService
chunkService interfaces.ChunkService
taskEnqueuer interfaces.TaskEnqueuer
redisClient *redis.Client
}
func NewKnowledgePostProcessService(
@@ -26,12 +28,14 @@ func NewKnowledgePostProcessService(
kbService interfaces.KnowledgeBaseService,
chunkService interfaces.ChunkService,
taskEnqueuer interfaces.TaskEnqueuer,
redisClient *redis.Client,
) interfaces.TaskHandler {
return &KnowledgePostProcessService{
knowledgeRepo: knowledgeRepo,
kbService: kbService,
chunkService: chunkService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
}
}
@@ -117,7 +121,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// 6. Spawn Wiki Ingest Task for wiki-type knowledge bases
if kb.Type == types.KnowledgeBaseTypeWiki && len(textChunks) > 0 {
EnqueueWikiIngest(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.redisClient, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
}
return nil
@@ -227,12 +227,12 @@ func (s *sessionService) buildAgentConfig(
Thinking: customAgent.Config.Thinking,
RetrieveKBOnlyWhenMentioned: customAgent.Config.RetrieveKBOnlyWhenMentioned,
LLMCallTimeout: customAgent.Config.LLMCallTimeout,
RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory,
}
// Falls back to global configuration if no specific timeout is set for the agent.
if agentConfig.LLMCallTimeout == 0 && s.cfg.Agent != nil && s.cfg.Agent.LLMCallTimeout > 0 {
agentConfig.LLMCallTimeout = s.cfg.Agent.LLMCallTimeout
RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory,
}
// Configure skills based on CustomAgentConfig
+19 -15
View File
@@ -33,11 +33,15 @@ const (
// WikiLintIssue represents a single lint finding
type WikiLintIssue struct {
Type WikiLintIssueType `json:"type"`
Severity WikiLintIssueSeverity `json:"severity"`
PageSlug string `json:"page_slug"`
Description string `json:"description"`
AutoFixable bool `json:"auto_fixable"`
Type WikiLintIssueType `json:"type"`
Severity WikiLintIssueSeverity `json:"severity"`
PageSlug string `json:"page_slug"`
// TargetSlug identifies the other page involved in the issue (e.g. the
// broken link target, or the entity slug for a missing cross-ref). It is
// the structured field used by AutoFix instead of parsing Description.
TargetSlug string `json:"target_slug,omitempty"`
Description string `json:"description"`
AutoFixable bool `json:"auto_fixable"`
}
// WikiLintReport is the complete lint report for a wiki KB
@@ -131,6 +135,7 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe
Type: LintIssueBrokenLink,
Severity: SeverityError,
PageSlug: page.Slug,
TargetSlug: outLink,
Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink),
AutoFixable: true,
})
@@ -178,8 +183,9 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe
Type: LintIssueMissingCrossRef,
Severity: SeverityInfo,
PageSlug: page.Slug,
TargetSlug: slug,
Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug),
AutoFixable: true,
AutoFixable: false,
})
}
}
@@ -275,19 +281,17 @@ func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error)
switch issue.Type {
case LintIssueBrokenLink:
// Remove broken links from page content
// Replace [[broken-slug]] with plain text so the reference text is
// preserved but no longer renders as a dangling wiki link.
if issue.TargetSlug == "" {
continue
}
page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
if err != nil {
continue
}
// Extract the broken slug from the description
parts := strings.Split(issue.Description, "[[")
if len(parts) < 2 {
continue
}
brokenSlug := strings.Split(parts[1], "]]")[0]
// Remove the [[broken-link]] from content
page.Content = strings.ReplaceAll(page.Content, "[["+brokenSlug+"]]", brokenSlug)
target := issue.TargetSlug
page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target)
if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
fixed++
}
@@ -1,9 +1,9 @@
-- Migration: 000032_wiki_pages (rollback)
-- Migration: 000036_wiki_pages (rollback)
-- Description: Remove wiki_pages table and wiki_config column from knowledge_bases.
DO $$ BEGIN RAISE NOTICE '[Migration 000032] Removing wiki_pages table and wiki_config column'; END $$;
DO $$ BEGIN RAISE NOTICE '[Migration 000036] Removing wiki_pages table and wiki_config column'; END $$;
DROP TABLE IF EXISTS wiki_pages;
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS wiki_config;
DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column removed successfully'; END $$;
DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column removed successfully'; END $$;
@@ -1,7 +1,7 @@
-- Migration: 000032_wiki_pages
-- Migration: 000036_wiki_pages
-- Description: Add wiki_pages table and wiki_config column to knowledge_bases.
-- Wiki pages are LLM-generated, interlinked markdown documents that form a persistent wiki.
DO $$ BEGIN RAISE NOTICE '[Migration 000032] Creating wiki_pages table and adding wiki_config column'; END $$;
DO $$ BEGIN RAISE NOTICE '[Migration 000036] Creating wiki_pages table and adding wiki_config column'; END $$;
-- Add wiki_config column to knowledge_bases
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS wiki_config JSONB;
@@ -55,4 +55,4 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_deleted_at
CREATE INDEX IF NOT EXISTS idx_wiki_pages_fulltext
ON wiki_pages USING GIN (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')));
DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column created successfully'; END $$;
DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column created successfully'; END $$;