From 99cc721140390b51cbeaa2783e35b516f71bdf59 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 17 Apr 2026 21:16:35 +0800 Subject: [PATCH] feat: Enhance wiki API and repository with slug encoding and improved search functionality - Introduced `encodeSlugPath` function to safely encode wiki slugs for API requests, preserving hierarchical routing. - Updated API functions to utilize the new slug encoding for `getWikiPage`, `updateWikiPage`, and `deleteWikiPage`. - Enhanced `ListBySourceRef` method in the repository to safely handle source knowledge IDs with JSON marshalling, improving query security. - Added `escapeLikePattern` function to ensure safe concatenation of LIKE patterns in SQL queries, preventing unintended matches. - Updated `KnowledgePostProcessService` to include Redis client for task management, improving performance in handling wiki ingest tasks. - Enhanced `WikiLintIssue` structure to include `TargetSlug` for better issue tracking and auto-fixing capabilities. These changes significantly improve the robustness and security of the wiki management system, enhancing user experience and data integrity. --- frontend/src/api/wiki/index.ts | 14 ++++-- internal/application/repository/wiki_page.go | 47 +++++++++++++++++-- .../service/chat_pipeline/wiki_boost.go | 32 ++++++++----- .../service/knowledge_post_process.go | 6 ++- .../application/service/session_agent_qa.go | 2 +- internal/application/service/wiki_lint.go | 34 ++++++++------ .../versioned/000036_wiki_pages.down.sql | 6 +-- migrations/versioned/000036_wiki_pages.up.sql | 6 +-- 8 files changed, 104 insertions(+), 43 deletions(-) diff --git a/frontend/src/api/wiki/index.ts b/frontend/src/api/wiki/index.ts index 21ac8302..0c8ee283 100644 --- a/frontend/src/api/wiki/index.ts +++ b/frontend/src/api/wiki/index.ts @@ -1,5 +1,13 @@ import { get, post, put, del } from "../../utils/request"; +// encodeSlugPath encodes each segment of a hierarchical wiki slug (e.g. +// "foo/bar baz?") so the URL is safe while preserving the "/" separators +// between segments. Using encodeURIComponent on the whole slug would also +// escape the "/" and break hierarchical routing on the backend. +function encodeSlugPath(slug: string): string { + return slug.split("/").map(encodeURIComponent).join("/"); +} + // Wiki Page Types export interface WikiPage { id: string; @@ -86,15 +94,15 @@ export function createWikiPage(kbId: string, data: Partial) { } export function getWikiPage(kbId: string, slug: string) { - return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`); + return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`); } export function updateWikiPage(kbId: string, slug: string, data: Partial) { - return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`, data); + return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`, data); } export function deleteWikiPage(kbId: string, slug: string) { - return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`); + return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`); } export function getWikiIndex(kbId: string) { diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index b533d7d9..0f2d7d12 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -2,8 +2,10 @@ package repository import ( "context" + "encoding/json" "errors" "fmt" + "strings" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -179,13 +181,35 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy // ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID. // Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array. func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) { + // Build the JSON needle safely so arbitrary IDs cannot break out of the + // quoted string (e.g. ids containing quotes or backslashes). + needle, err := json.Marshal([]string{sourceKnowledgeID}) + if err != nil { + return nil, fmt.Errorf("marshal source ref needle: %w", err) + } + + // For the "knowledgeID|title" prefix form, match against the JSON-encoded + // value: json.Marshal escapes special chars so the LIKE pattern is safe. + prefix, err := json.Marshal(sourceKnowledgeID + "|") + if err != nil { + return nil, fmt.Errorf("marshal source ref prefix: %w", err) + } + // prefix is a JSON string including the surrounding quotes; e.g. "abc|". + // We strip the trailing quote so LIKE can continue into the title portion. + prefixStr := string(prefix) + if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' { + prefixStr = prefixStr[:len(prefixStr)-1] + } + // Escape LIKE metacharacters in the already-JSON-escaped prefix, then wrap + // with %…% to match anywhere in the serialized JSON array. + likePattern := "%" + escapeLikePattern(prefixStr) + "%" + var pages []*types.WikiPage - // Match either exact "knowledgeID" or "knowledgeID|..." prefix in the JSON array if err := r.db.WithContext(ctx). - Where("knowledge_base_id = ? AND (source_refs @> ? OR source_refs::text LIKE ?)", + Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)", kbID, - fmt.Sprintf(`["%s"]`, sourceKnowledgeID), - fmt.Sprintf(`%%"%s|%%`, sourceKnowledgeID), + string(needle), + likePattern, ). Find(&pages).Error; err != nil { return nil, err @@ -233,7 +257,20 @@ func (r *wikiPageRepository) DeleteByID(ctx context.Context, id string) error { return nil } -// Search performs full-text search on wiki pages within a knowledge base +// escapeLikePattern escapes LIKE / ILIKE metacharacters so the returned string +// can be safely concatenated with % wildcards without unintended matches. +// Order matters: escape the backslash first, then the wildcards. +func escapeLikePattern(s string) string { + replacer := strings.NewReplacer( + `\`, `\\`, + `%`, `\%`, + `_`, `\_`, + ) + return replacer.Replace(s) +} + +// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base. +// The query is interpreted as a PostgreSQL regular expression (via ~*). func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) { if limit <= 0 { limit = 10 diff --git a/internal/application/service/chat_pipeline/wiki_boost.go b/internal/application/service/chat_pipeline/wiki_boost.go index c922942a..edd6c93d 100644 --- a/internal/application/service/chat_pipeline/wiki_boost.go +++ b/internal/application/service/chat_pipeline/wiki_boost.go @@ -2,6 +2,7 @@ package chatpipeline import ( "context" + "sort" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -49,16 +50,28 @@ func (p *PluginWikiBoost) OnEvent( return err } - // Check if any search target is a wiki KB + // Fast path: skip all work if there are no wiki_page chunks in the result set. + // This avoids hitting the KB service on every chat turn for non-wiki queries. + hasWikiChunk := false + for i := range chatManage.RerankResult { + if chatManage.RerankResult[i].ChunkType == types.ChunkTypeWikiPage { + hasWikiChunk = true + break + } + } + if !hasWikiChunk { + return nil + } + + // Confirm at least one search target is actually a wiki KB. hasWikiKB := false for _, target := range chatManage.SearchTargets { kb, err := p.kbService.GetKnowledgeBaseByIDOnly(ctx, target.KnowledgeBaseID) - if err == nil && kb.IsWikiEnabled() { + if err == nil && kb != nil && kb.IsWikiEnabled() { hasWikiKB = true break } } - if !hasWikiKB { return nil } @@ -74,15 +87,10 @@ func (p *PluginWikiBoost) OnEvent( if boostedCount > 0 { logger.Infof(ctx, "WikiBoost: boosted %d wiki page chunks by %.1fx", boostedCount, wikiBoostFactor) - - // Re-sort by score after boosting - for i := 0; i < len(chatManage.RerankResult); i++ { - for j := i + 1; j < len(chatManage.RerankResult); j++ { - if chatManage.RerankResult[j].Score > chatManage.RerankResult[i].Score { - chatManage.RerankResult[i], chatManage.RerankResult[j] = chatManage.RerankResult[j], chatManage.RerankResult[i] - } - } - } + // Re-sort by score after boosting; stable sort preserves ordering for ties. + sort.SliceStable(chatManage.RerankResult, func(i, j int) bool { + return chatManage.RerankResult[i].Score > chatManage.RerankResult[j].Score + }) } return nil diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index 2a356311..5dfa0e69 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" ) // KnowledgePostProcessService acts as an orchestrator for all post-processing tasks @@ -19,6 +20,7 @@ type KnowledgePostProcessService struct { kbService interfaces.KnowledgeBaseService chunkService interfaces.ChunkService taskEnqueuer interfaces.TaskEnqueuer + redisClient *redis.Client } func NewKnowledgePostProcessService( @@ -26,12 +28,14 @@ func NewKnowledgePostProcessService( kbService interfaces.KnowledgeBaseService, chunkService interfaces.ChunkService, taskEnqueuer interfaces.TaskEnqueuer, + redisClient *redis.Client, ) interfaces.TaskHandler { return &KnowledgePostProcessService{ knowledgeRepo: knowledgeRepo, kbService: kbService, chunkService: chunkService, taskEnqueuer: taskEnqueuer, + redisClient: redisClient, } } @@ -117,7 +121,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // 6. Spawn Wiki Ingest Task for wiki-type knowledge bases if kb.Type == types.KnowledgeBaseTypeWiki && len(textChunks) > 0 { - EnqueueWikiIngest(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) + EnqueueWikiIngest(ctx, s.taskEnqueuer, s.redisClient, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) } return nil diff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.go index 24c97ba2..4e8592aa 100644 --- a/internal/application/service/session_agent_qa.go +++ b/internal/application/service/session_agent_qa.go @@ -227,12 +227,12 @@ func (s *sessionService) buildAgentConfig( Thinking: customAgent.Config.Thinking, RetrieveKBOnlyWhenMentioned: customAgent.Config.RetrieveKBOnlyWhenMentioned, LLMCallTimeout: customAgent.Config.LLMCallTimeout, + RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory, } // Falls back to global configuration if no specific timeout is set for the agent. if agentConfig.LLMCallTimeout == 0 && s.cfg.Agent != nil && s.cfg.Agent.LLMCallTimeout > 0 { agentConfig.LLMCallTimeout = s.cfg.Agent.LLMCallTimeout - RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory, } // Configure skills based on CustomAgentConfig diff --git a/internal/application/service/wiki_lint.go b/internal/application/service/wiki_lint.go index 4e73fb75..0f438749 100644 --- a/internal/application/service/wiki_lint.go +++ b/internal/application/service/wiki_lint.go @@ -33,11 +33,15 @@ const ( // WikiLintIssue represents a single lint finding type WikiLintIssue struct { - Type WikiLintIssueType `json:"type"` - Severity WikiLintIssueSeverity `json:"severity"` - PageSlug string `json:"page_slug"` - Description string `json:"description"` - AutoFixable bool `json:"auto_fixable"` + Type WikiLintIssueType `json:"type"` + Severity WikiLintIssueSeverity `json:"severity"` + PageSlug string `json:"page_slug"` + // TargetSlug identifies the other page involved in the issue (e.g. the + // broken link target, or the entity slug for a missing cross-ref). It is + // the structured field used by AutoFix instead of parsing Description. + TargetSlug string `json:"target_slug,omitempty"` + Description string `json:"description"` + AutoFixable bool `json:"auto_fixable"` } // WikiLintReport is the complete lint report for a wiki KB @@ -131,6 +135,7 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe Type: LintIssueBrokenLink, Severity: SeverityError, PageSlug: page.Slug, + TargetSlug: outLink, Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink), AutoFixable: true, }) @@ -178,8 +183,9 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe Type: LintIssueMissingCrossRef, Severity: SeverityInfo, PageSlug: page.Slug, + TargetSlug: slug, Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug), - AutoFixable: true, + AutoFixable: false, }) } } @@ -275,19 +281,17 @@ func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error) switch issue.Type { case LintIssueBrokenLink: - // Remove broken links from page content + // Replace [[broken-slug]] with plain text so the reference text is + // preserved but no longer renders as a dangling wiki link. + if issue.TargetSlug == "" { + continue + } page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug) if err != nil { continue } - // Extract the broken slug from the description - parts := strings.Split(issue.Description, "[[") - if len(parts) < 2 { - continue - } - brokenSlug := strings.Split(parts[1], "]]")[0] - // Remove the [[broken-link]] from content - page.Content = strings.ReplaceAll(page.Content, "[["+brokenSlug+"]]", brokenSlug) + target := issue.TargetSlug + page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target) if _, err := s.wikiService.UpdatePage(ctx, page); err == nil { fixed++ } diff --git a/migrations/versioned/000036_wiki_pages.down.sql b/migrations/versioned/000036_wiki_pages.down.sql index 10882ee5..a69575de 100644 --- a/migrations/versioned/000036_wiki_pages.down.sql +++ b/migrations/versioned/000036_wiki_pages.down.sql @@ -1,9 +1,9 @@ --- Migration: 000032_wiki_pages (rollback) +-- Migration: 000036_wiki_pages (rollback) -- Description: Remove wiki_pages table and wiki_config column from knowledge_bases. -DO $$ BEGIN RAISE NOTICE '[Migration 000032] Removing wiki_pages table and wiki_config column'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000036] Removing wiki_pages table and wiki_config column'; END $$; DROP TABLE IF EXISTS wiki_pages; ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS wiki_config; -DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column removed successfully'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column removed successfully'; END $$; diff --git a/migrations/versioned/000036_wiki_pages.up.sql b/migrations/versioned/000036_wiki_pages.up.sql index c4f68bc4..9de360a6 100644 --- a/migrations/versioned/000036_wiki_pages.up.sql +++ b/migrations/versioned/000036_wiki_pages.up.sql @@ -1,7 +1,7 @@ --- Migration: 000032_wiki_pages +-- Migration: 000036_wiki_pages -- Description: Add wiki_pages table and wiki_config column to knowledge_bases. -- Wiki pages are LLM-generated, interlinked markdown documents that form a persistent wiki. -DO $$ BEGIN RAISE NOTICE '[Migration 000032] Creating wiki_pages table and adding wiki_config column'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000036] Creating wiki_pages table and adding wiki_config column'; END $$; -- Add wiki_config column to knowledge_bases ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS wiki_config JSONB; @@ -55,4 +55,4 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_deleted_at CREATE INDEX IF NOT EXISTS idx_wiki_pages_fulltext ON wiki_pages USING GIN (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, ''))); -DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column created successfully'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column created successfully'; END $$;