feat: Enhance wiki API and repository with slug encoding and improved search functionality
- Introduced `encodeSlugPath` function to safely encode wiki slugs for API requests, preserving hierarchical routing. - Updated API functions to utilize the new slug encoding for `getWikiPage`, `updateWikiPage`, and `deleteWikiPage`. - Enhanced `ListBySourceRef` method in the repository to safely handle source knowledge IDs with JSON marshalling, improving query security. - Added `escapeLikePattern` function to ensure safe concatenation of LIKE patterns in SQL queries, preventing unintended matches. - Updated `KnowledgePostProcessService` to include Redis client for task management, improving performance in handling wiki ingest tasks. - Enhanced `WikiLintIssue` structure to include `TargetSlug` for better issue tracking and auto-fixing capabilities. These changes significantly improve the robustness and security of the wiki management system, enhancing user experience and data integrity.
This commit is contained in:
@@ -1,5 +1,13 @@
|
||||
import { get, post, put, del } from "../../utils/request";
|
||||
|
||||
// encodeSlugPath encodes each segment of a hierarchical wiki slug (e.g.
|
||||
// "foo/bar baz?") so the URL is safe while preserving the "/" separators
|
||||
// between segments. Using encodeURIComponent on the whole slug would also
|
||||
// escape the "/" and break hierarchical routing on the backend.
|
||||
function encodeSlugPath(slug: string): string {
|
||||
return slug.split("/").map(encodeURIComponent).join("/");
|
||||
}
|
||||
|
||||
// Wiki Page Types
|
||||
export interface WikiPage {
|
||||
id: string;
|
||||
@@ -86,15 +94,15 @@ export function createWikiPage(kbId: string, data: Partial<WikiPage>) {
|
||||
}
|
||||
|
||||
export function getWikiPage(kbId: string, slug: string) {
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`);
|
||||
return get(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
|
||||
}
|
||||
|
||||
export function updateWikiPage(kbId: string, slug: string, data: Partial<WikiPage>) {
|
||||
return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`, data);
|
||||
return put(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`, data);
|
||||
}
|
||||
|
||||
export function deleteWikiPage(kbId: string, slug: string) {
|
||||
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${slug}`);
|
||||
return del(`/api/v1/knowledgebase/${kbId}/wiki/pages/${encodeSlugPath(slug)}`);
|
||||
}
|
||||
|
||||
export function getWikiIndex(kbId: string) {
|
||||
|
||||
@@ -2,8 +2,10 @@ package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -179,13 +181,35 @@ func (r *wikiPageRepository) ListByType(ctx context.Context, kbID string, pageTy
|
||||
// ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID.
|
||||
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array.
|
||||
func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) {
|
||||
// Build the JSON needle safely so arbitrary IDs cannot break out of the
|
||||
// quoted string (e.g. ids containing quotes or backslashes).
|
||||
needle, err := json.Marshal([]string{sourceKnowledgeID})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal source ref needle: %w", err)
|
||||
}
|
||||
|
||||
// For the "knowledgeID|title" prefix form, match against the JSON-encoded
|
||||
// value: json.Marshal escapes special chars so the LIKE pattern is safe.
|
||||
prefix, err := json.Marshal(sourceKnowledgeID + "|")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal source ref prefix: %w", err)
|
||||
}
|
||||
// prefix is a JSON string including the surrounding quotes; e.g. "abc|".
|
||||
// We strip the trailing quote so LIKE can continue into the title portion.
|
||||
prefixStr := string(prefix)
|
||||
if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' {
|
||||
prefixStr = prefixStr[:len(prefixStr)-1]
|
||||
}
|
||||
// Escape LIKE metacharacters in the already-JSON-escaped prefix, then wrap
|
||||
// with %…% to match anywhere in the serialized JSON array.
|
||||
likePattern := "%" + escapeLikePattern(prefixStr) + "%"
|
||||
|
||||
var pages []*types.WikiPage
|
||||
// Match either exact "knowledgeID" or "knowledgeID|..." prefix in the JSON array
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("knowledge_base_id = ? AND (source_refs @> ? OR source_refs::text LIKE ?)",
|
||||
Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)",
|
||||
kbID,
|
||||
fmt.Sprintf(`["%s"]`, sourceKnowledgeID),
|
||||
fmt.Sprintf(`%%"%s|%%`, sourceKnowledgeID),
|
||||
string(needle),
|
||||
likePattern,
|
||||
).
|
||||
Find(&pages).Error; err != nil {
|
||||
return nil, err
|
||||
@@ -233,7 +257,20 @@ func (r *wikiPageRepository) DeleteByID(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search performs full-text search on wiki pages within a knowledge base
|
||||
// escapeLikePattern escapes LIKE / ILIKE metacharacters so the returned string
|
||||
// can be safely concatenated with % wildcards without unintended matches.
|
||||
// Order matters: escape the backslash first, then the wildcards.
|
||||
func escapeLikePattern(s string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
`\`, `\\`,
|
||||
`%`, `\%`,
|
||||
`_`, `\_`,
|
||||
)
|
||||
return replacer.Replace(s)
|
||||
}
|
||||
|
||||
// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base.
|
||||
// The query is interpreted as a PostgreSQL regular expression (via ~*).
|
||||
func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
|
||||
@@ -2,6 +2,7 @@ package chatpipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -49,16 +50,28 @@ func (p *PluginWikiBoost) OnEvent(
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if any search target is a wiki KB
|
||||
// Fast path: skip all work if there are no wiki_page chunks in the result set.
|
||||
// This avoids hitting the KB service on every chat turn for non-wiki queries.
|
||||
hasWikiChunk := false
|
||||
for i := range chatManage.RerankResult {
|
||||
if chatManage.RerankResult[i].ChunkType == types.ChunkTypeWikiPage {
|
||||
hasWikiChunk = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasWikiChunk {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Confirm at least one search target is actually a wiki KB.
|
||||
hasWikiKB := false
|
||||
for _, target := range chatManage.SearchTargets {
|
||||
kb, err := p.kbService.GetKnowledgeBaseByIDOnly(ctx, target.KnowledgeBaseID)
|
||||
if err == nil && kb.IsWikiEnabled() {
|
||||
if err == nil && kb != nil && kb.IsWikiEnabled() {
|
||||
hasWikiKB = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !hasWikiKB {
|
||||
return nil
|
||||
}
|
||||
@@ -74,15 +87,10 @@ func (p *PluginWikiBoost) OnEvent(
|
||||
|
||||
if boostedCount > 0 {
|
||||
logger.Infof(ctx, "WikiBoost: boosted %d wiki page chunks by %.1fx", boostedCount, wikiBoostFactor)
|
||||
|
||||
// Re-sort by score after boosting
|
||||
for i := 0; i < len(chatManage.RerankResult); i++ {
|
||||
for j := i + 1; j < len(chatManage.RerankResult); j++ {
|
||||
if chatManage.RerankResult[j].Score > chatManage.RerankResult[i].Score {
|
||||
chatManage.RerankResult[i], chatManage.RerankResult[j] = chatManage.RerankResult[j], chatManage.RerankResult[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
// Re-sort by score after boosting; stable sort preserves ordering for ties.
|
||||
sort.SliceStable(chatManage.RerankResult, func(i, j int) bool {
|
||||
return chatManage.RerankResult[i].Score > chatManage.RerankResult[j].Score
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// KnowledgePostProcessService acts as an orchestrator for all post-processing tasks
|
||||
@@ -19,6 +20,7 @@ type KnowledgePostProcessService struct {
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
chunkService interfaces.ChunkService
|
||||
taskEnqueuer interfaces.TaskEnqueuer
|
||||
redisClient *redis.Client
|
||||
}
|
||||
|
||||
func NewKnowledgePostProcessService(
|
||||
@@ -26,12 +28,14 @@ func NewKnowledgePostProcessService(
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
chunkService interfaces.ChunkService,
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
redisClient *redis.Client,
|
||||
) interfaces.TaskHandler {
|
||||
return &KnowledgePostProcessService{
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
kbService: kbService,
|
||||
chunkService: chunkService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
redisClient: redisClient,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,7 +121,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
|
||||
// 6. Spawn Wiki Ingest Task for wiki-type knowledge bases
|
||||
if kb.Type == types.KnowledgeBaseTypeWiki && len(textChunks) > 0 {
|
||||
EnqueueWikiIngest(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
|
||||
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.redisClient, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -227,12 +227,12 @@ func (s *sessionService) buildAgentConfig(
|
||||
Thinking: customAgent.Config.Thinking,
|
||||
RetrieveKBOnlyWhenMentioned: customAgent.Config.RetrieveKBOnlyWhenMentioned,
|
||||
LLMCallTimeout: customAgent.Config.LLMCallTimeout,
|
||||
RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory,
|
||||
}
|
||||
|
||||
// Falls back to global configuration if no specific timeout is set for the agent.
|
||||
if agentConfig.LLMCallTimeout == 0 && s.cfg.Agent != nil && s.cfg.Agent.LLMCallTimeout > 0 {
|
||||
agentConfig.LLMCallTimeout = s.cfg.Agent.LLMCallTimeout
|
||||
RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory,
|
||||
}
|
||||
|
||||
// Configure skills based on CustomAgentConfig
|
||||
|
||||
@@ -33,11 +33,15 @@ const (
|
||||
|
||||
// WikiLintIssue represents a single lint finding
|
||||
type WikiLintIssue struct {
|
||||
Type WikiLintIssueType `json:"type"`
|
||||
Severity WikiLintIssueSeverity `json:"severity"`
|
||||
PageSlug string `json:"page_slug"`
|
||||
Description string `json:"description"`
|
||||
AutoFixable bool `json:"auto_fixable"`
|
||||
Type WikiLintIssueType `json:"type"`
|
||||
Severity WikiLintIssueSeverity `json:"severity"`
|
||||
PageSlug string `json:"page_slug"`
|
||||
// TargetSlug identifies the other page involved in the issue (e.g. the
|
||||
// broken link target, or the entity slug for a missing cross-ref). It is
|
||||
// the structured field used by AutoFix instead of parsing Description.
|
||||
TargetSlug string `json:"target_slug,omitempty"`
|
||||
Description string `json:"description"`
|
||||
AutoFixable bool `json:"auto_fixable"`
|
||||
}
|
||||
|
||||
// WikiLintReport is the complete lint report for a wiki KB
|
||||
@@ -131,6 +135,7 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe
|
||||
Type: LintIssueBrokenLink,
|
||||
Severity: SeverityError,
|
||||
PageSlug: page.Slug,
|
||||
TargetSlug: outLink,
|
||||
Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink),
|
||||
AutoFixable: true,
|
||||
})
|
||||
@@ -178,8 +183,9 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe
|
||||
Type: LintIssueMissingCrossRef,
|
||||
Severity: SeverityInfo,
|
||||
PageSlug: page.Slug,
|
||||
TargetSlug: slug,
|
||||
Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug),
|
||||
AutoFixable: true,
|
||||
AutoFixable: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -275,19 +281,17 @@ func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error)
|
||||
|
||||
switch issue.Type {
|
||||
case LintIssueBrokenLink:
|
||||
// Remove broken links from page content
|
||||
// Replace [[broken-slug]] with plain text so the reference text is
|
||||
// preserved but no longer renders as a dangling wiki link.
|
||||
if issue.TargetSlug == "" {
|
||||
continue
|
||||
}
|
||||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Extract the broken slug from the description
|
||||
parts := strings.Split(issue.Description, "[[")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
brokenSlug := strings.Split(parts[1], "]]")[0]
|
||||
// Remove the [[broken-link]] from content
|
||||
page.Content = strings.ReplaceAll(page.Content, "[["+brokenSlug+"]]", brokenSlug)
|
||||
target := issue.TargetSlug
|
||||
page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target)
|
||||
if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
|
||||
fixed++
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
-- Migration: 000032_wiki_pages (rollback)
|
||||
-- Migration: 000036_wiki_pages (rollback)
|
||||
-- Description: Remove wiki_pages table and wiki_config column from knowledge_bases.
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000032] Removing wiki_pages table and wiki_config column'; END $$;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000036] Removing wiki_pages table and wiki_config column'; END $$;
|
||||
|
||||
DROP TABLE IF EXISTS wiki_pages;
|
||||
|
||||
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS wiki_config;
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column removed successfully'; END $$;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column removed successfully'; END $$;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
-- Migration: 000032_wiki_pages
|
||||
-- Migration: 000036_wiki_pages
|
||||
-- Description: Add wiki_pages table and wiki_config column to knowledge_bases.
|
||||
-- Wiki pages are LLM-generated, interlinked markdown documents that form a persistent wiki.
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000032] Creating wiki_pages table and adding wiki_config column'; END $$;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000036] Creating wiki_pages table and adding wiki_config column'; END $$;
|
||||
|
||||
-- Add wiki_config column to knowledge_bases
|
||||
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS wiki_config JSONB;
|
||||
@@ -55,4 +55,4 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_deleted_at
|
||||
CREATE INDEX IF NOT EXISTS idx_wiki_pages_fulltext
|
||||
ON wiki_pages USING GIN (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')));
|
||||
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000032] wiki_pages table and wiki_config column created successfully'; END $$;
|
||||
DO $$ BEGIN RAISE NOTICE '[Migration 000036] wiki_pages table and wiki_config column created successfully'; END $$;
|
||||
|
||||
Reference in New Issue
Block a user