Files
Enterprise-Knowledge-Base/internal/application/service/wiki_ingest.go
T
wizardchen a75233c90f refactor: Enhance deduplication logic and logging in wiki ingestion service
- Updated the `WikiDeduplicationPrompt` to clarify the criteria for identifying exact duplicates, emphasizing strict matching of real-world entities and concepts.
- Improved the `appendLogEntry` method to include a summary parameter for better context in log entries, enhancing traceability of operations.
- Consolidated log entry creation for both retraction and ingestion operations, ensuring chronological clarity in the log.
- Implemented validation for slug merges to prevent mismatches based on entity types, improving the accuracy of deduplication.

These changes enhance the clarity and reliability of the wiki ingestion process, leading to more accurate content management.
2026-04-22 21:17:34 +08:00

1109 lines
34 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"text/template"
"time"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
const (
// maxContentForWiki limits the document content sent to LLM for wiki generation
maxContentForWiki = 32768
// wikiPendingKeyPrefix is the Redis key prefix for pending wiki ingest document lists.
// Key format: wiki:pending:{kbID} → Redis List of knowledge IDs.
wikiPendingKeyPrefix = "wiki:pending:"
// wikiActiveKeyPrefix is the Redis key for the "batch in progress" flag.
// Key format: wiki:active:{kbID} → "1" with TTL. Prevents concurrent batches.
wikiActiveKeyPrefix = "wiki:active:"
// wikiIngestDelay is how long to wait after a document is added before
// the batch task fires. Debounces rapid uploads.
wikiIngestDelay = 30 * time.Second
// wikiPendingTTL prevents stale pending lists from accumulating.
wikiPendingTTL = 24 * time.Hour
// wikiMaxDocsPerBatch limits how many documents a single batch processes.
// Prevents unbounded execution time. Remaining docs stay in the pending list
// and are picked up by the follow-up task.
wikiMaxDocsPerBatch = 5
)
// WikiIngestPayload is the asynq task payload for wiki ingest batch trigger.
// The actual document IDs are stored in a Redis list (wiki:pending:{kbID}).
// KnowledgeID is only used as fallback in Lite mode (no Redis).
type WikiIngestPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Language string `json:"language,omitempty"`
// Fallback for Lite mode (no Redis)
LiteOps []WikiPendingOp `json:"lite_ops,omitempty"`
}
// WikiRetractPayload is the asynq task payload for wiki content retraction
type WikiRetractPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
DocTitle string `json:"doc_title"`
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
Language string `json:"language,omitempty"`
PageSlugs []string `json:"page_slugs"`
}
const (
WikiOpIngest = "ingest"
WikiOpRetract = "retract"
)
// WikiPendingOp represents a single operation in the Redis pending queue
type WikiPendingOp struct {
Op string `json:"op"`
KnowledgeID string `json:"knowledge_id"`
// Ingest fields
Language string `json:"language,omitempty"`
// Retract fields
DocTitle string `json:"doc_title,omitempty"`
DocSummary string `json:"doc_summary,omitempty"`
PageSlugs []string `json:"page_slugs,omitempty"`
}
// wikiIngestService handles the LLM-powered wiki generation pipeline
type wikiIngestService struct {
wikiService interfaces.WikiPageService
kbService interfaces.KnowledgeBaseService
knowledgeSvc interfaces.KnowledgeService
chunkRepo interfaces.ChunkRepository
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
redisClient *redis.Client // nil in Lite mode (no Redis)
}
// NewWikiIngestService creates a new wiki ingest service
func NewWikiIngestService(
wikiService interfaces.WikiPageService,
kbService interfaces.KnowledgeBaseService,
knowledgeSvc interfaces.KnowledgeService,
chunkRepo interfaces.ChunkRepository,
modelService interfaces.ModelService,
task interfaces.TaskEnqueuer,
redisClient *redis.Client,
) interfaces.TaskHandler {
svc := &wikiIngestService{
wikiService: wikiService,
kbService: kbService,
knowledgeSvc: knowledgeSvc,
chunkRepo: chunkRepo,
modelService: modelService,
task: task,
redisClient: redisClient,
}
return svc
}
// EnqueueWikiIngest adds a document to the wiki ingest queue.
//
// Architecture: each document upload pushes its knowledgeID to a Redis pending list,
// then schedules a delayed asynq task. When the task fires, it atomically drains the
// entire list and processes ALL pending documents in one batch.
//
// If multiple uploads happen within the delay window (30s), each one schedules a task,
// but the FIRST task to fire drains the list and processes everything. Subsequent tasks
// fire, find an empty list, and exit immediately (no-op). This gives us natural batching
// without any locks or task deduplication.
//
// t=0s doc1 → RPush + Enqueue(delay=30s, id=random1)
// t=5s doc2 → RPush + Enqueue(delay=30s, id=random2)
// t=10s doc3 → RPush + Enqueue(delay=30s, id=random3)
// t=30s random1 fires → drain [doc1,doc2,doc3] → process all
// t=35s random2 fires → drain [] → no-op return
// t=40s random3 fires → drain [] → no-op return
//
// In Lite mode (no Redis), falls back to immediate per-document execution.
func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, tenantID uint64, kbID, knowledgeID string) {
lang, _ := types.LanguageFromContext(ctx)
payload := WikiIngestPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
Language: lang,
}
// Push to Redis pending list (if Redis available)
if redisClient != nil {
pendingKey := wikiPendingKeyPrefix + kbID
op := WikiPendingOp{
Op: WikiOpIngest,
KnowledgeID: knowledgeID,
Language: lang,
}
opBytes, _ := json.Marshal(op)
redisClient.RPush(ctx, pendingKey, string(opBytes))
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
} else {
// Fallback for Lite mode (no Redis)
payload.LiteOps = []WikiPendingOp{{
Op: WikiOpIngest,
KnowledgeID: knowledgeID,
Language: lang,
}}
}
payloadBytes, _ := json.Marshal(payload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10), // Increased from 3 to 10 to ensure it can outlast the 5-minute active lock TTL
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiIngestDelay),
)
if _, err := task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to enqueue task: %v", err)
}
}
// EnqueueWikiRetract enqueues an async wiki content retraction task
func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, payload WikiRetractPayload) {
ingestPayload := WikiIngestPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Language: payload.Language,
}
op := WikiPendingOp{
Op: WikiOpRetract,
KnowledgeID: payload.KnowledgeID,
DocTitle: payload.DocTitle,
DocSummary: payload.DocSummary,
PageSlugs: payload.PageSlugs,
Language: payload.Language,
}
if redisClient != nil {
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
opBytes, _ := json.Marshal(op)
redisClient.RPush(ctx, pendingKey, string(opBytes))
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
} else {
// Fallback for Lite mode (no Redis)
ingestPayload.LiteOps = []WikiPendingOp{op}
}
payloadBytes, _ := json.Marshal(ingestPayload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly
)
if _, err := task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki retract: failed to enqueue task: %v", err)
}
}
// Handle implements interfaces.TaskHandler for asynq task processing.
// Wiki ingest tasks are debounced via asynq.Unique + ProcessIn, so at most
// one ingest task runs per KB at a time. No distributed lock needed.
func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error {
return s.ProcessWikiIngest(ctx, t)
}
// peekPendingList gets up to wikiMaxDocsPerBatch entries from the Redis pending list
// WITHOUT removing them. It returns the unique ops and the actual number of items peeked.
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string) ([]WikiPendingOp, int) {
if s.redisClient == nil {
return nil, 0
}
pendingKey := wikiPendingKeyPrefix + kbID
result, err := s.redisClient.LRange(ctx, pendingKey, 0, wikiMaxDocsPerBatch-1).Result()
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err)
return nil, 0
}
var ops []WikiPendingOp
for _, item := range result {
if !strings.HasPrefix(item, "{") {
// Backward compatibility: raw knowledgeID string
ops = append(ops, WikiPendingOp{
Op: WikiOpIngest,
KnowledgeID: item,
})
continue
}
var op WikiPendingOp
if err := json.Unmarshal([]byte(item), &op); err == nil {
ops = append(ops, op)
}
}
// Deduplicate by KnowledgeID, keeping only the *last* operation for each document.
// This optimizes out redundant sequences (e.g., upload then immediate delete: [ingest, retract] -> [retract]).
seen := make(map[string]bool)
var reversedUnique []WikiPendingOp
for i := len(ops) - 1; i >= 0; i-- {
op := ops[i]
if !seen[op.KnowledgeID] {
seen[op.KnowledgeID] = true
reversedUnique = append(reversedUnique, op)
}
}
// Reverse back to maintain chronological order
var unique []WikiPendingOp
for i := len(reversedUnique) - 1; i >= 0; i-- {
unique = append(unique, reversedUnique[i])
}
return unique, len(result)
}
// trimPendingList removes the first `count` items from the Redis pending list.
func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, count int) {
if s.redisClient == nil || count <= 0 {
return
}
pendingKey := wikiPendingKeyPrefix + kbID
if err := s.redisClient.LTrim(ctx, pendingKey, int64(count), -1).Err(); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to trim pending list: %v", err)
}
}
// docIngestResult captures per-document info for batch post-processing.
type docIngestResult struct {
KnowledgeID string
DocTitle string
Summary string // one-line summary of the document (from summary page)
Pages []string // affected page slugs
}
// WikiBatchContext holds shared data across Map and Reduce phases
type WikiBatchContext struct {
AllPages []*types.WikiPage
SlugTitleMap map[string]string
SummaryContentByKnowledgeID map[string]string
}
// SlugUpdate represents a single update operation for a specific slug
type SlugUpdate struct {
Slug string
Type string // "entity", "concept", "summary", "retract", "retractStale"
Item extractedItem // For entity/concept
DocTitle string
KnowledgeID string
SourceRef string
Language string
SummaryBody string // For summary
SummaryLine string // For summary
RetractDocContent string // For retract / retractStale
}
func previewText(s string, maxRunes int) string {
s = strings.TrimSpace(s)
s = strings.ReplaceAll(s, "\n", " ")
s = strings.ReplaceAll(s, "\t", " ")
for strings.Contains(s, " ") {
s = strings.ReplaceAll(s, " ", " ")
}
r := []rune(s)
if maxRunes <= 0 || len(r) <= maxRunes {
return s
}
return string(r[:maxRunes]) + "...(truncated)"
}
func previewStringSlice(items []string, limit int) string {
if len(items) == 0 {
return "[]"
}
if limit <= 0 {
limit = 1
}
n := len(items)
if n > limit {
items = items[:limit]
}
out := make([]string, 0, len(items))
for _, it := range items {
out = append(out, previewText(it, 48))
}
if n > limit {
return fmt.Sprintf("[%s ...(+%d)]", strings.Join(out, ", "), n-limit)
}
return fmt.Sprintf("[%s]", strings.Join(out, ", "))
}
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
// Scans all published pages, checks each out_link, and removes references to
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return
}
// Build set of live (non-archived, non-system) slugs
liveSlugs := make(map[string]bool)
for _, p := range allPages {
if p.Status != types.WikiPageStatusArchived {
liveSlugs[p.Slug] = true
}
}
var cleaned int
for _, p := range allPages {
if p.Status == types.WikiPageStatusArchived {
continue
}
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
content := p.Content
changed := false
// Find all [[slug]] references and remove dead ones
for _, outSlug := range p.OutLinks {
if liveSlugs[outSlug] {
continue // link is alive
}
// Dead link — remove the [[slug]] from content, keep the display text if any
linkPattern := "[[" + outSlug + "]]"
if strings.Contains(content, linkPattern) {
// Replace [[dead-slug]] with just the slug's readable part
parts := strings.Split(outSlug, "/")
readableName := parts[len(parts)-1]
readableName = strings.ReplaceAll(readableName, "-", " ")
content = strings.ReplaceAll(content, linkPattern, readableName)
changed = true
}
}
if changed {
p.Content = content
if _, err := s.wikiService.UpdatePage(ctx, p); err != nil {
logger.Warnf(ctx, "wiki: failed to clean dead links in page %s: %v", p.Slug, err)
} else {
cleaned++
}
}
}
if cleaned > 0 {
logger.Infof(ctx, "wiki: cleaned dead links in %d pages", cleaned)
}
}
// injectCrossLinks scans affected pages and injects [[wiki-links]] for mentions
// of other wiki page titles in the content. Pure text replacement, no LLM call.
// Processes entity/concept/synthesis/comparison and summary pages (not index/log).
func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) {
// Build a title→slug lookup from ALL pages in this KB
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) < 2 {
return
}
type pageRef struct {
slug string
matchText string
}
var allRefs []pageRef
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
if p.Title != "" {
allRefs = append(allRefs, pageRef{slug: p.Slug, matchText: p.Title})
}
for _, alias := range p.Aliases {
if alias != "" {
allRefs = append(allRefs, pageRef{slug: p.Slug, matchText: alias})
}
}
}
if len(allRefs) == 0 {
return
}
// Sort by title length descending — match longer names first to avoid
// partial matches (e.g. "北京邮电大学" before "北京")
for i := 0; i < len(allRefs); i++ {
for j := i + 1; j < len(allRefs); j++ {
if len([]rune(allRefs[j].matchText)) > len([]rune(allRefs[i].matchText)) {
allRefs[i], allRefs[j] = allRefs[j], allRefs[i]
}
}
}
// Process only the pages we just created/updated
affectedSet := make(map[string]bool, len(affectedSlugs))
for _, s := range affectedSlugs {
affectedSet[s] = true
}
var updated int
for _, p := range allPages {
if !affectedSet[p.Slug] {
continue
}
// Skip system pages (index, log). We allow summary to be processed so it gets links to other KB entities.
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
content := p.Content
changed := false
for _, ref := range allRefs {
if ref.slug == p.Slug {
continue
}
// Skip if already linked with this slug (either [[slug]] or [[slug|...]])
if strings.Contains(content, "[["+ref.slug+"|") || strings.Contains(content, "[["+ref.slug+"]]") {
continue
}
if strings.Contains(content, ref.matchText) {
content = replaceFirstOutsideLinks(content, ref.matchText, "[["+ref.slug+"|"+ref.matchText+"]]")
changed = true
}
}
if changed {
p.Content = content
if _, err := s.wikiService.UpdatePage(ctx, p); err != nil {
logger.Warnf(ctx, "wiki ingest: cross-link injection failed for %s: %v", p.Slug, err)
} else {
updated++
}
}
}
if updated > 0 {
logger.Infof(ctx, "wiki ingest: injected cross-links in %d pages", updated)
}
}
// replaceFirstOutsideLinks replaces the first occurrence of `old` with `new` in s,
// but only if it's not already inside a [[...]] wiki link.
func replaceFirstOutsideLinks(s, old, newStr string) string {
idx := 0
for {
pos := strings.Index(s[idx:], old)
if pos < 0 {
return s // not found
}
absPos := idx + pos
// Check if this occurrence is inside a [[ ... ]] link
// Look backwards for [[ without encountering ]]
insideLink := false
for i := absPos - 1; i >= 0 && i >= absPos-200; i-- {
if i > 0 && s[i-1:i+1] == "]]" {
break // closed link before us
}
if i > 0 && s[i-1:i+1] == "[[" {
insideLink = true
break
}
}
if !insideLink {
return s[:absPos] + newStr + s[absPos+len(old):]
}
// Skip this match, try next
idx = absPos + len(old)
}
}
// getExistingPageSlugsForKnowledge returns all page slugs that currently reference
// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest.
func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return nil
}
slugs := make(map[string]bool)
prefix := knowledgeID + "|"
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
for _, ref := range p.SourceRefs {
if ref == knowledgeID || strings.HasPrefix(ref, prefix) {
slugs[p.Slug] = true
break
}
}
}
return slugs
}
// retractStalePages handles pages that were previously linked to this document
// but are no longer produced by the updated extraction.
// - Single-source stale pages → deleted
// - Multi-source stale pages → LLM retract to clean content synchronously
// Build set of newly affected slugs (including summary)
// Stale = was in old set but not in new set
// Remove this doc's source ref
// No other sources → delete the page
// Multi-source → remove ref, queue retract
// extractedItem represents a single extracted entity or concept
type extractedItem struct {
Name string `json:"name"`
Slug string `json:"slug"`
Aliases []string `json:"aliases"`
Description string `json:"description"`
Details string `json:"details"`
}
// combinedExtraction represents the parsed result of the combined entity+concept extraction
type combinedExtraction struct {
Entities []extractedItem `json:"entities"`
Concepts []extractedItem `json:"concepts"`
}
// rebuildIndexPage regenerates the index page.
//
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
// Subsequent: incrementally updated with changeDescription.
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
if indexPage == nil {
return nil
}
// List all live pages
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
if err != nil {
return err
}
typeOrder := []string{
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
}
typeLabels := map[string]string{
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
types.WikiPageTypeComparison: "Comparison",
}
grouped := make(map[string][]*types.WikiPage)
totalPages := 0
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
if p.Status == types.WikiPageStatusArchived {
continue
}
grouped[p.PageType] = append(grouped[p.PageType], p)
totalPages++
}
// Build document summaries listing (only summary-type pages — they represent documents)
var docSummaries strings.Builder
for _, p := range grouped[types.WikiPageTypeSummary] {
fmt.Fprintf(&docSummaries, "<document>\n<title>%s</title>\n<summary>%s</summary>\n</document>\n\n", p.Title, p.Summary)
}
if docSummaries.Len() == 0 {
docSummaries.WriteString("(no documents yet)")
}
// Generate or update intro
existingIntro := indexPage.Summary
var intro string
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
// First time — generate intro from scratch
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
"DocumentSummaries": docSummaries.String(),
"Language": lang,
})
if genErr != nil {
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
} else {
intro = strings.TrimSpace(generatedIntro)
}
} else if changeDesc != "" {
// Incremental update — tell LLM what changed
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
"ExistingIntro": existingIntro,
"ChangeDescription": changeDesc,
"DocumentSummaries": docSummaries.String(),
"Language": lang,
})
if genErr != nil {
intro = existingIntro // keep existing on error
} else {
intro = strings.TrimSpace(updatedIntro)
}
} else {
intro = existingIntro // no change description, keep as-is
}
// Build directory (pure code, no LLM)
var dir strings.Builder
for _, pt := range typeOrder {
pages := grouped[pt]
if len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
for _, p := range pages {
summary := p.Summary
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
}
}
for pt, pages := range grouped {
inOrder := false
for _, o := range typeOrder {
if o == pt {
inOrder = true
break
}
}
if inOrder || len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
for _, p := range pages {
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
}
}
if totalPages == 0 {
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
}
indexPage.Content = intro + "\n" + dir.String()
indexPage.Summary = intro // persist intro for next incremental update
_, err = s.wikiService.UpdatePage(ctx, indexPage)
return err
}
// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output.
// Returns (summary, content). If no SUMMARY line found, summary is empty.
func splitSummaryLine(raw string) (summary string, content string) {
raw = strings.TrimSpace(raw)
if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") {
idx := strings.IndexByte(raw, '\n')
if idx < 0 {
// Only one line
return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), ""
}
summaryLine := raw[:idx]
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:])
}
return "", raw
}
// appendLogEntry appends a structured, grep-parseable entry to the log page.
// Format: ## [2026-04-07 19:50:02] action | title
// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean.
func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) {
logPage, _ := s.wikiService.GetLog(ctx, kbID)
if logPage == nil {
return
}
var sb strings.Builder
fmt.Fprintf(&sb, "\n## [%s] %s | %s\n",
time.Now().Format("2006-01-02 15:04:05"),
action,
docTitle,
)
if knowledgeID != "" {
fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID)
}
if summary != "" {
fmt.Fprintf(&sb, "- **Summary**: %s\n", summary)
}
if len(pagesAffected) > 0 {
fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", "))
}
logPage.Content = logPage.Content + sb.String()
if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to update log page: %v", err)
}
}
// publishDraftPages transitions draft pages to published status after ingest completes.
// This ensures users don't see half-built pages during the ingest process.
func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, slugs []string) {
for _, slug := range slugs {
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
if err != nil || page == nil {
continue
}
if page.Status == types.WikiPageStatusDraft {
page.Status = types.WikiPageStatusPublished
if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to publish page %s: %v", slug, err)
}
}
}
}
// deduplicateExtractedBatch deduplicates both entities and concepts against
// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid
// redundant DB queries. This replaces the two separate deduplicateItems calls
// that each queried ListAllPages + made a separate LLM call.
func (s *wikiIngestService) deduplicateExtractedBatch(
ctx context.Context,
chatModel chat.Chat,
entities, concepts []extractedItem,
allPages []*types.WikiPage,
) ([]extractedItem, []extractedItem) {
if len(entities) == 0 && len(concepts) == 0 {
return entities, concepts
}
var existingBuf strings.Builder
for _, p := range allPages {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
aliases := ""
if len(p.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(p.Aliases, ", "))
}
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s | type: %s%s\n", p.Slug, p.Title, p.PageType, aliases)
}
if existingBuf.Len() == 0 {
return entities, concepts
}
var newBuf strings.Builder
for _, item := range entities {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
}
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: entity%s\n", item.Slug, item.Name, aliases)
}
for _, item := range concepts {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
}
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: concept%s\n", item.Slug, item.Name, aliases)
}
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
"NewItems": newBuf.String(),
"ExistingPages": existingBuf.String(),
})
if err != nil {
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
return entities, concepts
}
dedupeJSON = cleanLLMJSON(dedupeJSON)
var dedupeResult struct {
Merges map[string]string `json:"merges"`
}
if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON)
return entities, concepts
}
if len(dedupeResult.Merges) == 0 {
return entities, concepts
}
existingSlugs := make(map[string]bool, len(allPages))
for _, p := range allPages {
existingSlugs[p.Slug] = true
}
validMerge := func(srcSlug, dstSlug string) bool {
if !existingSlugs[dstSlug] {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug)
return false
}
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
if srcPrefix != dstPrefix {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
return false
}
return true
}
for i, item := range entities {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
entities[i].Slug = existingSlug
}
}
for i, item := range concepts {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
concepts[i].Slug = existingSlug
}
}
return entities, concepts
}
// generateWithTemplate executes a prompt template and calls the LLM
func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) {
tmpl, err := template.New("wiki").Parse(promptTpl)
if err != nil {
return "", fmt.Errorf("parse template: %w", err)
}
var buf strings.Builder
if err := tmpl.Execute(&buf, data); err != nil {
return "", fmt.Errorf("execute template: %w", err)
}
prompt := buf.String()
thinking := false
response, err := chatModel.Chat(ctx, []chat.Message{
{Role: "user", Content: prompt},
}, &chat.ChatOptions{
Temperature: 0.3,
Thinking: &thinking,
})
if err != nil {
return "", fmt.Errorf("LLM call failed: %w", err)
}
return response.Content, nil
}
// --- Helpers ---
// reconstructContent rebuilds document text from chunks
func reconstructContent(chunks []*types.Chunk) string {
var textChunks []*types.Chunk
for _, c := range chunks {
if c.ChunkType == types.ChunkTypeText || c.ChunkType == "" {
textChunks = append(textChunks, c)
}
}
// Sort by StartAt, then ChunkIndex
sort.Slice(textChunks, func(i, j int) bool {
if textChunks[i].StartAt == textChunks[j].StartAt {
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
}
return textChunks[i].StartAt < textChunks[j].StartAt
})
var sb strings.Builder
lastEndAt := -1
for _, c := range textChunks {
toAppend := c.Content
if c.StartAt > lastEndAt || c.EndAt == 0 {
// Non-overlapping or missing position info
if sb.Len() > 0 {
sb.WriteString("\n")
}
sb.WriteString(toAppend)
if c.EndAt > 0 {
lastEndAt = c.EndAt
}
} else if c.EndAt > lastEndAt {
// Partial overlap
contentRunes := []rune(toAppend)
offset := len(contentRunes) - (c.EndAt - lastEndAt)
if offset >= 0 && offset < len(contentRunes) {
sb.WriteString(string(contentRunes[offset:]))
} else {
// Fallback if offset calculation is invalid
if sb.Len() > 0 {
sb.WriteString("\n")
}
sb.WriteString(toAppend)
}
lastEndAt = c.EndAt
}
// If c.EndAt <= lastEndAt, it's fully contained, so skip appending text
}
// Append image information at the end to avoid interrupting text flow
var hasImages bool
seenURLs := make(map[string]bool)
for _, c := range textChunks {
if c.ImageInfo != "" {
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(c.ImageInfo), &imageInfos); err == nil && len(imageInfos) > 0 {
for _, img := range imageInfos {
// Deduplicate images by URL to avoid printing the same image multiple times from overlapping chunks
if img.URL != "" {
if seenURLs[img.URL] {
continue
}
seenURLs[img.URL] = true
}
if !hasImages {
sb.WriteString("\n\n<images>\n")
hasImages = true
} else {
sb.WriteString("\n")
}
sb.WriteString("<image>\n")
if img.URL != "" {
sb.WriteString(fmt.Sprintf(" <url>%s</url>\n", img.URL))
}
if img.Caption != "" {
sb.WriteString(fmt.Sprintf(" <caption>%s</caption>\n", img.Caption))
}
if img.OCRText != "" {
sb.WriteString(fmt.Sprintf(" <ocr_text>%s</ocr_text>\n", img.OCRText))
}
sb.WriteString("</image>\n")
}
}
}
}
if hasImages {
sb.WriteString("</images>\n")
}
return sb.String()
}
// slugify creates a URL-friendly slug from a string
func slugify(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
s = strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '/' {
return r
}
if r == ' ' || r == '_' {
return '-'
}
// Keep CJK characters
if r >= 0x4E00 && r <= 0x9FFF {
return r
}
return -1
}, s)
// Collapse multiple hyphens
for strings.Contains(s, "--") {
s = strings.ReplaceAll(s, "--", "-")
}
s = strings.Trim(s, "-")
if len(s) > 200 {
s = s[:200]
}
return s
}
// truncateString truncates a string to maxLen runes
func truncateString(s string, maxLen int) string {
runes := []rune(s)
if len(runes) <= maxLen {
return s
}
return string(runes[:maxLen]) + "..."
}
// appendUnique appends a string to a StringArray if not already present
func appendUnique(arr types.StringArray, s string) types.StringArray {
for _, v := range arr {
if v == s {
return arr
}
}
return append(arr, s)
}
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
// from LLM-generated JSON output so it can be safely unmarshalled.
func cleanLLMJSON(s string) string {
s = strings.TrimSpace(s)
s = strings.TrimPrefix(s, "```json")
s = strings.TrimPrefix(s, "```")
s = strings.TrimSuffix(s, "```")
s = strings.TrimSpace(s)
return sanitizeJSONString(s)
}
// sanitizeJSONString sanitizes a string that is intended to be parsed as JSON,
// by properly escaping unescaped control characters (like newlines) inside string literals.
func sanitizeJSONString(s string) string {
var buf strings.Builder
buf.Grow(len(s))
inString := false
escape := false
for _, r := range s {
if escape {
if r == '\n' {
buf.WriteString(`n`)
} else if r == '\r' {
buf.WriteString(`r`)
} else if r == '\t' {
buf.WriteString(`t`)
} else {
buf.WriteRune(r)
}
escape = false
continue
}
if r == '\\' {
escape = true
buf.WriteRune(r)
continue
}
if r == '"' {
inString = !inString
buf.WriteRune(r)
continue
}
if inString {
if r == '\n' {
buf.WriteString(`\n`)
continue
}
if r == '\r' {
buf.WriteString(`\r`)
continue
}
if r == '\t' {
buf.WriteString(`\t`)
continue
}
}
buf.WriteRune(r)
}
return buf.String()
}