fdccb68ee4
- Enhanced the `cleanupWikiOnKnowledgeDelete` function to pass the full knowledge object, allowing for more accurate title and summary retrieval during wiki cleanup. - Introduced a tombstone mechanism in Redis to mark recently deleted knowledge, enabling in-flight wiki ingest tasks to skip processing for deleted documents. - Updated the `isKnowledgeGone` function to check for deleted knowledge using the tombstone, improving efficiency in handling concurrent delete and ingest operations. - Added logic in the `wikiIngestService` to filter out updates for deleted knowledge, ensuring that no ghost references are created during the ingest process. - Implemented lint checks for stale source references in the `WikiLintService`, allowing for automatic cleanup of pages referencing deleted knowledge. These changes significantly enhance the robustness of the wiki system, ensuring consistency and accuracy in handling knowledge deletions and related wiki pages.
1149 lines
36 KiB
Go
1149 lines
36 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"text/template"
|
||
"time"
|
||
|
||
"github.com/Tencent/WeKnora/internal/agent"
|
||
"github.com/Tencent/WeKnora/internal/logger"
|
||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||
"github.com/Tencent/WeKnora/internal/searchutil"
|
||
"github.com/Tencent/WeKnora/internal/types"
|
||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||
"github.com/hibiken/asynq"
|
||
"github.com/redis/go-redis/v9"
|
||
)
|
||
|
||
const (
|
||
// maxContentForWiki limits the document content sent to LLM for wiki generation
|
||
maxContentForWiki = 32768
|
||
|
||
// wikiPendingKeyPrefix is the Redis key prefix for pending wiki ingest document lists.
|
||
// Key format: wiki:pending:{kbID} → Redis List of knowledge IDs.
|
||
wikiPendingKeyPrefix = "wiki:pending:"
|
||
|
||
// wikiActiveKeyPrefix is the Redis key for the "batch in progress" flag.
|
||
// Key format: wiki:active:{kbID} → "1" with TTL. Prevents concurrent batches.
|
||
wikiActiveKeyPrefix = "wiki:active:"
|
||
|
||
// wikiIngestDelay is how long to wait after a document is added before
|
||
// the batch task fires. Debounces rapid uploads.
|
||
wikiIngestDelay = 30 * time.Second
|
||
|
||
// wikiPendingTTL prevents stale pending lists from accumulating.
|
||
wikiPendingTTL = 24 * time.Hour
|
||
|
||
// wikiMaxDocsPerBatch limits how many documents a single batch processes.
|
||
// Prevents unbounded execution time. Remaining docs stay in the pending list
|
||
// and are picked up by the follow-up task.
|
||
wikiMaxDocsPerBatch = 5
|
||
|
||
// wikiDeletedKeyPrefix is the Redis key prefix for "recently deleted
|
||
// knowledge" tombstones. Key: wiki:deleted:{kbID}:{knowledgeID}. Written
|
||
// by cleanupWikiOnKnowledgeDelete so that any wiki_ingest task still in
|
||
// flight (or queued) for this knowledge can fast-path skip without
|
||
// hitting the DB. TTL > wikiIngestDelay so it's guaranteed to outlast
|
||
// any in-flight ingest.
|
||
wikiDeletedKeyPrefix = "wiki:deleted:"
|
||
|
||
// wikiDeletedTTL bounds how long we remember a deletion. Must comfortably
|
||
// exceed the longest plausible ingest run (LLM extraction + reduce).
|
||
wikiDeletedTTL = 1 * time.Hour
|
||
)
|
||
|
||
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
|
||
// recently deleted, so wiki_ingest tasks in flight can short-circuit. Exposed
|
||
// so knowledgeService.cleanupWikiOnKnowledgeDelete can write the same key
|
||
// without duplicating the format string.
|
||
func WikiDeletedTombstoneKey(kbID, knowledgeID string) string {
|
||
return wikiDeletedKeyPrefix + kbID + ":" + knowledgeID
|
||
}
|
||
|
||
// WikiIngestPayload is the asynq task payload for wiki ingest batch trigger.
|
||
// The actual document IDs are stored in a Redis list (wiki:pending:{kbID}).
|
||
// KnowledgeID is only used as fallback in Lite mode (no Redis).
|
||
type WikiIngestPayload struct {
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
Language string `json:"language,omitempty"`
|
||
// Fallback for Lite mode (no Redis)
|
||
LiteOps []WikiPendingOp `json:"lite_ops,omitempty"`
|
||
}
|
||
|
||
// WikiRetractPayload is the asynq task payload for wiki content retraction
|
||
type WikiRetractPayload struct {
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
DocTitle string `json:"doc_title"`
|
||
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
|
||
Language string `json:"language,omitempty"`
|
||
PageSlugs []string `json:"page_slugs"`
|
||
}
|
||
|
||
const (
|
||
WikiOpIngest = "ingest"
|
||
WikiOpRetract = "retract"
|
||
)
|
||
|
||
// WikiPendingOp represents a single operation in the Redis pending queue
|
||
type WikiPendingOp struct {
|
||
Op string `json:"op"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
// Ingest fields
|
||
Language string `json:"language,omitempty"`
|
||
// Retract fields
|
||
DocTitle string `json:"doc_title,omitempty"`
|
||
DocSummary string `json:"doc_summary,omitempty"`
|
||
PageSlugs []string `json:"page_slugs,omitempty"`
|
||
}
|
||
|
||
// wikiIngestService handles the LLM-powered wiki generation pipeline
|
||
type wikiIngestService struct {
|
||
wikiService interfaces.WikiPageService
|
||
kbService interfaces.KnowledgeBaseService
|
||
knowledgeSvc interfaces.KnowledgeService
|
||
chunkRepo interfaces.ChunkRepository
|
||
modelService interfaces.ModelService
|
||
task interfaces.TaskEnqueuer
|
||
redisClient *redis.Client // nil in Lite mode (no Redis)
|
||
}
|
||
|
||
// NewWikiIngestService creates a new wiki ingest service
|
||
func NewWikiIngestService(
|
||
wikiService interfaces.WikiPageService,
|
||
kbService interfaces.KnowledgeBaseService,
|
||
knowledgeSvc interfaces.KnowledgeService,
|
||
chunkRepo interfaces.ChunkRepository,
|
||
modelService interfaces.ModelService,
|
||
task interfaces.TaskEnqueuer,
|
||
redisClient *redis.Client,
|
||
) interfaces.TaskHandler {
|
||
svc := &wikiIngestService{
|
||
wikiService: wikiService,
|
||
kbService: kbService,
|
||
knowledgeSvc: knowledgeSvc,
|
||
chunkRepo: chunkRepo,
|
||
modelService: modelService,
|
||
task: task,
|
||
redisClient: redisClient,
|
||
}
|
||
return svc
|
||
}
|
||
|
||
// EnqueueWikiIngest adds a document to the wiki ingest queue.
|
||
//
|
||
// Architecture: each document upload pushes its knowledgeID to a Redis pending list,
|
||
// then schedules a delayed asynq task. When the task fires, it atomically drains the
|
||
// entire list and processes ALL pending documents in one batch.
|
||
//
|
||
// If multiple uploads happen within the delay window (30s), each one schedules a task,
|
||
// but the FIRST task to fire drains the list and processes everything. Subsequent tasks
|
||
// fire, find an empty list, and exit immediately (no-op). This gives us natural batching
|
||
// without any locks or task deduplication.
|
||
//
|
||
// t=0s doc1 → RPush + Enqueue(delay=30s, id=random1)
|
||
// t=5s doc2 → RPush + Enqueue(delay=30s, id=random2)
|
||
// t=10s doc3 → RPush + Enqueue(delay=30s, id=random3)
|
||
// t=30s random1 fires → drain [doc1,doc2,doc3] → process all
|
||
// t=35s random2 fires → drain [] → no-op return
|
||
// t=40s random3 fires → drain [] → no-op return
|
||
//
|
||
// In Lite mode (no Redis), falls back to immediate per-document execution.
|
||
func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, tenantID uint64, kbID, knowledgeID string) {
|
||
lang, _ := types.LanguageFromContext(ctx)
|
||
|
||
payload := WikiIngestPayload{
|
||
TenantID: tenantID,
|
||
KnowledgeBaseID: kbID,
|
||
Language: lang,
|
||
}
|
||
|
||
// Push to Redis pending list (if Redis available)
|
||
if redisClient != nil {
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
op := WikiPendingOp{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: knowledgeID,
|
||
Language: lang,
|
||
}
|
||
opBytes, _ := json.Marshal(op)
|
||
redisClient.RPush(ctx, pendingKey, string(opBytes))
|
||
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
|
||
} else {
|
||
// Fallback for Lite mode (no Redis)
|
||
payload.LiteOps = []WikiPendingOp{{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: knowledgeID,
|
||
Language: lang,
|
||
}}
|
||
}
|
||
|
||
payloadBytes, _ := json.Marshal(payload)
|
||
|
||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||
asynq.Queue("low"),
|
||
asynq.MaxRetry(10), // Increased from 3 to 10 to ensure it can outlast the 5-minute active lock TTL
|
||
asynq.Timeout(60*time.Minute),
|
||
asynq.ProcessIn(wikiIngestDelay),
|
||
)
|
||
if _, err := task.Enqueue(t); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to enqueue task: %v", err)
|
||
}
|
||
}
|
||
|
||
// EnqueueWikiRetract enqueues an async wiki content retraction task
|
||
func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, payload WikiRetractPayload) {
|
||
ingestPayload := WikiIngestPayload{
|
||
TenantID: payload.TenantID,
|
||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||
Language: payload.Language,
|
||
}
|
||
|
||
op := WikiPendingOp{
|
||
Op: WikiOpRetract,
|
||
KnowledgeID: payload.KnowledgeID,
|
||
DocTitle: payload.DocTitle,
|
||
DocSummary: payload.DocSummary,
|
||
PageSlugs: payload.PageSlugs,
|
||
Language: payload.Language,
|
||
}
|
||
|
||
if redisClient != nil {
|
||
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
|
||
opBytes, _ := json.Marshal(op)
|
||
redisClient.RPush(ctx, pendingKey, string(opBytes))
|
||
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
|
||
} else {
|
||
// Fallback for Lite mode (no Redis)
|
||
ingestPayload.LiteOps = []WikiPendingOp{op}
|
||
}
|
||
|
||
payloadBytes, _ := json.Marshal(ingestPayload)
|
||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||
asynq.Queue("low"),
|
||
asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
|
||
asynq.Timeout(60*time.Minute),
|
||
asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly
|
||
)
|
||
if _, err := task.Enqueue(t); err != nil {
|
||
logger.Warnf(ctx, "wiki retract: failed to enqueue task: %v", err)
|
||
}
|
||
}
|
||
|
||
// Handle implements interfaces.TaskHandler for asynq task processing.
|
||
// Wiki ingest tasks are debounced via asynq.Unique + ProcessIn, so at most
|
||
// one ingest task runs per KB at a time. No distributed lock needed.
|
||
func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error {
|
||
return s.ProcessWikiIngest(ctx, t)
|
||
}
|
||
|
||
// peekPendingList gets up to wikiMaxDocsPerBatch entries from the Redis pending list
|
||
// WITHOUT removing them. It returns the unique ops and the actual number of items peeked.
|
||
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string) ([]WikiPendingOp, int) {
|
||
if s.redisClient == nil {
|
||
return nil, 0
|
||
}
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
|
||
result, err := s.redisClient.LRange(ctx, pendingKey, 0, wikiMaxDocsPerBatch-1).Result()
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err)
|
||
return nil, 0
|
||
}
|
||
|
||
var ops []WikiPendingOp
|
||
for _, item := range result {
|
||
if !strings.HasPrefix(item, "{") {
|
||
// Backward compatibility: raw knowledgeID string
|
||
ops = append(ops, WikiPendingOp{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: item,
|
||
})
|
||
continue
|
||
}
|
||
var op WikiPendingOp
|
||
if err := json.Unmarshal([]byte(item), &op); err == nil {
|
||
ops = append(ops, op)
|
||
}
|
||
}
|
||
|
||
// Deduplicate by KnowledgeID, keeping only the *last* operation for each document.
|
||
// This optimizes out redundant sequences (e.g., upload then immediate delete: [ingest, retract] -> [retract]).
|
||
seen := make(map[string]bool)
|
||
var reversedUnique []WikiPendingOp
|
||
for i := len(ops) - 1; i >= 0; i-- {
|
||
op := ops[i]
|
||
if !seen[op.KnowledgeID] {
|
||
seen[op.KnowledgeID] = true
|
||
reversedUnique = append(reversedUnique, op)
|
||
}
|
||
}
|
||
|
||
// Reverse back to maintain chronological order
|
||
var unique []WikiPendingOp
|
||
for i := len(reversedUnique) - 1; i >= 0; i-- {
|
||
unique = append(unique, reversedUnique[i])
|
||
}
|
||
|
||
return unique, len(result)
|
||
}
|
||
|
||
// trimPendingList removes the first `count` items from the Redis pending list.
|
||
func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, count int) {
|
||
if s.redisClient == nil || count <= 0 {
|
||
return
|
||
}
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
if err := s.redisClient.LTrim(ctx, pendingKey, int64(count), -1).Err(); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to trim pending list: %v", err)
|
||
}
|
||
}
|
||
|
||
// docIngestResult captures per-document info for batch post-processing.
|
||
type docIngestResult struct {
|
||
KnowledgeID string
|
||
DocTitle string
|
||
Summary string // one-line summary of the document (from summary page)
|
||
Pages []string // affected page slugs
|
||
}
|
||
|
||
// WikiBatchContext holds shared data across Map and Reduce phases
|
||
type WikiBatchContext struct {
|
||
AllPages []*types.WikiPage
|
||
SlugTitleMap map[string]string
|
||
SummaryContentByKnowledgeID map[string]string
|
||
}
|
||
|
||
// SlugUpdate represents a single update operation for a specific slug
|
||
type SlugUpdate struct {
|
||
Slug string
|
||
Type string // "entity", "concept", "summary", "retract", "retractStale"
|
||
Item extractedItem // For entity/concept
|
||
DocTitle string
|
||
KnowledgeID string
|
||
SourceRef string
|
||
Language string
|
||
SummaryBody string // For summary
|
||
SummaryLine string // For summary
|
||
RetractDocContent string // For retract / retractStale
|
||
}
|
||
|
||
func previewText(s string, maxRunes int) string {
|
||
s = strings.TrimSpace(s)
|
||
s = strings.ReplaceAll(s, "\n", " ")
|
||
s = strings.ReplaceAll(s, "\t", " ")
|
||
for strings.Contains(s, " ") {
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
}
|
||
r := []rune(s)
|
||
if maxRunes <= 0 || len(r) <= maxRunes {
|
||
return s
|
||
}
|
||
return string(r[:maxRunes]) + "...(truncated)"
|
||
}
|
||
|
||
func previewStringSlice(items []string, limit int) string {
|
||
if len(items) == 0 {
|
||
return "[]"
|
||
}
|
||
if limit <= 0 {
|
||
limit = 1
|
||
}
|
||
n := len(items)
|
||
if n > limit {
|
||
items = items[:limit]
|
||
}
|
||
out := make([]string, 0, len(items))
|
||
for _, it := range items {
|
||
out = append(out, previewText(it, 48))
|
||
}
|
||
if n > limit {
|
||
return fmt.Sprintf("[%s ...(+%d)]", strings.Join(out, ", "), n-limit)
|
||
}
|
||
return fmt.Sprintf("[%s]", strings.Join(out, ", "))
|
||
}
|
||
|
||
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
|
||
// Scans all published pages, checks each out_link, and removes references to
|
||
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
|
||
func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) == 0 {
|
||
return
|
||
}
|
||
|
||
// Build set of live (non-archived, non-system) slugs
|
||
liveSlugs := make(map[string]bool)
|
||
for _, p := range allPages {
|
||
if p.Status != types.WikiPageStatusArchived {
|
||
liveSlugs[p.Slug] = true
|
||
}
|
||
}
|
||
|
||
var cleaned int
|
||
for _, p := range allPages {
|
||
if p.Status == types.WikiPageStatusArchived {
|
||
continue
|
||
}
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
|
||
content := p.Content
|
||
changed := false
|
||
|
||
// Find all [[slug]] references and remove dead ones
|
||
for _, outSlug := range p.OutLinks {
|
||
if liveSlugs[outSlug] {
|
||
continue // link is alive
|
||
}
|
||
// Dead link — remove the [[slug]] from content, keep the display text if any
|
||
linkPattern := "[[" + outSlug + "]]"
|
||
if strings.Contains(content, linkPattern) {
|
||
// Replace [[dead-slug]] with just the slug's readable part
|
||
parts := strings.Split(outSlug, "/")
|
||
readableName := parts[len(parts)-1]
|
||
readableName = strings.ReplaceAll(readableName, "-", " ")
|
||
content = strings.ReplaceAll(content, linkPattern, readableName)
|
||
changed = true
|
||
}
|
||
}
|
||
|
||
if changed {
|
||
p.Content = content
|
||
if _, err := s.wikiService.UpdatePage(ctx, p); err != nil {
|
||
logger.Warnf(ctx, "wiki: failed to clean dead links in page %s: %v", p.Slug, err)
|
||
} else {
|
||
cleaned++
|
||
}
|
||
}
|
||
}
|
||
|
||
if cleaned > 0 {
|
||
logger.Infof(ctx, "wiki: cleaned dead links in %d pages", cleaned)
|
||
}
|
||
}
|
||
|
||
// injectCrossLinks scans affected pages and injects [[wiki-links]] for mentions
|
||
// of other wiki page titles in the content. Pure text replacement, no LLM call.
|
||
// Processes entity/concept/synthesis/comparison and summary pages (not index/log).
|
||
//
|
||
// The actual matching is delegated to linkifyContent, which handles code-block
|
||
// and existing-link exclusions plus ASCII word-boundary checks.
|
||
func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) < 2 {
|
||
return
|
||
}
|
||
|
||
refs := collectLinkRefs(allPages)
|
||
if len(refs) == 0 {
|
||
return
|
||
}
|
||
|
||
affectedSet := make(map[string]bool, len(affectedSlugs))
|
||
for _, slug := range affectedSlugs {
|
||
affectedSet[slug] = true
|
||
}
|
||
|
||
var updated int
|
||
for _, p := range allPages {
|
||
if !affectedSet[p.Slug] {
|
||
continue
|
||
}
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
|
||
newContent, changed := linkifyContent(p.Content, refs, p.Slug)
|
||
if !changed {
|
||
continue
|
||
}
|
||
p.Content = newContent
|
||
if _, err := s.wikiService.UpdatePage(ctx, p); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: cross-link injection failed for %s: %v", p.Slug, err)
|
||
continue
|
||
}
|
||
updated++
|
||
}
|
||
|
||
if updated > 0 {
|
||
logger.Infof(ctx, "wiki ingest: injected cross-links in %d pages", updated)
|
||
}
|
||
}
|
||
|
||
// collectLinkRefs flattens (title + aliases) of all non-system pages into a
|
||
// single linkRef slice suitable for linkifyContent.
|
||
func collectLinkRefs(pages []*types.WikiPage) []linkRef {
|
||
refs := make([]linkRef, 0, len(pages)*2)
|
||
for _, p := range pages {
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
if p.Title != "" {
|
||
refs = append(refs, linkRef{slug: p.Slug, matchText: p.Title})
|
||
}
|
||
for _, alias := range p.Aliases {
|
||
if alias != "" {
|
||
refs = append(refs, linkRef{slug: p.Slug, matchText: alias})
|
||
}
|
||
}
|
||
}
|
||
return refs
|
||
}
|
||
|
||
// getExistingPageSlugsForKnowledge returns all page slugs that currently reference
|
||
// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest.
|
||
func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) == 0 {
|
||
return nil
|
||
}
|
||
|
||
slugs := make(map[string]bool)
|
||
prefix := knowledgeID + "|"
|
||
for _, p := range allPages {
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
for _, ref := range p.SourceRefs {
|
||
if ref == knowledgeID || strings.HasPrefix(ref, prefix) {
|
||
slugs[p.Slug] = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return slugs
|
||
}
|
||
|
||
// retractStalePages handles pages that were previously linked to this document
|
||
// but are no longer produced by the updated extraction.
|
||
// - Single-source stale pages → deleted
|
||
// - Multi-source stale pages → LLM retract to clean content synchronously
|
||
|
||
// Build set of newly affected slugs (including summary)
|
||
|
||
// Stale = was in old set but not in new set
|
||
|
||
// Remove this doc's source ref
|
||
|
||
// No other sources → delete the page
|
||
|
||
// Multi-source → remove ref, queue retract
|
||
|
||
// extractedItem represents a single extracted entity or concept
|
||
type extractedItem struct {
|
||
Name string `json:"name"`
|
||
Slug string `json:"slug"`
|
||
Aliases []string `json:"aliases"`
|
||
Description string `json:"description"`
|
||
Details string `json:"details"`
|
||
}
|
||
|
||
// combinedExtraction represents the parsed result of the combined entity+concept extraction
|
||
type combinedExtraction struct {
|
||
Entities []extractedItem `json:"entities"`
|
||
Concepts []extractedItem `json:"concepts"`
|
||
}
|
||
|
||
// rebuildIndexPage regenerates the index page.
|
||
//
|
||
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
|
||
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
|
||
// Subsequent: incrementally updated with changeDescription.
|
||
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
|
||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
|
||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||
if indexPage == nil {
|
||
return nil
|
||
}
|
||
|
||
// List all live pages
|
||
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
typeOrder := []string{
|
||
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
|
||
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
|
||
}
|
||
typeLabels := map[string]string{
|
||
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
|
||
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
|
||
types.WikiPageTypeComparison: "Comparison",
|
||
}
|
||
|
||
grouped := make(map[string][]*types.WikiPage)
|
||
totalPages := 0
|
||
for _, p := range allPages {
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
if p.Status == types.WikiPageStatusArchived {
|
||
continue
|
||
}
|
||
grouped[p.PageType] = append(grouped[p.PageType], p)
|
||
totalPages++
|
||
}
|
||
|
||
// Build document summaries listing (only summary-type pages — they represent documents)
|
||
var docSummaries strings.Builder
|
||
for _, p := range grouped[types.WikiPageTypeSummary] {
|
||
fmt.Fprintf(&docSummaries, "<document>\n<title>%s</title>\n<summary>%s</summary>\n</document>\n\n", p.Title, p.Summary)
|
||
}
|
||
if docSummaries.Len() == 0 {
|
||
docSummaries.WriteString("(no documents yet)")
|
||
}
|
||
|
||
// Generate or update intro
|
||
existingIntro := indexPage.Summary
|
||
var intro string
|
||
|
||
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
|
||
// First time — generate intro from scratch
|
||
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
|
||
"DocumentSummaries": docSummaries.String(),
|
||
"Language": lang,
|
||
})
|
||
if genErr != nil {
|
||
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
|
||
} else {
|
||
intro = strings.TrimSpace(generatedIntro)
|
||
}
|
||
} else if changeDesc != "" {
|
||
// Incremental update — tell LLM what changed
|
||
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
|
||
"ExistingIntro": existingIntro,
|
||
"ChangeDescription": changeDesc,
|
||
"DocumentSummaries": docSummaries.String(),
|
||
"Language": lang,
|
||
})
|
||
if genErr != nil {
|
||
intro = existingIntro // keep existing on error
|
||
} else {
|
||
intro = strings.TrimSpace(updatedIntro)
|
||
}
|
||
} else {
|
||
intro = existingIntro // no change description, keep as-is
|
||
}
|
||
|
||
// Build directory (pure code, no LLM)
|
||
var dir strings.Builder
|
||
for _, pt := range typeOrder {
|
||
pages := grouped[pt]
|
||
if len(pages) == 0 {
|
||
continue
|
||
}
|
||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
|
||
for _, p := range pages {
|
||
summary := p.Summary
|
||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
|
||
}
|
||
}
|
||
for pt, pages := range grouped {
|
||
inOrder := false
|
||
for _, o := range typeOrder {
|
||
if o == pt {
|
||
inOrder = true
|
||
break
|
||
}
|
||
}
|
||
if inOrder || len(pages) == 0 {
|
||
continue
|
||
}
|
||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
|
||
for _, p := range pages {
|
||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
|
||
}
|
||
}
|
||
if totalPages == 0 {
|
||
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
|
||
}
|
||
|
||
indexPage.Content = intro + "\n" + dir.String()
|
||
indexPage.Summary = intro // persist intro for next incremental update
|
||
_, err = s.wikiService.UpdatePage(ctx, indexPage)
|
||
return err
|
||
}
|
||
|
||
// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output.
|
||
// Returns (summary, content). If no SUMMARY line found, summary is empty.
|
||
func splitSummaryLine(raw string) (summary string, content string) {
|
||
raw = strings.TrimSpace(raw)
|
||
if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") {
|
||
idx := strings.IndexByte(raw, '\n')
|
||
if idx < 0 {
|
||
// Only one line
|
||
return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), ""
|
||
}
|
||
summaryLine := raw[:idx]
|
||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||
return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:])
|
||
}
|
||
return "", raw
|
||
}
|
||
|
||
// appendLogEntry appends a structured, grep-parseable entry to the log page.
|
||
// Format: ## [2026-04-07 19:50:02] action | title
|
||
// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean.
|
||
func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) {
|
||
logPage, _ := s.wikiService.GetLog(ctx, kbID)
|
||
if logPage == nil {
|
||
return
|
||
}
|
||
|
||
var sb strings.Builder
|
||
fmt.Fprintf(&sb, "\n## [%s] %s | %s\n",
|
||
time.Now().Format("2006-01-02 15:04:05"),
|
||
action,
|
||
docTitle,
|
||
)
|
||
if knowledgeID != "" {
|
||
fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID)
|
||
}
|
||
if summary != "" {
|
||
fmt.Fprintf(&sb, "- **Summary**: %s\n", summary)
|
||
}
|
||
if len(pagesAffected) > 0 {
|
||
fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", "))
|
||
}
|
||
|
||
logPage.Content = logPage.Content + sb.String()
|
||
if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to update log page: %v", err)
|
||
}
|
||
}
|
||
|
||
// publishDraftPages transitions draft pages to published status after ingest completes.
|
||
// This ensures users don't see half-built pages during the ingest process.
|
||
func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, slugs []string) {
|
||
for _, slug := range slugs {
|
||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
|
||
if err != nil || page == nil {
|
||
continue
|
||
}
|
||
if page.Status == types.WikiPageStatusDraft {
|
||
page.Status = types.WikiPageStatusPublished
|
||
if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to publish page %s: %v", slug, err)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// writeDedupItemXML renders a single entity/concept entry as a structured XML
|
||
// block for the deduplication prompt. Structured form (versus a single
|
||
// pipe-separated line) helps the LLM reliably tell name / aliases / type apart
|
||
// and reduces nonsensical merges like "居民身份证" → "工作居住证".
|
||
func writeDedupItemXML(buf *strings.Builder, slug, name, itemType string, aliases []string) {
|
||
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", slug, itemType)
|
||
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(name))
|
||
for _, alias := range aliases {
|
||
if alias == "" {
|
||
continue
|
||
}
|
||
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
|
||
}
|
||
buf.WriteString(" </item>\n")
|
||
}
|
||
|
||
// xmlEscape escapes the minimal set of characters that can break XML text
|
||
// content. Slugs are ASCII-only so they don't need escaping when used as
|
||
// attribute values.
|
||
func xmlEscape(s string) string {
|
||
s = strings.ReplaceAll(s, "&", "&")
|
||
s = strings.ReplaceAll(s, "<", "<")
|
||
s = strings.ReplaceAll(s, ">", ">")
|
||
return s
|
||
}
|
||
|
||
// deduplicateExtractedBatch deduplicates both entities and concepts against
|
||
// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid
|
||
// redundant DB queries. This replaces the two separate deduplicateItems calls
|
||
// that each queried ListAllPages + made a separate LLM call.
|
||
func (s *wikiIngestService) deduplicateExtractedBatch(
|
||
ctx context.Context,
|
||
chatModel chat.Chat,
|
||
entities, concepts []extractedItem,
|
||
allPages []*types.WikiPage,
|
||
) ([]extractedItem, []extractedItem) {
|
||
if len(entities) == 0 && len(concepts) == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
var existingBuf strings.Builder
|
||
for _, p := range allPages {
|
||
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
|
||
continue
|
||
}
|
||
writeDedupItemXML(&existingBuf, p.Slug, p.Title, string(p.PageType), []string(p.Aliases))
|
||
}
|
||
if existingBuf.Len() == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
var newBuf strings.Builder
|
||
for _, item := range entities {
|
||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "entity", item.Aliases)
|
||
}
|
||
for _, item := range concepts {
|
||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "concept", item.Aliases)
|
||
}
|
||
|
||
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
|
||
"NewItems": newBuf.String(),
|
||
"ExistingPages": existingBuf.String(),
|
||
})
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
|
||
return entities, concepts
|
||
}
|
||
|
||
dedupeJSON = cleanLLMJSON(dedupeJSON)
|
||
|
||
var dedupeResult struct {
|
||
Merges map[string]string `json:"merges"`
|
||
}
|
||
if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON)
|
||
return entities, concepts
|
||
}
|
||
|
||
if len(dedupeResult.Merges) == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
existingSlugs := make(map[string]bool, len(allPages))
|
||
for _, p := range allPages {
|
||
existingSlugs[p.Slug] = true
|
||
}
|
||
|
||
validMerge := func(srcSlug, dstSlug string) bool {
|
||
if !existingSlugs[dstSlug] {
|
||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug)
|
||
return false
|
||
}
|
||
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
|
||
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
|
||
if srcPrefix != dstPrefix {
|
||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
for i, item := range entities {
|
||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
|
||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||
entities[i].Slug = existingSlug
|
||
}
|
||
}
|
||
for i, item := range concepts {
|
||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
|
||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||
concepts[i].Slug = existingSlug
|
||
}
|
||
}
|
||
|
||
return entities, concepts
|
||
}
|
||
|
||
// generateWithTemplate executes a prompt template and calls the LLM
|
||
func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) {
|
||
tmpl, err := template.New("wiki").Parse(promptTpl)
|
||
if err != nil {
|
||
return "", fmt.Errorf("parse template: %w", err)
|
||
}
|
||
|
||
var buf strings.Builder
|
||
if err := tmpl.Execute(&buf, data); err != nil {
|
||
return "", fmt.Errorf("execute template: %w", err)
|
||
}
|
||
|
||
prompt := buf.String()
|
||
thinking := false
|
||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||
{Role: "user", Content: prompt},
|
||
}, &chat.ChatOptions{
|
||
Temperature: 0.3,
|
||
Thinking: &thinking,
|
||
})
|
||
if err != nil {
|
||
return "", fmt.Errorf("LLM call failed: %w", err)
|
||
}
|
||
|
||
return response.Content, nil
|
||
}
|
||
|
||
// --- Helpers ---
|
||
|
||
// isKnowledgeGone returns true if the given knowledge has been deleted or is
|
||
// in the middle of being deleted. It first consults the Redis tombstone
|
||
// (written by cleanupWikiOnKnowledgeDelete) as a fast path, then falls back
|
||
// to the DB. A nil result from GetKnowledgeByIDOnly also counts as gone: the
|
||
// repo layer uses GORM First() which filters soft-deleted rows, so a
|
||
// soft-deleted knowledge surfaces as "not found" here — exactly what we want.
|
||
func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) bool {
|
||
if knowledgeID == "" {
|
||
return true
|
||
}
|
||
if s.redisClient != nil {
|
||
if exists, err := s.redisClient.Exists(ctx, WikiDeletedTombstoneKey(kbID, knowledgeID)).Result(); err == nil && exists > 0 {
|
||
return true
|
||
}
|
||
}
|
||
kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID)
|
||
if err != nil || kn == nil {
|
||
return true
|
||
}
|
||
return kn.ParseStatus == types.ParseStatusDeleting
|
||
}
|
||
|
||
// filterLiveUpdates drops additions/summaries whose source knowledge has been
|
||
// deleted since the Map phase finished. Retract updates are preserved so
|
||
// pages still get cleaned up. Caches per-knowledge results to avoid DB
|
||
// hammering when a single reduce slug carries many updates for the same doc.
|
||
func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string, updates []SlugUpdate) []SlugUpdate {
|
||
if len(updates) == 0 {
|
||
return updates
|
||
}
|
||
goneCache := make(map[string]bool)
|
||
isGone := func(kid string) bool {
|
||
if kid == "" {
|
||
return false
|
||
}
|
||
if v, ok := goneCache[kid]; ok {
|
||
return v
|
||
}
|
||
v := s.isKnowledgeGone(ctx, kbID, kid)
|
||
goneCache[kid] = v
|
||
return v
|
||
}
|
||
filtered := make([]SlugUpdate, 0, len(updates))
|
||
dropped := 0
|
||
for _, u := range updates {
|
||
switch u.Type {
|
||
case "retract", "retractStale":
|
||
filtered = append(filtered, u)
|
||
default:
|
||
if isGone(u.KnowledgeID) {
|
||
dropped++
|
||
continue
|
||
}
|
||
filtered = append(filtered, u)
|
||
}
|
||
}
|
||
if dropped > 0 {
|
||
logger.Infof(ctx, "wiki ingest: reduce dropped %d updates for deleted knowledge(s)", dropped)
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
// reconstructContent rebuilds document text from chunks.
|
||
//
|
||
// This only concatenates text-type chunks — image OCR / caption information is
|
||
// stored on image_ocr / image_caption child chunks (see image_multimodal.go),
|
||
// not on the parent text chunk's ImageInfo field. Callers that need the full
|
||
// enriched content (with OCR / captions inlined) should call
|
||
// reconstructEnrichedContent instead so image info is fetched from child
|
||
// chunks and embedded alongside Markdown image links.
|
||
func reconstructContent(chunks []*types.Chunk) string {
|
||
var textChunks []*types.Chunk
|
||
for _, c := range chunks {
|
||
if c.ChunkType == types.ChunkTypeText || c.ChunkType == "" {
|
||
textChunks = append(textChunks, c)
|
||
}
|
||
}
|
||
|
||
// Sort by StartAt, then ChunkIndex
|
||
sort.Slice(textChunks, func(i, j int) bool {
|
||
if textChunks[i].StartAt == textChunks[j].StartAt {
|
||
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
|
||
}
|
||
return textChunks[i].StartAt < textChunks[j].StartAt
|
||
})
|
||
|
||
var sb strings.Builder
|
||
lastEndAt := -1
|
||
for _, c := range textChunks {
|
||
toAppend := c.Content
|
||
|
||
if c.StartAt > lastEndAt || c.EndAt == 0 {
|
||
// Non-overlapping or missing position info
|
||
if sb.Len() > 0 {
|
||
sb.WriteString("\n")
|
||
}
|
||
sb.WriteString(toAppend)
|
||
if c.EndAt > 0 {
|
||
lastEndAt = c.EndAt
|
||
}
|
||
} else if c.EndAt > lastEndAt {
|
||
// Partial overlap
|
||
contentRunes := []rune(toAppend)
|
||
offset := len(contentRunes) - (c.EndAt - lastEndAt)
|
||
if offset >= 0 && offset < len(contentRunes) {
|
||
sb.WriteString(string(contentRunes[offset:]))
|
||
} else {
|
||
// Fallback if offset calculation is invalid
|
||
if sb.Len() > 0 {
|
||
sb.WriteString("\n")
|
||
}
|
||
sb.WriteString(toAppend)
|
||
}
|
||
lastEndAt = c.EndAt
|
||
}
|
||
// If c.EndAt <= lastEndAt, it's fully contained, so skip appending text
|
||
}
|
||
|
||
return sb.String()
|
||
}
|
||
|
||
// reconstructEnrichedContent rebuilds document text and inlines image_info
|
||
// (OCR text + caption) pulled from image_ocr / image_caption child chunks.
|
||
//
|
||
// Without this enrichment, image-heavy documents (e.g. a scanned PDF or a
|
||
// standalone .jpg) reach the LLM as bare Markdown image links, causing
|
||
// extraction / summarization to produce empty or "no textual content" output.
|
||
func reconstructEnrichedContent(
|
||
ctx context.Context,
|
||
chunkRepo interfaces.ChunkRepository,
|
||
tenantID uint64,
|
||
chunks []*types.Chunk,
|
||
) string {
|
||
content := reconstructContent(chunks)
|
||
|
||
var textChunkIDs []string
|
||
for _, c := range chunks {
|
||
if c.ChunkType == types.ChunkTypeText || c.ChunkType == "" {
|
||
if c.ID != "" {
|
||
textChunkIDs = append(textChunkIDs, c.ID)
|
||
}
|
||
}
|
||
}
|
||
if len(textChunkIDs) == 0 || chunkRepo == nil {
|
||
return content
|
||
}
|
||
|
||
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, chunkRepo, tenantID, textChunkIDs)
|
||
mergedImageInfo := searchutil.MergeImageInfoJSON(imageInfoMap)
|
||
if mergedImageInfo == "" {
|
||
return content
|
||
}
|
||
return searchutil.EnrichContentWithImageInfo(content, mergedImageInfo)
|
||
}
|
||
|
||
// slugify creates a URL-friendly slug from a string
|
||
func slugify(s string) string {
|
||
s = strings.ToLower(strings.TrimSpace(s))
|
||
s = strings.Map(func(r rune) rune {
|
||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '/' {
|
||
return r
|
||
}
|
||
if r == ' ' || r == '_' {
|
||
return '-'
|
||
}
|
||
// Keep CJK characters
|
||
if r >= 0x4E00 && r <= 0x9FFF {
|
||
return r
|
||
}
|
||
return -1
|
||
}, s)
|
||
// Collapse multiple hyphens
|
||
for strings.Contains(s, "--") {
|
||
s = strings.ReplaceAll(s, "--", "-")
|
||
}
|
||
s = strings.Trim(s, "-")
|
||
if len(s) > 200 {
|
||
s = s[:200]
|
||
}
|
||
return s
|
||
}
|
||
|
||
// truncateString truncates a string to maxLen runes
|
||
func truncateString(s string, maxLen int) string {
|
||
runes := []rune(s)
|
||
if len(runes) <= maxLen {
|
||
return s
|
||
}
|
||
return string(runes[:maxLen]) + "..."
|
||
}
|
||
|
||
// appendUnique appends a string to a StringArray if not already present
|
||
func appendUnique(arr types.StringArray, s string) types.StringArray {
|
||
for _, v := range arr {
|
||
if v == s {
|
||
return arr
|
||
}
|
||
}
|
||
return append(arr, s)
|
||
}
|
||
|
||
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
|
||
// from LLM-generated JSON output so it can be safely unmarshalled.
|
||
func cleanLLMJSON(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
s = strings.TrimPrefix(s, "```json")
|
||
s = strings.TrimPrefix(s, "```")
|
||
s = strings.TrimSuffix(s, "```")
|
||
s = strings.TrimSpace(s)
|
||
return sanitizeJSONString(s)
|
||
}
|
||
|
||
// sanitizeJSONString sanitizes a string that is intended to be parsed as JSON,
|
||
// by properly escaping unescaped control characters (like newlines) inside string literals.
|
||
func sanitizeJSONString(s string) string {
|
||
var buf strings.Builder
|
||
buf.Grow(len(s))
|
||
inString := false
|
||
escape := false
|
||
for _, r := range s {
|
||
if escape {
|
||
if r == '\n' {
|
||
buf.WriteString(`n`)
|
||
} else if r == '\r' {
|
||
buf.WriteString(`r`)
|
||
} else if r == '\t' {
|
||
buf.WriteString(`t`)
|
||
} else {
|
||
buf.WriteRune(r)
|
||
}
|
||
escape = false
|
||
continue
|
||
}
|
||
if r == '\\' {
|
||
escape = true
|
||
buf.WriteRune(r)
|
||
continue
|
||
}
|
||
if r == '"' {
|
||
inString = !inString
|
||
buf.WriteRune(r)
|
||
continue
|
||
}
|
||
if inString {
|
||
if r == '\n' {
|
||
buf.WriteString(`\n`)
|
||
continue
|
||
}
|
||
if r == '\r' {
|
||
buf.WriteString(`\r`)
|
||
continue
|
||
}
|
||
if r == '\t' {
|
||
buf.WriteString(`\t`)
|
||
continue
|
||
}
|
||
}
|
||
buf.WriteRune(r)
|
||
}
|
||
return buf.String()
|
||
}
|