b7388f10e1
When a doc's entity/concept page generation hit a transient LLM error in
the reduce phase, the page never got written, but the doc's summary page
(generated in parallel with the slug list baked into its content) was
already persisted with [[entity/foo|name]] links pointing at the missing
page. The wiki log feed also surfaced the failed slugs as clickable
entries that 404'd. cleanDeadLinks only ran for retract batches, so the
debris persisted indefinitely.
reduceSlugUpdates now reports addition-path failures back to the batch
driver, which collects them and:
- rewrites this batch's summary pages, replacing dead [[slug|display]]
refs with plain display text (or a humanized slug tail);
- filters dead slugs out of docResults[].Pages before the wiki log
entries are flushed;
- broadens the cleanDeadLinks trigger to fire whenever any page was
touched or any slug failed, as a long-tail safety net.
Pure text replacement; no extra LLM calls. Failed slugs are picked up
naturally on the next ingest of the same document via the existing slug-
continuity rules, so the system self-heals without manual reingest.
1589 lines
56 KiB
Go
1589 lines
56 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"text/template"
|
||
"time"
|
||
"unicode/utf8"
|
||
|
||
"github.com/Tencent/WeKnora/internal/agent"
|
||
"github.com/Tencent/WeKnora/internal/logger"
|
||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||
"github.com/Tencent/WeKnora/internal/searchutil"
|
||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||
"github.com/Tencent/WeKnora/internal/types"
|
||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||
"github.com/hibiken/asynq"
|
||
"github.com/redis/go-redis/v9"
|
||
)
|
||
|
||
// ErrWikiIngestConcurrent is returned by the wiki ingest handler when another
|
||
// batch is already running for the same KB (i.e. the `wiki:active:<kbID>`
|
||
// Redis lock is held). The asynq server's RetryDelayFunc uses errors.Is on
|
||
// this sentinel to apply a short, fixed retry delay instead of asynq's default
|
||
// exponential backoff — otherwise a freshly orphaned lock (e.g. from a crash
|
||
// or restart) would force newcomers to wait minutes even after the lock
|
||
// naturally expires.
|
||
var ErrWikiIngestConcurrent = errors.New("concurrent wiki task active")
|
||
|
||
const (
|
||
// maxContentForWiki limits the document content sent to LLM for wiki generation
|
||
maxContentForWiki = 32768
|
||
|
||
// wikiPendingKeyPrefix is the Redis key prefix for pending wiki ingest document lists.
|
||
// Key format: wiki:pending:{kbID} → Redis List of knowledge IDs.
|
||
wikiPendingKeyPrefix = "wiki:pending:"
|
||
|
||
// wikiActiveKeyPrefix is the Redis key for the "batch in progress" flag.
|
||
// Key format: wiki:active:{kbID} → "1" with TTL. Prevents concurrent batches.
|
||
wikiActiveKeyPrefix = "wiki:active:"
|
||
|
||
// wikiIngestDelay is how long to wait after a document is added before
|
||
// the batch task fires. Debounces rapid uploads.
|
||
wikiIngestDelay = 30 * time.Second
|
||
|
||
// wikiPendingTTL prevents stale pending lists from accumulating.
|
||
wikiPendingTTL = 24 * time.Hour
|
||
|
||
// wikiMaxDocsPerBatch limits how many documents a single batch processes.
|
||
// Prevents unbounded execution time. Remaining docs stay in the pending list
|
||
// and are picked up by the follow-up task.
|
||
wikiMaxDocsPerBatch = 5
|
||
|
||
// wikiFailCountKeyPrefix is the Redis key prefix for per-document failure
|
||
// counters. Key: wiki:failcount:{kbID}:{knowledgeID} → integer.
|
||
// Incremented each time requeueFailedOps retries an op; reset to 0 on
|
||
// successful ingest. Once the counter exceeds wikiMaxFailRetries the op is
|
||
// dropped instead of re-queued, preventing LLM-timeout storms from causing
|
||
// unbounded wiki:pending growth.
|
||
wikiFailCountKeyPrefix = "wiki:failcount:"
|
||
|
||
// wikiMaxFailRetries is the maximum number of times a single document op
|
||
// may be re-queued via requeueFailedOps before it is permanently dropped.
|
||
// 5 retries ≈ five full batch cycles (each with a ~30 s delay), giving
|
||
// transient LLM errors a fair chance to recover without letting a
|
||
// persistently-broken doc clog the queue indefinitely.
|
||
wikiMaxFailRetries = 5
|
||
|
||
// wikiIngestMaxRetry controls asynq retry budget for wiki:ingest tasks.
|
||
// Keep this moderate: lock conflicts already retry every 15s via
|
||
// asynqRetryDelayFunc, and follow-up/retract paths fire quickly.
|
||
wikiIngestMaxRetry = 10
|
||
|
||
// wikiDeletedKeyPrefix is the Redis key prefix for "recently deleted
|
||
// knowledge" tombstones. Key: wiki:deleted:{kbID}:{knowledgeID}. Written
|
||
// by cleanupWikiOnKnowledgeDelete so that any wiki_ingest task still in
|
||
// flight (or queued) for this knowledge can fast-path skip without
|
||
// hitting the DB. TTL > wikiIngestDelay so it's guaranteed to outlast
|
||
// any in-flight ingest.
|
||
wikiDeletedKeyPrefix = "wiki:deleted:"
|
||
|
||
// wikiDeletedTTL bounds how long we remember a deletion. Must comfortably
|
||
// exceed the longest plausible ingest run (LLM extraction + reduce).
|
||
wikiDeletedTTL = 1 * time.Hour
|
||
|
||
// wikiActiveLockTTL is the TTL for the per-KB "batch in progress" flag.
|
||
// Kept short (relative to total batch runtime) so that if the owning
|
||
// process crashes without running its `defer Del`, the orphaned lock
|
||
// expires quickly and newcomers aren't blocked. A periodic renew
|
||
// (wikiActiveLockRenew) keeps the lock alive while the handler is
|
||
// genuinely still running.
|
||
wikiActiveLockTTL = 60 * time.Second
|
||
|
||
// wikiActiveLockRenew is how often the in-flight handler bumps the TTL.
|
||
// Must be comfortably shorter than wikiActiveLockTTL so a single missed
|
||
// tick (GC pause, Redis blip) doesn't let the lock slip out from under a
|
||
// live handler.
|
||
wikiActiveLockRenew = 20 * time.Second
|
||
|
||
// wikiLLMMaxAttempts is the total attempt count (initial + retries) for
|
||
// every LLM call routed through generateWithTemplate. 3 was chosen to
|
||
// absorb transient 504/timeouts from upstream gateways without
|
||
// materially prolonging task runtime when the remote is genuinely down.
|
||
wikiLLMMaxAttempts = 3
|
||
|
||
// wikiLLMBackoffBase is the base delay for the exponential backoff
|
||
// between retry attempts. The nth retry waits base << (n-1) — so with
|
||
// a 2s base we wait 2s, 4s, 8s between attempts.
|
||
wikiLLMBackoffBase = 2 * time.Second
|
||
)
|
||
|
||
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
|
||
// recently deleted, so wiki_ingest tasks in flight can short-circuit. Exposed
|
||
// so knowledgeService.cleanupWikiOnKnowledgeDelete can write the same key
|
||
// without duplicating the format string.
|
||
func WikiDeletedTombstoneKey(kbID, knowledgeID string) string {
|
||
return wikiDeletedKeyPrefix + kbID + ":" + knowledgeID
|
||
}
|
||
|
||
// WikiIngestPayload is the asynq task payload for wiki ingest batch trigger.
|
||
// The actual document IDs are stored in a Redis list (wiki:pending:{kbID}).
|
||
// KnowledgeID is only used as fallback in Lite mode (no Redis).
|
||
type WikiIngestPayload struct {
|
||
types.TracingContext
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
Language string `json:"language,omitempty"`
|
||
// Fallback for Lite mode (no Redis)
|
||
LiteOps []WikiPendingOp `json:"lite_ops,omitempty"`
|
||
}
|
||
|
||
// WikiRetractPayload is the asynq task payload for wiki content retraction
|
||
type WikiRetractPayload struct {
|
||
types.TracingContext
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
DocTitle string `json:"doc_title"`
|
||
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
|
||
Language string `json:"language,omitempty"`
|
||
PageSlugs []string `json:"page_slugs"`
|
||
}
|
||
|
||
const (
|
||
WikiOpIngest = "ingest"
|
||
WikiOpRetract = "retract"
|
||
)
|
||
|
||
// WikiPendingOp represents a single operation in the Redis pending queue
|
||
type WikiPendingOp struct {
|
||
Op string `json:"op"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
// Ingest fields
|
||
Language string `json:"language,omitempty"`
|
||
// Retract fields
|
||
DocTitle string `json:"doc_title,omitempty"`
|
||
DocSummary string `json:"doc_summary,omitempty"`
|
||
PageSlugs []string `json:"page_slugs,omitempty"`
|
||
}
|
||
|
||
// wikiIngestService handles the LLM-powered wiki generation pipeline
|
||
type wikiIngestService struct {
|
||
wikiService interfaces.WikiPageService
|
||
kbService interfaces.KnowledgeBaseService
|
||
knowledgeSvc interfaces.KnowledgeService
|
||
chunkRepo interfaces.ChunkRepository
|
||
modelService interfaces.ModelService
|
||
task interfaces.TaskEnqueuer
|
||
logEntrySvc interfaces.WikiLogEntryService
|
||
redisClient *redis.Client // nil in Lite mode (no Redis)
|
||
// liteLocks provides per-KB mutual exclusion in Lite mode (no Redis).
|
||
// Keys are kbID strings; values are unused (presence = locked).
|
||
liteLocks sync.Map
|
||
}
|
||
|
||
// NewWikiIngestService creates a new wiki ingest service
|
||
func NewWikiIngestService(
|
||
wikiService interfaces.WikiPageService,
|
||
kbService interfaces.KnowledgeBaseService,
|
||
knowledgeSvc interfaces.KnowledgeService,
|
||
chunkRepo interfaces.ChunkRepository,
|
||
modelService interfaces.ModelService,
|
||
task interfaces.TaskEnqueuer,
|
||
logEntrySvc interfaces.WikiLogEntryService,
|
||
redisClient *redis.Client,
|
||
) interfaces.TaskHandler {
|
||
svc := &wikiIngestService{
|
||
wikiService: wikiService,
|
||
kbService: kbService,
|
||
knowledgeSvc: knowledgeSvc,
|
||
chunkRepo: chunkRepo,
|
||
modelService: modelService,
|
||
task: task,
|
||
logEntrySvc: logEntrySvc,
|
||
redisClient: redisClient,
|
||
}
|
||
return svc
|
||
}
|
||
|
||
// EnqueueWikiIngest adds a document to the wiki ingest queue.
|
||
//
|
||
// Architecture: each document upload pushes its knowledgeID to a Redis pending list,
|
||
// then schedules a delayed asynq task. When the task fires, it atomically drains the
|
||
// entire list and processes ALL pending documents in one batch.
|
||
//
|
||
// If multiple uploads happen within the delay window (30s), each one schedules a task,
|
||
// but the FIRST task to fire drains the list and processes everything. Subsequent tasks
|
||
// fire, find an empty list, and exit immediately (no-op). This gives us natural batching
|
||
// without any locks or task deduplication.
|
||
//
|
||
// t=0s doc1 → RPush + Enqueue(delay=30s, id=random1)
|
||
// t=5s doc2 → RPush + Enqueue(delay=30s, id=random2)
|
||
// t=10s doc3 → RPush + Enqueue(delay=30s, id=random3)
|
||
// t=30s random1 fires → drain [doc1,doc2,doc3] → process all
|
||
// t=35s random2 fires → drain [] → no-op return
|
||
// t=40s random3 fires → drain [] → no-op return
|
||
//
|
||
// In Lite mode (no Redis), falls back to immediate per-document execution.
|
||
func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, tenantID uint64, kbID, knowledgeID string) {
|
||
lang, _ := types.LanguageFromContext(ctx)
|
||
|
||
payload := WikiIngestPayload{
|
||
TenantID: tenantID,
|
||
KnowledgeBaseID: kbID,
|
||
Language: lang,
|
||
}
|
||
|
||
// Push to Redis pending list (if Redis available)
|
||
if redisClient != nil {
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
// Reset stale fail counter for fresh user-triggered ingest enqueue.
|
||
// This gives a newly re-ingested document a full retry budget.
|
||
failKey := wikiFailCountKeyPrefix + kbID + ":" + knowledgeID
|
||
redisClient.Del(ctx, failKey)
|
||
op := WikiPendingOp{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: knowledgeID,
|
||
Language: lang,
|
||
}
|
||
opBytes, _ := json.Marshal(op)
|
||
redisClient.RPush(ctx, pendingKey, string(opBytes))
|
||
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
|
||
} else {
|
||
// Fallback for Lite mode (no Redis)
|
||
payload.LiteOps = []WikiPendingOp{{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: knowledgeID,
|
||
Language: lang,
|
||
}}
|
||
}
|
||
|
||
langfuse.InjectTracing(ctx, &payload)
|
||
payloadBytes, _ := json.Marshal(payload)
|
||
|
||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||
asynq.Queue("low"),
|
||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||
asynq.Timeout(60*time.Minute),
|
||
asynq.ProcessIn(wikiIngestDelay),
|
||
)
|
||
if _, err := task.Enqueue(t); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to enqueue task: %v", err)
|
||
}
|
||
}
|
||
|
||
// EnqueueWikiRetract enqueues an async wiki content retraction task
|
||
func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, payload WikiRetractPayload) {
|
||
ingestPayload := WikiIngestPayload{
|
||
TenantID: payload.TenantID,
|
||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||
Language: payload.Language,
|
||
}
|
||
|
||
op := WikiPendingOp{
|
||
Op: WikiOpRetract,
|
||
KnowledgeID: payload.KnowledgeID,
|
||
DocTitle: payload.DocTitle,
|
||
DocSummary: payload.DocSummary,
|
||
PageSlugs: payload.PageSlugs,
|
||
Language: payload.Language,
|
||
}
|
||
|
||
if redisClient != nil {
|
||
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
|
||
opBytes, _ := json.Marshal(op)
|
||
redisClient.RPush(ctx, pendingKey, string(opBytes))
|
||
redisClient.Expire(ctx, pendingKey, wikiPendingTTL)
|
||
} else {
|
||
// Fallback for Lite mode (no Redis)
|
||
ingestPayload.LiteOps = []WikiPendingOp{op}
|
||
}
|
||
|
||
langfuse.InjectTracing(ctx, &ingestPayload)
|
||
payloadBytes, _ := json.Marshal(ingestPayload)
|
||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||
asynq.Queue("low"),
|
||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||
asynq.Timeout(60*time.Minute),
|
||
asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly
|
||
)
|
||
if _, err := task.Enqueue(t); err != nil {
|
||
logger.Warnf(ctx, "wiki retract: failed to enqueue task: %v", err)
|
||
}
|
||
}
|
||
|
||
// Handle implements interfaces.TaskHandler for asynq task processing.
|
||
// Wiki ingest tasks are debounced via asynq.Unique + ProcessIn, so at most
|
||
// one ingest task runs per KB at a time. No distributed lock needed.
|
||
func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error {
|
||
return s.ProcessWikiIngest(ctx, t)
|
||
}
|
||
|
||
// peekPendingList gets up to wikiMaxDocsPerBatch entries from the Redis pending list
|
||
// WITHOUT removing them. It returns the unique ops and the actual number of items peeked.
|
||
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string) ([]WikiPendingOp, int) {
|
||
if s.redisClient == nil {
|
||
return nil, 0
|
||
}
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
|
||
result, err := s.redisClient.LRange(ctx, pendingKey, 0, wikiMaxDocsPerBatch-1).Result()
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err)
|
||
return nil, 0
|
||
}
|
||
|
||
var ops []WikiPendingOp
|
||
for _, item := range result {
|
||
if !strings.HasPrefix(item, "{") {
|
||
// Backward compatibility: raw knowledgeID string
|
||
ops = append(ops, WikiPendingOp{
|
||
Op: WikiOpIngest,
|
||
KnowledgeID: item,
|
||
})
|
||
continue
|
||
}
|
||
var op WikiPendingOp
|
||
if err := json.Unmarshal([]byte(item), &op); err == nil {
|
||
ops = append(ops, op)
|
||
} else {
|
||
logger.Warnf(ctx, "wiki ingest: failed to unmarshal pending op: %v, raw_item: %.100s", err, item)
|
||
}
|
||
}
|
||
|
||
// Deduplicate by KnowledgeID, keeping only the *last* operation for each document.
|
||
// This optimizes out redundant sequences (e.g., upload then immediate delete: [ingest, retract] -> [retract]).
|
||
seen := make(map[string]bool)
|
||
var reversedUnique []WikiPendingOp
|
||
for i := len(ops) - 1; i >= 0; i-- {
|
||
op := ops[i]
|
||
if !seen[op.KnowledgeID] {
|
||
seen[op.KnowledgeID] = true
|
||
reversedUnique = append(reversedUnique, op)
|
||
}
|
||
}
|
||
|
||
// Reverse back to maintain chronological order
|
||
var unique []WikiPendingOp
|
||
for i := len(reversedUnique) - 1; i >= 0; i-- {
|
||
unique = append(unique, reversedUnique[i])
|
||
}
|
||
|
||
return unique, len(result)
|
||
}
|
||
|
||
// trimPendingList removes the first `count` items from the Redis pending list.
|
||
func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, count int) {
|
||
if s.redisClient == nil || count <= 0 {
|
||
return
|
||
}
|
||
pendingKey := wikiPendingKeyPrefix + kbID
|
||
if err := s.redisClient.LTrim(ctx, pendingKey, int64(count), -1).Err(); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to trim pending list: %v", err)
|
||
}
|
||
}
|
||
|
||
// requeueFailedOps re-enqueues failed operations for retry.
|
||
//
|
||
// Redis mode: appends ops back to the pending list tail so the next follow-up
|
||
// batch picks them up.
|
||
//
|
||
// Lite mode (no Redis): enqueues a new asynq task per failed op with a short
|
||
// delay, since there is no shared pending list to append to.
|
||
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) {
|
||
if s.redisClient != nil {
|
||
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
|
||
for _, op := range ops {
|
||
// Increment failure counter; drop the op permanently once it
|
||
// exceeds wikiMaxFailRetries to prevent unbounded queue growth
|
||
// caused by persistent LLM timeouts or extraction errors.
|
||
failKey := wikiFailCountKeyPrefix + payload.KnowledgeBaseID + ":" + op.KnowledgeID
|
||
count, err := s.redisClient.Incr(ctx, failKey).Result()
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s: %v", op.KnowledgeID, err)
|
||
// Fall through and requeue anyway — better to retry than silently drop
|
||
} else {
|
||
// Refresh TTL on every update so the key doesn't outlast its usefulness
|
||
s.redisClient.Expire(ctx, failKey, wikiPendingTTL)
|
||
if count > wikiMaxFailRetries {
|
||
logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries)
|
||
// Drop-path cleanup: avoid leaking stale counters that can
|
||
// poison future manual re-ingest within TTL window.
|
||
s.redisClient.Del(ctx, failKey)
|
||
continue
|
||
}
|
||
}
|
||
|
||
data, err := json.Marshal(op)
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err)
|
||
continue
|
||
}
|
||
if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err)
|
||
continue
|
||
}
|
||
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries)
|
||
}
|
||
return
|
||
}
|
||
|
||
// Lite mode: re-enqueue each failed op as a new asynq task.
|
||
for _, op := range ops {
|
||
retryPayload := WikiIngestPayload{
|
||
TenantID: payload.TenantID,
|
||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||
Language: op.Language,
|
||
LiteOps: []WikiPendingOp{op},
|
||
}
|
||
langfuse.InjectTracing(ctx, &retryPayload)
|
||
payloadBytes, _ := json.Marshal(retryPayload)
|
||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||
asynq.Queue("low"),
|
||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||
asynq.Timeout(60*time.Minute),
|
||
asynq.ProcessIn(wikiIngestDelay),
|
||
)
|
||
if _, err := s.task.Enqueue(t); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to requeue lite op %s: %v", op.KnowledgeID, err)
|
||
continue
|
||
}
|
||
logger.Infof(ctx, "wiki ingest: re-queued failed lite op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
|
||
}
|
||
}
|
||
|
||
// docIngestResult captures per-document info for batch post-processing.
|
||
type docIngestResult struct {
|
||
KnowledgeID string
|
||
DocTitle string
|
||
Summary string // one-line summary of the document (from summary page)
|
||
// Pages records the wiki pages this document touched, carrying both
|
||
// the slug (for navigation / retract lookups) and the human-readable
|
||
// title captured at ingest time (for the log feed's display layer).
|
||
Pages []types.WikiLogPageRef
|
||
}
|
||
|
||
// WikiBatchContext holds shared data across Map and Reduce phases
|
||
type WikiBatchContext struct {
|
||
AllPages []*types.WikiPage
|
||
SlugTitleMap map[string]string
|
||
SummaryContentByKnowledgeID map[string]string
|
||
// ExtractionGranularity drives Pass 0 (candidate slug extraction)
|
||
// aggressiveness. Resolved once per batch from the KnowledgeBase's
|
||
// WikiConfig so every doc in the batch sees the same scope rules.
|
||
// Already Normalize()'d — consumers can assume it is one of the
|
||
// three valid values.
|
||
ExtractionGranularity types.WikiExtractionGranularity
|
||
}
|
||
|
||
// SlugUpdate represents a single update operation for a specific slug
|
||
type SlugUpdate struct {
|
||
Slug string
|
||
Type string // "entity", "concept", "summary", "retract", "retractStale"
|
||
Item extractedItem // For entity/concept
|
||
DocTitle string
|
||
KnowledgeID string
|
||
SourceRef string
|
||
Language string
|
||
SummaryBody string // For summary
|
||
SummaryLine string // For summary
|
||
RetractDocContent string // For retract / retractStale
|
||
// SourceChunks lists the chunk IDs (within KnowledgeID) that substantively
|
||
// support this update. Mirrors Item.SourceChunks for convenience — the
|
||
// Reduce phase reads from here to avoid an extra field hop.
|
||
SourceChunks []string
|
||
// DocSummary is the document-level summary body produced by
|
||
// WikiSummaryPrompt (everything after the SUMMARY: ... headline, falling
|
||
// back to the raw output if no headline could be parsed out). Carried
|
||
// here so the Reduce phase can frame cited chunks with a rich
|
||
// <source_context> block that tells the editor model what the document
|
||
// is about AND what kind of document it is (resume vs announcement vs
|
||
// product page). The one-line headline alone was too terse to keep the
|
||
// editor grounded on longer / multi-topic source documents.
|
||
DocSummary string
|
||
}
|
||
|
||
func previewText(s string, maxRunes int) string {
|
||
s = strings.TrimSpace(s)
|
||
s = strings.ReplaceAll(s, "\n", " ")
|
||
s = strings.ReplaceAll(s, "\t", " ")
|
||
for strings.Contains(s, " ") {
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
}
|
||
r := []rune(s)
|
||
if maxRunes <= 0 || len(r) <= maxRunes {
|
||
return s
|
||
}
|
||
return string(r[:maxRunes]) + "...(truncated)"
|
||
}
|
||
|
||
func previewStringSlice(items []string, limit int) string {
|
||
if len(items) == 0 {
|
||
return "[]"
|
||
}
|
||
if limit <= 0 {
|
||
limit = 1
|
||
}
|
||
n := len(items)
|
||
if n > limit {
|
||
items = items[:limit]
|
||
}
|
||
out := make([]string, 0, len(items))
|
||
for _, it := range items {
|
||
out = append(out, previewText(it, 48))
|
||
}
|
||
if n > limit {
|
||
return fmt.Sprintf("[%s ...(+%d)]", strings.Join(out, ", "), n-limit)
|
||
}
|
||
return fmt.Sprintf("[%s]", strings.Join(out, ", "))
|
||
}
|
||
|
||
// wikiLinkRE matches `[[slug]]` and `[[slug|display text]]` references
|
||
// inside wiki page content. The slug capture group rejects whitespace and
|
||
// the closing-bracket / pipe characters so we don't accidentally swallow
|
||
// adjacent text. Display text (group 2) is optional.
|
||
var wikiLinkRE = regexp.MustCompile(`\[\[([^\[\]\|\s]+)(?:\|([^\]]+))?\]\]`)
|
||
|
||
// sanitizeDeadSummaryLinks rewrites the summary pages produced by THIS
|
||
// batch to remove `[[slug]]` / `[[slug|display]]` references that point
|
||
// at slugs whose entity/concept page generation failed in reduce.
|
||
//
|
||
// Background: WikiSummaryPrompt instructs the LLM to embed wiki links
|
||
// for every extracted slug it knows about, but slug extraction happens
|
||
// during map (parallel with summary generation) and the actual page
|
||
// creation happens later in reduce. When reduce's WikiPageModifyPrompt
|
||
// fails on an entity/concept slug the page never gets written — and
|
||
// the already-persisted summary is left holding a `[[entity/foo|name]]`
|
||
// link that 404s. This pass replaces those dead links with their
|
||
// display text so the summary degrades gracefully.
|
||
//
|
||
// Pure text replacement, no LLM call. Scoped to the doc-summary slugs
|
||
// in this batch (`summary/<slugify(knowledgeID)>`), keeping the work
|
||
// proportional to batch size.
|
||
func (s *wikiIngestService) sanitizeDeadSummaryLinks(
|
||
ctx context.Context,
|
||
kbID string,
|
||
docResults []*docIngestResult,
|
||
failedSlugs map[string]struct{},
|
||
) {
|
||
if len(failedSlugs) == 0 || len(docResults) == 0 {
|
||
return
|
||
}
|
||
for _, r := range docResults {
|
||
if r == nil || r.KnowledgeID == "" {
|
||
continue
|
||
}
|
||
summarySlug := "summary/" + slugify(r.KnowledgeID)
|
||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, summarySlug)
|
||
if err != nil || page == nil {
|
||
continue
|
||
}
|
||
newContent, changed := stripDeadWikiLinks(page.Content, failedSlugs)
|
||
if !changed {
|
||
continue
|
||
}
|
||
page.Content = newContent
|
||
if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to sanitize dead links in summary %s: %v", summarySlug, err)
|
||
continue
|
||
}
|
||
logger.Infof(ctx, "wiki ingest: sanitized dead [[slug]] refs in summary %s", summarySlug)
|
||
}
|
||
}
|
||
|
||
// stripDeadWikiLinks replaces `[[slug]]` / `[[slug|display]]` references
|
||
// with plain text whenever `slug` is in the dead set. The display text
|
||
// (when present) is preserved verbatim; otherwise the slug's last path
|
||
// segment is humanized into a fallback label so the surrounding sentence
|
||
// still reads naturally.
|
||
func stripDeadWikiLinks(content string, deadSlugs map[string]struct{}) (string, bool) {
|
||
if len(deadSlugs) == 0 || content == "" {
|
||
return content, false
|
||
}
|
||
changed := false
|
||
out := wikiLinkRE.ReplaceAllStringFunc(content, func(match string) string {
|
||
sub := wikiLinkRE.FindStringSubmatch(match)
|
||
if len(sub) < 2 {
|
||
return match
|
||
}
|
||
slug := sub[1]
|
||
if _, dead := deadSlugs[slug]; !dead {
|
||
return match
|
||
}
|
||
changed = true
|
||
display := ""
|
||
if len(sub) >= 3 {
|
||
display = strings.TrimSpace(sub[2])
|
||
}
|
||
if display != "" {
|
||
return display
|
||
}
|
||
// Derive a readable label from the slug's tail segment so we
|
||
// don't leave a raw "entity/foo-bar" smear in the prose.
|
||
parts := strings.Split(slug, "/")
|
||
label := parts[len(parts)-1]
|
||
label = strings.ReplaceAll(label, "-", " ")
|
||
return label
|
||
})
|
||
return out, changed
|
||
}
|
||
|
||
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
|
||
// Scans all published pages, checks each out_link, and removes references to
|
||
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
|
||
func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) == 0 {
|
||
return
|
||
}
|
||
|
||
// Build set of live (non-archived, non-system) slugs
|
||
liveSlugs := make(map[string]bool)
|
||
for _, p := range allPages {
|
||
if p.Status != types.WikiPageStatusArchived {
|
||
liveSlugs[p.Slug] = true
|
||
}
|
||
}
|
||
|
||
var cleaned int
|
||
for _, p := range allPages {
|
||
if p.Status == types.WikiPageStatusArchived {
|
||
continue
|
||
}
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
|
||
content := p.Content
|
||
changed := false
|
||
|
||
// Find all [[slug]] references and remove dead ones
|
||
for _, outSlug := range p.OutLinks {
|
||
if liveSlugs[outSlug] {
|
||
continue // link is alive
|
||
}
|
||
// Dead link — remove the [[slug]] from content, keep the display text if any
|
||
linkPattern := "[[" + outSlug + "]]"
|
||
if strings.Contains(content, linkPattern) {
|
||
// Replace [[dead-slug]] with just the slug's readable part
|
||
parts := strings.Split(outSlug, "/")
|
||
readableName := parts[len(parts)-1]
|
||
readableName = strings.ReplaceAll(readableName, "-", " ")
|
||
content = strings.ReplaceAll(content, linkPattern, readableName)
|
||
changed = true
|
||
}
|
||
}
|
||
|
||
if changed {
|
||
p.Content = content
|
||
if err := s.wikiService.UpdateAutoLinkedContent(ctx, p); err != nil {
|
||
logger.Warnf(ctx, "wiki: failed to clean dead links in page %s: %v", p.Slug, err)
|
||
} else {
|
||
cleaned++
|
||
}
|
||
}
|
||
}
|
||
|
||
if cleaned > 0 {
|
||
logger.Infof(ctx, "wiki: cleaned dead links in %d pages", cleaned)
|
||
}
|
||
}
|
||
|
||
// injectCrossLinks scans affected pages and injects [[wiki-links]] for mentions
|
||
// of other wiki page titles in the content. Pure text replacement, no LLM call.
|
||
// Processes entity/concept/synthesis/comparison and summary pages (not index/log).
|
||
//
|
||
// The actual matching is delegated to linkifyContent, which handles code-block
|
||
// and existing-link exclusions plus ASCII word-boundary checks.
|
||
func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) < 2 {
|
||
return
|
||
}
|
||
|
||
refs := collectLinkRefs(allPages)
|
||
if len(refs) == 0 {
|
||
return
|
||
}
|
||
|
||
affectedSet := make(map[string]bool, len(affectedSlugs))
|
||
for _, slug := range affectedSlugs {
|
||
affectedSet[slug] = true
|
||
}
|
||
|
||
var updated int
|
||
for _, p := range allPages {
|
||
if !affectedSet[p.Slug] {
|
||
continue
|
||
}
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
|
||
newContent, changed := linkifyContent(p.Content, refs, p.Slug)
|
||
if !changed {
|
||
continue
|
||
}
|
||
p.Content = newContent
|
||
if err := s.wikiService.UpdateAutoLinkedContent(ctx, p); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: cross-link injection failed for %s: %v", p.Slug, err)
|
||
continue
|
||
}
|
||
updated++
|
||
}
|
||
|
||
if updated > 0 {
|
||
logger.Infof(ctx, "wiki ingest: injected cross-links in %d pages", updated)
|
||
}
|
||
}
|
||
|
||
// collectLinkRefs flattens (title + aliases) of all non-system pages into a
|
||
// single linkRef slice suitable for linkifyContent.
|
||
func collectLinkRefs(pages []*types.WikiPage) []linkRef {
|
||
refs := make([]linkRef, 0, len(pages)*2)
|
||
for _, p := range pages {
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
if p.Title != "" {
|
||
refs = append(refs, linkRef{slug: p.Slug, matchText: p.Title})
|
||
}
|
||
for _, alias := range p.Aliases {
|
||
if alias != "" {
|
||
refs = append(refs, linkRef{slug: p.Slug, matchText: alias})
|
||
}
|
||
}
|
||
}
|
||
return refs
|
||
}
|
||
|
||
// getExistingPageSlugsForKnowledge returns all page slugs that currently reference
|
||
// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest.
|
||
func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool {
|
||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||
if err != nil || len(allPages) == 0 {
|
||
return nil
|
||
}
|
||
|
||
slugs := make(map[string]bool)
|
||
prefix := knowledgeID + "|"
|
||
for _, p := range allPages {
|
||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||
continue
|
||
}
|
||
for _, ref := range p.SourceRefs {
|
||
if ref == knowledgeID || strings.HasPrefix(ref, prefix) {
|
||
slugs[p.Slug] = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return slugs
|
||
}
|
||
|
||
// retractStalePages handles pages that were previously linked to this document
|
||
// but are no longer produced by the updated extraction.
|
||
// - Single-source stale pages → deleted
|
||
// - Multi-source stale pages → LLM retract to clean content synchronously
|
||
|
||
// Build set of newly affected slugs (including summary)
|
||
|
||
// Stale = was in old set but not in new set
|
||
|
||
// Remove this doc's source ref
|
||
|
||
// No other sources → delete the page
|
||
|
||
// Multi-source → remove ref, queue retract
|
||
|
||
// extractedItem represents a single extracted entity or concept.
|
||
//
|
||
// SourceChunks holds the stable chunk IDs (from the source document) that
|
||
// substantively discuss this item. Populated by the chunk-citation pass; when
|
||
// non-empty the Reduce phase uses these chunks verbatim as the item's
|
||
// evidence instead of the shorter Description/Details fields.
|
||
type extractedItem struct {
|
||
Name string `json:"name"`
|
||
Slug string `json:"slug"`
|
||
Aliases []string `json:"aliases"`
|
||
Description string `json:"description"`
|
||
Details string `json:"details"`
|
||
SourceChunks []string `json:"source_chunks,omitempty"`
|
||
}
|
||
|
||
// combinedExtraction represents the parsed result of the combined entity+concept extraction
|
||
type combinedExtraction struct {
|
||
Entities []extractedItem `json:"entities"`
|
||
Concepts []extractedItem `json:"concepts"`
|
||
}
|
||
|
||
// rebuildIndexPage refreshes the LLM-generated intro that sits on the
|
||
// index wiki_pages row.
|
||
//
|
||
// History: the index page used to store "intro + full directory listing" as
|
||
// a single multi-MB markdown blob in content. Every ingest batch rewrote
|
||
// the whole column, which on KBs with tens of thousands of pages caused
|
||
// O(N) TOAST writes per batch. The directory was lifted out into the
|
||
// structured GET /wiki/index endpoint (see wikiPageService.GetIndexView),
|
||
// and this method now only maintains the intro.
|
||
//
|
||
// Intro lifecycle:
|
||
// - First time (empty or legacy placeholder): generate from all document
|
||
// summaries via WikiIndexIntroPrompt.
|
||
// - Subsequent calls with a change description: incremental update via
|
||
// WikiIndexIntroUpdatePrompt so the intro reflects what just landed.
|
||
// - No change description: keep the existing intro untouched.
|
||
//
|
||
// The new intro is written to both Content and Summary so readers that
|
||
// still fall back to Summary (older clients, legacy migrations) stay in
|
||
// sync with the column the view actually renders.
|
||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
|
||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||
if indexPage == nil {
|
||
return nil
|
||
}
|
||
|
||
// Gather just the Summary-type pages the LLM needs for intro framing.
|
||
// We do NOT load or iterate every wiki page here — the directory is
|
||
// now assembled on demand by GetIndexView, which is the only reader
|
||
// that needs the full enumeration.
|
||
summaries, err := s.wikiService.ListByType(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
var docSummaries strings.Builder
|
||
for _, p := range summaries {
|
||
if p == nil || p.Status == types.WikiPageStatusArchived {
|
||
continue
|
||
}
|
||
fmt.Fprintf(&docSummaries, "<document>\n<title>%s</title>\n<summary>%s</summary>\n</document>\n\n", p.Title, p.Summary)
|
||
}
|
||
if docSummaries.Len() == 0 {
|
||
docSummaries.WriteString("(no documents yet)")
|
||
}
|
||
|
||
// The intro lives on both Content and Summary. Prefer Content since
|
||
// that's what the new index view returns; fall back to Summary for
|
||
// rows written before this refactor so the incremental-update prompt
|
||
// has something to work with.
|
||
existingIntro := strings.TrimSpace(indexPage.Content)
|
||
if existingIntro == "" {
|
||
existingIntro = strings.TrimSpace(indexPage.Summary)
|
||
}
|
||
// Detect the legacy "intro + directory" payload. Such rows embed the
|
||
// fence-separated "## Summary" sections right after the intro, so we
|
||
// clip everything from the first directory heading onward to keep the
|
||
// intro length bounded when we feed it back into the update prompt.
|
||
if idx := strings.Index(existingIntro, "\n## "); idx >= 0 {
|
||
existingIntro = strings.TrimSpace(existingIntro[:idx])
|
||
}
|
||
|
||
var intro string
|
||
switch {
|
||
case existingIntro == "" || existingIntro == "Wiki index - table of contents":
|
||
// First time — generate intro from scratch.
|
||
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
|
||
"DocumentSummaries": docSummaries.String(),
|
||
"Language": lang,
|
||
})
|
||
if genErr != nil {
|
||
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
|
||
} else {
|
||
intro = strings.TrimSpace(generatedIntro)
|
||
}
|
||
case changeDesc != "":
|
||
// Incremental update — tell the LLM what changed.
|
||
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
|
||
"ExistingIntro": existingIntro,
|
||
"ChangeDescription": changeDesc,
|
||
"DocumentSummaries": docSummaries.String(),
|
||
"Language": lang,
|
||
})
|
||
if genErr != nil {
|
||
intro = existingIntro // keep existing on error
|
||
} else {
|
||
intro = strings.TrimSpace(updatedIntro)
|
||
}
|
||
default:
|
||
// No change description and an existing intro: leave it as-is so
|
||
// we don't bump the version for a no-op.
|
||
intro = existingIntro
|
||
}
|
||
|
||
// Defensive: some LLM outputs occasionally bleed into a directory-
|
||
// like section even when the intro prompt doesn't ask for one. If
|
||
// the freshly-generated intro starts to look like a legacy payload,
|
||
// clip it at the first "\n## " just like we did on the read path
|
||
// above. This keeps indexPage.Content a bounded intro-only blob.
|
||
if idx := strings.Index(intro, "\n## "); idx >= 0 {
|
||
intro = strings.TrimSpace(intro[:idx])
|
||
}
|
||
|
||
indexPage.Content = intro
|
||
indexPage.Summary = intro
|
||
_, err = s.wikiService.UpdatePage(ctx, indexPage)
|
||
return err
|
||
}
|
||
|
||
// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output.
|
||
// Returns (summary, content). If no SUMMARY line found, summary is empty.
|
||
func splitSummaryLine(raw string) (summary string, content string) {
|
||
raw = strings.TrimSpace(raw)
|
||
if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") {
|
||
idx := strings.IndexByte(raw, '\n')
|
||
if idx < 0 {
|
||
// Only one line
|
||
return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), ""
|
||
}
|
||
summaryLine := raw[:idx]
|
||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||
return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:])
|
||
}
|
||
return "", raw
|
||
}
|
||
|
||
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
|
||
// pure (no DB access) so callers can accumulate entries cheaply under their
|
||
// lock and flush them in a single AppendBatch call at the end of the batch.
|
||
//
|
||
// Historically this was a per-event `GetLog + UpdatePage` round trip, which
|
||
// rewrote the entire log page's TEXT column on every ingest/retract op —
|
||
// O(n^2) write amplification as the log grew. The batch writer now uses
|
||
// wikiLogEntryService.AppendBatch instead; see ProcessWikiIngest.
|
||
func (s *wikiIngestService) buildLogEntry(tenantID uint64, kbID, action, knowledgeID, docTitle, summary string, pagesAffected []types.WikiLogPageRef) *types.WikiLogEntry {
|
||
// Copy pagesAffected so the entry does not alias caller-owned slices.
|
||
// The batch accumulates SlugUpdate results that may be reused downstream.
|
||
var pages types.WikiLogPageRefs
|
||
if len(pagesAffected) > 0 {
|
||
pages = make(types.WikiLogPageRefs, len(pagesAffected))
|
||
copy(pages, pagesAffected)
|
||
}
|
||
return &types.WikiLogEntry{
|
||
TenantID: tenantID,
|
||
KnowledgeBaseID: kbID,
|
||
Action: action,
|
||
KnowledgeID: knowledgeID,
|
||
DocTitle: docTitle,
|
||
Summary: summary,
|
||
PagesAffected: pages,
|
||
CreatedAt: time.Now(),
|
||
}
|
||
}
|
||
|
||
// publishDraftPages transitions draft pages to published status after ingest completes.
|
||
// This ensures users don't see half-built pages during the ingest process.
|
||
func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, slugs []string) {
|
||
for _, slug := range slugs {
|
||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
|
||
if err != nil || page == nil {
|
||
continue
|
||
}
|
||
if page.Status == types.WikiPageStatusDraft {
|
||
page.Status = types.WikiPageStatusPublished
|
||
if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to publish page %s: %v", slug, err)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// writeDedupItemXML renders a single entity/concept entry as a structured XML
|
||
// block for the deduplication prompt. Structured form (versus a single
|
||
// pipe-separated line) helps the LLM reliably tell name / aliases / type apart
|
||
// and reduces nonsensical merges like "居民身份证" → "工作居住证".
|
||
func writeDedupItemXML(buf *strings.Builder, slug, name, itemType string, aliases []string) {
|
||
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", slug, itemType)
|
||
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(name))
|
||
for _, alias := range aliases {
|
||
if alias == "" {
|
||
continue
|
||
}
|
||
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
|
||
}
|
||
buf.WriteString(" </item>\n")
|
||
}
|
||
|
||
// xmlEscape escapes the minimal set of characters that can break XML text
|
||
// content. Slugs are ASCII-only so they don't need escaping when used as
|
||
// attribute values.
|
||
func xmlEscape(s string) string {
|
||
s = strings.ReplaceAll(s, "&", "&")
|
||
s = strings.ReplaceAll(s, "<", "<")
|
||
s = strings.ReplaceAll(s, ">", ">")
|
||
return s
|
||
}
|
||
|
||
// deduplicateExtractedBatch deduplicates both entities and concepts against
|
||
// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid
|
||
// redundant DB queries. This replaces the two separate deduplicateItems calls
|
||
// that each queried ListAllPages + made a separate LLM call.
|
||
func (s *wikiIngestService) deduplicateExtractedBatch(
|
||
ctx context.Context,
|
||
chatModel chat.Chat,
|
||
entities, concepts []extractedItem,
|
||
allPages []*types.WikiPage,
|
||
) ([]extractedItem, []extractedItem) {
|
||
if len(entities) == 0 && len(concepts) == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
if len(allPages) == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
// Pre-filter the candidate existing-pages set. Passing the full corpus
|
||
// to the LLM on large KBs bloats the prompt and empirically lets the
|
||
// model hallucinate merges between totally unrelated slugs. The filter
|
||
// is conservative (high recall) and the downstream validMerge check
|
||
// remains in place for defense in depth.
|
||
newItems := make([]extractedItem, 0, len(entities)+len(concepts))
|
||
newItems = append(newItems, entities...)
|
||
newItems = append(newItems, concepts...)
|
||
candidatePages := selectDedupCandidatePages(newItems, allPages)
|
||
if len(candidatePages) == 0 {
|
||
return entities, concepts
|
||
}
|
||
if origCount := countEntityConceptPages(allPages); origCount > len(candidatePages) {
|
||
logger.Infof(ctx, "wiki ingest: dedup candidate filter kept %d/%d existing pages for %d new items",
|
||
len(candidatePages), origCount, len(newItems))
|
||
}
|
||
|
||
var existingBuf strings.Builder
|
||
for _, p := range candidatePages {
|
||
writeDedupItemXML(&existingBuf, p.Slug, p.Title, string(p.PageType), []string(p.Aliases))
|
||
}
|
||
if existingBuf.Len() == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
var newBuf strings.Builder
|
||
for _, item := range entities {
|
||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "entity", item.Aliases)
|
||
}
|
||
for _, item := range concepts {
|
||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "concept", item.Aliases)
|
||
}
|
||
|
||
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
|
||
"NewItems": newBuf.String(),
|
||
"ExistingPages": existingBuf.String(),
|
||
})
|
||
if err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
|
||
return entities, concepts
|
||
}
|
||
|
||
dedupeJSON = cleanLLMJSON(dedupeJSON)
|
||
|
||
var dedupeResult struct {
|
||
Merges map[string]string `json:"merges"`
|
||
}
|
||
if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil {
|
||
logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON)
|
||
return entities, concepts
|
||
}
|
||
|
||
if len(dedupeResult.Merges) == 0 {
|
||
return entities, concepts
|
||
}
|
||
|
||
existingSlugs := make(map[string]bool, len(allPages))
|
||
for _, p := range allPages {
|
||
existingSlugs[p.Slug] = true
|
||
}
|
||
|
||
validMerge := func(srcSlug, dstSlug string) bool {
|
||
if !existingSlugs[dstSlug] {
|
||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug)
|
||
return false
|
||
}
|
||
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
|
||
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
|
||
if srcPrefix != dstPrefix {
|
||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
for i, item := range entities {
|
||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
|
||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||
entities[i].Slug = existingSlug
|
||
}
|
||
}
|
||
for i, item := range concepts {
|
||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
|
||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||
concepts[i].Slug = existingSlug
|
||
}
|
||
}
|
||
|
||
return entities, concepts
|
||
}
|
||
|
||
// generateWithTemplate executes a prompt template and calls the LLM with
|
||
// bounded exponential-backoff retries for transient infrastructure errors.
|
||
//
|
||
// Retry policy:
|
||
// - Up to wikiLLMMaxAttempts total attempts (initial + retries).
|
||
// - Only retry errors classified as transient by isTransientLLMError:
|
||
// HTTP 408/429/5xx, context deadline exceeded (when the parent ctx is
|
||
// still alive), or generic "timeout"/"connection reset" wording.
|
||
// 4xx (except 408/429) is a caller-side fault and fails fast.
|
||
// - Backoff is exponential base 2s: 2s, 4s, 8s — roughly wikiLLMBackoffBase
|
||
// - 2^(attempt-1). Honors ctx cancellation so the task can abort.
|
||
//
|
||
// This exists because wiki ingest makes several independent LLM calls per
|
||
// document (extraction, summary, dedup, citations, intro) and a single
|
||
// transient 504 from the upstream gateway used to drop the document's
|
||
// summary page permanently. Retries plus failedOps requeuing (see
|
||
// mapOneDocument) turn those events into at-most-a-few-minute hiccups.
|
||
func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel chat.Chat, promptTpl string, data map[string]string) (string, error) {
|
||
tmpl, err := template.New("wiki").Parse(promptTpl)
|
||
if err != nil {
|
||
return "", fmt.Errorf("parse template: %w", err)
|
||
}
|
||
|
||
var buf strings.Builder
|
||
if err := tmpl.Execute(&buf, data); err != nil {
|
||
return "", fmt.Errorf("execute template: %w", err)
|
||
}
|
||
|
||
prompt := buf.String()
|
||
thinking := false
|
||
|
||
var lastErr error
|
||
for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ {
|
||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||
{Role: "user", Content: prompt},
|
||
}, &chat.ChatOptions{
|
||
Temperature: 0.3,
|
||
Thinking: &thinking,
|
||
})
|
||
if err == nil {
|
||
return response.Content, nil
|
||
}
|
||
lastErr = err
|
||
|
||
// Abort immediately on non-retryable errors (4xx except 408/429,
|
||
// parse/marshal failures, tool-side bugs, etc.). Retrying a
|
||
// hard "invalid arguments" error just wastes the model's budget.
|
||
if !isTransientLLMError(ctx, err) {
|
||
return "", fmt.Errorf("LLM call failed: %w", err)
|
||
}
|
||
if attempt == wikiLLMMaxAttempts {
|
||
break
|
||
}
|
||
|
||
backoff := wikiLLMBackoffBase << (attempt - 1)
|
||
logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v",
|
||
attempt, wikiLLMMaxAttempts, backoff, err)
|
||
select {
|
||
case <-ctx.Done():
|
||
return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err())
|
||
case <-time.After(backoff):
|
||
}
|
||
}
|
||
return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr)
|
||
}
|
||
|
||
// isTransientLLMError reports whether an error from the chat provider
|
||
// looks like an infrastructure hiccup worth retrying. Classification is
|
||
// intentionally conservative: the truthful "could not tell, assume
|
||
// permanent" choice keeps retries cheap and avoids masking real bugs.
|
||
//
|
||
// We treat the following as transient:
|
||
// - HTTP 408 (client request timeout — upstream usually didn't process),
|
||
// 429 (rate-limited — retry after backoff may succeed), 5xx (any
|
||
// server-side fault, including the 504 "Remote error, timeout with
|
||
// 60" we see from the gateway in front of several LLM providers).
|
||
// - Wrapped context.DeadlineExceeded when the parent ctx is still alive
|
||
// (nested per-call timeouts).
|
||
// - Substring matches on the error text for common transport failures
|
||
// ("timeout", "connection reset", "EOF") that providers surface
|
||
// without a structured status code.
|
||
func isTransientLLMError(ctx context.Context, err error) bool {
|
||
if err == nil {
|
||
return false
|
||
}
|
||
// Never retry after the parent ctx itself expired — the task is
|
||
// being cancelled and the next attempt would just fail again.
|
||
if ctx.Err() != nil {
|
||
return false
|
||
}
|
||
|
||
msg := err.Error()
|
||
// Providers that bubble HTTP status up formatted as
|
||
// "API request failed with status NNN: ..." — match that first.
|
||
for _, s := range []string{
|
||
"status 408", "status 429",
|
||
"status 500", "status 501", "status 502", "status 503", "status 504",
|
||
"status 520", "status 521", "status 522", "status 523", "status 524",
|
||
} {
|
||
if strings.Contains(msg, s) {
|
||
return true
|
||
}
|
||
}
|
||
|
||
lower := strings.ToLower(msg)
|
||
for _, s := range []string{
|
||
"timeout",
|
||
"timed out",
|
||
"connection reset",
|
||
"connection refused",
|
||
"broken pipe",
|
||
"no such host", // DNS hiccup
|
||
"i/o timeout",
|
||
"unexpected eof",
|
||
"tls handshake",
|
||
"context deadline exceeded", // nested per-call deadline
|
||
} {
|
||
if strings.Contains(lower, s) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- Helpers ---
|
||
|
||
// isKnowledgeGone returns true if the given knowledge has been deleted or is
|
||
// in the middle of being deleted. It first consults the Redis tombstone
|
||
// (written by cleanupWikiOnKnowledgeDelete) as a fast path, then falls back
|
||
// to the DB. A nil result from GetKnowledgeByIDOnly also counts as gone: the
|
||
// repo layer uses GORM First() which filters soft-deleted rows, so a
|
||
// soft-deleted knowledge surfaces as "not found" here — exactly what we want.
|
||
func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) bool {
|
||
if knowledgeID == "" {
|
||
return true
|
||
}
|
||
if s.redisClient != nil {
|
||
if exists, err := s.redisClient.Exists(ctx, WikiDeletedTombstoneKey(kbID, knowledgeID)).Result(); err == nil && exists > 0 {
|
||
return true
|
||
}
|
||
}
|
||
kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID)
|
||
if err != nil || kn == nil {
|
||
return true
|
||
}
|
||
return kn.ParseStatus == types.ParseStatusDeleting
|
||
}
|
||
|
||
// filterLiveUpdates drops additions/summaries whose source knowledge has been
|
||
// deleted since the Map phase finished. Retract updates are preserved so
|
||
// pages still get cleaned up. Caches per-knowledge results to avoid DB
|
||
// hammering when a single reduce slug carries many updates for the same doc.
|
||
func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string, updates []SlugUpdate) []SlugUpdate {
|
||
if len(updates) == 0 {
|
||
return updates
|
||
}
|
||
goneCache := make(map[string]bool)
|
||
isGone := func(kid string) bool {
|
||
if kid == "" {
|
||
return false
|
||
}
|
||
if v, ok := goneCache[kid]; ok {
|
||
return v
|
||
}
|
||
v := s.isKnowledgeGone(ctx, kbID, kid)
|
||
goneCache[kid] = v
|
||
return v
|
||
}
|
||
filtered := make([]SlugUpdate, 0, len(updates))
|
||
dropped := 0
|
||
for _, u := range updates {
|
||
switch u.Type {
|
||
case "retract", "retractStale":
|
||
filtered = append(filtered, u)
|
||
default:
|
||
if isGone(u.KnowledgeID) {
|
||
dropped++
|
||
continue
|
||
}
|
||
filtered = append(filtered, u)
|
||
}
|
||
}
|
||
if dropped > 0 {
|
||
logger.Infof(ctx, "wiki ingest: reduce dropped %d updates for deleted knowledge(s)", dropped)
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
// reconstructContent rebuilds document text from chunks.
|
||
//
|
||
// This only concatenates text-type chunks — image OCR / caption information is
|
||
// stored on image_ocr / image_caption child chunks (see image_multimodal.go),
|
||
// not on the parent text chunk's ImageInfo field. Callers that need the full
|
||
// enriched content (with OCR / captions inlined) should call
|
||
// reconstructEnrichedContent instead so image info is fetched from child
|
||
// chunks and embedded alongside Markdown image links.
|
||
func reconstructContent(chunks []*types.Chunk) string {
|
||
var textChunks []*types.Chunk
|
||
for _, c := range chunks {
|
||
if c.ChunkType == types.ChunkTypeText || c.ChunkType == "" {
|
||
textChunks = append(textChunks, c)
|
||
}
|
||
}
|
||
|
||
// Sort by StartAt, then ChunkIndex
|
||
sort.Slice(textChunks, func(i, j int) bool {
|
||
if textChunks[i].StartAt == textChunks[j].StartAt {
|
||
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
|
||
}
|
||
return textChunks[i].StartAt < textChunks[j].StartAt
|
||
})
|
||
|
||
var sb strings.Builder
|
||
lastEndAt := -1
|
||
for _, c := range textChunks {
|
||
toAppend := c.Content
|
||
|
||
if c.StartAt > lastEndAt || c.EndAt == 0 {
|
||
// Non-overlapping or missing position info
|
||
if sb.Len() > 0 {
|
||
sb.WriteString("\n")
|
||
}
|
||
sb.WriteString(toAppend)
|
||
if c.EndAt > 0 {
|
||
lastEndAt = c.EndAt
|
||
}
|
||
} else if c.EndAt > lastEndAt {
|
||
// Partial overlap
|
||
contentRunes := []rune(toAppend)
|
||
offset := len(contentRunes) - (c.EndAt - lastEndAt)
|
||
if offset >= 0 && offset < len(contentRunes) {
|
||
sb.WriteString(string(contentRunes[offset:]))
|
||
} else {
|
||
// Fallback if offset calculation is invalid
|
||
if sb.Len() > 0 {
|
||
sb.WriteString("\n")
|
||
}
|
||
sb.WriteString(toAppend)
|
||
}
|
||
lastEndAt = c.EndAt
|
||
}
|
||
// If c.EndAt <= lastEndAt, it's fully contained, so skip appending text
|
||
}
|
||
|
||
return sb.String()
|
||
}
|
||
|
||
// reconstructEnrichedContent rebuilds document text and inlines image_info
|
||
// (OCR text + caption) pulled from image_ocr / image_caption child chunks.
|
||
//
|
||
// Without this enrichment, image-heavy documents (e.g. a scanned PDF or a
|
||
// standalone .jpg) reach the LLM as bare Markdown image links, causing
|
||
// extraction / summarization to produce empty or "no textual content" output.
|
||
func reconstructEnrichedContent(
|
||
ctx context.Context,
|
||
chunkRepo interfaces.ChunkRepository,
|
||
tenantID uint64,
|
||
chunks []*types.Chunk,
|
||
) string {
|
||
content := reconstructContent(chunks)
|
||
|
||
var textChunkIDs []string
|
||
for _, c := range chunks {
|
||
if c.ChunkType == types.ChunkTypeText || c.ChunkType == "" {
|
||
if c.ID != "" {
|
||
textChunkIDs = append(textChunkIDs, c.ID)
|
||
}
|
||
}
|
||
}
|
||
if len(textChunkIDs) == 0 || chunkRepo == nil {
|
||
return content
|
||
}
|
||
|
||
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, chunkRepo, tenantID, textChunkIDs)
|
||
mergedImageInfo := searchutil.MergeImageInfoJSON(imageInfoMap)
|
||
if mergedImageInfo == "" {
|
||
return content
|
||
}
|
||
return searchutil.EnrichContentWithImageInfo(content, mergedImageInfo)
|
||
}
|
||
|
||
// slugify creates a URL-friendly slug from a string
|
||
func slugify(s string) string {
|
||
s = strings.ToLower(strings.TrimSpace(s))
|
||
s = strings.Map(func(r rune) rune {
|
||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '/' {
|
||
return r
|
||
}
|
||
if r == ' ' || r == '_' {
|
||
return '-'
|
||
}
|
||
// Keep CJK characters
|
||
if r >= 0x4E00 && r <= 0x9FFF {
|
||
return r
|
||
}
|
||
return -1
|
||
}, s)
|
||
// Collapse multiple hyphens
|
||
for strings.Contains(s, "--") {
|
||
s = strings.ReplaceAll(s, "--", "-")
|
||
}
|
||
s = strings.Trim(s, "-")
|
||
if len(s) > 200 {
|
||
s = s[:200]
|
||
}
|
||
return s
|
||
}
|
||
|
||
// truncateString truncates a string to maxLen runes
|
||
func truncateString(s string, maxLen int) string {
|
||
runes := []rune(s)
|
||
if len(runes) <= maxLen {
|
||
return s
|
||
}
|
||
return string(runes[:maxLen]) + "..."
|
||
}
|
||
|
||
// appendUnique appends a string to a StringArray if not already present
|
||
func appendUnique(arr types.StringArray, s string) types.StringArray {
|
||
for _, v := range arr {
|
||
if v == s {
|
||
return arr
|
||
}
|
||
}
|
||
return append(arr, s)
|
||
}
|
||
|
||
// minTextContentRunes is the minimum number of non-whitespace, non-image-reference
|
||
// runes required for content to be considered substantive enough for LLM
|
||
// summarization or wiki extraction. Documents below this threshold (e.g. a
|
||
// scanned PDF where OCR yielded nothing AND no caption either) are routed to
|
||
// a deterministic empty-content fallback instead of being passed to the LLM,
|
||
// which would otherwise hallucinate based on metadata alone.
|
||
//
|
||
// The threshold is intentionally low: legitimate short documents (brief
|
||
// memos, single-line notes) must still pass. The goal is only to catch
|
||
// the empty-image-only case.
|
||
//
|
||
// Declared as a var (not const) so tests can override it and future config
|
||
// plumbing can adjust it at runtime without a rebuild.
|
||
var minTextContentRunes = 10
|
||
|
||
var (
|
||
// Markdown image references like  — pure visual placeholders
|
||
// with no extractable text, so the whole reference is removed.
|
||
mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
|
||
|
||
// <image_original>...</image_original> blocks wrap the verbatim Markdown
|
||
// image reference inside an enriched <image> block (see
|
||
// searchutil.EnrichContentWithImageInfo). The content is just a redundant
|
||
// copy of an already-stripped image link, so the whole block (tags +
|
||
// content) is removed.
|
||
imageOriginalBlockRE = regexp.MustCompile(`(?is)<image_original\b[^>]*>.*?</image_original>`)
|
||
|
||
// Self-closing or attribute-only HTML <img> tags.
|
||
htmlImgTagRE = regexp.MustCompile(`(?i)<img\b[^>]*/?>`)
|
||
|
||
// Wrapper-style <image>, <images>, <image_caption>, <image_ocr> tags
|
||
// (opening or closing). Matches ONLY the tag; the text content between
|
||
// open and close tags is preserved. This is critical: VLM-generated OCR
|
||
// and caption text live inside <image_ocr>...</image_ocr> and
|
||
// <image_caption>...</image_caption> blocks, and stripping the content
|
||
// would silently destroy the very text we want to keep.
|
||
imageWrapperTagRE = regexp.MustCompile(`(?i)</?image[a-z_]*\b[^>]*/?>`)
|
||
)
|
||
|
||
// stripImageMarkup removes image-only placeholders (Markdown image refs,
|
||
// <img> tags, <image_original> redundancy blocks) and unwraps the
|
||
// <image>/<image_caption>/<image_ocr> XML wrappers produced by the search
|
||
// enrichment layer, leaving any OCR or caption text as plain inline text.
|
||
//
|
||
// This shape matters: when VLM OCR succeeds on a scanned PDF page, the
|
||
// extracted text reaches downstream code wrapped in <image_ocr> tags inside
|
||
// an <image> block. A naive "strip the whole <image>...</image> block"
|
||
// approach would discard the OCR text — the exact opposite of what we want.
|
||
func stripImageMarkup(s string) string {
|
||
s = imageOriginalBlockRE.ReplaceAllString(s, "")
|
||
s = mdImageRefRE.ReplaceAllString(s, "")
|
||
s = htmlImgTagRE.ReplaceAllString(s, "")
|
||
s = imageWrapperTagRE.ReplaceAllString(s, "")
|
||
return s
|
||
}
|
||
|
||
// extractRealText returns the trimmed content with image markup stripped.
|
||
// Cached at the call site for use both in the threshold check and in any
|
||
// subsequent log message, avoiding redundant regex passes over large docs.
|
||
func extractRealText(content string) string {
|
||
return strings.TrimSpace(stripImageMarkup(content))
|
||
}
|
||
|
||
// hasSufficientTextContent reports whether the given content carries enough
|
||
// real text (after image markup is stripped, with OCR/caption text retained)
|
||
// to warrant an LLM call. It is the primary defence against filename-driven
|
||
// hallucinations on scanned PDFs that have NO usable text at all.
|
||
func hasSufficientTextContent(content string) bool {
|
||
return realTextRuneCount(content) >= minTextContentRunes
|
||
}
|
||
|
||
// realTextRuneCount returns the rune length of the content after image
|
||
// markup is stripped. Uses utf8.RuneCountInString to avoid allocating a
|
||
// rune slice for the count.
|
||
func realTextRuneCount(content string) int {
|
||
return utf8.RuneCountInString(extractRealText(content))
|
||
}
|
||
|
||
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
|
||
// from LLM-generated JSON output so it can be safely unmarshalled.
|
||
func cleanLLMJSON(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
s = strings.TrimPrefix(s, "```json")
|
||
s = strings.TrimPrefix(s, "```")
|
||
s = strings.TrimSuffix(s, "```")
|
||
s = strings.TrimSpace(s)
|
||
return sanitizeJSONString(s)
|
||
}
|
||
|
||
// sanitizeJSONString sanitizes a string that is intended to be parsed as JSON,
|
||
// by properly escaping unescaped control characters (like newlines) inside string literals.
|
||
func sanitizeJSONString(s string) string {
|
||
var buf strings.Builder
|
||
buf.Grow(len(s))
|
||
inString := false
|
||
escape := false
|
||
for _, r := range s {
|
||
if escape {
|
||
if r == '\n' {
|
||
buf.WriteString(`n`)
|
||
} else if r == '\r' {
|
||
buf.WriteString(`r`)
|
||
} else if r == '\t' {
|
||
buf.WriteString(`t`)
|
||
} else {
|
||
buf.WriteRune(r)
|
||
}
|
||
escape = false
|
||
continue
|
||
}
|
||
if r == '\\' {
|
||
escape = true
|
||
buf.WriteRune(r)
|
||
continue
|
||
}
|
||
if r == '"' {
|
||
inString = !inString
|
||
buf.WriteRune(r)
|
||
continue
|
||
}
|
||
if inString {
|
||
if r == '\n' {
|
||
buf.WriteString(`\n`)
|
||
continue
|
||
}
|
||
if r == '\r' {
|
||
buf.WriteString(`\r`)
|
||
continue
|
||
}
|
||
if r == '\t' {
|
||
buf.WriteString(`\t`)
|
||
continue
|
||
}
|
||
}
|
||
buf.WriteRune(r)
|
||
}
|
||
return buf.String()
|
||
}
|