From 82947c726ee9cf573ae4efc4a8a89a55e03c7199 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Sat, 9 May 2026 23:59:12 +0800 Subject: [PATCH] fix(wiki-ingest,asynqdl): review fixups for PR #1241 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - repo: drop r.db.Debug() from FindSimilarPages — it was dumping every trigram probe's SQL+args (per-alias, per-item) into production logs. - wiki_ingest dedup: fix Printf format string ("selected for %d new items" had two args), and harden validMerge against un-prefixed slugs whose strings.Index returned -1 and silently passed the type check. - wiki_ingest_batch: drop the duplicated loggedBatchSize/MapPar/ ReducePar assignments. - asynqdl: record the real attempt count (retried + 1) on the dead letter row instead of a hard-coded 0; tighten payloadProbe to the set of field names with consistent semantics across payloads (drop source_id/target_id/target_kb_id which differ by task type). - asynqdl tests: update for the trimmed probe and assert FailCount=0 outside an asynq worker ctx so the semantics stay pinned. --- internal/application/repository/wiki_page.go | 2 +- internal/application/service/wiki_ingest.go | 17 +++++- .../application/service/wiki_ingest_batch.go | 11 ++-- internal/middleware/asynqdl/asynqdl.go | 55 ++++++++++++------- internal/middleware/asynqdl/asynqdl_test.go | 13 +++-- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index 8b3e96d0..e3bbb2cd 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -628,7 +628,7 @@ func (r *wikiPageRepository) FindSimilarPages( q := strings.ToLower(strings.TrimSpace(query)) var rows []types.WikiPageLite - if err := r.db.Debug().WithContext(ctx). + if err := r.db.WithContext(ctx). Model(&types.WikiPage{}). Select("slug, title, page_type, status, aliases, out_links, similarity(lower(title), ?) AS sim", q). Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND lower(title) % ?", diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index c21b1e4e..b2652282 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -1404,7 +1404,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch( logger.Infof(ctx, "wiki ingest: no similar existing pages found for %d new items", len(entities)+len(concepts)) return entities, concepts } - logger.Infof(ctx, "wiki ingest: similar existing pages selected for %d new items", + logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items", len(candidatePages), len(entities)+len(concepts)) var existingBuf strings.Builder @@ -1461,8 +1461,19 @@ func (s *wikiIngestService) deduplicateExtractedBatch( logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug) return false } - srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1] - dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1] + srcSlash := strings.Index(srcSlug, "/") + dstSlash := strings.Index(dstSlug, "/") + if srcSlash <= 0 || dstSlash <= 0 { + // A type-prefixed slug must look like "entity/foo" or + // "concept/bar". An LLM that emits an un-prefixed slug + // here is hallucinating; reject rather than fall through + // the prefix-equality check (which would treat both empty + // prefixes as a match). + logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug) + return false + } + srcPrefix := srcSlug[:srcSlash+1] + dstPrefix := dstSlug[:dstSlash+1] if srcPrefix != dstPrefix { logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix) return false diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 187df45f..800452d6 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -223,9 +223,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task loggedBatchSize = batchSize loggedMapPar = mapParallel loggedReducePar = reduceParallel - loggedBatchSize = batchSize - loggedMapPar = mapParallel - loggedReducePar = reduceParallel lang := types.LanguageNameFromContext(ctx) @@ -254,9 +251,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // callers pay only for the slugs / knowledge ids they actually // reach for. Cache hits keep repeat lookups within the batch free. var ( - fetchMu sync.Mutex - slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing - summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing + fetchMu sync.Mutex + slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing + summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing ) resolveSlugs := func(ctx context.Context, slugs []string) map[string]string { @@ -351,7 +348,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task m := resolveSlugs(ctx, []string{slug}) return m[slug] }, - SlugTitleMany: resolveSlugs, + SlugTitleMany: resolveSlugs, SummaryContentByKnowledgeID: func(ctx context.Context, kid string) string { m := resolveSummaries(ctx, []string{kid}) return m[kid] diff --git a/internal/middleware/asynqdl/asynqdl.go b/internal/middleware/asynqdl/asynqdl.go index c2276c8c..20f63fc2 100644 --- a/internal/middleware/asynqdl/asynqdl.go +++ b/internal/middleware/asynqdl/asynqdl.go @@ -59,7 +59,19 @@ func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc { if repo == nil { return err } - dl := buildDeadLetter(t, err) + // Capture the actual attempt count from asynq so the + // dead-letter row reflects how many tries were made + // before the task was archived. On the final attempt + // asynq's counter satisfies retried == maxRetry, so the + // total attempt count is retried + 1 (initial try + + // retries). Both helpers can fail outside a worker ctx; + // in that case we record 0 so it's clear the count + // wasn't observable. + attempts := 0 + if retried, ok := asynq.GetRetryCount(ctx); ok { + attempts = retried + 1 + } + dl := buildDeadLetter(t, err, attempts) if dl == nil { return err } @@ -103,7 +115,7 @@ func isFinalAttempt(ctx context.Context) bool { // Returns nil only if the task itself is nil — every other failure mode // (unparseable payload, unknown task type) still produces a row, with // scope="unknown" so operators can find it. -func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter { +func buildDeadLetter(t *asynq.Task, taskErr error, attempts int) *types.TaskDeadLetter { if t == nil { return nil } @@ -113,12 +125,6 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter { scope, scopeID := inferScope(probe) relatedID := probe.KnowledgeID - retryCount := 0 - // asynq doesn't expose the captured retry on the Task itself, so we - // fall back to the probe's RetryCount (often 0 — most payloads don't - // carry it). The middleware caller has already verified that we're - // at the final attempt; the absolute number is informational. - // asynq stores the payload as raw bytes; preserve it verbatim so a // future requeue can reuse it directly. rawPayload := t.Payload() @@ -134,23 +140,35 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter { RelatedID: relatedID, Payload: json.RawMessage(rawPayload), LastError: truncateError(taskErr.Error(), 8192), - FailCount: retryCount, + FailCount: attempts, } } // payloadProbe is a best-effort decoder that extracts the common -// identifier fields from any project payload. Every payload type is a -// JSON object with some subset of these keys; missing keys decode to -// zero values, which we then fall back through. +// identifier fields from any project payload. We deliberately limit +// the probe to field names that have a consistent meaning across every +// existing payload type: +// +// - tenant_id — uniformly the tenant uint64 +// - knowledge_base_id — wiki / chunk / image / post-process / ... +// - kb_id — FAQImportPayload's alias for the same thing +// - knowledge_id — per-document scope +// - source_kb_id — KnowledgeMovePayload (semantic: source KB) +// +// Other plausible-looking keys (`source_id`, `target_id`) are NOT +// probed: different payloads use them with different meanings (KB +// clone vs data-source sync), and a wrong scope inference would route +// dead letters under a misleading bucket. New payloads that need a +// specific scope should reuse one of the canonical keys above. +// +// Missing keys decode to zero values, which we then fall back through +// in inferScope. type payloadProbe struct { TenantID uint64 `json:"tenant_id,omitempty"` KnowledgeBaseID string `json:"knowledge_base_id,omitempty"` - KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name + KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name KnowledgeID string `json:"knowledge_id,omitempty"` - SourceID string `json:"source_id,omitempty"` // KBClonePayload - TargetID string `json:"target_id,omitempty"` // KBClonePayload - SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload - TargetKBID string `json:"target_kb_id,omitempty"` // KnowledgeMovePayload + SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload } // inferScope picks the most-specific scope tuple available in the @@ -172,9 +190,6 @@ func inferScope(p payloadProbe) (string, string) { // blast-radius indicator (it's where the work was being read // from when the failure occurred). return types.TaskScopeKnowledgeBase, p.SourceKBID - case p.SourceID != "": - // KB clone: same reasoning — record against the source. - return types.TaskScopeKnowledgeBase, p.SourceID case p.KnowledgeID != "": return types.TaskScopeKnowledge, p.KnowledgeID case p.TenantID != 0: diff --git a/internal/middleware/asynqdl/asynqdl_test.go b/internal/middleware/asynqdl/asynqdl_test.go index ad772d64..11961780 100644 --- a/internal/middleware/asynqdl/asynqdl_test.go +++ b/internal/middleware/asynqdl/asynqdl_test.go @@ -16,9 +16,9 @@ import ( // the subset of TaskDeadLetterRepository the middleware uses; the // Lister methods are unused in the middleware path. type fakeRepo struct { - mu sync.Mutex - rows []*types.TaskDeadLetter - fail error + mu sync.Mutex + rows []*types.TaskDeadLetter + fail error } func (f *fakeRepo) Insert(ctx context.Context, dl *types.TaskDeadLetter) error { @@ -112,6 +112,11 @@ func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) { if row.LastError != "boom" { t.Errorf("last_error: got %q", row.LastError) } + // Outside an asynq worker ctx GetRetryCount returns !ok, so the + // middleware records 0 attempts (clearer than a misleading 1). + if row.FailCount != 0 { + t.Errorf("fail_count: expected 0 outside worker ctx, got %d", row.FailCount) + } } func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) { @@ -193,7 +198,7 @@ func TestInferScope_Priority(t *testing.T) { }, { name: "source_kb_id from KnowledgeMovePayload", - probe: payloadProbe{SourceKBID: "src-kb", TargetKBID: "tgt-kb"}, + probe: payloadProbe{SourceKBID: "src-kb"}, wantScope: types.TaskScopeKnowledgeBase, wantID: "src-kb", },