fix(wiki-ingest,asynqdl): review fixups for PR #1241

- repo: drop r.db.Debug() from FindSimilarPages — it was dumping every
  trigram probe's SQL+args (per-alias, per-item) into production logs.
- wiki_ingest dedup: fix Printf format string ("selected for %d new
  items" had two args), and harden validMerge against un-prefixed
  slugs whose strings.Index returned -1 and silently passed the type
  check.
- wiki_ingest_batch: drop the duplicated loggedBatchSize/MapPar/
  ReducePar assignments.
- asynqdl: record the real attempt count (retried + 1) on the dead
  letter row instead of a hard-coded 0; tighten payloadProbe to the
  set of field names with consistent semantics across payloads
  (drop source_id/target_id/target_kb_id which differ by task type).
- asynqdl tests: update for the trimmed probe and assert FailCount=0
  outside an asynq worker ctx so the semantics stay pinned.
This commit is contained in:
wizardchen
2026-05-09 23:59:12 +08:00
committed by lyingbug
parent 04e8105728
commit 82947c726e
5 changed files with 63 additions and 35 deletions
+1 -1
View File
@@ -628,7 +628,7 @@ func (r *wikiPageRepository) FindSimilarPages(
q := strings.ToLower(strings.TrimSpace(query))
var rows []types.WikiPageLite
if err := r.db.Debug().WithContext(ctx).
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Select("slug, title, page_type, status, aliases, out_links, similarity(lower(title), ?) AS sim", q).
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND lower(title) % ?",
+14 -3
View File
@@ -1404,7 +1404,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
logger.Infof(ctx, "wiki ingest: no similar existing pages found for %d new items", len(entities)+len(concepts))
return entities, concepts
}
logger.Infof(ctx, "wiki ingest: similar existing pages selected for %d new items",
logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items",
len(candidatePages), len(entities)+len(concepts))
var existingBuf strings.Builder
@@ -1461,8 +1461,19 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug)
return false
}
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
srcSlash := strings.Index(srcSlug, "/")
dstSlash := strings.Index(dstSlug, "/")
if srcSlash <= 0 || dstSlash <= 0 {
// A type-prefixed slug must look like "entity/foo" or
// "concept/bar". An LLM that emits an un-prefixed slug
// here is hallucinating; reject rather than fall through
// the prefix-equality check (which would treat both empty
// prefixes as a match).
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug)
return false
}
srcPrefix := srcSlug[:srcSlash+1]
dstPrefix := dstSlug[:dstSlash+1]
if srcPrefix != dstPrefix {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
return false
@@ -223,9 +223,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
loggedBatchSize = batchSize
loggedMapPar = mapParallel
loggedReducePar = reduceParallel
loggedBatchSize = batchSize
loggedMapPar = mapParallel
loggedReducePar = reduceParallel
lang := types.LanguageNameFromContext(ctx)
@@ -254,9 +251,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// callers pay only for the slugs / knowledge ids they actually
// reach for. Cache hits keep repeat lookups within the batch free.
var (
fetchMu sync.Mutex
slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing
summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing
fetchMu sync.Mutex
slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing
summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing
)
resolveSlugs := func(ctx context.Context, slugs []string) map[string]string {
@@ -351,7 +348,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
m := resolveSlugs(ctx, []string{slug})
return m[slug]
},
SlugTitleMany: resolveSlugs,
SlugTitleMany: resolveSlugs,
SummaryContentByKnowledgeID: func(ctx context.Context, kid string) string {
m := resolveSummaries(ctx, []string{kid})
return m[kid]
+35 -20
View File
@@ -59,7 +59,19 @@ func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc {
if repo == nil {
return err
}
dl := buildDeadLetter(t, err)
// Capture the actual attempt count from asynq so the
// dead-letter row reflects how many tries were made
// before the task was archived. On the final attempt
// asynq's counter satisfies retried == maxRetry, so the
// total attempt count is retried + 1 (initial try +
// retries). Both helpers can fail outside a worker ctx;
// in that case we record 0 so it's clear the count
// wasn't observable.
attempts := 0
if retried, ok := asynq.GetRetryCount(ctx); ok {
attempts = retried + 1
}
dl := buildDeadLetter(t, err, attempts)
if dl == nil {
return err
}
@@ -103,7 +115,7 @@ func isFinalAttempt(ctx context.Context) bool {
// Returns nil only if the task itself is nil — every other failure mode
// (unparseable payload, unknown task type) still produces a row, with
// scope="unknown" so operators can find it.
func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
func buildDeadLetter(t *asynq.Task, taskErr error, attempts int) *types.TaskDeadLetter {
if t == nil {
return nil
}
@@ -113,12 +125,6 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
scope, scopeID := inferScope(probe)
relatedID := probe.KnowledgeID
retryCount := 0
// asynq doesn't expose the captured retry on the Task itself, so we
// fall back to the probe's RetryCount (often 0 — most payloads don't
// carry it). The middleware caller has already verified that we're
// at the final attempt; the absolute number is informational.
// asynq stores the payload as raw bytes; preserve it verbatim so a
// future requeue can reuse it directly.
rawPayload := t.Payload()
@@ -134,23 +140,35 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
RelatedID: relatedID,
Payload: json.RawMessage(rawPayload),
LastError: truncateError(taskErr.Error(), 8192),
FailCount: retryCount,
FailCount: attempts,
}
}
// payloadProbe is a best-effort decoder that extracts the common
// identifier fields from any project payload. Every payload type is a
// JSON object with some subset of these keys; missing keys decode to
// zero values, which we then fall back through.
// identifier fields from any project payload. We deliberately limit
// the probe to field names that have a consistent meaning across every
// existing payload type:
//
// - tenant_id — uniformly the tenant uint64
// - knowledge_base_id — wiki / chunk / image / post-process / ...
// - kb_id — FAQImportPayload's alias for the same thing
// - knowledge_id — per-document scope
// - source_kb_id — KnowledgeMovePayload (semantic: source KB)
//
// Other plausible-looking keys (`source_id`, `target_id`) are NOT
// probed: different payloads use them with different meanings (KB
// clone vs data-source sync), and a wrong scope inference would route
// dead letters under a misleading bucket. New payloads that need a
// specific scope should reuse one of the canonical keys above.
//
// Missing keys decode to zero values, which we then fall back through
// in inferScope.
type payloadProbe struct {
TenantID uint64 `json:"tenant_id,omitempty"`
KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`
KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name
KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name
KnowledgeID string `json:"knowledge_id,omitempty"`
SourceID string `json:"source_id,omitempty"` // KBClonePayload
TargetID string `json:"target_id,omitempty"` // KBClonePayload
SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload
TargetKBID string `json:"target_kb_id,omitempty"` // KnowledgeMovePayload
SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload
}
// inferScope picks the most-specific scope tuple available in the
@@ -172,9 +190,6 @@ func inferScope(p payloadProbe) (string, string) {
// blast-radius indicator (it's where the work was being read
// from when the failure occurred).
return types.TaskScopeKnowledgeBase, p.SourceKBID
case p.SourceID != "":
// KB clone: same reasoning — record against the source.
return types.TaskScopeKnowledgeBase, p.SourceID
case p.KnowledgeID != "":
return types.TaskScopeKnowledge, p.KnowledgeID
case p.TenantID != 0:
+9 -4
View File
@@ -16,9 +16,9 @@ import (
// the subset of TaskDeadLetterRepository the middleware uses; the
// Lister methods are unused in the middleware path.
type fakeRepo struct {
mu sync.Mutex
rows []*types.TaskDeadLetter
fail error
mu sync.Mutex
rows []*types.TaskDeadLetter
fail error
}
func (f *fakeRepo) Insert(ctx context.Context, dl *types.TaskDeadLetter) error {
@@ -112,6 +112,11 @@ func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) {
if row.LastError != "boom" {
t.Errorf("last_error: got %q", row.LastError)
}
// Outside an asynq worker ctx GetRetryCount returns !ok, so the
// middleware records 0 attempts (clearer than a misleading 1).
if row.FailCount != 0 {
t.Errorf("fail_count: expected 0 outside worker ctx, got %d", row.FailCount)
}
}
func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) {
@@ -193,7 +198,7 @@ func TestInferScope_Priority(t *testing.T) {
},
{
name: "source_kb_id from KnowledgeMovePayload",
probe: payloadProbe{SourceKBID: "src-kb", TargetKBID: "tgt-kb"},
probe: payloadProbe{SourceKBID: "src-kb"},
wantScope: types.TaskScopeKnowledgeBase,
wantID: "src-kb",
},