fix(wiki-ingest,asynqdl): review fixups for PR #1241
- repo: drop r.db.Debug() from FindSimilarPages — it was dumping every
trigram probe's SQL+args (per-alias, per-item) into production logs.
- wiki_ingest dedup: fix Printf format string ("selected for %d new
items" had two args), and harden validMerge against un-prefixed
slugs whose strings.Index returned -1 and silently passed the type
check.
- wiki_ingest_batch: drop the duplicated loggedBatchSize/MapPar/
ReducePar assignments.
- asynqdl: record the real attempt count (retried + 1) on the dead
letter row instead of a hard-coded 0; tighten payloadProbe to the
set of field names with consistent semantics across payloads
(drop source_id/target_id/target_kb_id which differ by task type).
- asynqdl tests: update for the trimmed probe and assert FailCount=0
outside an asynq worker ctx so the semantics stay pinned.
This commit is contained in:
@@ -628,7 +628,7 @@ func (r *wikiPageRepository) FindSimilarPages(
|
||||
q := strings.ToLower(strings.TrimSpace(query))
|
||||
|
||||
var rows []types.WikiPageLite
|
||||
if err := r.db.Debug().WithContext(ctx).
|
||||
if err := r.db.WithContext(ctx).
|
||||
Model(&types.WikiPage{}).
|
||||
Select("slug, title, page_type, status, aliases, out_links, similarity(lower(title), ?) AS sim", q).
|
||||
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND lower(title) % ?",
|
||||
|
||||
@@ -1404,7 +1404,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
logger.Infof(ctx, "wiki ingest: no similar existing pages found for %d new items", len(entities)+len(concepts))
|
||||
return entities, concepts
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: similar existing pages selected for %d new items",
|
||||
logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items",
|
||||
len(candidatePages), len(entities)+len(concepts))
|
||||
|
||||
var existingBuf strings.Builder
|
||||
@@ -1461,8 +1461,19 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug)
|
||||
return false
|
||||
}
|
||||
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
|
||||
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
|
||||
srcSlash := strings.Index(srcSlug, "/")
|
||||
dstSlash := strings.Index(dstSlug, "/")
|
||||
if srcSlash <= 0 || dstSlash <= 0 {
|
||||
// A type-prefixed slug must look like "entity/foo" or
|
||||
// "concept/bar". An LLM that emits an un-prefixed slug
|
||||
// here is hallucinating; reject rather than fall through
|
||||
// the prefix-equality check (which would treat both empty
|
||||
// prefixes as a match).
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug)
|
||||
return false
|
||||
}
|
||||
srcPrefix := srcSlug[:srcSlash+1]
|
||||
dstPrefix := dstSlug[:dstSlash+1]
|
||||
if srcPrefix != dstPrefix {
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
|
||||
return false
|
||||
|
||||
@@ -223,9 +223,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
loggedBatchSize = batchSize
|
||||
loggedMapPar = mapParallel
|
||||
loggedReducePar = reduceParallel
|
||||
loggedBatchSize = batchSize
|
||||
loggedMapPar = mapParallel
|
||||
loggedReducePar = reduceParallel
|
||||
|
||||
lang := types.LanguageNameFromContext(ctx)
|
||||
|
||||
@@ -254,9 +251,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// callers pay only for the slugs / knowledge ids they actually
|
||||
// reach for. Cache hits keep repeat lookups within the batch free.
|
||||
var (
|
||||
fetchMu sync.Mutex
|
||||
slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing
|
||||
summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing
|
||||
fetchMu sync.Mutex
|
||||
slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing
|
||||
summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing
|
||||
)
|
||||
|
||||
resolveSlugs := func(ctx context.Context, slugs []string) map[string]string {
|
||||
@@ -351,7 +348,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
m := resolveSlugs(ctx, []string{slug})
|
||||
return m[slug]
|
||||
},
|
||||
SlugTitleMany: resolveSlugs,
|
||||
SlugTitleMany: resolveSlugs,
|
||||
SummaryContentByKnowledgeID: func(ctx context.Context, kid string) string {
|
||||
m := resolveSummaries(ctx, []string{kid})
|
||||
return m[kid]
|
||||
|
||||
@@ -59,7 +59,19 @@ func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc {
|
||||
if repo == nil {
|
||||
return err
|
||||
}
|
||||
dl := buildDeadLetter(t, err)
|
||||
// Capture the actual attempt count from asynq so the
|
||||
// dead-letter row reflects how many tries were made
|
||||
// before the task was archived. On the final attempt
|
||||
// asynq's counter satisfies retried == maxRetry, so the
|
||||
// total attempt count is retried + 1 (initial try +
|
||||
// retries). Both helpers can fail outside a worker ctx;
|
||||
// in that case we record 0 so it's clear the count
|
||||
// wasn't observable.
|
||||
attempts := 0
|
||||
if retried, ok := asynq.GetRetryCount(ctx); ok {
|
||||
attempts = retried + 1
|
||||
}
|
||||
dl := buildDeadLetter(t, err, attempts)
|
||||
if dl == nil {
|
||||
return err
|
||||
}
|
||||
@@ -103,7 +115,7 @@ func isFinalAttempt(ctx context.Context) bool {
|
||||
// Returns nil only if the task itself is nil — every other failure mode
|
||||
// (unparseable payload, unknown task type) still produces a row, with
|
||||
// scope="unknown" so operators can find it.
|
||||
func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
|
||||
func buildDeadLetter(t *asynq.Task, taskErr error, attempts int) *types.TaskDeadLetter {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -113,12 +125,6 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
|
||||
scope, scopeID := inferScope(probe)
|
||||
relatedID := probe.KnowledgeID
|
||||
|
||||
retryCount := 0
|
||||
// asynq doesn't expose the captured retry on the Task itself, so we
|
||||
// fall back to the probe's RetryCount (often 0 — most payloads don't
|
||||
// carry it). The middleware caller has already verified that we're
|
||||
// at the final attempt; the absolute number is informational.
|
||||
|
||||
// asynq stores the payload as raw bytes; preserve it verbatim so a
|
||||
// future requeue can reuse it directly.
|
||||
rawPayload := t.Payload()
|
||||
@@ -134,23 +140,35 @@ func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter {
|
||||
RelatedID: relatedID,
|
||||
Payload: json.RawMessage(rawPayload),
|
||||
LastError: truncateError(taskErr.Error(), 8192),
|
||||
FailCount: retryCount,
|
||||
FailCount: attempts,
|
||||
}
|
||||
}
|
||||
|
||||
// payloadProbe is a best-effort decoder that extracts the common
|
||||
// identifier fields from any project payload. Every payload type is a
|
||||
// JSON object with some subset of these keys; missing keys decode to
|
||||
// zero values, which we then fall back through.
|
||||
// identifier fields from any project payload. We deliberately limit
|
||||
// the probe to field names that have a consistent meaning across every
|
||||
// existing payload type:
|
||||
//
|
||||
// - tenant_id — uniformly the tenant uint64
|
||||
// - knowledge_base_id — wiki / chunk / image / post-process / ...
|
||||
// - kb_id — FAQImportPayload's alias for the same thing
|
||||
// - knowledge_id — per-document scope
|
||||
// - source_kb_id — KnowledgeMovePayload (semantic: source KB)
|
||||
//
|
||||
// Other plausible-looking keys (`source_id`, `target_id`) are NOT
|
||||
// probed: different payloads use them with different meanings (KB
|
||||
// clone vs data-source sync), and a wrong scope inference would route
|
||||
// dead letters under a misleading bucket. New payloads that need a
|
||||
// specific scope should reuse one of the canonical keys above.
|
||||
//
|
||||
// Missing keys decode to zero values, which we then fall back through
|
||||
// in inferScope.
|
||||
type payloadProbe struct {
|
||||
TenantID uint64 `json:"tenant_id,omitempty"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`
|
||||
KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name
|
||||
KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
SourceID string `json:"source_id,omitempty"` // KBClonePayload
|
||||
TargetID string `json:"target_id,omitempty"` // KBClonePayload
|
||||
SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload
|
||||
TargetKBID string `json:"target_kb_id,omitempty"` // KnowledgeMovePayload
|
||||
SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload
|
||||
}
|
||||
|
||||
// inferScope picks the most-specific scope tuple available in the
|
||||
@@ -172,9 +190,6 @@ func inferScope(p payloadProbe) (string, string) {
|
||||
// blast-radius indicator (it's where the work was being read
|
||||
// from when the failure occurred).
|
||||
return types.TaskScopeKnowledgeBase, p.SourceKBID
|
||||
case p.SourceID != "":
|
||||
// KB clone: same reasoning — record against the source.
|
||||
return types.TaskScopeKnowledgeBase, p.SourceID
|
||||
case p.KnowledgeID != "":
|
||||
return types.TaskScopeKnowledge, p.KnowledgeID
|
||||
case p.TenantID != 0:
|
||||
|
||||
@@ -16,9 +16,9 @@ import (
|
||||
// the subset of TaskDeadLetterRepository the middleware uses; the
|
||||
// Lister methods are unused in the middleware path.
|
||||
type fakeRepo struct {
|
||||
mu sync.Mutex
|
||||
rows []*types.TaskDeadLetter
|
||||
fail error
|
||||
mu sync.Mutex
|
||||
rows []*types.TaskDeadLetter
|
||||
fail error
|
||||
}
|
||||
|
||||
func (f *fakeRepo) Insert(ctx context.Context, dl *types.TaskDeadLetter) error {
|
||||
@@ -112,6 +112,11 @@ func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) {
|
||||
if row.LastError != "boom" {
|
||||
t.Errorf("last_error: got %q", row.LastError)
|
||||
}
|
||||
// Outside an asynq worker ctx GetRetryCount returns !ok, so the
|
||||
// middleware records 0 attempts (clearer than a misleading 1).
|
||||
if row.FailCount != 0 {
|
||||
t.Errorf("fail_count: expected 0 outside worker ctx, got %d", row.FailCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) {
|
||||
@@ -193,7 +198,7 @@ func TestInferScope_Priority(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "source_kb_id from KnowledgeMovePayload",
|
||||
probe: payloadProbe{SourceKBID: "src-kb", TargetKBID: "tgt-kb"},
|
||||
probe: payloadProbe{SourceKBID: "src-kb"},
|
||||
wantScope: types.TaskScopeKnowledgeBase,
|
||||
wantID: "src-kb",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user