414749681f
Addresses three review concerns from the prior PR:
1. No tests existed for any of the stuck-parsing fixes (PR① / PR②.5).
This commit adds coverage for the four most regression-prone
surfaces: span repo upsert/cascade/attempt isolation, span
tracker cascade-cancel and cross-process LookupStage,
housekeeping false-kill protection, and handler tree assembly.
2. Housekeeping was using only knowledge.updated_at as its staleness
signal, but knowledge.updated_at advances only at parse_status
transitions — a long DocReader call (or large embedding batch)
can run for an hour with no updated_at change, so a tight
DocumentProcessTimeout setting would falsely flip an actively
running parse to "failed".
The sweep now does a two-stage check: candidates by knowledge
updated_at, then filtered by MAX(spans.updated_at). Every
SpanTracker Begin/End/Fail/Skip now also pokes
knowledge.updated_at as a side-channel heartbeat, so the
filter sees recent activity even when no parse_status
transition has fired.
3. parseHeartbeatTime accepts the timestamp formats both Postgres
and SQLite emit for an aggregated MAX() column (the SQLite
driver doesn't auto-cast aggregates to time.Time the way
Postgres does), so the same code path works in Lite mode.
The new TestHousekeeping_NoFalseKill_ActiveSpan is the regression
test for the user-flagged scenario: a 3-hour-stale knowledge.updated_at
combined with a 2-minute-fresh span row must NOT be killed.
278 lines
10 KiB
Go
278 lines
10 KiB
Go
// Package service: knowledge housekeeping.
|
|
//
|
|
// HousekeepingService periodically scans for knowledge rows that have been
|
|
// stuck in "processing" longer than any reasonable execution window and
|
|
// marks them as failed. This is the safety net that catches anything the
|
|
// other defences (asynq retry, dead-letter callback, image_multimodal
|
|
// finalize-on-last-attempt) miss — for example:
|
|
//
|
|
// - Worker process killed mid-handler before any defer could run.
|
|
// - DocReader call genuinely exceeding DocReaderCallTimeout AND the
|
|
// worker subsequently being lost before retry kicks in.
|
|
// - Multimodal Redis counter set to N but ALL N image tasks failing in
|
|
// ways that bypass finalize (extremely rare; defence-in-depth here).
|
|
//
|
|
// Without this sweep, a single unlucky failure mode can leave a knowledge
|
|
// row in "processing" forever — invisible to users except as a permanent
|
|
// spinner. With this sweep the worst-case latency from stall to user-
|
|
// visible failure is bounded to ~1 stale-threshold + 1 sweep interval.
|
|
package service
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/robfig/cron/v3"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// HousekeepingService runs background sweeps to recover stuck rows.
|
|
type HousekeepingService struct {
|
|
db *gorm.DB
|
|
cfg *config.Config
|
|
cron *cron.Cron
|
|
|
|
mu sync.Mutex
|
|
started bool
|
|
}
|
|
|
|
// NewHousekeepingService constructs a HousekeepingService. It does NOT start
|
|
// the cron — call Start in the application bootstrap so a misconfigured
|
|
// cron schedule cannot prevent the rest of the service from coming up.
|
|
func NewHousekeepingService(db *gorm.DB, cfg *config.Config) *HousekeepingService {
|
|
return &HousekeepingService{
|
|
db: db,
|
|
cfg: cfg,
|
|
cron: cron.New(cron.WithSeconds(), cron.WithChain(
|
|
cron.Recover(cron.DefaultLogger),
|
|
)),
|
|
}
|
|
}
|
|
|
|
// Start registers the sweep schedule and begins the background runner.
|
|
// Idempotent — repeated calls are a no-op so wiring code can call Start
|
|
// without coordinating ordering.
|
|
func (h *HousekeepingService) Start(ctx context.Context) error {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.started {
|
|
return nil
|
|
}
|
|
if !housekeepingEnabled() {
|
|
logger.Infof(ctx, "[Housekeeping] disabled via WEKNORA_HOUSEKEEPING_ENABLED=false")
|
|
return nil
|
|
}
|
|
// Every 5 minutes — frequent enough that user-visible recovery latency
|
|
// is acceptable, infrequent enough that the SQL sweep is invisible to
|
|
// query load even on large knowledge tables.
|
|
if _, err := h.cron.AddFunc("0 */5 * * * *", func() {
|
|
// Use Background so a cancelled bootstrap ctx doesn't stop sweeps.
|
|
h.runSweep(context.Background())
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
h.cron.Start()
|
|
h.started = true
|
|
logger.Infof(ctx, "[Housekeeping] started with 5-minute sweep")
|
|
return nil
|
|
}
|
|
|
|
// Stop halts the cron and waits for in-flight sweeps to finish.
|
|
func (h *HousekeepingService) Stop() {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if !h.started {
|
|
return
|
|
}
|
|
c := h.cron.Stop()
|
|
<-c.Done()
|
|
h.started = false
|
|
}
|
|
|
|
// runSweep is exported on the type for testability — tests can drive a
|
|
// single sweep without waiting for the cron tick.
|
|
func (h *HousekeepingService) runSweep(ctx context.Context) {
|
|
threshold := h.staleThreshold()
|
|
cutoff := time.Now().Add(-threshold)
|
|
|
|
// Sweep A: knowledge stuck in "processing".
|
|
//
|
|
// Two-stage check is critical here: knowledge.updated_at advances
|
|
// only at parse_status transitions, but a long stage (DocReader on
|
|
// a 500MB PDF, embedding 5K chunks) can run for an hour with no
|
|
// status change. Using updated_at alone would falsely kill that
|
|
// run. So we OR-combine knowledge.updated_at with the most recent
|
|
// span row's updated_at — every Begin/End/Fail/Skip from
|
|
// SpanTracker bumps the span row, so an actively-progressing
|
|
// pipeline always has a recent span heartbeat even when the
|
|
// parent knowledge row is "frozen" mid-stage.
|
|
//
|
|
// Knowledge rows with no spans at all (lite mode, in-flight tasks
|
|
// from before this code shipped) fall back to the simple
|
|
// updated_at check — they have no heartbeat to consult.
|
|
var candidates []types.Knowledge
|
|
if err := h.db.WithContext(ctx).
|
|
Where("parse_status = ? AND updated_at < ?", types.ParseStatusProcessing, cutoff).
|
|
Find(&candidates).Error; err != nil {
|
|
logger.Warnf(ctx, "[Housekeeping] knowledge candidate query failed: %v", err)
|
|
return
|
|
}
|
|
|
|
stuck := h.filterByLastSpanActivity(ctx, candidates, cutoff)
|
|
if len(stuck) > 0 {
|
|
stuckIDs := make([]string, 0, len(stuck))
|
|
for _, k := range stuck {
|
|
stuckIDs = append(stuckIDs, k.ID)
|
|
}
|
|
res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("id IN ? AND parse_status = ?", stuckIDs, types.ParseStatusProcessing).
|
|
Updates(map[string]interface{}{
|
|
"parse_status": types.ParseStatusFailed,
|
|
"error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping",
|
|
})
|
|
if res.Error != nil {
|
|
logger.Warnf(ctx, "[Housekeeping] knowledge sweep update failed: %v", res.Error)
|
|
} else if res.RowsAffected > 0 {
|
|
logger.Infof(ctx, "[Housekeeping] recovered %d stuck knowledge rows (threshold=%s)",
|
|
res.RowsAffected, threshold)
|
|
}
|
|
}
|
|
if active := len(candidates) - len(stuck); active > 0 {
|
|
// Visibility into "we considered killing N rows but their
|
|
// span tree showed they're still progressing". Ops can grep
|
|
// for this if they suspect housekeeping over- or under-fires.
|
|
logger.Infof(ctx,
|
|
"[Housekeeping] %d candidate(s) skipped — span heartbeat within threshold",
|
|
active)
|
|
}
|
|
|
|
// Sweep B: knowledge summary stuck. Summary is post-parse; threshold
|
|
// is shorter because summary tasks are bounded by a single LLM call.
|
|
// No span heartbeat exists for the summary stage (it lives in a
|
|
// downstream asynq task), so we accept the original simple check.
|
|
summaryCutoff := time.Now().Add(-1 * time.Hour)
|
|
resSummary := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("summary_status = ? AND updated_at < ?", types.SummaryStatusProcessing, summaryCutoff).
|
|
Update("summary_status", types.SummaryStatusFailed)
|
|
if resSummary.Error != nil {
|
|
logger.Warnf(ctx, "[Housekeeping] summary sweep failed: %v", resSummary.Error)
|
|
} else if resSummary.RowsAffected > 0 {
|
|
logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)
|
|
}
|
|
}
|
|
|
|
// filterByLastSpanActivity returns the subset of candidates whose most
|
|
// recent span row predates `cutoff` — i.e. genuinely stuck. Candidates
|
|
// with no span rows at all also pass through (they're lite-mode or
|
|
// pre-instrumentation tasks; the simple updated_at check already proved
|
|
// them stuck and we have no heartbeat to override that).
|
|
func (h *HousekeepingService) filterByLastSpanActivity(ctx context.Context, candidates []types.Knowledge, cutoff time.Time) []types.Knowledge {
|
|
if len(candidates) == 0 {
|
|
return candidates
|
|
}
|
|
ids := make([]string, 0, len(candidates))
|
|
for _, k := range candidates {
|
|
ids = append(ids, k.ID)
|
|
}
|
|
|
|
// We scan MAX(updated_at) as string then parse client-side. That
|
|
// dodges the SQLite driver's well-known refusal to auto-convert
|
|
// aggregate datetime values into time.Time on its own — Postgres
|
|
// happily round-trips, but the same query shape must work in
|
|
// Lite mode too. Since we only compare against a cutoff, the
|
|
// parse layer below tries the formats both Postgres and SQLite
|
|
// emit and takes the first that parses.
|
|
type spanHeartbeat struct {
|
|
KnowledgeID string `gorm:"column:knowledge_id"`
|
|
LastSeen string `gorm:"column:last_seen"`
|
|
}
|
|
var beats []spanHeartbeat
|
|
err := h.db.WithContext(ctx).
|
|
Table("knowledge_processing_spans").
|
|
Select("knowledge_id, MAX(updated_at) AS last_seen").
|
|
Where("knowledge_id IN ?", ids).
|
|
Group("knowledge_id").
|
|
Find(&beats).Error
|
|
if err != nil {
|
|
// On query failure, fail safe — assume nothing has a
|
|
// heartbeat (so all candidates are "stuck"). This matches
|
|
// the previous-version behaviour and never under-recovers.
|
|
logger.Warnf(ctx, "[Housekeeping] span heartbeat query failed: %v (will fail safe and recover all candidates)", err)
|
|
return candidates
|
|
}
|
|
heartbeat := make(map[string]time.Time, len(beats))
|
|
for _, b := range beats {
|
|
if t, ok := parseHeartbeatTime(b.LastSeen); ok {
|
|
heartbeat[b.KnowledgeID] = t
|
|
}
|
|
}
|
|
|
|
out := candidates[:0]
|
|
for _, k := range candidates {
|
|
if last, ok := heartbeat[k.ID]; ok && last.After(cutoff) {
|
|
// Active span heartbeat — leave alone.
|
|
continue
|
|
}
|
|
out = append(out, k)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// parseHeartbeatTime accepts the timestamp formats Postgres and SQLite
|
|
// emit for a TIMESTAMP column read back through MAX(). Returns false if
|
|
// none parse — the caller treats unparseable rows as "no heartbeat",
|
|
// which fails safe (the row gets recovered as stuck rather than
|
|
// silently preserved).
|
|
func parseHeartbeatTime(s string) (time.Time, bool) {
|
|
if s == "" {
|
|
return time.Time{}, false
|
|
}
|
|
for _, layout := range []string{
|
|
time.RFC3339Nano,
|
|
time.RFC3339,
|
|
"2006-01-02 15:04:05.999999999-07:00",
|
|
"2006-01-02 15:04:05.999999999",
|
|
"2006-01-02 15:04:05.999999",
|
|
"2006-01-02 15:04:05",
|
|
} {
|
|
if t, err := time.Parse(layout, s); err == nil {
|
|
return t, true
|
|
}
|
|
}
|
|
return time.Time{}, false
|
|
}
|
|
|
|
// staleThreshold returns how long a "processing" row may sit untouched
|
|
// before housekeeping treats it as orphaned. The floor is 1 hour so that a
|
|
// genuinely slow large-PDF parse cannot be killed mid-flight; the ceiling
|
|
// scales with the operator-configured DocumentProcessTimeout plus 10 minute
|
|
// buffer to absorb scheduling jitter.
|
|
func (h *HousekeepingService) staleThreshold() time.Duration {
|
|
base := 1 * time.Hour
|
|
if h.cfg != nil && h.cfg.KnowledgeBase != nil && h.cfg.KnowledgeBase.DocumentProcessTimeout > base {
|
|
base = h.cfg.KnowledgeBase.DocumentProcessTimeout
|
|
}
|
|
return base + 10*time.Minute
|
|
}
|
|
|
|
func housekeepingEnabled() bool {
|
|
// Default-on: missing/empty env enables the sweep. Operators must
|
|
// explicitly set "false" to opt out, matching the plan's commitment
|
|
// that no env change is required for the safety net to engage.
|
|
v := strings.TrimSpace(os.Getenv("WEKNORA_HOUSEKEEPING_ENABLED"))
|
|
if v == "" {
|
|
return true
|
|
}
|
|
switch strings.ToLower(v) {
|
|
case "0", "false", "off", "no":
|
|
return false
|
|
}
|
|
return true
|
|
}
|