Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_housekeeping_test.go
T
wizardchen 414749681f test(knowledge): add coverage and protect housekeeping from false-killing long stages
Addresses three review concerns from the prior PR:

  1. No tests existed for any of the stuck-parsing fixes (PR① / PR②.5).
     This commit adds coverage for the four most regression-prone
     surfaces: span repo upsert/cascade/attempt isolation, span
     tracker cascade-cancel and cross-process LookupStage,
     housekeeping false-kill protection, and handler tree assembly.

  2. Housekeeping was using only knowledge.updated_at as its staleness
     signal, but knowledge.updated_at advances only at parse_status
     transitions — a long DocReader call (or large embedding batch)
     can run for an hour with no updated_at change, so a tight
     DocumentProcessTimeout setting would falsely flip an actively
     running parse to "failed".

     The sweep now does a two-stage check: candidates by knowledge
     updated_at, then filtered by MAX(spans.updated_at). Every
     SpanTracker Begin/End/Fail/Skip now also pokes
     knowledge.updated_at as a side-channel heartbeat, so the
     filter sees recent activity even when no parse_status
     transition has fired.

  3. parseHeartbeatTime accepts the timestamp formats both Postgres
     and SQLite emit for an aggregated MAX() column (the SQLite
     driver doesn't auto-cast aggregates to time.Time the way
     Postgres does), so the same code path works in Lite mode.

The new TestHousekeeping_NoFalseKill_ActiveSpan is the regression
test for the user-flagged scenario: a 3-hour-stale knowledge.updated_at
combined with a 2-minute-fresh span row must NOT be killed.
2026-05-28 15:14:45 +08:00

189 lines
7.1 KiB
Go

package service
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// knowledgeTestDDL is the minimal subset of the knowledge schema this
// suite needs. We avoid AutoMigrate because Knowledge carries multiple
// JSONB-tagged fields whose SQLite mapping is fragile.
//
// Table name is `knowledges` (plural) — that's what migration 000000
// creates and what GORM's default pluralization expects when the
// service code uses Model(&types.Knowledge{}).
const knowledgeTestDDL = `
CREATE TABLE IF NOT EXISTS knowledges (
id VARCHAR(64) PRIMARY KEY,
tenant_id INTEGER NOT NULL DEFAULT 0,
knowledge_base_id VARCHAR(64),
parse_status VARCHAR(32) NOT NULL DEFAULT 'pending',
summary_status VARCHAR(32) NOT NULL DEFAULT 'none',
error_message TEXT,
title TEXT,
file_type TEXT,
enable_status TEXT NOT NULL DEFAULT 'enabled',
type TEXT NOT NULL DEFAULT 'document',
embedding_model_id TEXT NOT NULL DEFAULT '',
storage_size BIGINT NOT NULL DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
`
const housekeepingSpansDDL = `
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
knowledge_id VARCHAR(64) NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
span_id VARCHAR(64) NOT NULL,
parent_span_id VARCHAR(64),
name VARCHAR(64) NOT NULL,
kind VARCHAR(16) NOT NULL,
status VARCHAR(16) NOT NULL,
input TEXT,
output TEXT,
metadata TEXT,
error_code VARCHAR(64),
error_message TEXT,
error_detail TEXT,
started_at DATETIME,
finished_at DATETIME,
duration_ms BIGINT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE (knowledge_id, attempt, span_id)
);
`
func setupHousekeepingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(knowledgeTestDDL).Error)
require.NoError(t, db.Exec(housekeepingSpansDDL).Error)
return db
}
// insertKnowledge writes a knowledge row at the given updated_at. We
// can't pass updated_at through GORM defaults since CURRENT_TIMESTAMP
// would override our test fixture; raw SQL keeps the timestamp.
func insertKnowledge(t *testing.T, db *gorm.DB, id, status string, updatedAt time.Time) {
t.Helper()
require.NoError(t, db.Exec(
`INSERT INTO knowledges (id, parse_status, updated_at) VALUES (?, ?, ?)`,
id, status, updatedAt,
).Error)
}
func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, status string, updatedAt time.Time) {
t.Helper()
require.NoError(t, db.Exec(
`INSERT INTO knowledge_processing_spans (knowledge_id, attempt, span_id, name, kind, status, updated_at)
VALUES (?, ?, ?, 'docreader', 'stage', ?, ?)`,
kid, attempt, spanID, status, updatedAt,
).Error)
}
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
cfg := &config.Config{KnowledgeBase: &config.KnowledgeBaseConfig{
// 1h floor + 10min buffer = 70min cutoff. Tight enough to keep
// the test's relative timestamps in seconds; the production
// default of 2h+10min is just a constant scale factor.
DocumentProcessTimeout: 1 * time.Hour,
}}
return NewHousekeepingService(db, cfg)
}
// TestHousekeeping_RecoversAbandoned exercises the happy path: a
// knowledge stuck at "processing" with no recent heartbeat (no spans,
// stale knowledge.updated_at) MUST be flipped to failed.
func TestHousekeeping_RecoversAbandoned(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
stale := time.Now().Add(-3 * time.Hour) // well past 70min cutoff
insertKnowledge(t, db, "kid-abandoned", types.ParseStatusProcessing, stale)
svc.runSweep(context.Background())
var status, errMsg string
require.NoError(t, db.Raw(
`SELECT parse_status, error_message FROM knowledges WHERE id = ?`, "kid-abandoned",
).Row().Scan(&status, &errMsg))
assert.Equal(t, types.ParseStatusFailed, status)
assert.Contains(t, errMsg, "stuck in processing")
}
// TestHousekeeping_NoFalseKill_ActiveSpan is the regression test for
// the "long DocReader silently runs longer than DocumentProcessTimeout"
// scenario the user flagged. A knowledge whose knowledge.updated_at
// looks stale BUT whose span tree shows recent activity must NOT be
// killed.
func TestHousekeeping_NoFalseKill_ActiveSpan(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
stale := time.Now().Add(-3 * time.Hour)
insertKnowledge(t, db, "kid-active", types.ParseStatusProcessing, stale)
// Span heartbeat well within the 70min cutoff — it represents
// "we're STILL working, the worker just hasn't transitioned the
// parse_status column yet".
insertSpan(t, db, "kid-active", 1, "docreader-1", types.SpanStatusRunning, time.Now().Add(-2*time.Minute))
svc.runSweep(context.Background())
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-active",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusProcessing, status,
"knowledge with recent span heartbeat must NOT be flipped to failed")
}
// TestHousekeeping_NoFalseKill_StaleSpanRecovers confirms the inverse:
// a knowledge whose span tree has ALSO gone silent past the threshold
// is genuinely stuck and must be recovered.
func TestHousekeeping_NoFalseKill_StaleSpanRecovers(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
stale := time.Now().Add(-3 * time.Hour)
insertKnowledge(t, db, "kid-stuck", types.ParseStatusProcessing, stale)
// Span row stale by the same amount — no recent activity anywhere.
insertSpan(t, db, "kid-stuck", 1, "docreader-1", types.SpanStatusRunning, stale)
svc.runSweep(context.Background())
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-stuck",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusFailed, status,
"genuinely stuck knowledge (knowledge AND spans both stale) must still be recovered")
}
// TestHousekeeping_PreservesRecentlyTouched: any knowledge whose
// updated_at is within the cutoff is left alone — that's the cheap
// fast path that doesn't even consult the spans table.
func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
insertKnowledge(t, db, "kid-fresh", types.ParseStatusProcessing, time.Now().Add(-30*time.Second))
svc.runSweep(context.Background())
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-fresh",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusProcessing, status,
"knowledge updated within the cutoff must be left alone")
}