姚章浩
1d6ba6dc2e
- The attention lane was derived from page 1 of the queue (limit 50, newest by updated_at), so failures ranked past 50 silently disappeared from "needs attention" as the completed backlog grew. - Fetch attention items with a dedicated request: the API now accepts a comma-separated processing_state list (failed,stalled) and the view prefers that result set over the paged queue. - Batch retry/cancel targets are computed from the same failure set instead of the visible page, so actions reach every failing item. - 全部任务 shows 已显示 N / 共 M plus 加载更多 so rows past 50 are reachable. - Fix pre-existing ui_settings integration test: strict JSON decoding rejects the legacy updated_by field (actor comes from the session). - Tests: view regression (attention item outside the page, load-more affordance) + handler multi-state filter + isolated-DB set filter.
185 lines
8.4 KiB
Go
185 lines
8.4 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
// Documents whose source is permanently broken (DingTalk exports an empty
|
|
// body) must be exitable without deleting anything: the operator cancels the
|
|
// publication, its queued upload job is withdrawn, its open dead letter is
|
|
// abandoned, the attention lane drops it, and re-marking can still create a
|
|
// fresh publication later.
|
|
func TestCancelBatchRetiresFailuresWithoutLosingSource(t *testing.T) {
|
|
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
|
if databaseURL == "" {
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
|
}
|
|
ctx := context.Background()
|
|
pool, err := pgxpool.New(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
if err := ApplyMigrations(ctx, pool); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = pool.Exec(ctx, `
|
|
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-cancel','dingtalk','取消来源',true);
|
|
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-cancel',repeat('b',64),4681,'verified/cancel.docx','verified');
|
|
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
|
|
VALUES('item-cancel','conn-cancel','ext-cancel',repeat('c',64),'开箱.adoc','document','r1','internal','manual');
|
|
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('siv-cancel','item-cancel','r1','co-cancel','verified');
|
|
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,last_error,updated_at) VALUES
|
|
('tp-cancel-1','siv-cancel','kb-1','uploaded','failed','unpublished','internal','remote_parse_failed','2026-09-15T04:00:00Z'),
|
|
('tp-cancel-2','siv-cancel','kb-2','uploaded','stalled','unpublished','internal','stalled','2026-09-15T04:00:00Z');`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// tp-cancel-1 also owns one open dead letter and one queued upload job.
|
|
uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-cancel-1"})
|
|
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key) VALUES
|
|
('job-cancel-dlq','test.upload',$1,'failed',5,5,'k-cancel-dlq'),
|
|
('job-cancel-queued','test.upload',$1,'pending',0,5,'k-cancel-queued')`, uploadPayload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO job_failures(job_id,job_type,publication_id,error_code,last_error,attempt,max_attempts_at_failure,state,created_at)
|
|
VALUES('job-cancel-dlq','test.upload','tp-cancel-1','remote_parse_failed','remote_parse_failed',5,5,'open',NOW())`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := NewStore(pool)
|
|
now := time.Date(2026, 9, 15, 9, 0, 0, 0, time.UTC)
|
|
|
|
result, replayed, err := store.CancelPublicationsBatchWithOperation(ctx, governance.CancelPublicationsBatchRequest{
|
|
PublicationIDs: []string{"tp-cancel-1", "tp-cancel-2", "tp-nope"},
|
|
OperatorRef: "admin-cancel", Reason: "源头为空文档，不再重试", Now: now,
|
|
}, "op-cancel-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if replayed {
|
|
t.Fatal("first cancel must not be a replay")
|
|
}
|
|
if result.Started != 2 || result.NotRetryable != 1 {
|
|
t.Fatalf("result=%#v", result)
|
|
}
|
|
|
|
var processingState string
|
|
if err := pool.QueryRow(ctx, `SELECT processing_state FROM test_publications WHERE id='tp-cancel-1'`).Scan(&processingState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if processingState != "cancelled" {
|
|
t.Fatalf("tp-1 state = %s, want cancelled", processingState)
|
|
}
|
|
// Queued upload withdrawn, dead letter abandoned with operator+reason.
|
|
var jobState string
|
|
if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-cancel-queued'`).Scan(&jobState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if jobState != "cancelled" {
|
|
t.Fatalf("queued job state = %s, want cancelled", jobState)
|
|
}
|
|
var failureState, actor, reason string
|
|
if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-cancel-dlq'`).Scan(&failureState, &actor, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if failureState != "abandoned" || actor != "admin-cancel" || reason != "源头为空文档，不再重试" {
|
|
t.Fatalf("failure row = (%s,%s,%s)", failureState, actor, reason)
|
|
}
|
|
// Source records and the spool artifact survive: nothing destructive.
|
|
var spoolRef string
|
|
if err := pool.QueryRow(ctx, `SELECT local_spool_ref FROM content_objects WHERE id='co-cancel'`).Scan(&spoolRef); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if spoolRef != "verified/cancel.docx" {
|
|
t.Fatalf("spool ref = %s, must remain for re-marking", spoolRef)
|
|
}
|
|
// Attention projection: cancelled rows are neither retryable nor startable.
|
|
page, err := store.ListTestPublications(ctx, governance.TestPublicationFilter{ProcessingState: "cancelled", Limit: 10})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if page.Total != 2 || len(page.Items) != 2 {
|
|
t.Fatalf("cancelled page = %d/%d, want 2", len(page.Items), page.Total)
|
|
}
|
|
for _, item := range page.Items {
|
|
if item.CanRetry || item.CanStart {
|
|
t.Fatalf("cancelled publication %s still actionable (can_retry=%v can_start=%v)", item.ID, item.CanRetry, item.CanStart)
|
|
}
|
|
}
|
|
// Replaying the same operation is idempotent and reports replayed.
|
|
replay, replayed, err := store.CancelPublicationsBatchWithOperation(ctx, governance.CancelPublicationsBatchRequest{
|
|
PublicationIDs: []string{"tp-cancel-1", "tp-cancel-2", "tp-nope"},
|
|
OperatorRef: "admin-cancel", Reason: "源头为空文档，不再重试", Now: now,
|
|
}, "op-cancel-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !replayed || replay.Started != 2 {
|
|
t.Fatalf("replay=%#v replayed=%v", replay, replayed)
|
|
}
|
|
// A cancelled publication is no longer retryable, but re-marking the
|
|
// source item can supersede it: the row stays linked to the version.
|
|
var versionID string
|
|
if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE id='tp-cancel-1'`).Scan(&versionID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if versionID != "siv-cancel" {
|
|
t.Fatal("cancelled publication lost its version link")
|
|
}
|
|
}
|
|
|
|
// The attention lane lists every failing publication in one request, so the
|
|
// processing_state filter has to behave as a set — a single literal would make
|
|
// failures silently vanish again as soon as the queue grows past one page.
|
|
func TestListPublicationsAcceptsStateSet(t *testing.T) {
|
|
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
|
if databaseURL == "" {
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
|
}
|
|
ctx := context.Background()
|
|
pool, err := pgxpool.New(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
if err := ApplyMigrations(ctx, pool); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `
|
|
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-set','dingtalk','集合来源',true);
|
|
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-set',repeat('d',64),4891,'verified/set.docx','verified');
|
|
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
|
|
VALUES('item-set','conn-set','ext-set',repeat('e',64),'FAQ.adoc','document','r1','internal','manual');
|
|
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('siv-set','item-set','r1','co-set','verified');
|
|
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,last_error,updated_at) VALUES
|
|
('tp-set-failed','siv-set','kb-set-1','uploaded','failed','unpublished','internal','remote_parse_failed',NOW()),
|
|
('tp-set-stalled','siv-set','kb-set-2','uploaded','stalled','unpublished','internal','stalled',NOW()),
|
|
('tp-set-done','siv-set','kb-set-3','uploaded','completed','enabled','internal','',NOW());`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := NewStore(pool)
|
|
for _, testCase := range []struct {
|
|
filter string
|
|
want int
|
|
}{
|
|
{"failed,stalled", 2},
|
|
{"failed", 1},
|
|
{"cancelled", 0},
|
|
{"", 3},
|
|
} {
|
|
page, err := store.ListTestPublications(ctx, governance.TestPublicationFilter{ProcessingState: testCase.filter, Limit: 50})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if page.Total != testCase.want || len(page.Items) != testCase.want {
|
|
t.Fatalf("filter %q = %d items / total %d, want %d", testCase.filter, len(page.Items), page.Total, testCase.want)
|
|
}
|
|
}
|
|
}
|