feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestC4ConcurrentOperationCreatesOnceAndSurvivesPoolRestart(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp','pp','C4 scan',true)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 9, 11, 13, 0, 0, 0, time.UTC)
|
||||
store := NewStore(pool)
|
||||
|
||||
// 10 concurrent identical requests with the same Idempotency-Key: exactly one must create.
|
||||
var wg sync.WaitGroup
|
||||
type outcome struct {
|
||||
scanID string
|
||||
replayed bool
|
||||
err error
|
||||
}
|
||||
results := make([]outcome, 10)
|
||||
for i := range results {
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
scan, replayed, err := store.TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now, "op-c4-scan-1")
|
||||
results[index] = outcome{scanID: scan.ID, replayed: replayed, err: err}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var created int
|
||||
var firstID string
|
||||
for _, result := range results {
|
||||
if result.err != nil {
|
||||
t.Fatalf("concurrent operation error = %v", result.err)
|
||||
}
|
||||
if !result.replayed {
|
||||
created++
|
||||
firstID = result.scanID
|
||||
}
|
||||
if result.scanID != firstID && firstID != "" {
|
||||
t.Fatalf("replay returned different scan id: %s vs %s", result.scanID, firstID)
|
||||
}
|
||||
}
|
||||
if created != 1 {
|
||||
t.Fatalf("created=%d, want exactly 1", created)
|
||||
}
|
||||
var scans, jobs, operations int
|
||||
if err := pool.QueryRow(ctx, `SELECT (SELECT COUNT(*) FROM source_scans),(SELECT COUNT(*) FROM jobs WHERE type='source.scan.pp'),(SELECT COUNT(*) FROM operation_requests WHERE operation_id='op-c4-scan-1')`).Scan(&scans, &jobs, &operations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scans != 1 || jobs != 1 || operations != 1 {
|
||||
t.Fatalf("scans=%d jobs=%d operations=%d", scans, jobs, operations)
|
||||
}
|
||||
|
||||
// Restart: close the pool and reconnect, then replay with the same key.
|
||||
pool.Close()
|
||||
restarted, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer restarted.Close()
|
||||
scan, replayed, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now.Add(time.Hour), "op-c4-scan-1")
|
||||
if err != nil || !replayed || scan.ID != firstID {
|
||||
t.Fatalf("post-restart replay scan=%#v replayed=%v err=%v want id=%s", scan, replayed, err, firstID)
|
||||
}
|
||||
|
||||
// Same key, different payload (different connection): must conflict.
|
||||
if _, err := restarted.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp2','pp','C4 scan 2',true)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp2", now, "op-c4-scan-1"); err == nil {
|
||||
t.Fatal("same key different payload should conflict")
|
||||
} else if !errors.Is(err, ErrOperationRequestConflict) {
|
||||
t.Fatalf("conflict error=%v", err)
|
||||
}
|
||||
var freshOperations int
|
||||
if err := restarted.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests`).Scan(&freshOperations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if freshOperations != 1 {
|
||||
t.Fatalf("operations=%d want 1", freshOperations)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
var ErrCatalogChangeSetNotFound = errors.New("catalog change set not found")
|
||||
|
||||
func (s *Store) GetLatestCatalogChangeSet(ctx context.Context, filter governance.CatalogChangeSetFilter) (governance.CatalogChangeSetDetail, error) {
|
||||
if !validRouteID(filter.ConnectionID) || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 || !validCatalogChangeAction(filter.Action) {
|
||||
return governance.CatalogChangeSetDetail{}, errors.New("invalid catalog change set filter")
|
||||
}
|
||||
detail := governance.CatalogChangeSetDetail{Items: []governance.CatalogChangeItem{}, Limit: filter.Limit, Offset: filter.Offset}
|
||||
err := s.db.QueryRow(ctx, `SELECT change_set.id,change_set.source_connection_id,connection.name,change_set.state,
|
||||
change_set.add_count,change_set.update_count,change_set.move_count,change_set.move_scope_count,
|
||||
change_set.delete_count,change_set.no_change_count,change_set.created_at
|
||||
FROM source_catalog_change_sets change_set
|
||||
JOIN source_connections connection ON connection.id=change_set.source_connection_id
|
||||
WHERE change_set.source_connection_id=$1
|
||||
ORDER BY change_set.created_at DESC,change_set.id DESC
|
||||
LIMIT 1`, filter.ConnectionID).Scan(
|
||||
&detail.ID, &detail.ConnectionID, &detail.SourceName, &detail.State,
|
||||
&detail.AddCount, &detail.UpdateCount, &detail.MoveCount, &detail.MoveScopeCount,
|
||||
&detail.DeleteCount, &detail.NoChangeCount, &detail.CreatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.CatalogChangeSetDetail{}, ErrCatalogChangeSetNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return governance.CatalogChangeSetDetail{}, fmt.Errorf("get latest catalog change set: %w", err)
|
||||
}
|
||||
if err := s.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND ($2='' OR action=$2)`, detail.ID, filter.Action).Scan(&detail.Total); err != nil {
|
||||
return governance.CatalogChangeSetDetail{}, fmt.Errorf("count catalog change items: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT action,title,old_path,new_path,old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,COALESCE(old_content_sha256::text,''),COALESCE(new_content_sha256::text,''),reason_code,item_seq,COALESCE(old_scope,''),COALESCE(new_scope,''),COALESCE(old_version_id,''),COALESCE(new_version_id,'')
|
||||
FROM source_catalog_change_items
|
||||
WHERE change_set_id=$1 AND ($2='' OR action=$2)
|
||||
ORDER BY CASE action WHEN 'delete' THEN 1 WHEN 'update' THEN 2 WHEN 'move_scope' THEN 3 WHEN 'add' THEN 4 ELSE 5 END,
|
||||
lower(title),source_locator_hash
|
||||
LIMIT $3 OFFSET $4`, detail.ID, filter.Action, filter.Limit, filter.Offset)
|
||||
if err != nil {
|
||||
return governance.CatalogChangeSetDetail{}, fmt.Errorf("list catalog change items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.CatalogChangeItem
|
||||
if err := rows.Scan(&item.Action, &item.Title, &item.OldPath, &item.NewPath, &item.OldSizeBytes, &item.NewSizeBytes, &item.OldModified, &item.NewModified, &item.OldContentSHA256, &item.NewContentSHA256, &item.ReasonCode, &item.Seq, &item.OldScope, &item.NewScope, &item.OldVersionID, &item.NewVersionID); err != nil {
|
||||
return governance.CatalogChangeSetDetail{}, fmt.Errorf("scan catalog change item: %w", err)
|
||||
}
|
||||
detail.Items = append(detail.Items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.CatalogChangeSetDetail{}, fmt.Errorf("iterate catalog change items: %w", err)
|
||||
}
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
func validCatalogChangeAction(action string) bool {
|
||||
switch action {
|
||||
case "", "add", "update", "move", "move_scope", "delete", "no_change":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestCatalogChangeSetPersistsSummaryAndFilteredPreview(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-change','alist','国内工业',true);
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-old','conn-change',repeat('1',64),'国内工业',2,1,1,'2026-09-10T12:00:00Z'),('snapshot-new','conn-change',repeat('2',64),'国内工业',3,1,2,'2026-09-10T12:00:00Z');
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,no_change_count,created_at)
|
||||
VALUES('cs-change','conn-change','snapshot-old','snapshot-new','planned',1,1,0,1,5,'2026-09-10T12:00:00Z');
|
||||
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,old_size_bytes,new_size_bytes,created_at)
|
||||
VALUES('cs-change',repeat('a',64),'update','新版手册.pdf','手册.pdf','新版手册.pdf',10,12,'2026-09-10T12:00:00Z'),
|
||||
('cs-change',repeat('b',64),'delete','旧手册.pdf','旧手册.pdf','',20,NULL,'2026-09-10T12:00:00Z');`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
detail, err := store.GetLatestCatalogChangeSet(ctx, governance.CatalogChangeSetFilter{ConnectionID: "conn-change", Action: "delete", Limit: 20})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.SourceName != "国内工业" || detail.AddCount != 1 || detail.UpdateCount != 1 || detail.DeleteCount != 1 || detail.Total != 1 || len(detail.Items) != 1 || detail.Items[0].Title != "旧手册.pdf" || detail.Items[0].Action != "delete" {
|
||||
t.Fatalf("detail=%#v", detail)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
)
|
||||
|
||||
func (s *Store) UpdateCatalogRefreshProgress(ctx context.Context, jobID, owner string, progress catalogrefresh.Progress, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE jobs SET payload=payload||jsonb_build_object(
|
||||
'stage',$3::text,'progress',$4::integer,'message',$5::text,'processed',$6::integer,'total',$7::integer,
|
||||
'current_directory',NULLIF($9,''),'scanned_directories',$10::integer,'discovered_directories',$11::integer,
|
||||
'scanned_files',$12::integer,'estimated_total_files',$13::integer,'total_is_estimate',$14::boolean,
|
||||
'source_name',NULLIF($15,'')),updated_at=$8 WHERE id=$1 AND lease_owner=$2 AND state='running'`,
|
||||
jobID, owner, progress.Stage, progress.Progress, progress.Message, progress.Processed, progress.Total, now,
|
||||
progress.CurrentDirectory, progress.ScannedDirectories, progress.DiscoveredDirectories,
|
||||
progress.ScannedFiles, progress.EstimatedTotalFiles, progress.TotalIsEstimate, progress.SourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update catalog refresh progress: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ErrLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func (s *Store) RouteCatalogFiles(ctx context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
|
||||
if err := validateCatalogRouteRequest(request); err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return governance.CatalogRouteResult{}, errors.New("catalog route transactions are unavailable")
|
||||
}
|
||||
tx, err := beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("begin catalog route: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
result, err := routeCatalogFilesTx(ctx, tx, request)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("commit catalog route: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateCatalogRouteRequest(request governance.CatalogRouteRequest) error {
|
||||
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
|
||||
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
|
||||
}
|
||||
if !validRouteID(request.ConnectionID) || len(request.NodeIDs) == 0 || len(request.NodeIDs) > 100 ||
|
||||
!validCatalogPolicy(request.Visibility, targetKnowledgeBaseIDs) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 ||
|
||||
strings.TrimSpace(request.Reason) == "" || len(request.Reason) > 500 || request.Now.IsZero() {
|
||||
return errors.New("invalid catalog route request")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range request.NodeIDs {
|
||||
if !validRouteID(id) || seen[id] {
|
||||
return errors.New("invalid catalog route nodes")
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
|
||||
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
|
||||
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
|
||||
}
|
||||
nodeIDs := append([]string(nil), request.NodeIDs...)
|
||||
selectedRows, err := tx.Query(ctx, `SELECT node.node_id,node.node_type,node.relative_path
|
||||
FROM source_catalog_heads head
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
|
||||
ORDER BY node.node_id
|
||||
FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route nodes: %w", err)
|
||||
}
|
||||
selectedCount := 0
|
||||
directoryPaths := make([]string, 0)
|
||||
for selectedRows.Next() {
|
||||
var nodeID, nodeType, directoryPath string
|
||||
if err := selectedRows.Scan(&nodeID, &nodeType, &directoryPath); err != nil {
|
||||
selectedRows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route node: %w", err)
|
||||
}
|
||||
selectedCount++
|
||||
if nodeType == "folder" {
|
||||
directoryPaths = append(directoryPaths, directoryPath)
|
||||
}
|
||||
}
|
||||
if err := selectedRows.Err(); err != nil {
|
||||
selectedRows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route nodes: %w", err)
|
||||
}
|
||||
selectedRows.Close()
|
||||
if selectedCount != len(nodeIDs) {
|
||||
return governance.CatalogRouteResult{}, errors.New("catalog route contains missing nodes")
|
||||
}
|
||||
for _, directoryPath := range directoryPaths {
|
||||
if err := setCatalogDirectoryPolicy(ctx, tx, request, directoryPath); err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `WITH RECURSIVE selected AS (
|
||||
SELECT node.snapshot_id,node.node_id,node.node_type
|
||||
FROM source_catalog_heads head
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
|
||||
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
|
||||
), descendants AS (
|
||||
SELECT snapshot_id,node_id,node_type FROM selected
|
||||
UNION
|
||||
SELECT child.snapshot_id,child.node_id,child.node_type
|
||||
FROM descendants parent
|
||||
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
|
||||
WHERE parent.node_type='folder'
|
||||
)
|
||||
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
|
||||
WHERE node.node_type='file'
|
||||
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
|
||||
}
|
||||
type selectedFile struct {
|
||||
nodeID string
|
||||
sourceItemID *string
|
||||
direct bool
|
||||
}
|
||||
files := make([]selectedFile, 0, len(nodeIDs))
|
||||
for rows.Next() {
|
||||
var file selectedFile
|
||||
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
|
||||
rows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
|
||||
routedItems := map[string]bool{}
|
||||
for _, file := range files {
|
||||
if file.sourceItemID == nil {
|
||||
result.UnboundFiles++
|
||||
continue
|
||||
}
|
||||
itemID := *file.sourceItemID
|
||||
if routedItems[itemID] {
|
||||
continue
|
||||
}
|
||||
routedItems[itemID] = true
|
||||
if file.direct {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
|
||||
}
|
||||
eventID, err := newOpaqueID("se-")
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
|
||||
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
|
||||
}
|
||||
}
|
||||
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
|
||||
var currentVersionID *string
|
||||
var contentState, spoolRef string
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
|
||||
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.id=$1`, itemID).Scan(¤tVersionID, &contentState, &spoolRef)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
result.UnavailableFiles++
|
||||
case err != nil:
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
|
||||
case contentState == string(governance.ContentVerified) && spoolRef != "":
|
||||
result.RoutableFiles++
|
||||
default:
|
||||
result.HashUnknownFiles++
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO directory_scope_rules(source_connection_id,directory_path,visibility,updated_by,reason,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$6)
|
||||
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`,
|
||||
request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return fmt.Errorf("set routed directory scope: %w", err)
|
||||
}
|
||||
eventID, err := newOpaqueID("se-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_connection_id,directory_path,action,decided_visibility,operator_ref,reason,created_at)
|
||||
VALUES($1,'directory',$2,$3,'set',$4,$5,$6,$7)`, eventID, request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return fmt.Errorf("audit routed directory scope: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validCatalogPolicy(policy governance.Visibility, targetKnowledgeBaseIDs []string) bool {
|
||||
expectedTargets := 1
|
||||
switch policy {
|
||||
case governance.VisibilityInternal, governance.VisibilityExternal:
|
||||
case governance.VisibilityBoth:
|
||||
expectedTargets = 2
|
||||
case governance.VisibilityBlocked:
|
||||
expectedTargets = 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if len(targetKnowledgeBaseIDs) != expectedTargets {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]bool, len(targetKnowledgeBaseIDs))
|
||||
for _, target := range targetKnowledgeBaseIDs {
|
||||
if !validRouteID(target) || seen[target] {
|
||||
return false
|
||||
}
|
||||
seen[target] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validRouteID(value string) bool {
|
||||
if value == "" || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if unicode.IsLetter(character) || unicode.IsDigit(character) || character == '-' || character == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
|
||||
// It never creates or mutates test_publications and never enqueues jobs.
|
||||
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err = ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-route','pp','Route PP',true);
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-route',repeat('c',64),10,'verified/file.pdf','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-route','conn-route','external-route',repeat('b',64),'工业产品/CRA系列','手册.pdf','document','v1','external','pp_public_tag');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-route','item-route','v1','content-route','verified');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-route','conn-route',repeat('a',64),'Route PP',4,2,2,NOW());
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES
|
||||
('snapshot-route','root-route',NULL,'folder','Route PP','',repeat('1',64),NULL,NULL,2,2,20),
|
||||
('snapshot-route','folder-route','root-route','folder','工业产品','工业产品',repeat('4',64),NULL,NULL,1,1,10),
|
||||
('snapshot-route','file-route','folder-route','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-route',10,0,1,10),
|
||||
('snapshot-route','file-unmapped','root-route','file','未接入.pdf','未接入.pdf',repeat('3',64),NULL,10,0,1,10);
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-route','snapshot-route',NOW());`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
|
||||
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
// Desired state saved, recursion intact.
|
||||
var directoryPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if directoryPolicy != "internal" {
|
||||
t.Fatalf("directory policy=%q", directoryPolicy)
|
||||
}
|
||||
// Zero side effects: no publications, no jobs.
|
||||
var publicationCount, jobCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publicationCount != 0 || jobCount != 0 {
|
||||
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
|
||||
}
|
||||
|
||||
// Re-annotating the same directory is idempotent for rules and still side-effect free.
|
||||
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
|
||||
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
|
||||
t.Fatalf("again=%#v err=%v", again, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
|
||||
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
|
||||
}
|
||||
var ruleCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
|
||||
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
|
||||
}
|
||||
|
||||
// Switching policy updates desired state without retiring or removing anything.
|
||||
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
|
||||
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
|
||||
t.Fatalf("both=%#v err=%v", both, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
|
||||
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
|
||||
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocked.SelectedFiles != 1 {
|
||||
t.Fatalf("blocked=%#v", blocked)
|
||||
}
|
||||
var retrievalState string
|
||||
var removalJobs int
|
||||
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retrievalState != "enabled" || removalJobs != 0 {
|
||||
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
|
||||
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Unbound file and hash-unknown file are reported, never planned.
|
||||
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
|
||||
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
|
||||
t.Fatalf("unbound=%#v err=%v", unbound, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
|
||||
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
|
||||
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
|
||||
}
|
||||
// Overrides are still recorded for unbound/pending items so the desired state survives.
|
||||
var overrideCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
|
||||
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
|
||||
}
|
||||
// Audit trail exists for every annotation action.
|
||||
var events int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
|
||||
t.Fatalf("scope events=%d err=%v", events, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestRouteCatalogFilesWithOperationReplaysOriginalResultAndRejectsConflictingPayload(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-op','pp','Operation PP',true);
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-op',repeat('d',64),10,'verified/operation.pdf','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-op','conn-op','external-op',repeat('e',64),'工业产品','手册.pdf','document','v1','external','pp_public_tag');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-op','item-op','v1','content-op','verified');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-op','conn-op',repeat('f',64),'Operation PP',2,1,1,NOW());
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES
|
||||
('snapshot-op','root-op',NULL,'folder','Operation PP','',repeat('1',64),NULL,NULL,1,1,10),
|
||||
('snapshot-op','file-op','root-op','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-op',10,0,1,10);
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-op','snapshot-op',NOW());`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
|
||||
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
|
||||
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
if err != nil || replayed || first.RoutableFiles != 1 {
|
||||
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
|
||||
}
|
||||
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
if err != nil || !replayed || second != first {
|
||||
t.Fatalf("replay result=%#v replayed=%v error=%v", second, replayed, err)
|
||||
}
|
||||
var publications, operations int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publications); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publications != 0 || operations != 1 {
|
||||
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
|
||||
}
|
||||
conflicting := request
|
||||
conflicting.Reason = "不同的操作内容"
|
||||
_, _, err = store.RouteCatalogFilesWithOperation(ctx, conflicting, "op-route-1")
|
||||
if !errors.Is(err, ErrOperationRequestConflict) {
|
||||
t.Fatalf("conflicting operation error=%v, want ErrOperationRequestConflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueCatalogRefreshWithOperationReplaysTheOriginalJob(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-refresh-op','pp','Refresh operation',true)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 11, 7, 0, 0, 0, time.UTC)
|
||||
source := governance.CatalogSource{ConnectionID: "conn-refresh-op", SourceType: governance.SourceTypePP, Name: "Refresh operation"}
|
||||
first, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now)
|
||||
if err != nil || replayed || first.Type != "source.catalog.refresh" {
|
||||
t.Fatalf("first job=%#v replayed=%v error=%v", first, replayed, err)
|
||||
}
|
||||
second, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now.Add(time.Minute))
|
||||
if err != nil || !replayed || second.ID != first.ID || second.Type != first.Type {
|
||||
t.Fatalf("replay job=%#v replayed=%v error=%v", second, replayed, err)
|
||||
}
|
||||
var jobs, operations int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh'`).Scan(&jobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.refresh' AND operation_id='op-refresh-1'`).Scan(&operations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobs != 1 || operations != 1 {
|
||||
t.Fatalf("jobs=%d operations=%d", jobs, operations)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// ConfirmCatalogChangeSet executes ONLY the human-selected items of one planned
|
||||
// ChangeSet: it creates publications + confirmations + test.upload jobs, or
|
||||
// disables retrieval and enqueues test.remove for deletes. NO_CHANGE never
|
||||
// enqueues. Re-confirmation replays the first stored outcome.
|
||||
func (s *Store) ConfirmCatalogChangeSet(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) {
|
||||
operator := strings.TrimSpace(request.OperatorRef)
|
||||
reason := strings.TrimSpace(request.Reason)
|
||||
if !validRouteID(request.ChangeSetID) || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() || len(request.ItemSeqs) == 0 || len(request.ItemSeqs) > 200 {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set confirmation")
|
||||
}
|
||||
seen := make(map[int64]bool, len(request.ItemSeqs))
|
||||
for _, seq := range request.ItemSeqs {
|
||||
if seq < 1 || seen[seq] {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set item selections")
|
||||
}
|
||||
seen[seq] = true
|
||||
}
|
||||
var tx pgx.Tx
|
||||
ownsTransaction := false
|
||||
if existing, ok := s.db.(pgx.Tx); ok {
|
||||
tx = existing
|
||||
} else {
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation transactions are unavailable")
|
||||
}
|
||||
var err error
|
||||
tx, err = beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("begin change set confirmation: %w", err)
|
||||
}
|
||||
ownsTransaction = true
|
||||
defer tx.Rollback(ctx)
|
||||
}
|
||||
result, err := confirmCatalogChangeSetTx(ctx, tx, request)
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
if ownsTransaction {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("commit change set confirmation: %w", err)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func confirmCatalogChangeSetTx(ctx context.Context, tx pgx.Tx, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) {
|
||||
result := governance.ConfirmCatalogChangeSetResult{ChangeSetID: request.ChangeSetID}
|
||||
selections := append([]int64(nil), request.ItemSeqs...)
|
||||
var state string
|
||||
err := tx.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id=$1 FOR UPDATE`, request.ChangeSetID).Scan(&state)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, ErrCatalogChangeSetNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("lock change set for confirmation: %w", err)
|
||||
}
|
||||
if state != "planned" {
|
||||
// Even on replay the requested locators must belong to this change set.
|
||||
var owned int
|
||||
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND item_seq=ANY($2::bigint[])`, request.ChangeSetID, selections).Scan(&owned); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set item check: %w", err)
|
||||
}
|
||||
if owned != len(selections) {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items")
|
||||
}
|
||||
// Replay the first stored outcome from persisted evidence.
|
||||
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM processing_confirmations WHERE change_set_id=$1`, request.ChangeSetID).Scan(&result.ConfirmedItems); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set confirmation: %w", err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.UploadsCreated); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set uploads: %w", err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.remove' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.RemovalsRequested); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set removals: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT item.item_seq,item.action,COALESCE(item.new_scope,''),COALESCE(item.old_scope,''),item.title,
|
||||
COALESCE(item.new_source_item_id,item.old_source_item_id) AS source_item_id,
|
||||
version.id AS version_id,
|
||||
version.content_state='verified' AND content.content_state='verified' AND content.local_spool_ref<>'' AS artifact_ready
|
||||
FROM source_catalog_change_items item
|
||||
LEFT JOIN source_items source ON source.id=COALESCE(item.new_source_item_id,item.old_source_item_id)
|
||||
LEFT JOIN source_item_versions version ON version.source_item_id=source.id AND version.revision=source.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.change_set_id=$1 AND item.item_seq=ANY($2::bigint[])
|
||||
ORDER BY item.item_seq`, request.ChangeSetID, selections)
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("select change set items: %w", err)
|
||||
}
|
||||
type changeRow struct {
|
||||
seq int64
|
||||
action string
|
||||
newScope string
|
||||
oldScope string
|
||||
sourceItemID *string
|
||||
versionID *string
|
||||
artifactReady bool
|
||||
}
|
||||
changes := make([]changeRow, 0, len(selections))
|
||||
for rows.Next() {
|
||||
var item changeRow
|
||||
var title string
|
||||
var versionID *string
|
||||
var artifactReady *bool
|
||||
if err := rows.Scan(&item.seq, &item.action, &item.newScope, &item.oldScope, &title, &item.sourceItemID, &versionID, &artifactReady); err != nil {
|
||||
rows.Close()
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("scan change set item: %w", err)
|
||||
}
|
||||
if versionID != nil {
|
||||
item.versionID = versionID
|
||||
}
|
||||
if artifactReady != nil {
|
||||
item.artifactReady = *artifactReady
|
||||
}
|
||||
changes = append(changes, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("iterate change set items: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
if len(changes) != len(selections) {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items")
|
||||
}
|
||||
|
||||
for _, item := range changes {
|
||||
if item.action == "no_change" {
|
||||
result.NoChangeIgnored++
|
||||
continue
|
||||
}
|
||||
if item.action == "delete" || item.newScope == string(governance.VisibilityBlocked) {
|
||||
if item.sourceItemID == nil {
|
||||
result.SkippedNoArtifact++
|
||||
continue
|
||||
}
|
||||
removals, err := retireItemPublicationsForChange(ctx, tx, *item.sourceItemID, request)
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
result.RemovalsRequested += removals
|
||||
result.ConfirmedItems++
|
||||
continue
|
||||
}
|
||||
targets := make([]string, 0, 2)
|
||||
switch item.newScope {
|
||||
case string(governance.VisibilityInternal):
|
||||
if request.InternalTargetKBID == "" {
|
||||
result.SkippedNoTarget++
|
||||
continue
|
||||
}
|
||||
targets = append(targets, request.InternalTargetKBID)
|
||||
case string(governance.VisibilityExternal):
|
||||
if request.ExternalTargetKBID == "" {
|
||||
result.SkippedNoTarget++
|
||||
continue
|
||||
}
|
||||
targets = append(targets, request.ExternalTargetKBID)
|
||||
case string(governance.VisibilityBoth):
|
||||
if request.InternalTargetKBID != "" {
|
||||
targets = append(targets, request.InternalTargetKBID)
|
||||
}
|
||||
if request.ExternalTargetKBID != "" {
|
||||
targets = append(targets, request.ExternalTargetKBID)
|
||||
}
|
||||
if len(targets) < 2 {
|
||||
result.SkippedNoTarget++
|
||||
continue
|
||||
}
|
||||
default:
|
||||
result.SkippedNoTarget++
|
||||
continue
|
||||
}
|
||||
if item.versionID == nil || !item.artifactReady {
|
||||
result.SkippedNoArtifact++
|
||||
continue
|
||||
}
|
||||
result.ConfirmedItems++
|
||||
for _, target := range targets {
|
||||
scope := item.newScope
|
||||
if scope == string(governance.VisibilityBoth) {
|
||||
scope = string(governance.VisibilityInternal)
|
||||
if target == request.ExternalTargetKBID {
|
||||
scope = string(governance.VisibilityExternal)
|
||||
}
|
||||
}
|
||||
var existingPublication string
|
||||
err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existingPublication)
|
||||
if err == nil {
|
||||
if err := ensureChangeSetConfirmation(ctx, tx, existingPublication, request); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
result.UploadsExisting++
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("inspect change set publication: %w", err)
|
||||
}
|
||||
publicationID, err := newOpaqueID("tp-")
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,change_set_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$6)
|
||||
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
|
||||
publicationID, *item.versionID, target, scope, request.ChangeSetID, request.Now)
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("create change set publication: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
// A concurrent confirmation already owns this version+target.
|
||||
var existing string
|
||||
if err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existing); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("resolve concurrent publication: %w", err)
|
||||
}
|
||||
if err := ensureChangeSetConfirmation(ctx, tx, existing, request); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
result.UploadsExisting++
|
||||
continue
|
||||
}
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{"publication_id": publicationID, "change_set_id": request.ChangeSetID, "item_seq": strconv.FormatInt(item.seq, 10), "target_scope": scope, "target_kb_id": target})
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, errors.New("encode change set upload job")
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3,$5,$6,$7,'upload')
|
||||
ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "test-start:"+publicationID, request.ChangeSetID, *item.versionID, target); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("enqueue change set upload: %w", err)
|
||||
}
|
||||
if err := ensureChangeSetConfirmation(ctx, tx, publicationID, request); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, err
|
||||
}
|
||||
result.UploadsCreated++
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_catalog_change_sets SET state='confirmed',confirmed_at=$2 WHERE id=$1 AND state='planned'`, request.ChangeSetID, request.Now); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("confirm change set: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ensureChangeSetConfirmation(ctx context.Context, tx pgx.Tx, publicationID string, request governance.ConfirmCatalogChangeSetRequest) error {
|
||||
confirmationID, err := newOpaqueID("pc-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,change_set_id) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT (publication_id) DO NOTHING`,
|
||||
confirmationID, publicationID, request.OperatorRef, request.Reason, request.Now, request.ChangeSetID); err != nil {
|
||||
return fmt.Errorf("record change set confirmation: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retireItemPublicationsForChange(ctx context.Context, tx pgx.Tx, itemID string, request governance.ConfirmCatalogChangeSetRequest) (int, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT publication.id,publication.knowledge_id,publication.retrieval_state
|
||||
FROM test_publications publication
|
||||
JOIN source_item_versions version ON version.id=publication.source_item_version_id
|
||||
WHERE version.source_item_id=$1 AND publication.retrieval_state NOT IN ('disabling','disabled','unpublished')
|
||||
FOR UPDATE OF publication`, itemID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("inspect change set removals: %w", err)
|
||||
}
|
||||
type target struct{ id, knowledgeID string }
|
||||
targets := make([]target, 0)
|
||||
for rows.Next() {
|
||||
var item target
|
||||
if err := rows.Scan(&item.id, &item.knowledgeID, new(string)); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("scan change set removal: %w", err)
|
||||
}
|
||||
targets = append(targets, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("iterate change set removals: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
removals := 0
|
||||
for _, item := range targets {
|
||||
if item.knowledgeID == "" {
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET ingest_state='failed',processing_state='failed',retrieval_state='disabled',last_error='blocked_by_changeset',updated_at=$2 WHERE id=$1 AND knowledge_id=''`, item.id, request.Now); err != nil {
|
||||
return 0, fmt.Errorf("stop pre-upload change set removal: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET retrieval_state='disabling',last_error='',updated_at=$2 WHERE id=$1`, item.id, request.Now); err != nil {
|
||||
return 0, fmt.Errorf("mark change set removal: %w", err)
|
||||
}
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{"publication_id": item.id, "change_set_id": request.ChangeSetID})
|
||||
if err != nil {
|
||||
return 0, errors.New("encode change set remove job")
|
||||
}
|
||||
// Removals key on the item (version-independent): a confirmed delete must
|
||||
// not enqueue twice even if version rows rotate between deliveries.
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES($1,'test.remove',$2,'pending',20,$3,10,$4,$3,$3,$5,$6,'','remove')
|
||||
ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "changeset-remove:"+item.id+":"+request.ChangeSetID, request.ChangeSetID, item.id); err != nil {
|
||||
return 0, fmt.Errorf("enqueue change set removal: %w", err)
|
||||
}
|
||||
removals++
|
||||
}
|
||||
return removals, nil
|
||||
}
|
||||
|
||||
func (s *Store) ConfirmCatalogChangeSetWithOperation(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest, operationID string) (governance.ConfirmCatalogChangeSetResult, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ChangeSetID string `json:"change_set_id"`
|
||||
ItemSeqs []int64 `json:"item_seqs"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
}{request.ChangeSetID, request.ItemSeqs, request.OperatorRef, request.Reason})
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, false, err
|
||||
}
|
||||
raw, replayed, err := s.executeOperation(ctx, "changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := NewStore(tx).ConfirmCatalogChangeSet(ctx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, false, err
|
||||
}
|
||||
var result governance.ConfirmCatalogChangeSetResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return governance.ConfirmCatalogChangeSetResult{}, false, fmt.Errorf("decode stored change set result: %w", err)
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func isolatedExternalIntegrationDatabaseURL(t *testing.T, databaseURL string) string {
|
||||
t.Helper()
|
||||
if databaseURL == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(databaseURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(t.Name()))
|
||||
schema := "kgtestx_" + hex.EncodeToString(digest[:8])
|
||||
ctx := context.Background()
|
||||
admin, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect integration database: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
|
||||
t.Fatalf("create isolated integration schema: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup, err := pgxpool.New(context.Background(), databaseURL)
|
||||
if err == nil {
|
||||
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
|
||||
cleanup.Close()
|
||||
}
|
||||
})
|
||||
query := parsed.Query()
|
||||
query.Set("search_path", schema)
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func isolatedIntegrationDatabaseURL(t *testing.T, databaseURL string) string {
|
||||
t.Helper()
|
||||
if databaseURL == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(databaseURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(t.Name()))
|
||||
schema := "kgtest_" + hex.EncodeToString(digest[:8])
|
||||
ctx := context.Background()
|
||||
admin, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect integration database: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
|
||||
t.Fatalf("create isolated integration schema: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup, err := pgxpool.New(context.Background(), databaseURL)
|
||||
if err == nil {
|
||||
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
|
||||
cleanup.Close()
|
||||
}
|
||||
})
|
||||
query := parsed.Query()
|
||||
query.Set("search_path", schema)
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
const listJobFailuresSQL = `
|
||||
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
|
||||
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
|
||||
jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at,
|
||||
COALESCE(siv.source_item_id,'')
|
||||
FROM job_failures jf
|
||||
LEFT JOIN test_publications p ON p.id = jf.publication_id
|
||||
LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id
|
||||
WHERE ($1 = '' OR jf.state = $1)
|
||||
ORDER BY jf.created_at DESC, jf.id DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
`
|
||||
|
||||
const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)`
|
||||
|
||||
// ListJobFailures is the DLQ view (P6): projections of terminal attempts. The
|
||||
// original jobs and publication audit rows are never moved or deleted.
|
||||
func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) {
|
||||
state := filter.State
|
||||
if state == "" {
|
||||
state = "open"
|
||||
}
|
||||
if state != "open" && state != "requeued" && state != "abandoned" {
|
||||
return governance.JobFailurePage{}, errors.New("invalid failure state filter")
|
||||
}
|
||||
limit, offset := filter.Limit, filter.Offset
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 100 {
|
||||
return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100")
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
var total int
|
||||
if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil {
|
||||
return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset)
|
||||
if err != nil {
|
||||
return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]governance.JobFailure, 0, limit)
|
||||
for rows.Next() {
|
||||
var item governance.JobFailure
|
||||
if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode,
|
||||
&item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID,
|
||||
&item.State, &item.CreatedAt, &item.SourceItemID); err != nil {
|
||||
return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err)
|
||||
}
|
||||
return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
|
||||
}
|
||||
|
||||
const failurePublicationStateSQL = `
|
||||
SELECT p.processing_state, p.ingest_state,
|
||||
EXISTS (
|
||||
SELECT 1 FROM source_item_versions siv
|
||||
JOIN content_objects co ON co.id = siv.content_object_id
|
||||
WHERE siv.id = p.source_item_version_id
|
||||
AND co.content_state = 'verified'
|
||||
AND co.local_spool_ref <> ''
|
||||
)
|
||||
FROM test_publications p
|
||||
WHERE p.id = $1
|
||||
`
|
||||
|
||||
const resolvePublicationFailuresSQL = `
|
||||
UPDATE job_failures
|
||||
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
|
||||
WHERE publication_id = $1 AND state = 'open'
|
||||
`
|
||||
|
||||
const recordRetryAuditSQL = `
|
||||
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (batch_key, publication_id) DO NOTHING
|
||||
`
|
||||
|
||||
// RetryPublicationsBatchWithOperation performs the whole selection server-side
|
||||
// (P6): one operation envelope, one transaction, per-item outcome, atomic audit.
|
||||
func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
|
||||
if operationID == "" || len(operationID) > 128 {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
|
||||
}
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
|
||||
}
|
||||
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
|
||||
}
|
||||
seen := make(map[string]bool, len(request.PublicationIDs))
|
||||
ids := make([]string, 0, len(request.PublicationIDs))
|
||||
for _, id := range request.PublicationIDs {
|
||||
if id == "" || len(id) > 64 || seen[id] {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
|
||||
}
|
||||
seen[id] = true
|
||||
ids = append(ids, id)
|
||||
}
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
PublicationIDs []string `json:"publication_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
}{ids, request.OperatorRef, request.Reason})
|
||||
if err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, err
|
||||
}
|
||||
raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, err
|
||||
}
|
||||
var result governance.RetryPublicationsBatchResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result")
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
|
||||
// Stable lock order prevents cross-batch deadlocks.
|
||||
sorted := append([]string(nil), ids...)
|
||||
for i := 1; i < len(sorted); i++ {
|
||||
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
|
||||
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
|
||||
}
|
||||
}
|
||||
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
|
||||
store := NewStore(tx)
|
||||
for _, id := range sorted {
|
||||
var processingState, ingestState string
|
||||
var spoolAvailable bool
|
||||
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable)
|
||||
outcome := governance.RetryOutcome{PublicationID: id}
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication not found"
|
||||
case err != nil:
|
||||
return result, fmt.Errorf("classify publication retry: %w", err)
|
||||
case processingState == "processing" || processingState == "finalizing":
|
||||
outcome.Result = governance.RetryAlreadyRunning
|
||||
case processingState == "pending" && ingestState != "failed":
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication is already queued"
|
||||
case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"):
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "publication is not in a failed state"
|
||||
case !spoolAvailable:
|
||||
outcome.Result = governance.RetryNotRetryable
|
||||
outcome.Detail = "no verified spool artifact"
|
||||
default:
|
||||
if err := store.RetryPublication(ctx, id, request.Now); err != nil {
|
||||
return result, fmt.Errorf("retry publication %s: %w", id, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
|
||||
return result, fmt.Errorf("resolve job failures: %w", err)
|
||||
}
|
||||
outcome.Result = governance.RetryStarted
|
||||
}
|
||||
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
|
||||
return result, fmt.Errorf("record retry audit: %w", err)
|
||||
}
|
||||
switch outcome.Result {
|
||||
case governance.RetryStarted:
|
||||
result.Started++
|
||||
case governance.RetryAlreadyRunning:
|
||||
result.AlreadyRunning++
|
||||
default:
|
||||
result.NotRetryable++
|
||||
}
|
||||
result.Outcomes = append(result.Outcomes, outcome)
|
||||
}
|
||||
result.Total = len(sorted)
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
var ErrInvalidManualPublicationSeed = errors.New("invalid manual publication seed")
|
||||
|
||||
type ManualPublicationSeedRequest struct {
|
||||
TestKBID string
|
||||
Title string
|
||||
SHA256 string
|
||||
SizeBytes int64
|
||||
MIME string
|
||||
ValidatedFormat string
|
||||
LocalSpoolRef string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ManualPublicationSeedResult struct {
|
||||
PublicationID string
|
||||
JobID string
|
||||
}
|
||||
|
||||
const seedManualPublicationSQL = `
|
||||
WITH ensured_connection AS (
|
||||
INSERT INTO source_connections (
|
||||
id, source_type, name, base_url, secret_ref, enabled,
|
||||
schedule, last_health, created_at, updated_at
|
||||
) VALUES ($3, 'manual', '本地手工验收', '', '', FALSE,
|
||||
'{}'::jsonb, '{}'::jsonb, $2, $2)
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
SET updated_at = source_connections.updated_at
|
||||
RETURNING id
|
||||
), ensured_content AS (
|
||||
INSERT INTO content_objects (
|
||||
id, sha256, size_bytes, mime, validated_format,
|
||||
local_spool_ref, content_state,
|
||||
first_discovered_at, last_discovered_at, created_at, updated_at
|
||||
) VALUES ($10, $11, $8, $9, $12, $13, 'verified', $2, $2, $2, $2)
|
||||
ON CONFLICT (sha256) DO UPDATE
|
||||
SET last_discovered_at = EXCLUDED.last_discovered_at,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
local_spool_ref = CASE
|
||||
WHEN content_objects.local_spool_ref = '' THEN EXCLUDED.local_spool_ref
|
||||
ELSE content_objects.local_spool_ref
|
||||
END
|
||||
WHERE content_objects.size_bytes = EXCLUDED.size_bytes
|
||||
RETURNING id
|
||||
), ensured_item AS (
|
||||
INSERT INTO source_items (
|
||||
id, source_connection_id, stable_external_id, canonical_uri_hash,
|
||||
normalized_path, title, item_type, size_bytes, mime, revision,
|
||||
visibility, visibility_origin, source_change, created_at, updated_at,
|
||||
identity_confidence
|
||||
)
|
||||
SELECT $4, id, $5, $6, '', $7, 'document', $8, $9, $15,
|
||||
'internal', 'manual', 'added', $2, $2, 'content_hash'
|
||||
FROM ensured_connection
|
||||
ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE
|
||||
SET updated_at = source_items.updated_at,
|
||||
identity_confidence = CASE
|
||||
WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence
|
||||
ELSE EXCLUDED.identity_confidence
|
||||
END
|
||||
RETURNING id
|
||||
), ensured_version AS (
|
||||
INSERT INTO source_item_versions (
|
||||
id, source_item_id, revision, content_object_id, content_state,
|
||||
download_error, raw_record_summary, created_at, updated_at
|
||||
)
|
||||
SELECT $14, ensured_item.id, $15, ensured_content.id, 'verified', '',
|
||||
'{"origin":"manual_e2e"}'::jsonb, $2, $2
|
||||
FROM ensured_item CROSS JOIN ensured_content
|
||||
ON CONFLICT (source_item_id, revision) DO UPDATE
|
||||
SET updated_at = source_item_versions.updated_at
|
||||
RETURNING id
|
||||
), ensured_publication AS (
|
||||
INSERT INTO test_publications (
|
||||
id, source_item_version_id, test_kb_id,
|
||||
ingest_state, processing_state, retrieval_state,
|
||||
created_at, updated_at
|
||||
)
|
||||
SELECT $16, id, $17, 'pending', 'pending', 'unpublished', $2, $2
|
||||
FROM ensured_version
|
||||
ON CONFLICT (source_item_version_id, test_kb_id) DO UPDATE
|
||||
SET updated_at = test_publications.updated_at
|
||||
RETURNING id
|
||||
), ensured_job AS (
|
||||
INSERT INTO jobs (
|
||||
id, type, payload, state, priority, available_at,
|
||||
max_attempts, idempotency_key, created_at, updated_at
|
||||
)
|
||||
SELECT $18, 'test.upload', $1::jsonb, 'pending', 10, $2,
|
||||
5, $19, $2, $2
|
||||
FROM ensured_publication
|
||||
ON CONFLICT (idempotency_key) DO UPDATE
|
||||
SET idempotency_key = jobs.idempotency_key
|
||||
RETURNING id
|
||||
)
|
||||
SELECT ensured_publication.id, ensured_job.id
|
||||
FROM ensured_publication CROSS JOIN ensured_job
|
||||
`
|
||||
|
||||
func (s *Store) SeedManualPublication(ctx context.Context, request ManualPublicationSeedRequest) (ManualPublicationSeedResult, error) {
|
||||
if !validManualPublicationSeed(request) {
|
||||
return ManualPublicationSeedResult{}, ErrInvalidManualPublicationSeed
|
||||
}
|
||||
identity := request.SHA256 + ":" + request.TestKBID
|
||||
connectionID := "conn-manual-local"
|
||||
sourceItemID := deterministicGovernanceID("si-", request.SHA256)
|
||||
contentID := deterministicGovernanceID("co-", request.SHA256)
|
||||
versionID := deterministicGovernanceID("sv-", request.SHA256)
|
||||
publicationID := deterministicGovernanceID("tp-", identity)
|
||||
jobID := deterministicGovernanceID("job-", "test.upload:"+publicationID)
|
||||
stableExternalID := "manual-e2e:" + request.SHA256
|
||||
canonicalDigest := sha256.Sum256([]byte(stableExternalID))
|
||||
revision := "sha256:" + request.SHA256
|
||||
payload, err := json.Marshal(struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
}{PublicationID: publicationID})
|
||||
if err != nil {
|
||||
return ManualPublicationSeedResult{}, errors.New("encode manual publication upload job")
|
||||
}
|
||||
idempotencyKey := "test-upload:" + publicationID
|
||||
|
||||
result := ManualPublicationSeedResult{}
|
||||
err = s.db.QueryRow(
|
||||
ctx,
|
||||
seedManualPublicationSQL,
|
||||
payload,
|
||||
request.Now,
|
||||
connectionID,
|
||||
sourceItemID,
|
||||
stableExternalID,
|
||||
hex.EncodeToString(canonicalDigest[:]),
|
||||
request.Title,
|
||||
request.SizeBytes,
|
||||
request.MIME,
|
||||
contentID,
|
||||
request.SHA256,
|
||||
request.ValidatedFormat,
|
||||
request.LocalSpoolRef,
|
||||
versionID,
|
||||
revision,
|
||||
publicationID,
|
||||
request.TestKBID,
|
||||
jobID,
|
||||
idempotencyKey,
|
||||
).Scan(&result.PublicationID, &result.JobID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ManualPublicationSeedResult{}, ErrPublicationStateConflict
|
||||
}
|
||||
if err != nil {
|
||||
return ManualPublicationSeedResult{}, fmt.Errorf("seed manual test publication: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validManualPublicationSeed(request ManualPublicationSeedRequest) bool {
|
||||
if !governance.ValidSHA256(request.SHA256) || request.SizeBytes <= 0 || request.Now.IsZero() {
|
||||
return false
|
||||
}
|
||||
if len(request.TestKBID) == 0 || len(request.TestKBID) > 255 || strings.ContainsAny(request.TestKBID, "/\\\x00\t\r\n") {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(request.Title) == "" || len(request.Title) > 512 || strings.ContainsRune(request.Title, '\x00') {
|
||||
return false
|
||||
}
|
||||
if request.MIME == "" || len(request.MIME) > 128 || strings.ContainsAny(request.MIME, "\x00\r\n") || !validErrorCode(request.ValidatedFormat) {
|
||||
return false
|
||||
}
|
||||
clean := path.Clean(request.LocalSpoolRef)
|
||||
return len(request.LocalSpoolRef) <= 512 && clean == request.LocalSpoolRef && clean != "." && !strings.HasPrefix(clean, "/") && !strings.HasPrefix(clean, "../")
|
||||
}
|
||||
|
||||
func deterministicGovernanceID(prefix string, parts ...string) string {
|
||||
digest := sha256.New()
|
||||
for _, part := range parts {
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(part))
|
||||
}
|
||||
return prefix + hex.EncodeToString(digest.Sum(nil))[:32]
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSeedManualPublicationCreatesAtomicUploadChain(t *testing.T) {
|
||||
db := &recordingDB{row: manualSeedRow{publicationID: "tp-seeded", jobID: "job-seeded"}}
|
||||
store := NewStore(db)
|
||||
now := time.Date(2026, 9, 5, 5, 0, 0, 0, time.UTC)
|
||||
|
||||
result, err := store.SeedManualPublication(context.Background(), ManualPublicationSeedRequest{
|
||||
TestKBID: "c12ff4d5-dd87-423b-ab0f-af745b9d54a7",
|
||||
Title: "Governance P3 E2E",
|
||||
SHA256: strings.Repeat("a", 64),
|
||||
SizeBytes: 128,
|
||||
MIME: "text/markdown",
|
||||
ValidatedFormat: "markdown",
|
||||
LocalSpoolRef: "manual/e2e-a.md",
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SeedManualPublication() error = %v", err)
|
||||
}
|
||||
if result.PublicationID != "tp-seeded" || result.JobID != "job-seeded" {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
lowerSQL := strings.ToLower(db.queryRowSQL)
|
||||
for _, required := range []string{
|
||||
"insert into source_connections",
|
||||
"insert into source_items",
|
||||
"insert into content_objects",
|
||||
"insert into source_item_versions",
|
||||
"insert into test_publications",
|
||||
"insert into jobs",
|
||||
"on conflict",
|
||||
"returning",
|
||||
} {
|
||||
if !strings.Contains(lowerSQL, required) {
|
||||
t.Errorf("seed SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(db.queryRowArgs[0].([]byte)), `"publication_id"`) || strings.Contains(string(db.queryRowArgs[0].([]byte)), "c12ff4d5") {
|
||||
t.Fatalf("job payload = %s, want publication_id only", db.queryRowArgs[0])
|
||||
}
|
||||
}
|
||||
|
||||
type manualSeedRow struct {
|
||||
publicationID string
|
||||
jobID string
|
||||
}
|
||||
|
||||
func (row manualSeedRow) Scan(dest ...any) error {
|
||||
*(dest[0].(*string)) = row.publicationID
|
||||
*(dest[1].(*string)) = row.jobID
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFS embed.FS
|
||||
|
||||
type migration struct {
|
||||
Version int64
|
||||
UpSQL string
|
||||
DownSQL string
|
||||
}
|
||||
|
||||
func loadMigrations() ([]migration, error) {
|
||||
paths, err := fs.Glob(migrationFS, "migrations/*.up.sql")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("glob migrations: %w", err)
|
||||
}
|
||||
result := make([]migration, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
base := strings.TrimPrefix(path, "migrations/")
|
||||
versionText, _, ok := strings.Cut(base, "_")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid migration filename %q", path)
|
||||
}
|
||||
version, err := strconv.ParseInt(versionText, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse migration version %q: %w", path, err)
|
||||
}
|
||||
up, err := migrationFS.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read up migration %q: %w", path, err)
|
||||
}
|
||||
downPath := strings.TrimSuffix(path, ".up.sql") + ".down.sql"
|
||||
down, err := migrationFS.ReadFile(downPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read down migration %q: %w", downPath, err)
|
||||
}
|
||||
result = append(result, migration{Version: version, UpSQL: string(up), DownSQL: string(down)})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].Version < result[j].Version })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const migrationLockID int64 = 7346729101
|
||||
|
||||
func ApplyMigrations(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
if _, err := pool.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil {
|
||||
return fmt.Errorf("lock migrations: %w", err)
|
||||
}
|
||||
defer func() { _, _ = pool.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockID) }()
|
||||
|
||||
if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version BIGINT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)`); err != nil {
|
||||
return fmt.Errorf("create migration ledger: %w", err)
|
||||
}
|
||||
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range migrations {
|
||||
var applied bool
|
||||
if err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = $1)", item.Version).Scan(&applied); err != nil {
|
||||
return fmt.Errorf("check migration %d: %w", item.Version, err)
|
||||
}
|
||||
if applied {
|
||||
continue
|
||||
}
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin migration %d: %w", item.Version, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, item.UpSQL); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("apply migration %d: %w", item.Version, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version) VALUES ($1)", item.Version); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("record migration %d: %w", item.Version, err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit migration %d: %w", item.Version, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RollbackLatestMigration(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var version int64
|
||||
if err := pool.QueryRow(ctx, "SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1").Scan(&version); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read latest migration: %w", err)
|
||||
}
|
||||
var selected *migration
|
||||
for i := range migrations {
|
||||
if migrations[i].Version == version {
|
||||
selected = &migrations[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if selected == nil {
|
||||
return fmt.Errorf("missing down migration for version %d", version)
|
||||
}
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin rollback %d: %w", version, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, selected.DownSQL); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("rollback migration %d: %w", version, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "DELETE FROM schema_migrations WHERE version = $1", version); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("delete migration ledger %d: %w", version, err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit rollback %d: %w", version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS jobs;
|
||||
@@ -0,0 +1,31 @@
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version BIGINT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
type VARCHAR(100) NOT NULL,
|
||||
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
state VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed')),
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
lease_owner VARCHAR(255),
|
||||
lease_until TIMESTAMPTZ,
|
||||
attempt INTEGER NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5 CHECK (max_attempts > 0),
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
idempotency_key VARCHAR(255) NOT NULL,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_claim
|
||||
ON jobs (priority DESC, available_at, created_at, id)
|
||||
WHERE state IN ('pending', 'retry', 'running');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_state_updated
|
||||
ON jobs (state, updated_at DESC);
|
||||
@@ -0,0 +1,5 @@
|
||||
DROP TABLE IF EXISTS test_publications;
|
||||
DROP TABLE IF EXISTS source_item_versions;
|
||||
DROP TABLE IF EXISTS content_objects;
|
||||
DROP TABLE IF EXISTS source_items;
|
||||
DROP TABLE IF EXISTS source_connections;
|
||||
@@ -0,0 +1,114 @@
|
||||
CREATE TABLE IF NOT EXISTS source_connections (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_type VARCHAR(20) NOT NULL
|
||||
CHECK (source_type IN ('pp', 'alist', 'dingtalk')),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
CHECK (jsonb_typeof(schedule) = 'object'),
|
||||
last_health JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
CHECK (jsonb_typeof(last_health) = 'object'),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_items (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
stable_external_id TEXT NOT NULL,
|
||||
canonical_uri_hash CHAR(64) NOT NULL
|
||||
CHECK (canonical_uri_hash ~ '^[0-9a-f]{64}$'),
|
||||
parent_external_id TEXT NOT NULL DEFAULT '',
|
||||
normalized_path TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL,
|
||||
item_type VARCHAR(20) NOT NULL
|
||||
CHECK (item_type IN ('document', 'file', 'link', 'video')),
|
||||
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
|
||||
mime TEXT NOT NULL DEFAULT '',
|
||||
revision TEXT NOT NULL DEFAULT '',
|
||||
updated_at_source TIMESTAMPTZ,
|
||||
visibility VARCHAR(20) NOT NULL DEFAULT 'internal'
|
||||
CHECK (visibility IN ('internal', 'external')),
|
||||
visibility_origin VARCHAR(30) NOT NULL DEFAULT 'manual'
|
||||
CHECK (visibility_origin IN ('pp_public_tag', 'configured_scope', 'manual')),
|
||||
last_seen_scan_id VARCHAR(36) NOT NULL DEFAULT '',
|
||||
source_change VARCHAR(20) NOT NULL DEFAULT 'unchanged'
|
||||
CHECK (source_change IN ('unchanged', 'added', 'updated', 'deleted', 'unavailable')),
|
||||
absence_streak INTEGER NOT NULL DEFAULT 0 CHECK (absence_streak >= 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (source_connection_id, stable_external_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS content_objects (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
sha256 CHAR(64) NOT NULL
|
||||
CHECK (sha256 ~ '^[0-9a-f]{64}$'),
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
|
||||
mime TEXT NOT NULL DEFAULT '',
|
||||
validated_format VARCHAR(50) NOT NULL DEFAULT '',
|
||||
local_spool_ref TEXT NOT NULL DEFAULT '',
|
||||
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
|
||||
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
|
||||
first_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (sha256)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_item_versions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_item_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
revision TEXT NOT NULL,
|
||||
content_object_id VARCHAR(36)
|
||||
REFERENCES content_objects(id) ON DELETE RESTRICT,
|
||||
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
|
||||
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
|
||||
download_error TEXT NOT NULL DEFAULT '',
|
||||
raw_record_summary JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
CHECK (jsonb_typeof(raw_record_summary) = 'object'),
|
||||
source_updated_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (source_item_id, revision)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_publications (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_item_version_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
|
||||
test_kb_id VARCHAR(255) NOT NULL,
|
||||
knowledge_id VARCHAR(255) NOT NULL DEFAULT '',
|
||||
ingest_state VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (ingest_state IN ('pending', 'uploading', 'uploaded', 'failed')),
|
||||
processing_state VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')),
|
||||
retrieval_state VARCHAR(20) NOT NULL DEFAULT 'unpublished'
|
||||
CHECK (retrieval_state IN ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed')),
|
||||
parser_generation VARCHAR(255) NOT NULL DEFAULT '',
|
||||
index_generation VARCHAR(255) NOT NULL DEFAULT '',
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (source_item_version_id, test_kb_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_items_change
|
||||
ON source_items (source_connection_id, source_change, updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_items_visibility
|
||||
ON source_items (visibility, updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_item_versions_content
|
||||
ON source_item_versions (content_object_id)
|
||||
WHERE content_object_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_test_publications_state
|
||||
ON test_publications (processing_state, retrieval_state, updated_at DESC);
|
||||
@@ -0,0 +1,12 @@
|
||||
DROP TABLE IF EXISTS acceptance_results;
|
||||
DROP INDEX IF EXISTS idx_test_publications_poll;
|
||||
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS superseded_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS supersedes_publication_id;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS stalled_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS completed_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS processing_started_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_observed_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_updated_at;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_status;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS poll_sequence;
|
||||
@@ -0,0 +1,33 @@
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS poll_sequence INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (poll_sequence >= 0);
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_status VARCHAR(32) NOT NULL DEFAULT '';
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_updated_at TIMESTAMPTZ;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_observed_at TIMESTAMPTZ;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS stalled_at TIMESTAMPTZ;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS supersedes_publication_id VARCHAR(36)
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT;
|
||||
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS superseded_at TIMESTAMPTZ;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_results (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
publication_id VARCHAR(36) NOT NULL
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT,
|
||||
run_id VARCHAR(36) NOT NULL,
|
||||
check_type VARCHAR(32) NOT NULL
|
||||
CHECK (check_type IN ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches')),
|
||||
state VARCHAR(20) NOT NULL
|
||||
CHECK (state IN ('pending', 'passed', 'failed', 'manual_required')),
|
||||
policy_version VARCHAR(64) NOT NULL,
|
||||
evidence_summary JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
CHECK (jsonb_typeof(evidence_summary) = 'object'),
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (publication_id, run_id, check_type)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_test_publications_poll
|
||||
ON test_publications (processing_state, last_observed_at, updated_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_acceptance_results_publication
|
||||
ON acceptance_results (publication_id, observed_at DESC);
|
||||
@@ -0,0 +1,18 @@
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM source_connections
|
||||
WHERE source_type = 'manual'
|
||||
) THEN
|
||||
RAISE EXCEPTION 'cannot downgrade while manual source connections exist';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
ALTER TABLE source_connections
|
||||
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
|
||||
|
||||
ALTER TABLE source_connections
|
||||
ADD CONSTRAINT source_connections_source_type_check
|
||||
CHECK (source_type IN ('pp', 'alist', 'dingtalk'));
|
||||
@@ -0,0 +1,6 @@
|
||||
ALTER TABLE source_connections
|
||||
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
|
||||
|
||||
ALTER TABLE source_connections
|
||||
ADD CONSTRAINT source_connections_source_type_check
|
||||
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual'));
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS source_scan_changes;
|
||||
DROP TABLE IF EXISTS source_scans;
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE IF NOT EXISTS source_scans (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
source_type VARCHAR(32) NOT NULL CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
|
||||
state VARCHAR(16) NOT NULL CHECK (state IN ('pending', 'running', 'completed', 'failed')),
|
||||
snapshot_sha256 VARCHAR(64) NOT NULL DEFAULT '' CHECK (snapshot_sha256 = '' OR snapshot_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
observed_items INTEGER NOT NULL DEFAULT 0 CHECK (observed_items >= 0),
|
||||
unchanged_items INTEGER NOT NULL DEFAULT 0 CHECK (unchanged_items >= 0),
|
||||
added_items INTEGER NOT NULL DEFAULT 0 CHECK (added_items >= 0),
|
||||
updated_items INTEGER NOT NULL DEFAULT 0 CHECK (updated_items >= 0),
|
||||
over_limit_items INTEGER NOT NULL DEFAULT 0 CHECK (over_limit_items >= 0),
|
||||
missing_items INTEGER NOT NULL DEFAULT 0 CHECK (missing_items >= 0),
|
||||
planned_uploads INTEGER NOT NULL DEFAULT 0 CHECK (planned_uploads >= 0),
|
||||
error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_scans_one_active
|
||||
ON source_scans (source_connection_id)
|
||||
WHERE state IN ('pending', 'running');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_scans_connection_created
|
||||
ON source_scans (source_connection_id, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_scan_changes (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
source_scan_id VARCHAR(64) NOT NULL REFERENCES source_scans(id) ON DELETE RESTRICT,
|
||||
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
stable_external_id VARCHAR(512) NOT NULL,
|
||||
source_item_id VARCHAR(64) REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
change_type VARCHAR(16) NOT NULL CHECK (change_type IN ('added', 'updated', 'unavailable')),
|
||||
previous_revision VARCHAR(512) NOT NULL DEFAULT '',
|
||||
observed_revision VARCHAR(512) NOT NULL DEFAULT '',
|
||||
observed_size_bytes BIGINT CHECK (observed_size_bytes IS NULL OR observed_size_bytes >= 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (source_scan_id, stable_external_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_scan_changes_pending
|
||||
ON source_scan_changes (source_connection_id, change_type, created_at DESC, id DESC);
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS release_batch_items;
|
||||
DROP TABLE IF EXISTS release_batches;
|
||||
DROP TABLE IF EXISTS publication_reviews;
|
||||
@@ -0,0 +1,58 @@
|
||||
CREATE TABLE IF NOT EXISTS publication_reviews (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
publication_id VARCHAR(36) NOT NULL
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT,
|
||||
decision VARCHAR(16) NOT NULL
|
||||
CHECK (decision IN ('approved', 'rejected')),
|
||||
decided_visibility VARCHAR(20) NOT NULL
|
||||
CHECK (decided_visibility IN ('internal', 'external')),
|
||||
reviewer_ref VARCHAR(128) NOT NULL,
|
||||
decision_code VARCHAR(64) NOT NULL
|
||||
CHECK (decision_code ~ '^[a-z0-9_]+$'),
|
||||
acceptance_run_id VARCHAR(36) NOT NULL,
|
||||
publication_updated_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (publication_id, acceptance_run_id, publication_updated_at,
|
||||
reviewer_ref, decision, decided_visibility, decision_code)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_publication_reviews_latest
|
||||
ON publication_reviews (publication_id, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS release_batches (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'draft'
|
||||
CHECK (state IN ('draft', 'sealed', 'cancelled')),
|
||||
policy_version VARCHAR(64) NOT NULL,
|
||||
created_by VARCHAR(128) NOT NULL,
|
||||
sealed_by VARCHAR(128) NOT NULL DEFAULT '',
|
||||
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
sealed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_batches_state_created
|
||||
ON release_batches (state, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS release_batch_items (
|
||||
release_batch_id VARCHAR(36) NOT NULL
|
||||
REFERENCES release_batches(id) ON DELETE RESTRICT,
|
||||
publication_id VARCHAR(36) NOT NULL
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT,
|
||||
review_id VARCHAR(36) NOT NULL
|
||||
REFERENCES publication_reviews(id) ON DELETE RESTRICT,
|
||||
source_item_version_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
|
||||
acceptance_run_id VARCHAR(36) NOT NULL,
|
||||
publication_updated_at TIMESTAMPTZ NOT NULL,
|
||||
decided_visibility VARCHAR(20) NOT NULL
|
||||
CHECK (decided_visibility IN ('internal', 'external')),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (release_batch_id, publication_id),
|
||||
UNIQUE (publication_id),
|
||||
UNIQUE (review_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_batch_items_batch
|
||||
ON release_batch_items (release_batch_id, publication_id);
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS release_item_executions;
|
||||
DROP TABLE IF EXISTS release_executions;
|
||||
@@ -0,0 +1,65 @@
|
||||
CREATE TABLE IF NOT EXISTS release_executions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
release_batch_id VARCHAR(36) NOT NULL
|
||||
REFERENCES release_batches(id) ON DELETE RESTRICT,
|
||||
operation VARCHAR(16) NOT NULL
|
||||
CHECK (operation IN ('publish', 'rollback')),
|
||||
prior_execution_id VARCHAR(36)
|
||||
REFERENCES release_executions(id) ON DELETE RESTRICT,
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'running', 'completed', 'failed')),
|
||||
target_environment VARCHAR(16) NOT NULL DEFAULT 'production'
|
||||
CHECK (target_environment = 'production'),
|
||||
authorized_by VARCHAR(128) NOT NULL,
|
||||
authorization_code VARCHAR(64) NOT NULL,
|
||||
job_id VARCHAR(36) NOT NULL UNIQUE
|
||||
REFERENCES jobs(id) ON DELETE RESTRICT,
|
||||
manifest_sha256 CHAR(64) NOT NULL DEFAULT ''
|
||||
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
|
||||
completed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (completed_item_count >= 0),
|
||||
failed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (failed_item_count >= 0),
|
||||
error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
CHECK (operation = 'publish' OR prior_execution_id IS NOT NULL),
|
||||
CHECK (completed_item_count + failed_item_count <= item_count)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_publish_batch
|
||||
ON release_executions (release_batch_id)
|
||||
WHERE operation = 'publish';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_rollback_prior
|
||||
ON release_executions (prior_execution_id)
|
||||
WHERE operation = 'rollback';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_executions_batch_created
|
||||
ON release_executions (release_batch_id, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_release_executions_state_created
|
||||
ON release_executions (state, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS release_item_executions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
release_execution_id VARCHAR(36) NOT NULL
|
||||
REFERENCES release_executions(id) ON DELETE RESTRICT,
|
||||
publication_id VARCHAR(36) NOT NULL
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT,
|
||||
state VARCHAR(24) NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'staged', 'activated', 'verified', 'failed', 'rolled_back')),
|
||||
bundle_sha256 CHAR(64) NOT NULL DEFAULT ''
|
||||
CHECK (bundle_sha256 = '' OR bundle_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
previous_production_revision VARCHAR(128) NOT NULL DEFAULT '',
|
||||
production_revision VARCHAR(128) NOT NULL DEFAULT '',
|
||||
error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
UNIQUE (release_execution_id, publication_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_item_executions_execution
|
||||
ON release_item_executions (release_execution_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_release_item_executions_publication
|
||||
ON release_item_executions (publication_id, created_at DESC);
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS scope_change_events;
|
||||
DROP TABLE IF EXISTS source_item_scope_overrides;
|
||||
DROP TABLE IF EXISTS directory_scope_rules;
|
||||
@@ -0,0 +1,57 @@
|
||||
CREATE TABLE IF NOT EXISTS directory_scope_rules (
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
directory_path TEXT NOT NULL DEFAULT '',
|
||||
visibility VARCHAR(20) NOT NULL
|
||||
CHECK (visibility IN ('internal', 'external')),
|
||||
updated_by VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (source_connection_id, directory_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_item_scope_overrides (
|
||||
source_item_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
visibility VARCHAR(20) NOT NULL
|
||||
CHECK (visibility IN ('internal', 'external')),
|
||||
updated_by VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (source_item_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scope_change_events (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
target_type VARCHAR(16) NOT NULL
|
||||
CHECK (target_type IN ('directory', 'item')),
|
||||
source_connection_id VARCHAR(36)
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
directory_path TEXT NOT NULL DEFAULT '',
|
||||
source_item_id VARCHAR(36)
|
||||
REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
action VARCHAR(16) NOT NULL
|
||||
CHECK (action IN ('set', 'clear')),
|
||||
previous_visibility VARCHAR(20)
|
||||
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external')),
|
||||
decided_visibility VARCHAR(20)
|
||||
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external')),
|
||||
operator_ref VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
CHECK (
|
||||
(target_type = 'directory' AND source_connection_id IS NOT NULL AND source_item_id IS NULL)
|
||||
OR
|
||||
(target_type = 'item' AND source_connection_id IS NULL AND source_item_id IS NOT NULL)
|
||||
),
|
||||
CHECK (
|
||||
(action = 'set' AND decided_visibility IS NOT NULL)
|
||||
OR
|
||||
(action = 'clear' AND decided_visibility IS NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scope_change_events_target
|
||||
ON scope_change_events (target_type, source_connection_id, source_item_id, created_at DESC);
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS source_catalog_heads;
|
||||
DROP TABLE IF EXISTS source_catalog_nodes;
|
||||
DROP TABLE IF EXISTS source_catalog_snapshots;
|
||||
@@ -0,0 +1,56 @@
|
||||
CREATE TABLE IF NOT EXISTS source_catalog_snapshots (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
snapshot_sha256 CHAR(64) NOT NULL
|
||||
CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
source_label VARCHAR(255) NOT NULL,
|
||||
node_count INTEGER NOT NULL CHECK (node_count >= 1),
|
||||
folder_count INTEGER NOT NULL CHECK (folder_count >= 1),
|
||||
file_count INTEGER NOT NULL CHECK (file_count >= 0),
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (source_connection_id, snapshot_sha256),
|
||||
UNIQUE (source_connection_id, id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_catalog_nodes (
|
||||
snapshot_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
|
||||
node_id VARCHAR(36) NOT NULL,
|
||||
parent_node_id VARCHAR(36),
|
||||
node_type VARCHAR(16) NOT NULL
|
||||
CHECK (node_type IN ('folder', 'file')),
|
||||
name TEXT NOT NULL CHECK (name <> '' AND length(name) <= 1024),
|
||||
relative_path TEXT NOT NULL CHECK (length(relative_path) <= 4096),
|
||||
source_locator_hash CHAR(64) NOT NULL
|
||||
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
|
||||
source_item_id VARCHAR(36)
|
||||
REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
|
||||
modified_at TIMESTAMPTZ,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
direct_child_count INTEGER NOT NULL DEFAULT 0 CHECK (direct_child_count >= 0),
|
||||
subtree_file_count INTEGER NOT NULL DEFAULT 0 CHECK (subtree_file_count >= 0),
|
||||
subtree_bytes BIGINT NOT NULL DEFAULT 0 CHECK (subtree_bytes >= 0),
|
||||
PRIMARY KEY (snapshot_id, node_id),
|
||||
FOREIGN KEY (snapshot_id, parent_node_id)
|
||||
REFERENCES source_catalog_nodes(snapshot_id, node_id) ON DELETE RESTRICT
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_parent
|
||||
ON source_catalog_nodes (snapshot_id, parent_node_id, node_type, sort_order, name, node_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_source_item
|
||||
ON source_catalog_nodes (source_item_id)
|
||||
WHERE source_item_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_catalog_heads (
|
||||
source_connection_id VARCHAR(36) PRIMARY KEY
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
snapshot_id VARCHAR(36) NOT NULL,
|
||||
activated_at TIMESTAMPTZ NOT NULL,
|
||||
FOREIGN KEY (source_connection_id, snapshot_id)
|
||||
REFERENCES source_catalog_snapshots(source_connection_id, id) ON DELETE RESTRICT
|
||||
);
|
||||
@@ -0,0 +1,36 @@
|
||||
UPDATE scope_change_events
|
||||
SET previous_visibility = 'internal'
|
||||
WHERE previous_visibility IN ('both', 'blocked');
|
||||
UPDATE scope_change_events
|
||||
SET decided_visibility = 'internal'
|
||||
WHERE decided_visibility IN ('both', 'blocked');
|
||||
UPDATE source_item_scope_overrides
|
||||
SET visibility = 'internal'
|
||||
WHERE visibility IN ('both', 'blocked');
|
||||
UPDATE directory_scope_rules
|
||||
SET visibility = 'internal'
|
||||
WHERE visibility IN ('both', 'blocked');
|
||||
|
||||
ALTER TABLE scope_change_events
|
||||
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
|
||||
ALTER TABLE scope_change_events
|
||||
ADD CONSTRAINT scope_change_events_decided_visibility_check
|
||||
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external'));
|
||||
|
||||
ALTER TABLE scope_change_events
|
||||
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
|
||||
ALTER TABLE scope_change_events
|
||||
ADD CONSTRAINT scope_change_events_previous_visibility_check
|
||||
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external'));
|
||||
|
||||
ALTER TABLE source_item_scope_overrides
|
||||
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
|
||||
ALTER TABLE source_item_scope_overrides
|
||||
ADD CONSTRAINT source_item_scope_overrides_visibility_check
|
||||
CHECK (visibility IN ('internal', 'external'));
|
||||
|
||||
ALTER TABLE directory_scope_rules
|
||||
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
|
||||
ALTER TABLE directory_scope_rules
|
||||
ADD CONSTRAINT directory_scope_rules_visibility_check
|
||||
CHECK (visibility IN ('internal', 'external'));
|
||||
@@ -0,0 +1,25 @@
|
||||
-- Visibility policies used by the source catalog. Publication reviews remain
|
||||
-- scoped to one concrete internal or external test publication.
|
||||
ALTER TABLE directory_scope_rules
|
||||
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
|
||||
ALTER TABLE directory_scope_rules
|
||||
ADD CONSTRAINT directory_scope_rules_visibility_check
|
||||
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
|
||||
|
||||
ALTER TABLE source_item_scope_overrides
|
||||
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
|
||||
ALTER TABLE source_item_scope_overrides
|
||||
ADD CONSTRAINT source_item_scope_overrides_visibility_check
|
||||
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
|
||||
|
||||
ALTER TABLE scope_change_events
|
||||
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
|
||||
ALTER TABLE scope_change_events
|
||||
ADD CONSTRAINT scope_change_events_previous_visibility_check
|
||||
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external', 'both', 'blocked'));
|
||||
|
||||
ALTER TABLE scope_change_events
|
||||
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
|
||||
ALTER TABLE scope_change_events
|
||||
ADD CONSTRAINT scope_change_events_decided_visibility_check
|
||||
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external', 'both', 'blocked'));
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS processing_confirmations;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS target_scope;
|
||||
@@ -0,0 +1,36 @@
|
||||
ALTER TABLE test_publications
|
||||
ADD COLUMN IF NOT EXISTS target_scope VARCHAR(16) NOT NULL DEFAULT 'internal'
|
||||
CHECK (target_scope IN ('internal', 'external'));
|
||||
|
||||
UPDATE test_publications AS publication
|
||||
SET target_scope = COALESCE(
|
||||
(
|
||||
SELECT review.decided_visibility
|
||||
FROM publication_reviews AS review
|
||||
WHERE review.publication_id = publication.id
|
||||
AND review.decision = 'approved'
|
||||
AND review.decided_visibility IN ('internal', 'external')
|
||||
ORDER BY review.created_at DESC, review.id DESC
|
||||
LIMIT 1
|
||||
),
|
||||
CASE WHEN item_rule.visibility IN ('internal', 'external') THEN item_rule.visibility END,
|
||||
source.visibility
|
||||
)
|
||||
FROM source_item_versions AS version
|
||||
JOIN source_items AS source ON source.id = version.source_item_id
|
||||
LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id
|
||||
WHERE publication.source_item_version_id = version.id;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS processing_confirmations (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
publication_id VARCHAR(36) NOT NULL
|
||||
REFERENCES test_publications(id) ON DELETE RESTRICT,
|
||||
operator_ref VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL,
|
||||
confirmed_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (publication_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_confirmed
|
||||
ON processing_confirmations (confirmed_at DESC, publication_id);
|
||||
@@ -0,0 +1,7 @@
|
||||
ALTER TABLE release_batch_items
|
||||
ADD CONSTRAINT release_batch_items_publication_id_key UNIQUE (publication_id);
|
||||
ALTER TABLE release_batch_items
|
||||
ADD CONSTRAINT release_batch_items_review_id_key UNIQUE (review_id);
|
||||
|
||||
ALTER TABLE release_item_executions DROP COLUMN IF EXISTS action;
|
||||
ALTER TABLE release_batch_items DROP COLUMN IF EXISTS action;
|
||||
@@ -0,0 +1,12 @@
|
||||
ALTER TABLE release_batch_items
|
||||
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
|
||||
CHECK (action IN ('upsert', 'delete'));
|
||||
|
||||
ALTER TABLE release_item_executions
|
||||
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
|
||||
CHECK (action IN ('upsert', 'delete'));
|
||||
|
||||
ALTER TABLE release_batch_items
|
||||
DROP CONSTRAINT IF EXISTS release_batch_items_publication_id_key;
|
||||
ALTER TABLE release_batch_items
|
||||
DROP CONSTRAINT IF EXISTS release_batch_items_review_id_key;
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS ui_setting_revisions;
|
||||
@@ -0,0 +1,24 @@
|
||||
CREATE TABLE IF NOT EXISTS ui_setting_revisions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
state VARCHAR(16) NOT NULL CHECK (state IN ('validated', 'active', 'retired', 'failed')),
|
||||
document_refresh_seconds INTEGER NOT NULL CHECK (document_refresh_seconds IN (15, 30, 60)),
|
||||
resource_refresh_seconds INTEGER NOT NULL CHECK (resource_refresh_seconds IN (30, 60)),
|
||||
stalled_after_seconds INTEGER NOT NULL CHECK (stalled_after_seconds IN (1800, 3600)),
|
||||
finalizing_after_seconds INTEGER NOT NULL CHECK (finalizing_after_seconds IN (7200, 14400)),
|
||||
updated_by VARCHAR(128) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
activated_at TIMESTAMPTZ,
|
||||
retired_at TIMESTAMPTZ,
|
||||
error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$')
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_ui_setting_active
|
||||
ON ui_setting_revisions ((state)) WHERE state = 'active';
|
||||
|
||||
INSERT INTO ui_setting_revisions (
|
||||
id, state, document_refresh_seconds, resource_refresh_seconds,
|
||||
stalled_after_seconds, finalizing_after_seconds, updated_by, activated_at
|
||||
) VALUES (
|
||||
'settings-default', 'active', 15, 30, 1800, 7200, 'system:migration', NOW()
|
||||
) ON CONFLICT (id) DO NOTHING;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE jobs
|
||||
DROP COLUMN IF EXISTS started_at;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE jobs
|
||||
ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS source_catalog_change_items;
|
||||
DROP TABLE IF EXISTS source_catalog_change_sets;
|
||||
@@ -0,0 +1,42 @@
|
||||
CREATE TABLE IF NOT EXISTS source_catalog_change_sets (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE RESTRICT,
|
||||
from_snapshot_id VARCHAR(36)
|
||||
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
|
||||
to_snapshot_id VARCHAR(36) NOT NULL UNIQUE
|
||||
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'planned'
|
||||
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled')),
|
||||
add_count INTEGER NOT NULL DEFAULT 0 CHECK (add_count >= 0),
|
||||
update_count INTEGER NOT NULL DEFAULT 0 CHECK (update_count >= 0),
|
||||
move_scope_count INTEGER NOT NULL DEFAULT 0 CHECK (move_scope_count >= 0),
|
||||
delete_count INTEGER NOT NULL DEFAULT 0 CHECK (delete_count >= 0),
|
||||
no_change_count INTEGER NOT NULL DEFAULT 0 CHECK (no_change_count >= 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
confirmed_at TIMESTAMPTZ,
|
||||
applied_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_catalog_change_items (
|
||||
change_set_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
|
||||
source_locator_hash CHAR(64) NOT NULL
|
||||
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
|
||||
action VARCHAR(16) NOT NULL
|
||||
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change')),
|
||||
title TEXT NOT NULL,
|
||||
old_path TEXT NOT NULL DEFAULT '',
|
||||
new_path TEXT NOT NULL DEFAULT '',
|
||||
old_size_bytes BIGINT CHECK (old_size_bytes IS NULL OR old_size_bytes >= 0),
|
||||
new_size_bytes BIGINT CHECK (new_size_bytes IS NULL OR new_size_bytes >= 0),
|
||||
old_modified_at TIMESTAMPTZ,
|
||||
new_modified_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (change_set_id, source_locator_hash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_sets_source
|
||||
ON source_catalog_change_sets (source_connection_id, created_at DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_items_action
|
||||
ON source_catalog_change_items (change_set_id, action, title);
|
||||
@@ -0,0 +1,4 @@
|
||||
DELETE FROM source_catalog_change_items
|
||||
WHERE change_set_id IN (SELECT id FROM source_catalog_change_sets WHERE origin='backfill');
|
||||
DELETE FROM source_catalog_change_sets WHERE origin='backfill';
|
||||
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS origin;
|
||||
@@ -0,0 +1,87 @@
|
||||
ALTER TABLE source_catalog_change_sets
|
||||
ADD COLUMN IF NOT EXISTS origin VARCHAR(16) NOT NULL DEFAULT 'refresh'
|
||||
CHECK (origin IN ('refresh', 'backfill'));
|
||||
|
||||
WITH current_targets AS (
|
||||
SELECT head.source_connection_id,head.snapshot_id AS to_snapshot_id,
|
||||
(
|
||||
SELECT prior.id
|
||||
FROM source_catalog_snapshots prior
|
||||
JOIN source_catalog_snapshots current ON current.id=head.snapshot_id
|
||||
WHERE prior.source_connection_id=head.source_connection_id
|
||||
AND prior.id<>head.snapshot_id
|
||||
AND prior.observed_at<=current.observed_at
|
||||
ORDER BY prior.observed_at DESC,prior.id DESC
|
||||
LIMIT 1
|
||||
) AS from_snapshot_id
|
||||
FROM source_catalog_heads head
|
||||
)
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,origin,created_at)
|
||||
SELECT 'cs-'||md5(target.source_connection_id||'|'||target.to_snapshot_id),
|
||||
target.source_connection_id,target.from_snapshot_id,target.to_snapshot_id,'planned','backfill',NOW()
|
||||
FROM current_targets target
|
||||
ON CONFLICT (to_snapshot_id) DO NOTHING;
|
||||
|
||||
WITH change_sets AS (
|
||||
SELECT id,from_snapshot_id,to_snapshot_id
|
||||
FROM source_catalog_change_sets
|
||||
WHERE origin='backfill'
|
||||
), old_files AS (
|
||||
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
|
||||
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
|
||||
FROM change_sets change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
|
||||
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
|
||||
), new_files AS (
|
||||
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
|
||||
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
|
||||
FROM change_sets change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
|
||||
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
|
||||
), compared AS (
|
||||
SELECT COALESCE(new_files.change_set_id,old_files.change_set_id) AS change_set_id,
|
||||
COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
|
||||
COALESCE(new_files.name,old_files.name) AS title,
|
||||
COALESCE(old_files.relative_path,'') AS old_path,
|
||||
COALESCE(new_files.relative_path,'') AS new_path,
|
||||
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
|
||||
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
|
||||
CASE
|
||||
WHEN old_files.source_locator_hash IS NULL THEN 'add'
|
||||
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
|
||||
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
|
||||
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
|
||||
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
|
||||
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
|
||||
ELSE 'no_change'
|
||||
END AS action
|
||||
FROM old_files FULL OUTER JOIN new_files
|
||||
ON new_files.change_set_id=old_files.change_set_id
|
||||
AND new_files.source_locator_hash=old_files.source_locator_hash
|
||||
)
|
||||
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
|
||||
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,created_at)
|
||||
SELECT change_set_id,source_locator_hash,action,title,old_path,new_path,
|
||||
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,NOW()
|
||||
FROM compared
|
||||
ON CONFLICT (change_set_id,source_locator_hash) DO NOTHING;
|
||||
|
||||
UPDATE source_catalog_change_sets change_set SET
|
||||
add_count=summary.add_count,
|
||||
update_count=summary.update_count,
|
||||
move_scope_count=summary.move_scope_count,
|
||||
delete_count=summary.delete_count,
|
||||
no_change_count=summary.no_change_count
|
||||
FROM (
|
||||
SELECT change_set.id AS change_set_id,
|
||||
COUNT(item.*) FILTER (WHERE item.action='add')::integer AS add_count,
|
||||
COUNT(item.*) FILTER (WHERE item.action='update')::integer AS update_count,
|
||||
COUNT(item.*) FILTER (WHERE item.action='move_scope')::integer AS move_scope_count,
|
||||
COUNT(item.*) FILTER (WHERE item.action='delete')::integer AS delete_count,
|
||||
COUNT(item.*) FILTER (WHERE item.action='no_change')::integer AS no_change_count
|
||||
FROM source_catalog_change_sets change_set
|
||||
LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
|
||||
WHERE change_set.origin='backfill'
|
||||
GROUP BY change_set.id
|
||||
) summary
|
||||
WHERE change_set.id=summary.change_set_id;
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE release_executions
|
||||
DROP CONSTRAINT IF EXISTS release_execution_batch_progress_check,
|
||||
DROP COLUMN IF EXISTS total_batches,
|
||||
DROP COLUMN IF EXISTS current_batch,
|
||||
DROP COLUMN IF EXISTS batch_size;
|
||||
@@ -0,0 +1,16 @@
|
||||
ALTER TABLE release_executions
|
||||
ADD COLUMN IF NOT EXISTS batch_size INTEGER NOT NULL DEFAULT 50
|
||||
CHECK (batch_size BETWEEN 1 AND 100),
|
||||
ADD COLUMN IF NOT EXISTS current_batch INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (current_batch >= 0),
|
||||
ADD COLUMN IF NOT EXISTS total_batches INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (total_batches >= 0),
|
||||
ADD CONSTRAINT release_execution_batch_progress_check
|
||||
CHECK (current_batch <= total_batches OR total_batches = 0);
|
||||
|
||||
UPDATE release_executions
|
||||
SET total_batches = CASE WHEN item_count = 0 THEN 0 ELSE CEIL(item_count::numeric / batch_size)::integer END,
|
||||
current_batch = CASE
|
||||
WHEN state IN ('completed', 'failed') AND item_count > 0 THEN CEIL(item_count::numeric / batch_size)::integer
|
||||
ELSE current_batch
|
||||
END;
|
||||
@@ -0,0 +1,11 @@
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM jobs WHERE state = 'cancelled') THEN
|
||||
RAISE EXCEPTION 'cannot roll back job cancelled state while cancelled jobs exist';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
|
||||
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
|
||||
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed'));
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
|
||||
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
|
||||
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed', 'cancelled'));
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS operation_requests;
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE IF NOT EXISTS operation_requests (
|
||||
operation_type TEXT NOT NULL CHECK (operation_type <> '' AND char_length(operation_type) <= 100),
|
||||
operation_id TEXT NOT NULL CHECK (operation_id <> '' AND char_length(operation_id) <= 128),
|
||||
request_fingerprint_sha256 CHAR(64) NOT NULL CHECK (request_fingerprint_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
state TEXT NOT NULL DEFAULT 'completed' CHECK (state IN ('completed')),
|
||||
result JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
completed_at TIMESTAMPTZ NOT NULL,
|
||||
UNIQUE (operation_type, operation_id)
|
||||
);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE source_catalog_nodes DROP COLUMN IF EXISTS content_sha256;
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE source_catalog_nodes
|
||||
ADD COLUMN IF NOT EXISTS content_sha256 CHAR(64)
|
||||
CHECK (content_sha256 IS NULL OR content_sha256 ~ '^[0-9a-f]{64}$');
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS reason_code, DROP COLUMN IF EXISTS new_content_sha256, DROP COLUMN IF EXISTS old_content_sha256;
|
||||
ALTER TABLE source_items DROP COLUMN IF EXISTS identity_confidence;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- P2 stable identity: source items record how strongly their identity is anchored.
|
||||
ALTER TABLE source_items
|
||||
ADD COLUMN IF NOT EXISTS identity_confidence VARCHAR(16) NOT NULL DEFAULT 'path'
|
||||
CHECK (identity_confidence IN ('stable_id','object_id','content_hash','path'));
|
||||
|
||||
-- Change items gain content-hash evidence and a reason code so a rename can be
|
||||
-- classified as a proven move instead of delete+add when the bytes match 1:1.
|
||||
ALTER TABLE source_catalog_change_items
|
||||
ADD COLUMN IF NOT EXISTS old_content_sha256 CHAR(64) CHECK (old_content_sha256 IS NULL OR old_content_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
ADD COLUMN IF NOT EXISTS new_content_sha256 CHAR(64) CHECK (new_content_sha256 IS NULL OR new_content_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
ADD COLUMN IF NOT EXISTS reason_code VARCHAR(32) NOT NULL DEFAULT 'locator'
|
||||
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
|
||||
@@ -0,0 +1 @@
|
||||
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
|
||||
@@ -0,0 +1,99 @@
|
||||
-- P2/S8: canonical snapshot-diff classifier lives in the database so the
|
||||
-- importer, backfills and future ChangeSet execution share one source of truth.
|
||||
-- Two sequential statements (not a data-modifying CTE) so the recount sees the rows just written.
|
||||
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
|
||||
BEGIN
|
||||
WITH change_set AS (
|
||||
SELECT id,from_snapshot_id,to_snapshot_id FROM source_catalog_change_sets
|
||||
WHERE id=p_change_set_id
|
||||
), old_files AS (
|
||||
SELECT DISTINCT ON (node.source_locator_hash)
|
||||
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
|
||||
FROM change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
|
||||
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
|
||||
), new_files AS (
|
||||
SELECT DISTINCT ON (node.source_locator_hash)
|
||||
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
|
||||
FROM change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
|
||||
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
|
||||
), proven_moves AS (
|
||||
-- A content hash proves a move only when exactly one old path carries it and
|
||||
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
|
||||
SELECT content_sha256 FROM (
|
||||
SELECT old_files.content_sha256 FROM old_files WHERE old_files.content_sha256 IS NOT NULL
|
||||
GROUP BY old_files.content_sha256 HAVING COUNT(*)=1
|
||||
) one_old
|
||||
JOIN (
|
||||
SELECT new_files.content_sha256 FROM new_files WHERE new_files.content_sha256 IS NOT NULL
|
||||
GROUP BY new_files.content_sha256 HAVING COUNT(*)=1
|
||||
) one_new USING (content_sha256)
|
||||
), compared AS (
|
||||
SELECT COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
|
||||
COALESCE(new_files.name,old_files.name) AS title,
|
||||
COALESCE(old_files.relative_path,'') AS old_path,
|
||||
COALESCE(new_files.relative_path,'') AS new_path,
|
||||
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
|
||||
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
|
||||
old_files.content_sha256 AS old_content_sha256,new_files.content_sha256 AS new_content_sha256,
|
||||
CASE
|
||||
WHEN old_files.source_locator_hash IS NULL THEN 'add'
|
||||
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
|
||||
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
|
||||
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
|
||||
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
|
||||
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
|
||||
ELSE 'no_change'
|
||||
END AS action,
|
||||
CASE
|
||||
WHEN old_files.source_locator_hash IS NULL AND new_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
|
||||
THEN 'content_hash_move'
|
||||
WHEN new_files.source_locator_hash IS NULL AND old_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
|
||||
THEN 'content_hash_move'
|
||||
WHEN old_files.source_locator_hash IS NULL THEN 'locator'
|
||||
WHEN new_files.source_locator_hash IS NULL THEN 'source_missing'
|
||||
WHEN old_files.content_sha256 IS DISTINCT FROM new_files.content_sha256
|
||||
AND (old_files.relative_path<>new_files.relative_path OR old_files.name<>new_files.name)
|
||||
THEN 'content_hash_conflict'
|
||||
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path OR old_files.name IS DISTINCT FROM new_files.name
|
||||
THEN 'path_change'
|
||||
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes THEN 'size_change'
|
||||
WHEN old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'content_change'
|
||||
ELSE 'locator'
|
||||
END AS reason_code
|
||||
FROM old_files FULL OUTER JOIN new_files USING(source_locator_hash)
|
||||
)
|
||||
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
|
||||
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,created_at)
|
||||
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
|
||||
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
|
||||
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,NOW()
|
||||
FROM change_set CROSS JOIN compared
|
||||
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
|
||||
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
|
||||
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
|
||||
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
|
||||
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
|
||||
reason_code=EXCLUDED.reason_code;
|
||||
|
||||
UPDATE source_catalog_change_sets change_set SET
|
||||
add_count=summary.add_count,
|
||||
update_count=summary.update_count,
|
||||
move_scope_count=summary.move_scope_count,
|
||||
delete_count=summary.delete_count,
|
||||
no_change_count=summary.no_change_count
|
||||
FROM (
|
||||
SELECT items.change_set_id,
|
||||
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
|
||||
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
|
||||
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
|
||||
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
|
||||
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
|
||||
FROM source_catalog_change_items items
|
||||
WHERE items.change_set_id=p_change_set_id
|
||||
GROUP BY items.change_set_id
|
||||
) summary
|
||||
WHERE change_set.id=summary.change_set_id;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Revert S8 contract. Change rows written with new vocabulary remain but become
|
||||
-- unclassifiable only if constraints reject them; we keep the checks permissive
|
||||
-- first, then drop added columns.
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_version_id;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_version_id;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_source_item_id;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_source_item_id;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_scope;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_scope;
|
||||
UPDATE source_catalog_change_items SET action='move_scope' WHERE action='move';
|
||||
UPDATE source_catalog_change_items SET reason_code='path_change' WHERE reason_code='scope_change';
|
||||
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
|
||||
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
|
||||
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
|
||||
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
|
||||
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
|
||||
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change'));
|
||||
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS failure_reason;
|
||||
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS cancelled_at;
|
||||
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS applying_at;
|
||||
UPDATE source_catalog_change_sets SET move_scope_count=move_scope_count+move_count, move_count=0;
|
||||
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS move_count;
|
||||
UPDATE source_catalog_change_sets SET state='confirmed' WHERE state IN ('applying','failed');
|
||||
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
|
||||
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
|
||||
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled'));
|
||||
@@ -0,0 +1,167 @@
|
||||
-- S8 complete ChangeSet contract: full lifecycle, distinct move vs move_scope,
|
||||
-- old/new effective scope, and version identity evidence.
|
||||
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
|
||||
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
|
||||
CHECK (state IN ('planned', 'confirmed', 'applying', 'applied', 'cancelled', 'failed'));
|
||||
ALTER TABLE source_catalog_change_sets
|
||||
ADD COLUMN IF NOT EXISTS move_count INTEGER NOT NULL DEFAULT 0 CHECK (move_count >= 0),
|
||||
ADD COLUMN IF NOT EXISTS applying_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS cancelled_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS failure_reason TEXT NOT NULL DEFAULT '';
|
||||
|
||||
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
|
||||
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
|
||||
CHECK (action IN ('add', 'update', 'move', 'move_scope', 'delete', 'no_change'));
|
||||
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
|
||||
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
|
||||
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing','scope_change'));
|
||||
ALTER TABLE source_catalog_change_items
|
||||
ADD COLUMN IF NOT EXISTS old_scope VARCHAR(16),
|
||||
ADD COLUMN IF NOT EXISTS new_scope VARCHAR(16),
|
||||
ADD COLUMN IF NOT EXISTS old_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
ADD COLUMN IF NOT EXISTS new_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
|
||||
ADD COLUMN IF NOT EXISTS old_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT,
|
||||
ADD COLUMN IF NOT EXISTS new_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT;
|
||||
|
||||
-- Canonical classifier v2 (single source of truth; importer and backfills call it).
|
||||
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
|
||||
BEGIN
|
||||
WITH change_set AS (
|
||||
SELECT id,from_snapshot_id,to_snapshot_id,source_connection_id FROM source_catalog_change_sets
|
||||
WHERE id=p_change_set_id
|
||||
), old_files AS (
|
||||
SELECT DISTINCT ON (node.source_locator_hash)
|
||||
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
node.source_item_id
|
||||
FROM change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
|
||||
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
|
||||
), new_files AS (
|
||||
SELECT DISTINCT ON (node.source_locator_hash)
|
||||
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
node.source_item_id
|
||||
FROM change_set
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
|
||||
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
|
||||
)
|
||||
-- Old side reports the source-recorded visibility (what the last sync stored);
|
||||
-- new side reports the currently effective policy (override > item). A policy
|
||||
-- change between snapshots therefore surfaces as move_scope, not a fake move.
|
||||
, scoped_old AS (
|
||||
SELECT old_files.*,
|
||||
(SELECT item.visibility FROM source_items item WHERE item.id=old_files.source_item_id) AS item_scope
|
||||
FROM old_files
|
||||
), scoped_new AS (
|
||||
SELECT new_files.*,
|
||||
(SELECT COALESCE(override.visibility,item.visibility)
|
||||
FROM source_items item
|
||||
LEFT JOIN source_item_scope_overrides override ON override.source_item_id=item.id
|
||||
WHERE item.id=new_files.source_item_id) AS item_scope
|
||||
FROM new_files
|
||||
), with_rule_scope AS (
|
||||
SELECT f.*,COALESCE(f.item_scope,(
|
||||
SELECT rule.visibility FROM directory_scope_rules rule
|
||||
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
|
||||
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
|
||||
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
|
||||
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
|
||||
FROM (SELECT * FROM scoped_old) f
|
||||
), with_rule_scope_new AS (
|
||||
SELECT f.*,COALESCE(f.item_scope,(
|
||||
SELECT rule.visibility FROM directory_scope_rules rule
|
||||
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
|
||||
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
|
||||
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
|
||||
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
|
||||
FROM (SELECT * FROM scoped_new) f
|
||||
), proven_moves AS (
|
||||
-- A content hash proves a move only when exactly one old path carries it and
|
||||
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
|
||||
SELECT content_sha256 FROM (
|
||||
SELECT content_sha256 FROM scoped_old WHERE content_sha256 IS NOT NULL
|
||||
GROUP BY content_sha256 HAVING COUNT(*)=1
|
||||
) one_old
|
||||
JOIN (
|
||||
SELECT content_sha256 FROM scoped_new WHERE content_sha256 IS NOT NULL
|
||||
GROUP BY content_sha256 HAVING COUNT(*)=1
|
||||
) one_new USING (content_sha256)
|
||||
), compared AS (
|
||||
SELECT COALESCE(n.source_locator_hash,o.source_locator_hash) AS source_locator_hash,
|
||||
COALESCE(n.name,o.name) AS title,
|
||||
COALESCE(o.relative_path,'') AS old_path,
|
||||
COALESCE(n.relative_path,'') AS new_path,
|
||||
o.size_bytes AS old_size_bytes,n.size_bytes AS new_size_bytes,
|
||||
o.modified_at AS old_modified_at,n.modified_at AS new_modified_at,
|
||||
o.content_sha256 AS old_content_sha256,n.content_sha256 AS new_content_sha256,
|
||||
o.effective_scope AS old_scope,n.effective_scope AS new_scope,
|
||||
o.source_item_id AS old_source_item_id,n.source_item_id AS new_source_item_id,
|
||||
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=o.source_item_id AND i.revision=v.revision WHERE v.source_item_id=o.source_item_id LIMIT 1) AS old_version_id,
|
||||
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=n.source_item_id AND i.revision=v.revision WHERE v.source_item_id=n.source_item_id LIMIT 1) AS new_version_id,
|
||||
CASE
|
||||
WHEN o.source_locator_hash IS NULL THEN 'add'
|
||||
WHEN n.source_locator_hash IS NULL THEN 'delete'
|
||||
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'move_scope'
|
||||
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256
|
||||
AND (o.relative_path<>n.relative_path OR o.name<>n.name) THEN 'move'
|
||||
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256<>n.content_sha256 THEN 'update'
|
||||
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'move_scope'
|
||||
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes OR o.modified_at IS DISTINCT FROM n.modified_at THEN 'update'
|
||||
ELSE 'no_change'
|
||||
END AS action,
|
||||
CASE
|
||||
WHEN o.source_locator_hash IS NULL AND n.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
|
||||
THEN 'content_hash_move'
|
||||
WHEN n.source_locator_hash IS NULL AND o.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
|
||||
THEN 'content_hash_move'
|
||||
WHEN o.source_locator_hash IS NULL THEN 'locator'
|
||||
WHEN n.source_locator_hash IS NULL THEN 'source_missing'
|
||||
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'scope_change'
|
||||
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256 THEN 'content_hash_move'
|
||||
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL THEN 'content_hash_conflict'
|
||||
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'path_change'
|
||||
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes THEN 'size_change'
|
||||
WHEN o.modified_at IS DISTINCT FROM n.modified_at THEN 'content_change'
|
||||
ELSE 'locator'
|
||||
END AS reason_code
|
||||
FROM with_rule_scope o FULL OUTER JOIN with_rule_scope_new n USING(source_locator_hash)
|
||||
)
|
||||
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
|
||||
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,
|
||||
old_scope,new_scope,old_source_item_id,new_source_item_id,old_version_id,new_version_id,created_at)
|
||||
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
|
||||
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
|
||||
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,
|
||||
compared.old_scope,compared.new_scope,compared.old_source_item_id,compared.new_source_item_id,compared.old_version_id,compared.new_version_id,NOW()
|
||||
FROM change_set CROSS JOIN compared
|
||||
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
|
||||
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
|
||||
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
|
||||
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
|
||||
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
|
||||
reason_code=EXCLUDED.reason_code,
|
||||
old_scope=EXCLUDED.old_scope,new_scope=EXCLUDED.new_scope,
|
||||
old_source_item_id=EXCLUDED.old_source_item_id,new_source_item_id=EXCLUDED.new_source_item_id,
|
||||
old_version_id=EXCLUDED.old_version_id,new_version_id=EXCLUDED.new_version_id;
|
||||
|
||||
UPDATE source_catalog_change_sets change_set SET
|
||||
add_count=summary.add_count,
|
||||
update_count=summary.update_count,
|
||||
move_count=summary.move_count,
|
||||
move_scope_count=summary.move_scope_count,
|
||||
delete_count=summary.delete_count,
|
||||
no_change_count=summary.no_change_count
|
||||
FROM (
|
||||
SELECT items.change_set_id,
|
||||
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
|
||||
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
|
||||
COUNT(*) FILTER (WHERE items.action='move')::integer AS move_count,
|
||||
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
|
||||
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
|
||||
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
|
||||
FROM source_catalog_change_items items
|
||||
WHERE items.change_set_id=p_change_set_id
|
||||
GROUP BY items.change_set_id
|
||||
) summary
|
||||
WHERE change_set.id=summary.change_set_id;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,5 @@
|
||||
DROP INDEX IF EXISTS idx_source_catalog_change_items_seq;
|
||||
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS item_seq;
|
||||
ALTER TABLE test_publications DROP COLUMN IF EXISTS change_locator_hash, DROP COLUMN IF EXISTS change_set_id;
|
||||
DROP INDEX IF EXISTS idx_processing_confirmations_change_set;
|
||||
ALTER TABLE processing_confirmations DROP COLUMN IF EXISTS change_set_id;
|
||||
@@ -0,0 +1,14 @@
|
||||
-- P3: confirmations bind to the ChangeSet that produced them; publications record
|
||||
-- provenance so every upload/removal job can be traced back to change evidence.
|
||||
ALTER TABLE processing_confirmations
|
||||
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT;
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_change_set
|
||||
ON processing_confirmations (change_set_id) WHERE change_set_id IS NOT NULL;
|
||||
|
||||
-- Surrogate public handle for selecting change items without leaking locator hashes.
|
||||
ALTER TABLE source_catalog_change_items ADD COLUMN IF NOT EXISTS item_seq BIGSERIAL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_catalog_change_items_seq ON source_catalog_change_items (item_seq);
|
||||
|
||||
ALTER TABLE test_publications
|
||||
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
|
||||
ADD COLUMN IF NOT EXISTS change_locator_hash CHAR(64) CHECK (change_locator_hash IS NULL OR change_locator_hash ~ '^[0-9a-f]{64}$');
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS jobs_processing_identity;
|
||||
ALTER TABLE jobs DROP COLUMN IF EXISTS business_action, DROP COLUMN IF EXISTS business_target_kb_id, DROP COLUMN IF EXISTS business_version_id, DROP COLUMN IF EXISTS business_change_set_id;
|
||||
@@ -0,0 +1,13 @@
|
||||
-- P5 formal processing identity: every confirmed change-set job carries an
|
||||
-- explicit business identity (publication uniqueness already exists since
|
||||
-- migration 000002) so a duplicate delivery or worker restart cannot enqueue
|
||||
-- the same action twice.
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_change_set_id VARCHAR(36);
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_version_id VARCHAR(36);
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_target_kb_id VARCHAR(255);
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_action VARCHAR(20)
|
||||
CHECK (business_action IS NULL OR business_action IN ('upload', 'remove'));
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS jobs_processing_identity
|
||||
ON jobs (business_change_set_id, business_version_id, business_target_kb_id, business_action)
|
||||
WHERE business_change_set_id IS NOT NULL;
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS job_retry_operations;
|
||||
DROP TABLE IF EXISTS job_failures;
|
||||
ALTER TABLE jobs DROP COLUMN IF EXISTS last_error_code;
|
||||
@@ -0,0 +1,48 @@
|
||||
-- P6 DLQ: every terminal job failure keeps its immutable jobs audit row AND
|
||||
-- gains a dead-letter projection. Requeued failures flip to 'requeued'; the
|
||||
-- history is append-only. Retrying resets the attempt budget on the SAME job
|
||||
-- row so the P5 business identity can never be duplicated.
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS last_error_code VARCHAR(64) NOT NULL DEFAULT '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_failures (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
job_id VARCHAR(36) NOT NULL REFERENCES jobs(id),
|
||||
job_type VARCHAR(100) NOT NULL,
|
||||
publication_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
error_code VARCHAR(64) NOT NULL,
|
||||
last_error TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL CHECK (attempt > 0),
|
||||
max_attempts_at_failure INTEGER NOT NULL CHECK (max_attempts_at_failure > 0),
|
||||
business_change_set_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
business_version_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
business_action VARCHAR(16) NOT NULL DEFAULT '',
|
||||
business_target_kb_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'open'
|
||||
CHECK (state IN ('open', 'requeued', 'abandoned')),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
resolved_at TIMESTAMPTZ,
|
||||
resolution_actor_ref VARCHAR(255) NOT NULL DEFAULT '',
|
||||
resolution_reason TEXT NOT NULL DEFAULT '',
|
||||
UNIQUE (job_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_job_failures_open
|
||||
ON job_failures (created_at DESC) WHERE state = 'open';
|
||||
|
||||
-- Retry audit: who re-queued which publication and with what reason
|
||||
-- (batch-shared key). job_id is not stored: it is derivable from the
|
||||
-- publication and would break the audit row on cross-batch duplicates.
|
||||
CREATE TABLE IF NOT EXISTS job_retry_operations (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
batch_key VARCHAR(255) NOT NULL,
|
||||
publication_id VARCHAR(64) NOT NULL,
|
||||
actor_ref VARCHAR(255) NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
outcome VARCHAR(24) NOT NULL
|
||||
CHECK (outcome IN ('started', 'already_running', 'not_retryable')),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (batch_key, publication_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_job_retry_operations_publication
|
||||
ON job_retry_operations (publication_id, created_at DESC);
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS worker_heartbeats;
|
||||
@@ -0,0 +1,9 @@
|
||||
-- P10 worker liveness: refreshed whenever a worker touches the queue (claim
|
||||
-- attempts), read by the API to decide whether the worker process is alive.
|
||||
CREATE TABLE IF NOT EXISTS worker_heartbeats (
|
||||
worker_id VARCHAR(128) PRIMARY KEY,
|
||||
job_types TEXT NOT NULL DEFAULT '',
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_heartbeats_seen ON worker_heartbeats(last_seen_at DESC);
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE release_batches DROP COLUMN IF EXISTS release_change_set_id;
|
||||
DROP TABLE IF EXISTS release_change_set_items;
|
||||
DROP TABLE IF EXISTS release_change_sets;
|
||||
@@ -0,0 +1,50 @@
|
||||
-- R1 persistent release ChangeSet: the real-time release plan becomes an
|
||||
-- immutable, snapshot-bound object. Batches reference CONFIRMED sets; later
|
||||
-- source refreshes can never mutate an existing set or its manifest.
|
||||
CREATE TABLE IF NOT EXISTS release_change_sets (
|
||||
id VARCHAR(40) PRIMARY KEY,
|
||||
name VARCHAR(200) NOT NULL,
|
||||
number INTEGER NOT NULL,
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'planned'
|
||||
CHECK (state IN ('planned','confirmed','consumed','cancelled')),
|
||||
source_snapshot_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
source_change_set_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
acceptance_cutoff TIMESTAMPTZ NOT NULL,
|
||||
production_baseline_ref VARCHAR(64) NOT NULL DEFAULT '',
|
||||
item_count INTEGER NOT NULL DEFAULT 0,
|
||||
add_count INTEGER NOT NULL DEFAULT 0,
|
||||
update_count INTEGER NOT NULL DEFAULT 0,
|
||||
delete_count INTEGER NOT NULL DEFAULT 0,
|
||||
blocked_count INTEGER NOT NULL DEFAULT 0,
|
||||
manifest_sha256 CHAR(64) NOT NULL
|
||||
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
created_by VARCHAR(120) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
confirmed_by VARCHAR(120) NOT NULL DEFAULT '',
|
||||
confirmed_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS release_change_sets_number_key
|
||||
ON release_change_sets(number);
|
||||
|
||||
-- Frozen per-item plan: every field is written once at generation and never
|
||||
-- updated (delete protection + immutability are enforced by having no UPDATE
|
||||
-- path and a state machine on the parent).
|
||||
CREATE TABLE IF NOT EXISTS release_change_set_items (
|
||||
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
|
||||
seq INTEGER NOT NULL,
|
||||
publication_id VARCHAR(64) NOT NULL,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
target_scope VARCHAR(16) NOT NULL CHECK (target_scope IN ('internal','external')),
|
||||
action VARCHAR(16) NOT NULL CHECK (action IN ('add','update','delete','blocked','unchanged')),
|
||||
test_state VARCHAR(16) NOT NULL,
|
||||
production_state VARCHAR(16) NOT NULL,
|
||||
can_release BOOLEAN NOT NULL DEFAULT false,
|
||||
blocker VARCHAR(32) NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (change_set_id, seq)
|
||||
);
|
||||
|
||||
-- Batches optionally bind to a confirmed change set (formal releases).
|
||||
ALTER TABLE release_batches
|
||||
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
|
||||
REFERENCES release_change_sets(id);
|
||||
CREATE INDEX IF NOT EXISTS release_batches_change_set ON release_batches(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Revert R4 policy. Rollback-stage rows are folded back to running/completed so
|
||||
-- the old CHECK can be restored without data loss surprises in test schemas.
|
||||
UPDATE release_executions SET state='running' WHERE state='rolling_back';
|
||||
UPDATE release_executions SET state='completed' WHERE state='rolled_back';
|
||||
UPDATE release_executions SET state='failed' WHERE state='rollback_failed';
|
||||
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
|
||||
ALTER TABLE release_executions
|
||||
ADD CONSTRAINT release_executions_state_check CHECK (state IN ('pending','running','completed','failed'));
|
||||
ALTER TABLE release_executions DROP COLUMN IF EXISTS snapshot_verification_id;
|
||||
ALTER TABLE release_executions DROP COLUMN IF EXISTS restore_trigger;
|
||||
ALTER TABLE release_executions DROP COLUMN IF EXISTS failure_level;
|
||||
DROP INDEX IF EXISTS release_executions_rollback_state;
|
||||
@@ -0,0 +1,22 @@
|
||||
-- R4 three-level rollback policy:
|
||||
-- single_item: one publication permanently failed; later items were skipped
|
||||
-- and (for publish) compensated by an automatic rollback.
|
||||
-- batch: remaining batches were never started after a permanent failure.
|
||||
-- overall: restore the entire release from the pre-release Production
|
||||
-- Snapshot; only a production authorizer may trigger it, and it
|
||||
-- requires an R2 snapshot verification id as evidence.
|
||||
-- The rollback execution walks pending -> rolling_back -> rolled_back /
|
||||
-- rollback_failed so the UI can show "compensating now" distinctly.
|
||||
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
|
||||
ALTER TABLE release_executions
|
||||
ADD CONSTRAINT release_executions_state_check CHECK (state IN (
|
||||
'pending','running','rolling_back','completed','failed','rolled_back','rollback_failed'));
|
||||
|
||||
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS failure_level VARCHAR(16) NOT NULL DEFAULT ''
|
||||
CHECK (failure_level IN ('','single_item','batch','overall'));
|
||||
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS restore_trigger VARCHAR(32) NOT NULL DEFAULT ''
|
||||
CHECK (restore_trigger IN ('','auto_rollback_after_failure','snapshot_restore'));
|
||||
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS snapshot_verification_id VARCHAR(64) NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS release_executions_rollback_state
|
||||
ON release_executions(prior_execution_id) WHERE operation = 'rollback';
|
||||
@@ -0,0 +1,4 @@
|
||||
DROP TABLE IF EXISTS release_change_set_catalog_links;
|
||||
DROP TABLE IF EXISTS release_change_set_snapshot_links;
|
||||
DROP INDEX IF EXISTS release_executions_change_set;
|
||||
ALTER TABLE release_executions DROP COLUMN IF EXISTS release_change_set_id;
|
||||
@@ -0,0 +1,46 @@
|
||||
-- R6 release audit chain: promote the evidence links that used to live only in
|
||||
-- JSONB into real, immutable foreign keys so any execution can be traced
|
||||
-- end-to-end (execution -> batch -> release change set -> source snapshots ->
|
||||
-- source change sets -> review/acceptance -> source item/version).
|
||||
ALTER TABLE release_executions
|
||||
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
|
||||
REFERENCES release_change_sets(id);
|
||||
CREATE INDEX IF NOT EXISTS release_executions_change_set
|
||||
ON release_executions(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
|
||||
|
||||
-- Backfill the denormalized execution binding from its batch (R1 binding).
|
||||
UPDATE release_executions AS execution
|
||||
SET release_change_set_id = batch.release_change_set_id
|
||||
FROM release_batches AS batch
|
||||
WHERE execution.release_batch_id = batch.id
|
||||
AND batch.release_change_set_id IS NOT NULL
|
||||
AND execution.release_change_set_id IS NULL;
|
||||
|
||||
-- Normalized snapshot bindings (display JSONB stays, but FK rows are truth).
|
||||
CREATE TABLE IF NOT EXISTS release_change_set_snapshot_links (
|
||||
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
|
||||
snapshot_id VARCHAR(36) NOT NULL REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (change_set_id, snapshot_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS release_change_set_catalog_links (
|
||||
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
|
||||
catalog_change_set_id VARCHAR(36) NOT NULL REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (change_set_id, catalog_change_set_id)
|
||||
);
|
||||
|
||||
-- Backfill from the JSONB captured at generation time (skip ids that no longer
|
||||
-- exist: RESTRICT keeps live rows, deleted ones simply do not link).
|
||||
INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id)
|
||||
SELECT rcs.id, elem.value
|
||||
FROM release_change_sets rcs
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_snapshot_ids) AS elem(value)
|
||||
JOIN source_catalog_snapshots s ON s.id = elem.value
|
||||
ON CONFLICT DO NOTHING;
|
||||
INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id)
|
||||
SELECT rcs.id, elem.value
|
||||
FROM release_change_sets rcs
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_change_set_ids) AS elem(value)
|
||||
JOIN source_catalog_change_sets c ON c.id = elem.value
|
||||
ON CONFLICT DO NOTHING;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Revert V1 revisions.
|
||||
ALTER TABLE source_connections DROP COLUMN IF EXISTS current_revision_id,
|
||||
DROP COLUMN IF EXISTS updated_by,
|
||||
DROP COLUMN IF EXISTS last_scheduled_at,
|
||||
DROP COLUMN IF EXISTS next_refresh_at;
|
||||
DROP TABLE IF EXISTS source_connection_revisions CASCADE;
|
||||
@@ -0,0 +1,64 @@
|
||||
-- V1 source connection revisions: settings edits become validated -> active -> retired
|
||||
-- versions; source_connections keeps ONE authoritative active snapshot that runtime
|
||||
-- schedulers and existing FKs read. Secrets stay references only: base_url is visible,
|
||||
-- secret_ref names where a credential lives and is never resolved through this table.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_connection_revisions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
connection_id VARCHAR(36) NOT NULL REFERENCES source_connections(id) ON DELETE CASCADE,
|
||||
revision_number INTEGER NOT NULL CHECK (revision_number > 0),
|
||||
state VARCHAR(16) NOT NULL
|
||||
CHECK (state IN ('validated', 'active', 'retired', 'failed')),
|
||||
source_type VARCHAR(20) NOT NULL
|
||||
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
CHECK (jsonb_typeof(schedule) = 'object'),
|
||||
created_by VARCHAR(128) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
validated_at TIMESTAMPTZ,
|
||||
activated_at TIMESTAMPTZ,
|
||||
retired_at TIMESTAMPTZ,
|
||||
error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$'),
|
||||
UNIQUE (connection_id, revision_number)
|
||||
);
|
||||
|
||||
-- Exactly one active revision per connection.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_source_connection_active_revision
|
||||
ON source_connection_revisions (connection_id) WHERE state = 'active';
|
||||
|
||||
-- Scheduler state lives next to the active snapshot so the existing FKs and UI
|
||||
-- keep reading source_connections. next_refresh_at is NULL until the operator
|
||||
-- sets a refresh interval, and it is ALWAYS derived from the schedule on the
|
||||
-- active revision (never trusted from clients).
|
||||
ALTER TABLE source_connections
|
||||
ADD COLUMN IF NOT EXISTS next_refresh_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS last_scheduled_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS current_revision_id VARCHAR(36) REFERENCES source_connection_revisions(id),
|
||||
ADD COLUMN IF NOT EXISTS updated_by VARCHAR(128) NOT NULL DEFAULT 'system';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_connections_due
|
||||
ON source_connections (next_refresh_at)
|
||||
WHERE enabled AND next_refresh_at IS NOT NULL;
|
||||
|
||||
-- Seed: every existing connection gets revision 1 reflecting its current row,
|
||||
-- marked active, created by the migration itself.
|
||||
INSERT INTO source_connection_revisions (
|
||||
id, connection_id, revision_number, state, source_type, name, base_url,
|
||||
secret_ref, enabled, schedule, created_by, validated_at, activated_at
|
||||
)
|
||||
SELECT
|
||||
'scr-' || md5(c.id), c.id, 1, 'active', c.source_type, c.name, c.base_url,
|
||||
c.secret_ref, c.enabled, c.schedule, 'migration-31', NOW(), NOW()
|
||||
FROM source_connections AS c
|
||||
ON CONFLICT (connection_id, revision_number) DO NOTHING;
|
||||
|
||||
UPDATE source_connections AS c
|
||||
SET current_revision_id = COALESCE(c.current_revision_id, 'scr-' || md5(c.id))
|
||||
FROM source_connection_revisions AS r
|
||||
WHERE r.connection_id = c.id AND r.revision_number = 1
|
||||
AND c.current_revision_id IS NULL;
|
||||
@@ -0,0 +1,633 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 31 {
|
||||
t.Fatalf("migration count = %d, want 24", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
}
|
||||
if migrations[1].Version != 2 || !strings.Contains(migrations[1].UpSQL, "CREATE TABLE IF NOT EXISTS source_items") {
|
||||
t.Fatalf("migration 2 = %#v", migrations[1])
|
||||
}
|
||||
if migrations[2].Version != 3 || !strings.Contains(migrations[2].UpSQL, "CREATE TABLE IF NOT EXISTS acceptance_results") {
|
||||
t.Fatalf("migration 3 = %#v", migrations[2])
|
||||
}
|
||||
if migrations[3].Version != 4 || !strings.Contains(migrations[3].UpSQL, "source_connections_source_type_check") {
|
||||
t.Fatalf("migration 4 = %#v", migrations[3])
|
||||
}
|
||||
if migrations[4].Version != 5 || !strings.Contains(migrations[4].UpSQL, "CREATE TABLE IF NOT EXISTS source_scans") {
|
||||
t.Fatalf("migration 5 = %#v", migrations[4])
|
||||
}
|
||||
if migrations[5].Version != 6 || !strings.Contains(migrations[5].UpSQL, "CREATE TABLE IF NOT EXISTS publication_reviews") {
|
||||
t.Fatalf("migration 6 = %#v", migrations[5])
|
||||
}
|
||||
if migrations[7].Version != 8 || !strings.Contains(migrations[7].UpSQL, "CREATE TABLE IF NOT EXISTS directory_scope_rules") {
|
||||
t.Fatalf("migration 8 = %#v", migrations[7])
|
||||
}
|
||||
if migrations[8].Version != 9 || !strings.Contains(migrations[8].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_snapshots") {
|
||||
t.Fatalf("migration 9 = %#v", migrations[8])
|
||||
}
|
||||
if migrations[9].Version != 10 || !strings.Contains(migrations[9].UpSQL, "Visibility policies used by the source catalog") {
|
||||
t.Fatalf("migration 10 = %#v", migrations[9])
|
||||
}
|
||||
if migrations[10].Version != 11 || !strings.Contains(migrations[10].UpSQL, "CREATE TABLE IF NOT EXISTS processing_confirmations") {
|
||||
t.Fatalf("migration 11 = %#v", migrations[10])
|
||||
}
|
||||
if migrations[13].Version != 14 || !strings.Contains(migrations[13].UpSQL, "ADD COLUMN IF NOT EXISTS started_at") {
|
||||
t.Fatalf("migration 14 = %#v", migrations[13])
|
||||
}
|
||||
if migrations[14].Version != 15 || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_sets") || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_items") {
|
||||
t.Fatalf("migration 15 = %#v", migrations[14])
|
||||
}
|
||||
if migrations[15].Version != 16 || !strings.Contains(migrations[15].UpSQL, "origin") || !strings.Contains(migrations[15].UpSQL, "source_catalog_change_items") {
|
||||
t.Fatalf("migration 16 = %#v", migrations[15])
|
||||
}
|
||||
if migrations[16].Version != 17 || !strings.Contains(migrations[16].UpSQL, "current_batch") || !strings.Contains(migrations[16].UpSQL, "total_batches") {
|
||||
t.Fatalf("migration 17 = %#v", migrations[16])
|
||||
}
|
||||
if migrations[17].Version != 18 || !strings.Contains(migrations[17].UpSQL, "jobs_state_check") || !strings.Contains(migrations[17].UpSQL, "cancelled") {
|
||||
t.Fatalf("migration 18 = %#v", migrations[17])
|
||||
}
|
||||
if migrations[18].Version != 19 || !strings.Contains(migrations[18].UpSQL, "CREATE TABLE IF NOT EXISTS operation_requests") || !strings.Contains(migrations[18].UpSQL, "UNIQUE (operation_type, operation_id)") {
|
||||
t.Fatalf("migration 19 = %#v", migrations[18])
|
||||
}
|
||||
if migrations[19].Version != 20 || !strings.Contains(migrations[19].UpSQL, "content_sha256") || !strings.Contains(migrations[19].UpSQL, "IF NOT EXISTS") {
|
||||
t.Fatalf("migration 20 = %#v", migrations[19])
|
||||
}
|
||||
if migrations[20].Version != 21 || !strings.Contains(migrations[20].UpSQL, "identity_confidence") || !strings.Contains(migrations[20].UpSQL, "reason_code") {
|
||||
t.Fatalf("migration 21 = %#v", migrations[20])
|
||||
}
|
||||
if migrations[21].Version != 22 || !strings.Contains(migrations[21].UpSQL, "compare_catalog_change_set") {
|
||||
t.Fatalf("migration 22 = %#v", migrations[21])
|
||||
}
|
||||
if migrations[22].Version != 23 || !strings.Contains(migrations[22].UpSQL, "move_count") || !strings.Contains(migrations[22].UpSQL, "'applying'") || !strings.Contains(migrations[22].UpSQL, "old_scope") {
|
||||
t.Fatalf("migration 23 = %#v", migrations[22])
|
||||
}
|
||||
if migrations[23].Version != 24 || !strings.Contains(migrations[23].UpSQL, "item_seq") || !strings.Contains(migrations[23].UpSQL, "change_set_id") {
|
||||
t.Fatalf("migration 24 = %#v", migrations[23])
|
||||
}
|
||||
if migrations[24].Version != 25 || !strings.Contains(migrations[24].UpSQL, "business_change_set_id") || !strings.Contains(migrations[24].UpSQL, "jobs_processing_identity") {
|
||||
t.Fatalf("migration 25 = %#v", migrations[24])
|
||||
}
|
||||
if migrations[25].Version != 26 || !strings.Contains(migrations[25].UpSQL, "job_failures") || !strings.Contains(migrations[25].UpSQL, "last_error_code") {
|
||||
t.Fatalf("migration 26 = %#v", migrations[25])
|
||||
}
|
||||
if migrations[26].Version != 27 || !strings.Contains(migrations[26].UpSQL, "worker_heartbeats") {
|
||||
t.Fatalf("migration 27 = %#v", migrations[26])
|
||||
}
|
||||
if migrations[27].Version != 28 || !strings.Contains(migrations[27].UpSQL, "release_change_sets") || !strings.Contains(migrations[27].UpSQL, "manifest_sha256") {
|
||||
t.Fatalf("migration 28 = %#v", migrations[27])
|
||||
}
|
||||
if migrations[28].Version != 29 || !strings.Contains(migrations[28].UpSQL, "rolling_back") || !strings.Contains(migrations[28].UpSQL, "failure_level") {
|
||||
t.Fatalf("migration 29 = %#v", migrations[28])
|
||||
}
|
||||
if migrations[29].Version != 30 || !strings.Contains(migrations[29].UpSQL, "release_change_set_snapshot_links") || !strings.Contains(migrations[29].UpSQL, "release_change_set_catalog_links") {
|
||||
t.Fatalf("migration 30 = %#v", migrations[29])
|
||||
}
|
||||
if migrations[30].Version != 31 || !strings.Contains(migrations[30].UpSQL, "source_connection_revisions") || !strings.Contains(migrations[30].UpSQL, "uq_source_connection_active_revision") || !strings.Contains(migrations[30].UpSQL, "next_refresh_at") {
|
||||
t.Fatalf("migration 31 = %#v", migrations[30])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000011_processing_confirmations.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000011_processing_confirmations.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"add column if not exists target_scope",
|
||||
"update test_publications as publication",
|
||||
"review.decided_visibility",
|
||||
"source.visibility",
|
||||
"check (target_scope in ('internal', 'external'))",
|
||||
"create table if not exists processing_confirmations",
|
||||
"references test_publications(id) on delete restrict",
|
||||
"operator_ref",
|
||||
"reason",
|
||||
"unique (publication_id)",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("processing confirmation migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
if !strings.Contains(downSQL, "drop table if exists processing_confirmations") || !strings.Contains(downSQL, "drop column if exists target_scope") {
|
||||
t.Fatal("processing confirmation down migration must remove only migration 11 objects")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionedUISystemSettingsMigration(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000013_ui_system_settings.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000013_ui_system_settings.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{"create table if not exists ui_setting_revisions", "'validated', 'active', 'retired', 'failed'", "create unique index", "where state = 'active'", "insert into ui_setting_revisions"} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Fatalf("UI settings migration missing %q", required)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(down)), "drop table if exists ui_setting_revisions") {
|
||||
t.Fatal("UI settings rollback must remove only its revision table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseActionMigrationSeparatesUpsertAndDelete(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000012_release_actions.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000012_release_actions.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"alter table release_batch_items", "alter table release_item_executions",
|
||||
"add column if not exists action", "check (action in ('upsert', 'delete'))",
|
||||
"drop constraint if exists release_batch_items_publication_id_key",
|
||||
"drop constraint if exists release_batch_items_review_id_key",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Fatalf("release action migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
if !strings.Contains(downSQL, "drop column if exists action") {
|
||||
t.Fatal("release action rollback must remove action columns")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogPolicyMigrationAllowsBothAndBlockedWithoutChangingPublicationReviewScope(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000010_catalog_policies.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000010_catalog_policies.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"visibility policies used by the source catalog",
|
||||
"directory_scope_rules_visibility_check",
|
||||
"source_item_scope_overrides_visibility_check",
|
||||
"scope_change_events_previous_visibility_check",
|
||||
"scope_change_events_decided_visibility_check",
|
||||
"'internal', 'external', 'both', 'blocked'",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("catalog policy migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
if !strings.Contains(downSQL, "update source_item_scope_overrides") || !strings.Contains(downSQL, "update directory_scope_rules") {
|
||||
t.Fatalf("catalog policy down migration must normalize extended policy values before restoring constraints")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceCatalogMigrationDefinesImmutableHierarchicalSnapshots(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000009_source_catalog.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000009_source_catalog.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"create table if not exists source_catalog_snapshots",
|
||||
"create table if not exists source_catalog_nodes",
|
||||
"create table if not exists source_catalog_heads",
|
||||
"check (node_type in ('folder', 'file'))",
|
||||
"primary key (snapshot_id, node_id)",
|
||||
"foreign key (snapshot_id, parent_node_id)",
|
||||
"references source_items(id) on delete restrict",
|
||||
"create index if not exists idx_source_catalog_nodes_parent",
|
||||
"unique (source_connection_id, snapshot_sha256)",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("source catalog migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
ordered := []string{"drop table if exists source_catalog_heads", "drop table if exists source_catalog_nodes", "drop table if exists source_catalog_snapshots"}
|
||||
previous := -1
|
||||
for _, statement := range ordered {
|
||||
position := strings.Index(downSQL, statement)
|
||||
if position < 0 || position <= previous {
|
||||
t.Fatalf("source catalog down order invalid for %q", statement)
|
||||
}
|
||||
previous = position
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopeRulesMigrationDefinesAuditedDirectoryAndFileOverrides(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000008_scope_rules.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000008_scope_rules.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"create table if not exists directory_scope_rules",
|
||||
"create table if not exists source_item_scope_overrides",
|
||||
"create table if not exists scope_change_events",
|
||||
"primary key (source_connection_id, directory_path)",
|
||||
"primary key (source_item_id)",
|
||||
"check (visibility in ('internal', 'external'))",
|
||||
"check (target_type in ('directory', 'item'))",
|
||||
"references source_connections(id) on delete restrict",
|
||||
"references source_items(id) on delete restrict",
|
||||
"create index if not exists idx_scope_change_events_target",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("scope migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
for _, required := range []string{
|
||||
"drop table if exists scope_change_events",
|
||||
"drop table if exists source_item_scope_overrides",
|
||||
"drop table if exists directory_scope_rules",
|
||||
} {
|
||||
if !strings.Contains(downSQL, required) {
|
||||
t.Errorf("scope down migration missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobsMigrationDefinesDurableLeaseQueue(t *testing.T) {
|
||||
body, err := os.ReadFile("migrations/000001_jobs.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read jobs migration: %v", err)
|
||||
}
|
||||
sql := strings.ToLower(string(body))
|
||||
for _, required := range []string{
|
||||
"create table if not exists schema_migrations",
|
||||
"create table if not exists jobs",
|
||||
"idempotency_key",
|
||||
"lease_owner",
|
||||
"lease_until",
|
||||
"available_at",
|
||||
"max_attempts",
|
||||
"check (state in",
|
||||
"unique (idempotency_key)",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("migration missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceGovernanceMigrationDefinesIdentityAndStateContracts(t *testing.T) {
|
||||
body, err := os.ReadFile("migrations/000002_source_governance.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read source governance migration: %v", err)
|
||||
}
|
||||
sql := strings.ToLower(string(body))
|
||||
for _, required := range []string{
|
||||
"create table if not exists source_connections",
|
||||
"create table if not exists source_items",
|
||||
"create table if not exists content_objects",
|
||||
"create table if not exists source_item_versions",
|
||||
"create table if not exists test_publications",
|
||||
"unique (source_connection_id, stable_external_id)",
|
||||
"unique (sha256)",
|
||||
"unique (source_item_id, revision)",
|
||||
"unique (source_item_version_id, test_kb_id)",
|
||||
"check (source_type in ('pp', 'alist', 'dingtalk'))",
|
||||
"check (sha256 ~ '^[0-9a-f]{64}$')",
|
||||
"check (source_change in ('unchanged', 'added', 'updated', 'deleted', 'unavailable'))",
|
||||
"check (content_state in ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed'))",
|
||||
"check (ingest_state in ('pending', 'uploading', 'uploaded', 'failed'))",
|
||||
"check (processing_state in ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'))",
|
||||
"check (retrieval_state in ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed'))",
|
||||
"references source_connections(id)",
|
||||
"references source_items(id)",
|
||||
"references content_objects(id)",
|
||||
"references source_item_versions(id)",
|
||||
"raw_record_summary jsonb",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("source governance migration missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceGovernanceDownMigrationDropsTablesInDependencyOrder(t *testing.T) {
|
||||
body, err := os.ReadFile("migrations/000002_source_governance.down.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read source governance down migration: %v", err)
|
||||
}
|
||||
sql := strings.ToLower(string(body))
|
||||
ordered := []string{
|
||||
"drop table if exists test_publications",
|
||||
"drop table if exists source_item_versions",
|
||||
"drop table if exists content_objects",
|
||||
"drop table if exists source_items",
|
||||
"drop table if exists source_connections",
|
||||
}
|
||||
previous := -1
|
||||
for _, statement := range ordered {
|
||||
position := strings.Index(sql, statement)
|
||||
if position < 0 {
|
||||
t.Fatalf("down migration missing %q", statement)
|
||||
}
|
||||
if position <= previous {
|
||||
t.Fatalf("%q is not in reverse dependency order", statement)
|
||||
}
|
||||
previous = position
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestOrchestrationMigrationDefinesPollingAndAcceptanceContracts(t *testing.T) {
|
||||
body, err := os.ReadFile("migrations/000003_test_orchestration.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read test orchestration migration: %v", err)
|
||||
}
|
||||
sql := strings.ToLower(string(body))
|
||||
for _, required := range []string{
|
||||
"alter table test_publications add column if not exists poll_sequence",
|
||||
"alter table test_publications add column if not exists last_remote_status",
|
||||
"alter table test_publications add column if not exists last_observed_at",
|
||||
"alter table test_publications add column if not exists supersedes_publication_id",
|
||||
"references test_publications(id) on delete restrict",
|
||||
"create table if not exists acceptance_results",
|
||||
"references test_publications(id) on delete restrict",
|
||||
"check (check_type in ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches'))",
|
||||
"check (state in ('pending', 'passed', 'failed', 'manual_required'))",
|
||||
"check (jsonb_typeof(evidence_summary) = 'object')",
|
||||
"unique (publication_id, run_id, check_type)",
|
||||
"create index if not exists idx_test_publications_poll",
|
||||
"create index if not exists idx_acceptance_results_publication",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("test orchestration migration missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestOrchestrationDownMigrationRemovesDependentObjectsFirst(t *testing.T) {
|
||||
body, err := os.ReadFile("migrations/000003_test_orchestration.down.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read test orchestration down migration: %v", err)
|
||||
}
|
||||
sql := strings.ToLower(string(body))
|
||||
acceptance := strings.Index(sql, "drop table if exists acceptance_results")
|
||||
publicationColumns := strings.Index(sql, "alter table test_publications")
|
||||
if acceptance < 0 || publicationColumns < 0 {
|
||||
t.Fatalf("down migration missing dependent object removal: %s", sql)
|
||||
}
|
||||
if acceptance > publicationColumns {
|
||||
t.Fatal("acceptance_results must be dropped before test_publications columns")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualSourceMigrationExpandsWithoutRewritingAndDowngradesFailClosed(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000004_manual_source.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read manual source up migration: %v", err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000004_manual_source.down.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read manual source down migration: %v", err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"drop constraint if exists source_connections_source_type_check",
|
||||
"check (source_type in ('pp', 'alist', 'dingtalk', 'manual'))",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("manual source up migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
for _, required := range []string{
|
||||
"if exists",
|
||||
"source_type = 'manual'",
|
||||
"raise exception",
|
||||
"check (source_type in ('pp', 'alist', 'dingtalk'))",
|
||||
} {
|
||||
if !strings.Contains(downSQL, required) {
|
||||
t.Errorf("manual source down migration missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceScanMigrationDefinesAuditableSingleActiveScan(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000005_source_scans.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read source scans up migration: %v", err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000005_source_scans.down.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read source scans down migration: %v", err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"create table if not exists source_scans",
|
||||
"create table if not exists source_scan_changes",
|
||||
"references source_connections(id) on delete restrict",
|
||||
"references source_scans(id) on delete restrict",
|
||||
"check (state in ('pending', 'running', 'completed', 'failed'))",
|
||||
"check (change_type in ('added', 'updated', 'unavailable'))",
|
||||
"check (snapshot_sha256 = '' or snapshot_sha256 ~ '^[0-9a-f]{64}$')",
|
||||
"check (observed_items >= 0)",
|
||||
"check (unchanged_items >= 0)",
|
||||
"check (added_items >= 0)",
|
||||
"check (updated_items >= 0)",
|
||||
"check (over_limit_items >= 0)",
|
||||
"check (missing_items >= 0)",
|
||||
"check (planned_uploads >= 0)",
|
||||
"create unique index if not exists idx_source_scans_one_active",
|
||||
"where state in ('pending', 'running')",
|
||||
"unique (source_scan_id, stable_external_id)",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("source scans up migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
changes := strings.Index(downSQL, "drop table if exists source_scan_changes")
|
||||
scans := strings.Index(downSQL, "drop table if exists source_scans")
|
||||
if changes < 0 || scans < 0 || changes > scans {
|
||||
t.Fatal("source scans down migration must drop changes before scans")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewAndSealMigrationDefinesImmutableHumanGate(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000006_review_and_seal.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read review and seal up migration: %v", err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000006_review_and_seal.down.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read review and seal down migration: %v", err)
|
||||
}
|
||||
upSQL := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"create table if not exists publication_reviews",
|
||||
"references test_publications(id) on delete restrict",
|
||||
"check (decision in ('approved', 'rejected'))",
|
||||
"check (decided_visibility in ('internal', 'external'))",
|
||||
"acceptance_run_id",
|
||||
"publication_updated_at",
|
||||
"create table if not exists release_batches",
|
||||
"check (state in ('draft', 'sealed', 'cancelled'))",
|
||||
"policy_version",
|
||||
"create table if not exists release_batch_items",
|
||||
"references publication_reviews(id) on delete restrict",
|
||||
"unique (publication_id)",
|
||||
"decided_visibility",
|
||||
} {
|
||||
if !strings.Contains(upSQL, required) {
|
||||
t.Errorf("review and seal migration missing %q", required)
|
||||
}
|
||||
}
|
||||
downSQL := strings.ToLower(string(down))
|
||||
ordered := []string{
|
||||
"drop table if exists release_batch_items",
|
||||
"drop table if exists release_batches",
|
||||
"drop table if exists publication_reviews",
|
||||
}
|
||||
previous := -1
|
||||
for _, statement := range ordered {
|
||||
position := strings.Index(downSQL, statement)
|
||||
if position < 0 || position <= previous {
|
||||
t.Fatalf("review and seal down migration has invalid dependency order for %q", statement)
|
||||
}
|
||||
previous = position
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
|
||||
up, err := os.ReadFile("migrations/000007_release_executions.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
down, err := os.ReadFile("migrations/000007_release_executions.down.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.ToLower(string(up))
|
||||
for _, required := range []string{
|
||||
"create table if not exists release_executions",
|
||||
"operation in ('publish', 'rollback')",
|
||||
"state in ('pending', 'running', 'completed', 'failed')",
|
||||
"target_environment = 'production'",
|
||||
"references release_batches(id) on delete restrict",
|
||||
"references jobs(id) on delete restrict",
|
||||
"create unique index if not exists uq_release_publish_batch",
|
||||
"create table if not exists release_item_executions",
|
||||
"bundle_sha256", "previous_production_revision", "production_revision", "error_code",
|
||||
} {
|
||||
if !strings.Contains(text, required) {
|
||||
t.Fatalf("release execution migration missing %q", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, "api_key") || strings.Contains(text, "password") || strings.Contains(text, "secret") {
|
||||
t.Fatal("release execution migration must not persist production credentials")
|
||||
}
|
||||
downText := strings.ToLower(string(down))
|
||||
items := strings.Index(downText, "drop table if exists release_item_executions")
|
||||
executions := strings.Index(downText, "drop table if exists release_executions")
|
||||
if items < 0 || executions < 0 || items > executions {
|
||||
t.Fatal("release execution down migration must drop item executions before executions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
|
||||
sql := strings.ToLower(claimJobSQL)
|
||||
for _, required := range []string{
|
||||
"for update skip locked",
|
||||
"state in ('pending', 'retry')",
|
||||
"state = 'running'",
|
||||
"lease_until <",
|
||||
"attempt = j.attempt + 1",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("claim SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
required []string
|
||||
}{
|
||||
{
|
||||
name: "enqueue",
|
||||
sql: enqueueJobSQL,
|
||||
required: []string{
|
||||
"on conflict (idempotency_key)",
|
||||
"do update set idempotency_key = excluded.idempotency_key",
|
||||
"returning id",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "complete",
|
||||
sql: completeJobSQL,
|
||||
required: []string{
|
||||
"state = 'running'",
|
||||
"lease_owner = $2",
|
||||
"state = 'completed'",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "heartbeat",
|
||||
sql: heartbeatJobSQL,
|
||||
required: []string{
|
||||
"state = 'running'",
|
||||
"lease_owner = $2",
|
||||
"lease_until = $4",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fail",
|
||||
sql: failJobSQL,
|
||||
required: []string{
|
||||
"case when attempt >= max_attempts then 'failed'",
|
||||
"else 'retry'",
|
||||
"lease_owner = null",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sql := strings.ToLower(tt.sql)
|
||||
for _, required := range tt.required {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
var ErrOperationRequestConflict = errors.New("operation request conflicts with an existing request")
|
||||
|
||||
type operationExecutor func(pgx.Tx) (json.RawMessage, error)
|
||||
|
||||
func (s *Store) RouteCatalogFilesWithOperation(ctx context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) {
|
||||
if err := validateCatalogRouteRequest(request); err != nil {
|
||||
return governance.CatalogRouteResult{}, false, err
|
||||
}
|
||||
fingerprint, err := catalogRouteFingerprint(request)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, false, err
|
||||
}
|
||||
resultJSON, replayed, err := s.executeOperation(ctx, "catalog.route", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := routeCatalogFilesTx(ctx, tx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoded, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode catalog route result: %w", err)
|
||||
}
|
||||
return encoded, nil
|
||||
})
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, false, err
|
||||
}
|
||||
var result governance.CatalogRouteResult
|
||||
if err := json.Unmarshal(resultJSON, &result); err != nil {
|
||||
return governance.CatalogRouteResult{}, false, fmt.Errorf("decode stored catalog route result: %w", err)
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func catalogRouteFingerprint(request governance.CatalogRouteRequest) (string, error) {
|
||||
targets := request.TargetKnowledgeBaseIDs
|
||||
if len(targets) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targets = []string{request.TargetKnowledgeBaseID}
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
NodeIDs []string `json:"node_ids"`
|
||||
Visibility governance.Visibility `json:"visibility"`
|
||||
Targets []string `json:"target_knowledge_base_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
}{
|
||||
ConnectionID: request.ConnectionID,
|
||||
NodeIDs: request.NodeIDs,
|
||||
Visibility: request.Visibility,
|
||||
Targets: targets,
|
||||
OperatorRef: strings.TrimSpace(request.OperatorRef),
|
||||
Reason: strings.TrimSpace(request.Reason),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode catalog route fingerprint: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func (s *Store) EnqueueCatalogRefreshWithOperation(ctx context.Context, source governance.CatalogSource, operationID string, now time.Time) (jobs.Job, bool, error) {
|
||||
if !validRouteID(source.ConnectionID) || strings.TrimSpace(source.Name) == "" || now.IsZero() {
|
||||
return jobs.Job{}, false, errors.New("invalid catalog refresh request")
|
||||
}
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
}{ConnectionID: source.ConnectionID})
|
||||
if err != nil {
|
||||
return jobs.Job{}, false, err
|
||||
}
|
||||
resultJSON, replayed, err := s.executeOperation(ctx, "catalog.refresh", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"connection_id": source.ConnectionID, "source_type": source.SourceType, "source_name": source.Name, "stage": "queued", "progress": 0, "message": "等待后台刷新", "processed": 0, "total": 0})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode catalog refresh job: %w", err)
|
||||
}
|
||||
job, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, jobID, "source.catalog.refresh", payload, 20, now, 3, "catalog-refresh:"+jobID, now))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("enqueue catalog refresh job: %w", err)
|
||||
}
|
||||
encoded, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode catalog refresh result: %w", err)
|
||||
}
|
||||
return encoded, nil
|
||||
})
|
||||
if err != nil {
|
||||
return jobs.Job{}, false, err
|
||||
}
|
||||
var result jobs.Job
|
||||
if err := json.Unmarshal(resultJSON, &result); err != nil {
|
||||
return jobs.Job{}, false, fmt.Errorf("decode stored catalog refresh result: %w", err)
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func operationFingerprint(value any) (string, error) {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode operation fingerprint: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func (s *Store) executeOperation(ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute operationExecutor) (json.RawMessage, bool, error) {
|
||||
if strings.TrimSpace(operationType) == "" || len(operationType) > 100 || strings.TrimSpace(operationID) == "" || len(operationID) > 128 || len(fingerprint) != 64 || now.IsZero() || execute == nil {
|
||||
return nil, false, errors.New("invalid operation request")
|
||||
}
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return nil, false, errors.New("operation request transactions are unavailable")
|
||||
}
|
||||
tx, err := beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("begin operation request: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO operation_requests(operation_type,operation_id,request_fingerprint_sha256,state,result,created_at,completed_at)
|
||||
VALUES($1,$2,$3,'completed','{}'::jsonb,$4,$4)
|
||||
ON CONFLICT(operation_type,operation_id) DO NOTHING`, operationType, operationID, fingerprint, now)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("create operation request: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
var storedFingerprint string
|
||||
var storedResult json.RawMessage
|
||||
if err := tx.QueryRow(ctx, `SELECT request_fingerprint_sha256,result
|
||||
FROM operation_requests
|
||||
WHERE operation_type=$1 AND operation_id=$2
|
||||
FOR UPDATE`, operationType, operationID).Scan(&storedFingerprint, &storedResult); err != nil {
|
||||
return nil, false, fmt.Errorf("load operation request: %w", err)
|
||||
}
|
||||
if storedFingerprint != fingerprint {
|
||||
return nil, false, ErrOperationRequestConflict
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, false, fmt.Errorf("commit replay operation request: %w", err)
|
||||
}
|
||||
return storedResult, true, nil
|
||||
}
|
||||
result, err := execute(tx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !json.Valid(result) {
|
||||
return nil, false, errors.New("operation result must be valid JSON")
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE operation_requests SET result=$3::jsonb,completed_at=$4
|
||||
WHERE operation_type=$1 AND operation_id=$2`, operationType, operationID, result, now); err != nil {
|
||||
return nil, false, fmt.Errorf("complete operation request: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, false, fmt.Errorf("commit operation request: %w", err)
|
||||
}
|
||||
return result, false, nil
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func runOperation[T any](s *Store, ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute func(pgx.Tx) (T, error)) (T, bool, error) {
|
||||
var zero T
|
||||
raw, replayed, err := s.executeOperation(ctx, operationType, operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := execute(tx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
var result T
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
|
||||
func (s *Store) TriggerPPSourceScanWithOperation(ctx context.Context, connectionID string, now time.Time, operationID string) (governance.SourceScan, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
}{connectionID})
|
||||
if err != nil {
|
||||
return governance.SourceScan{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "source.scan.trigger", operationID, fingerprint, now, func(tx pgx.Tx) (governance.SourceScan, error) {
|
||||
return NewStore(tx).TriggerPPSourceScan(ctx, connectionID, now)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) CreateUISettingsCandidateWithOperation(ctx context.Context, request governance.CreateUISettingsCandidateRequest, operationID string) (governance.UISettings, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
Document int `json:"document_refresh_seconds"`
|
||||
Resource int `json:"resource_refresh_seconds"`
|
||||
Stalled int `json:"stalled_after_seconds"`
|
||||
Finalize int `json:"finalizing_after_seconds"`
|
||||
By string `json:"updated_by"`
|
||||
}{request.DocumentRefreshSeconds, request.ResourceRefreshSeconds, request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy})
|
||||
if err != nil {
|
||||
return governance.UISettings{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "ui_settings.candidate.create", operationID, fingerprint, request.Now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) {
|
||||
return NewStore(tx).CreateUISettingsCandidate(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) ActivateUISettingsWithOperation(ctx context.Context, revisionID string, now time.Time, operationID string) (governance.UISettings, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
RevisionID string `json:"revision_id"`
|
||||
}{revisionID})
|
||||
if err != nil {
|
||||
return governance.UISettings{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "ui_settings.activate", operationID, fingerprint, now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) {
|
||||
return NewStore(tx).ActivateUISettings(ctx, revisionID, now)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) CreateReleaseBatchWithOperation(ctx context.Context, request governance.CreateReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
Name string `json:"name"`
|
||||
By string `json:"created_by"`
|
||||
Upsert []string `json:"publication_ids"`
|
||||
Delete []string `json:"delete_publication_ids"`
|
||||
Set string `json:"release_change_set_id"`
|
||||
}{request.Name, request.CreatedBy, request.PublicationIDs, request.DeletePublicationIDs, request.ReleaseChangeSetID})
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.batch.create", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) {
|
||||
return NewStore(tx).CreateReleaseBatch(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) GenerateReleaseChangeSetWithOperation(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
Name string `json:"name"`
|
||||
By string `json:"created_by"`
|
||||
}{request.Name, request.CreatedBy})
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.changeset.generate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) {
|
||||
return NewStore(tx).GenerateReleaseChangeSet(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) ConfirmReleaseChangeSetWithOperation(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ID string `json:"change_set_id"`
|
||||
By string `json:"confirmed_by"`
|
||||
}{request.ChangeSetID, request.ConfirmedBy})
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) {
|
||||
return NewStore(tx).ConfirmReleaseChangeSet(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) SealReleaseBatchWithOperation(ctx context.Context, request governance.SealReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
BatchID string `json:"batch_id"`
|
||||
SealedBy string `json:"sealed_by"`
|
||||
}{request.BatchID, request.SealedBy})
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.batch.seal", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) {
|
||||
return NewStore(tx).SealReleaseBatch(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) AuthorizeReleaseBatchWithOperation(ctx context.Context, request governance.AuthorizeReleaseBatchRequest, operationID string) (governance.ReleaseExecution, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
BatchID string `json:"batch_id"`
|
||||
AuthorizedBy string `json:"authorized_by"`
|
||||
}{request.BatchID, request.AuthorizedBy})
|
||||
if err != nil {
|
||||
return governance.ReleaseExecution{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.authorize.publish", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
|
||||
return NewStore(tx).AuthorizeReleaseBatch(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) AuthorizeReleaseRollbackWithOperation(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest, operationID string) (governance.ReleaseExecution, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
PriorID string `json:"prior_execution_id"`
|
||||
AuthorizedBy string `json:"authorized_by"`
|
||||
}{request.PriorExecutionID, request.AuthorizedBy})
|
||||
if err != nil {
|
||||
return governance.ReleaseExecution{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.authorize.rollback", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
|
||||
return NewStore(tx).AuthorizeReleaseRollback(ctx, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) RetryReleaseExecutionWithOperation(ctx context.Context, request governance.RetryReleaseExecutionRequest, operationID string) (governance.ReleaseExecution, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
ExecutionID string `json:"execution_id"`
|
||||
}{request.ExecutionID})
|
||||
if err != nil {
|
||||
return governance.ReleaseExecution{}, false, err
|
||||
}
|
||||
return runOperation(s, ctx, "release.execution.retry", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
|
||||
return NewStore(tx).RetryReleaseExecution(ctx, request)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestP2StableIdentityAndProvenMove freezes docs plan P2:
|
||||
// identity_confidence is never downgraded from stable_id; the canonical
|
||||
// compare_catalog_change_set function classifies a 1:1 content-hash relocation
|
||||
// as proven move evidence, and never merges ambiguous copies.
|
||||
func TestP2StableIdentityAndProvenMove(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := strings.Repeat("c", 64)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id','pp','ID PP',true);
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snap-old','conn-id',repeat('1',64),'ID PP',2,1,1,'2026-09-11T00:00:00Z'),
|
||||
('snap-new','conn-id',repeat('2',64),'ID PP',2,1,1,'2026-09-12T00:00:00Z'),
|
||||
('snap-copy','conn-id',repeat('3',64),'ID PP',3,1,2,'2026-09-13T00:00:00Z');
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes,content_sha256)
|
||||
VALUES
|
||||
('snap-old','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL),
|
||||
('snap-old','f-old','root','file','手册.pdf','旧目录/手册.pdf',repeat('5',64),10,0,1,10,repeat('c',64)),
|
||||
('snap-new','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL),
|
||||
('snap-new','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)),
|
||||
('snap-copy','root',NULL,'folder','ID PP','',repeat('4',64),NULL,2,2,20,NULL),
|
||||
('snap-copy','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)),
|
||||
('snap-copy','f-copy','root','file','副本.pdf','新目录/副本.pdf',repeat('7',64),10,0,1,10,repeat('c',64));
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
|
||||
VALUES('cs-move','conn-id','snap-old','snap-new',NOW()),
|
||||
('cs-copy','conn-id','snap-new','snap-copy',NOW());
|
||||
SELECT compare_catalog_change_set('cs-move');
|
||||
SELECT compare_catalog_change_set('cs-copy');`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Proven move: the disappeared old locator is a delete carrying move evidence,
|
||||
// and the reappearing new locator is an add carrying the same content hash.
|
||||
var deleteReason, deleteOldHash string
|
||||
if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(old_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='delete'`).Scan(&deleteReason, &deleteOldHash); err != nil || deleteReason != "content_hash_move" || deleteOldHash != hash {
|
||||
t.Fatalf("cs-move delete=%q hash=%q err=%v", deleteReason, deleteOldHash, err)
|
||||
}
|
||||
var addReason, addNewHash string
|
||||
if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(new_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='add'`).Scan(&addReason, &addNewHash); err != nil || addReason != "content_hash_move" || addNewHash != hash {
|
||||
t.Fatalf("cs-move add=%q hash=%q err=%v", addReason, addNewHash, err)
|
||||
}
|
||||
// Counts were recalculated by the function.
|
||||
var addCount, deleteCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT add_count,delete_count FROM source_catalog_change_sets WHERE id='cs-move'`).Scan(&addCount, &deleteCount); err != nil || addCount != 1 || deleteCount != 1 {
|
||||
t.Fatalf("cs-move counts add=%d delete=%d err=%v", addCount, deleteCount, err)
|
||||
}
|
||||
|
||||
// Ambiguous copy: hash appears twice on the new side → NOT a proven move;
|
||||
// the copy is a plain add and the original stays untouched.
|
||||
var copyReason string
|
||||
if err := pool.QueryRow(ctx, `SELECT reason_code FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='add' AND new_path LIKE '%副本%'`).Scan(©Reason); err != nil || copyReason != "locator" {
|
||||
t.Fatalf("ambiguous copy reason=%q err=%v", copyReason, err)
|
||||
}
|
||||
var copyDeletes int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='delete'`).Scan(©Deletes); err != nil || copyDeletes != 0 {
|
||||
t.Fatalf("ambiguous copy must not delete the original: %d err=%v", copyDeletes, err)
|
||||
}
|
||||
|
||||
// identity_confidence upsert: stable_id never downgrades; path can rise.
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id2','pp','ID PP2',true) ON CONFLICT DO NOTHING;
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,identity_confidence)
|
||||
VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'稳.pdf','稳.pdf','document','v1','internal','pp_public_tag','stable_id') ON CONFLICT DO NOTHING;`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,source_change,identity_confidence)
|
||||
VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'','稳.pdf','document','v1','internal','manual','added','content_hash')
|
||||
ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE
|
||||
SET updated_at = source_items.updated_at,
|
||||
identity_confidence = CASE WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence ELSE EXCLUDED.identity_confidence END;`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var confidence string
|
||||
if err := pool.QueryRow(ctx, `SELECT identity_confidence FROM source_items WHERE id='item-stable'`).Scan(&confidence); err != nil || confidence != "stable_id" {
|
||||
t.Fatalf("stable_id downgraded to %q err=%v", confidence, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func seedP3Fixture(t *testing.T, pool *pgxpool.Pool) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
_, seedErr := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p3','alist','P3 来源',true);
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES
|
||||
('co-p3-a',repeat('a',64),10,'verified/a.pdf','verified'),
|
||||
('co-p3-b',repeat('b',64),12,'verified/b.pdf','verified'),
|
||||
('co-p3-c',repeat('c',64),20,'verified/c.pdf','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-a','conn-p3','ext-a',repeat('d',64),'新增.pdf','document','r1','internal','manual'),
|
||||
('item-b','conn-p3','ext-b',repeat('e',64),'更新.pdf','document','r2','internal','manual'),
|
||||
('item-c','conn-p3','ext-c',repeat('f',64),'删除.pdf','document','r3','internal','manual'),
|
||||
('item-d','conn-p3','ext-d',repeat('9',64),'未变.pdf','document','r1','internal','manual'),
|
||||
('item-e','conn-p3','ext-e',repeat('8',64),'内部专属.pdf','document','r1','internal','manual');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES
|
||||
('version-a','item-a','r1','co-p3-a','verified'),
|
||||
('version-b','item-b','r2','co-p3-b','verified'),
|
||||
('version-c','item-c','r3','co-p3-c','verified'),
|
||||
('version-d','item-d','r1','co-p3-a','verified'),
|
||||
('version-e','item-e','r1','co-p3-a','verified');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('p3-old','conn-p3',repeat('1',64),'P3 来源',1,1,0,'2026-09-11T00:00:00Z'),
|
||||
('p3-new','conn-p3',repeat('2',64),'P3 来源',6,1,5,'2026-09-12T00:00:00Z');
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,content_sha256)
|
||||
VALUES
|
||||
('p3-new','root',NULL,'folder','P3 来源','',repeat('3',64),NULL,NULL,NULL),
|
||||
('p3-new','n-a','root','file','新增.pdf','新增.pdf',repeat('a',64),'item-a',10,repeat('a',64)),
|
||||
('p3-new','n-b','root','file','更新.pdf','更新.pdf',repeat('b',64),'item-b',12,repeat('b',64)),
|
||||
('p3-new','n-d','root','file','未变.pdf','未变.pdf',repeat('c',64),'item-d',10,repeat('a',64)),
|
||||
('p3-new','n-e','root','file','内部专属.pdf','内部专属.pdf',repeat('e',64),'item-e',10,repeat('a',64));
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
|
||||
VALUES('tp-p3-c','version-c','kb-internal','internal','uploaded','completed','enabled','kn-c',NOW(),NOW());
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
|
||||
VALUES('cs-p3','conn-p3','p3-old','p3-new',NOW());
|
||||
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,new_scope,new_source_item_id,old_source_item_id,created_at)
|
||||
VALUES
|
||||
('cs-p3',repeat('a',64),'add','新增.pdf','','新增.pdf','internal','item-a',NULL,NOW()),
|
||||
('cs-p3',repeat('b',64),'update','更新.pdf','更新.pdf','更新.pdf','internal','item-b',NULL,NOW()),
|
||||
('cs-p3',repeat('c',64),'delete','删除.pdf','删除.pdf','','',NULL,'item-c',NOW()),
|
||||
('cs-p3',repeat('d',64),'no_change','未变.pdf','未变.pdf','未变.pdf','internal','item-d',NULL,NOW()),
|
||||
('cs-p3',repeat('e',64),'add','内部专属.pdf','','内部专属.pdf','blocked','item-e',NULL,NOW());`)
|
||||
if seedErr != nil {
|
||||
t.Fatal(seedErr)
|
||||
}
|
||||
return "cs-p3"
|
||||
}
|
||||
|
||||
func p3Seqs(t *testing.T, pool *pgxpool.Pool, changeSetID string) []int64 {
|
||||
t.Helper()
|
||||
rows, err := pool.Query(context.Background(), `SELECT item_seq FROM source_catalog_change_items WHERE change_set_id=$1 ORDER BY item_seq`, changeSetID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var seqs []int64
|
||||
for rows.Next() {
|
||||
var seq int64
|
||||
if err := rows.Scan(&seq); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seqs = append(seqs, seq)
|
||||
}
|
||||
if len(seqs) != 5 {
|
||||
t.Fatalf("expected 5 change items, got %d", len(seqs))
|
||||
}
|
||||
return seqs
|
||||
}
|
||||
|
||||
func TestP3ConfirmExecutesSelectedChangeItems(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := seedP3Fixture(t, pool)
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 12, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
// Jobs are zero before confirmation.
|
||||
var jobsBefore int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobsBefore); err != nil || jobsBefore != 0 {
|
||||
t.Fatalf("jobs before=%d err=%v", jobsBefore, err)
|
||||
}
|
||||
|
||||
// Partial selection: add, update, delete, no_change, blocked-delete.
|
||||
result, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
|
||||
ChangeSetID: id, ItemSeqs: p3Seqs(t, pool, "cs-p3"),
|
||||
OperatorRef: "engineer-1", Reason: "确认P3差异执行", Now: now,
|
||||
InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.ConfirmedItems != 4 || result.UploadsCreated != 2 || result.RemovalsRequested != 1 || result.NoChangeIgnored != 1 || result.SkippedNoArtifact != 0 || result.SkippedNoTarget != 0 {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
// blocked add became a removal request too (desired state blocked → disable path)
|
||||
var uploads, removals int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if uploads != 2 || removals != 1 {
|
||||
t.Fatalf("uploads=%d removals=%d", uploads, removals)
|
||||
}
|
||||
// Upload payloads carry the ChangeSet and item identity (P5 will make it a constraint).
|
||||
var payloadOK bool
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*)=2 FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'='cs-p3'`).Scan(&payloadOK); err != nil || !payloadOK {
|
||||
t.Fatalf("upload provenance missing ok=%v err=%v", payloadOK, err)
|
||||
}
|
||||
// Publications record their change provenance.
|
||||
var provenance int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE change_set_id='cs-p3'`).Scan(&provenance); err != nil || provenance != 2 {
|
||||
t.Fatalf("publication provenance=%d err=%v", provenance, err)
|
||||
}
|
||||
// Confirmations bind to the change set.
|
||||
var confirmations int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM processing_confirmations WHERE change_set_id='cs-p3'`).Scan(&confirmations); err != nil || confirmations != 2 {
|
||||
t.Fatalf("confirmations=%d err=%v", confirmations, err)
|
||||
}
|
||||
// Lifecycle moved to confirmed.
|
||||
var state string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id='cs-p3'`).Scan(&state); err != nil || state != "confirmed" {
|
||||
t.Fatalf("state=%q err=%v", state, err)
|
||||
}
|
||||
|
||||
// Re-confirming the same items replays stored counts; no new jobs or rows.
|
||||
replaySeqs := p3Seqs(t, pool, "cs-p3")[:1]
|
||||
replay, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
|
||||
ChangeSetID: id, ItemSeqs: replaySeqs,
|
||||
OperatorRef: "engineer-2", Reason: "重复确认", Now: now.Add(time.Minute),
|
||||
InternalTargetKBID: "kb-internal",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if replay.UploadsCreated != 2 || replay.RemovalsRequested != 1 {
|
||||
t.Fatalf("replay=%#v", replay)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil || uploads != 2 {
|
||||
t.Fatalf("replay created uploads=%d err=%v", uploads, err)
|
||||
}
|
||||
|
||||
// Unknown locator belongs to nobody: rejected before any enqueue.
|
||||
if _, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
|
||||
ChangeSetID: id, ItemSeqs: []int64{999999},
|
||||
OperatorRef: "engineer-2", Reason: "越界确认", Now: now, InternalTargetKBID: "kb-internal",
|
||||
}); err == nil {
|
||||
t.Fatal("unknown item selection accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestP3ConcurrentConfirmationsEnqueueOnce(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := seedP3Fixture(t, pool)
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 12, 4, 0, 0, 0, time.UTC)
|
||||
|
||||
const workers = 8
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, workers)
|
||||
for index := 0; index < workers; index++ {
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
_, errs[index] = store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
|
||||
ChangeSetID: id,
|
||||
ItemSeqs: p3Seqs(t, pool, "cs-p3"),
|
||||
OperatorRef: "engineer-1", Reason: "并发确认", Now: now.Add(time.Duration(index) * time.Second),
|
||||
InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external",
|
||||
})
|
||||
}(index)
|
||||
}
|
||||
wg.Wait()
|
||||
for index, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("worker %d err=%v", index, err)
|
||||
}
|
||||
}
|
||||
// Exactly one enqueue per publication regardless of concurrent confirmations
|
||||
// (FOR UPDATE on the set serializes; later workers replay from stored counts).
|
||||
var uploads, removals int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if uploads != 2 || removals != 1 {
|
||||
t.Fatalf("concurrent enqueue: uploads=%d removals=%d", uploads, removals)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestP5ProcessingJobIdentity freezes docs plan P5: the database rejects a
|
||||
// duplicate delivery of the same (change_set, version, target, action), while
|
||||
// a different ChangeSet or action for the same version+target remains legal and
|
||||
// identity-less jobs never collide.
|
||||
func TestP5ProcessingJobIdentity(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p5','alist','P5 来源',true);
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p5',repeat('a',64),10,'verified/a.pdf','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-p5','conn-p5','ext-p5',repeat('b',64),'P5.pdf','document','r1','internal','manual');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-p5','item-p5','r1','co-p5','verified');
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope) VALUES('tp-p5','version-p5','kb-internal','internal');
|
||||
INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES('job-p5-1','test.upload','{}','p5:first','cs-p5-a','version-p5','kb-internal','upload')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Duplicate delivery of the same processing identity must be rejected.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES('job-p5-2','test.upload','{}','p5:replay','cs-p5-a','version-p5','kb-internal','upload')`); err == nil || !strings.Contains(err.Error(), "duplicate key value") {
|
||||
t.Fatalf("duplicate identity err = %v, want unique violation", err)
|
||||
}
|
||||
|
||||
// A later ChangeSet touching the same version+target is a different identity.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES('job-p5-3','test.upload','{}','p5:next','cs-p5-b','version-p5','kb-internal','upload')`); err != nil {
|
||||
t.Fatalf("different change set rejected: %v", err)
|
||||
}
|
||||
|
||||
// A different action on the same identity tuple is legal.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
|
||||
VALUES('job-p5-4','test.remove','{}','p5:remove','cs-p5-a','version-p5','kb-internal','remove')`); err != nil {
|
||||
t.Fatalf("different action rejected: %v", err)
|
||||
}
|
||||
|
||||
// Jobs without business identity never collide with the partial index.
|
||||
for _, id := range []string{"job-p5-5", "job-p5-6"} {
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key) VALUES($1,'test.poll','{}',$2)`, id, "plain-"+id); err != nil {
|
||||
t.Fatalf("unidentified job %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM jobs WHERE business_version_id='version-p5'`).Scan(&count); err != nil || count != 3 {
|
||||
t.Fatalf("identified jobs = %d err = %v, want 3", count, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// TestP6DeadLetterAndServerSideBatchRetry freezes docs plan P6: an ordinary
|
||||
// failure that exhausts its budget lands in the DLQ projection without
|
||||
// touching the original audit row, a permanent failure lands immediately, and
|
||||
// the server-side batch retry returns mixed outcomes atomically with replay.
|
||||
func TestP6DeadLetterAndServerSideBatchRetry(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
_, seedErr := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p6','alist','P6 来源',true);
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p6',repeat('a',64),10,'verified/p6.pdf','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-p6','conn-p6','ext-p6',repeat('d',64),'P6.pdf','document','r1','internal','manual');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES
|
||||
('siv-1','item-p6','r1','co-p6','verified'),
|
||||
('siv-2','item-p6','r2','co-p6','verified'),
|
||||
('siv-3','item-p6','r3','co-p6','verified'),
|
||||
('siv-4','item-p6','r4','co-p6','verified');
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,updated_at) VALUES
|
||||
('tp-1','siv-1','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
|
||||
('tp-2','siv-2','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
|
||||
('tp-3','siv-3','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
|
||||
('tp-5','siv-4','kb-1','uploaded','processing','unpublished','internal','2026-09-12T01:00:00Z');`)
|
||||
if seedErr != nil {
|
||||
t.Fatalf("seed: %v", seedErr)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
base := time.Date(2026, time.September, 12, 1, 0, 0, 0, time.UTC)
|
||||
|
||||
// 1) Ordinary failure exhausting the budget projects into the DLQ.
|
||||
uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-1"})
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-1','test.upload',$1,'running',5,5,'k-dlq-1','w1','cs-p6','siv-1','kb-1','upload',$2)`, uploadPayload, base); err != nil {
|
||||
t.Fatalf("seed job: %v", err)
|
||||
}
|
||||
if err := store.Fail(ctx, "job-dlq-1", "w1", "remote parse failed", "validation_failed", base, base.Add(time.Minute)); err != nil {
|
||||
t.Fatalf("exhausting fail: %v", err)
|
||||
}
|
||||
var jobState, errorCode, failureState string
|
||||
var attempt int
|
||||
if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState, &errorCode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobState != "failed" || errorCode != "validation_failed" {
|
||||
t.Fatalf("exhausted job = (%s,%s)", jobState, errorCode)
|
||||
}
|
||||
var failedPublicationID string
|
||||
if err := pool.QueryRow(ctx, `SELECT state,attempt,error_code,publication_id FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &attempt, &errorCode, &failedPublicationID); err != nil {
|
||||
t.Fatalf("dlq projection missing: %v", err)
|
||||
}
|
||||
if failureState != "open" || attempt != 5 || errorCode != "validation_failed" || failedPublicationID != "tp-1" {
|
||||
t.Fatalf("dlq row = (%s,%d,%s,%s)", failureState, attempt, errorCode, failedPublicationID)
|
||||
}
|
||||
|
||||
// 2) Permanent failure parks immediately and projects.
|
||||
removePayload, _ := json.Marshal(map[string]string{"publication_id": "tp-3"})
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-3','test.remove',$1,'running',1,10,'k-dlq-3','w1','cs-p6','siv-3','','remove',$2)`, removePayload, base); err != nil {
|
||||
t.Fatalf("seed remove job: %v", err)
|
||||
}
|
||||
var _ jobs.TerminalFailer = store
|
||||
if err := store.FailTerminal(ctx, "job-dlq-3", "w1", "remote rejected", "remote_permanent", base); err != nil {
|
||||
t.Fatalf("terminal fail: %v", err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-3'`).Scan(&jobState, &errorCode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobState != "failed" || errorCode != "remote_permanent" {
|
||||
t.Fatalf("terminal job = (%s,%s)", jobState, errorCode)
|
||||
}
|
||||
|
||||
// 3) DLQ query lists both open projections.
|
||||
page, err := store.ListJobFailures(ctx, governance.JobFailureFilter{Limit: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list failures: %v", err)
|
||||
}
|
||||
if page.Total != 2 || len(page.Items) != 2 {
|
||||
t.Fatalf("open failures = %d/%d, want 2", len(page.Items), page.Total)
|
||||
}
|
||||
|
||||
// 4) Server-side batch retry: tp-1 + tp-2 started, unknown id not_retryable.
|
||||
result, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
|
||||
PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
|
||||
OperatorRef: "姚工",
|
||||
Reason: "解析服务恢复",
|
||||
Now: base.Add(2 * time.Minute),
|
||||
}, "op-retry-batch-1")
|
||||
if err != nil || replayed {
|
||||
t.Fatalf("batch retry: %v replayed=%v", err, replayed)
|
||||
}
|
||||
if result.Total != 4 || result.Started != 2 || result.NotRetryable != 1 || result.AlreadyRunning != 1 {
|
||||
t.Fatalf("batch result = %+v", result)
|
||||
}
|
||||
if result.Outcomes[0].PublicationID != "tp-1" || result.Outcomes[0].Result != governance.RetryStarted {
|
||||
t.Fatalf("tp-1 outcome = %+v", result.Outcomes[0])
|
||||
}
|
||||
if result.Outcomes[1].PublicationID != "tp-2" || result.Outcomes[1].Result != governance.RetryStarted {
|
||||
t.Fatalf("tp-2 outcome = %+v", result.Outcomes[1])
|
||||
}
|
||||
if result.Outcomes[2].PublicationID != "tp-4-missing" || result.Outcomes[2].Result != governance.RetryNotRetryable || result.Outcomes[2].Detail != "publication not found" {
|
||||
t.Fatalf("missing outcome = %+v", result.Outcomes[2])
|
||||
}
|
||||
if result.Outcomes[3].PublicationID != "tp-5" || result.Outcomes[3].Result != governance.RetryAlreadyRunning {
|
||||
t.Fatalf("tp-5 outcome = %+v", result.Outcomes[3])
|
||||
}
|
||||
|
||||
// tp-1's open DLQ row flipped to requeued with operator and reason.
|
||||
var actor, reason string
|
||||
if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &actor, &reason); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failureState != "requeued" || actor != "姚工" || reason != "解析服务恢复" {
|
||||
t.Fatalf("tp-1 failure after retry = (%s,%s,%s)", failureState, actor, reason)
|
||||
}
|
||||
// tp-3's failure stays open (not part of this batch).
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE job_id='job-dlq-3'`).Scan(&failureState); err != nil || failureState != "open" {
|
||||
t.Fatalf("tp-3 failure state = %s err=%v", failureState, err)
|
||||
}
|
||||
|
||||
// Audit rows recorded atomically for every selection.
|
||||
var auditCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM job_retry_operations WHERE batch_key='op-retry-batch-1'`).Scan(&auditCount); err != nil || auditCount != 4 {
|
||||
t.Fatalf("retry audit rows = %d err=%v", auditCount, err)
|
||||
}
|
||||
// Fresh retry jobs exist and the original exhausted job stays untouched.
|
||||
var newJobs int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND state='pending' AND idempotency_key LIKE 'test-retry:tp-1:%'`).Scan(&newJobs); err != nil || newJobs != 1 {
|
||||
t.Fatalf("tp-1 retry jobs = %d err=%v", newJobs, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState); err != nil || jobState != "failed" {
|
||||
t.Fatalf("original job state = %q err=%v, want failed (untouched)", jobState, err)
|
||||
}
|
||||
|
||||
// 5) Replaying the same operation id returns the identical stored result.
|
||||
replay, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
|
||||
PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
|
||||
OperatorRef: "姚工",
|
||||
Reason: "解析服务恢复",
|
||||
Now: base.Add(9 * time.Minute),
|
||||
}, "op-retry-batch-1")
|
||||
if err != nil || !replayed {
|
||||
t.Fatalf("replay: %v replayed=%v", err, replayed)
|
||||
}
|
||||
if replay.Started != result.Started || replay.Total != result.Total {
|
||||
t.Fatalf("replay result = %+v, want %+v", replay, result)
|
||||
}
|
||||
|
||||
// 6) Same key with a different body conflicts.
|
||||
if _, _, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
|
||||
PublicationIDs: []string{"tp-2"},
|
||||
OperatorRef: "姚工",
|
||||
Reason: "解析服务恢复",
|
||||
Now: base.Add(10 * time.Minute),
|
||||
}, "op-retry-batch-1"); err == nil {
|
||||
t.Fatal("expected operation request conflict for same key different body")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
)
|
||||
|
||||
type LegacyImportReport struct {
|
||||
Connections int
|
||||
SourceItems int
|
||||
ContentObjects int
|
||||
Versions int
|
||||
Publications int
|
||||
}
|
||||
|
||||
func LoadPPCurrentItems(ctx context.Context, pool *pgxpool.Pool, connectionID string) (map[string]ppconnector.CurrentItem, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT stable_external_id, revision
|
||||
FROM source_items
|
||||
WHERE source_connection_id = $1
|
||||
ORDER BY stable_external_id`, connectionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query current PP items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make(map[string]ppconnector.CurrentItem)
|
||||
for rows.Next() {
|
||||
var sourceHash string
|
||||
var item ppconnector.CurrentItem
|
||||
if err := rows.Scan(&sourceHash, &item.Revision); err != nil {
|
||||
return nil, fmt.Errorf("scan current PP item: %w", err)
|
||||
}
|
||||
items[sourceHash] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate current PP items: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func ImportPPLegacy(ctx context.Context, pool *pgxpool.Pool, plan ppconnector.ImportPlan) (LegacyImportReport, error) {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("begin PP legacy import: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
report := LegacyImportReport{}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO source_connections (
|
||||
id, source_type, name, base_url, secret_ref, enabled, schedule, last_health
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, '{}'::jsonb, '{}'::jsonb)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
plan.Connection.ID,
|
||||
plan.Connection.Type,
|
||||
plan.Connection.Name,
|
||||
plan.Connection.BaseURL,
|
||||
plan.Connection.SecretRef,
|
||||
plan.Connection.Enabled,
|
||||
)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("insert PP source connection: %w", err)
|
||||
}
|
||||
report.Connections = int(tag.RowsAffected())
|
||||
|
||||
for _, content := range plan.ContentObjects {
|
||||
tag, err = tx.Exec(ctx, `INSERT INTO content_objects (
|
||||
id, sha256, size_bytes, mime, validated_format, local_spool_ref, content_state
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
content.ID,
|
||||
content.SHA256,
|
||||
content.SizeBytes,
|
||||
content.MIME,
|
||||
content.ValidatedFormat,
|
||||
content.LocalSpoolRef,
|
||||
content.State,
|
||||
)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("insert PP content object: %w", err)
|
||||
}
|
||||
report.ContentObjects += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
for _, item := range plan.SourceItems {
|
||||
tag, err = tx.Exec(ctx, `INSERT INTO source_items (
|
||||
id, source_connection_id, stable_external_id, canonical_uri_hash,
|
||||
parent_external_id, normalized_path, title, item_type, size_bytes,
|
||||
mime, revision, visibility, visibility_origin, last_seen_scan_id,
|
||||
source_change, absence_streak, identity_confidence
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
$10, $11, $12, $13, $14, $15, $16, 'stable_id'
|
||||
)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
item.ID,
|
||||
item.SourceConnectionID,
|
||||
item.StableExternalID,
|
||||
item.CanonicalURIHash,
|
||||
item.ParentExternalID,
|
||||
item.NormalizedPath,
|
||||
item.Title,
|
||||
item.ItemType,
|
||||
item.SizeBytes,
|
||||
item.MIME,
|
||||
item.Revision,
|
||||
item.Visibility,
|
||||
item.VisibilityOrigin,
|
||||
item.LastSeenScanID,
|
||||
item.SourceChange,
|
||||
item.AbsenceStreak,
|
||||
)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("insert PP source item: %w", err)
|
||||
}
|
||||
report.SourceItems += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
for _, version := range plan.Versions {
|
||||
tag, err = tx.Exec(ctx, `INSERT INTO source_item_versions (
|
||||
id, source_item_id, revision, content_object_id, content_state,
|
||||
download_error, raw_record_summary, source_updated_at
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, CAST($7 AS JSONB), $8)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
version.ID,
|
||||
version.SourceItemID,
|
||||
version.Revision,
|
||||
version.ContentObjectID,
|
||||
version.ContentState,
|
||||
version.DownloadError,
|
||||
string(version.RawRecordSummary),
|
||||
version.SourceUpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("insert PP source item version: %w", err)
|
||||
}
|
||||
report.Versions += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
for _, publication := range plan.Publications {
|
||||
tag, err = tx.Exec(ctx, `INSERT INTO test_publications (
|
||||
id, source_item_version_id, test_kb_id, knowledge_id,
|
||||
ingest_state, processing_state, retrieval_state,
|
||||
parser_generation, index_generation, last_error, retry_count
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
publication.ID,
|
||||
publication.SourceItemVersionID,
|
||||
publication.TestKBID,
|
||||
publication.KnowledgeID,
|
||||
publication.IngestState,
|
||||
publication.ProcessingState,
|
||||
publication.RetrievalState,
|
||||
publication.ParserGeneration,
|
||||
publication.IndexGeneration,
|
||||
publication.LastError,
|
||||
publication.RetryCount,
|
||||
)
|
||||
if err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("insert PP test publication: %w", err)
|
||||
}
|
||||
report.Publications += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return LegacyImportReport{}, fmt.Errorf("commit PP legacy import: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestImportPPLegacyIsIdempotent(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("pgxpool.New() error = %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("ApplyMigrations() error = %v", err)
|
||||
}
|
||||
|
||||
sourceCanonical := strings.Repeat("a", 64)
|
||||
sourceAlias := strings.Repeat("b", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
|
||||
{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
|
||||
{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildImportPlan() error = %v", err)
|
||||
}
|
||||
|
||||
first, err := ImportPPLegacy(ctx, pool, plan)
|
||||
if err != nil {
|
||||
t.Fatalf("first ImportPPLegacy() error = %v", err)
|
||||
}
|
||||
if first != (LegacyImportReport{Connections: 1, SourceItems: 2, ContentObjects: 1, Versions: 2, Publications: 1}) {
|
||||
t.Fatalf("first ImportPPLegacy() report = %#v", first)
|
||||
}
|
||||
second, err := ImportPPLegacy(ctx, pool, plan)
|
||||
if err != nil {
|
||||
t.Fatalf("second ImportPPLegacy() error = %v", err)
|
||||
}
|
||||
if second != (LegacyImportReport{}) {
|
||||
t.Fatalf("second ImportPPLegacy() report = %#v, want zero inserts", second)
|
||||
}
|
||||
|
||||
var counts [5]int
|
||||
if err := pool.QueryRow(ctx, `SELECT
|
||||
(SELECT COUNT(*) FROM source_connections),
|
||||
(SELECT COUNT(*) FROM source_items),
|
||||
(SELECT COUNT(*) FROM content_objects),
|
||||
(SELECT COUNT(*) FROM source_item_versions),
|
||||
(SELECT COUNT(*) FROM test_publications)`).Scan(&counts[0], &counts[1], &counts[2], &counts[3], &counts[4]); err != nil {
|
||||
t.Fatalf("query imported counts: %v", err)
|
||||
}
|
||||
if counts != [5]int{1, 2, 1, 2, 1} {
|
||||
t.Fatalf("imported counts = %#v", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPPCurrentItemsSupportsZeroUploadReconciliation(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
source := strings.Repeat("d", 64)
|
||||
contentHash := strings.Repeat("e", 64)
|
||||
snapshot := ppconnector.Snapshot{KBID: "kb-reconcile", Items: []ppconnector.Item{{
|
||||
KBID: "kb-reconcile", SourceURLHash: source, SourceIDs: []string{"pp-3"}, Bytes: 12,
|
||||
SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-3",
|
||||
}}}
|
||||
plan, err := ppconnector.BuildImportPlan(snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ImportPPLegacy(ctx, pool, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
current, err := LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
summary, err := ppconnector.PlanReconciliation(snapshot, current)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary.UploadRequired != 0 || summary.Unchanged != 1 {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreListsSourceAssetsWithSummaryAndStateFilter(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-assets", Items: []ppconnector.Item{
|
||||
{KBID: "kb-assets", SourceURLHash: strings.Repeat("1", 64), SourceIDs: []string{"pp-5"}, Bytes: 10, SHA256: strings.Repeat("3", 64), Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-5"},
|
||||
{KBID: "kb-assets", SourceURLHash: strings.Repeat("2", 64), SourceIDs: []string{"pp-6"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range plan.SourceItems {
|
||||
plan.SourceItems[index].Title = "合成手册.pdf"
|
||||
plan.SourceItems[index].NormalizedPath = "使用指南"
|
||||
}
|
||||
if _, err := ImportPPLegacy(ctx, pool, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
page, err := NewStore(pool).ListSourceAssets(ctx, governance.SourceAssetFilter{State: "over_limit", Limit: 25})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if page.Summary != (governance.SourceAssetSummary{Total: 2, Completed: 1, OverLimit: 1}) {
|
||||
t.Fatalf("summary = %#v", page.Summary)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].ContentState != governance.ContentOverLimit || page.Items[0].HasPublication {
|
||||
t.Fatalf("items = %#v", page.Items)
|
||||
}
|
||||
item := page.Items[0]
|
||||
if item.Title != "合成手册.pdf" || item.DirectoryPath != "使用指南" || item.ConnectionID != plan.Connection.ID || item.VisibilityOrigin != "pp_public_tag" || item.CanRetry {
|
||||
t.Fatalf("document navigation metadata mismatch: %#v", item)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func (s *Store) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) {
|
||||
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
|
||||
operator := strings.TrimSpace(request.OperatorRef)
|
||||
reason := strings.TrimSpace(request.Reason)
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
|
||||
}
|
||||
seen := make(map[string]bool, len(request.PublicationIDs))
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
if !validRouteID(publicationID) || seen[publicationID] {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("invalid processing publication IDs")
|
||||
}
|
||||
seen[publicationID] = true
|
||||
}
|
||||
|
||||
var tx pgx.Tx
|
||||
var err error
|
||||
ownsTransaction := false
|
||||
if existing, ok := s.db.(pgx.Tx); ok {
|
||||
tx = existing
|
||||
} else {
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("processing confirmation transactions are unavailable")
|
||||
}
|
||||
var err error
|
||||
tx, err = beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("begin processing confirmation: %w", err)
|
||||
}
|
||||
ownsTransaction = true
|
||||
defer tx.Rollback(ctx)
|
||||
}
|
||||
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
var ready, confirmed, uploadJobExists bool
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT p.ingest_state='pending'
|
||||
AND p.processing_state='pending'
|
||||
AND p.retrieval_state='unpublished'
|
||||
AND p.knowledge_id=''
|
||||
AND version.content_state='verified'
|
||||
AND content.content_state='verified'
|
||||
AND content.local_spool_ref<>''
|
||||
AND content.size_bytes>0 AS ready,
|
||||
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
|
||||
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
||||
JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE p.id=$1
|
||||
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("inspect processing plan: %w", err)
|
||||
}
|
||||
if confirmed || uploadJobExists {
|
||||
result.AlreadyStarted++
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
confirmationID, err := newOpaqueID("pc-")
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, err
|
||||
}
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{"publication_id": publicationID})
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("encode processing upload job")
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,created_at) VALUES($1,$2,$3,$4,$5,$5)`, confirmationID, publicationID, operator, reason, request.Now); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("record processing confirmation: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
||||
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3)`, jobID, payload, request.Now, "test-start:"+publicationID); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("enqueue confirmed processing: %w", err)
|
||||
}
|
||||
result.Started++
|
||||
}
|
||||
if ownsTransaction {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("commit processing confirmation: %w", err)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) {
|
||||
fingerprint, err := operationFingerprint(struct {
|
||||
PublicationIDs []string `json:"publication_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
}{request.PublicationIDs, request.OperatorRef, request.Reason})
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, false, err
|
||||
}
|
||||
raw, replayed, err := s.executeOperation(ctx, "test.publications.start", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
||||
result, err := NewStore(tx).StartTestPublications(ctx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(result)
|
||||
})
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, false, err
|
||||
}
|
||||
var result governance.StartTestPublicationsResult
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, false, fmt.Errorf("decode stored processing result: %w", err)
|
||||
}
|
||||
return result, replayed, nil
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
func TestProductionDeleteBatchIsReversibleAndReconcilesPlan(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 10, 5, 0, 0, 0, time.UTC)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publicationID := "tp-delete-e2e"
|
||||
insertReviewFixture(t, ctx, pool, 7, publicationID, now)
|
||||
review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initialBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Initial publish", CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: initialBatch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initialExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: initialBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initialPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: initialExecution.ID, JobID: initialExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
|
||||
if err != nil || len(initialPlan) != 1 || initialPlan[0].Action != "upsert" {
|
||||
t.Fatalf("initial plan=%#v error=%v", initialPlan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: initialExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "absent", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, initialExecution.ID, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-7','blocked','engineer-1','确认删除生产知识',$1,$1)`, now.Add(8*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deletePlan, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != publicationID {
|
||||
t.Fatalf("delete plan=%#v error=%v", deletePlan, err)
|
||||
}
|
||||
deleteBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Delete production knowledge", CreatedBy: "engineer-1", DeletePublicationIDs: []string{publicationID}, Now: now.Add(9 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.GetReleaseBatch(ctx, deleteBatch.ID)
|
||||
if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" {
|
||||
t.Fatalf("delete batch=%#v error=%v", detail, err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: deleteBatch.ID, SealedBy: "engineer-2", Now: now.Add(10 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleteExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: deleteBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(11 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-7'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12 * time.Minute)}); !errors.Is(err, ErrReleaseExecutionNotClaimable) {
|
||||
t.Fatalf("unavailable source start error=%v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-7'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleteItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12*time.Minute + time.Second)})
|
||||
if err != nil || len(deleteItems) != 1 || deleteItems[0].Action != "delete" || deleteItems[0].PreviousProductionRevision != review.ID {
|
||||
t.Fatalf("delete items=%#v error=%v", deleteItems, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: deleteExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: review.ID, ProductionRevision: "absent", Now: now.Add(13 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, deleteExecution.ID, now.Add(14*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterDelete, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(afterDelete.Items) != 0 {
|
||||
t.Fatalf("plan after delete=%#v error=%v", afterDelete, err)
|
||||
}
|
||||
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: deleteExecution.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(15 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rollbackItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)})
|
||||
if err != nil || len(rollbackItems) != 1 || rollbackItems[0].PreviousProductionRevision != review.ID {
|
||||
t.Fatalf("rollback items=%#v error=%v", rollbackItems, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: publicationID, PreviousProductionRevision: review.ID, ProductionRevision: review.ID, Now: now.Add(17 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterRollback, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(afterRollback.Items) != 1 || afterRollback.Items[0].Action != "delete" {
|
||||
t.Fatalf("plan after rollback=%#v error=%v", afterRollback, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type PublicationForRemoval struct {
|
||||
ID string
|
||||
TestKBID string
|
||||
KnowledgeID string
|
||||
RetrievalState governance.RetrievalState
|
||||
}
|
||||
|
||||
func (s *Store) LoadPublicationForRemoval(ctx context.Context, publicationID string) (PublicationForRemoval, error) {
|
||||
var candidate PublicationForRemoval
|
||||
var retrievalState string
|
||||
err := s.db.QueryRow(ctx, `SELECT id,test_kb_id,knowledge_id,retrieval_state
|
||||
FROM test_publications
|
||||
WHERE id=$1 AND knowledge_id<>'' AND retrieval_state='disabling'`, publicationID).Scan(
|
||||
&candidate.ID, &candidate.TestKBID, &candidate.KnowledgeID, &retrievalState,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return PublicationForRemoval{}, ErrPublicationStateConflict
|
||||
}
|
||||
if err != nil {
|
||||
return PublicationForRemoval{}, fmt.Errorf("load publication for removal: %w", err)
|
||||
}
|
||||
candidate.RetrievalState = governance.RetrievalState(retrievalState)
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func (s *Store) RecordRemovalCompleted(ctx context.Context, publicationID string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE test_publications
|
||||
SET retrieval_state='disabled',last_error='',updated_at=$2
|
||||
WHERE id=$1 AND retrieval_state='disabling'`, publicationID, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record publication removal: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ErrPublicationStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) RecordRemovalFailure(ctx context.Context, publicationID, code string, now time.Time) error {
|
||||
if code != "removal_identity_invalid" && code != "removal_remote_rejected" {
|
||||
return ErrInvalidErrorCode
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `UPDATE test_publications
|
||||
SET retrieval_state='failed',last_error=$2,updated_at=$3
|
||||
WHERE id=$1 AND retrieval_state='disabling'`, publicationID, code, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record publication removal failure: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return ErrPublicationStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package postgres
|
||||
|
||||
const enqueueJobSQL = `
|
||||
INSERT INTO jobs (
|
||||
id, type, payload, state, priority, available_at,
|
||||
max_attempts, idempotency_key, created_at, updated_at
|
||||
) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8, $8)
|
||||
ON CONFLICT (idempotency_key)
|
||||
DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key
|
||||
RETURNING id, type, payload, state, priority, available_at,
|
||||
lease_owner, lease_until, attempt, max_attempts,
|
||||
last_error, idempotency_key, created_at, updated_at
|
||||
`
|
||||
|
||||
const claimJobSQL = `
|
||||
WITH candidate AS (
|
||||
SELECT id
|
||||
FROM jobs
|
||||
WHERE (
|
||||
(state IN ('pending', 'retry') AND available_at <= $2)
|
||||
OR (state = 'running' AND lease_until < $2)
|
||||
)
|
||||
AND attempt < max_attempts
|
||||
AND type = ANY($4::text[])
|
||||
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE jobs AS j
|
||||
SET state = 'running',
|
||||
started_at = $2,
|
||||
lease_owner = $1,
|
||||
lease_until = $3,
|
||||
attempt = j.attempt + 1,
|
||||
updated_at = $2
|
||||
FROM candidate
|
||||
WHERE j.id = candidate.id
|
||||
RETURNING j.id, j.type, j.payload, j.state, j.priority, j.available_at,
|
||||
j.lease_owner, j.lease_until, j.attempt, j.max_attempts,
|
||||
j.last_error, j.idempotency_key, j.created_at, j.updated_at
|
||||
`
|
||||
|
||||
const completeJobSQL = `
|
||||
UPDATE jobs
|
||||
SET state = 'completed',
|
||||
completed_at = $3,
|
||||
lease_owner = NULL,
|
||||
lease_until = NULL,
|
||||
updated_at = $3
|
||||
WHERE id = $1
|
||||
AND lease_owner = $2
|
||||
AND state = 'running'
|
||||
`
|
||||
|
||||
const heartbeatJobSQL = `
|
||||
UPDATE jobs
|
||||
SET lease_until = $4,
|
||||
updated_at = $3
|
||||
WHERE id = $1
|
||||
AND lease_owner = $2
|
||||
AND state = 'running'
|
||||
`
|
||||
|
||||
const failJobSQL = `
|
||||
WITH updated AS (
|
||||
UPDATE jobs
|
||||
SET state = CASE WHEN attempt >= max_attempts THEN 'failed' ELSE 'retry' END,
|
||||
available_at = $6,
|
||||
last_error = $3,
|
||||
last_error_code = CASE WHEN $5 <> '' THEN $5 ELSE last_error_code END,
|
||||
lease_owner = NULL,
|
||||
lease_until = NULL,
|
||||
updated_at = $4
|
||||
WHERE id = $1
|
||||
AND lease_owner = $2
|
||||
AND state = 'running'
|
||||
RETURNING id, type, payload, last_error,
|
||||
CASE WHEN last_error_code = '' THEN 'handler_error' ELSE last_error_code END AS error_code,
|
||||
attempt, max_attempts,
|
||||
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
|
||||
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt,
|
||||
state
|
||||
)
|
||||
INSERT INTO job_failures (
|
||||
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
|
||||
business_change_set_id, business_version_id, business_action, business_target_kb_id,
|
||||
created_at
|
||||
)
|
||||
SELECT updated.id, updated.type, COALESCE(updated.payload->>'publication_id',''), updated.error_code,
|
||||
updated.last_error, updated.attempt, updated.max_attempts,
|
||||
updated.bc, updated.bv, updated.ba, updated.bt, $4
|
||||
FROM updated
|
||||
WHERE updated.state = 'failed' AND updated.attempt > 0
|
||||
ON CONFLICT (job_id, attempt) DO UPDATE
|
||||
SET publication_id = EXCLUDED.publication_id,
|
||||
error_code = EXCLUDED.error_code,
|
||||
last_error = EXCLUDED.last_error,
|
||||
max_attempts_at_failure = EXCLUDED.max_attempts_at_failure,
|
||||
business_change_set_id = EXCLUDED.business_change_set_id,
|
||||
business_version_id = EXCLUDED.business_version_id,
|
||||
business_action = EXCLUDED.business_action,
|
||||
business_target_kb_id = EXCLUDED.business_target_kb_id,
|
||||
state = 'open',
|
||||
resolved_at = NULL,
|
||||
resolution_actor_ref = '',
|
||||
resolution_reason = '',
|
||||
created_at = EXCLUDED.created_at
|
||||
`
|
||||
|
||||
const listJobsSQL = `
|
||||
SELECT id, type, payload, state, priority, available_at,
|
||||
lease_owner, lease_until, attempt, max_attempts,
|
||||
last_error, idempotency_key, created_at, updated_at
|
||||
FROM jobs
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT $1
|
||||
`
|
||||
|
||||
// P6: a permanent failure parks the job in 'failed' immediately and projects
|
||||
// the terminal attempt into the dead-letter table.
|
||||
const failTerminalJobSQL = `
|
||||
WITH parked AS (
|
||||
UPDATE jobs
|
||||
SET state = 'failed',
|
||||
available_at = $5,
|
||||
last_error = $3,
|
||||
last_error_code = CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END,
|
||||
lease_owner = NULL,
|
||||
lease_until = NULL,
|
||||
updated_at = $5
|
||||
WHERE id = $1
|
||||
AND lease_owner = $2
|
||||
AND lease_owner IS NOT NULL
|
||||
AND state = 'running'
|
||||
RETURNING id, type, payload, last_error,
|
||||
CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END AS error_code,
|
||||
attempt, max_attempts,
|
||||
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
|
||||
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt
|
||||
)
|
||||
INSERT INTO job_failures (
|
||||
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
|
||||
business_change_set_id, business_version_id, business_action, business_target_kb_id,
|
||||
created_at
|
||||
)
|
||||
SELECT parked.id, parked.type, COALESCE(parked.payload->>'publication_id',''), parked.error_code,
|
||||
parked.last_error, GREATEST(parked.attempt,1), parked.max_attempts,
|
||||
parked.bc, parked.bv, parked.ba, parked.bt, $5
|
||||
FROM parked
|
||||
ON CONFLICT (job_id, attempt) DO UPDATE
|
||||
SET publication_id = EXCLUDED.publication_id,
|
||||
error_code = EXCLUDED.error_code,
|
||||
last_error = EXCLUDED.last_error,
|
||||
max_attempts_at_failure = EXCLUDED.max_attempts_at_failure,
|
||||
state = 'open',
|
||||
resolved_at = NULL,
|
||||
resolution_actor_ref = '',
|
||||
resolution_reason = '',
|
||||
created_at = EXCLUDED.created_at
|
||||
`
|
||||
|
||||
// P10 liveness: every claim attempt refreshes the worker row so the API can
|
||||
// distinguish "worker loop alive" from "process dead" without an HTTP port.
|
||||
const workerHeartbeatSQL = `
|
||||
INSERT INTO worker_heartbeats(worker_id, job_types, started_at, last_seen_at)
|
||||
VALUES($1, $2, $3, $3)
|
||||
ON CONFLICT (worker_id) DO UPDATE
|
||||
SET job_types = EXCLUDED.job_types, last_seen_at = EXCLUDED.last_seen_at
|
||||
`
|
||||
@@ -0,0 +1,317 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// seedR1ReleaseFixture gives us:
|
||||
// - tp-keep: fully review+acceptance approved, present source -> add
|
||||
// - tp-drop: was released to prod, source now deleted -> delete
|
||||
// - tp-block: source unavailable -> blocked
|
||||
// - tp-unch: already released to prod with same revision -> unchanged
|
||||
func seedR1ReleaseFixture(t *testing.T, pool *pgxpool.Pool, ctx context.Context) {
|
||||
t.Helper()
|
||||
_, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name) VALUES
|
||||
('conn-r1','alist','R1来源');
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES
|
||||
('co-r1',repeat('a',64),10,'spool/r1','verified');
|
||||
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,normalized_path,revision,visibility,visibility_origin,source_change,identity_confidence) VALUES
|
||||
('item-keep','conn-r1','ext-keep',repeat('1',64),'新增手册','document','新增手册.pdf','v1','internal','manual','unchanged','stable_id'),
|
||||
('item-drop','conn-r1','ext-drop',repeat('2',64),'已删手册','document','已删手册.pdf','v1','internal','manual','deleted','stable_id'),
|
||||
('item-block','conn-r1','ext-block',repeat('3',64),'不可用手册','document','不可用手册.pdf','v1','internal','manual','unavailable','stable_id'),
|
||||
('item-unch','conn-r1','ext-unch',repeat('4',64),'不变手册','document','不变手册.pdf','v1','internal','manual','unchanged','stable_id');
|
||||
INSERT INTO source_item_versions(id,source_item_id,revision,content_state) VALUES
|
||||
('siv-keep','item-keep','v1','verified'),
|
||||
('siv-drop','item-drop','v1','verified'),
|
||||
('siv-block','item-block','v1','verified'),
|
||||
('siv-unch','item-unch','v1','verified');
|
||||
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,processing_state,ingest_state,retrieval_state,updated_at) VALUES
|
||||
('tp-keep','siv-keep','kb-r1','internal','completed','uploaded','unpublished',now() - interval '1 hour'),
|
||||
('tp-drop','siv-drop','kb-r1','internal','completed','uploaded','unpublished',now() - interval '2 hours'),
|
||||
('tp-block','siv-block','kb-r1','internal','completed','uploaded','unpublished',now() - interval '3 hours'),
|
||||
('tp-unch','siv-unch','kb-r1','internal','completed','uploaded','unpublished',now() - interval '4 hours');
|
||||
INSERT INTO acceptance_results(id,publication_id,run_id,check_type,state,policy_version,observed_at) VALUES
|
||||
('ace-keep-1','tp-keep','run-keep','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-keep-2','tp-keep','run-keep','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-keep-3','tp-keep','run-keep','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-unch-1','tp-unch','run-unch','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-unch-2','tp-unch','run-unch','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-unch-3','tp-unch','run-unch','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-drop-1','tp-drop','run-drop','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-drop-2','tp-drop','run-drop','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
|
||||
('ace-drop-3','tp-drop','run-drop','retrieval_hit','passed','structural-v1',now() - interval '50 minutes');
|
||||
INSERT INTO publication_reviews(id,publication_id,decision,decided_visibility,reviewer_ref,decision_code,acceptance_run_id,publication_updated_at,created_at)
|
||||
SELECT 'rev-'||p.id, p.id, 'approved', 'internal', 'reviewer-r1', 'auto_approve', split_part(a.run_id,'',1), p.updated_at, now()
|
||||
FROM test_publications p
|
||||
JOIN LATERAL (SELECT CASE p.id WHEN 'tp-keep' THEN 'run-keep' WHEN 'tp-unch' THEN 'run-unch' ELSE 'run-drop' END AS run_id) a ON TRUE
|
||||
WHERE p.id IN ('tp-keep','tp-unch','tp-drop');
|
||||
INSERT INTO release_batches(id,name,state,policy_version,created_by,item_count,created_at,sealed_at,release_change_set_id) VALUES
|
||||
('rb-seed','历史批次','sealed','release-gate-v1','seed',2,now() - interval '1 day',now() - interval '1 day',NULL);
|
||||
INSERT INTO release_batch_items(release_batch_id,publication_id,review_id,source_item_version_id,acceptance_run_id,publication_updated_at,decided_visibility,action,created_at) VALUES
|
||||
('rb-seed','tp-unch','rev-tp-unch','siv-unch','run-unch',(SELECT updated_at FROM test_publications WHERE id='tp-unch'),'internal','upsert',now() - interval '1 day'),
|
||||
('rb-seed','tp-drop','rev-tp-drop','siv-drop','run-drop',(SELECT updated_at FROM test_publications WHERE id='tp-drop'),'internal','upsert',now() - interval '1 day');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES
|
||||
('snap-r1','conn-r1',repeat('7',64),'R1快照',2,1,1,now() - interval '1 day');
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES
|
||||
('conn-r1','snap-r1',now() - interval '1 day');
|
||||
INSERT INTO jobs(id,type,payload,state,available_at,attempt,max_attempts,idempotency_key) VALUES
|
||||
('job-r1-seed','release.publish','{}','completed',now() - interval '23 hours',1,5,'kr1-seed');
|
||||
INSERT INTO release_executions(id,release_batch_id,operation,state,authorized_by,authorization_code,job_id,item_count,started_at,completed_at) VALUES
|
||||
('re-seed','rb-seed','publish','completed','seed','seed_auth',
|
||||
'job-r1-seed',2,now() - interval '1 day',now() - interval '23 hours');
|
||||
INSERT INTO release_item_executions(id,release_execution_id,publication_id,production_revision,state,completed_at)
|
||||
VALUES ('rie-seed','re-seed','tp-unch','rev-tp-unch','verified',now() - interval '23 hours'),
|
||||
('rie-seed-2','re-seed','tp-drop','rev-tp-drop','verified',now() - interval '23 hours');`)
|
||||
if err != nil {
|
||||
t.Fatalf("seed r1 fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestR1ReleaseChangeSetLifecycle freezes docs plan R1: generation snapshots the
|
||||
// live plan immutably, confirmation is required before formal batching, and
|
||||
// later source refreshes never move a frozen manifest.
|
||||
func TestR1ReleaseChangeSetLifecycle(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, time.September, 12, 7, 0, 0, 0, time.UTC)
|
||||
|
||||
plan, err := store.ListReleasePlan(ctx)
|
||||
if err == nil && len(plan.Items) > 0 {
|
||||
t.Skip("fixture isolation failed: live plan not empty")
|
||||
}
|
||||
|
||||
seedR1ReleaseFixture(t, pool, ctx)
|
||||
first, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布", CreatedBy: "yaozhanghao", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.State != "planned" || first.Number != 1 || first.ManifestSHA256 == "" || len(first.ManifestSHA256) != 64 {
|
||||
t.Fatalf("first set = %#v", first)
|
||||
}
|
||||
if len(first.SourceSnapshotIDs) == 0 {
|
||||
t.Fatalf("R1 core: change set must bind the head catalog snapshots it was computed against")
|
||||
}
|
||||
bound := false
|
||||
for _, id := range first.SourceSnapshotIDs {
|
||||
if id == "snap-r1" {
|
||||
bound = true
|
||||
}
|
||||
}
|
||||
if !bound {
|
||||
t.Fatalf("head snapshot not bound: %#v", first.SourceSnapshotIDs)
|
||||
}
|
||||
|
||||
detail, err := store.GetReleaseChangeSet(ctx, first.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actions := map[string]string{}
|
||||
blockedRows := 0
|
||||
for _, item := range detail.Items {
|
||||
actions[item.PublicationID] = item.Action
|
||||
if item.Action == "blocked" {
|
||||
blockedRows++
|
||||
}
|
||||
}
|
||||
if actions["tp-keep"] != "add" || actions["tp-drop"] != "delete" {
|
||||
t.Fatalf("frozen actions = %#v", actions)
|
||||
}
|
||||
// R1 core rule: an unavailable source can ONLY surface as blocked (never a
|
||||
// deletable/updatable item with a real publication binding).
|
||||
if blockedRows != 1 || actions["tp-block"] != "" && actions["tp-block"] != "blocked" {
|
||||
t.Fatalf("blocked handling = %#v rows=%d", actions, blockedRows)
|
||||
}
|
||||
if _, ok := actions["tp-unch"]; ok {
|
||||
t.Fatal("unchanged items must not enter the manifest")
|
||||
}
|
||||
// Manifest recompute must match byte-exact (stable ordering + delimiter).
|
||||
if got := ComputeReleaseChangeSetManifest(detail.Items); got != first.ManifestSHA256 {
|
||||
t.Fatalf("manifest not reproducible: %s vs %s", got, first.ManifestSHA256)
|
||||
}
|
||||
|
||||
// Source mutates AFTER freezing: dropping an item that already exists in
|
||||
// production turns its live-plan row into delete — but the frozen manifest
|
||||
// must not move (plan changes, set does not).
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='deleted' WHERE id='item-unch'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := store.GetReleaseChangeSet(ctx, first.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after.ManifestSHA256 != first.ManifestSHA256 || len(after.Items) != len(detail.Items) {
|
||||
t.Fatal("frozen change set mutated after source change")
|
||||
}
|
||||
// Fresh live plan reflects the mutation (proof the set is NOT live-derived).
|
||||
mutated, err := store.ListReleasePlan(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mutationSeen := false
|
||||
for _, item := range mutated.Items {
|
||||
if item.PublicationID == "tp-unch" && item.Action == "delete" {
|
||||
mutationSeen = true
|
||||
}
|
||||
}
|
||||
if !mutationSeen {
|
||||
t.Fatal("live plan failed to reflect post-freeze source change")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='item-unch'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Dedupe on the SAME plan: repeated generation reuses the latest PLANNED
|
||||
// set (concurrent clicks must not pile up versions).
|
||||
deduped, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "重复", CreatedBy: "yaozhanghao", Now: now.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deduped.ID != first.ID || deduped.Number != 1 {
|
||||
t.Fatalf("dedupe failed: %#v vs %#v", deduped, first)
|
||||
}
|
||||
|
||||
// Batching BEFORE confirmation is rejected with the honest reason.
|
||||
_, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
|
||||
Name: "太早了", CreatedBy: "yaozhanghao",
|
||||
PublicationIDs: []string{"tp-keep"},
|
||||
ReleaseChangeSetID: first.ID, Now: now,
|
||||
})
|
||||
if !errors.Is(err, ErrReleaseChangeSetUnconfirmed) {
|
||||
t.Fatalf("unconfirmed batching err = %v", err)
|
||||
}
|
||||
|
||||
// Confirm (idempotent replay on re-confirm).
|
||||
confirmed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "boss", Now: now})
|
||||
if err != nil || confirmed.State != "confirmed" || confirmed.ConfirmedBy != "boss" {
|
||||
t.Fatalf("confirm = %#v err = %v", confirmed, err)
|
||||
}
|
||||
replayed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "someone-else", Now: now.Add(time.Minute)})
|
||||
if err != nil || replayed.ConfirmedBy != "boss" {
|
||||
t.Fatalf("re-confirm must replay stored confirmation: %#v err=%v", replayed, err)
|
||||
}
|
||||
|
||||
// A new generation after confirmation takes the next version number — the
|
||||
// confirmed set stays untouched forever (page refresh cannot alter it).
|
||||
second, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布v2", CreatedBy: "yaozhanghao", Now: now.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Number != 2 || second.ID == first.ID {
|
||||
t.Fatalf("post-confirm generation = %#v", second)
|
||||
}
|
||||
stillFirst, err := store.GetReleaseChangeSet(ctx, first.ID)
|
||||
if err != nil || stillFirst.ManifestSHA256 != first.ManifestSHA256 || stillFirst.State != "confirmed" {
|
||||
t.Fatalf("confirmed set mutated: %#v err=%v", stillFirst.ReleaseChangeSet, err)
|
||||
}
|
||||
|
||||
// The formal batch citing the CONFIRMED set succeeds and keeps its binding.
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
|
||||
Name: "正式批次", CreatedBy: "yaozhanghao",
|
||||
PublicationIDs: []string{"tp-keep"},
|
||||
ReleaseChangeSetID: first.ID, Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("bound batch: %v", err)
|
||||
}
|
||||
if batch.ReleaseChangeSetID != first.ID {
|
||||
t.Fatalf("batch lost its binding: %#v", batch)
|
||||
}
|
||||
|
||||
// A batch citing items NOT in the confirmed manifest is rejected.
|
||||
_, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
|
||||
Name: "越界批次", CreatedBy: "yaozhanghao",
|
||||
PublicationIDs: []string{"tp-block"},
|
||||
ReleaseChangeSetID: first.ID, Now: now,
|
||||
})
|
||||
if !errors.Is(err, ErrReleaseBatchNotEligible) && !errors.Is(err, ErrReleaseChangeSetUnconfirmed) {
|
||||
t.Fatalf("out-of-manifest batch err = %v", err)
|
||||
}
|
||||
|
||||
// Concurrency: 8 parallel generations on the SAME plan produce exactly one
|
||||
// new version (dedupe) or all fail on the gate — never corrupt duplicates.
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM release_batch_items WHERE release_batch_id IN (SELECT id FROM release_batches WHERE release_change_set_id IS NOT NULL);
|
||||
DELETE FROM release_batches WHERE release_change_set_id IS NOT NULL;
|
||||
DELETE FROM release_change_sets`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
ids := make([]string, 8)
|
||||
errs := make([]error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
set, genErr := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "并发", CreatedBy: "yaozhanghao", Now: now})
|
||||
ids[index], errs[index] = set.ID, genErr
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
unique := map[string]bool{}
|
||||
for index, genErr := range errs {
|
||||
if genErr != nil {
|
||||
t.Fatalf("concurrent generate[%d]: %v", index, genErr)
|
||||
}
|
||||
unique[ids[index]] = true
|
||||
}
|
||||
var rowCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&rowCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rowCount != len(unique) {
|
||||
t.Fatalf("concurrent generation produced %d rows but %d distinct ids", rowCount, len(unique))
|
||||
}
|
||||
if len(unique) > 2 {
|
||||
t.Fatalf("dedupe collapsed under concurrency: %d distinct sets", len(unique))
|
||||
}
|
||||
|
||||
// The concurrent winner set is a valid, stable, hex manifest.
|
||||
var winnerID, manifest string
|
||||
if err := pool.QueryRow(ctx, `SELECT id, manifest_sha256 FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&winnerID, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(manifest) != 64 || strings.TrimLeft(manifest, "0123456789abcdef") != "" {
|
||||
t.Fatalf("manifest shape broken: %q", manifest)
|
||||
}
|
||||
winner, err := store.GetReleaseChangeSet(ctx, winnerID)
|
||||
if err != nil || len(winner.Items) == 0 {
|
||||
t.Fatalf("winner detail = %#v err=%v", winner, err)
|
||||
}
|
||||
if ComputeReleaseChangeSetManifest(winner.Items) != manifest {
|
||||
t.Fatal("stored manifest does not recompute from stored items")
|
||||
}
|
||||
|
||||
// Page listing.
|
||||
page, err := store.ListReleaseChangeSets(ctx, governance.ReleaseBatchFilter{Limit: 10})
|
||||
if err != nil || page.Total < 1 {
|
||||
t.Fatalf("page = %#v err = %v", page, err)
|
||||
}
|
||||
if _, err := store.LatestReleaseChangeSet(ctx); err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if _, err := store.GetReleaseChangeSet(ctx, "rcs-missing"); !errors.Is(err, ErrReleaseChangeSetNotFound) {
|
||||
t.Fatalf("missing detail err = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
// TestR4RollbackLevels freezes docs plan R4: publish failures grade into
|
||||
// single_item/batch, rollbacks walk rolling_back -> rolled_back /
|
||||
// rollback_failed, a failed rollback escalates the prior publish to overall,
|
||||
// and an overall-graded publish can no longer be compensated by plain
|
||||
// rollback authorization (snapshot restore is R2 territory and must be gated).
|
||||
func TestR4RollbackLevels(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC)
|
||||
// The shared review fixture binds to conn-manual and packs its URI hash via
|
||||
// repeat(<hex>, 64), so indexes must stay single-hex-digit.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs := []string{"tp-r4-a", "tp-r4-b"}
|
||||
for index, publicationID := range refs {
|
||||
insertReviewFixture(t, ctx, pool, index+1, publicationID, now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次", CreatedBy: "engineer-1", PublicationIDs: refs, Now: now.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
|
||||
if err != nil || len(plan) != 2 {
|
||||
t.Fatalf("publish plan=%#v err=%v", plan, err)
|
||||
}
|
||||
|
||||
// ---- single_item grade: first item fails permanently, nothing verified.
|
||||
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Second item also fails so FinishExecution can settle the execution.
|
||||
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var level string
|
||||
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if level != "single_item" {
|
||||
t.Fatalf("zero-progress failure must grade single_item, got %q", level)
|
||||
}
|
||||
// Nothing was ever verified: auto rollback is a no-op (idempotent, no rows).
|
||||
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rollbackRows int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_executions WHERE prior_execution_id=$1`, publish.ID).Scan(&rollbackRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rollbackRows != 0 {
|
||||
t.Fatalf("single_item with zero verified must schedule no rollback, got %d", rollbackRows)
|
||||
}
|
||||
|
||||
// ---- batch grade: reset items, retry, verify one, fail the other.
|
||||
// (The job is still pending in this store-level fixture; a real worker run
|
||||
// would have parked it failed. Retry requires a terminal job, so we set it.)
|
||||
if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, publish.JobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: publish.ID, Now: now.Add(10 * time.Minute)})
|
||||
if err != nil || retried.State != "pending" {
|
||||
t.Fatalf("retry publish = %#v err=%v", retried, err)
|
||||
}
|
||||
plan, err = store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(11 * time.Minute)})
|
||||
if err != nil || len(plan) != 2 {
|
||||
t.Fatalf("retry plan=%#v err=%v", plan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(12 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(12 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, publish.ID, now.Add(13*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if level != "batch" {
|
||||
t.Fatalf("partial-progress failure must grade batch, got %q", level)
|
||||
}
|
||||
|
||||
// ---- batch rollback: rolling_back -> rolled_back with restart re-claim.
|
||||
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(14*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10})
|
||||
if err != nil || rollbacks.Total != 1 {
|
||||
t.Fatalf("rollbacks=%#v err=%v", rollbacks, err)
|
||||
}
|
||||
rollback := rollbacks.Items[0]
|
||||
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(15 * time.Minute)})
|
||||
if err != nil || len(rollbackPlan) != 1 {
|
||||
t.Fatalf("rollback plan=%#v err=%v", rollbackPlan, err)
|
||||
}
|
||||
// R4: a claimed rollback runs in the dedicated rolling_back stage.
|
||||
var stage string
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stage != "rolling_back" {
|
||||
t.Fatalf("rollback claim must enter rolling_back, got %q", stage)
|
||||
}
|
||||
// Worker restart mid-rollback: the same execution is re-claimable.
|
||||
if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)}); err != nil {
|
||||
t.Fatalf("re-claim after restart: %v", err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: rollbackPlan[0].PublicationID, BundleSHA256: "", PreviousProductionRevision: "prod-before", ProductionRevision: "restored-v0", Now: now.Add(17 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stage != "rolled_back" {
|
||||
t.Fatalf("successful compensation must land rolled_back, got %q", stage)
|
||||
}
|
||||
|
||||
// ---- second batch with another verified item so we can fail the rollback.
|
||||
refs2 := []string{"tp-r4-b"}
|
||||
batch2, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次2", CreatedBy: "engineer-1", PublicationIDs: refs2, DeletePublicationIDs: []string{}, Now: now.Add(19 * time.Minute)})
|
||||
if err == nil {
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch2.ID, SealedBy: "engineer-2", Now: now.Add(20 * time.Minute)}); err != nil {
|
||||
t.Fatalf("seal batch2: %v", err)
|
||||
}
|
||||
publish2, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(21 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("authorize batch2: %v", err)
|
||||
}
|
||||
plan2, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish2.ID, JobID: publish2.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(22 * time.Minute)})
|
||||
if err != nil || len(plan2) != 1 {
|
||||
t.Fatalf("publish2 plan=%#v err=%v", plan2, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish2.ID, PublicationID: plan2[0].PublicationID, BundleSHA256: strings.Repeat("c", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan2[0].ReleaseRevision, Now: now.Add(23 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Finish publish2 successfully so its item stays verified.
|
||||
if err := store.FinishExecution(ctx, publish2.ID, now.Add(24*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Authorize a rollback for the completed publish2 and make it fail.
|
||||
manual, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(25 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("authorize rollback: %v", err)
|
||||
}
|
||||
manualPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: manual.ID, JobID: manual.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(26 * time.Minute)})
|
||||
_ = manualPlan
|
||||
if err != nil {
|
||||
t.Fatalf("start manual rollback: %v", err)
|
||||
}
|
||||
// The compensated publication now has a newer publication row; fail the
|
||||
// rollback item permanently.
|
||||
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: manual.ID, PublicationID: plan2[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(27 * time.Minute)}); err != nil {
|
||||
t.Fatalf("fail rollback item: %v", err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, manual.ID, now.Add(28*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, manual.ID).Scan(&stage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stage != "rollback_failed" {
|
||||
t.Fatalf("failed rollback must land rollback_failed, got %q", stage)
|
||||
}
|
||||
// Escalation: the prior publish is now 'overall' (needs snapshot restore).
|
||||
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish2.ID).Scan(&level); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if level != "overall" {
|
||||
t.Fatalf("failed rollback must escalate prior publish to overall, got %q", level)
|
||||
}
|
||||
// Overall-graded publish can no longer be plain-rollback authorized.
|
||||
if _, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(29 * time.Minute)}); err == nil {
|
||||
t.Fatal("overall-graded publish must not accept plain rollback authorization")
|
||||
}
|
||||
// A permanently failed rollback stays retriable (re-runs compensation).
|
||||
if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, manual.JobID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retriedRollback, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: manual.ID, Now: now.Add(30 * time.Minute)})
|
||||
if err != nil || retriedRollback.State != "pending" {
|
||||
t.Fatalf("retry failed rollback = %#v err=%v", retriedRollback, err)
|
||||
}
|
||||
} else {
|
||||
// Batch creation legitimately refused only when tp-r4-b has no
|
||||
// publish-eligible state; in this fixture it should succeed.
|
||||
t.Fatalf("batch2 create: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
// TestR5ExecutionManifestSealAndMismatch freezes docs plan R5: finishing an
|
||||
// execution seals a deterministic manifest of what actually happened, and a
|
||||
// "success" that does not match the frozen release change set intent is
|
||||
// downgraded to failed — never reported as completed.
|
||||
func TestR5ExecutionManifestSealAndMismatch(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, time.September, 12, 13, 0, 0, 0, time.UTC)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertReviewFixture(t, ctx, pool, 1, "tp-r5-a", now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
runExecution := func(publicationID, tag string) (executionID, jobID string) {
|
||||
t.Helper()
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R5批次-" + tag, CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
|
||||
if err != nil || len(plan) != 1 {
|
||||
t.Fatalf("plan=%#v err=%v", plan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("d", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return publish.ID, publish.JobID
|
||||
}
|
||||
|
||||
// Happy path (unbound batch): manifest is sealed, deterministic, non-empty.
|
||||
executionID, _ := runExecution("tp-r5-a", "happy")
|
||||
if err := store.FinishExecution(ctx, executionID, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sealed, state, publicationID, bundleSHA, revision string
|
||||
if err := pool.QueryRow(ctx, `SELECT e.manifest_sha256, e.state, i.publication_id, i.bundle_sha256, i.production_revision
|
||||
FROM release_executions e JOIN release_item_executions i ON i.release_execution_id=e.id WHERE e.id=$1`, executionID).Scan(&sealed, &state, &publicationID, &bundleSHA, &revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != "completed" {
|
||||
t.Fatalf("unbound success must stay completed, got %q", state)
|
||||
}
|
||||
if len(sealed) != 64 || sealed != ComputeReleaseExecutionManifest("publish", "", []string{publicationID + "|" + bundleSHA + "|" + revision}) {
|
||||
t.Fatalf("manifest not sealed/deterministic: %q", sealed)
|
||||
}
|
||||
// Re-seal must be a no-op (immutable once set).
|
||||
if err := store.FinishExecution(ctx, executionID, now.Add(8*time.Minute)); err == nil {
|
||||
t.Fatal("settled execution must not finish twice")
|
||||
}
|
||||
|
||||
// Mismatch path: simulate a tampered/legacy row where the bound frozen
|
||||
// change set does NOT contain the publication that was "successfully"
|
||||
// released — R5 forbids reporting success.
|
||||
insertReviewFixture(t, ctx, pool, 2, "tp-r5-b", now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-b", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondExecution, jobID := runExecution("tp-r5-b", "mismatch")
|
||||
executionID = secondExecution
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO release_change_sets(id,name,number,state,manifest_sha256,created_by,acceptance_cutoff) VALUES ('rcs-r5-x','不相干计划集',99,'confirmed',repeat('a',64),'tamper',$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES ('rcs-r5-x',1,'tp-other','别人','internal','add','ready','absent',true,'')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE release_executions SET release_change_set_id='rcs-r5-x' WHERE id=$1`, executionID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reset to running so FinishExecution can settle again for this scenario.
|
||||
if _, err := pool.Exec(ctx, `UPDATE release_executions SET state='running', manifest_sha256='', completed_at=NULL WHERE id=$1`, executionID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, executionID, now.Add(9*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT state, error_code, manifest_sha256 FROM release_executions WHERE id=$1`, executionID).Scan(&state, &bundleSHA, &sealed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state != "failed" || bundleSHA != "release_manifest_mismatch" {
|
||||
t.Fatalf("mismatch must downgrade to failed, got state=%q code=%q", state, bundleSHA)
|
||||
}
|
||||
if len(sealed) != 64 {
|
||||
t.Fatalf("mismatched execution must still seal its actual manifest, got %q", sealed)
|
||||
}
|
||||
_ = jobID
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
// TestR6ReleaseAuditChain freezes docs plan R6: one execution traces through
|
||||
// real FK rows to batch -> confirmed release change set -> bound snapshots and
|
||||
// catalog change sets -> per-item review/acceptance/source identity, and the
|
||||
// linked evidence cannot be deleted underneath the chain.
|
||||
func TestR6ReleaseAuditChain(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, time.September, 12, 11, 0, 0, 0, time.UTC)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual');
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES
|
||||
('snap-r6','conn-manual',repeat('e',64),'R6快照',3,2,1,now() - interval '2 hours');
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES
|
||||
('conn-manual','snap-r6',now() - interval '2 hours');
|
||||
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,confirmed_at) VALUES
|
||||
('cs-r6','conn-manual',NULL,'snap-r6','confirmed',1,0,0,0,now() - interval '90 minutes');`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertReviewFixture(t, ctx, pool, 1, "tp-r6-a", now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r6-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
set, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "R6发布", CreatedBy: "yaozhanghao", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: set.ID, ConfirmedBy: "boss", Now: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R6批次", CreatedBy: "yaozhanghao", PublicationIDs: []string{"tp-r6-a"}, ReleaseChangeSetID: set.ID, Now: now.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(2 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(3 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(4 * time.Minute)})
|
||||
if err != nil || len(plan) != 1 {
|
||||
t.Fatalf("plan=%#v err=%v", plan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(5 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, publish.ID, now.Add(6*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
chain, err := store.GetReleaseAuditChain(ctx, publish.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chain.ExecutionID != publish.ID || chain.Operation != "publish" || chain.State != "completed" {
|
||||
t.Fatalf("head = %#v", chain)
|
||||
}
|
||||
if chain.BatchID != batch.ID || chain.BatchName != "R6批次" || chain.CreatedBy != "yaozhanghao" || chain.SealedBy != "engineer-2" || chain.AuthorizedBy != "release-owner" {
|
||||
t.Fatalf("batch links = %#v", chain)
|
||||
}
|
||||
if chain.ReleaseChangeSetID != set.ID || chain.ChangeSetName != "R6发布" || chain.ManifestSHA256 != set.ManifestSHA256 || chain.ChangeSetConfirmedBy != "boss" {
|
||||
t.Fatalf("change set links = %#v", chain)
|
||||
}
|
||||
if len(chain.Snapshots) != 1 || chain.Snapshots[0].ID != "snap-r6" || chain.Snapshots[0].SourceName != "Manual" || chain.Snapshots[0].FileCount != 1 {
|
||||
t.Fatalf("snapshot links = %#v", chain.Snapshots)
|
||||
}
|
||||
if len(chain.CatalogChangeSets) != 1 || chain.CatalogChangeSets[0].ID != "cs-r6" || chain.CatalogChangeSets[0].State != "confirmed" {
|
||||
t.Fatalf("catalog links = %#v", chain.CatalogChangeSets)
|
||||
}
|
||||
if len(chain.ManifestSHA256) != 64 {
|
||||
t.Fatalf("manifest = %q", chain.ManifestSHA256)
|
||||
}
|
||||
if len(chain.Items) != 1 {
|
||||
t.Fatalf("items = %#v", chain.Items)
|
||||
}
|
||||
item := chain.Items[0]
|
||||
if item.PublicationID != "tp-r6-a" || item.Action != "upsert" || item.ItemState != "verified" || item.ProductionRevision != plan[0].ReleaseRevision {
|
||||
t.Fatalf("item = %#v", item)
|
||||
}
|
||||
if item.ReviewID == "" || item.AcceptanceRunID == "" || item.SourceItemID == "" || item.SourceVersionID == "" {
|
||||
t.Fatalf("item evidence missing: %#v", item)
|
||||
}
|
||||
|
||||
// R6 integrity: FK rows protect the evidence — the bound snapshot and the
|
||||
// confirmed release change set cannot be deleted while the chain exists.
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM source_catalog_snapshots WHERE id='snap-r6'`); err == nil {
|
||||
t.Fatal("snapshot delete must be RESTRICTed by the audit chain")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM release_change_sets WHERE id=$1`, set.ID); err == nil {
|
||||
t.Fatal("release change set delete must be RESTRICTed by batch/execution bindings")
|
||||
}
|
||||
// The catalog chain row links are real FK rows, not just JSONB.
|
||||
var linkRows int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_set_snapshot_links WHERE change_set_id=$1`, set.ID).Scan(&linkRows); err != nil || linkRows != 1 {
|
||||
t.Fatalf("snapshot link rows = %d err=%v", linkRows, err)
|
||||
}
|
||||
|
||||
// Rollback chain: authorize a rollback of the completed publish and see it
|
||||
// appear on the original execution's projection.
|
||||
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(7 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publishChain, err := store.GetReleaseAuditChain(ctx, publish.ID)
|
||||
if err != nil || publishChain.RollbackExecutionID != rollback.ID {
|
||||
t.Fatalf("publish chain rollback = %#v err=%v", publishChain, err)
|
||||
}
|
||||
rollbackChain, err := store.GetReleaseAuditChain(ctx, rollback.ID)
|
||||
if err != nil || rollbackChain.PriorExecutionID != publish.ID || rollbackChain.ReleaseChangeSetID != set.ID {
|
||||
t.Fatalf("rollback chain = %#v err=%v", rollbackChain, err)
|
||||
}
|
||||
if _, err := store.GetReleaseAuditChain(ctx, "rx-unknown"); err != ErrReleaseAuditChainNotFound {
|
||||
t.Fatalf("unknown chain err = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// ErrReleaseAuditChainNotFound is returned for an unknown execution id.
|
||||
var ErrReleaseAuditChainNotFound = errors.New("release audit chain not found")
|
||||
|
||||
const releaseAuditChainHeadSQL = `
|
||||
SELECT e.id, e.operation, e.state, e.failure_level,
|
||||
COALESCE(e.release_batch_id,''), COALESCE(b.name,''), COALESCE(b.created_by,''), COALESCE(b.sealed_by,''),
|
||||
COALESCE(e.authorized_by,''), COALESCE(e.release_change_set_id::text,''),
|
||||
COALESCE(rcs.name,''), COALESCE(rcs.number,0), COALESCE(rcs.state,''),
|
||||
COALESCE(rcs.created_by,''), COALESCE(rcs.confirmed_by,''), rcs.confirmed_at,
|
||||
COALESCE(rcs.manifest_sha256,''), COALESCE(e.prior_execution_id,''),
|
||||
COALESCE((SELECT rb.id FROM release_executions AS rb
|
||||
WHERE rb.prior_execution_id = e.id AND rb.operation = 'rollback'
|
||||
ORDER BY rb.created_at DESC LIMIT 1),'')
|
||||
FROM release_executions AS e
|
||||
LEFT JOIN release_batches AS b ON b.id = e.release_batch_id
|
||||
LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id
|
||||
WHERE e.id = $1
|
||||
`
|
||||
|
||||
const releaseAuditSnapshotLinksSQL = `
|
||||
SELECT snap.id, COALESCE(conn.name,''), snap.node_count, snap.file_count, snap.observed_at
|
||||
FROM release_change_set_snapshot_links AS link
|
||||
JOIN source_catalog_snapshots AS snap ON snap.id = link.snapshot_id
|
||||
LEFT JOIN source_connections AS conn ON conn.id = snap.source_connection_id
|
||||
WHERE link.change_set_id = $1
|
||||
ORDER BY snap.id
|
||||
`
|
||||
|
||||
const releaseAuditCatalogLinksSQL = `
|
||||
SELECT cset.id, cset.state, cset.add_count, cset.update_count,
|
||||
cset.move_count, cset.delete_count
|
||||
FROM release_change_set_catalog_links AS link
|
||||
JOIN source_catalog_change_sets AS cset ON cset.id = link.catalog_change_set_id
|
||||
WHERE link.change_set_id = $1
|
||||
ORDER BY cset.id
|
||||
`
|
||||
|
||||
const releaseAuditItemsSQL = `
|
||||
SELECT COALESCE(batch_item.publication_id, item.publication_id),
|
||||
COALESCE(version_title.title, ''),
|
||||
COALESCE(item.action, batch_item.action),
|
||||
COALESCE(item.state, 'pending'),
|
||||
COALESCE(batch_item.review_id, ''),
|
||||
COALESCE(batch_item.acceptance_run_id, ''),
|
||||
COALESCE(version.source_item_id, ''),
|
||||
COALESCE(batch_item.source_item_version_id, ''),
|
||||
COALESCE(item.production_revision, ''),
|
||||
COALESCE(item.error_code, '')
|
||||
FROM release_item_executions AS item
|
||||
JOIN test_publications AS publication ON publication.id = item.publication_id
|
||||
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
|
||||
JOIN source_items AS version_title ON version_title.id = version.source_item_id
|
||||
LEFT JOIN release_batch_items AS batch_item
|
||||
ON batch_item.release_batch_id = $2 AND batch_item.publication_id = item.publication_id
|
||||
WHERE item.release_execution_id = $1
|
||||
ORDER BY item.id
|
||||
`
|
||||
|
||||
// GetReleaseAuditChain projects the complete R6 evidence chain for one
|
||||
// execution: execution -> batch -> release change set -> bound source
|
||||
// snapshots and catalog change sets -> per-item review/acceptance/source
|
||||
// identity, plus the compensating rollback row when one exists. Every link is
|
||||
// read through real foreign keys (migration 30), not denormalized JSON. Only
|
||||
// safe identifiers, labels, counts and timestamps are selected — never bundle
|
||||
// paths, credentials, base URLs, or signed object URLs.
|
||||
func (s *Store) GetReleaseAuditChain(ctx context.Context, executionID string) (governance.ReleaseAuditChain, error) {
|
||||
if !validShortID(executionID) {
|
||||
return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound
|
||||
}
|
||||
var chain governance.ReleaseAuditChain
|
||||
err := s.db.QueryRow(ctx, releaseAuditChainHeadSQL, executionID).Scan(
|
||||
&chain.ExecutionID, &chain.Operation, &chain.State, &chain.FailureLevel,
|
||||
&chain.BatchID, &chain.BatchName, &chain.CreatedBy, &chain.SealedBy,
|
||||
&chain.AuthorizedBy, &chain.ReleaseChangeSetID,
|
||||
&chain.ChangeSetName, &chain.ChangeSetNumber, &chain.ChangeSetState,
|
||||
&chain.ChangeSetCreatedBy, &chain.ChangeSetConfirmedBy, &chain.ChangeSetConfirmedAt,
|
||||
&chain.ManifestSHA256, &chain.PriorExecutionID,
|
||||
&chain.RollbackExecutionID,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("load release audit chain: %w", err)
|
||||
}
|
||||
chain.Snapshots = []governance.AuditSnapshot{}
|
||||
chain.CatalogChangeSets = []governance.AuditCatalogSet{}
|
||||
chain.Items = []governance.AuditChainItem{}
|
||||
if chain.ReleaseChangeSetID != "" {
|
||||
rows, err := s.db.Query(ctx, releaseAuditSnapshotLinksSQL, chain.ReleaseChangeSetID)
|
||||
if err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("list release snapshot links: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var snap governance.AuditSnapshot
|
||||
if err := rows.Scan(&snap.ID, &snap.SourceName, &snap.NodeCount, &snap.FileCount, &snap.ObservedAt); err != nil {
|
||||
rows.Close()
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release snapshot link: %w", err)
|
||||
}
|
||||
chain.Snapshots = append(chain.Snapshots, snap)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release snapshot links: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
rows, err = s.db.Query(ctx, releaseAuditCatalogLinksSQL, chain.ReleaseChangeSetID)
|
||||
if err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("list release catalog links: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var set governance.AuditCatalogSet
|
||||
if err := rows.Scan(&set.ID, &set.State, &set.AddCount, &set.UpdateCount, &set.MoveCount, &set.DeleteCount); err != nil {
|
||||
rows.Close()
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release catalog link: %w", err)
|
||||
}
|
||||
chain.CatalogChangeSets = append(chain.CatalogChangeSets, set)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release catalog links: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
rows, err := s.db.Query(ctx, releaseAuditItemsSQL, executionID, chain.BatchID)
|
||||
if err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("list release audit items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.AuditChainItem
|
||||
if err := rows.Scan(&item.PublicationID, &item.Title, &item.Action, &item.ItemState,
|
||||
&item.ReviewID, &item.AcceptanceRunID, &item.SourceItemID, &item.SourceVersionID,
|
||||
&item.ProductionRevision, &item.ErrorCode); err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release audit item: %w", err)
|
||||
}
|
||||
chain.Items = append(chain.Items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release audit items: %w", err)
|
||||
}
|
||||
return chain, nil
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
func TestReleaseAuthorizationPersistentIntegration(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertReviewFixture(t, ctx, pool, 10, "tp-auth", now)
|
||||
review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{
|
||||
PublicationID: "tp-auth", Decision: governance.ReviewApproved,
|
||||
DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1",
|
||||
DecisionCode: "approved_for_release", Now: now.Add(time.Minute),
|
||||
})
|
||||
if err != nil || review.ID == "" {
|
||||
t.Fatalf("review=%#v error=%v", review, err)
|
||||
}
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
|
||||
Name: "Authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auth"}, Now: now.Add(2 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := governance.AuthorizeReleaseBatchRequest{
|
||||
BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute),
|
||||
}
|
||||
first, err := store.AuthorizeReleaseBatch(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := store.AuthorizeReleaseBatch(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.ID != second.ID || first.JobID != second.JobID || first.ItemCount != 1 {
|
||||
t.Fatalf("first=%#v second=%#v", first, second)
|
||||
}
|
||||
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{
|
||||
ExecutionID: first.ID, JobID: first.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute),
|
||||
})
|
||||
if err != nil || len(plan) != 1 || plan[0].PublicationID != "tp-auth" {
|
||||
t.Fatalf("publish plan=%#v error=%v", plan, err)
|
||||
}
|
||||
if plan[0].ReleaseExecutionID != first.ID || plan[0].SourceRevision != "revision-10" || plan[0].ParseRevision != "parser-v1@index-v1" || plan[0].ReleaseRevision != review.ID {
|
||||
t.Fatalf("publish revisions=%#v", plan[0])
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{
|
||||
ExecutionID: first.ID, PublicationID: "tp-auth", BundleSHA256: strings.Repeat("a", 64),
|
||||
PreviousProductionRevision: "prod-v0", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, first.ID, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
currentPlan, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(currentPlan.Items) != 0 {
|
||||
t.Fatalf("current release plan=%#v error=%v", currentPlan, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-10'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unavailablePlan, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(unavailablePlan.Items) != 1 || unavailablePlan.Items[0].Action != "blocked" || unavailablePlan.Items[0].CanRelease || unavailablePlan.Items[0].Blocker != "source_unavailable" {
|
||||
t.Fatalf("unavailable release plan=%#v error=%v", unavailablePlan, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-10'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-10','blocked','engineer-1','禁止进入生产',$1,$1)`, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deletePlan, err := store.ListReleasePlan(ctx)
|
||||
if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != "tp-auth" || deletePlan.Items[0].Blocker != "" {
|
||||
t.Fatalf("delete release plan=%#v error=%v", deletePlan, err)
|
||||
}
|
||||
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{
|
||||
PriorExecutionID: first.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(8 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{
|
||||
ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(9 * time.Minute),
|
||||
})
|
||||
if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PreviousProductionRevision != "prod-v0" {
|
||||
t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{
|
||||
ExecutionID: rollback.ID, PublicationID: "tp-auth", ProductionRevision: "prod-v0", Now: now.Add(10 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, rollback.ID, now.Add(11*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mutation := range []struct {
|
||||
sql string
|
||||
args []any
|
||||
}{
|
||||
{`UPDATE release_item_executions SET state='failed', error_code='release_verification_failed', completed_at=$2 WHERE release_execution_id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}},
|
||||
{`UPDATE release_executions SET state='failed', completed_item_count=0, failed_item_count=1, error_code='release_item_failed', completed_at=$2 WHERE id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}},
|
||||
{`UPDATE jobs SET state='completed', attempt=3, completed_at=$2 WHERE id=$1`, []any{rollback.JobID, now.Add(12 * time.Minute)}},
|
||||
} {
|
||||
if _, err := pool.Exec(ctx, mutation.sql, mutation.args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: rollback.ID, Now: now.Add(13 * time.Minute)})
|
||||
if err != nil || retried.State != governance.ReleaseExecutionPending || retried.FailedItemCount != 0 {
|
||||
t.Fatalf("retried=%#v error=%v", retried, err)
|
||||
}
|
||||
var executionState, itemState, jobState string
|
||||
var attempt int
|
||||
if err := pool.QueryRow(ctx, `SELECT execution.state, item.state, job.state, job.attempt FROM release_executions execution JOIN release_item_executions item ON item.release_execution_id=execution.id JOIN jobs job ON job.id=execution.job_id WHERE execution.id=$1`, rollback.ID).Scan(&executionState, &itemState, &jobState, &attempt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executionState != "pending" || itemState != "pending" || jobState != "pending" || attempt != 0 {
|
||||
t.Fatalf("states=%s/%s/%s attempt=%d", executionState, itemState, jobState, attempt)
|
||||
}
|
||||
|
||||
insertReviewFixture(t, ctx, pool, 11, "tp-stale-auth", now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{
|
||||
PublicationID: "tp-stale-auth", Decision: governance.ReviewApproved,
|
||||
DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1",
|
||||
DecisionCode: "approved_for_release", Now: now.Add(5 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
staleBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
|
||||
Name: "Stale authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-stale-auth"}, Now: now.Add(6 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: staleBatch.ID, SealedBy: "engineer-2", Now: now.Add(7 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE test_publications SET updated_at = updated_at + INTERVAL '1 second' WHERE id = 'tp-stale-auth'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{
|
||||
BatchID: staleBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(8 * time.Minute),
|
||||
})
|
||||
if !errors.Is(err, ErrReleaseAuthorizationGateConflict) {
|
||||
t.Fatalf("stale authorization error=%v", err)
|
||||
}
|
||||
|
||||
var executions, jobs, itemExecutions, completed, rolledBack, pending int
|
||||
if err := pool.QueryRow(ctx, `SELECT
|
||||
(SELECT COUNT(*)::INTEGER FROM release_executions),
|
||||
(SELECT COUNT(*)::INTEGER FROM jobs WHERE type = 'release.publish'),
|
||||
(SELECT COUNT(*)::INTEGER FROM release_item_executions),
|
||||
(SELECT COUNT(*)::INTEGER FROM release_executions WHERE state = 'completed'),
|
||||
(SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'rolled_back'),
|
||||
(SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'pending')`).Scan(&executions, &jobs, &itemExecutions, &completed, &rolledBack, &pending); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if executions != 2 || jobs != 1 || itemExecutions != 2 || completed != 1 || rolledBack != 0 || pending != 1 {
|
||||
t.Fatalf("executions=%d publishJobs=%d itemExecutions=%d completed=%d rolledBack=%d pending=%d", executions, jobs, itemExecutions, completed, rolledBack, pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedPublishSchedulesIdempotentRollbackForVerifiedItemsOnly(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 10, 3, 0, 0, 0, time.UTC)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index, publicationID := range []string{"tp-auto-a", "tp-auto-b"} {
|
||||
insertReviewFixture(t, ctx, pool, 8+index, publicationID, now)
|
||||
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Automatic rollback batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auto-a", "tp-auto-b"}, Now: now.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
|
||||
if err != nil || len(plan) != 2 {
|
||||
t.Fatalf("publish plan=%#v error=%v", plan, err)
|
||||
}
|
||||
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10})
|
||||
if err != nil || rollbacks.Total != 1 || len(rollbacks.Items) != 1 || rollbacks.Items[0].ItemCount != 1 || rollbacks.Items[0].AuthorizationCode != "automatic_rollback_after_failure" {
|
||||
t.Fatalf("rollbacks=%#v error=%v", rollbacks, err)
|
||||
}
|
||||
rollback := rollbacks.Items[0]
|
||||
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(10 * time.Minute)})
|
||||
if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PublicationID != plan[0].PublicationID || rollbackPlan[0].PreviousProductionRevision != "prod-before" {
|
||||
t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
const releaseGatePolicyVersion = "release-gate-v1"
|
||||
|
||||
var (
|
||||
ErrInvalidReleaseBatch = errors.New("invalid release batch")
|
||||
ErrInvalidReleaseBatchFilter = errors.New("invalid release batch filter")
|
||||
ErrReleaseBatchNotEligible = errors.New("release batch contains ineligible publications")
|
||||
ErrReleaseBatchGateConflict = errors.New("release batch gate conflict")
|
||||
ErrReleaseBatchNotFound = errors.New("release batch not found")
|
||||
ErrReleaseChangeSetUnconfirmed = errors.New("release change set must be confirmed before batching")
|
||||
)
|
||||
|
||||
const releaseGateCTEs = `
|
||||
acceptance_run_summary AS (
|
||||
SELECT publication_id, run_id, MAX(observed_at) AS observed_at,
|
||||
COALESCE(MAX(state) FILTER (WHERE check_type = 'knowledge_exists'), '') AS knowledge_exists,
|
||||
COALESCE(MAX(state) FILTER (WHERE check_type = 'chunks_present'), '') AS chunks_present,
|
||||
COALESCE(MAX(state) FILTER (WHERE check_type = 'retrieval_hit'), '') AS retrieval_hit
|
||||
FROM acceptance_results
|
||||
GROUP BY publication_id, run_id
|
||||
), latest_acceptance AS (
|
||||
SELECT DISTINCT ON (publication_id)
|
||||
publication_id, run_id, knowledge_exists, chunks_present, retrieval_hit, observed_at
|
||||
FROM acceptance_run_summary
|
||||
ORDER BY publication_id, observed_at DESC, run_id DESC
|
||||
), latest_review AS (
|
||||
SELECT DISTINCT ON (publication_id)
|
||||
id, publication_id, decision, decided_visibility, acceptance_run_id,
|
||||
publication_updated_at, created_at
|
||||
FROM publication_reviews
|
||||
ORDER BY publication_id, created_at DESC, id DESC
|
||||
), release_ready AS (
|
||||
SELECT publication.id AS publication_id,
|
||||
publication.source_item_version_id,
|
||||
publication.updated_at AS publication_updated_at,
|
||||
review.id AS review_id,
|
||||
review.acceptance_run_id,
|
||||
review.decided_visibility
|
||||
FROM test_publications AS publication
|
||||
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
|
||||
JOIN source_items AS source ON source.id = version.source_item_id
|
||||
JOIN latest_acceptance AS acceptance ON acceptance.publication_id = publication.id
|
||||
JOIN publication_reviews AS review ON review.id = (
|
||||
SELECT latest_review.id FROM latest_review WHERE latest_review.publication_id = publication.id
|
||||
)
|
||||
WHERE review.decision = 'approved'
|
||||
AND review.acceptance_run_id = acceptance.run_id
|
||||
AND review.publication_updated_at = publication.updated_at
|
||||
AND publication.ingest_state = 'uploaded'
|
||||
AND publication.processing_state = 'completed'
|
||||
AND publication.retrieval_state = 'unpublished'
|
||||
AND version.revision = source.revision
|
||||
AND source.source_change NOT IN ('deleted', 'unavailable')
|
||||
AND acceptance.knowledge_exists = 'passed'
|
||||
AND acceptance.chunks_present = 'passed'
|
||||
AND acceptance.retrieval_hit IN ('passed', 'manual_required')
|
||||
)
|
||||
`
|
||||
|
||||
const releaseDeleteReadyCTEs = `
|
||||
source_truth AS (
|
||||
SELECT source.id, source.source_change,
|
||||
COALESCE(item_rule.visibility, directory_rule.visibility, source.visibility) AS desired_policy
|
||||
FROM source_items AS source
|
||||
LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT rule.visibility
|
||||
FROM directory_scope_rules AS rule
|
||||
WHERE rule.source_connection_id = source.source_connection_id
|
||||
AND (rule.directory_path = '' OR source.normalized_path = rule.directory_path OR source.normalized_path LIKE rule.directory_path || '/%')
|
||||
ORDER BY LENGTH(rule.directory_path) DESC, rule.directory_path DESC
|
||||
LIMIT 1
|
||||
) AS directory_rule ON TRUE
|
||||
), latest_production AS (
|
||||
SELECT DISTINCT ON (version.source_item_id, publication.target_scope)
|
||||
publication.id AS publication_id, version.source_item_id, publication.target_scope,
|
||||
execution_item.production_revision,
|
||||
batch_item.review_id, batch_item.source_item_version_id,
|
||||
batch_item.acceptance_run_id, batch_item.publication_updated_at,
|
||||
batch_item.decided_visibility
|
||||
FROM release_item_executions AS execution_item
|
||||
JOIN release_executions AS execution ON execution.id = execution_item.release_execution_id
|
||||
JOIN test_publications AS publication ON publication.id = execution_item.publication_id
|
||||
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
|
||||
JOIN release_batch_items AS batch_item
|
||||
ON batch_item.release_batch_id = execution.release_batch_id
|
||||
AND batch_item.publication_id = publication.id
|
||||
WHERE execution.state IN ('completed', 'rolled_back')
|
||||
AND execution_item.state IN ('verified', 'rolled_back')
|
||||
ORDER BY version.source_item_id, publication.target_scope,
|
||||
execution.completed_at DESC, execution_item.completed_at DESC, execution_item.id DESC
|
||||
), release_delete_ready AS (
|
||||
SELECT production.publication_id, production.source_item_version_id,
|
||||
production.publication_updated_at, production.review_id,
|
||||
production.acceptance_run_id, production.decided_visibility,
|
||||
'delete' AS action
|
||||
FROM latest_production AS production
|
||||
JOIN source_truth AS truth ON truth.id = production.source_item_id
|
||||
WHERE production.production_revision <> 'absent'
|
||||
AND truth.source_change <> 'unavailable'
|
||||
AND (
|
||||
truth.source_change = 'deleted'
|
||||
OR truth.desired_policy = 'blocked'
|
||||
OR (production.target_scope = 'internal' AND truth.desired_policy NOT IN ('internal', 'both'))
|
||||
OR (production.target_scope = 'external' AND truth.desired_policy NOT IN ('external', 'both'))
|
||||
)
|
||||
), release_upsert_ready AS (
|
||||
SELECT ready.*
|
||||
FROM release_ready AS ready
|
||||
JOIN test_publications AS publication ON publication.id = ready.publication_id
|
||||
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
|
||||
JOIN source_truth AS truth ON truth.id = version.source_item_id
|
||||
LEFT JOIN latest_production AS production
|
||||
ON production.source_item_id = version.source_item_id
|
||||
AND production.target_scope = publication.target_scope
|
||||
WHERE truth.source_change NOT IN ('deleted', 'unavailable')
|
||||
AND truth.desired_policy <> 'blocked'
|
||||
AND publication.target_scope = ready.decided_visibility
|
||||
AND (
|
||||
(publication.target_scope = 'internal' AND truth.desired_policy IN ('internal', 'both'))
|
||||
OR (publication.target_scope = 'external' AND truth.desired_policy IN ('external', 'both'))
|
||||
)
|
||||
AND (production.production_revision IS NULL OR production.production_revision = 'absent' OR production.production_revision <> ready.review_id)
|
||||
)
|
||||
`
|
||||
|
||||
const createReleaseBatchSQL = `
|
||||
WITH requested AS (
|
||||
SELECT value AS publication_id, 'upsert' AS action
|
||||
FROM jsonb_array_elements_text($5::jsonb)
|
||||
UNION ALL
|
||||
SELECT value AS publication_id, 'delete' AS action
|
||||
FROM jsonb_array_elements_text($6::jsonb)
|
||||
), ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
|
||||
request_stats AS (
|
||||
SELECT COUNT(*)::INTEGER AS requested_count,
|
||||
COUNT(DISTINCT publication_id)::INTEGER AS distinct_count
|
||||
FROM requested
|
||||
), binding AS (
|
||||
-- R1: when a batch cites a release change set, the set must be CONFIRMED
|
||||
-- and every requested publication must exist in its frozen manifest as a
|
||||
-- releasable item. Source refreshes can never move this goalpost.
|
||||
SELECT 1
|
||||
WHERE $7::text = '' OR EXISTS (
|
||||
SELECT 1 FROM release_change_sets rcs
|
||||
WHERE rcs.id = $7 AND rcs.state = 'confirmed'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM requested q
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM release_change_set_items frozen
|
||||
WHERE frozen.change_set_id = rcs.id
|
||||
AND frozen.publication_id = q.publication_id
|
||||
AND frozen.can_release
|
||||
)
|
||||
)
|
||||
)
|
||||
), eligible AS (
|
||||
SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at,
|
||||
ready.review_id, ready.acceptance_run_id, ready.decided_visibility,
|
||||
requested.action
|
||||
FROM release_upsert_ready AS ready
|
||||
JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'upsert'
|
||||
UNION ALL
|
||||
SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at,
|
||||
ready.review_id, ready.acceptance_run_id, ready.decided_visibility,
|
||||
ready.action
|
||||
FROM release_delete_ready AS ready
|
||||
JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'delete'
|
||||
), created_batch AS (
|
||||
INSERT INTO release_batches (
|
||||
id, name, state, policy_version, created_by, item_count, created_at, release_change_set_id
|
||||
)
|
||||
SELECT $1, $2, 'draft', 'release-gate-v1', $3, request_stats.requested_count, $4, NULLIF($7::text, '')
|
||||
FROM request_stats
|
||||
WHERE EXISTS (SELECT 1 FROM binding)
|
||||
AND request_stats.requested_count BETWEEN 1 AND 100
|
||||
AND request_stats.requested_count = request_stats.distinct_count
|
||||
AND request_stats.requested_count = (SELECT COUNT(*) FROM eligible)
|
||||
RETURNING id, name, state, policy_version, created_by, sealed_by,
|
||||
item_count, created_at, sealed_at, release_change_set_id
|
||||
), inserted_items AS (
|
||||
INSERT INTO release_batch_items (
|
||||
release_batch_id, publication_id, review_id, source_item_version_id,
|
||||
acceptance_run_id, publication_updated_at, decided_visibility, action, created_at
|
||||
)
|
||||
SELECT created_batch.id, eligible.publication_id, eligible.review_id,
|
||||
eligible.source_item_version_id, eligible.acceptance_run_id,
|
||||
eligible.publication_updated_at, eligible.decided_visibility, eligible.action, $4
|
||||
FROM created_batch CROSS JOIN eligible
|
||||
RETURNING release_batch_id
|
||||
)
|
||||
SELECT id, name, state, policy_version, created_by, sealed_by,
|
||||
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
|
||||
FROM created_batch
|
||||
WHERE item_count = (SELECT COUNT(*) FROM inserted_items)
|
||||
`
|
||||
|
||||
const sealReleaseBatchSQL = `
|
||||
WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
|
||||
item_stats AS (
|
||||
SELECT batch.id AS batch_id,
|
||||
COUNT(item.publication_id)::INTEGER AS total_count,
|
||||
COUNT(*) FILTER (WHERE
|
||||
(item.action = 'upsert' AND ready.publication_id IS NOT NULL)
|
||||
OR (item.action = 'delete' AND delete_ready.publication_id IS NOT NULL)
|
||||
)::INTEGER AS valid_count
|
||||
FROM release_batches AS batch
|
||||
LEFT JOIN release_batch_items AS item ON item.release_batch_id = batch.id
|
||||
LEFT JOIN release_upsert_ready AS ready
|
||||
ON ready.publication_id = item.publication_id
|
||||
AND item.action = 'upsert'
|
||||
AND ready.review_id = item.review_id
|
||||
AND ready.acceptance_run_id = item.acceptance_run_id
|
||||
AND ready.publication_updated_at = item.publication_updated_at
|
||||
AND ready.decided_visibility = item.decided_visibility
|
||||
LEFT JOIN release_delete_ready AS delete_ready
|
||||
ON delete_ready.publication_id = item.publication_id
|
||||
AND item.action = 'delete'
|
||||
AND delete_ready.review_id = item.review_id
|
||||
AND delete_ready.acceptance_run_id = item.acceptance_run_id
|
||||
AND delete_ready.publication_updated_at = item.publication_updated_at
|
||||
AND delete_ready.decided_visibility = item.decided_visibility
|
||||
WHERE batch.id = $1 AND batch.state = 'draft'
|
||||
GROUP BY batch.id
|
||||
), sealed AS (
|
||||
UPDATE release_batches AS batch
|
||||
SET state = 'sealed', sealed_by = $2, sealed_at = $3
|
||||
FROM item_stats
|
||||
WHERE batch.id = item_stats.batch_id
|
||||
AND batch.policy_version = 'release-gate-v1'
|
||||
AND item_stats.total_count > 0
|
||||
AND item_stats.total_count = item_stats.valid_count
|
||||
AND batch.item_count = item_stats.total_count
|
||||
RETURNING batch.id, batch.name, batch.state, batch.policy_version,
|
||||
batch.created_by, batch.sealed_by, batch.item_count,
|
||||
batch.created_at, batch.sealed_at, COALESCE(batch.release_change_set_id::text,'')
|
||||
), already_sealed AS (
|
||||
SELECT id, name, state, policy_version, created_by, sealed_by,
|
||||
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
|
||||
FROM release_batches
|
||||
WHERE id = $1 AND state = 'sealed'
|
||||
)
|
||||
SELECT * FROM sealed
|
||||
UNION ALL
|
||||
SELECT * FROM already_sealed
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
const countReleaseBatchesSQL = `
|
||||
SELECT COUNT(*)::INTEGER
|
||||
FROM release_batches
|
||||
WHERE $1 = '' OR state = $1
|
||||
`
|
||||
|
||||
const listReleaseBatchesSQL = `
|
||||
SELECT id, name, state, policy_version, created_by, sealed_by,
|
||||
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
|
||||
FROM release_batches
|
||||
WHERE $1 = '' OR state = $1
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
`
|
||||
|
||||
const getReleaseBatchSQL = `
|
||||
SELECT id, name, state, policy_version, created_by, sealed_by,
|
||||
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
|
||||
FROM release_batches
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
const listReleaseBatchItemsSQL = `
|
||||
SELECT item.publication_id, source.title, item.decided_visibility, item.action
|
||||
FROM release_batch_items AS item
|
||||
JOIN test_publications AS publication ON publication.id = item.publication_id
|
||||
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
|
||||
JOIN source_items AS source ON source.id = version.source_item_id
|
||||
WHERE item.release_batch_id = $1
|
||||
ORDER BY item.publication_id
|
||||
`
|
||||
|
||||
func (s *Store) CreateReleaseBatch(ctx context.Context, request governance.CreateReleaseBatchRequest) (governance.ReleaseBatch, error) {
|
||||
if !validCreateReleaseBatch(request) {
|
||||
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
|
||||
}
|
||||
batchID, err := newOpaqueID("rb-")
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, err
|
||||
}
|
||||
publicationIDs, err := json.Marshal(append([]string{}, request.PublicationIDs...))
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
|
||||
}
|
||||
deletePublicationIDs, err := json.Marshal(append([]string{}, request.DeletePublicationIDs...))
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
|
||||
}
|
||||
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, createReleaseBatchSQL,
|
||||
batchID, request.Name, request.CreatedBy, request.Now, publicationIDs, deletePublicationIDs,
|
||||
strings.TrimSpace(request.ReleaseChangeSetID),
|
||||
))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// R1: distinguish an unconfirmed/non-covering change set from plain
|
||||
// gate ineligibility so operators get the honest reason.
|
||||
if request.ReleaseChangeSetID != "" {
|
||||
var state string
|
||||
if lookup := s.db.QueryRow(ctx, `SELECT state FROM release_change_sets WHERE id=$1`, request.ReleaseChangeSetID).Scan(&state); lookup != nil || state != "confirmed" {
|
||||
return governance.ReleaseBatch{}, ErrReleaseChangeSetUnconfirmed
|
||||
}
|
||||
}
|
||||
return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, fmt.Errorf("create release batch: %w", err)
|
||||
}
|
||||
return batch, nil
|
||||
}
|
||||
|
||||
func (s *Store) SealReleaseBatch(ctx context.Context, request governance.SealReleaseBatchRequest) (governance.ReleaseBatch, error) {
|
||||
if !validShortID(request.BatchID) || !validReviewerRef(request.SealedBy) || request.Now.IsZero() {
|
||||
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
|
||||
}
|
||||
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, sealReleaseBatchSQL, request.BatchID, request.SealedBy, request.Now))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ReleaseBatch{}, ErrReleaseBatchGateConflict
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseBatch{}, fmt.Errorf("seal release batch: %w", err)
|
||||
}
|
||||
return batch, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListReleaseBatches(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseBatchPage, error) {
|
||||
if filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 ||
|
||||
filter.State != "" && filter.State != string(governance.ReleaseBatchDraft) &&
|
||||
filter.State != string(governance.ReleaseBatchSealed) && filter.State != string(governance.ReleaseBatchCancelled) {
|
||||
return governance.ReleaseBatchPage{}, ErrInvalidReleaseBatchFilter
|
||||
}
|
||||
page := governance.ReleaseBatchPage{Items: []governance.ReleaseBatch{}, Limit: filter.Limit, Offset: filter.Offset}
|
||||
if err := s.db.QueryRow(ctx, countReleaseBatchesSQL, filter.State).Scan(&page.Total); err != nil {
|
||||
return governance.ReleaseBatchPage{}, fmt.Errorf("count release batches: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(ctx, listReleaseBatchesSQL, filter.State, filter.Limit, filter.Offset)
|
||||
if err != nil {
|
||||
return governance.ReleaseBatchPage{}, fmt.Errorf("list release batches: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
batch, err := scanReleaseBatch(rows)
|
||||
if err != nil {
|
||||
return governance.ReleaseBatchPage{}, fmt.Errorf("scan release batch: %w", err)
|
||||
}
|
||||
page.Items = append(page.Items, batch)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.ReleaseBatchPage{}, fmt.Errorf("iterate release batches: %w", err)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetReleaseBatch(ctx context.Context, batchID string) (governance.ReleaseBatchDetail, error) {
|
||||
if !validShortID(batchID) {
|
||||
return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound
|
||||
}
|
||||
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, getReleaseBatchSQL, batchID))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseBatchDetail{}, fmt.Errorf("get release batch: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(ctx, listReleaseBatchItemsSQL, batchID)
|
||||
if err != nil {
|
||||
return governance.ReleaseBatchDetail{}, fmt.Errorf("list release batch items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
detail := governance.ReleaseBatchDetail{ReleaseBatch: batch, Items: []governance.ReleaseBatchItemView{}}
|
||||
for rows.Next() {
|
||||
var item governance.ReleaseBatchItemView
|
||||
var visibility string
|
||||
if err := rows.Scan(&item.PublicationID, &item.Title, &visibility, &item.Action); err != nil {
|
||||
return governance.ReleaseBatchDetail{}, fmt.Errorf("scan release batch item: %w", err)
|
||||
}
|
||||
item.DecidedVisibility = governance.Visibility(visibility)
|
||||
if (item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) || (item.Action != "upsert" && item.Action != "delete") {
|
||||
return governance.ReleaseBatchDetail{}, errors.New("invalid release batch item visibility")
|
||||
}
|
||||
detail.Items = append(detail.Items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.ReleaseBatchDetail{}, fmt.Errorf("iterate release batch items: %w", err)
|
||||
}
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
func validCreateReleaseBatch(request governance.CreateReleaseBatchRequest) bool {
|
||||
if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 255 ||
|
||||
strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) ||
|
||||
request.Now.IsZero() || len(request.PublicationIDs)+len(request.DeletePublicationIDs) < 1 || len(request.PublicationIDs)+len(request.DeletePublicationIDs) > 100 {
|
||||
return false
|
||||
}
|
||||
if request.ReleaseChangeSetID != "" && !validRouteID(request.ReleaseChangeSetID) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(request.PublicationIDs)+len(request.DeletePublicationIDs))
|
||||
for _, publicationID := range append(append([]string{}, request.PublicationIDs...), request.DeletePublicationIDs...) {
|
||||
if !validShortID(publicationID) {
|
||||
return false
|
||||
}
|
||||
if _, exists := seen[publicationID]; exists {
|
||||
return false
|
||||
}
|
||||
seen[publicationID] = struct{}{}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func scanReleaseBatch(row rowScanner) (governance.ReleaseBatch, error) {
|
||||
var batch governance.ReleaseBatch
|
||||
var state string
|
||||
if err := row.Scan(
|
||||
&batch.ID, &batch.Name, &state, &batch.PolicyVersion,
|
||||
&batch.CreatedBy, &batch.SealedBy, &batch.ItemCount,
|
||||
&batch.CreatedAt, &batch.SealedAt, &batch.ReleaseChangeSetID,
|
||||
); err != nil {
|
||||
return governance.ReleaseBatch{}, err
|
||||
}
|
||||
batch.State = governance.ReleaseBatchState(state)
|
||||
if !batch.State.Valid() || batch.PolicyVersion != releaseGatePolicyVersion {
|
||||
return governance.ReleaseBatch{}, errors.New("invalid release batch row")
|
||||
}
|
||||
return batch, nil
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestCreateReleaseBatchAtomicallyLocksOnlyApprovedCurrentItems(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC)
|
||||
db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "draft", policy: "release-gate-v1", createdBy: "engineer-1", itemCount: 3, createdAt: now}}
|
||||
batch, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{
|
||||
Name: "Synthetic batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-b"}, DeletePublicationIDs: []string{"tp-c"}, Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateReleaseBatch() error = %v", err)
|
||||
}
|
||||
if batch.ID != "rb-safe" || batch.State != governance.ReleaseBatchDraft || batch.ItemCount != 3 {
|
||||
t.Fatalf("batch = %#v", batch)
|
||||
}
|
||||
sql := strings.ToLower(db.queryRowSQL)
|
||||
for _, required := range []string{
|
||||
"jsonb_array_elements_text", "publication_reviews", "decision = 'approved'",
|
||||
"acceptance_run_id", "publication_updated_at", "knowledge_exists = 'passed'",
|
||||
"chunks_present = 'passed'", "retrieval_hit in ('passed', 'manual_required')",
|
||||
"insert into release_batches", "insert into release_batch_items", "count(distinct publication_id)", "returning",
|
||||
"release_delete_ready", "'delete' as action", "eligible.action",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("create batch SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "insert into jobs") || strings.Contains(sql, "release.probe") {
|
||||
t.Fatal("P5 batch creation must not enqueue release jobs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateReleaseBatchRejectsInvalidOrDuplicateInput(t *testing.T) {
|
||||
now := time.Now()
|
||||
requests := []governance.CreateReleaseBatchRequest{
|
||||
{Name: "", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: now},
|
||||
{Name: "Batch", CreatedBy: "", PublicationIDs: []string{"tp-a"}, Now: now},
|
||||
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: nil, Now: now},
|
||||
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-a"}, Now: now},
|
||||
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, DeletePublicationIDs: []string{"tp-a"}, Now: now},
|
||||
}
|
||||
for _, request := range requests {
|
||||
if _, err := NewStore(&recordingDB{}).CreateReleaseBatch(context.Background(), request); !errors.Is(err, ErrInvalidReleaseBatch) {
|
||||
t.Fatalf("request=%#v error=%v, want ErrInvalidReleaseBatch", request, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateReleaseBatchReturnsNotEligibleWhenAnyItemFailsGate(t *testing.T) {
|
||||
db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}}
|
||||
_, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{
|
||||
Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: time.Now(),
|
||||
})
|
||||
if !errors.Is(err, ErrReleaseBatchNotEligible) {
|
||||
t.Fatalf("error=%v, want ErrReleaseBatchNotEligible", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealReleaseBatchRevalidatesEveryLockedItemWithoutEnqueue(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 9, 10, 0, 0, time.UTC)
|
||||
db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 2, createdAt: now.Add(-time.Minute), sealedAt: &now}}
|
||||
batch, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-2", Now: now})
|
||||
if err != nil {
|
||||
t.Fatalf("SealReleaseBatch() error = %v", err)
|
||||
}
|
||||
if batch.State != governance.ReleaseBatchSealed || batch.SealedBy != "engineer-2" || batch.SealedAt == nil {
|
||||
t.Fatalf("batch = %#v", batch)
|
||||
}
|
||||
sql := strings.ToLower(db.queryRowSQL)
|
||||
for _, required := range []string{
|
||||
"join release_batch_items", "join publication_reviews",
|
||||
"ready.review_id = item.review_id", "ready.acceptance_run_id = item.acceptance_run_id", "ready.publication_updated_at = item.publication_updated_at",
|
||||
"publication.ingest_state = 'uploaded'", "publication.processing_state = 'completed'",
|
||||
"publication.retrieval_state = 'unpublished'", "state = 'sealed'", "state = 'draft'",
|
||||
"release_delete_ready", "item.action = 'upsert'", "item.action = 'delete'",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Errorf("seal SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "insert into jobs") {
|
||||
t.Fatal("P5 seal must not enqueue a production release")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealReleaseBatchReturnsGateConflict(t *testing.T) {
|
||||
db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}}
|
||||
_, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-1", Now: time.Now()})
|
||||
if !errors.Is(err, ErrReleaseBatchGateConflict) {
|
||||
t.Fatalf("error=%v, want ErrReleaseBatchGateConflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListReleaseBatchesAndDetailReturnSafeProjection(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 9, 20, 0, 0, time.UTC)
|
||||
db := &recordingDB{
|
||||
row: releaseCountRow{count: 1},
|
||||
rows: &releaseBatchRows{values: []releaseBatchRow{{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now}}},
|
||||
}
|
||||
page, err := NewStore(db).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "sealed", Limit: 20})
|
||||
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].State != governance.ReleaseBatchSealed {
|
||||
t.Fatalf("page=%#v error=%v", page, err)
|
||||
}
|
||||
joined := strings.ToLower(db.queryRowSQL + "\n" + db.querySQL)
|
||||
if !strings.Contains(joined, "from release_batches") || !strings.Contains(joined, "state = $1") {
|
||||
t.Fatalf("batch list SQL=%s", joined)
|
||||
}
|
||||
|
||||
detailDB := &releaseBatchReadDB{
|
||||
recordingDB: &recordingDB{},
|
||||
row: releaseBatchRow{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now},
|
||||
rows: &releaseBatchItemRows{values: []releaseBatchItemRow{{publicationID: "tp-safe", title: "Synthetic", visibility: "internal", action: "delete"}}},
|
||||
}
|
||||
detail, err := NewStore(detailDB).GetReleaseBatch(context.Background(), "rb-safe")
|
||||
if err != nil || len(detail.Items) != 1 || detail.Items[0].Title != "Synthetic" || detail.Items[0].Action != "delete" {
|
||||
t.Fatalf("detail=%#v error=%v", detail, err)
|
||||
}
|
||||
joined = strings.ToLower(detailDB.queryRowSQL + "\n" + detailDB.querySQL)
|
||||
for _, forbidden := range []string{"knowledge_id", "test_kb_id", "acceptance_run_id", "publication_updated_at", "stable_external_id", "local_spool_ref"} {
|
||||
if strings.Contains(joined, forbidden) {
|
||||
t.Errorf("batch read SQL includes forbidden projection %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListReleaseBatchesRejectsInvalidFilter(t *testing.T) {
|
||||
_, err := NewStore(&recordingDB{}).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "released", Limit: 20})
|
||||
if !errors.Is(err, ErrInvalidReleaseBatchFilter) {
|
||||
t.Fatalf("error=%v, want ErrInvalidReleaseBatchFilter", err)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseBatchRow struct {
|
||||
id, name, state, policy, createdBy, sealedBy string
|
||||
itemCount int
|
||||
createdAt time.Time
|
||||
sealedAt *time.Time
|
||||
}
|
||||
|
||||
func (r releaseBatchRow) Scan(dest ...any) error {
|
||||
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = r.id, r.name, r.state, r.policy
|
||||
*dest[4].(*string), *dest[5].(*string), *dest[6].(*int) = r.createdBy, r.sealedBy, r.itemCount
|
||||
*dest[7].(*time.Time), *dest[8].(**time.Time) = r.createdAt, r.sealedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
type releaseCountRow struct{ count int }
|
||||
|
||||
func (r releaseCountRow) Scan(dest ...any) error {
|
||||
*dest[0].(*int) = r.count
|
||||
return nil
|
||||
}
|
||||
|
||||
type releaseBatchRows struct {
|
||||
values []releaseBatchRow
|
||||
index int
|
||||
}
|
||||
|
||||
func (r *releaseBatchRows) Close() {}
|
||||
func (r *releaseBatchRows) Err() error { return nil }
|
||||
func (r *releaseBatchRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
|
||||
func (r *releaseBatchRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
|
||||
func (r *releaseBatchRows) RawValues() [][]byte { return nil }
|
||||
func (r *releaseBatchRows) Conn() *pgx.Conn { return nil }
|
||||
func (r *releaseBatchRows) Next() bool {
|
||||
if r.index >= len(r.values) {
|
||||
return false
|
||||
}
|
||||
r.index++
|
||||
return true
|
||||
}
|
||||
func (r *releaseBatchRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) }
|
||||
func (r *releaseBatchRows) Values() ([]any, error) { return nil, nil }
|
||||
|
||||
type releaseBatchItemRow struct{ publicationID, title, visibility, action string }
|
||||
|
||||
type releaseBatchItemRows struct {
|
||||
values []releaseBatchItemRow
|
||||
index int
|
||||
}
|
||||
|
||||
func (r *releaseBatchItemRows) Close() {}
|
||||
func (r *releaseBatchItemRows) Err() error { return nil }
|
||||
func (r *releaseBatchItemRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
|
||||
func (r *releaseBatchItemRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
|
||||
func (r *releaseBatchItemRows) RawValues() [][]byte { return nil }
|
||||
func (r *releaseBatchItemRows) Conn() *pgx.Conn { return nil }
|
||||
func (r *releaseBatchItemRows) Next() bool {
|
||||
if r.index >= len(r.values) {
|
||||
return false
|
||||
}
|
||||
r.index++
|
||||
return true
|
||||
}
|
||||
func (r *releaseBatchItemRows) Scan(dest ...any) error {
|
||||
v := r.values[r.index-1]
|
||||
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = v.publicationID, v.title, v.visibility, v.action
|
||||
return nil
|
||||
}
|
||||
func (r *releaseBatchItemRows) Values() ([]any, error) { return nil, nil }
|
||||
|
||||
type releaseBatchReadDB struct {
|
||||
*recordingDB
|
||||
row pgx.Row
|
||||
rows pgx.Rows
|
||||
}
|
||||
|
||||
func (d *releaseBatchReadDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row {
|
||||
d.queryRowSQL, d.queryRowArgs = sql, args
|
||||
return d.row
|
||||
}
|
||||
|
||||
func (d *releaseBatchReadDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) {
|
||||
d.querySQL, d.queryArgs = sql, args
|
||||
return d.rows, nil
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// Errors for R1 release ChangeSets.
|
||||
var (
|
||||
ErrInvalidReleaseChangeSet = errors.New("invalid release change set request")
|
||||
ErrReleaseChangeSetNotFound = errors.New("release change set not found")
|
||||
ErrReleaseChangeSetStateConflict = errors.New("release change set state conflict")
|
||||
ErrReleaseChangeSetEmpty = errors.New("release plan has no releasable items")
|
||||
)
|
||||
|
||||
const releaseChangeSetColumns = `id,number,name,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at,confirmed_by,COALESCE(confirmed_at::text,'')`
|
||||
|
||||
func scanReleaseChangeSet(row pgx.Row) (governance.ReleaseChangeSet, error) {
|
||||
var item governance.ReleaseChangeSet
|
||||
var snapshots, changeSets []byte
|
||||
var confirmedAt string
|
||||
err := row.Scan(&item.ID, &item.Number, &item.Name, &item.State, &snapshots, &changeSets, &item.AcceptanceCutoff, &item.ProductionBaselineRef,
|
||||
&item.ItemCount, &item.AddCount, &item.UpdateCount, &item.DeleteCount, &item.BlockedCount, &item.ManifestSHA256, &item.CreatedBy, &item.CreatedAt, &item.ConfirmedBy, &confirmedAt)
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, err
|
||||
}
|
||||
_ = json.Unmarshal(snapshots, &item.SourceSnapshotIDs)
|
||||
_ = json.Unmarshal(changeSets, &item.SourceChangeSetIDs)
|
||||
if item.SourceSnapshotIDs == nil {
|
||||
item.SourceSnapshotIDs = []string{}
|
||||
}
|
||||
if item.SourceChangeSetIDs == nil {
|
||||
item.SourceChangeSetIDs = []string{}
|
||||
}
|
||||
if confirmedAt != "" {
|
||||
if parsed, parseErr := time.Parse(time.RFC3339Nano, confirmedAt); parseErr == nil {
|
||||
item.ConfirmedAt = &parsed
|
||||
}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// ComputeReleaseChangeSetManifest is exported for tests: the manifest seals
|
||||
// every frozen decision in seq order with an exact delimiter.
|
||||
func ComputeReleaseChangeSetManifest(items []governance.ReleaseChangeSetItem) string {
|
||||
lines := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
lines = append(lines, strings.Join([]string{
|
||||
strconv.Itoa(item.Seq), item.PublicationID, item.TargetScope, item.Action,
|
||||
item.TestState, item.ProductionState, strconv.FormatBool(item.CanRelease), item.Blocker,
|
||||
}, "|"))
|
||||
}
|
||||
digest := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
// GenerateReleaseChangeSet freezes the current live plan into an immutable
|
||||
// versioned set. Repeated generation with an identical plan reuses the latest
|
||||
// PLANNED set (dedup), so concurrent generation cannot create duplicates.
|
||||
func (s *Store) GenerateReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) {
|
||||
if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 200 ||
|
||||
strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) || request.Now.IsZero() {
|
||||
return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet
|
||||
}
|
||||
plan, err := s.ListReleasePlan(ctx)
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, err
|
||||
}
|
||||
items := make([]governance.ReleaseChangeSetItem, 0, len(plan.Items))
|
||||
for index, planItem := range plan.Items {
|
||||
items = append(items, governance.ReleaseChangeSetItem{Seq: index + 1, PublicationID: planItem.PublicationID, Title: planItem.Title, TargetScope: planItem.TargetScope, Action: planItem.Action, TestState: planItem.TestState, ProductionState: planItem.ProductionState, CanRelease: planItem.CanRelease, Blocker: planItem.Blocker})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return governance.ReleaseChangeSet{}, ErrReleaseChangeSetEmpty
|
||||
}
|
||||
manifest := ComputeReleaseChangeSetManifest(items)
|
||||
counts := map[string]int{}
|
||||
for _, item := range items {
|
||||
counts[item.Action]++
|
||||
}
|
||||
blocked := 0
|
||||
for _, item := range items {
|
||||
if item.Action == "blocked" {
|
||||
blocked++
|
||||
}
|
||||
}
|
||||
|
||||
// Same plan twice (sequentially) reuses the latest PLANNED set instead of
|
||||
// piling up identical versions. Concurrent racers fall through to the
|
||||
// unique-number conflict path below.
|
||||
if existing, lookupErr := s.findPlannedByManifest(ctx, manifest); lookupErr == nil && existing.ManifestSHA256 == manifest {
|
||||
return existing, nil
|
||||
} else if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) {
|
||||
return governance.ReleaseChangeSet{}, lookupErr
|
||||
}
|
||||
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
result, retry, err := s.tryInsertReleaseChangeSet(ctx, request, items, manifest, counts["add"], counts["update"], counts["delete"], blocked)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
if !retry {
|
||||
return governance.ReleaseChangeSet{}, err
|
||||
}
|
||||
// A concurrent identical generation won; return its set (dedup) if the
|
||||
// manifest matches, otherwise loop to take the next version number.
|
||||
existing, lookupErr := s.findPlannedByManifest(ctx, manifest)
|
||||
if lookupErr == nil && existing.ManifestSHA256 == manifest {
|
||||
return existing, nil
|
||||
}
|
||||
if lookupErr != nil && !errors.Is(lookupErr, ErrReleaseChangeSetNotFound) {
|
||||
return governance.ReleaseChangeSet{}, lookupErr
|
||||
}
|
||||
}
|
||||
return governance.ReleaseChangeSet{}, fmt.Errorf("release change set generation kept conflicting")
|
||||
}
|
||||
|
||||
func (s *Store) tryInsertReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, items []governance.ReleaseChangeSetItem, manifest string, add, update, del, blocked int) (governance.ReleaseChangeSet, bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
id, err := newOpaqueID("rcs-")
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, err
|
||||
}
|
||||
set, err := scanReleaseChangeSet(tx.QueryRow(ctx, `
|
||||
INSERT INTO release_change_sets(id,name,number,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at)
|
||||
SELECT $1,$2,(SELECT COALESCE(MAX(number),0)+1 FROM release_change_sets),'planned',
|
||||
COALESCE((SELECT jsonb_agg(snapshot_id ORDER BY snapshot_id) FROM (SELECT DISTINCT snapshot_id FROM source_catalog_heads) s),'[]'::jsonb),
|
||||
COALESCE((SELECT jsonb_agg(change_set_id ORDER BY change_set_id) FROM (SELECT DISTINCT id AS change_set_id FROM source_catalog_change_sets WHERE state IN ('confirmed','applied')) c),'[]'::jsonb),
|
||||
COALESCE((SELECT MAX(updated_at) FROM test_publications),(SELECT now())),
|
||||
(SELECT COALESCE(MAX(completed_at)::text,'') FROM release_executions WHERE state='completed'),
|
||||
$3,$4,$5,$6,$7,$8,$9,$10
|
||||
RETURNING `+releaseChangeSetColumns,
|
||||
id, request.Name, len(items), add, update, del, blocked, manifest, request.CreatedBy, request.Now))
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return governance.ReleaseChangeSet{}, true, err
|
||||
}
|
||||
return governance.ReleaseChangeSet{}, false, fmt.Errorf("create release change set: %w", err)
|
||||
}
|
||||
// R6 audit chain: mirror the JSONB binding into real FK rows so the trace
|
||||
// survives even if someone later edits denormalized display fields.
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id)
|
||||
SELECT $1, elem.value
|
||||
FROM jsonb_array_elements_text($2::jsonb) AS elem(value)
|
||||
ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceSnapshotIDs)); err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release snapshots: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id)
|
||||
SELECT $1, elem.value
|
||||
FROM jsonb_array_elements_text($2::jsonb) AS elem(value)
|
||||
ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceChangeSetIDs)); err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release catalog sets: %w", err)
|
||||
}
|
||||
for _, item := range items {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`,
|
||||
set.ID, item.Seq, item.PublicationID, item.Title, item.TargetScope, item.Action, item.TestState, item.ProductionState, item.CanRelease, item.Blocker); err != nil {
|
||||
return governance.ReleaseChangeSet{}, false, fmt.Errorf("insert release change set item: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return governance.ReleaseChangeSet{}, true, err
|
||||
}
|
||||
return governance.ReleaseChangeSet{}, false, err
|
||||
}
|
||||
return set, false, nil
|
||||
}
|
||||
|
||||
func (s *Store) findPlannedByManifest(ctx context.Context, manifest string) (governance.ReleaseChangeSet, error) {
|
||||
return scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE manifest_sha256=$1 AND state='planned' ORDER BY number DESC LIMIT 1`, manifest))
|
||||
}
|
||||
|
||||
// ConfirmReleaseChangeSet freezes a planned set for batching. Re-confirming an
|
||||
// already-confirmed set replays the stored confirmation (immutability).
|
||||
func (s *Store) ConfirmReleaseChangeSet(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) {
|
||||
if !validRouteID(request.ChangeSetID) || !validReviewerRef(request.ConfirmedBy) || request.Now.IsZero() {
|
||||
return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet
|
||||
}
|
||||
set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `UPDATE release_change_sets SET state='confirmed',confirmed_by=$2,confirmed_at=$3 WHERE id=$1 AND state='planned' RETURNING `+releaseChangeSetColumns, request.ChangeSetID, request.ConfirmedBy, request.Now))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
current, lookupErr := s.GetReleaseChangeSet(ctx, request.ChangeSetID)
|
||||
if lookupErr != nil {
|
||||
return governance.ReleaseChangeSet{}, lookupErr
|
||||
}
|
||||
if current.State == "confirmed" || current.State == "consumed" {
|
||||
return current.ReleaseChangeSet, nil
|
||||
}
|
||||
return governance.ReleaseChangeSet{}, ErrReleaseChangeSetStateConflict
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSet{}, fmt.Errorf("confirm release change set: %w", err)
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetReleaseChangeSet(ctx context.Context, id string) (governance.ReleaseChangeSetDetail, error) {
|
||||
set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE id=$1`, id))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSetDetail{}, err
|
||||
}
|
||||
detail := governance.ReleaseChangeSetDetail{ReleaseChangeSet: set, Items: []governance.ReleaseChangeSetItem{}}
|
||||
rows, err := s.db.Query(ctx, `SELECT seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker FROM release_change_set_items WHERE change_set_id=$1 ORDER BY seq`, id)
|
||||
if err != nil {
|
||||
return governance.ReleaseChangeSetDetail{}, fmt.Errorf("list release change set items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.ReleaseChangeSetItem
|
||||
if err := rows.Scan(&item.Seq, &item.PublicationID, &item.Title, &item.TargetScope, &item.Action, &item.TestState, &item.ProductionState, &item.CanRelease, &item.Blocker); err != nil {
|
||||
return governance.ReleaseChangeSetDetail{}, err
|
||||
}
|
||||
detail.Items = append(detail.Items, item)
|
||||
}
|
||||
return detail, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ListReleaseChangeSets(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseChangeSetPage, error) {
|
||||
limit, offset := filter.Limit, filter.Offset
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
page := governance.ReleaseChangeSetPage{Items: []governance.ReleaseChangeSet{}, Limit: limit, Offset: offset}
|
||||
if limit > 100 {
|
||||
return page, errors.New("release change set limit must be between 1 and 100")
|
||||
}
|
||||
var total int
|
||||
if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&total); err != nil {
|
||||
return page, err
|
||||
}
|
||||
page.Total = total
|
||||
rows, err := s.db.Query(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets ORDER BY number DESC LIMIT $1 OFFSET $2`, limit, offset)
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
item, err := scanReleaseChangeSet(rows)
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
page.Items = append(page.Items, item)
|
||||
}
|
||||
return page, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) LatestReleaseChangeSet(ctx context.Context) (governance.ReleaseChangeSetDetail, error) {
|
||||
var id string
|
||||
if err := s.db.QueryRow(ctx, `SELECT id FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&id); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound
|
||||
}
|
||||
return governance.ReleaseChangeSetDetail{}, err
|
||||
}
|
||||
return s.GetReleaseChangeSet(ctx, id)
|
||||
}
|
||||
|
||||
func mustMarshalJSON(values []string) []byte {
|
||||
raw, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return []byte("[]")
|
||||
}
|
||||
return raw
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestListAndGetReleaseExecutionsReturnsAuditableProjection(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC)
|
||||
row := releaseExecutionRow{id: "rx-safe", batchID: "rb-safe", operation: "publish", state: "completed", target: "production", authorizedBy: "owner", authorizationCode: "production_release_authorized", jobID: "job-safe", manifestSHA: strings.Repeat("a", 64), itemCount: 1, completedCount: 1, createdAt: now, completedAt: &now}
|
||||
db := &releaseExecutionReadDB{countRow: releaseExecutionCountRow{count: 1}, row: row, rows: &releaseExecutionRows{values: []releaseExecutionRow{row}}}
|
||||
store := NewStore(db)
|
||||
page, err := store.ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "publish", State: "completed", Limit: 20})
|
||||
if err != nil || page.Total != 1 || len(page.Items) != 1 {
|
||||
t.Fatalf("page=%#v error=%v", page, err)
|
||||
}
|
||||
db.rows = &releaseItemExecutionRows{values: []releaseItemExecutionRow{{publicationID: "tp-safe", title: "Synthetic", action: "delete", state: "verified", previousRevision: "prod-v0", productionRevision: "absent", completedAt: &now}}}
|
||||
detail, err := store.GetReleaseExecution(context.Background(), "rx-safe")
|
||||
if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" || detail.Items[0].ProductionRevision != "absent" {
|
||||
t.Fatalf("detail=%#v error=%v", detail, err)
|
||||
}
|
||||
joined := strings.ToLower(strings.Join(db.queries, "\n"))
|
||||
for _, required := range []string{"from release_executions", "operation = $1", "state = $2", "from release_item_executions", "publication_id", "item.action"} {
|
||||
if !strings.Contains(joined, required) {
|
||||
t.Fatalf("release read SQL missing %q", required)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge_id", "test_kb_id", "physical_path", "api_key", "secret"} {
|
||||
if strings.Contains(joined, forbidden) {
|
||||
t.Fatalf("release read SQL contains forbidden %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListReleaseExecutionsRejectsInvalidFilter(t *testing.T) {
|
||||
_, err := NewStore(&recordingDB{}).ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "deploy", Limit: 20})
|
||||
if !errors.Is(err, ErrInvalidReleaseExecutionFilter) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseExecutionCountRow struct{ count int }
|
||||
|
||||
func (r releaseExecutionCountRow) Scan(dest ...any) error { *dest[0].(*int) = r.count; return nil }
|
||||
|
||||
type releaseExecutionRows struct {
|
||||
values []releaseExecutionRow
|
||||
index int
|
||||
}
|
||||
|
||||
func (r *releaseExecutionRows) Close() {}
|
||||
func (r *releaseExecutionRows) Err() error { return nil }
|
||||
func (r *releaseExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
|
||||
func (r *releaseExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
|
||||
func (r *releaseExecutionRows) RawValues() [][]byte { return nil }
|
||||
func (r *releaseExecutionRows) Conn() *pgx.Conn { return nil }
|
||||
func (r *releaseExecutionRows) Next() bool {
|
||||
if r.index >= len(r.values) {
|
||||
return false
|
||||
}
|
||||
r.index++
|
||||
return true
|
||||
}
|
||||
func (r *releaseExecutionRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) }
|
||||
func (r *releaseExecutionRows) Values() ([]any, error) { return nil, nil }
|
||||
|
||||
type releaseItemExecutionRow struct {
|
||||
publicationID, title, action, state, previousRevision, productionRevision, errorCode string
|
||||
createdAt time.Time
|
||||
completedAt *time.Time
|
||||
}
|
||||
type releaseItemExecutionRows struct {
|
||||
values []releaseItemExecutionRow
|
||||
index int
|
||||
}
|
||||
|
||||
func (r *releaseItemExecutionRows) Close() {}
|
||||
func (r *releaseItemExecutionRows) Err() error { return nil }
|
||||
func (r *releaseItemExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
|
||||
func (r *releaseItemExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
|
||||
func (r *releaseItemExecutionRows) RawValues() [][]byte { return nil }
|
||||
func (r *releaseItemExecutionRows) Conn() *pgx.Conn { return nil }
|
||||
func (r *releaseItemExecutionRows) Next() bool {
|
||||
if r.index >= len(r.values) {
|
||||
return false
|
||||
}
|
||||
r.index++
|
||||
return true
|
||||
}
|
||||
func (r *releaseItemExecutionRows) Scan(dest ...any) error {
|
||||
value := r.values[r.index-1]
|
||||
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = value.publicationID, value.title, value.action, value.state
|
||||
*dest[4].(*string), *dest[5].(*string), *dest[6].(*string) = value.previousRevision, value.productionRevision, value.errorCode
|
||||
*dest[7].(*time.Time), *dest[8].(**time.Time) = value.createdAt, value.completedAt
|
||||
return nil
|
||||
}
|
||||
func (r *releaseItemExecutionRows) Values() ([]any, error) { return nil, nil }
|
||||
|
||||
type releaseExecutionReadDB struct {
|
||||
countRow, row pgx.Row
|
||||
rows pgx.Rows
|
||||
queries []string
|
||||
queryRows int
|
||||
}
|
||||
|
||||
func (d *releaseExecutionReadDB) Begin(context.Context) (pgx.Tx, error) {
|
||||
return nil, errors.New("transactions are not supported by the reading fake")
|
||||
}
|
||||
|
||||
func (d *releaseExecutionReadDB) QueryRow(_ context.Context, sql string, _ ...any) pgx.Row {
|
||||
d.queries = append(d.queries, sql)
|
||||
d.queryRows++
|
||||
if d.queryRows == 1 {
|
||||
return d.countRow
|
||||
}
|
||||
return d.row
|
||||
}
|
||||
func (d *releaseExecutionReadDB) Query(_ context.Context, sql string, _ ...any) (pgx.Rows, error) {
|
||||
d.queries = append(d.queries, sql)
|
||||
return d.rows, nil
|
||||
}
|
||||
func (d *releaseExecutionReadDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) {
|
||||
return pgconn.CommandTag{}, nil
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
var ErrReleaseExecutionNotClaimable = errors.New("release execution is not claimable")
|
||||
|
||||
const startReleaseExecutionSQL = `
|
||||
WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
|
||||
execution_gate AS (
|
||||
SELECT execution.id, execution.operation,
|
||||
COUNT(item.publication_id)::INTEGER AS item_count,
|
||||
COUNT(*) FILTER (
|
||||
WHERE (item.action = 'upsert'
|
||||
AND ready.publication_id IS NOT NULL
|
||||
AND ready.review_id = item.review_id
|
||||
AND ready.acceptance_run_id = item.acceptance_run_id
|
||||
AND ready.publication_updated_at = item.publication_updated_at
|
||||
AND ready.decided_visibility = item.decided_visibility)
|
||||
OR (item.action = 'delete'
|
||||
AND delete_ready.publication_id IS NOT NULL
|
||||
AND delete_ready.review_id = item.review_id
|
||||
AND delete_ready.acceptance_run_id = item.acceptance_run_id
|
||||
AND delete_ready.publication_updated_at = item.publication_updated_at
|
||||
AND delete_ready.decided_visibility = item.decided_visibility)
|
||||
)::INTEGER AS valid_items
|
||||
FROM release_executions AS execution
|
||||
JOIN release_batch_items AS item ON item.release_batch_id = execution.release_batch_id
|
||||
LEFT JOIN release_upsert_ready AS ready ON ready.publication_id = item.publication_id AND item.action = 'upsert'
|
||||
LEFT JOIN release_delete_ready AS delete_ready ON delete_ready.publication_id = item.publication_id AND item.action = 'delete'
|
||||
WHERE execution.id = $1
|
||||
GROUP BY execution.id, execution.operation
|
||||
), started AS (
|
||||
UPDATE release_executions AS execution
|
||||
SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolling_back' ELSE 'running' END,
|
||||
started_at = COALESCE(started_at, $4::TIMESTAMPTZ)
|
||||
FROM execution_gate
|
||||
WHERE execution.id = $1::VARCHAR
|
||||
AND execution_gate.id = execution.id
|
||||
AND (execution_gate.operation = 'rollback' OR execution_gate.valid_items = execution_gate.item_count)
|
||||
AND execution.job_id = $2::VARCHAR
|
||||
AND execution.operation = $3::VARCHAR
|
||||
AND execution.state IN ('pending', 'running', 'rolling_back')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM release_batch_items AS gate_item
|
||||
JOIN test_publications AS gate_publication ON gate_publication.id = gate_item.publication_id
|
||||
JOIN source_item_versions AS gate_version ON gate_version.id = gate_item.source_item_version_id
|
||||
WHERE gate_item.release_batch_id = execution.release_batch_id
|
||||
AND (gate_version.revision = '' OR gate_publication.parser_generation = '' OR gate_publication.index_generation = '')
|
||||
)
|
||||
RETURNING execution.id, execution.release_batch_id, execution.operation, execution.prior_execution_id
|
||||
), seeded AS (
|
||||
INSERT INTO release_item_executions (
|
||||
id, release_execution_id, publication_id, action, state, created_at, updated_at
|
||||
)
|
||||
SELECT 'ri-' || SUBSTR(MD5(started.id || ':' || item.publication_id), 1, 32),
|
||||
started.id, item.publication_id, item.action, 'pending', $4::TIMESTAMPTZ, $4::TIMESTAMPTZ
|
||||
FROM started
|
||||
JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id
|
||||
WHERE started.operation = 'publish' OR EXISTS (
|
||||
SELECT 1
|
||||
FROM release_item_executions AS prior_seed
|
||||
WHERE prior_seed.release_execution_id = started.prior_execution_id
|
||||
AND prior_seed.publication_id = item.publication_id
|
||||
AND prior_seed.state = 'verified'
|
||||
)
|
||||
ON CONFLICT (release_execution_id, publication_id) DO NOTHING
|
||||
RETURNING publication_id, action
|
||||
), active_items AS (
|
||||
SELECT publication_id, action FROM seeded
|
||||
UNION
|
||||
SELECT current_item.publication_id, current_item.action
|
||||
FROM started
|
||||
JOIN release_item_executions AS current_item
|
||||
ON current_item.release_execution_id = started.id
|
||||
WHERE current_item.state = 'pending'
|
||||
), plan AS (
|
||||
SELECT started.id AS release_execution_id,
|
||||
item.publication_id,
|
||||
active_items.action,
|
||||
item.source_item_version_id,
|
||||
current_version.revision AS source_revision,
|
||||
publication.parser_generation || '@' || publication.index_generation AS parse_revision,
|
||||
item.review_id AS release_revision,
|
||||
publication.test_kb_id,
|
||||
publication.knowledge_id,
|
||||
item.decided_visibility,
|
||||
COALESCE(previous.production_revision, '') AS previous_production_revision
|
||||
FROM started
|
||||
JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id
|
||||
JOIN active_items ON active_items.publication_id = item.publication_id
|
||||
|
||||
JOIN test_publications AS publication ON publication.id = item.publication_id
|
||||
JOIN source_item_versions AS current_version ON current_version.id = item.source_item_version_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT CASE
|
||||
WHEN started.operation = 'rollback' THEN prior_item.previous_production_revision
|
||||
ELSE prior_item.production_revision
|
||||
END AS production_revision
|
||||
FROM release_item_executions AS prior_item
|
||||
JOIN release_executions AS prior_execution ON prior_execution.id = prior_item.release_execution_id
|
||||
JOIN test_publications AS prior_publication ON prior_publication.id = prior_item.publication_id
|
||||
JOIN source_item_versions AS prior_version ON prior_version.id = prior_publication.source_item_version_id
|
||||
WHERE prior_item.state = 'verified'
|
||||
AND prior_execution.operation = 'publish'
|
||||
AND prior_execution.state IN ('completed', 'failed')
|
||||
AND (
|
||||
(started.operation = 'rollback'
|
||||
AND prior_execution.id = started.prior_execution_id
|
||||
AND prior_item.publication_id = item.publication_id)
|
||||
OR
|
||||
(started.operation = 'publish'
|
||||
AND prior_version.source_item_id = current_version.source_item_id
|
||||
AND (
|
||||
(active_items.action = 'delete' AND prior_item.publication_id = item.publication_id)
|
||||
OR (active_items.action = 'upsert' AND prior_item.publication_id <> item.publication_id)
|
||||
))
|
||||
)
|
||||
ORDER BY prior_item.completed_at DESC, prior_item.id DESC
|
||||
LIMIT 1
|
||||
) AS previous ON TRUE
|
||||
ORDER BY item.publication_id
|
||||
)
|
||||
SELECT COUNT(DISTINCT started.id)::INTEGER,
|
||||
COALESCE(
|
||||
JSONB_AGG(JSONB_BUILD_OBJECT(
|
||||
'release_execution_id', plan.release_execution_id,
|
||||
'publication_id', plan.publication_id,
|
||||
'action', plan.action,
|
||||
'source_item_version_id', plan.source_item_version_id,
|
||||
'source_revision', plan.source_revision,
|
||||
'parse_revision', plan.parse_revision,
|
||||
'release_revision', plan.release_revision,
|
||||
'test_knowledge_base_id', plan.test_kb_id,
|
||||
'knowledge_id', plan.knowledge_id,
|
||||
'decided_visibility', plan.decided_visibility,
|
||||
'previous_production_revision', plan.previous_production_revision
|
||||
) ORDER BY plan.publication_id) FILTER (WHERE plan.publication_id IS NOT NULL),
|
||||
'[]'::JSONB
|
||||
)
|
||||
FROM started
|
||||
LEFT JOIN plan ON TRUE
|
||||
`
|
||||
|
||||
const beginReleaseBatchSQL = `
|
||||
UPDATE release_executions
|
||||
SET current_batch = $2,
|
||||
total_batches = $3
|
||||
WHERE id = $1
|
||||
AND state IN ('running', 'rolling_back')
|
||||
AND (total_batches = 0 OR total_batches = $3)
|
||||
AND ($2 = current_batch OR $2 = current_batch + 1)
|
||||
`
|
||||
|
||||
const completeReleaseItemSQL = `
|
||||
UPDATE release_item_executions AS item
|
||||
SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolled_back' ELSE 'verified' END,
|
||||
bundle_sha256 = $3,
|
||||
previous_production_revision = $4,
|
||||
production_revision = $5,
|
||||
error_code = '',
|
||||
updated_at = $6,
|
||||
completed_at = $6
|
||||
FROM release_executions AS execution
|
||||
WHERE item.release_execution_id = $1
|
||||
AND item.publication_id = $2
|
||||
AND execution.id = item.release_execution_id
|
||||
AND execution.state IN ('running', 'rolling_back')
|
||||
AND item.state IN ('pending', 'staged', 'activated', 'verified', 'rolled_back')
|
||||
`
|
||||
|
||||
const failReleaseItemSQL = `
|
||||
UPDATE release_item_executions AS item
|
||||
SET state = 'failed',
|
||||
error_code = $3,
|
||||
updated_at = $4,
|
||||
completed_at = $4
|
||||
FROM release_executions AS execution
|
||||
WHERE item.release_execution_id = $1
|
||||
AND item.publication_id = $2
|
||||
AND execution.id = item.release_execution_id
|
||||
AND execution.state IN ('running', 'rolling_back')
|
||||
AND item.state IN ('pending', 'staged', 'activated', 'failed')
|
||||
`
|
||||
|
||||
const finishReleaseExecutionSQL = `
|
||||
WITH item_counts AS (
|
||||
SELECT release_execution_id,
|
||||
COUNT(*)::INTEGER AS total_items,
|
||||
COUNT(*) FILTER (WHERE state IN ('verified', 'rolled_back'))::INTEGER AS completed_items,
|
||||
COUNT(*) FILTER (WHERE state = 'failed')::INTEGER AS failed_items
|
||||
FROM release_item_executions
|
||||
WHERE release_execution_id = $1
|
||||
GROUP BY release_execution_id
|
||||
)
|
||||
UPDATE release_executions AS execution
|
||||
SET state = CASE
|
||||
WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'rollback_failed'
|
||||
WHEN execution.operation = 'rollback' THEN 'rolled_back'
|
||||
WHEN item_counts.failed_items > 0 THEN 'failed'
|
||||
ELSE 'completed'
|
||||
END,
|
||||
completed_item_count = item_counts.completed_items,
|
||||
failed_item_count = item_counts.failed_items,
|
||||
-- R4 failure grading on publish: nothing published yet = single_item
|
||||
-- (compensate that item only); progress existed = batch (stop later
|
||||
-- batches, compensate this batch's verified items). A failed ROLLBACK
|
||||
-- escalates the prior publish to overall (snapshot restore required).
|
||||
failure_level = CASE
|
||||
WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 AND item_counts.completed_items > 0 THEN 'batch'
|
||||
WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 THEN 'single_item'
|
||||
ELSE execution.failure_level
|
||||
END,
|
||||
error_code = CASE
|
||||
WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'release_rollback_failed'
|
||||
WHEN item_counts.failed_items > 0 THEN 'release_item_failed'
|
||||
ELSE ''
|
||||
END,
|
||||
completed_at = $2
|
||||
FROM item_counts
|
||||
WHERE execution.id = item_counts.release_execution_id
|
||||
AND execution.state IN ('running', 'rolling_back')
|
||||
AND item_counts.total_items = execution.item_count
|
||||
AND item_counts.completed_items + item_counts.failed_items = execution.item_count
|
||||
`
|
||||
|
||||
// R4: a permanently failed rollback means compensation alone is no longer
|
||||
// enough: the prior publish escalates to overall (snapshot restore).
|
||||
const escalatePriorOverallSQL = `
|
||||
UPDATE release_executions AS prior
|
||||
SET failure_level = 'overall'
|
||||
FROM release_executions AS rollback
|
||||
WHERE rollback.id = $1
|
||||
AND rollback.operation = 'rollback'
|
||||
AND rollback.state = 'rollback_failed'
|
||||
AND prior.id = rollback.prior_execution_id
|
||||
AND prior.operation = 'publish'
|
||||
`
|
||||
|
||||
func (s *Store) StartExecution(ctx context.Context, request releaseworkflow.StartExecutionRequest) ([]releaseworkflow.PlanItem, error) {
|
||||
if !validReleaseExecutionRequest(request) {
|
||||
return nil, ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
var executionCount int
|
||||
var payload []byte
|
||||
if err := s.db.QueryRow(ctx, startReleaseExecutionSQL,
|
||||
request.ExecutionID, request.JobID, request.Operation, request.Now.UTC(),
|
||||
).Scan(&executionCount, &payload); err != nil {
|
||||
return nil, fmt.Errorf("start release execution: %w", err)
|
||||
}
|
||||
if executionCount != 1 {
|
||||
return nil, ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
var items []releaseworkflow.PlanItem
|
||||
if err := json.Unmarshal(payload, &items); err != nil {
|
||||
return nil, errors.New("decode release execution plan")
|
||||
}
|
||||
for _, item := range items {
|
||||
if !validShortID(item.PublicationID) || !validShortID(item.SourceItemVersionID) ||
|
||||
!validRemoteIdentifier(item.TestKnowledgeBaseID) || !validRemoteIdentifier(item.KnowledgeID) ||
|
||||
(item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) ||
|
||||
len(item.PreviousProductionRevision) > 128 {
|
||||
return nil, errors.New("invalid release execution plan")
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Store) BeginBatch(ctx context.Context, executionID string, batch, total int, now time.Time) error {
|
||||
if !validShortID(executionID) || batch < 1 || total < batch || total > 100000 || now.IsZero() {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
result, err := s.db.Exec(ctx, beginReleaseBatchSQL, executionID, batch, total)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin release batch: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CompleteItem(ctx context.Context, request releaseworkflow.CompleteItemRequest) error {
|
||||
if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) ||
|
||||
(request.BundleSHA256 != "" && !validLowerSHA256(request.BundleSHA256)) ||
|
||||
!validRevision(request.ProductionRevision) || !validOptionalRevision(request.PreviousProductionRevision) || request.Now.IsZero() {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
result, err := s.db.Exec(ctx, completeReleaseItemSQL, request.ExecutionID, request.PublicationID,
|
||||
request.BundleSHA256, request.PreviousProductionRevision, request.ProductionRevision, request.Now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete release item: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) FailItem(ctx context.Context, request releaseworkflow.FailItemRequest) error {
|
||||
if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) || !releaseworkflow.ValidErrorCode(request.ErrorCode) || request.Now.IsZero() {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
result, err := s.db.Exec(ctx, failReleaseItemSQL, request.ExecutionID, request.PublicationID, request.ErrorCode, request.Now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail release item: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ComputeReleaseExecutionManifest seals what ACTUALLY happened: one line per
|
||||
// item (sorted by publication id) plus the bound release change set digest.
|
||||
// Publish lines carry the verified bundle digest; rollback lines carry the
|
||||
// restored revision. The digest is deterministic and re-computable for audits.
|
||||
func ComputeReleaseExecutionManifest(operation, changeSetManifest string, lines []string) string {
|
||||
digest := sha256.New()
|
||||
bound := changeSetManifest
|
||||
if bound == "" {
|
||||
bound = "none"
|
||||
}
|
||||
fmt.Fprintf(digest, "op:%s\nchangeset:%s\n", operation, bound)
|
||||
for _, line := range lines {
|
||||
fmt.Fprintf(digest, "%s\n", line)
|
||||
}
|
||||
return hex.EncodeToString(digest.Sum(nil))
|
||||
}
|
||||
|
||||
func (s *Store) FinishExecution(ctx context.Context, executionID string, now time.Time) error {
|
||||
if !validShortID(executionID) || now.IsZero() {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
result, err := s.db.Exec(ctx, finishReleaseExecutionSQL, executionID, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("finish release execution: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
// R4: a permanently failed rollback escalates the prior publish to the
|
||||
// overall level (snapshot restore becomes the only sanctioned path).
|
||||
if _, err := s.db.Exec(ctx, escalatePriorOverallSQL, executionID); err != nil {
|
||||
return fmt.Errorf("escalate rollback failure: %w", err)
|
||||
}
|
||||
// R5: seal the execution manifest and reconcile it against the confirmed
|
||||
// release change set. A success that does not match the frozen manifest is
|
||||
// downgraded to failed (release_manifest_mismatch) — never completed.
|
||||
if err := s.sealExecutionManifest(ctx, executionID); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) sealExecutionManifest(ctx context.Context, executionID string) error {
|
||||
var operation, state, changeSetManifest, changeSetID string
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT e.operation, e.state, COALESCE(rcs.manifest_sha256,''), COALESCE(e.release_change_set_id::text,'')
|
||||
FROM release_executions AS e
|
||||
LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id
|
||||
WHERE e.id = $1`, executionID).Scan(&operation, &state, &changeSetManifest, &changeSetID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrReleaseExecutionNotClaimable
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("load execution for manifest seal: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT item.publication_id, item.action, item.state, COALESCE(item.bundle_sha256,''), COALESCE(item.production_revision,'')
|
||||
FROM release_item_executions AS item
|
||||
WHERE item.release_execution_id = $1
|
||||
ORDER BY item.publication_id`, executionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load execution items for manifest seal: %w", err)
|
||||
}
|
||||
type itemRow struct {
|
||||
publicationID, action, itemState, bundleSHA, revision string
|
||||
}
|
||||
items := []itemRow{}
|
||||
for rows.Next() {
|
||||
var item itemRow
|
||||
if err := rows.Scan(&item.publicationID, &item.action, &item.itemState, &item.bundleSHA, &item.revision); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("scan execution item for manifest seal: %w", err)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("iterate execution items: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
lines := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if operation == "rollback" {
|
||||
lines = append(lines, fmt.Sprintf("%s|%s", item.publicationID, item.revision))
|
||||
} else {
|
||||
lines = append(lines, fmt.Sprintf("%s|%s|%s", item.publicationID, item.bundleSHA, item.revision))
|
||||
}
|
||||
}
|
||||
// Only a terminal execution seals its manifest; mid-batch re-finishes must
|
||||
// never freeze a partial snapshot (the seal below is first-write-wins).
|
||||
switch state {
|
||||
case "completed", "failed", "rolled_back", "rollback_failed":
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
manifest := ComputeReleaseExecutionManifest(operation, changeSetManifest, lines)
|
||||
if _, err := s.db.Exec(ctx, `UPDATE release_executions SET manifest_sha256 = $2 WHERE id = $1 AND manifest_sha256 = ''`, executionID, manifest); err != nil {
|
||||
return fmt.Errorf("seal execution manifest: %w", err)
|
||||
}
|
||||
if state != "completed" || changeSetID == "" {
|
||||
// Failed/rolled-back terminal states are honest audit facts; R4
|
||||
// overall escalation is already recorded. Nothing further to verify.
|
||||
return nil
|
||||
}
|
||||
// Success must equal the frozen change-set intent for the publications it
|
||||
// actually touched.
|
||||
frozen, err := s.changeSetItemActions(ctx, changeSetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
frozenAction, exists := frozen[item.publicationID]
|
||||
// The frozen change set records PLAN actions (add/update/delete);
|
||||
// executions carry EFFECTIVE actions (upsert/delete). Map equivalently.
|
||||
consistent := exists && item.itemState == "verified" &&
|
||||
((item.action == "delete" && frozenAction == "delete") ||
|
||||
(item.action == "upsert" && (frozenAction == "add" || frozenAction == "update")))
|
||||
if !consistent {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE release_executions SET state='failed', error_code='release_manifest_mismatch' WHERE id=$1 AND state='completed'`, executionID); err != nil {
|
||||
return fmt.Errorf("downgrade manifest mismatch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) changeSetItemActions(ctx context.Context, changeSetID string) (map[string]string, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT publication_id, action FROM release_change_set_items WHERE change_set_id = $1`, changeSetID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load release change set items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
actions := map[string]string{}
|
||||
for rows.Next() {
|
||||
var publicationID, action string
|
||||
if err := rows.Scan(&publicationID, &action); err != nil {
|
||||
return nil, fmt.Errorf("scan release change set item: %w", err)
|
||||
}
|
||||
actions[publicationID] = action
|
||||
}
|
||||
return actions, rows.Err()
|
||||
}
|
||||
|
||||
func validReleaseExecutionRequest(request releaseworkflow.StartExecutionRequest) bool {
|
||||
return validShortID(request.ExecutionID) && validShortID(request.JobID) && request.Operation.Valid() && !request.Now.IsZero()
|
||||
}
|
||||
|
||||
func validRemoteIdentifier(value string) bool {
|
||||
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\")
|
||||
}
|
||||
|
||||
func validRevision(value string) bool {
|
||||
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t")
|
||||
}
|
||||
|
||||
func validOptionalRevision(value string) bool {
|
||||
return value == "" || validRevision(value)
|
||||
}
|
||||
|
||||
func validLowerSHA256(value string) bool {
|
||||
if len(value) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user