feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
@@ -0,0 +1,104 @@
package postgres
import (
"context"
"errors"
"os"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestC4ConcurrentOperationCreatesOnceAndSurvivesPoolRestart(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp','pp','C4 scan',true)`); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 9, 11, 13, 0, 0, 0, time.UTC)
store := NewStore(pool)
// 10 concurrent identical requests with the same Idempotency-Key: exactly one must create.
var wg sync.WaitGroup
type outcome struct {
scanID string
replayed bool
err error
}
results := make([]outcome, 10)
for i := range results {
wg.Add(1)
go func(index int) {
defer wg.Done()
scan, replayed, err := store.TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now, "op-c4-scan-1")
results[index] = outcome{scanID: scan.ID, replayed: replayed, err: err}
}(i)
}
wg.Wait()
var created int
var firstID string
for _, result := range results {
if result.err != nil {
t.Fatalf("concurrent operation error = %v", result.err)
}
if !result.replayed {
created++
firstID = result.scanID
}
if result.scanID != firstID && firstID != "" {
t.Fatalf("replay returned different scan id: %s vs %s", result.scanID, firstID)
}
}
if created != 1 {
t.Fatalf("created=%d, want exactly 1", created)
}
var scans, jobs, operations int
if err := pool.QueryRow(ctx, `SELECT (SELECT COUNT(*) FROM source_scans),(SELECT COUNT(*) FROM jobs WHERE type='source.scan.pp'),(SELECT COUNT(*) FROM operation_requests WHERE operation_id='op-c4-scan-1')`).Scan(&scans, &jobs, &operations); err != nil {
t.Fatal(err)
}
if scans != 1 || jobs != 1 || operations != 1 {
t.Fatalf("scans=%d jobs=%d operations=%d", scans, jobs, operations)
}
// Restart: close the pool and reconnect, then replay with the same key.
pool.Close()
restarted, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer restarted.Close()
scan, replayed, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now.Add(time.Hour), "op-c4-scan-1")
if err != nil || !replayed || scan.ID != firstID {
t.Fatalf("post-restart replay scan=%#v replayed=%v err=%v want id=%s", scan, replayed, err, firstID)
}
// Same key, different payload (different connection): must conflict.
if _, err := restarted.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp2','pp','C4 scan 2',true)`); err != nil {
t.Fatal(err)
}
if _, _, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp2", now, "op-c4-scan-1"); err == nil {
t.Fatal("same key different payload should conflict")
} else if !errors.Is(err, ErrOperationRequestConflict) {
t.Fatalf("conflict error=%v", err)
}
var freshOperations int
if err := restarted.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests`).Scan(&freshOperations); err != nil {
t.Fatal(err)
}
if freshOperations != 1 {
t.Fatalf("operations=%d want 1", freshOperations)
}
}
@@ -0,0 +1,71 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
var ErrCatalogChangeSetNotFound = errors.New("catalog change set not found")
func (s *Store) GetLatestCatalogChangeSet(ctx context.Context, filter governance.CatalogChangeSetFilter) (governance.CatalogChangeSetDetail, error) {
if !validRouteID(filter.ConnectionID) || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 || !validCatalogChangeAction(filter.Action) {
return governance.CatalogChangeSetDetail{}, errors.New("invalid catalog change set filter")
}
detail := governance.CatalogChangeSetDetail{Items: []governance.CatalogChangeItem{}, Limit: filter.Limit, Offset: filter.Offset}
err := s.db.QueryRow(ctx, `SELECT change_set.id,change_set.source_connection_id,connection.name,change_set.state,
change_set.add_count,change_set.update_count,change_set.move_count,change_set.move_scope_count,
change_set.delete_count,change_set.no_change_count,change_set.created_at
FROM source_catalog_change_sets change_set
JOIN source_connections connection ON connection.id=change_set.source_connection_id
WHERE change_set.source_connection_id=$1
ORDER BY change_set.created_at DESC,change_set.id DESC
LIMIT 1`, filter.ConnectionID).Scan(
&detail.ID, &detail.ConnectionID, &detail.SourceName, &detail.State,
&detail.AddCount, &detail.UpdateCount, &detail.MoveCount, &detail.MoveScopeCount,
&detail.DeleteCount, &detail.NoChangeCount, &detail.CreatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return governance.CatalogChangeSetDetail{}, ErrCatalogChangeSetNotFound
}
if err != nil {
return governance.CatalogChangeSetDetail{}, fmt.Errorf("get latest catalog change set: %w", err)
}
if err := s.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND ($2='' OR action=$2)`, detail.ID, filter.Action).Scan(&detail.Total); err != nil {
return governance.CatalogChangeSetDetail{}, fmt.Errorf("count catalog change items: %w", err)
}
rows, err := s.db.Query(ctx, `SELECT action,title,old_path,new_path,old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,COALESCE(old_content_sha256::text,''),COALESCE(new_content_sha256::text,''),reason_code,item_seq,COALESCE(old_scope,''),COALESCE(new_scope,''),COALESCE(old_version_id,''),COALESCE(new_version_id,'')
FROM source_catalog_change_items
WHERE change_set_id=$1 AND ($2='' OR action=$2)
ORDER BY CASE action WHEN 'delete' THEN 1 WHEN 'update' THEN 2 WHEN 'move_scope' THEN 3 WHEN 'add' THEN 4 ELSE 5 END,
lower(title),source_locator_hash
LIMIT $3 OFFSET $4`, detail.ID, filter.Action, filter.Limit, filter.Offset)
if err != nil {
return governance.CatalogChangeSetDetail{}, fmt.Errorf("list catalog change items: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.CatalogChangeItem
if err := rows.Scan(&item.Action, &item.Title, &item.OldPath, &item.NewPath, &item.OldSizeBytes, &item.NewSizeBytes, &item.OldModified, &item.NewModified, &item.OldContentSHA256, &item.NewContentSHA256, &item.ReasonCode, &item.Seq, &item.OldScope, &item.NewScope, &item.OldVersionID, &item.NewVersionID); err != nil {
return governance.CatalogChangeSetDetail{}, fmt.Errorf("scan catalog change item: %w", err)
}
detail.Items = append(detail.Items, item)
}
if err := rows.Err(); err != nil {
return governance.CatalogChangeSetDetail{}, fmt.Errorf("iterate catalog change items: %w", err)
}
return detail, nil
}
func validCatalogChangeAction(action string) bool {
switch action {
case "", "add", "update", "move", "move_scope", "delete", "no_change":
return true
default:
return false
}
}
@@ -0,0 +1,47 @@
package postgres
import (
"context"
"os"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
func TestCatalogChangeSetPersistsSummaryAndFilteredPreview(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-change','alist','国内工业',true);
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-old','conn-change',repeat('1',64),'国内工业',2,1,1,'2026-09-10T12:00:00Z'),('snapshot-new','conn-change',repeat('2',64),'国内工业',3,1,2,'2026-09-10T12:00:00Z');
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,no_change_count,created_at)
VALUES('cs-change','conn-change','snapshot-old','snapshot-new','planned',1,1,0,1,5,'2026-09-10T12:00:00Z');
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,old_size_bytes,new_size_bytes,created_at)
VALUES('cs-change',repeat('a',64),'update','新版手册.pdf','手册.pdf','新版手册.pdf',10,12,'2026-09-10T12:00:00Z'),
('cs-change',repeat('b',64),'delete','旧手册.pdf','旧手册.pdf','',20,NULL,'2026-09-10T12:00:00Z');`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
detail, err := store.GetLatestCatalogChangeSet(ctx, governance.CatalogChangeSetFilter{ConnectionID: "conn-change", Action: "delete", Limit: 20})
if err != nil {
t.Fatal(err)
}
if detail.SourceName != "国内工业" || detail.AddCount != 1 || detail.UpdateCount != 1 || detail.DeleteCount != 1 || detail.Total != 1 || len(detail.Items) != 1 || detail.Items[0].Title != "旧手册.pdf" || detail.Items[0].Action != "delete" {
t.Fatalf("detail=%#v", detail)
}
}
@@ -0,0 +1,27 @@
package postgres
import (
"context"
"fmt"
"time"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
)
func (s *Store) UpdateCatalogRefreshProgress(ctx context.Context, jobID, owner string, progress catalogrefresh.Progress, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE jobs SET payload=payload||jsonb_build_object(
'stage',$3::text,'progress',$4::integer,'message',$5::text,'processed',$6::integer,'total',$7::integer,
'current_directory',NULLIF($9,''),'scanned_directories',$10::integer,'discovered_directories',$11::integer,
'scanned_files',$12::integer,'estimated_total_files',$13::integer,'total_is_estimate',$14::boolean,
'source_name',NULLIF($15,'')),updated_at=$8 WHERE id=$1 AND lease_owner=$2 AND state='running'`,
jobID, owner, progress.Stage, progress.Progress, progress.Message, progress.Processed, progress.Total, now,
progress.CurrentDirectory, progress.ScannedDirectories, progress.DiscoveredDirectories,
progress.ScannedFiles, progress.EstimatedTotalFiles, progress.TotalIsEstimate, progress.SourceName)
if err != nil {
return fmt.Errorf("update catalog refresh progress: %w", err)
}
if tag.RowsAffected() != 1 {
return ErrLeaseLost
}
return nil
}
@@ -0,0 +1,243 @@
package postgres
import (
"context"
"errors"
"fmt"
"strings"
"unicode"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) RouteCatalogFiles(ctx context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
if err := validateCatalogRouteRequest(request); err != nil {
return governance.CatalogRouteResult{}, err
}
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return governance.CatalogRouteResult{}, errors.New("catalog route transactions are unavailable")
}
tx, err := beginner.Begin(ctx)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("begin catalog route: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
result, err := routeCatalogFilesTx(ctx, tx, request)
if err != nil {
return governance.CatalogRouteResult{}, err
}
if err := tx.Commit(ctx); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("commit catalog route: %w", err)
}
return result, nil
}
func validateCatalogRouteRequest(request governance.CatalogRouteRequest) error {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
if !validRouteID(request.ConnectionID) || len(request.NodeIDs) == 0 || len(request.NodeIDs) > 100 ||
!validCatalogPolicy(request.Visibility, targetKnowledgeBaseIDs) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 ||
strings.TrimSpace(request.Reason) == "" || len(request.Reason) > 500 || request.Now.IsZero() {
return errors.New("invalid catalog route request")
}
seen := map[string]bool{}
for _, id := range request.NodeIDs {
if !validRouteID(id) || seen[id] {
return errors.New("invalid catalog route nodes")
}
seen[id] = true
}
return nil
}
func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
nodeIDs := append([]string(nil), request.NodeIDs...)
selectedRows, err := tx.Query(ctx, `SELECT node.node_id,node.node_type,node.relative_path
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
ORDER BY node.node_id
FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route nodes: %w", err)
}
selectedCount := 0
directoryPaths := make([]string, 0)
for selectedRows.Next() {
var nodeID, nodeType, directoryPath string
if err := selectedRows.Scan(&nodeID, &nodeType, &directoryPath); err != nil {
selectedRows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route node: %w", err)
}
selectedCount++
if nodeType == "folder" {
directoryPaths = append(directoryPaths, directoryPath)
}
}
if err := selectedRows.Err(); err != nil {
selectedRows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route nodes: %w", err)
}
selectedRows.Close()
if selectedCount != len(nodeIDs) {
return governance.CatalogRouteResult{}, errors.New("catalog route contains missing nodes")
}
for _, directoryPath := range directoryPaths {
if err := setCatalogDirectoryPolicy(ctx, tx, request, directoryPath); err != nil {
return governance.CatalogRouteResult{}, err
}
}
rows, err := tx.Query(ctx, `WITH RECURSIVE selected AS (
SELECT node.snapshot_id,node.node_id,node.node_type
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[])
), descendants AS (
SELECT snapshot_id,node_id,node_type FROM selected
UNION
SELECT child.snapshot_id,child.node_id,child.node_type
FROM descendants parent
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
WHERE parent.node_type='folder'
)
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
}
type selectedFile struct {
nodeID string
sourceItemID *string
direct bool
}
files := make([]selectedFile, 0, len(nodeIDs))
for rows.Next() {
var file selectedFile
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
}
files = append(files, file)
}
if err := rows.Err(); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
}
rows.Close()
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
routedItems := map[string]bool{}
for _, file := range files {
if file.sourceItemID == nil {
result.UnboundFiles++
continue
}
itemID := *file.sourceItemID
if routedItems[itemID] {
continue
}
routedItems[itemID] = true
if file.direct {
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
}
eventID, err := newOpaqueID("se-")
if err != nil {
return governance.CatalogRouteResult{}, err
}
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
}
}
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
var currentVersionID *string
var contentState, spoolRef string
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.id=$1`, itemID).Scan(&currentVersionID, &contentState, &spoolRef)
switch {
case errors.Is(err, pgx.ErrNoRows):
result.UnavailableFiles++
case err != nil:
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
case contentState == string(governance.ContentVerified) && spoolRef != "":
result.RoutableFiles++
default:
result.HashUnknownFiles++
}
}
return result, nil
}
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
if _, err := tx.Exec(ctx, `INSERT INTO directory_scope_rules(source_connection_id,directory_path,visibility,updated_by,reason,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$6,$6)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return fmt.Errorf("set routed directory scope: %w", err)
}
eventID, err := newOpaqueID("se-")
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_connection_id,directory_path,action,decided_visibility,operator_ref,reason,created_at)
VALUES($1,'directory',$2,$3,'set',$4,$5,$6,$7)`, eventID, request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return fmt.Errorf("audit routed directory scope: %w", err)
}
return nil
}
func validCatalogPolicy(policy governance.Visibility, targetKnowledgeBaseIDs []string) bool {
expectedTargets := 1
switch policy {
case governance.VisibilityInternal, governance.VisibilityExternal:
case governance.VisibilityBoth:
expectedTargets = 2
case governance.VisibilityBlocked:
expectedTargets = 0
default:
return false
}
if len(targetKnowledgeBaseIDs) != expectedTargets {
return false
}
seen := make(map[string]bool, len(targetKnowledgeBaseIDs))
for _, target := range targetKnowledgeBaseIDs {
if !validRouteID(target) || seen[target] {
return false
}
seen[target] = true
}
return true
}
func validRouteID(value string) bool {
if value == "" || len(value) > 128 {
return false
}
for _, character := range value {
if unicode.IsLetter(character) || unicode.IsDigit(character) || character == '-' || character == '_' {
continue
}
return false
}
return true
}
@@ -0,0 +1,158 @@
package postgres
import (
"context"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
// It never creates or mutates test_publications and never enqueues jobs.
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err = ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-route','pp','Route PP',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-route',repeat('c',64),10,'verified/file.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-route','conn-route','external-route',repeat('b',64),'工业产品/CRA系列','手册.pdf','document','v1','external','pp_public_tag');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-route','item-route','v1','content-route','verified');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-route','conn-route',repeat('a',64),'Route PP',4,2,2,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES
('snapshot-route','root-route',NULL,'folder','Route PP','',repeat('1',64),NULL,NULL,2,2,20),
('snapshot-route','folder-route','root-route','folder','工业产品','工业产品',repeat('4',64),NULL,NULL,1,1,10),
('snapshot-route','file-route','folder-route','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-route',10,0,1,10),
('snapshot-route','file-unmapped','root-route','file','未接入.pdf','未接入.pdf',repeat('3',64),NULL,10,0,1,10);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-route','snapshot-route',NOW());`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
if err != nil {
t.Fatal(err)
}
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
t.Fatalf("result=%#v", result)
}
// Desired state saved, recursion intact.
var directoryPolicy string
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
t.Fatal(err)
}
if directoryPolicy != "internal" {
t.Fatalf("directory policy=%q", directoryPolicy)
}
// Zero side effects: no publications, no jobs.
var publicationCount, jobCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
t.Fatal(err)
}
if publicationCount != 0 || jobCount != 0 {
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
}
// Re-annotating the same directory is idempotent for rules and still side-effect free.
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
t.Fatalf("again=%#v err=%v", again, err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
}
var ruleCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
}
// Switching policy updates desired state without retiring or removing anything.
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
t.Fatalf("both=%#v err=%v", both, err)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
}
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
t.Fatal(err)
}
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if blocked.SelectedFiles != 1 {
t.Fatalf("blocked=%#v", blocked)
}
var retrievalState string
var removalJobs int
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil {
t.Fatal(err)
}
if retrievalState != "enabled" || removalJobs != 0 {
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
}
// Unbound file and hash-unknown file are reported, never planned.
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
t.Fatalf("unbound=%#v err=%v", unbound, err)
}
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
t.Fatal(err)
}
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
}
// Overrides are still recorded for unbound/pending items so the desired state survives.
var overrideCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
}
// Audit trail exists for every annotation action.
var events int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
t.Fatalf("scope events=%d err=%v", events, err)
}
}
@@ -0,0 +1,113 @@
package postgres
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
func TestRouteCatalogFilesWithOperationReplaysOriginalResultAndRejectsConflictingPayload(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-op','pp','Operation PP',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-op',repeat('d',64),10,'verified/operation.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-op','conn-op','external-op',repeat('e',64),'工业产品','手册.pdf','document','v1','external','pp_public_tag');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-op','item-op','v1','content-op','verified');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-op','conn-op',repeat('f',64),'Operation PP',2,1,1,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES
('snapshot-op','root-op',NULL,'folder','Operation PP','',repeat('1',64),NULL,NULL,1,1,10),
('snapshot-op','file-op','root-op','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-op',10,0,1,10);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-op','snapshot-op',NOW());`); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
if err != nil || replayed || first.RoutableFiles != 1 {
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
}
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
if err != nil || !replayed || second != first {
t.Fatalf("replay result=%#v replayed=%v error=%v", second, replayed, err)
}
var publications, operations int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publications); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
t.Fatal(err)
}
if publications != 0 || operations != 1 {
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
}
conflicting := request
conflicting.Reason = "不同的操作内容"
_, _, err = store.RouteCatalogFilesWithOperation(ctx, conflicting, "op-route-1")
if !errors.Is(err, ErrOperationRequestConflict) {
t.Fatalf("conflicting operation error=%v, want ErrOperationRequestConflict", err)
}
}
func TestEnqueueCatalogRefreshWithOperationReplaysTheOriginalJob(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-refresh-op','pp','Refresh operation',true)`); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 11, 7, 0, 0, 0, time.UTC)
source := governance.CatalogSource{ConnectionID: "conn-refresh-op", SourceType: governance.SourceTypePP, Name: "Refresh operation"}
first, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now)
if err != nil || replayed || first.Type != "source.catalog.refresh" {
t.Fatalf("first job=%#v replayed=%v error=%v", first, replayed, err)
}
second, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now.Add(time.Minute))
if err != nil || !replayed || second.ID != first.ID || second.Type != first.Type {
t.Fatalf("replay job=%#v replayed=%v error=%v", second, replayed, err)
}
var jobs, operations int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh'`).Scan(&jobs); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.refresh' AND operation_id='op-refresh-1'`).Scan(&operations); err != nil {
t.Fatal(err)
}
if jobs != 1 || operations != 1 {
t.Fatalf("jobs=%d operations=%d", jobs, operations)
}
}
@@ -0,0 +1,358 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// ConfirmCatalogChangeSet executes ONLY the human-selected items of one planned
// ChangeSet: it creates publications + confirmations + test.upload jobs, or
// disables retrieval and enqueues test.remove for deletes. NO_CHANGE never
// enqueues. Re-confirmation replays the first stored outcome.
func (s *Store) ConfirmCatalogChangeSet(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) {
operator := strings.TrimSpace(request.OperatorRef)
reason := strings.TrimSpace(request.Reason)
if !validRouteID(request.ChangeSetID) || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() || len(request.ItemSeqs) == 0 || len(request.ItemSeqs) > 200 {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set confirmation")
}
seen := make(map[int64]bool, len(request.ItemSeqs))
for _, seq := range request.ItemSeqs {
if seq < 1 || seen[seq] {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set item selections")
}
seen[seq] = true
}
var tx pgx.Tx
ownsTransaction := false
if existing, ok := s.db.(pgx.Tx); ok {
tx = existing
} else {
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation transactions are unavailable")
}
var err error
tx, err = beginner.Begin(ctx)
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("begin change set confirmation: %w", err)
}
ownsTransaction = true
defer tx.Rollback(ctx)
}
result, err := confirmCatalogChangeSetTx(ctx, tx, request)
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
if ownsTransaction {
if err := tx.Commit(ctx); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("commit change set confirmation: %w", err)
}
}
return result, nil
}
func confirmCatalogChangeSetTx(ctx context.Context, tx pgx.Tx, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) {
result := governance.ConfirmCatalogChangeSetResult{ChangeSetID: request.ChangeSetID}
selections := append([]int64(nil), request.ItemSeqs...)
var state string
err := tx.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id=$1 FOR UPDATE`, request.ChangeSetID).Scan(&state)
if errors.Is(err, pgx.ErrNoRows) {
return governance.ConfirmCatalogChangeSetResult{}, ErrCatalogChangeSetNotFound
}
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("lock change set for confirmation: %w", err)
}
if state != "planned" {
// Even on replay the requested locators must belong to this change set.
var owned int
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND item_seq=ANY($2::bigint[])`, request.ChangeSetID, selections).Scan(&owned); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set item check: %w", err)
}
if owned != len(selections) {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items")
}
// Replay the first stored outcome from persisted evidence.
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM processing_confirmations WHERE change_set_id=$1`, request.ChangeSetID).Scan(&result.ConfirmedItems); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set confirmation: %w", err)
}
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.UploadsCreated); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set uploads: %w", err)
}
if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.remove' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.RemovalsRequested); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set removals: %w", err)
}
return result, nil
}
rows, err := tx.Query(ctx, `
SELECT item.item_seq,item.action,COALESCE(item.new_scope,''),COALESCE(item.old_scope,''),item.title,
COALESCE(item.new_source_item_id,item.old_source_item_id) AS source_item_id,
version.id AS version_id,
version.content_state='verified' AND content.content_state='verified' AND content.local_spool_ref<>'' AS artifact_ready
FROM source_catalog_change_items item
LEFT JOIN source_items source ON source.id=COALESCE(item.new_source_item_id,item.old_source_item_id)
LEFT JOIN source_item_versions version ON version.source_item_id=source.id AND version.revision=source.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.change_set_id=$1 AND item.item_seq=ANY($2::bigint[])
ORDER BY item.item_seq`, request.ChangeSetID, selections)
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("select change set items: %w", err)
}
type changeRow struct {
seq int64
action string
newScope string
oldScope string
sourceItemID *string
versionID *string
artifactReady bool
}
changes := make([]changeRow, 0, len(selections))
for rows.Next() {
var item changeRow
var title string
var versionID *string
var artifactReady *bool
if err := rows.Scan(&item.seq, &item.action, &item.newScope, &item.oldScope, &title, &item.sourceItemID, &versionID, &artifactReady); err != nil {
rows.Close()
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("scan change set item: %w", err)
}
if versionID != nil {
item.versionID = versionID
}
if artifactReady != nil {
item.artifactReady = *artifactReady
}
changes = append(changes, item)
}
if err := rows.Err(); err != nil {
rows.Close()
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("iterate change set items: %w", err)
}
rows.Close()
if len(changes) != len(selections) {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items")
}
for _, item := range changes {
if item.action == "no_change" {
result.NoChangeIgnored++
continue
}
if item.action == "delete" || item.newScope == string(governance.VisibilityBlocked) {
if item.sourceItemID == nil {
result.SkippedNoArtifact++
continue
}
removals, err := retireItemPublicationsForChange(ctx, tx, *item.sourceItemID, request)
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
result.RemovalsRequested += removals
result.ConfirmedItems++
continue
}
targets := make([]string, 0, 2)
switch item.newScope {
case string(governance.VisibilityInternal):
if request.InternalTargetKBID == "" {
result.SkippedNoTarget++
continue
}
targets = append(targets, request.InternalTargetKBID)
case string(governance.VisibilityExternal):
if request.ExternalTargetKBID == "" {
result.SkippedNoTarget++
continue
}
targets = append(targets, request.ExternalTargetKBID)
case string(governance.VisibilityBoth):
if request.InternalTargetKBID != "" {
targets = append(targets, request.InternalTargetKBID)
}
if request.ExternalTargetKBID != "" {
targets = append(targets, request.ExternalTargetKBID)
}
if len(targets) < 2 {
result.SkippedNoTarget++
continue
}
default:
result.SkippedNoTarget++
continue
}
if item.versionID == nil || !item.artifactReady {
result.SkippedNoArtifact++
continue
}
result.ConfirmedItems++
for _, target := range targets {
scope := item.newScope
if scope == string(governance.VisibilityBoth) {
scope = string(governance.VisibilityInternal)
if target == request.ExternalTargetKBID {
scope = string(governance.VisibilityExternal)
}
}
var existingPublication string
err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existingPublication)
if err == nil {
if err := ensureChangeSetConfirmation(ctx, tx, existingPublication, request); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
result.UploadsExisting++
continue
}
if !errors.Is(err, pgx.ErrNoRows) {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("inspect change set publication: %w", err)
}
publicationID, err := newOpaqueID("tp-")
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,change_set_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$6)
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
publicationID, *item.versionID, target, scope, request.ChangeSetID, request.Now)
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("create change set publication: %w", err)
}
if tag.RowsAffected() == 0 {
// A concurrent confirmation already owns this version+target.
var existing string
if err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existing); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("resolve concurrent publication: %w", err)
}
if err := ensureChangeSetConfirmation(ctx, tx, existing, request); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
result.UploadsExisting++
continue
}
jobID, err := newOpaqueID("job-")
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
payload, err := json.Marshal(map[string]string{"publication_id": publicationID, "change_set_id": request.ChangeSetID, "item_seq": strconv.FormatInt(item.seq, 10), "target_scope": scope, "target_kb_id": target})
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, errors.New("encode change set upload job")
}
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3,$5,$6,$7,'upload')
ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "test-start:"+publicationID, request.ChangeSetID, *item.versionID, target); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("enqueue change set upload: %w", err)
}
if err := ensureChangeSetConfirmation(ctx, tx, publicationID, request); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, err
}
result.UploadsCreated++
}
}
if _, err := tx.Exec(ctx, `UPDATE source_catalog_change_sets SET state='confirmed',confirmed_at=$2 WHERE id=$1 AND state='planned'`, request.ChangeSetID, request.Now); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("confirm change set: %w", err)
}
return result, nil
}
func ensureChangeSetConfirmation(ctx context.Context, tx pgx.Tx, publicationID string, request governance.ConfirmCatalogChangeSetRequest) error {
confirmationID, err := newOpaqueID("pc-")
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,change_set_id) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT (publication_id) DO NOTHING`,
confirmationID, publicationID, request.OperatorRef, request.Reason, request.Now, request.ChangeSetID); err != nil {
return fmt.Errorf("record change set confirmation: %w", err)
}
return nil
}
func retireItemPublicationsForChange(ctx context.Context, tx pgx.Tx, itemID string, request governance.ConfirmCatalogChangeSetRequest) (int, error) {
rows, err := tx.Query(ctx, `SELECT publication.id,publication.knowledge_id,publication.retrieval_state
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
WHERE version.source_item_id=$1 AND publication.retrieval_state NOT IN ('disabling','disabled','unpublished')
FOR UPDATE OF publication`, itemID)
if err != nil {
return 0, fmt.Errorf("inspect change set removals: %w", err)
}
type target struct{ id, knowledgeID string }
targets := make([]target, 0)
for rows.Next() {
var item target
if err := rows.Scan(&item.id, &item.knowledgeID, new(string)); err != nil {
rows.Close()
return 0, fmt.Errorf("scan change set removal: %w", err)
}
targets = append(targets, item)
}
if err := rows.Err(); err != nil {
rows.Close()
return 0, fmt.Errorf("iterate change set removals: %w", err)
}
rows.Close()
removals := 0
for _, item := range targets {
if item.knowledgeID == "" {
if _, err := tx.Exec(ctx, `UPDATE test_publications SET ingest_state='failed',processing_state='failed',retrieval_state='disabled',last_error='blocked_by_changeset',updated_at=$2 WHERE id=$1 AND knowledge_id=''`, item.id, request.Now); err != nil {
return 0, fmt.Errorf("stop pre-upload change set removal: %w", err)
}
continue
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET retrieval_state='disabling',last_error='',updated_at=$2 WHERE id=$1`, item.id, request.Now); err != nil {
return 0, fmt.Errorf("mark change set removal: %w", err)
}
jobID, err := newOpaqueID("job-")
if err != nil {
return 0, err
}
payload, err := json.Marshal(map[string]string{"publication_id": item.id, "change_set_id": request.ChangeSetID})
if err != nil {
return 0, errors.New("encode change set remove job")
}
// Removals key on the item (version-independent): a confirmed delete must
// not enqueue twice even if version rows rotate between deliveries.
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES($1,'test.remove',$2,'pending',20,$3,10,$4,$3,$3,$5,$6,'','remove')
ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "changeset-remove:"+item.id+":"+request.ChangeSetID, request.ChangeSetID, item.id); err != nil {
return 0, fmt.Errorf("enqueue change set removal: %w", err)
}
removals++
}
return removals, nil
}
func (s *Store) ConfirmCatalogChangeSetWithOperation(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest, operationID string) (governance.ConfirmCatalogChangeSetResult, bool, error) {
fingerprint, err := operationFingerprint(struct {
ChangeSetID string `json:"change_set_id"`
ItemSeqs []int64 `json:"item_seqs"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{request.ChangeSetID, request.ItemSeqs, request.OperatorRef, request.Reason})
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := NewStore(tx).ConfirmCatalogChangeSet(ctx, request)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.ConfirmCatalogChangeSetResult{}, false, err
}
var result governance.ConfirmCatalogChangeSetResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.ConfirmCatalogChangeSetResult{}, false, fmt.Errorf("decode stored change set result: %w", err)
}
return result, replayed, nil
}
@@ -0,0 +1,44 @@
package postgres_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"net/url"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
func isolatedExternalIntegrationDatabaseURL(t *testing.T, databaseURL string) string {
t.Helper()
if databaseURL == "" {
return ""
}
parsed, err := url.Parse(databaseURL)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err)
}
digest := sha256.Sum256([]byte(t.Name()))
schema := "kgtestx_" + hex.EncodeToString(digest[:8])
ctx := context.Background()
admin, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect integration database: %v", err)
}
defer admin.Close()
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
t.Fatalf("create isolated integration schema: %v", err)
}
t.Cleanup(func() {
cleanup, err := pgxpool.New(context.Background(), databaseURL)
if err == nil {
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
cleanup.Close()
}
})
query := parsed.Query()
query.Set("search_path", schema)
parsed.RawQuery = query.Encode()
return parsed.String()
}
@@ -0,0 +1,44 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"net/url"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
func isolatedIntegrationDatabaseURL(t *testing.T, databaseURL string) string {
t.Helper()
if databaseURL == "" {
return ""
}
parsed, err := url.Parse(databaseURL)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err)
}
digest := sha256.Sum256([]byte(t.Name()))
schema := "kgtest_" + hex.EncodeToString(digest[:8])
ctx := context.Background()
admin, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect integration database: %v", err)
}
defer admin.Close()
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
t.Fatalf("create isolated integration schema: %v", err)
}
t.Cleanup(func() {
cleanup, err := pgxpool.New(context.Background(), databaseURL)
if err == nil {
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
cleanup.Close()
}
})
query := parsed.Query()
query.Set("search_path", schema)
parsed.RawQuery = query.Encode()
return parsed.String()
}
+201
View File
@@ -0,0 +1,201 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
const listJobFailuresSQL = `
SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code,
jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id,
jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at,
COALESCE(siv.source_item_id,'')
FROM job_failures jf
LEFT JOIN test_publications p ON p.id = jf.publication_id
LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id
WHERE ($1 = '' OR jf.state = $1)
ORDER BY jf.created_at DESC, jf.id DESC
LIMIT $2 OFFSET $3
`
const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)`
// ListJobFailures is the DLQ view (P6): projections of terminal attempts. The
// original jobs and publication audit rows are never moved or deleted.
func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) {
state := filter.State
if state == "" {
state = "open"
}
if state != "open" && state != "requeued" && state != "abandoned" {
return governance.JobFailurePage{}, errors.New("invalid failure state filter")
}
limit, offset := filter.Limit, filter.Offset
if limit <= 0 {
limit = 50
}
if limit > 100 {
return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100")
}
if offset < 0 {
offset = 0
}
var total int
if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err)
}
rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset)
if err != nil {
return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err)
}
defer rows.Close()
items := make([]governance.JobFailure, 0, limit)
for rows.Next() {
var item governance.JobFailure
if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode,
&item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID,
&item.State, &item.CreatedAt, &item.SourceItemID); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err)
}
return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
const failurePublicationStateSQL = `
SELECT p.processing_state, p.ingest_state,
EXISTS (
SELECT 1 FROM source_item_versions siv
JOIN content_objects co ON co.id = siv.content_object_id
WHERE siv.id = p.source_item_version_id
AND co.content_state = 'verified'
AND co.local_spool_ref <> ''
)
FROM test_publications p
WHERE p.id = $1
`
const resolvePublicationFailuresSQL = `
UPDATE job_failures
SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4
WHERE publication_id = $1 AND state = 'open'
`
const recordRetryAuditSQL = `
INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (batch_key, publication_id) DO NOTHING
`
// RetryPublicationsBatchWithOperation performs the whole selection server-side
// (P6): one operation envelope, one transaction, per-item outcome, atomic audit.
func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) {
if operationID == "" || len(operationID) > 128 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id")
}
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items")
}
if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() {
return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required")
}
seen := make(map[string]bool, len(request.PublicationIDs))
ids := make([]string, 0, len(request.PublicationIDs))
for _, id := range request.PublicationIDs {
if id == "" || len(id) > 64 || seen[id] {
return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry")
}
seen[id] = true
ids = append(ids, id)
}
fingerprint, err := operationFingerprint(struct {
PublicationIDs []string `json:"publication_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{ids, request.OperatorRef, request.Reason})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.RetryPublicationsBatchResult{}, false, err
}
var result governance.RetryPublicationsBatchResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result")
}
return result, replayed, nil
}
func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) {
// Stable lock order prevents cross-batch deadlocks.
sorted := append([]string(nil), ids...)
for i := 1; i < len(sorted); i++ {
for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
}
}
result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))}
store := NewStore(tx)
for _, id := range sorted {
var processingState, ingestState string
var spoolAvailable bool
err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable)
outcome := governance.RetryOutcome{PublicationID: id}
switch {
case errors.Is(err, pgx.ErrNoRows):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication not found"
case err != nil:
return result, fmt.Errorf("classify publication retry: %w", err)
case processingState == "processing" || processingState == "finalizing":
outcome.Result = governance.RetryAlreadyRunning
case processingState == "pending" && ingestState != "failed":
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication is already queued"
case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"):
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "publication is not in a failed state"
case !spoolAvailable:
outcome.Result = governance.RetryNotRetryable
outcome.Detail = "no verified spool artifact"
default:
if err := store.RetryPublication(ctx, id, request.Now); err != nil {
return result, fmt.Errorf("retry publication %s: %w", id, err)
}
if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil {
return result, fmt.Errorf("resolve job failures: %w", err)
}
outcome.Result = governance.RetryStarted
}
if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil {
return result, fmt.Errorf("record retry audit: %w", err)
}
switch outcome.Result {
case governance.RetryStarted:
result.Started++
case governance.RetryAlreadyRunning:
result.AlreadyRunning++
default:
result.NotRetryable++
}
result.Outcomes = append(result.Outcomes, outcome)
}
result.Total = len(sorted)
return result, nil
}
@@ -0,0 +1,196 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
var ErrInvalidManualPublicationSeed = errors.New("invalid manual publication seed")
type ManualPublicationSeedRequest struct {
TestKBID string
Title string
SHA256 string
SizeBytes int64
MIME string
ValidatedFormat string
LocalSpoolRef string
Now time.Time
}
type ManualPublicationSeedResult struct {
PublicationID string
JobID string
}
const seedManualPublicationSQL = `
WITH ensured_connection AS (
INSERT INTO source_connections (
id, source_type, name, base_url, secret_ref, enabled,
schedule, last_health, created_at, updated_at
) VALUES ($3, 'manual', '本地手工验收', '', '', FALSE,
'{}'::jsonb, '{}'::jsonb, $2, $2)
ON CONFLICT (id) DO UPDATE
SET updated_at = source_connections.updated_at
RETURNING id
), ensured_content AS (
INSERT INTO content_objects (
id, sha256, size_bytes, mime, validated_format,
local_spool_ref, content_state,
first_discovered_at, last_discovered_at, created_at, updated_at
) VALUES ($10, $11, $8, $9, $12, $13, 'verified', $2, $2, $2, $2)
ON CONFLICT (sha256) DO UPDATE
SET last_discovered_at = EXCLUDED.last_discovered_at,
updated_at = EXCLUDED.updated_at,
local_spool_ref = CASE
WHEN content_objects.local_spool_ref = '' THEN EXCLUDED.local_spool_ref
ELSE content_objects.local_spool_ref
END
WHERE content_objects.size_bytes = EXCLUDED.size_bytes
RETURNING id
), ensured_item AS (
INSERT INTO source_items (
id, source_connection_id, stable_external_id, canonical_uri_hash,
normalized_path, title, item_type, size_bytes, mime, revision,
visibility, visibility_origin, source_change, created_at, updated_at,
identity_confidence
)
SELECT $4, id, $5, $6, '', $7, 'document', $8, $9, $15,
'internal', 'manual', 'added', $2, $2, 'content_hash'
FROM ensured_connection
ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE
SET updated_at = source_items.updated_at,
identity_confidence = CASE
WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence
ELSE EXCLUDED.identity_confidence
END
RETURNING id
), ensured_version AS (
INSERT INTO source_item_versions (
id, source_item_id, revision, content_object_id, content_state,
download_error, raw_record_summary, created_at, updated_at
)
SELECT $14, ensured_item.id, $15, ensured_content.id, 'verified', '',
'{"origin":"manual_e2e"}'::jsonb, $2, $2
FROM ensured_item CROSS JOIN ensured_content
ON CONFLICT (source_item_id, revision) DO UPDATE
SET updated_at = source_item_versions.updated_at
RETURNING id
), ensured_publication AS (
INSERT INTO test_publications (
id, source_item_version_id, test_kb_id,
ingest_state, processing_state, retrieval_state,
created_at, updated_at
)
SELECT $16, id, $17, 'pending', 'pending', 'unpublished', $2, $2
FROM ensured_version
ON CONFLICT (source_item_version_id, test_kb_id) DO UPDATE
SET updated_at = test_publications.updated_at
RETURNING id
), ensured_job AS (
INSERT INTO jobs (
id, type, payload, state, priority, available_at,
max_attempts, idempotency_key, created_at, updated_at
)
SELECT $18, 'test.upload', $1::jsonb, 'pending', 10, $2,
5, $19, $2, $2
FROM ensured_publication
ON CONFLICT (idempotency_key) DO UPDATE
SET idempotency_key = jobs.idempotency_key
RETURNING id
)
SELECT ensured_publication.id, ensured_job.id
FROM ensured_publication CROSS JOIN ensured_job
`
func (s *Store) SeedManualPublication(ctx context.Context, request ManualPublicationSeedRequest) (ManualPublicationSeedResult, error) {
if !validManualPublicationSeed(request) {
return ManualPublicationSeedResult{}, ErrInvalidManualPublicationSeed
}
identity := request.SHA256 + ":" + request.TestKBID
connectionID := "conn-manual-local"
sourceItemID := deterministicGovernanceID("si-", request.SHA256)
contentID := deterministicGovernanceID("co-", request.SHA256)
versionID := deterministicGovernanceID("sv-", request.SHA256)
publicationID := deterministicGovernanceID("tp-", identity)
jobID := deterministicGovernanceID("job-", "test.upload:"+publicationID)
stableExternalID := "manual-e2e:" + request.SHA256
canonicalDigest := sha256.Sum256([]byte(stableExternalID))
revision := "sha256:" + request.SHA256
payload, err := json.Marshal(struct {
PublicationID string `json:"publication_id"`
}{PublicationID: publicationID})
if err != nil {
return ManualPublicationSeedResult{}, errors.New("encode manual publication upload job")
}
idempotencyKey := "test-upload:" + publicationID
result := ManualPublicationSeedResult{}
err = s.db.QueryRow(
ctx,
seedManualPublicationSQL,
payload,
request.Now,
connectionID,
sourceItemID,
stableExternalID,
hex.EncodeToString(canonicalDigest[:]),
request.Title,
request.SizeBytes,
request.MIME,
contentID,
request.SHA256,
request.ValidatedFormat,
request.LocalSpoolRef,
versionID,
revision,
publicationID,
request.TestKBID,
jobID,
idempotencyKey,
).Scan(&result.PublicationID, &result.JobID)
if errors.Is(err, pgx.ErrNoRows) {
return ManualPublicationSeedResult{}, ErrPublicationStateConflict
}
if err != nil {
return ManualPublicationSeedResult{}, fmt.Errorf("seed manual test publication: %w", err)
}
return result, nil
}
func validManualPublicationSeed(request ManualPublicationSeedRequest) bool {
if !governance.ValidSHA256(request.SHA256) || request.SizeBytes <= 0 || request.Now.IsZero() {
return false
}
if len(request.TestKBID) == 0 || len(request.TestKBID) > 255 || strings.ContainsAny(request.TestKBID, "/\\\x00\t\r\n") {
return false
}
if strings.TrimSpace(request.Title) == "" || len(request.Title) > 512 || strings.ContainsRune(request.Title, '\x00') {
return false
}
if request.MIME == "" || len(request.MIME) > 128 || strings.ContainsAny(request.MIME, "\x00\r\n") || !validErrorCode(request.ValidatedFormat) {
return false
}
clean := path.Clean(request.LocalSpoolRef)
return len(request.LocalSpoolRef) <= 512 && clean == request.LocalSpoolRef && clean != "." && !strings.HasPrefix(clean, "/") && !strings.HasPrefix(clean, "../")
}
func deterministicGovernanceID(prefix string, parts ...string) string {
digest := sha256.New()
for _, part := range parts {
_, _ = digest.Write([]byte{0})
_, _ = digest.Write([]byte(part))
}
return prefix + hex.EncodeToString(digest.Sum(nil))[:32]
}
@@ -0,0 +1,60 @@
package postgres
import (
"context"
"strings"
"testing"
"time"
)
func TestSeedManualPublicationCreatesAtomicUploadChain(t *testing.T) {
db := &recordingDB{row: manualSeedRow{publicationID: "tp-seeded", jobID: "job-seeded"}}
store := NewStore(db)
now := time.Date(2026, 9, 5, 5, 0, 0, 0, time.UTC)
result, err := store.SeedManualPublication(context.Background(), ManualPublicationSeedRequest{
TestKBID: "c12ff4d5-dd87-423b-ab0f-af745b9d54a7",
Title: "Governance P3 E2E",
SHA256: strings.Repeat("a", 64),
SizeBytes: 128,
MIME: "text/markdown",
ValidatedFormat: "markdown",
LocalSpoolRef: "manual/e2e-a.md",
Now: now,
})
if err != nil {
t.Fatalf("SeedManualPublication() error = %v", err)
}
if result.PublicationID != "tp-seeded" || result.JobID != "job-seeded" {
t.Fatalf("result = %#v", result)
}
lowerSQL := strings.ToLower(db.queryRowSQL)
for _, required := range []string{
"insert into source_connections",
"insert into source_items",
"insert into content_objects",
"insert into source_item_versions",
"insert into test_publications",
"insert into jobs",
"on conflict",
"returning",
} {
if !strings.Contains(lowerSQL, required) {
t.Errorf("seed SQL missing %q", required)
}
}
if !strings.Contains(string(db.queryRowArgs[0].([]byte)), `"publication_id"`) || strings.Contains(string(db.queryRowArgs[0].([]byte)), "c12ff4d5") {
t.Fatalf("job payload = %s, want publication_id only", db.queryRowArgs[0])
}
}
type manualSeedRow struct {
publicationID string
jobID string
}
func (row manualSeedRow) Scan(dest ...any) error {
*(dest[0].(*string)) = row.publicationID
*(dest[1].(*string)) = row.jobID
return nil
}
+140
View File
@@ -0,0 +1,140 @@
package postgres
import (
"context"
"embed"
"fmt"
"io/fs"
"sort"
"strconv"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrationFS embed.FS
type migration struct {
Version int64
UpSQL string
DownSQL string
}
func loadMigrations() ([]migration, error) {
paths, err := fs.Glob(migrationFS, "migrations/*.up.sql")
if err != nil {
return nil, fmt.Errorf("glob migrations: %w", err)
}
result := make([]migration, 0, len(paths))
for _, path := range paths {
base := strings.TrimPrefix(path, "migrations/")
versionText, _, ok := strings.Cut(base, "_")
if !ok {
return nil, fmt.Errorf("invalid migration filename %q", path)
}
version, err := strconv.ParseInt(versionText, 10, 64)
if err != nil {
return nil, fmt.Errorf("parse migration version %q: %w", path, err)
}
up, err := migrationFS.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read up migration %q: %w", path, err)
}
downPath := strings.TrimSuffix(path, ".up.sql") + ".down.sql"
down, err := migrationFS.ReadFile(downPath)
if err != nil {
return nil, fmt.Errorf("read down migration %q: %w", downPath, err)
}
result = append(result, migration{Version: version, UpSQL: string(up), DownSQL: string(down)})
}
sort.Slice(result, func(i, j int) bool { return result[i].Version < result[j].Version })
return result, nil
}
const migrationLockID int64 = 7346729101
func ApplyMigrations(ctx context.Context, pool *pgxpool.Pool) error {
if _, err := pool.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil {
return fmt.Errorf("lock migrations: %w", err)
}
defer func() { _, _ = pool.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockID) }()
if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
version BIGINT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)`); err != nil {
return fmt.Errorf("create migration ledger: %w", err)
}
migrations, err := loadMigrations()
if err != nil {
return err
}
for _, item := range migrations {
var applied bool
if err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = $1)", item.Version).Scan(&applied); err != nil {
return fmt.Errorf("check migration %d: %w", item.Version, err)
}
if applied {
continue
}
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return fmt.Errorf("begin migration %d: %w", item.Version, err)
}
if _, err := tx.Exec(ctx, item.UpSQL); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("apply migration %d: %w", item.Version, err)
}
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version) VALUES ($1)", item.Version); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("record migration %d: %w", item.Version, err)
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit migration %d: %w", item.Version, err)
}
}
return nil
}
func RollbackLatestMigration(ctx context.Context, pool *pgxpool.Pool) error {
migrations, err := loadMigrations()
if err != nil {
return err
}
var version int64
if err := pool.QueryRow(ctx, "SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1").Scan(&version); err != nil {
if err == pgx.ErrNoRows {
return nil
}
return fmt.Errorf("read latest migration: %w", err)
}
var selected *migration
for i := range migrations {
if migrations[i].Version == version {
selected = &migrations[i]
break
}
}
if selected == nil {
return fmt.Errorf("missing down migration for version %d", version)
}
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return fmt.Errorf("begin rollback %d: %w", version, err)
}
if _, err := tx.Exec(ctx, selected.DownSQL); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("rollback migration %d: %w", version, err)
}
if _, err := tx.Exec(ctx, "DELETE FROM schema_migrations WHERE version = $1", version); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("delete migration ledger %d: %w", version, err)
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit rollback %d: %w", version, err)
}
return nil
}
@@ -0,0 +1 @@
DROP TABLE IF EXISTS jobs;
@@ -0,0 +1,31 @@
CREATE TABLE IF NOT EXISTS schema_migrations (
version BIGINT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS jobs (
id VARCHAR(36) PRIMARY KEY,
type VARCHAR(100) NOT NULL,
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed')),
priority INTEGER NOT NULL DEFAULT 0,
available_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
lease_owner VARCHAR(255),
lease_until TIMESTAMPTZ,
attempt INTEGER NOT NULL DEFAULT 0 CHECK (attempt >= 0),
max_attempts INTEGER NOT NULL DEFAULT 5 CHECK (max_attempts > 0),
last_error TEXT NOT NULL DEFAULT '',
idempotency_key VARCHAR(255) NOT NULL,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (idempotency_key)
);
CREATE INDEX IF NOT EXISTS idx_jobs_claim
ON jobs (priority DESC, available_at, created_at, id)
WHERE state IN ('pending', 'retry', 'running');
CREATE INDEX IF NOT EXISTS idx_jobs_state_updated
ON jobs (state, updated_at DESC);
@@ -0,0 +1,5 @@
DROP TABLE IF EXISTS test_publications;
DROP TABLE IF EXISTS source_item_versions;
DROP TABLE IF EXISTS content_objects;
DROP TABLE IF EXISTS source_items;
DROP TABLE IF EXISTS source_connections;
@@ -0,0 +1,114 @@
CREATE TABLE IF NOT EXISTS source_connections (
id VARCHAR(36) PRIMARY KEY,
source_type VARCHAR(20) NOT NULL
CHECK (source_type IN ('pp', 'alist', 'dingtalk')),
name VARCHAR(255) NOT NULL,
base_url TEXT NOT NULL DEFAULT '',
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
enabled BOOLEAN NOT NULL DEFAULT FALSE,
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(schedule) = 'object'),
last_health JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(last_health) = 'object'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS source_items (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
stable_external_id TEXT NOT NULL,
canonical_uri_hash CHAR(64) NOT NULL
CHECK (canonical_uri_hash ~ '^[0-9a-f]{64}$'),
parent_external_id TEXT NOT NULL DEFAULT '',
normalized_path TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL,
item_type VARCHAR(20) NOT NULL
CHECK (item_type IN ('document', 'file', 'link', 'video')),
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
mime TEXT NOT NULL DEFAULT '',
revision TEXT NOT NULL DEFAULT '',
updated_at_source TIMESTAMPTZ,
visibility VARCHAR(20) NOT NULL DEFAULT 'internal'
CHECK (visibility IN ('internal', 'external')),
visibility_origin VARCHAR(30) NOT NULL DEFAULT 'manual'
CHECK (visibility_origin IN ('pp_public_tag', 'configured_scope', 'manual')),
last_seen_scan_id VARCHAR(36) NOT NULL DEFAULT '',
source_change VARCHAR(20) NOT NULL DEFAULT 'unchanged'
CHECK (source_change IN ('unchanged', 'added', 'updated', 'deleted', 'unavailable')),
absence_streak INTEGER NOT NULL DEFAULT 0 CHECK (absence_streak >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_connection_id, stable_external_id)
);
CREATE TABLE IF NOT EXISTS content_objects (
id VARCHAR(36) PRIMARY KEY,
sha256 CHAR(64) NOT NULL
CHECK (sha256 ~ '^[0-9a-f]{64}$'),
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
mime TEXT NOT NULL DEFAULT '',
validated_format VARCHAR(50) NOT NULL DEFAULT '',
local_spool_ref TEXT NOT NULL DEFAULT '',
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
first_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (sha256)
);
CREATE TABLE IF NOT EXISTS source_item_versions (
id VARCHAR(36) PRIMARY KEY,
source_item_id VARCHAR(36) NOT NULL
REFERENCES source_items(id) ON DELETE RESTRICT,
revision TEXT NOT NULL,
content_object_id VARCHAR(36)
REFERENCES content_objects(id) ON DELETE RESTRICT,
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
download_error TEXT NOT NULL DEFAULT '',
raw_record_summary JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(raw_record_summary) = 'object'),
source_updated_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_item_id, revision)
);
CREATE TABLE IF NOT EXISTS test_publications (
id VARCHAR(36) PRIMARY KEY,
source_item_version_id VARCHAR(36) NOT NULL
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
test_kb_id VARCHAR(255) NOT NULL,
knowledge_id VARCHAR(255) NOT NULL DEFAULT '',
ingest_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (ingest_state IN ('pending', 'uploading', 'uploaded', 'failed')),
processing_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')),
retrieval_state VARCHAR(20) NOT NULL DEFAULT 'unpublished'
CHECK (retrieval_state IN ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed')),
parser_generation VARCHAR(255) NOT NULL DEFAULT '',
index_generation VARCHAR(255) NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_item_version_id, test_kb_id)
);
CREATE INDEX IF NOT EXISTS idx_source_items_change
ON source_items (source_connection_id, source_change, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_source_items_visibility
ON source_items (visibility, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_source_item_versions_content
ON source_item_versions (content_object_id)
WHERE content_object_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_test_publications_state
ON test_publications (processing_state, retrieval_state, updated_at DESC);
@@ -0,0 +1,12 @@
DROP TABLE IF EXISTS acceptance_results;
DROP INDEX IF EXISTS idx_test_publications_poll;
ALTER TABLE test_publications DROP COLUMN IF EXISTS superseded_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS supersedes_publication_id;
ALTER TABLE test_publications DROP COLUMN IF EXISTS stalled_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS completed_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS processing_started_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_observed_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_updated_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_status;
ALTER TABLE test_publications DROP COLUMN IF EXISTS poll_sequence;
@@ -0,0 +1,33 @@
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS poll_sequence INTEGER NOT NULL DEFAULT 0
CHECK (poll_sequence >= 0);
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_status VARCHAR(32) NOT NULL DEFAULT '';
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_updated_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_observed_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS stalled_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS supersedes_publication_id VARCHAR(36)
REFERENCES test_publications(id) ON DELETE RESTRICT;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS superseded_at TIMESTAMPTZ;
CREATE TABLE IF NOT EXISTS acceptance_results (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
run_id VARCHAR(36) NOT NULL,
check_type VARCHAR(32) NOT NULL
CHECK (check_type IN ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches')),
state VARCHAR(20) NOT NULL
CHECK (state IN ('pending', 'passed', 'failed', 'manual_required')),
policy_version VARCHAR(64) NOT NULL,
evidence_summary JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(evidence_summary) = 'object'),
observed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id, run_id, check_type)
);
CREATE INDEX IF NOT EXISTS idx_test_publications_poll
ON test_publications (processing_state, last_observed_at, updated_at);
CREATE INDEX IF NOT EXISTS idx_acceptance_results_publication
ON acceptance_results (publication_id, observed_at DESC);
@@ -0,0 +1,18 @@
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM source_connections
WHERE source_type = 'manual'
) THEN
RAISE EXCEPTION 'cannot downgrade while manual source connections exist';
END IF;
END
$$;
ALTER TABLE source_connections
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
ALTER TABLE source_connections
ADD CONSTRAINT source_connections_source_type_check
CHECK (source_type IN ('pp', 'alist', 'dingtalk'));
@@ -0,0 +1,6 @@
ALTER TABLE source_connections
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
ALTER TABLE source_connections
ADD CONSTRAINT source_connections_source_type_check
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual'));
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS source_scan_changes;
DROP TABLE IF EXISTS source_scans;
@@ -0,0 +1,43 @@
CREATE TABLE IF NOT EXISTS source_scans (
id VARCHAR(64) PRIMARY KEY,
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
source_type VARCHAR(32) NOT NULL CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
state VARCHAR(16) NOT NULL CHECK (state IN ('pending', 'running', 'completed', 'failed')),
snapshot_sha256 VARCHAR(64) NOT NULL DEFAULT '' CHECK (snapshot_sha256 = '' OR snapshot_sha256 ~ '^[0-9a-f]{64}$'),
observed_items INTEGER NOT NULL DEFAULT 0 CHECK (observed_items >= 0),
unchanged_items INTEGER NOT NULL DEFAULT 0 CHECK (unchanged_items >= 0),
added_items INTEGER NOT NULL DEFAULT 0 CHECK (added_items >= 0),
updated_items INTEGER NOT NULL DEFAULT 0 CHECK (updated_items >= 0),
over_limit_items INTEGER NOT NULL DEFAULT 0 CHECK (over_limit_items >= 0),
missing_items INTEGER NOT NULL DEFAULT 0 CHECK (missing_items >= 0),
planned_uploads INTEGER NOT NULL DEFAULT 0 CHECK (planned_uploads >= 0),
error_code VARCHAR(64) NOT NULL DEFAULT '',
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_scans_one_active
ON source_scans (source_connection_id)
WHERE state IN ('pending', 'running');
CREATE INDEX IF NOT EXISTS idx_source_scans_connection_created
ON source_scans (source_connection_id, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS source_scan_changes (
id VARCHAR(64) PRIMARY KEY,
source_scan_id VARCHAR(64) NOT NULL REFERENCES source_scans(id) ON DELETE RESTRICT,
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
stable_external_id VARCHAR(512) NOT NULL,
source_item_id VARCHAR(64) REFERENCES source_items(id) ON DELETE RESTRICT,
change_type VARCHAR(16) NOT NULL CHECK (change_type IN ('added', 'updated', 'unavailable')),
previous_revision VARCHAR(512) NOT NULL DEFAULT '',
observed_revision VARCHAR(512) NOT NULL DEFAULT '',
observed_size_bytes BIGINT CHECK (observed_size_bytes IS NULL OR observed_size_bytes >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_scan_id, stable_external_id)
);
CREATE INDEX IF NOT EXISTS idx_source_scan_changes_pending
ON source_scan_changes (source_connection_id, change_type, created_at DESC, id DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS release_batch_items;
DROP TABLE IF EXISTS release_batches;
DROP TABLE IF EXISTS publication_reviews;
@@ -0,0 +1,58 @@
CREATE TABLE IF NOT EXISTS publication_reviews (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
decision VARCHAR(16) NOT NULL
CHECK (decision IN ('approved', 'rejected')),
decided_visibility VARCHAR(20) NOT NULL
CHECK (decided_visibility IN ('internal', 'external')),
reviewer_ref VARCHAR(128) NOT NULL,
decision_code VARCHAR(64) NOT NULL
CHECK (decision_code ~ '^[a-z0-9_]+$'),
acceptance_run_id VARCHAR(36) NOT NULL,
publication_updated_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id, acceptance_run_id, publication_updated_at,
reviewer_ref, decision, decided_visibility, decision_code)
);
CREATE INDEX IF NOT EXISTS idx_publication_reviews_latest
ON publication_reviews (publication_id, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS release_batches (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
state VARCHAR(16) NOT NULL DEFAULT 'draft'
CHECK (state IN ('draft', 'sealed', 'cancelled')),
policy_version VARCHAR(64) NOT NULL,
created_by VARCHAR(128) NOT NULL,
sealed_by VARCHAR(128) NOT NULL DEFAULT '',
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
sealed_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_release_batches_state_created
ON release_batches (state, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS release_batch_items (
release_batch_id VARCHAR(36) NOT NULL
REFERENCES release_batches(id) ON DELETE RESTRICT,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
review_id VARCHAR(36) NOT NULL
REFERENCES publication_reviews(id) ON DELETE RESTRICT,
source_item_version_id VARCHAR(36) NOT NULL
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
acceptance_run_id VARCHAR(36) NOT NULL,
publication_updated_at TIMESTAMPTZ NOT NULL,
decided_visibility VARCHAR(20) NOT NULL
CHECK (decided_visibility IN ('internal', 'external')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (release_batch_id, publication_id),
UNIQUE (publication_id),
UNIQUE (review_id)
);
CREATE INDEX IF NOT EXISTS idx_release_batch_items_batch
ON release_batch_items (release_batch_id, publication_id);
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS release_item_executions;
DROP TABLE IF EXISTS release_executions;
@@ -0,0 +1,65 @@
CREATE TABLE IF NOT EXISTS release_executions (
id VARCHAR(36) PRIMARY KEY,
release_batch_id VARCHAR(36) NOT NULL
REFERENCES release_batches(id) ON DELETE RESTRICT,
operation VARCHAR(16) NOT NULL
CHECK (operation IN ('publish', 'rollback')),
prior_execution_id VARCHAR(36)
REFERENCES release_executions(id) ON DELETE RESTRICT,
state VARCHAR(16) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'running', 'completed', 'failed')),
target_environment VARCHAR(16) NOT NULL DEFAULT 'production'
CHECK (target_environment = 'production'),
authorized_by VARCHAR(128) NOT NULL,
authorization_code VARCHAR(64) NOT NULL,
job_id VARCHAR(36) NOT NULL UNIQUE
REFERENCES jobs(id) ON DELETE RESTRICT,
manifest_sha256 CHAR(64) NOT NULL DEFAULT ''
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
completed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (completed_item_count >= 0),
failed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (failed_item_count >= 0),
error_code VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
CHECK (operation = 'publish' OR prior_execution_id IS NOT NULL),
CHECK (completed_item_count + failed_item_count <= item_count)
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_publish_batch
ON release_executions (release_batch_id)
WHERE operation = 'publish';
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_rollback_prior
ON release_executions (prior_execution_id)
WHERE operation = 'rollback';
CREATE INDEX IF NOT EXISTS idx_release_executions_batch_created
ON release_executions (release_batch_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_release_executions_state_created
ON release_executions (state, created_at);
CREATE TABLE IF NOT EXISTS release_item_executions (
id VARCHAR(36) PRIMARY KEY,
release_execution_id VARCHAR(36) NOT NULL
REFERENCES release_executions(id) ON DELETE RESTRICT,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
state VARCHAR(24) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'staged', 'activated', 'verified', 'failed', 'rolled_back')),
bundle_sha256 CHAR(64) NOT NULL DEFAULT ''
CHECK (bundle_sha256 = '' OR bundle_sha256 ~ '^[0-9a-f]{64}$'),
previous_production_revision VARCHAR(128) NOT NULL DEFAULT '',
production_revision VARCHAR(128) NOT NULL DEFAULT '',
error_code VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
UNIQUE (release_execution_id, publication_id)
);
CREATE INDEX IF NOT EXISTS idx_release_item_executions_execution
ON release_item_executions (release_execution_id, created_at);
CREATE INDEX IF NOT EXISTS idx_release_item_executions_publication
ON release_item_executions (publication_id, created_at DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS scope_change_events;
DROP TABLE IF EXISTS source_item_scope_overrides;
DROP TABLE IF EXISTS directory_scope_rules;
@@ -0,0 +1,57 @@
CREATE TABLE IF NOT EXISTS directory_scope_rules (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
directory_path TEXT NOT NULL DEFAULT '',
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external')),
updated_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, directory_path)
);
CREATE TABLE IF NOT EXISTS source_item_scope_overrides (
source_item_id VARCHAR(36) NOT NULL
REFERENCES source_items(id) ON DELETE RESTRICT,
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external')),
updated_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_item_id)
);
CREATE TABLE IF NOT EXISTS scope_change_events (
id VARCHAR(36) PRIMARY KEY,
target_type VARCHAR(16) NOT NULL
CHECK (target_type IN ('directory', 'item')),
source_connection_id VARCHAR(36)
REFERENCES source_connections(id) ON DELETE RESTRICT,
directory_path TEXT NOT NULL DEFAULT '',
source_item_id VARCHAR(36)
REFERENCES source_items(id) ON DELETE RESTRICT,
action VARCHAR(16) NOT NULL
CHECK (action IN ('set', 'clear')),
previous_visibility VARCHAR(20)
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external')),
decided_visibility VARCHAR(20)
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external')),
operator_ref VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CHECK (
(target_type = 'directory' AND source_connection_id IS NOT NULL AND source_item_id IS NULL)
OR
(target_type = 'item' AND source_connection_id IS NULL AND source_item_id IS NOT NULL)
),
CHECK (
(action = 'set' AND decided_visibility IS NOT NULL)
OR
(action = 'clear' AND decided_visibility IS NULL)
)
);
CREATE INDEX IF NOT EXISTS idx_scope_change_events_target
ON scope_change_events (target_type, source_connection_id, source_item_id, created_at DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS source_catalog_heads;
DROP TABLE IF EXISTS source_catalog_nodes;
DROP TABLE IF EXISTS source_catalog_snapshots;
@@ -0,0 +1,56 @@
CREATE TABLE IF NOT EXISTS source_catalog_snapshots (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
snapshot_sha256 CHAR(64) NOT NULL
CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'),
source_label VARCHAR(255) NOT NULL,
node_count INTEGER NOT NULL CHECK (node_count >= 1),
folder_count INTEGER NOT NULL CHECK (folder_count >= 1),
file_count INTEGER NOT NULL CHECK (file_count >= 0),
observed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_connection_id, snapshot_sha256),
UNIQUE (source_connection_id, id)
);
CREATE TABLE IF NOT EXISTS source_catalog_nodes (
snapshot_id VARCHAR(36) NOT NULL
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
node_id VARCHAR(36) NOT NULL,
parent_node_id VARCHAR(36),
node_type VARCHAR(16) NOT NULL
CHECK (node_type IN ('folder', 'file')),
name TEXT NOT NULL CHECK (name <> '' AND length(name) <= 1024),
relative_path TEXT NOT NULL CHECK (length(relative_path) <= 4096),
source_locator_hash CHAR(64) NOT NULL
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
source_item_id VARCHAR(36)
REFERENCES source_items(id) ON DELETE RESTRICT,
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
modified_at TIMESTAMPTZ,
sort_order INTEGER NOT NULL DEFAULT 0,
direct_child_count INTEGER NOT NULL DEFAULT 0 CHECK (direct_child_count >= 0),
subtree_file_count INTEGER NOT NULL DEFAULT 0 CHECK (subtree_file_count >= 0),
subtree_bytes BIGINT NOT NULL DEFAULT 0 CHECK (subtree_bytes >= 0),
PRIMARY KEY (snapshot_id, node_id),
FOREIGN KEY (snapshot_id, parent_node_id)
REFERENCES source_catalog_nodes(snapshot_id, node_id) ON DELETE RESTRICT
DEFERRABLE INITIALLY DEFERRED
);
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_parent
ON source_catalog_nodes (snapshot_id, parent_node_id, node_type, sort_order, name, node_id);
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_source_item
ON source_catalog_nodes (source_item_id)
WHERE source_item_id IS NOT NULL;
CREATE TABLE IF NOT EXISTS source_catalog_heads (
source_connection_id VARCHAR(36) PRIMARY KEY
REFERENCES source_connections(id) ON DELETE RESTRICT,
snapshot_id VARCHAR(36) NOT NULL,
activated_at TIMESTAMPTZ NOT NULL,
FOREIGN KEY (source_connection_id, snapshot_id)
REFERENCES source_catalog_snapshots(source_connection_id, id) ON DELETE RESTRICT
);
@@ -0,0 +1,36 @@
UPDATE scope_change_events
SET previous_visibility = 'internal'
WHERE previous_visibility IN ('both', 'blocked');
UPDATE scope_change_events
SET decided_visibility = 'internal'
WHERE decided_visibility IN ('both', 'blocked');
UPDATE source_item_scope_overrides
SET visibility = 'internal'
WHERE visibility IN ('both', 'blocked');
UPDATE directory_scope_rules
SET visibility = 'internal'
WHERE visibility IN ('both', 'blocked');
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_decided_visibility_check
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_previous_visibility_check
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external'));
ALTER TABLE source_item_scope_overrides
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
ALTER TABLE source_item_scope_overrides
ADD CONSTRAINT source_item_scope_overrides_visibility_check
CHECK (visibility IN ('internal', 'external'));
ALTER TABLE directory_scope_rules
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
ALTER TABLE directory_scope_rules
ADD CONSTRAINT directory_scope_rules_visibility_check
CHECK (visibility IN ('internal', 'external'));
@@ -0,0 +1,25 @@
-- Visibility policies used by the source catalog. Publication reviews remain
-- scoped to one concrete internal or external test publication.
ALTER TABLE directory_scope_rules
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
ALTER TABLE directory_scope_rules
ADD CONSTRAINT directory_scope_rules_visibility_check
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE source_item_scope_overrides
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
ALTER TABLE source_item_scope_overrides
ADD CONSTRAINT source_item_scope_overrides_visibility_check
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_previous_visibility_check
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_decided_visibility_check
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external', 'both', 'blocked'));
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS processing_confirmations;
ALTER TABLE test_publications DROP COLUMN IF EXISTS target_scope;
@@ -0,0 +1,36 @@
ALTER TABLE test_publications
ADD COLUMN IF NOT EXISTS target_scope VARCHAR(16) NOT NULL DEFAULT 'internal'
CHECK (target_scope IN ('internal', 'external'));
UPDATE test_publications AS publication
SET target_scope = COALESCE(
(
SELECT review.decided_visibility
FROM publication_reviews AS review
WHERE review.publication_id = publication.id
AND review.decision = 'approved'
AND review.decided_visibility IN ('internal', 'external')
ORDER BY review.created_at DESC, review.id DESC
LIMIT 1
),
CASE WHEN item_rule.visibility IN ('internal', 'external') THEN item_rule.visibility END,
source.visibility
)
FROM source_item_versions AS version
JOIN source_items AS source ON source.id = version.source_item_id
LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id
WHERE publication.source_item_version_id = version.id;
CREATE TABLE IF NOT EXISTS processing_confirmations (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
operator_ref VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
confirmed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id)
);
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_confirmed
ON processing_confirmations (confirmed_at DESC, publication_id);
@@ -0,0 +1,7 @@
ALTER TABLE release_batch_items
ADD CONSTRAINT release_batch_items_publication_id_key UNIQUE (publication_id);
ALTER TABLE release_batch_items
ADD CONSTRAINT release_batch_items_review_id_key UNIQUE (review_id);
ALTER TABLE release_item_executions DROP COLUMN IF EXISTS action;
ALTER TABLE release_batch_items DROP COLUMN IF EXISTS action;
@@ -0,0 +1,12 @@
ALTER TABLE release_batch_items
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
CHECK (action IN ('upsert', 'delete'));
ALTER TABLE release_item_executions
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
CHECK (action IN ('upsert', 'delete'));
ALTER TABLE release_batch_items
DROP CONSTRAINT IF EXISTS release_batch_items_publication_id_key;
ALTER TABLE release_batch_items
DROP CONSTRAINT IF EXISTS release_batch_items_review_id_key;
@@ -0,0 +1 @@
DROP TABLE IF EXISTS ui_setting_revisions;
@@ -0,0 +1,24 @@
CREATE TABLE IF NOT EXISTS ui_setting_revisions (
id VARCHAR(36) PRIMARY KEY,
state VARCHAR(16) NOT NULL CHECK (state IN ('validated', 'active', 'retired', 'failed')),
document_refresh_seconds INTEGER NOT NULL CHECK (document_refresh_seconds IN (15, 30, 60)),
resource_refresh_seconds INTEGER NOT NULL CHECK (resource_refresh_seconds IN (30, 60)),
stalled_after_seconds INTEGER NOT NULL CHECK (stalled_after_seconds IN (1800, 3600)),
finalizing_after_seconds INTEGER NOT NULL CHECK (finalizing_after_seconds IN (7200, 14400)),
updated_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
activated_at TIMESTAMPTZ,
retired_at TIMESTAMPTZ,
error_code VARCHAR(64) NOT NULL DEFAULT '',
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$')
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_ui_setting_active
ON ui_setting_revisions ((state)) WHERE state = 'active';
INSERT INTO ui_setting_revisions (
id, state, document_refresh_seconds, resource_refresh_seconds,
stalled_after_seconds, finalizing_after_seconds, updated_by, activated_at
) VALUES (
'settings-default', 'active', 15, 30, 1800, 7200, 'system:migration', NOW()
) ON CONFLICT (id) DO NOTHING;
@@ -0,0 +1,2 @@
ALTER TABLE jobs
DROP COLUMN IF EXISTS started_at;
@@ -0,0 +1,2 @@
ALTER TABLE jobs
ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS source_catalog_change_items;
DROP TABLE IF EXISTS source_catalog_change_sets;
@@ -0,0 +1,42 @@
CREATE TABLE IF NOT EXISTS source_catalog_change_sets (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
from_snapshot_id VARCHAR(36)
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
to_snapshot_id VARCHAR(36) NOT NULL UNIQUE
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
state VARCHAR(16) NOT NULL DEFAULT 'planned'
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled')),
add_count INTEGER NOT NULL DEFAULT 0 CHECK (add_count >= 0),
update_count INTEGER NOT NULL DEFAULT 0 CHECK (update_count >= 0),
move_scope_count INTEGER NOT NULL DEFAULT 0 CHECK (move_scope_count >= 0),
delete_count INTEGER NOT NULL DEFAULT 0 CHECK (delete_count >= 0),
no_change_count INTEGER NOT NULL DEFAULT 0 CHECK (no_change_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
confirmed_at TIMESTAMPTZ,
applied_at TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS source_catalog_change_items (
change_set_id VARCHAR(36) NOT NULL
REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
source_locator_hash CHAR(64) NOT NULL
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
action VARCHAR(16) NOT NULL
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change')),
title TEXT NOT NULL,
old_path TEXT NOT NULL DEFAULT '',
new_path TEXT NOT NULL DEFAULT '',
old_size_bytes BIGINT CHECK (old_size_bytes IS NULL OR old_size_bytes >= 0),
new_size_bytes BIGINT CHECK (new_size_bytes IS NULL OR new_size_bytes >= 0),
old_modified_at TIMESTAMPTZ,
new_modified_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, source_locator_hash)
);
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_sets_source
ON source_catalog_change_sets (source_connection_id, created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_items_action
ON source_catalog_change_items (change_set_id, action, title);
@@ -0,0 +1,4 @@
DELETE FROM source_catalog_change_items
WHERE change_set_id IN (SELECT id FROM source_catalog_change_sets WHERE origin='backfill');
DELETE FROM source_catalog_change_sets WHERE origin='backfill';
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS origin;
@@ -0,0 +1,87 @@
ALTER TABLE source_catalog_change_sets
ADD COLUMN IF NOT EXISTS origin VARCHAR(16) NOT NULL DEFAULT 'refresh'
CHECK (origin IN ('refresh', 'backfill'));
WITH current_targets AS (
SELECT head.source_connection_id,head.snapshot_id AS to_snapshot_id,
(
SELECT prior.id
FROM source_catalog_snapshots prior
JOIN source_catalog_snapshots current ON current.id=head.snapshot_id
WHERE prior.source_connection_id=head.source_connection_id
AND prior.id<>head.snapshot_id
AND prior.observed_at<=current.observed_at
ORDER BY prior.observed_at DESC,prior.id DESC
LIMIT 1
) AS from_snapshot_id
FROM source_catalog_heads head
)
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,origin,created_at)
SELECT 'cs-'||md5(target.source_connection_id||'|'||target.to_snapshot_id),
target.source_connection_id,target.from_snapshot_id,target.to_snapshot_id,'planned','backfill',NOW()
FROM current_targets target
ON CONFLICT (to_snapshot_id) DO NOTHING;
WITH change_sets AS (
SELECT id,from_snapshot_id,to_snapshot_id
FROM source_catalog_change_sets
WHERE origin='backfill'
), old_files AS (
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
FROM change_sets change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
FROM change_sets change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
), compared AS (
SELECT COALESCE(new_files.change_set_id,old_files.change_set_id) AS change_set_id,
COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
COALESCE(new_files.name,old_files.name) AS title,
COALESCE(old_files.relative_path,'') AS old_path,
COALESCE(new_files.relative_path,'') AS new_path,
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
CASE
WHEN old_files.source_locator_hash IS NULL THEN 'add'
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
ELSE 'no_change'
END AS action
FROM old_files FULL OUTER JOIN new_files
ON new_files.change_set_id=old_files.change_set_id
AND new_files.source_locator_hash=old_files.source_locator_hash
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,created_at)
SELECT change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,NOW()
FROM compared
ON CONFLICT (change_set_id,source_locator_hash) DO NOTHING;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT change_set.id AS change_set_id,
COUNT(item.*) FILTER (WHERE item.action='add')::integer AS add_count,
COUNT(item.*) FILTER (WHERE item.action='update')::integer AS update_count,
COUNT(item.*) FILTER (WHERE item.action='move_scope')::integer AS move_scope_count,
COUNT(item.*) FILTER (WHERE item.action='delete')::integer AS delete_count,
COUNT(item.*) FILTER (WHERE item.action='no_change')::integer AS no_change_count
FROM source_catalog_change_sets change_set
LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
WHERE change_set.origin='backfill'
GROUP BY change_set.id
) summary
WHERE change_set.id=summary.change_set_id;
@@ -0,0 +1,5 @@
ALTER TABLE release_executions
DROP CONSTRAINT IF EXISTS release_execution_batch_progress_check,
DROP COLUMN IF EXISTS total_batches,
DROP COLUMN IF EXISTS current_batch,
DROP COLUMN IF EXISTS batch_size;
@@ -0,0 +1,16 @@
ALTER TABLE release_executions
ADD COLUMN IF NOT EXISTS batch_size INTEGER NOT NULL DEFAULT 50
CHECK (batch_size BETWEEN 1 AND 100),
ADD COLUMN IF NOT EXISTS current_batch INTEGER NOT NULL DEFAULT 0
CHECK (current_batch >= 0),
ADD COLUMN IF NOT EXISTS total_batches INTEGER NOT NULL DEFAULT 0
CHECK (total_batches >= 0),
ADD CONSTRAINT release_execution_batch_progress_check
CHECK (current_batch <= total_batches OR total_batches = 0);
UPDATE release_executions
SET total_batches = CASE WHEN item_count = 0 THEN 0 ELSE CEIL(item_count::numeric / batch_size)::integer END,
current_batch = CASE
WHEN state IN ('completed', 'failed') AND item_count > 0 THEN CEIL(item_count::numeric / batch_size)::integer
ELSE current_batch
END;
@@ -0,0 +1,11 @@
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM jobs WHERE state = 'cancelled') THEN
RAISE EXCEPTION 'cannot roll back job cancelled state while cancelled jobs exist';
END IF;
END
$$;
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed'));
@@ -0,0 +1,3 @@
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed', 'cancelled'));
@@ -0,0 +1 @@
DROP TABLE IF EXISTS operation_requests;
@@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS operation_requests (
operation_type TEXT NOT NULL CHECK (operation_type <> '' AND char_length(operation_type) <= 100),
operation_id TEXT NOT NULL CHECK (operation_id <> '' AND char_length(operation_id) <= 128),
request_fingerprint_sha256 CHAR(64) NOT NULL CHECK (request_fingerprint_sha256 ~ '^[0-9a-f]{64}$'),
state TEXT NOT NULL DEFAULT 'completed' CHECK (state IN ('completed')),
result JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ NOT NULL,
UNIQUE (operation_type, operation_id)
);
@@ -0,0 +1 @@
ALTER TABLE source_catalog_nodes DROP COLUMN IF EXISTS content_sha256;
@@ -0,0 +1,3 @@
ALTER TABLE source_catalog_nodes
ADD COLUMN IF NOT EXISTS content_sha256 CHAR(64)
CHECK (content_sha256 IS NULL OR content_sha256 ~ '^[0-9a-f]{64}$');
@@ -0,0 +1,2 @@
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS reason_code, DROP COLUMN IF EXISTS new_content_sha256, DROP COLUMN IF EXISTS old_content_sha256;
ALTER TABLE source_items DROP COLUMN IF EXISTS identity_confidence;
@@ -0,0 +1,12 @@
-- P2 stable identity: source items record how strongly their identity is anchored.
ALTER TABLE source_items
ADD COLUMN IF NOT EXISTS identity_confidence VARCHAR(16) NOT NULL DEFAULT 'path'
CHECK (identity_confidence IN ('stable_id','object_id','content_hash','path'));
-- Change items gain content-hash evidence and a reason code so a rename can be
-- classified as a proven move instead of delete+add when the bytes match 1:1.
ALTER TABLE source_catalog_change_items
ADD COLUMN IF NOT EXISTS old_content_sha256 CHAR(64) CHECK (old_content_sha256 IS NULL OR old_content_sha256 ~ '^[0-9a-f]{64}$'),
ADD COLUMN IF NOT EXISTS new_content_sha256 CHAR(64) CHECK (new_content_sha256 IS NULL OR new_content_sha256 ~ '^[0-9a-f]{64}$'),
ADD COLUMN IF NOT EXISTS reason_code VARCHAR(32) NOT NULL DEFAULT 'locator'
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
@@ -0,0 +1 @@
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
@@ -0,0 +1,99 @@
-- P2/S8: canonical snapshot-diff classifier lives in the database so the
-- importer, backfills and future ChangeSet execution share one source of truth.
-- Two sequential statements (not a data-modifying CTE) so the recount sees the rows just written.
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
WITH change_set AS (
SELECT id,from_snapshot_id,to_snapshot_id FROM source_catalog_change_sets
WHERE id=p_change_set_id
), old_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), proven_moves AS (
-- A content hash proves a move only when exactly one old path carries it and
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
SELECT content_sha256 FROM (
SELECT old_files.content_sha256 FROM old_files WHERE old_files.content_sha256 IS NOT NULL
GROUP BY old_files.content_sha256 HAVING COUNT(*)=1
) one_old
JOIN (
SELECT new_files.content_sha256 FROM new_files WHERE new_files.content_sha256 IS NOT NULL
GROUP BY new_files.content_sha256 HAVING COUNT(*)=1
) one_new USING (content_sha256)
), compared AS (
SELECT COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
COALESCE(new_files.name,old_files.name) AS title,
COALESCE(old_files.relative_path,'') AS old_path,
COALESCE(new_files.relative_path,'') AS new_path,
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
old_files.content_sha256 AS old_content_sha256,new_files.content_sha256 AS new_content_sha256,
CASE
WHEN old_files.source_locator_hash IS NULL THEN 'add'
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
ELSE 'no_change'
END AS action,
CASE
WHEN old_files.source_locator_hash IS NULL AND new_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN new_files.source_locator_hash IS NULL AND old_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN old_files.source_locator_hash IS NULL THEN 'locator'
WHEN new_files.source_locator_hash IS NULL THEN 'source_missing'
WHEN old_files.content_sha256 IS DISTINCT FROM new_files.content_sha256
AND (old_files.relative_path<>new_files.relative_path OR old_files.name<>new_files.name)
THEN 'content_hash_conflict'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path OR old_files.name IS DISTINCT FROM new_files.name
THEN 'path_change'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes THEN 'size_change'
WHEN old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'content_change'
ELSE 'locator'
END AS reason_code
FROM old_files FULL OUTER JOIN new_files USING(source_locator_hash)
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,created_at)
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,NOW()
FROM change_set CROSS JOIN compared
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
reason_code=EXCLUDED.reason_code;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT items.change_set_id,
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
FROM source_catalog_change_items items
WHERE items.change_set_id=p_change_set_id
GROUP BY items.change_set_id
) summary
WHERE change_set.id=summary.change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -0,0 +1,26 @@
-- Revert S8 contract. Change rows written with new vocabulary remain but become
-- unclassifiable only if constraints reject them; we keep the checks permissive
-- first, then drop added columns.
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_version_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_version_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_source_item_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_source_item_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_scope;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_scope;
UPDATE source_catalog_change_items SET action='move_scope' WHERE action='move';
UPDATE source_catalog_change_items SET reason_code='path_change' WHERE reason_code='scope_change';
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change'));
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS failure_reason;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS cancelled_at;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS applying_at;
UPDATE source_catalog_change_sets SET move_scope_count=move_scope_count+move_count, move_count=0;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS move_count;
UPDATE source_catalog_change_sets SET state='confirmed' WHERE state IN ('applying','failed');
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled'));
@@ -0,0 +1,167 @@
-- S8 complete ChangeSet contract: full lifecycle, distinct move vs move_scope,
-- old/new effective scope, and version identity evidence.
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
CHECK (state IN ('planned', 'confirmed', 'applying', 'applied', 'cancelled', 'failed'));
ALTER TABLE source_catalog_change_sets
ADD COLUMN IF NOT EXISTS move_count INTEGER NOT NULL DEFAULT 0 CHECK (move_count >= 0),
ADD COLUMN IF NOT EXISTS applying_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS cancelled_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS failure_reason TEXT NOT NULL DEFAULT '';
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
CHECK (action IN ('add', 'update', 'move', 'move_scope', 'delete', 'no_change'));
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing','scope_change'));
ALTER TABLE source_catalog_change_items
ADD COLUMN IF NOT EXISTS old_scope VARCHAR(16),
ADD COLUMN IF NOT EXISTS new_scope VARCHAR(16),
ADD COLUMN IF NOT EXISTS old_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS new_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS old_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS new_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT;
-- Canonical classifier v2 (single source of truth; importer and backfills call it).
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
WITH change_set AS (
SELECT id,from_snapshot_id,to_snapshot_id,source_connection_id FROM source_catalog_change_sets
WHERE id=p_change_set_id
), old_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_item_id
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_item_id
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
)
-- Old side reports the source-recorded visibility (what the last sync stored);
-- new side reports the currently effective policy (override > item). A policy
-- change between snapshots therefore surfaces as move_scope, not a fake move.
, scoped_old AS (
SELECT old_files.*,
(SELECT item.visibility FROM source_items item WHERE item.id=old_files.source_item_id) AS item_scope
FROM old_files
), scoped_new AS (
SELECT new_files.*,
(SELECT COALESCE(override.visibility,item.visibility)
FROM source_items item
LEFT JOIN source_item_scope_overrides override ON override.source_item_id=item.id
WHERE item.id=new_files.source_item_id) AS item_scope
FROM new_files
), with_rule_scope AS (
SELECT f.*,COALESCE(f.item_scope,(
SELECT rule.visibility FROM directory_scope_rules rule
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
FROM (SELECT * FROM scoped_old) f
), with_rule_scope_new AS (
SELECT f.*,COALESCE(f.item_scope,(
SELECT rule.visibility FROM directory_scope_rules rule
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
FROM (SELECT * FROM scoped_new) f
), proven_moves AS (
-- A content hash proves a move only when exactly one old path carries it and
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
SELECT content_sha256 FROM (
SELECT content_sha256 FROM scoped_old WHERE content_sha256 IS NOT NULL
GROUP BY content_sha256 HAVING COUNT(*)=1
) one_old
JOIN (
SELECT content_sha256 FROM scoped_new WHERE content_sha256 IS NOT NULL
GROUP BY content_sha256 HAVING COUNT(*)=1
) one_new USING (content_sha256)
), compared AS (
SELECT COALESCE(n.source_locator_hash,o.source_locator_hash) AS source_locator_hash,
COALESCE(n.name,o.name) AS title,
COALESCE(o.relative_path,'') AS old_path,
COALESCE(n.relative_path,'') AS new_path,
o.size_bytes AS old_size_bytes,n.size_bytes AS new_size_bytes,
o.modified_at AS old_modified_at,n.modified_at AS new_modified_at,
o.content_sha256 AS old_content_sha256,n.content_sha256 AS new_content_sha256,
o.effective_scope AS old_scope,n.effective_scope AS new_scope,
o.source_item_id AS old_source_item_id,n.source_item_id AS new_source_item_id,
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=o.source_item_id AND i.revision=v.revision WHERE v.source_item_id=o.source_item_id LIMIT 1) AS old_version_id,
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=n.source_item_id AND i.revision=v.revision WHERE v.source_item_id=n.source_item_id LIMIT 1) AS new_version_id,
CASE
WHEN o.source_locator_hash IS NULL THEN 'add'
WHEN n.source_locator_hash IS NULL THEN 'delete'
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'move_scope'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256
AND (o.relative_path<>n.relative_path OR o.name<>n.name) THEN 'move'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256<>n.content_sha256 THEN 'update'
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'move_scope'
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes OR o.modified_at IS DISTINCT FROM n.modified_at THEN 'update'
ELSE 'no_change'
END AS action,
CASE
WHEN o.source_locator_hash IS NULL AND n.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN n.source_locator_hash IS NULL AND o.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN o.source_locator_hash IS NULL THEN 'locator'
WHEN n.source_locator_hash IS NULL THEN 'source_missing'
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'scope_change'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256 THEN 'content_hash_move'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL THEN 'content_hash_conflict'
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'path_change'
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes THEN 'size_change'
WHEN o.modified_at IS DISTINCT FROM n.modified_at THEN 'content_change'
ELSE 'locator'
END AS reason_code
FROM with_rule_scope o FULL OUTER JOIN with_rule_scope_new n USING(source_locator_hash)
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,
old_scope,new_scope,old_source_item_id,new_source_item_id,old_version_id,new_version_id,created_at)
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,
compared.old_scope,compared.new_scope,compared.old_source_item_id,compared.new_source_item_id,compared.old_version_id,compared.new_version_id,NOW()
FROM change_set CROSS JOIN compared
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
reason_code=EXCLUDED.reason_code,
old_scope=EXCLUDED.old_scope,new_scope=EXCLUDED.new_scope,
old_source_item_id=EXCLUDED.old_source_item_id,new_source_item_id=EXCLUDED.new_source_item_id,
old_version_id=EXCLUDED.old_version_id,new_version_id=EXCLUDED.new_version_id;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_count=summary.move_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT items.change_set_id,
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
COUNT(*) FILTER (WHERE items.action='move')::integer AS move_count,
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
FROM source_catalog_change_items items
WHERE items.change_set_id=p_change_set_id
GROUP BY items.change_set_id
) summary
WHERE change_set.id=summary.change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -0,0 +1,5 @@
DROP INDEX IF EXISTS idx_source_catalog_change_items_seq;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS item_seq;
ALTER TABLE test_publications DROP COLUMN IF EXISTS change_locator_hash, DROP COLUMN IF EXISTS change_set_id;
DROP INDEX IF EXISTS idx_processing_confirmations_change_set;
ALTER TABLE processing_confirmations DROP COLUMN IF EXISTS change_set_id;
@@ -0,0 +1,14 @@
-- P3: confirmations bind to the ChangeSet that produced them; publications record
-- provenance so every upload/removal job can be traced back to change evidence.
ALTER TABLE processing_confirmations
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT;
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_change_set
ON processing_confirmations (change_set_id) WHERE change_set_id IS NOT NULL;
-- Surrogate public handle for selecting change items without leaking locator hashes.
ALTER TABLE source_catalog_change_items ADD COLUMN IF NOT EXISTS item_seq BIGSERIAL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_catalog_change_items_seq ON source_catalog_change_items (item_seq);
ALTER TABLE test_publications
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS change_locator_hash CHAR(64) CHECK (change_locator_hash IS NULL OR change_locator_hash ~ '^[0-9a-f]{64}$');
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS jobs_processing_identity;
ALTER TABLE jobs DROP COLUMN IF EXISTS business_action, DROP COLUMN IF EXISTS business_target_kb_id, DROP COLUMN IF EXISTS business_version_id, DROP COLUMN IF EXISTS business_change_set_id;
@@ -0,0 +1,13 @@
-- P5 formal processing identity: every confirmed change-set job carries an
-- explicit business identity (publication uniqueness already exists since
-- migration 000002) so a duplicate delivery or worker restart cannot enqueue
-- the same action twice.
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_change_set_id VARCHAR(36);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_version_id VARCHAR(36);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_target_kb_id VARCHAR(255);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_action VARCHAR(20)
CHECK (business_action IS NULL OR business_action IN ('upload', 'remove'));
CREATE UNIQUE INDEX IF NOT EXISTS jobs_processing_identity
ON jobs (business_change_set_id, business_version_id, business_target_kb_id, business_action)
WHERE business_change_set_id IS NOT NULL;
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS job_retry_operations;
DROP TABLE IF EXISTS job_failures;
ALTER TABLE jobs DROP COLUMN IF EXISTS last_error_code;
@@ -0,0 +1,48 @@
-- P6 DLQ: every terminal job failure keeps its immutable jobs audit row AND
-- gains a dead-letter projection. Requeued failures flip to 'requeued'; the
-- history is append-only. Retrying resets the attempt budget on the SAME job
-- row so the P5 business identity can never be duplicated.
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS last_error_code VARCHAR(64) NOT NULL DEFAULT '';
CREATE TABLE IF NOT EXISTS job_failures (
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
job_id VARCHAR(36) NOT NULL REFERENCES jobs(id),
job_type VARCHAR(100) NOT NULL,
publication_id VARCHAR(64) NOT NULL DEFAULT '',
error_code VARCHAR(64) NOT NULL,
last_error TEXT NOT NULL,
attempt INTEGER NOT NULL CHECK (attempt > 0),
max_attempts_at_failure INTEGER NOT NULL CHECK (max_attempts_at_failure > 0),
business_change_set_id VARCHAR(64) NOT NULL DEFAULT '',
business_version_id VARCHAR(64) NOT NULL DEFAULT '',
business_action VARCHAR(16) NOT NULL DEFAULT '',
business_target_kb_id VARCHAR(64) NOT NULL DEFAULT '',
state VARCHAR(16) NOT NULL DEFAULT 'open'
CHECK (state IN ('open', 'requeued', 'abandoned')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
resolved_at TIMESTAMPTZ,
resolution_actor_ref VARCHAR(255) NOT NULL DEFAULT '',
resolution_reason TEXT NOT NULL DEFAULT '',
UNIQUE (job_id, attempt)
);
CREATE INDEX IF NOT EXISTS idx_job_failures_open
ON job_failures (created_at DESC) WHERE state = 'open';
-- Retry audit: who re-queued which publication and with what reason
-- (batch-shared key). job_id is not stored: it is derivable from the
-- publication and would break the audit row on cross-batch duplicates.
CREATE TABLE IF NOT EXISTS job_retry_operations (
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
batch_key VARCHAR(255) NOT NULL,
publication_id VARCHAR(64) NOT NULL,
actor_ref VARCHAR(255) NOT NULL,
reason TEXT NOT NULL,
outcome VARCHAR(24) NOT NULL
CHECK (outcome IN ('started', 'already_running', 'not_retryable')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (batch_key, publication_id)
);
CREATE INDEX IF NOT EXISTS idx_job_retry_operations_publication
ON job_retry_operations (publication_id, created_at DESC);
@@ -0,0 +1 @@
DROP TABLE IF EXISTS worker_heartbeats;
@@ -0,0 +1,9 @@
-- P10 worker liveness: refreshed whenever a worker touches the queue (claim
-- attempts), read by the API to decide whether the worker process is alive.
CREATE TABLE IF NOT EXISTS worker_heartbeats (
worker_id VARCHAR(128) PRIMARY KEY,
job_types TEXT NOT NULL DEFAULT '',
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_worker_heartbeats_seen ON worker_heartbeats(last_seen_at DESC);
@@ -0,0 +1,3 @@
ALTER TABLE release_batches DROP COLUMN IF EXISTS release_change_set_id;
DROP TABLE IF EXISTS release_change_set_items;
DROP TABLE IF EXISTS release_change_sets;
@@ -0,0 +1,50 @@
-- R1 persistent release ChangeSet: the real-time release plan becomes an
-- immutable, snapshot-bound object. Batches reference CONFIRMED sets; later
-- source refreshes can never mutate an existing set or its manifest.
CREATE TABLE IF NOT EXISTS release_change_sets (
id VARCHAR(40) PRIMARY KEY,
name VARCHAR(200) NOT NULL,
number INTEGER NOT NULL,
state VARCHAR(16) NOT NULL DEFAULT 'planned'
CHECK (state IN ('planned','confirmed','consumed','cancelled')),
source_snapshot_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
source_change_set_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
acceptance_cutoff TIMESTAMPTZ NOT NULL,
production_baseline_ref VARCHAR(64) NOT NULL DEFAULT '',
item_count INTEGER NOT NULL DEFAULT 0,
add_count INTEGER NOT NULL DEFAULT 0,
update_count INTEGER NOT NULL DEFAULT 0,
delete_count INTEGER NOT NULL DEFAULT 0,
blocked_count INTEGER NOT NULL DEFAULT 0,
manifest_sha256 CHAR(64) NOT NULL
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
created_by VARCHAR(120) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
confirmed_by VARCHAR(120) NOT NULL DEFAULT '',
confirmed_at TIMESTAMPTZ
);
CREATE UNIQUE INDEX IF NOT EXISTS release_change_sets_number_key
ON release_change_sets(number);
-- Frozen per-item plan: every field is written once at generation and never
-- updated (delete protection + immutability are enforced by having no UPDATE
-- path and a state machine on the parent).
CREATE TABLE IF NOT EXISTS release_change_set_items (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
seq INTEGER NOT NULL,
publication_id VARCHAR(64) NOT NULL,
title VARCHAR(500) NOT NULL,
target_scope VARCHAR(16) NOT NULL CHECK (target_scope IN ('internal','external')),
action VARCHAR(16) NOT NULL CHECK (action IN ('add','update','delete','blocked','unchanged')),
test_state VARCHAR(16) NOT NULL,
production_state VARCHAR(16) NOT NULL,
can_release BOOLEAN NOT NULL DEFAULT false,
blocker VARCHAR(32) NOT NULL DEFAULT '',
PRIMARY KEY (change_set_id, seq)
);
-- Batches optionally bind to a confirmed change set (formal releases).
ALTER TABLE release_batches
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
REFERENCES release_change_sets(id);
CREATE INDEX IF NOT EXISTS release_batches_change_set ON release_batches(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
@@ -0,0 +1,12 @@
-- Revert R4 policy. Rollback-stage rows are folded back to running/completed so
-- the old CHECK can be restored without data loss surprises in test schemas.
UPDATE release_executions SET state='running' WHERE state='rolling_back';
UPDATE release_executions SET state='completed' WHERE state='rolled_back';
UPDATE release_executions SET state='failed' WHERE state='rollback_failed';
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
ALTER TABLE release_executions
ADD CONSTRAINT release_executions_state_check CHECK (state IN ('pending','running','completed','failed'));
ALTER TABLE release_executions DROP COLUMN IF EXISTS snapshot_verification_id;
ALTER TABLE release_executions DROP COLUMN IF EXISTS restore_trigger;
ALTER TABLE release_executions DROP COLUMN IF EXISTS failure_level;
DROP INDEX IF EXISTS release_executions_rollback_state;
@@ -0,0 +1,22 @@
-- R4 three-level rollback policy:
-- single_item: one publication permanently failed; later items were skipped
-- and (for publish) compensated by an automatic rollback.
-- batch: remaining batches were never started after a permanent failure.
-- overall: restore the entire release from the pre-release Production
-- Snapshot; only a production authorizer may trigger it, and it
-- requires an R2 snapshot verification id as evidence.
-- The rollback execution walks pending -> rolling_back -> rolled_back /
-- rollback_failed so the UI can show "compensating now" distinctly.
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
ALTER TABLE release_executions
ADD CONSTRAINT release_executions_state_check CHECK (state IN (
'pending','running','rolling_back','completed','failed','rolled_back','rollback_failed'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS failure_level VARCHAR(16) NOT NULL DEFAULT ''
CHECK (failure_level IN ('','single_item','batch','overall'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS restore_trigger VARCHAR(32) NOT NULL DEFAULT ''
CHECK (restore_trigger IN ('','auto_rollback_after_failure','snapshot_restore'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS snapshot_verification_id VARCHAR(64) NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS release_executions_rollback_state
ON release_executions(prior_execution_id) WHERE operation = 'rollback';
@@ -0,0 +1,4 @@
DROP TABLE IF EXISTS release_change_set_catalog_links;
DROP TABLE IF EXISTS release_change_set_snapshot_links;
DROP INDEX IF EXISTS release_executions_change_set;
ALTER TABLE release_executions DROP COLUMN IF EXISTS release_change_set_id;
@@ -0,0 +1,46 @@
-- R6 release audit chain: promote the evidence links that used to live only in
-- JSONB into real, immutable foreign keys so any execution can be traced
-- end-to-end (execution -> batch -> release change set -> source snapshots ->
-- source change sets -> review/acceptance -> source item/version).
ALTER TABLE release_executions
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
REFERENCES release_change_sets(id);
CREATE INDEX IF NOT EXISTS release_executions_change_set
ON release_executions(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
-- Backfill the denormalized execution binding from its batch (R1 binding).
UPDATE release_executions AS execution
SET release_change_set_id = batch.release_change_set_id
FROM release_batches AS batch
WHERE execution.release_batch_id = batch.id
AND batch.release_change_set_id IS NOT NULL
AND execution.release_change_set_id IS NULL;
-- Normalized snapshot bindings (display JSONB stays, but FK rows are truth).
CREATE TABLE IF NOT EXISTS release_change_set_snapshot_links (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
snapshot_id VARCHAR(36) NOT NULL REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, snapshot_id)
);
CREATE TABLE IF NOT EXISTS release_change_set_catalog_links (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
catalog_change_set_id VARCHAR(36) NOT NULL REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, catalog_change_set_id)
);
-- Backfill from the JSONB captured at generation time (skip ids that no longer
-- exist: RESTRICT keeps live rows, deleted ones simply do not link).
INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id)
SELECT rcs.id, elem.value
FROM release_change_sets rcs
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_snapshot_ids) AS elem(value)
JOIN source_catalog_snapshots s ON s.id = elem.value
ON CONFLICT DO NOTHING;
INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id)
SELECT rcs.id, elem.value
FROM release_change_sets rcs
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_change_set_ids) AS elem(value)
JOIN source_catalog_change_sets c ON c.id = elem.value
ON CONFLICT DO NOTHING;
@@ -0,0 +1,6 @@
-- Revert V1 revisions.
ALTER TABLE source_connections DROP COLUMN IF EXISTS current_revision_id,
DROP COLUMN IF EXISTS updated_by,
DROP COLUMN IF EXISTS last_scheduled_at,
DROP COLUMN IF EXISTS next_refresh_at;
DROP TABLE IF EXISTS source_connection_revisions CASCADE;
@@ -0,0 +1,64 @@
-- V1 source connection revisions: settings edits become validated -> active -> retired
-- versions; source_connections keeps ONE authoritative active snapshot that runtime
-- schedulers and existing FKs read. Secrets stay references only: base_url is visible,
-- secret_ref names where a credential lives and is never resolved through this table.
CREATE TABLE IF NOT EXISTS source_connection_revisions (
id VARCHAR(36) PRIMARY KEY,
connection_id VARCHAR(36) NOT NULL REFERENCES source_connections(id) ON DELETE CASCADE,
revision_number INTEGER NOT NULL CHECK (revision_number > 0),
state VARCHAR(16) NOT NULL
CHECK (state IN ('validated', 'active', 'retired', 'failed')),
source_type VARCHAR(20) NOT NULL
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
name VARCHAR(255) NOT NULL,
base_url TEXT NOT NULL DEFAULT '',
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
enabled BOOLEAN NOT NULL DEFAULT FALSE,
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(schedule) = 'object'),
created_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
validated_at TIMESTAMPTZ,
activated_at TIMESTAMPTZ,
retired_at TIMESTAMPTZ,
error_code VARCHAR(64) NOT NULL DEFAULT '',
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$'),
UNIQUE (connection_id, revision_number)
);
-- Exactly one active revision per connection.
CREATE UNIQUE INDEX IF NOT EXISTS uq_source_connection_active_revision
ON source_connection_revisions (connection_id) WHERE state = 'active';
-- Scheduler state lives next to the active snapshot so the existing FKs and UI
-- keep reading source_connections. next_refresh_at is NULL until the operator
-- sets a refresh interval, and it is ALWAYS derived from the schedule on the
-- active revision (never trusted from clients).
ALTER TABLE source_connections
ADD COLUMN IF NOT EXISTS next_refresh_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS last_scheduled_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS current_revision_id VARCHAR(36) REFERENCES source_connection_revisions(id),
ADD COLUMN IF NOT EXISTS updated_by VARCHAR(128) NOT NULL DEFAULT 'system';
CREATE INDEX IF NOT EXISTS idx_source_connections_due
ON source_connections (next_refresh_at)
WHERE enabled AND next_refresh_at IS NOT NULL;
-- Seed: every existing connection gets revision 1 reflecting its current row,
-- marked active, created by the migration itself.
INSERT INTO source_connection_revisions (
id, connection_id, revision_number, state, source_type, name, base_url,
secret_ref, enabled, schedule, created_by, validated_at, activated_at
)
SELECT
'scr-' || md5(c.id), c.id, 1, 'active', c.source_type, c.name, c.base_url,
c.secret_ref, c.enabled, c.schedule, 'migration-31', NOW(), NOW()
FROM source_connections AS c
ON CONFLICT (connection_id, revision_number) DO NOTHING;
UPDATE source_connections AS c
SET current_revision_id = COALESCE(c.current_revision_id, 'scr-' || md5(c.id))
FROM source_connection_revisions AS r
WHERE r.connection_id = c.id AND r.revision_number = 1
AND c.current_revision_id IS NULL;
@@ -0,0 +1,633 @@
package postgres
import (
"os"
"strings"
"testing"
)
func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
migrations, err := loadMigrations()
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 31 {
t.Fatalf("migration count = %d, want 24", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
}
if migrations[1].Version != 2 || !strings.Contains(migrations[1].UpSQL, "CREATE TABLE IF NOT EXISTS source_items") {
t.Fatalf("migration 2 = %#v", migrations[1])
}
if migrations[2].Version != 3 || !strings.Contains(migrations[2].UpSQL, "CREATE TABLE IF NOT EXISTS acceptance_results") {
t.Fatalf("migration 3 = %#v", migrations[2])
}
if migrations[3].Version != 4 || !strings.Contains(migrations[3].UpSQL, "source_connections_source_type_check") {
t.Fatalf("migration 4 = %#v", migrations[3])
}
if migrations[4].Version != 5 || !strings.Contains(migrations[4].UpSQL, "CREATE TABLE IF NOT EXISTS source_scans") {
t.Fatalf("migration 5 = %#v", migrations[4])
}
if migrations[5].Version != 6 || !strings.Contains(migrations[5].UpSQL, "CREATE TABLE IF NOT EXISTS publication_reviews") {
t.Fatalf("migration 6 = %#v", migrations[5])
}
if migrations[7].Version != 8 || !strings.Contains(migrations[7].UpSQL, "CREATE TABLE IF NOT EXISTS directory_scope_rules") {
t.Fatalf("migration 8 = %#v", migrations[7])
}
if migrations[8].Version != 9 || !strings.Contains(migrations[8].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_snapshots") {
t.Fatalf("migration 9 = %#v", migrations[8])
}
if migrations[9].Version != 10 || !strings.Contains(migrations[9].UpSQL, "Visibility policies used by the source catalog") {
t.Fatalf("migration 10 = %#v", migrations[9])
}
if migrations[10].Version != 11 || !strings.Contains(migrations[10].UpSQL, "CREATE TABLE IF NOT EXISTS processing_confirmations") {
t.Fatalf("migration 11 = %#v", migrations[10])
}
if migrations[13].Version != 14 || !strings.Contains(migrations[13].UpSQL, "ADD COLUMN IF NOT EXISTS started_at") {
t.Fatalf("migration 14 = %#v", migrations[13])
}
if migrations[14].Version != 15 || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_sets") || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_items") {
t.Fatalf("migration 15 = %#v", migrations[14])
}
if migrations[15].Version != 16 || !strings.Contains(migrations[15].UpSQL, "origin") || !strings.Contains(migrations[15].UpSQL, "source_catalog_change_items") {
t.Fatalf("migration 16 = %#v", migrations[15])
}
if migrations[16].Version != 17 || !strings.Contains(migrations[16].UpSQL, "current_batch") || !strings.Contains(migrations[16].UpSQL, "total_batches") {
t.Fatalf("migration 17 = %#v", migrations[16])
}
if migrations[17].Version != 18 || !strings.Contains(migrations[17].UpSQL, "jobs_state_check") || !strings.Contains(migrations[17].UpSQL, "cancelled") {
t.Fatalf("migration 18 = %#v", migrations[17])
}
if migrations[18].Version != 19 || !strings.Contains(migrations[18].UpSQL, "CREATE TABLE IF NOT EXISTS operation_requests") || !strings.Contains(migrations[18].UpSQL, "UNIQUE (operation_type, operation_id)") {
t.Fatalf("migration 19 = %#v", migrations[18])
}
if migrations[19].Version != 20 || !strings.Contains(migrations[19].UpSQL, "content_sha256") || !strings.Contains(migrations[19].UpSQL, "IF NOT EXISTS") {
t.Fatalf("migration 20 = %#v", migrations[19])
}
if migrations[20].Version != 21 || !strings.Contains(migrations[20].UpSQL, "identity_confidence") || !strings.Contains(migrations[20].UpSQL, "reason_code") {
t.Fatalf("migration 21 = %#v", migrations[20])
}
if migrations[21].Version != 22 || !strings.Contains(migrations[21].UpSQL, "compare_catalog_change_set") {
t.Fatalf("migration 22 = %#v", migrations[21])
}
if migrations[22].Version != 23 || !strings.Contains(migrations[22].UpSQL, "move_count") || !strings.Contains(migrations[22].UpSQL, "'applying'") || !strings.Contains(migrations[22].UpSQL, "old_scope") {
t.Fatalf("migration 23 = %#v", migrations[22])
}
if migrations[23].Version != 24 || !strings.Contains(migrations[23].UpSQL, "item_seq") || !strings.Contains(migrations[23].UpSQL, "change_set_id") {
t.Fatalf("migration 24 = %#v", migrations[23])
}
if migrations[24].Version != 25 || !strings.Contains(migrations[24].UpSQL, "business_change_set_id") || !strings.Contains(migrations[24].UpSQL, "jobs_processing_identity") {
t.Fatalf("migration 25 = %#v", migrations[24])
}
if migrations[25].Version != 26 || !strings.Contains(migrations[25].UpSQL, "job_failures") || !strings.Contains(migrations[25].UpSQL, "last_error_code") {
t.Fatalf("migration 26 = %#v", migrations[25])
}
if migrations[26].Version != 27 || !strings.Contains(migrations[26].UpSQL, "worker_heartbeats") {
t.Fatalf("migration 27 = %#v", migrations[26])
}
if migrations[27].Version != 28 || !strings.Contains(migrations[27].UpSQL, "release_change_sets") || !strings.Contains(migrations[27].UpSQL, "manifest_sha256") {
t.Fatalf("migration 28 = %#v", migrations[27])
}
if migrations[28].Version != 29 || !strings.Contains(migrations[28].UpSQL, "rolling_back") || !strings.Contains(migrations[28].UpSQL, "failure_level") {
t.Fatalf("migration 29 = %#v", migrations[28])
}
if migrations[29].Version != 30 || !strings.Contains(migrations[29].UpSQL, "release_change_set_snapshot_links") || !strings.Contains(migrations[29].UpSQL, "release_change_set_catalog_links") {
t.Fatalf("migration 30 = %#v", migrations[29])
}
if migrations[30].Version != 31 || !strings.Contains(migrations[30].UpSQL, "source_connection_revisions") || !strings.Contains(migrations[30].UpSQL, "uq_source_connection_active_revision") || !strings.Contains(migrations[30].UpSQL, "next_refresh_at") {
t.Fatalf("migration 31 = %#v", migrations[30])
}
}
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
up, err := os.ReadFile("migrations/000011_processing_confirmations.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000011_processing_confirmations.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"add column if not exists target_scope",
"update test_publications as publication",
"review.decided_visibility",
"source.visibility",
"check (target_scope in ('internal', 'external'))",
"create table if not exists processing_confirmations",
"references test_publications(id) on delete restrict",
"operator_ref",
"reason",
"unique (publication_id)",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("processing confirmation migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
if !strings.Contains(downSQL, "drop table if exists processing_confirmations") || !strings.Contains(downSQL, "drop column if exists target_scope") {
t.Fatal("processing confirmation down migration must remove only migration 11 objects")
}
}
func TestVersionedUISystemSettingsMigration(t *testing.T) {
up, err := os.ReadFile("migrations/000013_ui_system_settings.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000013_ui_system_settings.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{"create table if not exists ui_setting_revisions", "'validated', 'active', 'retired', 'failed'", "create unique index", "where state = 'active'", "insert into ui_setting_revisions"} {
if !strings.Contains(upSQL, required) {
t.Fatalf("UI settings migration missing %q", required)
}
}
if !strings.Contains(strings.ToLower(string(down)), "drop table if exists ui_setting_revisions") {
t.Fatal("UI settings rollback must remove only its revision table")
}
}
func TestReleaseActionMigrationSeparatesUpsertAndDelete(t *testing.T) {
up, err := os.ReadFile("migrations/000012_release_actions.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000012_release_actions.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"alter table release_batch_items", "alter table release_item_executions",
"add column if not exists action", "check (action in ('upsert', 'delete'))",
"drop constraint if exists release_batch_items_publication_id_key",
"drop constraint if exists release_batch_items_review_id_key",
} {
if !strings.Contains(upSQL, required) {
t.Fatalf("release action migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
if !strings.Contains(downSQL, "drop column if exists action") {
t.Fatal("release action rollback must remove action columns")
}
}
func TestCatalogPolicyMigrationAllowsBothAndBlockedWithoutChangingPublicationReviewScope(t *testing.T) {
up, err := os.ReadFile("migrations/000010_catalog_policies.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000010_catalog_policies.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"visibility policies used by the source catalog",
"directory_scope_rules_visibility_check",
"source_item_scope_overrides_visibility_check",
"scope_change_events_previous_visibility_check",
"scope_change_events_decided_visibility_check",
"'internal', 'external', 'both', 'blocked'",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("catalog policy migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
if !strings.Contains(downSQL, "update source_item_scope_overrides") || !strings.Contains(downSQL, "update directory_scope_rules") {
t.Fatalf("catalog policy down migration must normalize extended policy values before restoring constraints")
}
}
func TestSourceCatalogMigrationDefinesImmutableHierarchicalSnapshots(t *testing.T) {
up, err := os.ReadFile("migrations/000009_source_catalog.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000009_source_catalog.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"create table if not exists source_catalog_snapshots",
"create table if not exists source_catalog_nodes",
"create table if not exists source_catalog_heads",
"check (node_type in ('folder', 'file'))",
"primary key (snapshot_id, node_id)",
"foreign key (snapshot_id, parent_node_id)",
"references source_items(id) on delete restrict",
"create index if not exists idx_source_catalog_nodes_parent",
"unique (source_connection_id, snapshot_sha256)",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("source catalog migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
ordered := []string{"drop table if exists source_catalog_heads", "drop table if exists source_catalog_nodes", "drop table if exists source_catalog_snapshots"}
previous := -1
for _, statement := range ordered {
position := strings.Index(downSQL, statement)
if position < 0 || position <= previous {
t.Fatalf("source catalog down order invalid for %q", statement)
}
previous = position
}
}
func TestScopeRulesMigrationDefinesAuditedDirectoryAndFileOverrides(t *testing.T) {
up, err := os.ReadFile("migrations/000008_scope_rules.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000008_scope_rules.down.sql")
if err != nil {
t.Fatal(err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"create table if not exists directory_scope_rules",
"create table if not exists source_item_scope_overrides",
"create table if not exists scope_change_events",
"primary key (source_connection_id, directory_path)",
"primary key (source_item_id)",
"check (visibility in ('internal', 'external'))",
"check (target_type in ('directory', 'item'))",
"references source_connections(id) on delete restrict",
"references source_items(id) on delete restrict",
"create index if not exists idx_scope_change_events_target",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("scope migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
for _, required := range []string{
"drop table if exists scope_change_events",
"drop table if exists source_item_scope_overrides",
"drop table if exists directory_scope_rules",
} {
if !strings.Contains(downSQL, required) {
t.Errorf("scope down migration missing %q", required)
}
}
}
func TestJobsMigrationDefinesDurableLeaseQueue(t *testing.T) {
body, err := os.ReadFile("migrations/000001_jobs.up.sql")
if err != nil {
t.Fatalf("read jobs migration: %v", err)
}
sql := strings.ToLower(string(body))
for _, required := range []string{
"create table if not exists schema_migrations",
"create table if not exists jobs",
"idempotency_key",
"lease_owner",
"lease_until",
"available_at",
"max_attempts",
"check (state in",
"unique (idempotency_key)",
} {
if !strings.Contains(sql, required) {
t.Errorf("migration missing %q", required)
}
}
}
func TestSourceGovernanceMigrationDefinesIdentityAndStateContracts(t *testing.T) {
body, err := os.ReadFile("migrations/000002_source_governance.up.sql")
if err != nil {
t.Fatalf("read source governance migration: %v", err)
}
sql := strings.ToLower(string(body))
for _, required := range []string{
"create table if not exists source_connections",
"create table if not exists source_items",
"create table if not exists content_objects",
"create table if not exists source_item_versions",
"create table if not exists test_publications",
"unique (source_connection_id, stable_external_id)",
"unique (sha256)",
"unique (source_item_id, revision)",
"unique (source_item_version_id, test_kb_id)",
"check (source_type in ('pp', 'alist', 'dingtalk'))",
"check (sha256 ~ '^[0-9a-f]{64}$')",
"check (source_change in ('unchanged', 'added', 'updated', 'deleted', 'unavailable'))",
"check (content_state in ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed'))",
"check (ingest_state in ('pending', 'uploading', 'uploaded', 'failed'))",
"check (processing_state in ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'))",
"check (retrieval_state in ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed'))",
"references source_connections(id)",
"references source_items(id)",
"references content_objects(id)",
"references source_item_versions(id)",
"raw_record_summary jsonb",
} {
if !strings.Contains(sql, required) {
t.Errorf("source governance migration missing %q", required)
}
}
}
func TestSourceGovernanceDownMigrationDropsTablesInDependencyOrder(t *testing.T) {
body, err := os.ReadFile("migrations/000002_source_governance.down.sql")
if err != nil {
t.Fatalf("read source governance down migration: %v", err)
}
sql := strings.ToLower(string(body))
ordered := []string{
"drop table if exists test_publications",
"drop table if exists source_item_versions",
"drop table if exists content_objects",
"drop table if exists source_items",
"drop table if exists source_connections",
}
previous := -1
for _, statement := range ordered {
position := strings.Index(sql, statement)
if position < 0 {
t.Fatalf("down migration missing %q", statement)
}
if position <= previous {
t.Fatalf("%q is not in reverse dependency order", statement)
}
previous = position
}
}
func TestTestOrchestrationMigrationDefinesPollingAndAcceptanceContracts(t *testing.T) {
body, err := os.ReadFile("migrations/000003_test_orchestration.up.sql")
if err != nil {
t.Fatalf("read test orchestration migration: %v", err)
}
sql := strings.ToLower(string(body))
for _, required := range []string{
"alter table test_publications add column if not exists poll_sequence",
"alter table test_publications add column if not exists last_remote_status",
"alter table test_publications add column if not exists last_observed_at",
"alter table test_publications add column if not exists supersedes_publication_id",
"references test_publications(id) on delete restrict",
"create table if not exists acceptance_results",
"references test_publications(id) on delete restrict",
"check (check_type in ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches'))",
"check (state in ('pending', 'passed', 'failed', 'manual_required'))",
"check (jsonb_typeof(evidence_summary) = 'object')",
"unique (publication_id, run_id, check_type)",
"create index if not exists idx_test_publications_poll",
"create index if not exists idx_acceptance_results_publication",
} {
if !strings.Contains(sql, required) {
t.Errorf("test orchestration migration missing %q", required)
}
}
}
func TestTestOrchestrationDownMigrationRemovesDependentObjectsFirst(t *testing.T) {
body, err := os.ReadFile("migrations/000003_test_orchestration.down.sql")
if err != nil {
t.Fatalf("read test orchestration down migration: %v", err)
}
sql := strings.ToLower(string(body))
acceptance := strings.Index(sql, "drop table if exists acceptance_results")
publicationColumns := strings.Index(sql, "alter table test_publications")
if acceptance < 0 || publicationColumns < 0 {
t.Fatalf("down migration missing dependent object removal: %s", sql)
}
if acceptance > publicationColumns {
t.Fatal("acceptance_results must be dropped before test_publications columns")
}
}
func TestManualSourceMigrationExpandsWithoutRewritingAndDowngradesFailClosed(t *testing.T) {
up, err := os.ReadFile("migrations/000004_manual_source.up.sql")
if err != nil {
t.Fatalf("read manual source up migration: %v", err)
}
down, err := os.ReadFile("migrations/000004_manual_source.down.sql")
if err != nil {
t.Fatalf("read manual source down migration: %v", err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"drop constraint if exists source_connections_source_type_check",
"check (source_type in ('pp', 'alist', 'dingtalk', 'manual'))",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("manual source up migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
for _, required := range []string{
"if exists",
"source_type = 'manual'",
"raise exception",
"check (source_type in ('pp', 'alist', 'dingtalk'))",
} {
if !strings.Contains(downSQL, required) {
t.Errorf("manual source down migration missing %q", required)
}
}
}
func TestSourceScanMigrationDefinesAuditableSingleActiveScan(t *testing.T) {
up, err := os.ReadFile("migrations/000005_source_scans.up.sql")
if err != nil {
t.Fatalf("read source scans up migration: %v", err)
}
down, err := os.ReadFile("migrations/000005_source_scans.down.sql")
if err != nil {
t.Fatalf("read source scans down migration: %v", err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"create table if not exists source_scans",
"create table if not exists source_scan_changes",
"references source_connections(id) on delete restrict",
"references source_scans(id) on delete restrict",
"check (state in ('pending', 'running', 'completed', 'failed'))",
"check (change_type in ('added', 'updated', 'unavailable'))",
"check (snapshot_sha256 = '' or snapshot_sha256 ~ '^[0-9a-f]{64}$')",
"check (observed_items >= 0)",
"check (unchanged_items >= 0)",
"check (added_items >= 0)",
"check (updated_items >= 0)",
"check (over_limit_items >= 0)",
"check (missing_items >= 0)",
"check (planned_uploads >= 0)",
"create unique index if not exists idx_source_scans_one_active",
"where state in ('pending', 'running')",
"unique (source_scan_id, stable_external_id)",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("source scans up migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
changes := strings.Index(downSQL, "drop table if exists source_scan_changes")
scans := strings.Index(downSQL, "drop table if exists source_scans")
if changes < 0 || scans < 0 || changes > scans {
t.Fatal("source scans down migration must drop changes before scans")
}
}
func TestReviewAndSealMigrationDefinesImmutableHumanGate(t *testing.T) {
up, err := os.ReadFile("migrations/000006_review_and_seal.up.sql")
if err != nil {
t.Fatalf("read review and seal up migration: %v", err)
}
down, err := os.ReadFile("migrations/000006_review_and_seal.down.sql")
if err != nil {
t.Fatalf("read review and seal down migration: %v", err)
}
upSQL := strings.ToLower(string(up))
for _, required := range []string{
"create table if not exists publication_reviews",
"references test_publications(id) on delete restrict",
"check (decision in ('approved', 'rejected'))",
"check (decided_visibility in ('internal', 'external'))",
"acceptance_run_id",
"publication_updated_at",
"create table if not exists release_batches",
"check (state in ('draft', 'sealed', 'cancelled'))",
"policy_version",
"create table if not exists release_batch_items",
"references publication_reviews(id) on delete restrict",
"unique (publication_id)",
"decided_visibility",
} {
if !strings.Contains(upSQL, required) {
t.Errorf("review and seal migration missing %q", required)
}
}
downSQL := strings.ToLower(string(down))
ordered := []string{
"drop table if exists release_batch_items",
"drop table if exists release_batches",
"drop table if exists publication_reviews",
}
previous := -1
for _, statement := range ordered {
position := strings.Index(downSQL, statement)
if position < 0 || position <= previous {
t.Fatalf("review and seal down migration has invalid dependency order for %q", statement)
}
previous = position
}
}
func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
up, err := os.ReadFile("migrations/000007_release_executions.up.sql")
if err != nil {
t.Fatal(err)
}
down, err := os.ReadFile("migrations/000007_release_executions.down.sql")
if err != nil {
t.Fatal(err)
}
text := strings.ToLower(string(up))
for _, required := range []string{
"create table if not exists release_executions",
"operation in ('publish', 'rollback')",
"state in ('pending', 'running', 'completed', 'failed')",
"target_environment = 'production'",
"references release_batches(id) on delete restrict",
"references jobs(id) on delete restrict",
"create unique index if not exists uq_release_publish_batch",
"create table if not exists release_item_executions",
"bundle_sha256", "previous_production_revision", "production_revision", "error_code",
} {
if !strings.Contains(text, required) {
t.Fatalf("release execution migration missing %q", required)
}
}
if strings.Contains(text, "api_key") || strings.Contains(text, "password") || strings.Contains(text, "secret") {
t.Fatal("release execution migration must not persist production credentials")
}
downText := strings.ToLower(string(down))
items := strings.Index(downText, "drop table if exists release_item_executions")
executions := strings.Index(downText, "drop table if exists release_executions")
if items < 0 || executions < 0 || items > executions {
t.Fatal("release execution down migration must drop item executions before executions")
}
}
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
sql := strings.ToLower(claimJobSQL)
for _, required := range []string{
"for update skip locked",
"state in ('pending', 'retry')",
"state = 'running'",
"lease_until <",
"attempt = j.attempt + 1",
} {
if !strings.Contains(sql, required) {
t.Errorf("claim SQL missing %q", required)
}
}
}
func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) {
tests := []struct {
name string
sql string
required []string
}{
{
name: "enqueue",
sql: enqueueJobSQL,
required: []string{
"on conflict (idempotency_key)",
"do update set idempotency_key = excluded.idempotency_key",
"returning id",
},
},
{
name: "complete",
sql: completeJobSQL,
required: []string{
"state = 'running'",
"lease_owner = $2",
"state = 'completed'",
},
},
{
name: "heartbeat",
sql: heartbeatJobSQL,
required: []string{
"state = 'running'",
"lease_owner = $2",
"lease_until = $4",
},
},
{
name: "fail",
sql: failJobSQL,
required: []string{
"case when attempt >= max_attempts then 'failed'",
"else 'retry'",
"lease_owner = null",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sql := strings.ToLower(tt.sql)
for _, required := range tt.required {
if !strings.Contains(sql, required) {
t.Errorf("SQL missing %q", required)
}
}
})
}
}
@@ -0,0 +1,180 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
var ErrOperationRequestConflict = errors.New("operation request conflicts with an existing request")
type operationExecutor func(pgx.Tx) (json.RawMessage, error)
func (s *Store) RouteCatalogFilesWithOperation(ctx context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) {
if err := validateCatalogRouteRequest(request); err != nil {
return governance.CatalogRouteResult{}, false, err
}
fingerprint, err := catalogRouteFingerprint(request)
if err != nil {
return governance.CatalogRouteResult{}, false, err
}
resultJSON, replayed, err := s.executeOperation(ctx, "catalog.route", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := routeCatalogFilesTx(ctx, tx, request)
if err != nil {
return nil, err
}
encoded, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("encode catalog route result: %w", err)
}
return encoded, nil
})
if err != nil {
return governance.CatalogRouteResult{}, false, err
}
var result governance.CatalogRouteResult
if err := json.Unmarshal(resultJSON, &result); err != nil {
return governance.CatalogRouteResult{}, false, fmt.Errorf("decode stored catalog route result: %w", err)
}
return result, replayed, nil
}
func catalogRouteFingerprint(request governance.CatalogRouteRequest) (string, error) {
targets := request.TargetKnowledgeBaseIDs
if len(targets) == 0 && request.TargetKnowledgeBaseID != "" {
targets = []string{request.TargetKnowledgeBaseID}
}
payload, err := json.Marshal(struct {
ConnectionID string `json:"connection_id"`
NodeIDs []string `json:"node_ids"`
Visibility governance.Visibility `json:"visibility"`
Targets []string `json:"target_knowledge_base_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{
ConnectionID: request.ConnectionID,
NodeIDs: request.NodeIDs,
Visibility: request.Visibility,
Targets: targets,
OperatorRef: strings.TrimSpace(request.OperatorRef),
Reason: strings.TrimSpace(request.Reason),
})
if err != nil {
return "", fmt.Errorf("encode catalog route fingerprint: %w", err)
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func (s *Store) EnqueueCatalogRefreshWithOperation(ctx context.Context, source governance.CatalogSource, operationID string, now time.Time) (jobs.Job, bool, error) {
if !validRouteID(source.ConnectionID) || strings.TrimSpace(source.Name) == "" || now.IsZero() {
return jobs.Job{}, false, errors.New("invalid catalog refresh request")
}
fingerprint, err := operationFingerprint(struct {
ConnectionID string `json:"connection_id"`
}{ConnectionID: source.ConnectionID})
if err != nil {
return jobs.Job{}, false, err
}
resultJSON, replayed, err := s.executeOperation(ctx, "catalog.refresh", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) {
jobID, err := newOpaqueID("job-")
if err != nil {
return nil, err
}
payload, err := json.Marshal(map[string]any{"connection_id": source.ConnectionID, "source_type": source.SourceType, "source_name": source.Name, "stage": "queued", "progress": 0, "message": "等待后台刷新", "processed": 0, "total": 0})
if err != nil {
return nil, fmt.Errorf("encode catalog refresh job: %w", err)
}
job, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, jobID, "source.catalog.refresh", payload, 20, now, 3, "catalog-refresh:"+jobID, now))
if err != nil {
return nil, fmt.Errorf("enqueue catalog refresh job: %w", err)
}
encoded, err := json.Marshal(job)
if err != nil {
return nil, fmt.Errorf("encode catalog refresh result: %w", err)
}
return encoded, nil
})
if err != nil {
return jobs.Job{}, false, err
}
var result jobs.Job
if err := json.Unmarshal(resultJSON, &result); err != nil {
return jobs.Job{}, false, fmt.Errorf("decode stored catalog refresh result: %w", err)
}
return result, replayed, nil
}
func operationFingerprint(value any) (string, error) {
payload, err := json.Marshal(value)
if err != nil {
return "", fmt.Errorf("encode operation fingerprint: %w", err)
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func (s *Store) executeOperation(ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute operationExecutor) (json.RawMessage, bool, error) {
if strings.TrimSpace(operationType) == "" || len(operationType) > 100 || strings.TrimSpace(operationID) == "" || len(operationID) > 128 || len(fingerprint) != 64 || now.IsZero() || execute == nil {
return nil, false, errors.New("invalid operation request")
}
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return nil, false, errors.New("operation request transactions are unavailable")
}
tx, err := beginner.Begin(ctx)
if err != nil {
return nil, false, fmt.Errorf("begin operation request: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
tag, err := tx.Exec(ctx, `INSERT INTO operation_requests(operation_type,operation_id,request_fingerprint_sha256,state,result,created_at,completed_at)
VALUES($1,$2,$3,'completed','{}'::jsonb,$4,$4)
ON CONFLICT(operation_type,operation_id) DO NOTHING`, operationType, operationID, fingerprint, now)
if err != nil {
return nil, false, fmt.Errorf("create operation request: %w", err)
}
if tag.RowsAffected() == 0 {
var storedFingerprint string
var storedResult json.RawMessage
if err := tx.QueryRow(ctx, `SELECT request_fingerprint_sha256,result
FROM operation_requests
WHERE operation_type=$1 AND operation_id=$2
FOR UPDATE`, operationType, operationID).Scan(&storedFingerprint, &storedResult); err != nil {
return nil, false, fmt.Errorf("load operation request: %w", err)
}
if storedFingerprint != fingerprint {
return nil, false, ErrOperationRequestConflict
}
if err := tx.Commit(ctx); err != nil {
return nil, false, fmt.Errorf("commit replay operation request: %w", err)
}
return storedResult, true, nil
}
result, err := execute(tx)
if err != nil {
return nil, false, err
}
if !json.Valid(result) {
return nil, false, errors.New("operation result must be valid JSON")
}
if _, err := tx.Exec(ctx, `UPDATE operation_requests SET result=$3::jsonb,completed_at=$4
WHERE operation_type=$1 AND operation_id=$2`, operationType, operationID, result, now); err != nil {
return nil, false, fmt.Errorf("complete operation request: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, false, fmt.Errorf("commit operation request: %w", err)
}
return result, false, nil
}
@@ -0,0 +1,163 @@
package postgres
import (
"context"
"encoding/json"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func runOperation[T any](s *Store, ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute func(pgx.Tx) (T, error)) (T, bool, error) {
var zero T
raw, replayed, err := s.executeOperation(ctx, operationType, operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := execute(tx)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return zero, false, err
}
var result T
if err := json.Unmarshal(raw, &result); err != nil {
return zero, false, err
}
return result, replayed, nil
}
func (s *Store) TriggerPPSourceScanWithOperation(ctx context.Context, connectionID string, now time.Time, operationID string) (governance.SourceScan, bool, error) {
fingerprint, err := operationFingerprint(struct {
ConnectionID string `json:"connection_id"`
}{connectionID})
if err != nil {
return governance.SourceScan{}, false, err
}
return runOperation(s, ctx, "source.scan.trigger", operationID, fingerprint, now, func(tx pgx.Tx) (governance.SourceScan, error) {
return NewStore(tx).TriggerPPSourceScan(ctx, connectionID, now)
})
}
func (s *Store) CreateUISettingsCandidateWithOperation(ctx context.Context, request governance.CreateUISettingsCandidateRequest, operationID string) (governance.UISettings, bool, error) {
fingerprint, err := operationFingerprint(struct {
Document int `json:"document_refresh_seconds"`
Resource int `json:"resource_refresh_seconds"`
Stalled int `json:"stalled_after_seconds"`
Finalize int `json:"finalizing_after_seconds"`
By string `json:"updated_by"`
}{request.DocumentRefreshSeconds, request.ResourceRefreshSeconds, request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy})
if err != nil {
return governance.UISettings{}, false, err
}
return runOperation(s, ctx, "ui_settings.candidate.create", operationID, fingerprint, request.Now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) {
return NewStore(tx).CreateUISettingsCandidate(ctx, request)
})
}
func (s *Store) ActivateUISettingsWithOperation(ctx context.Context, revisionID string, now time.Time, operationID string) (governance.UISettings, bool, error) {
fingerprint, err := operationFingerprint(struct {
RevisionID string `json:"revision_id"`
}{revisionID})
if err != nil {
return governance.UISettings{}, false, err
}
return runOperation(s, ctx, "ui_settings.activate", operationID, fingerprint, now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) {
return NewStore(tx).ActivateUISettings(ctx, revisionID, now)
})
}
func (s *Store) CreateReleaseBatchWithOperation(ctx context.Context, request governance.CreateReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) {
fingerprint, err := operationFingerprint(struct {
Name string `json:"name"`
By string `json:"created_by"`
Upsert []string `json:"publication_ids"`
Delete []string `json:"delete_publication_ids"`
Set string `json:"release_change_set_id"`
}{request.Name, request.CreatedBy, request.PublicationIDs, request.DeletePublicationIDs, request.ReleaseChangeSetID})
if err != nil {
return governance.ReleaseBatch{}, false, err
}
return runOperation(s, ctx, "release.batch.create", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) {
return NewStore(tx).CreateReleaseBatch(ctx, request)
})
}
func (s *Store) GenerateReleaseChangeSetWithOperation(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) {
fingerprint, err := operationFingerprint(struct {
Name string `json:"name"`
By string `json:"created_by"`
}{request.Name, request.CreatedBy})
if err != nil {
return governance.ReleaseChangeSet{}, false, err
}
return runOperation(s, ctx, "release.changeset.generate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) {
return NewStore(tx).GenerateReleaseChangeSet(ctx, request)
})
}
func (s *Store) ConfirmReleaseChangeSetWithOperation(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) {
fingerprint, err := operationFingerprint(struct {
ID string `json:"change_set_id"`
By string `json:"confirmed_by"`
}{request.ChangeSetID, request.ConfirmedBy})
if err != nil {
return governance.ReleaseChangeSet{}, false, err
}
return runOperation(s, ctx, "release.changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) {
return NewStore(tx).ConfirmReleaseChangeSet(ctx, request)
})
}
func (s *Store) SealReleaseBatchWithOperation(ctx context.Context, request governance.SealReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) {
fingerprint, err := operationFingerprint(struct {
BatchID string `json:"batch_id"`
SealedBy string `json:"sealed_by"`
}{request.BatchID, request.SealedBy})
if err != nil {
return governance.ReleaseBatch{}, false, err
}
return runOperation(s, ctx, "release.batch.seal", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) {
return NewStore(tx).SealReleaseBatch(ctx, request)
})
}
func (s *Store) AuthorizeReleaseBatchWithOperation(ctx context.Context, request governance.AuthorizeReleaseBatchRequest, operationID string) (governance.ReleaseExecution, bool, error) {
fingerprint, err := operationFingerprint(struct {
BatchID string `json:"batch_id"`
AuthorizedBy string `json:"authorized_by"`
}{request.BatchID, request.AuthorizedBy})
if err != nil {
return governance.ReleaseExecution{}, false, err
}
return runOperation(s, ctx, "release.authorize.publish", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
return NewStore(tx).AuthorizeReleaseBatch(ctx, request)
})
}
func (s *Store) AuthorizeReleaseRollbackWithOperation(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest, operationID string) (governance.ReleaseExecution, bool, error) {
fingerprint, err := operationFingerprint(struct {
PriorID string `json:"prior_execution_id"`
AuthorizedBy string `json:"authorized_by"`
}{request.PriorExecutionID, request.AuthorizedBy})
if err != nil {
return governance.ReleaseExecution{}, false, err
}
return runOperation(s, ctx, "release.authorize.rollback", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
return NewStore(tx).AuthorizeReleaseRollback(ctx, request)
})
}
func (s *Store) RetryReleaseExecutionWithOperation(ctx context.Context, request governance.RetryReleaseExecutionRequest, operationID string) (governance.ReleaseExecution, bool, error) {
fingerprint, err := operationFingerprint(struct {
ExecutionID string `json:"execution_id"`
}{request.ExecutionID})
if err != nil {
return governance.ReleaseExecution{}, false, err
}
return runOperation(s, ctx, "release.execution.retry", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) {
return NewStore(tx).RetryReleaseExecution(ctx, request)
})
}
@@ -0,0 +1,100 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
// TestP2StableIdentityAndProvenMove freezes docs plan P2:
// identity_confidence is never downgraded from stable_id; the canonical
// compare_catalog_change_set function classifies a 1:1 content-hash relocation
// as proven move evidence, and never merges ambiguous copies.
func TestP2StableIdentityAndProvenMove(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
hash := strings.Repeat("c", 64)
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id','pp','ID PP',true);
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snap-old','conn-id',repeat('1',64),'ID PP',2,1,1,'2026-09-11T00:00:00Z'),
('snap-new','conn-id',repeat('2',64),'ID PP',2,1,1,'2026-09-12T00:00:00Z'),
('snap-copy','conn-id',repeat('3',64),'ID PP',3,1,2,'2026-09-13T00:00:00Z');
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes,content_sha256)
VALUES
('snap-old','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL),
('snap-old','f-old','root','file','手册.pdf','旧目录/手册.pdf',repeat('5',64),10,0,1,10,repeat('c',64)),
('snap-new','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL),
('snap-new','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)),
('snap-copy','root',NULL,'folder','ID PP','',repeat('4',64),NULL,2,2,20,NULL),
('snap-copy','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)),
('snap-copy','f-copy','root','file','副本.pdf','新目录/副本.pdf',repeat('7',64),10,0,1,10,repeat('c',64));
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
VALUES('cs-move','conn-id','snap-old','snap-new',NOW()),
('cs-copy','conn-id','snap-new','snap-copy',NOW());
SELECT compare_catalog_change_set('cs-move');
SELECT compare_catalog_change_set('cs-copy');`); err != nil {
t.Fatal(err)
}
// Proven move: the disappeared old locator is a delete carrying move evidence,
// and the reappearing new locator is an add carrying the same content hash.
var deleteReason, deleteOldHash string
if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(old_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='delete'`).Scan(&deleteReason, &deleteOldHash); err != nil || deleteReason != "content_hash_move" || deleteOldHash != hash {
t.Fatalf("cs-move delete=%q hash=%q err=%v", deleteReason, deleteOldHash, err)
}
var addReason, addNewHash string
if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(new_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='add'`).Scan(&addReason, &addNewHash); err != nil || addReason != "content_hash_move" || addNewHash != hash {
t.Fatalf("cs-move add=%q hash=%q err=%v", addReason, addNewHash, err)
}
// Counts were recalculated by the function.
var addCount, deleteCount int
if err := pool.QueryRow(ctx, `SELECT add_count,delete_count FROM source_catalog_change_sets WHERE id='cs-move'`).Scan(&addCount, &deleteCount); err != nil || addCount != 1 || deleteCount != 1 {
t.Fatalf("cs-move counts add=%d delete=%d err=%v", addCount, deleteCount, err)
}
// Ambiguous copy: hash appears twice on the new side → NOT a proven move;
// the copy is a plain add and the original stays untouched.
var copyReason string
if err := pool.QueryRow(ctx, `SELECT reason_code FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='add' AND new_path LIKE '%副本%'`).Scan(&copyReason); err != nil || copyReason != "locator" {
t.Fatalf("ambiguous copy reason=%q err=%v", copyReason, err)
}
var copyDeletes int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='delete'`).Scan(&copyDeletes); err != nil || copyDeletes != 0 {
t.Fatalf("ambiguous copy must not delete the original: %d err=%v", copyDeletes, err)
}
// identity_confidence upsert: stable_id never downgrades; path can rise.
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id2','pp','ID PP2',true) ON CONFLICT DO NOTHING;
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,identity_confidence)
VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'稳.pdf','稳.pdf','document','v1','internal','pp_public_tag','stable_id') ON CONFLICT DO NOTHING;`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,source_change,identity_confidence)
VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'','稳.pdf','document','v1','internal','manual','added','content_hash')
ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE
SET updated_at = source_items.updated_at,
identity_confidence = CASE WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence ELSE EXCLUDED.identity_confidence END;`); err != nil {
t.Fatal(err)
}
var confidence string
if err := pool.QueryRow(ctx, `SELECT identity_confidence FROM source_items WHERE id='item-stable'`).Scan(&confidence); err != nil || confidence != "stable_id" {
t.Fatalf("stable_id downgraded to %q err=%v", confidence, err)
}
}
@@ -0,0 +1,228 @@
package postgres
import (
"context"
"os"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
func seedP3Fixture(t *testing.T, pool *pgxpool.Pool) string {
t.Helper()
ctx := context.Background()
_, seedErr := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p3','alist','P3 来源',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES
('co-p3-a',repeat('a',64),10,'verified/a.pdf','verified'),
('co-p3-b',repeat('b',64),12,'verified/b.pdf','verified'),
('co-p3-c',repeat('c',64),20,'verified/c.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
VALUES('item-a','conn-p3','ext-a',repeat('d',64),'新增.pdf','document','r1','internal','manual'),
('item-b','conn-p3','ext-b',repeat('e',64),'更新.pdf','document','r2','internal','manual'),
('item-c','conn-p3','ext-c',repeat('f',64),'删除.pdf','document','r3','internal','manual'),
('item-d','conn-p3','ext-d',repeat('9',64),'未变.pdf','document','r1','internal','manual'),
('item-e','conn-p3','ext-e',repeat('8',64),'内部专属.pdf','document','r1','internal','manual');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES
('version-a','item-a','r1','co-p3-a','verified'),
('version-b','item-b','r2','co-p3-b','verified'),
('version-c','item-c','r3','co-p3-c','verified'),
('version-d','item-d','r1','co-p3-a','verified'),
('version-e','item-e','r1','co-p3-a','verified');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('p3-old','conn-p3',repeat('1',64),'P3 来源',1,1,0,'2026-09-11T00:00:00Z'),
('p3-new','conn-p3',repeat('2',64),'P3 来源',6,1,5,'2026-09-12T00:00:00Z');
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,content_sha256)
VALUES
('p3-new','root',NULL,'folder','P3 来源','',repeat('3',64),NULL,NULL,NULL),
('p3-new','n-a','root','file','新增.pdf','新增.pdf',repeat('a',64),'item-a',10,repeat('a',64)),
('p3-new','n-b','root','file','更新.pdf','更新.pdf',repeat('b',64),'item-b',12,repeat('b',64)),
('p3-new','n-d','root','file','未变.pdf','未变.pdf',repeat('c',64),'item-d',10,repeat('a',64)),
('p3-new','n-e','root','file','内部专属.pdf','内部专属.pdf',repeat('e',64),'item-e',10,repeat('a',64));
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
VALUES('tp-p3-c','version-c','kb-internal','internal','uploaded','completed','enabled','kn-c',NOW(),NOW());
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
VALUES('cs-p3','conn-p3','p3-old','p3-new',NOW());
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,new_scope,new_source_item_id,old_source_item_id,created_at)
VALUES
('cs-p3',repeat('a',64),'add','新增.pdf','','新增.pdf','internal','item-a',NULL,NOW()),
('cs-p3',repeat('b',64),'update','更新.pdf','更新.pdf','更新.pdf','internal','item-b',NULL,NOW()),
('cs-p3',repeat('c',64),'delete','删除.pdf','删除.pdf','','',NULL,'item-c',NOW()),
('cs-p3',repeat('d',64),'no_change','未变.pdf','未变.pdf','未变.pdf','internal','item-d',NULL,NOW()),
('cs-p3',repeat('e',64),'add','内部专属.pdf','','内部专属.pdf','blocked','item-e',NULL,NOW());`)
if seedErr != nil {
t.Fatal(seedErr)
}
return "cs-p3"
}
func p3Seqs(t *testing.T, pool *pgxpool.Pool, changeSetID string) []int64 {
t.Helper()
rows, err := pool.Query(context.Background(), `SELECT item_seq FROM source_catalog_change_items WHERE change_set_id=$1 ORDER BY item_seq`, changeSetID)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var seqs []int64
for rows.Next() {
var seq int64
if err := rows.Scan(&seq); err != nil {
t.Fatal(err)
}
seqs = append(seqs, seq)
}
if len(seqs) != 5 {
t.Fatalf("expected 5 change items, got %d", len(seqs))
}
return seqs
}
func TestP3ConfirmExecutesSelectedChangeItems(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
id := seedP3Fixture(t, pool)
store := NewStore(pool)
now := time.Date(2026, 9, 12, 3, 0, 0, 0, time.UTC)
// Jobs are zero before confirmation.
var jobsBefore int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobsBefore); err != nil || jobsBefore != 0 {
t.Fatalf("jobs before=%d err=%v", jobsBefore, err)
}
// Partial selection: add, update, delete, no_change, blocked-delete.
result, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
ChangeSetID: id, ItemSeqs: p3Seqs(t, pool, "cs-p3"),
OperatorRef: "engineer-1", Reason: "确认P3差异执行", Now: now,
InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external",
})
if err != nil {
t.Fatal(err)
}
if result.ConfirmedItems != 4 || result.UploadsCreated != 2 || result.RemovalsRequested != 1 || result.NoChangeIgnored != 1 || result.SkippedNoArtifact != 0 || result.SkippedNoTarget != 0 {
t.Fatalf("result=%#v", result)
}
// blocked add became a removal request too (desired state blocked → disable path)
var uploads, removals int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil {
t.Fatal(err)
}
if uploads != 2 || removals != 1 {
t.Fatalf("uploads=%d removals=%d", uploads, removals)
}
// Upload payloads carry the ChangeSet and item identity (P5 will make it a constraint).
var payloadOK bool
if err := pool.QueryRow(ctx, `SELECT COUNT(*)=2 FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'='cs-p3'`).Scan(&payloadOK); err != nil || !payloadOK {
t.Fatalf("upload provenance missing ok=%v err=%v", payloadOK, err)
}
// Publications record their change provenance.
var provenance int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE change_set_id='cs-p3'`).Scan(&provenance); err != nil || provenance != 2 {
t.Fatalf("publication provenance=%d err=%v", provenance, err)
}
// Confirmations bind to the change set.
var confirmations int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM processing_confirmations WHERE change_set_id='cs-p3'`).Scan(&confirmations); err != nil || confirmations != 2 {
t.Fatalf("confirmations=%d err=%v", confirmations, err)
}
// Lifecycle moved to confirmed.
var state string
if err := pool.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id='cs-p3'`).Scan(&state); err != nil || state != "confirmed" {
t.Fatalf("state=%q err=%v", state, err)
}
// Re-confirming the same items replays stored counts; no new jobs or rows.
replaySeqs := p3Seqs(t, pool, "cs-p3")[:1]
replay, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
ChangeSetID: id, ItemSeqs: replaySeqs,
OperatorRef: "engineer-2", Reason: "重复确认", Now: now.Add(time.Minute),
InternalTargetKBID: "kb-internal",
})
if err != nil {
t.Fatal(err)
}
if replay.UploadsCreated != 2 || replay.RemovalsRequested != 1 {
t.Fatalf("replay=%#v", replay)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil || uploads != 2 {
t.Fatalf("replay created uploads=%d err=%v", uploads, err)
}
// Unknown locator belongs to nobody: rejected before any enqueue.
if _, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
ChangeSetID: id, ItemSeqs: []int64{999999},
OperatorRef: "engineer-2", Reason: "越界确认", Now: now, InternalTargetKBID: "kb-internal",
}); err == nil {
t.Fatal("unknown item selection accepted")
}
}
func TestP3ConcurrentConfirmationsEnqueueOnce(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
id := seedP3Fixture(t, pool)
store := NewStore(pool)
now := time.Date(2026, 9, 12, 4, 0, 0, 0, time.UTC)
const workers = 8
var wg sync.WaitGroup
errs := make([]error, workers)
for index := 0; index < workers; index++ {
wg.Add(1)
go func(index int) {
defer wg.Done()
_, errs[index] = store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{
ChangeSetID: id,
ItemSeqs: p3Seqs(t, pool, "cs-p3"),
OperatorRef: "engineer-1", Reason: "并发确认", Now: now.Add(time.Duration(index) * time.Second),
InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external",
})
}(index)
}
wg.Wait()
for index, err := range errs {
if err != nil {
t.Fatalf("worker %d err=%v", index, err)
}
}
// Exactly one enqueue per publication regardless of concurrent confirmations
// (FOR UPDATE on the set serializes; later workers replay from stored counts).
var uploads, removals int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil {
t.Fatal(err)
}
if uploads != 2 || removals != 1 {
t.Fatalf("concurrent enqueue: uploads=%d removals=%d", uploads, removals)
}
}
@@ -0,0 +1,71 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
// TestP5ProcessingJobIdentity freezes docs plan P5: the database rejects a
// duplicate delivery of the same (change_set, version, target, action), while
// a different ChangeSet or action for the same version+target remains legal and
// identity-less jobs never collide.
func TestP5ProcessingJobIdentity(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p5','alist','P5 来源',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p5',repeat('a',64),10,'verified/a.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
VALUES('item-p5','conn-p5','ext-p5',repeat('b',64),'P5.pdf','document','r1','internal','manual');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-p5','item-p5','r1','co-p5','verified');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope) VALUES('tp-p5','version-p5','kb-internal','internal');
INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES('job-p5-1','test.upload','{}','p5:first','cs-p5-a','version-p5','kb-internal','upload')`); err != nil {
t.Fatal(err)
}
// Duplicate delivery of the same processing identity must be rejected.
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES('job-p5-2','test.upload','{}','p5:replay','cs-p5-a','version-p5','kb-internal','upload')`); err == nil || !strings.Contains(err.Error(), "duplicate key value") {
t.Fatalf("duplicate identity err = %v, want unique violation", err)
}
// A later ChangeSet touching the same version+target is a different identity.
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES('job-p5-3','test.upload','{}','p5:next','cs-p5-b','version-p5','kb-internal','upload')`); err != nil {
t.Fatalf("different change set rejected: %v", err)
}
// A different action on the same identity tuple is legal.
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action)
VALUES('job-p5-4','test.remove','{}','p5:remove','cs-p5-a','version-p5','kb-internal','remove')`); err != nil {
t.Fatalf("different action rejected: %v", err)
}
// Jobs without business identity never collide with the partial index.
for _, id := range []string{"job-p5-5", "job-p5-6"} {
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key) VALUES($1,'test.poll','{}',$2)`, id, "plain-"+id); err != nil {
t.Fatalf("unidentified job %s: %v", id, err)
}
}
var count int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM jobs WHERE business_version_id='version-p5'`).Scan(&count); err != nil || count != 3 {
t.Fatalf("identified jobs = %d err = %v, want 3", count, err)
}
}
@@ -0,0 +1,179 @@
package postgres
import (
"context"
"encoding/json"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// TestP6DeadLetterAndServerSideBatchRetry freezes docs plan P6: an ordinary
// failure that exhausts its budget lands in the DLQ projection without
// touching the original audit row, a permanent failure lands immediately, and
// the server-side batch retry returns mixed outcomes atomically with replay.
func TestP6DeadLetterAndServerSideBatchRetry(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("migrate: %v", err)
}
_, seedErr := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p6','alist','P6 来源',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p6',repeat('a',64),10,'verified/p6.pdf','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
VALUES('item-p6','conn-p6','ext-p6',repeat('d',64),'P6.pdf','document','r1','internal','manual');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES
('siv-1','item-p6','r1','co-p6','verified'),
('siv-2','item-p6','r2','co-p6','verified'),
('siv-3','item-p6','r3','co-p6','verified'),
('siv-4','item-p6','r4','co-p6','verified');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,updated_at) VALUES
('tp-1','siv-1','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
('tp-2','siv-2','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
('tp-3','siv-3','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'),
('tp-5','siv-4','kb-1','uploaded','processing','unpublished','internal','2026-09-12T01:00:00Z');`)
if seedErr != nil {
t.Fatalf("seed: %v", seedErr)
}
store := NewStore(pool)
base := time.Date(2026, time.September, 12, 1, 0, 0, 0, time.UTC)
// 1) Ordinary failure exhausting the budget projects into the DLQ.
uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-1"})
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-1','test.upload',$1,'running',5,5,'k-dlq-1','w1','cs-p6','siv-1','kb-1','upload',$2)`, uploadPayload, base); err != nil {
t.Fatalf("seed job: %v", err)
}
if err := store.Fail(ctx, "job-dlq-1", "w1", "remote parse failed", "validation_failed", base, base.Add(time.Minute)); err != nil {
t.Fatalf("exhausting fail: %v", err)
}
var jobState, errorCode, failureState string
var attempt int
if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState, &errorCode); err != nil {
t.Fatal(err)
}
if jobState != "failed" || errorCode != "validation_failed" {
t.Fatalf("exhausted job = (%s,%s)", jobState, errorCode)
}
var failedPublicationID string
if err := pool.QueryRow(ctx, `SELECT state,attempt,error_code,publication_id FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &attempt, &errorCode, &failedPublicationID); err != nil {
t.Fatalf("dlq projection missing: %v", err)
}
if failureState != "open" || attempt != 5 || errorCode != "validation_failed" || failedPublicationID != "tp-1" {
t.Fatalf("dlq row = (%s,%d,%s,%s)", failureState, attempt, errorCode, failedPublicationID)
}
// 2) Permanent failure parks immediately and projects.
removePayload, _ := json.Marshal(map[string]string{"publication_id": "tp-3"})
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-3','test.remove',$1,'running',1,10,'k-dlq-3','w1','cs-p6','siv-3','','remove',$2)`, removePayload, base); err != nil {
t.Fatalf("seed remove job: %v", err)
}
var _ jobs.TerminalFailer = store
if err := store.FailTerminal(ctx, "job-dlq-3", "w1", "remote rejected", "remote_permanent", base); err != nil {
t.Fatalf("terminal fail: %v", err)
}
if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-3'`).Scan(&jobState, &errorCode); err != nil {
t.Fatal(err)
}
if jobState != "failed" || errorCode != "remote_permanent" {
t.Fatalf("terminal job = (%s,%s)", jobState, errorCode)
}
// 3) DLQ query lists both open projections.
page, err := store.ListJobFailures(ctx, governance.JobFailureFilter{Limit: 50})
if err != nil {
t.Fatalf("list failures: %v", err)
}
if page.Total != 2 || len(page.Items) != 2 {
t.Fatalf("open failures = %d/%d, want 2", len(page.Items), page.Total)
}
// 4) Server-side batch retry: tp-1 + tp-2 started, unknown id not_retryable.
result, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
OperatorRef: "姚工",
Reason: "解析服务恢复",
Now: base.Add(2 * time.Minute),
}, "op-retry-batch-1")
if err != nil || replayed {
t.Fatalf("batch retry: %v replayed=%v", err, replayed)
}
if result.Total != 4 || result.Started != 2 || result.NotRetryable != 1 || result.AlreadyRunning != 1 {
t.Fatalf("batch result = %+v", result)
}
if result.Outcomes[0].PublicationID != "tp-1" || result.Outcomes[0].Result != governance.RetryStarted {
t.Fatalf("tp-1 outcome = %+v", result.Outcomes[0])
}
if result.Outcomes[1].PublicationID != "tp-2" || result.Outcomes[1].Result != governance.RetryStarted {
t.Fatalf("tp-2 outcome = %+v", result.Outcomes[1])
}
if result.Outcomes[2].PublicationID != "tp-4-missing" || result.Outcomes[2].Result != governance.RetryNotRetryable || result.Outcomes[2].Detail != "publication not found" {
t.Fatalf("missing outcome = %+v", result.Outcomes[2])
}
if result.Outcomes[3].PublicationID != "tp-5" || result.Outcomes[3].Result != governance.RetryAlreadyRunning {
t.Fatalf("tp-5 outcome = %+v", result.Outcomes[3])
}
// tp-1's open DLQ row flipped to requeued with operator and reason.
var actor, reason string
if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &actor, &reason); err != nil {
t.Fatal(err)
}
if failureState != "requeued" || actor != "姚工" || reason != "解析服务恢复" {
t.Fatalf("tp-1 failure after retry = (%s,%s,%s)", failureState, actor, reason)
}
// tp-3's failure stays open (not part of this batch).
if err := pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE job_id='job-dlq-3'`).Scan(&failureState); err != nil || failureState != "open" {
t.Fatalf("tp-3 failure state = %s err=%v", failureState, err)
}
// Audit rows recorded atomically for every selection.
var auditCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM job_retry_operations WHERE batch_key='op-retry-batch-1'`).Scan(&auditCount); err != nil || auditCount != 4 {
t.Fatalf("retry audit rows = %d err=%v", auditCount, err)
}
// Fresh retry jobs exist and the original exhausted job stays untouched.
var newJobs int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND state='pending' AND idempotency_key LIKE 'test-retry:tp-1:%'`).Scan(&newJobs); err != nil || newJobs != 1 {
t.Fatalf("tp-1 retry jobs = %d err=%v", newJobs, err)
}
if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState); err != nil || jobState != "failed" {
t.Fatalf("original job state = %q err=%v, want failed (untouched)", jobState, err)
}
// 5) Replaying the same operation id returns the identical stored result.
replay, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"},
OperatorRef: "姚工",
Reason: "解析服务恢复",
Now: base.Add(9 * time.Minute),
}, "op-retry-batch-1")
if err != nil || !replayed {
t.Fatalf("replay: %v replayed=%v", err, replayed)
}
if replay.Started != result.Started || replay.Total != result.Total {
t.Fatalf("replay result = %+v, want %+v", replay, result)
}
// 6) Same key with a different body conflicts.
if _, _, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{
PublicationIDs: []string{"tp-2"},
OperatorRef: "姚工",
Reason: "解析服务恢复",
Now: base.Add(10 * time.Minute),
}, "op-retry-batch-1"); err == nil {
t.Fatal("expected operation request conflict for same key different body")
}
}
+173
View File
@@ -0,0 +1,173 @@
package postgres
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
ppconnector "knowledge-governance/internal/connectors/pp"
)
type LegacyImportReport struct {
Connections int
SourceItems int
ContentObjects int
Versions int
Publications int
}
func LoadPPCurrentItems(ctx context.Context, pool *pgxpool.Pool, connectionID string) (map[string]ppconnector.CurrentItem, error) {
rows, err := pool.Query(ctx, `
SELECT stable_external_id, revision
FROM source_items
WHERE source_connection_id = $1
ORDER BY stable_external_id`, connectionID)
if err != nil {
return nil, fmt.Errorf("query current PP items: %w", err)
}
defer rows.Close()
items := make(map[string]ppconnector.CurrentItem)
for rows.Next() {
var sourceHash string
var item ppconnector.CurrentItem
if err := rows.Scan(&sourceHash, &item.Revision); err != nil {
return nil, fmt.Errorf("scan current PP item: %w", err)
}
items[sourceHash] = item
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate current PP items: %w", err)
}
return items, nil
}
func ImportPPLegacy(ctx context.Context, pool *pgxpool.Pool, plan ppconnector.ImportPlan) (LegacyImportReport, error) {
tx, err := pool.Begin(ctx)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("begin PP legacy import: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
report := LegacyImportReport{}
tag, err := tx.Exec(ctx, `INSERT INTO source_connections (
id, source_type, name, base_url, secret_ref, enabled, schedule, last_health
) VALUES ($1, $2, $3, $4, $5, $6, '{}'::jsonb, '{}'::jsonb)
ON CONFLICT DO NOTHING`,
plan.Connection.ID,
plan.Connection.Type,
plan.Connection.Name,
plan.Connection.BaseURL,
plan.Connection.SecretRef,
plan.Connection.Enabled,
)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("insert PP source connection: %w", err)
}
report.Connections = int(tag.RowsAffected())
for _, content := range plan.ContentObjects {
tag, err = tx.Exec(ctx, `INSERT INTO content_objects (
id, sha256, size_bytes, mime, validated_format, local_spool_ref, content_state
) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT DO NOTHING`,
content.ID,
content.SHA256,
content.SizeBytes,
content.MIME,
content.ValidatedFormat,
content.LocalSpoolRef,
content.State,
)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("insert PP content object: %w", err)
}
report.ContentObjects += int(tag.RowsAffected())
}
for _, item := range plan.SourceItems {
tag, err = tx.Exec(ctx, `INSERT INTO source_items (
id, source_connection_id, stable_external_id, canonical_uri_hash,
parent_external_id, normalized_path, title, item_type, size_bytes,
mime, revision, visibility, visibility_origin, last_seen_scan_id,
source_change, absence_streak, identity_confidence
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9,
$10, $11, $12, $13, $14, $15, $16, 'stable_id'
)
ON CONFLICT DO NOTHING`,
item.ID,
item.SourceConnectionID,
item.StableExternalID,
item.CanonicalURIHash,
item.ParentExternalID,
item.NormalizedPath,
item.Title,
item.ItemType,
item.SizeBytes,
item.MIME,
item.Revision,
item.Visibility,
item.VisibilityOrigin,
item.LastSeenScanID,
item.SourceChange,
item.AbsenceStreak,
)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("insert PP source item: %w", err)
}
report.SourceItems += int(tag.RowsAffected())
}
for _, version := range plan.Versions {
tag, err = tx.Exec(ctx, `INSERT INTO source_item_versions (
id, source_item_id, revision, content_object_id, content_state,
download_error, raw_record_summary, source_updated_at
) VALUES ($1, $2, $3, $4, $5, $6, CAST($7 AS JSONB), $8)
ON CONFLICT DO NOTHING`,
version.ID,
version.SourceItemID,
version.Revision,
version.ContentObjectID,
version.ContentState,
version.DownloadError,
string(version.RawRecordSummary),
version.SourceUpdatedAt,
)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("insert PP source item version: %w", err)
}
report.Versions += int(tag.RowsAffected())
}
for _, publication := range plan.Publications {
tag, err = tx.Exec(ctx, `INSERT INTO test_publications (
id, source_item_version_id, test_kb_id, knowledge_id,
ingest_state, processing_state, retrieval_state,
parser_generation, index_generation, last_error, retry_count
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT DO NOTHING`,
publication.ID,
publication.SourceItemVersionID,
publication.TestKBID,
publication.KnowledgeID,
publication.IngestState,
publication.ProcessingState,
publication.RetrievalState,
publication.ParserGeneration,
publication.IndexGeneration,
publication.LastError,
publication.RetryCount,
)
if err != nil {
return LegacyImportReport{}, fmt.Errorf("insert PP test publication: %w", err)
}
report.Publications += int(tag.RowsAffected())
}
if err := tx.Commit(ctx); err != nil {
return LegacyImportReport{}, fmt.Errorf("commit PP legacy import: %w", err)
}
return report, nil
}
@@ -0,0 +1,155 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
ppconnector "knowledge-governance/internal/connectors/pp"
"knowledge-governance/internal/governance"
)
func TestImportPPLegacyIsIdempotent(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("pgxpool.New() error = %v", err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("ApplyMigrations() error = %v", err)
}
sourceCanonical := strings.Repeat("a", 64)
sourceAlias := strings.Repeat("b", 64)
contentHash := strings.Repeat("c", 64)
plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical},
}})
if err != nil {
t.Fatalf("BuildImportPlan() error = %v", err)
}
first, err := ImportPPLegacy(ctx, pool, plan)
if err != nil {
t.Fatalf("first ImportPPLegacy() error = %v", err)
}
if first != (LegacyImportReport{Connections: 1, SourceItems: 2, ContentObjects: 1, Versions: 2, Publications: 1}) {
t.Fatalf("first ImportPPLegacy() report = %#v", first)
}
second, err := ImportPPLegacy(ctx, pool, plan)
if err != nil {
t.Fatalf("second ImportPPLegacy() error = %v", err)
}
if second != (LegacyImportReport{}) {
t.Fatalf("second ImportPPLegacy() report = %#v, want zero inserts", second)
}
var counts [5]int
if err := pool.QueryRow(ctx, `SELECT
(SELECT COUNT(*) FROM source_connections),
(SELECT COUNT(*) FROM source_items),
(SELECT COUNT(*) FROM content_objects),
(SELECT COUNT(*) FROM source_item_versions),
(SELECT COUNT(*) FROM test_publications)`).Scan(&counts[0], &counts[1], &counts[2], &counts[3], &counts[4]); err != nil {
t.Fatalf("query imported counts: %v", err)
}
if counts != [5]int{1, 2, 1, 2, 1} {
t.Fatalf("imported counts = %#v", counts)
}
}
func TestLoadPPCurrentItemsSupportsZeroUploadReconciliation(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
source := strings.Repeat("d", 64)
contentHash := strings.Repeat("e", 64)
snapshot := ppconnector.Snapshot{KBID: "kb-reconcile", Items: []ppconnector.Item{{
KBID: "kb-reconcile", SourceURLHash: source, SourceIDs: []string{"pp-3"}, Bytes: 12,
SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-3",
}}}
plan, err := ppconnector.BuildImportPlan(snapshot)
if err != nil {
t.Fatal(err)
}
if _, err := ImportPPLegacy(ctx, pool, plan); err != nil {
t.Fatal(err)
}
current, err := LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
if err != nil {
t.Fatal(err)
}
summary, err := ppconnector.PlanReconciliation(snapshot, current)
if err != nil {
t.Fatal(err)
}
if summary.UploadRequired != 0 || summary.Unchanged != 1 {
t.Fatalf("summary = %#v", summary)
}
}
func TestStoreListsSourceAssetsWithSummaryAndStateFilter(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-assets", Items: []ppconnector.Item{
{KBID: "kb-assets", SourceURLHash: strings.Repeat("1", 64), SourceIDs: []string{"pp-5"}, Bytes: 10, SHA256: strings.Repeat("3", 64), Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-5"},
{KBID: "kb-assets", SourceURLHash: strings.Repeat("2", 64), SourceIDs: []string{"pp-6"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
}})
if err != nil {
t.Fatal(err)
}
for index := range plan.SourceItems {
plan.SourceItems[index].Title = "合成手册.pdf"
plan.SourceItems[index].NormalizedPath = "使用指南"
}
if _, err := ImportPPLegacy(ctx, pool, plan); err != nil {
t.Fatal(err)
}
page, err := NewStore(pool).ListSourceAssets(ctx, governance.SourceAssetFilter{State: "over_limit", Limit: 25})
if err != nil {
t.Fatal(err)
}
if page.Summary != (governance.SourceAssetSummary{Total: 2, Completed: 1, OverLimit: 1}) {
t.Fatalf("summary = %#v", page.Summary)
}
if len(page.Items) != 1 || page.Items[0].ContentState != governance.ContentOverLimit || page.Items[0].HasPublication {
t.Fatalf("items = %#v", page.Items)
}
item := page.Items[0]
if item.Title != "合成手册.pdf" || item.DirectoryPath != "使用指南" || item.ConnectionID != plan.Connection.ID || item.VisibilityOrigin != "pp_public_tag" || item.CanRetry {
t.Fatalf("document navigation metadata mismatch: %#v", item)
}
}
@@ -0,0 +1,137 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) {
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
operator := strings.TrimSpace(request.OperatorRef)
reason := strings.TrimSpace(request.Reason)
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
}
seen := make(map[string]bool, len(request.PublicationIDs))
for _, publicationID := range request.PublicationIDs {
if !validRouteID(publicationID) || seen[publicationID] {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing publication IDs")
}
seen[publicationID] = true
}
var tx pgx.Tx
var err error
ownsTransaction := false
if existing, ok := s.db.(pgx.Tx); ok {
tx = existing
} else {
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return governance.StartTestPublicationsResult{}, errors.New("processing confirmation transactions are unavailable")
}
var err error
tx, err = beginner.Begin(ctx)
if err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("begin processing confirmation: %w", err)
}
ownsTransaction = true
defer tx.Rollback(ctx)
}
for _, publicationID := range request.PublicationIDs {
var ready, confirmed, uploadJobExists bool
err = tx.QueryRow(ctx, `
SELECT p.ingest_state='pending'
AND p.processing_state='pending'
AND p.retrieval_state='unpublished'
AND p.knowledge_id=''
AND version.content_state='verified'
AND content.content_state='verified'
AND content.local_spool_ref<>''
AND content.size_bytes>0 AS ready,
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN content_objects content ON content.id=version.content_object_id
WHERE p.id=$1
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
if errors.Is(err, pgx.ErrNoRows) {
result.Unavailable++
continue
}
if err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("inspect processing plan: %w", err)
}
if confirmed || uploadJobExists {
result.AlreadyStarted++
continue
}
if !ready {
result.Unavailable++
continue
}
confirmationID, err := newOpaqueID("pc-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
}
jobID, err := newOpaqueID("job-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
}
payload, err := json.Marshal(map[string]string{"publication_id": publicationID})
if err != nil {
return governance.StartTestPublicationsResult{}, errors.New("encode processing upload job")
}
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,created_at) VALUES($1,$2,$3,$4,$5,$5)`, confirmationID, publicationID, operator, reason, request.Now); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("record processing confirmation: %w", err)
}
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3)`, jobID, payload, request.Now, "test-start:"+publicationID); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("enqueue confirmed processing: %w", err)
}
result.Started++
}
if ownsTransaction {
if err := tx.Commit(ctx); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("commit processing confirmation: %w", err)
}
}
return result, nil
}
func (s *Store) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) {
fingerprint, err := operationFingerprint(struct {
PublicationIDs []string `json:"publication_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{request.PublicationIDs, request.OperatorRef, request.Reason})
if err != nil {
return governance.StartTestPublicationsResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "test.publications.start", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := NewStore(tx).StartTestPublications(ctx, request)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.StartTestPublicationsResult{}, false, err
}
var result governance.StartTestPublicationsResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.StartTestPublicationsResult{}, false, fmt.Errorf("decode stored processing result: %w", err)
}
return result, replayed, nil
}
@@ -0,0 +1,126 @@
package postgres
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
func TestProductionDeleteBatchIsReversibleAndReconcilesPlan(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 10, 5, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
t.Fatal(err)
}
publicationID := "tp-delete-e2e"
insertReviewFixture(t, ctx, pool, 7, publicationID, now)
review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)})
if err != nil {
t.Fatal(err)
}
initialBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Initial publish", CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: initialBatch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
t.Fatal(err)
}
initialExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: initialBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
if err != nil {
t.Fatal(err)
}
initialPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: initialExecution.ID, JobID: initialExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
if err != nil || len(initialPlan) != 1 || initialPlan[0].Action != "upsert" {
t.Fatalf("initial plan=%#v error=%v", initialPlan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: initialExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "absent", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, initialExecution.ID, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-7','blocked','engineer-1','确认删除生产知识',$1,$1)`, now.Add(8*time.Minute)); err != nil {
t.Fatal(err)
}
deletePlan, err := store.ListReleasePlan(ctx)
if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != publicationID {
t.Fatalf("delete plan=%#v error=%v", deletePlan, err)
}
deleteBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Delete production knowledge", CreatedBy: "engineer-1", DeletePublicationIDs: []string{publicationID}, Now: now.Add(9 * time.Minute)})
if err != nil {
t.Fatal(err)
}
detail, err := store.GetReleaseBatch(ctx, deleteBatch.ID)
if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" {
t.Fatalf("delete batch=%#v error=%v", detail, err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: deleteBatch.ID, SealedBy: "engineer-2", Now: now.Add(10 * time.Minute)}); err != nil {
t.Fatal(err)
}
deleteExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: deleteBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(11 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-7'`); err != nil {
t.Fatal(err)
}
if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12 * time.Minute)}); !errors.Is(err, ErrReleaseExecutionNotClaimable) {
t.Fatalf("unavailable source start error=%v", err)
}
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-7'`); err != nil {
t.Fatal(err)
}
deleteItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12*time.Minute + time.Second)})
if err != nil || len(deleteItems) != 1 || deleteItems[0].Action != "delete" || deleteItems[0].PreviousProductionRevision != review.ID {
t.Fatalf("delete items=%#v error=%v", deleteItems, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: deleteExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: review.ID, ProductionRevision: "absent", Now: now.Add(13 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, deleteExecution.ID, now.Add(14*time.Minute)); err != nil {
t.Fatal(err)
}
afterDelete, err := store.ListReleasePlan(ctx)
if err != nil || len(afterDelete.Items) != 0 {
t.Fatalf("plan after delete=%#v error=%v", afterDelete, err)
}
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: deleteExecution.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(15 * time.Minute)})
if err != nil {
t.Fatal(err)
}
rollbackItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)})
if err != nil || len(rollbackItems) != 1 || rollbackItems[0].PreviousProductionRevision != review.ID {
t.Fatalf("rollback items=%#v error=%v", rollbackItems, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: publicationID, PreviousProductionRevision: review.ID, ProductionRevision: review.ID, Now: now.Add(17 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil {
t.Fatal(err)
}
afterRollback, err := store.ListReleasePlan(ctx)
if err != nil || len(afterRollback.Items) != 1 || afterRollback.Items[0].Action != "delete" {
t.Fatalf("plan after rollback=%#v error=%v", afterRollback, err)
}
}
@@ -0,0 +1,65 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
type PublicationForRemoval struct {
ID string
TestKBID string
KnowledgeID string
RetrievalState governance.RetrievalState
}
func (s *Store) LoadPublicationForRemoval(ctx context.Context, publicationID string) (PublicationForRemoval, error) {
var candidate PublicationForRemoval
var retrievalState string
err := s.db.QueryRow(ctx, `SELECT id,test_kb_id,knowledge_id,retrieval_state
FROM test_publications
WHERE id=$1 AND knowledge_id<>'' AND retrieval_state='disabling'`, publicationID).Scan(
&candidate.ID, &candidate.TestKBID, &candidate.KnowledgeID, &retrievalState,
)
if errors.Is(err, pgx.ErrNoRows) {
return PublicationForRemoval{}, ErrPublicationStateConflict
}
if err != nil {
return PublicationForRemoval{}, fmt.Errorf("load publication for removal: %w", err)
}
candidate.RetrievalState = governance.RetrievalState(retrievalState)
return candidate, nil
}
func (s *Store) RecordRemovalCompleted(ctx context.Context, publicationID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE test_publications
SET retrieval_state='disabled',last_error='',updated_at=$2
WHERE id=$1 AND retrieval_state='disabling'`, publicationID, now)
if err != nil {
return fmt.Errorf("record publication removal: %w", err)
}
if tag.RowsAffected() != 1 {
return ErrPublicationStateConflict
}
return nil
}
func (s *Store) RecordRemovalFailure(ctx context.Context, publicationID, code string, now time.Time) error {
if code != "removal_identity_invalid" && code != "removal_remote_rejected" {
return ErrInvalidErrorCode
}
tag, err := s.db.Exec(ctx, `UPDATE test_publications
SET retrieval_state='failed',last_error=$2,updated_at=$3
WHERE id=$1 AND retrieval_state='disabling'`, publicationID, code, now)
if err != nil {
return fmt.Errorf("record publication removal failure: %w", err)
}
if tag.RowsAffected() != 1 {
return ErrPublicationStateConflict
}
return nil
}
+169
View File
@@ -0,0 +1,169 @@
package postgres
const enqueueJobSQL = `
INSERT INTO jobs (
id, type, payload, state, priority, available_at,
max_attempts, idempotency_key, created_at, updated_at
) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8, $8)
ON CONFLICT (idempotency_key)
DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key
RETURNING id, type, payload, state, priority, available_at,
lease_owner, lease_until, attempt, max_attempts,
last_error, idempotency_key, created_at, updated_at
`
const claimJobSQL = `
WITH candidate AS (
SELECT id
FROM jobs
WHERE (
(state IN ('pending', 'retry') AND available_at <= $2)
OR (state = 'running' AND lease_until < $2)
)
AND attempt < max_attempts
AND type = ANY($4::text[])
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1
)
UPDATE jobs AS j
SET state = 'running',
started_at = $2,
lease_owner = $1,
lease_until = $3,
attempt = j.attempt + 1,
updated_at = $2
FROM candidate
WHERE j.id = candidate.id
RETURNING j.id, j.type, j.payload, j.state, j.priority, j.available_at,
j.lease_owner, j.lease_until, j.attempt, j.max_attempts,
j.last_error, j.idempotency_key, j.created_at, j.updated_at
`
const completeJobSQL = `
UPDATE jobs
SET state = 'completed',
completed_at = $3,
lease_owner = NULL,
lease_until = NULL,
updated_at = $3
WHERE id = $1
AND lease_owner = $2
AND state = 'running'
`
const heartbeatJobSQL = `
UPDATE jobs
SET lease_until = $4,
updated_at = $3
WHERE id = $1
AND lease_owner = $2
AND state = 'running'
`
const failJobSQL = `
WITH updated AS (
UPDATE jobs
SET state = CASE WHEN attempt >= max_attempts THEN 'failed' ELSE 'retry' END,
available_at = $6,
last_error = $3,
last_error_code = CASE WHEN $5 <> '' THEN $5 ELSE last_error_code END,
lease_owner = NULL,
lease_until = NULL,
updated_at = $4
WHERE id = $1
AND lease_owner = $2
AND state = 'running'
RETURNING id, type, payload, last_error,
CASE WHEN last_error_code = '' THEN 'handler_error' ELSE last_error_code END AS error_code,
attempt, max_attempts,
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt,
state
)
INSERT INTO job_failures (
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
business_change_set_id, business_version_id, business_action, business_target_kb_id,
created_at
)
SELECT updated.id, updated.type, COALESCE(updated.payload->>'publication_id',''), updated.error_code,
updated.last_error, updated.attempt, updated.max_attempts,
updated.bc, updated.bv, updated.ba, updated.bt, $4
FROM updated
WHERE updated.state = 'failed' AND updated.attempt > 0
ON CONFLICT (job_id, attempt) DO UPDATE
SET publication_id = EXCLUDED.publication_id,
error_code = EXCLUDED.error_code,
last_error = EXCLUDED.last_error,
max_attempts_at_failure = EXCLUDED.max_attempts_at_failure,
business_change_set_id = EXCLUDED.business_change_set_id,
business_version_id = EXCLUDED.business_version_id,
business_action = EXCLUDED.business_action,
business_target_kb_id = EXCLUDED.business_target_kb_id,
state = 'open',
resolved_at = NULL,
resolution_actor_ref = '',
resolution_reason = '',
created_at = EXCLUDED.created_at
`
const listJobsSQL = `
SELECT id, type, payload, state, priority, available_at,
lease_owner, lease_until, attempt, max_attempts,
last_error, idempotency_key, created_at, updated_at
FROM jobs
ORDER BY created_at DESC, id DESC
LIMIT $1
`
// P6: a permanent failure parks the job in 'failed' immediately and projects
// the terminal attempt into the dead-letter table.
const failTerminalJobSQL = `
WITH parked AS (
UPDATE jobs
SET state = 'failed',
available_at = $5,
last_error = $3,
last_error_code = CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END,
lease_owner = NULL,
lease_until = NULL,
updated_at = $5
WHERE id = $1
AND lease_owner = $2
AND lease_owner IS NOT NULL
AND state = 'running'
RETURNING id, type, payload, last_error,
CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END AS error_code,
attempt, max_attempts,
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt
)
INSERT INTO job_failures (
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
business_change_set_id, business_version_id, business_action, business_target_kb_id,
created_at
)
SELECT parked.id, parked.type, COALESCE(parked.payload->>'publication_id',''), parked.error_code,
parked.last_error, GREATEST(parked.attempt,1), parked.max_attempts,
parked.bc, parked.bv, parked.ba, parked.bt, $5
FROM parked
ON CONFLICT (job_id, attempt) DO UPDATE
SET publication_id = EXCLUDED.publication_id,
error_code = EXCLUDED.error_code,
last_error = EXCLUDED.last_error,
max_attempts_at_failure = EXCLUDED.max_attempts_at_failure,
state = 'open',
resolved_at = NULL,
resolution_actor_ref = '',
resolution_reason = '',
created_at = EXCLUDED.created_at
`
// P10 liveness: every claim attempt refreshes the worker row so the API can
// distinguish "worker loop alive" from "process dead" without an HTTP port.
const workerHeartbeatSQL = `
INSERT INTO worker_heartbeats(worker_id, job_types, started_at, last_seen_at)
VALUES($1, $2, $3, $3)
ON CONFLICT (worker_id) DO UPDATE
SET job_types = EXCLUDED.job_types, last_seen_at = EXCLUDED.last_seen_at
`
@@ -0,0 +1,317 @@
package postgres
import (
"context"
"errors"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
// seedR1ReleaseFixture gives us:
// - tp-keep: fully review+acceptance approved, present source -> add
// - tp-drop: was released to prod, source now deleted -> delete
// - tp-block: source unavailable -> blocked
// - tp-unch: already released to prod with same revision -> unchanged
func seedR1ReleaseFixture(t *testing.T, pool *pgxpool.Pool, ctx context.Context) {
t.Helper()
_, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name) VALUES
('conn-r1','alist','R1来源');
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES
('co-r1',repeat('a',64),10,'spool/r1','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,normalized_path,revision,visibility,visibility_origin,source_change,identity_confidence) VALUES
('item-keep','conn-r1','ext-keep',repeat('1',64),'新增手册','document','新增手册.pdf','v1','internal','manual','unchanged','stable_id'),
('item-drop','conn-r1','ext-drop',repeat('2',64),'已删手册','document','已删手册.pdf','v1','internal','manual','deleted','stable_id'),
('item-block','conn-r1','ext-block',repeat('3',64),'不可用手册','document','不可用手册.pdf','v1','internal','manual','unavailable','stable_id'),
('item-unch','conn-r1','ext-unch',repeat('4',64),'不变手册','document','不变手册.pdf','v1','internal','manual','unchanged','stable_id');
INSERT INTO source_item_versions(id,source_item_id,revision,content_state) VALUES
('siv-keep','item-keep','v1','verified'),
('siv-drop','item-drop','v1','verified'),
('siv-block','item-block','v1','verified'),
('siv-unch','item-unch','v1','verified');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,processing_state,ingest_state,retrieval_state,updated_at) VALUES
('tp-keep','siv-keep','kb-r1','internal','completed','uploaded','unpublished',now() - interval '1 hour'),
('tp-drop','siv-drop','kb-r1','internal','completed','uploaded','unpublished',now() - interval '2 hours'),
('tp-block','siv-block','kb-r1','internal','completed','uploaded','unpublished',now() - interval '3 hours'),
('tp-unch','siv-unch','kb-r1','internal','completed','uploaded','unpublished',now() - interval '4 hours');
INSERT INTO acceptance_results(id,publication_id,run_id,check_type,state,policy_version,observed_at) VALUES
('ace-keep-1','tp-keep','run-keep','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
('ace-keep-2','tp-keep','run-keep','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
('ace-keep-3','tp-keep','run-keep','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'),
('ace-unch-1','tp-unch','run-unch','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
('ace-unch-2','tp-unch','run-unch','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
('ace-unch-3','tp-unch','run-unch','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'),
('ace-drop-1','tp-drop','run-drop','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'),
('ace-drop-2','tp-drop','run-drop','chunks_present','passed','structural-v1',now() - interval '50 minutes'),
('ace-drop-3','tp-drop','run-drop','retrieval_hit','passed','structural-v1',now() - interval '50 minutes');
INSERT INTO publication_reviews(id,publication_id,decision,decided_visibility,reviewer_ref,decision_code,acceptance_run_id,publication_updated_at,created_at)
SELECT 'rev-'||p.id, p.id, 'approved', 'internal', 'reviewer-r1', 'auto_approve', split_part(a.run_id,'',1), p.updated_at, now()
FROM test_publications p
JOIN LATERAL (SELECT CASE p.id WHEN 'tp-keep' THEN 'run-keep' WHEN 'tp-unch' THEN 'run-unch' ELSE 'run-drop' END AS run_id) a ON TRUE
WHERE p.id IN ('tp-keep','tp-unch','tp-drop');
INSERT INTO release_batches(id,name,state,policy_version,created_by,item_count,created_at,sealed_at,release_change_set_id) VALUES
('rb-seed','历史批次','sealed','release-gate-v1','seed',2,now() - interval '1 day',now() - interval '1 day',NULL);
INSERT INTO release_batch_items(release_batch_id,publication_id,review_id,source_item_version_id,acceptance_run_id,publication_updated_at,decided_visibility,action,created_at) VALUES
('rb-seed','tp-unch','rev-tp-unch','siv-unch','run-unch',(SELECT updated_at FROM test_publications WHERE id='tp-unch'),'internal','upsert',now() - interval '1 day'),
('rb-seed','tp-drop','rev-tp-drop','siv-drop','run-drop',(SELECT updated_at FROM test_publications WHERE id='tp-drop'),'internal','upsert',now() - interval '1 day');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES
('snap-r1','conn-r1',repeat('7',64),'R1快照',2,1,1,now() - interval '1 day');
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES
('conn-r1','snap-r1',now() - interval '1 day');
INSERT INTO jobs(id,type,payload,state,available_at,attempt,max_attempts,idempotency_key) VALUES
('job-r1-seed','release.publish','{}','completed',now() - interval '23 hours',1,5,'kr1-seed');
INSERT INTO release_executions(id,release_batch_id,operation,state,authorized_by,authorization_code,job_id,item_count,started_at,completed_at) VALUES
('re-seed','rb-seed','publish','completed','seed','seed_auth',
'job-r1-seed',2,now() - interval '1 day',now() - interval '23 hours');
INSERT INTO release_item_executions(id,release_execution_id,publication_id,production_revision,state,completed_at)
VALUES ('rie-seed','re-seed','tp-unch','rev-tp-unch','verified',now() - interval '23 hours'),
('rie-seed-2','re-seed','tp-drop','rev-tp-drop','verified',now() - interval '23 hours');`)
if err != nil {
t.Fatalf("seed r1 fixture: %v", err)
}
}
// TestR1ReleaseChangeSetLifecycle freezes docs plan R1: generation snapshots the
// live plan immutably, confirmation is required before formal batching, and
// later source refreshes never move a frozen manifest.
func TestR1ReleaseChangeSetLifecycle(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, time.September, 12, 7, 0, 0, 0, time.UTC)
plan, err := store.ListReleasePlan(ctx)
if err == nil && len(plan.Items) > 0 {
t.Skip("fixture isolation failed: live plan not empty")
}
seedR1ReleaseFixture(t, pool, ctx)
first, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布", CreatedBy: "yaozhanghao", Now: now})
if err != nil {
t.Fatal(err)
}
if first.State != "planned" || first.Number != 1 || first.ManifestSHA256 == "" || len(first.ManifestSHA256) != 64 {
t.Fatalf("first set = %#v", first)
}
if len(first.SourceSnapshotIDs) == 0 {
t.Fatalf("R1 core: change set must bind the head catalog snapshots it was computed against")
}
bound := false
for _, id := range first.SourceSnapshotIDs {
if id == "snap-r1" {
bound = true
}
}
if !bound {
t.Fatalf("head snapshot not bound: %#v", first.SourceSnapshotIDs)
}
detail, err := store.GetReleaseChangeSet(ctx, first.ID)
if err != nil {
t.Fatal(err)
}
actions := map[string]string{}
blockedRows := 0
for _, item := range detail.Items {
actions[item.PublicationID] = item.Action
if item.Action == "blocked" {
blockedRows++
}
}
if actions["tp-keep"] != "add" || actions["tp-drop"] != "delete" {
t.Fatalf("frozen actions = %#v", actions)
}
// R1 core rule: an unavailable source can ONLY surface as blocked (never a
// deletable/updatable item with a real publication binding).
if blockedRows != 1 || actions["tp-block"] != "" && actions["tp-block"] != "blocked" {
t.Fatalf("blocked handling = %#v rows=%d", actions, blockedRows)
}
if _, ok := actions["tp-unch"]; ok {
t.Fatal("unchanged items must not enter the manifest")
}
// Manifest recompute must match byte-exact (stable ordering + delimiter).
if got := ComputeReleaseChangeSetManifest(detail.Items); got != first.ManifestSHA256 {
t.Fatalf("manifest not reproducible: %s vs %s", got, first.ManifestSHA256)
}
// Source mutates AFTER freezing: dropping an item that already exists in
// production turns its live-plan row into delete — but the frozen manifest
// must not move (plan changes, set does not).
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='deleted' WHERE id='item-unch'`); err != nil {
t.Fatal(err)
}
after, err := store.GetReleaseChangeSet(ctx, first.ID)
if err != nil {
t.Fatal(err)
}
if after.ManifestSHA256 != first.ManifestSHA256 || len(after.Items) != len(detail.Items) {
t.Fatal("frozen change set mutated after source change")
}
// Fresh live plan reflects the mutation (proof the set is NOT live-derived).
mutated, err := store.ListReleasePlan(ctx)
if err != nil {
t.Fatal(err)
}
mutationSeen := false
for _, item := range mutated.Items {
if item.PublicationID == "tp-unch" && item.Action == "delete" {
mutationSeen = true
}
}
if !mutationSeen {
t.Fatal("live plan failed to reflect post-freeze source change")
}
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='item-unch'`); err != nil {
t.Fatal(err)
}
// Dedupe on the SAME plan: repeated generation reuses the latest PLANNED
// set (concurrent clicks must not pile up versions).
deduped, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "重复", CreatedBy: "yaozhanghao", Now: now.Add(time.Minute)})
if err != nil {
t.Fatal(err)
}
if deduped.ID != first.ID || deduped.Number != 1 {
t.Fatalf("dedupe failed: %#v vs %#v", deduped, first)
}
// Batching BEFORE confirmation is rejected with the honest reason.
_, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
Name: "太早了", CreatedBy: "yaozhanghao",
PublicationIDs: []string{"tp-keep"},
ReleaseChangeSetID: first.ID, Now: now,
})
if !errors.Is(err, ErrReleaseChangeSetUnconfirmed) {
t.Fatalf("unconfirmed batching err = %v", err)
}
// Confirm (idempotent replay on re-confirm).
confirmed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "boss", Now: now})
if err != nil || confirmed.State != "confirmed" || confirmed.ConfirmedBy != "boss" {
t.Fatalf("confirm = %#v err = %v", confirmed, err)
}
replayed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "someone-else", Now: now.Add(time.Minute)})
if err != nil || replayed.ConfirmedBy != "boss" {
t.Fatalf("re-confirm must replay stored confirmation: %#v err=%v", replayed, err)
}
// A new generation after confirmation takes the next version number — the
// confirmed set stays untouched forever (page refresh cannot alter it).
second, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布v2", CreatedBy: "yaozhanghao", Now: now.Add(2 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if second.Number != 2 || second.ID == first.ID {
t.Fatalf("post-confirm generation = %#v", second)
}
stillFirst, err := store.GetReleaseChangeSet(ctx, first.ID)
if err != nil || stillFirst.ManifestSHA256 != first.ManifestSHA256 || stillFirst.State != "confirmed" {
t.Fatalf("confirmed set mutated: %#v err=%v", stillFirst.ReleaseChangeSet, err)
}
// The formal batch citing the CONFIRMED set succeeds and keeps its binding.
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
Name: "正式批次", CreatedBy: "yaozhanghao",
PublicationIDs: []string{"tp-keep"},
ReleaseChangeSetID: first.ID, Now: now,
})
if err != nil {
t.Fatalf("bound batch: %v", err)
}
if batch.ReleaseChangeSetID != first.ID {
t.Fatalf("batch lost its binding: %#v", batch)
}
// A batch citing items NOT in the confirmed manifest is rejected.
_, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
Name: "越界批次", CreatedBy: "yaozhanghao",
PublicationIDs: []string{"tp-block"},
ReleaseChangeSetID: first.ID, Now: now,
})
if !errors.Is(err, ErrReleaseBatchNotEligible) && !errors.Is(err, ErrReleaseChangeSetUnconfirmed) {
t.Fatalf("out-of-manifest batch err = %v", err)
}
// Concurrency: 8 parallel generations on the SAME plan produce exactly one
// new version (dedupe) or all fail on the gate — never corrupt duplicates.
if _, err := pool.Exec(ctx, `DELETE FROM release_batch_items WHERE release_batch_id IN (SELECT id FROM release_batches WHERE release_change_set_id IS NOT NULL);
DELETE FROM release_batches WHERE release_change_set_id IS NOT NULL;
DELETE FROM release_change_sets`); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
ids := make([]string, 8)
errs := make([]error, 8)
for i := 0; i < 8; i++ {
wg.Add(1)
go func(index int) {
defer wg.Done()
set, genErr := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "并发", CreatedBy: "yaozhanghao", Now: now})
ids[index], errs[index] = set.ID, genErr
}(i)
}
wg.Wait()
unique := map[string]bool{}
for index, genErr := range errs {
if genErr != nil {
t.Fatalf("concurrent generate[%d]: %v", index, genErr)
}
unique[ids[index]] = true
}
var rowCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&rowCount); err != nil {
t.Fatal(err)
}
if rowCount != len(unique) {
t.Fatalf("concurrent generation produced %d rows but %d distinct ids", rowCount, len(unique))
}
if len(unique) > 2 {
t.Fatalf("dedupe collapsed under concurrency: %d distinct sets", len(unique))
}
// The concurrent winner set is a valid, stable, hex manifest.
var winnerID, manifest string
if err := pool.QueryRow(ctx, `SELECT id, manifest_sha256 FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&winnerID, &manifest); err != nil {
t.Fatal(err)
}
if len(manifest) != 64 || strings.TrimLeft(manifest, "0123456789abcdef") != "" {
t.Fatalf("manifest shape broken: %q", manifest)
}
winner, err := store.GetReleaseChangeSet(ctx, winnerID)
if err != nil || len(winner.Items) == 0 {
t.Fatalf("winner detail = %#v err=%v", winner, err)
}
if ComputeReleaseChangeSetManifest(winner.Items) != manifest {
t.Fatal("stored manifest does not recompute from stored items")
}
// Page listing.
page, err := store.ListReleaseChangeSets(ctx, governance.ReleaseBatchFilter{Limit: 10})
if err != nil || page.Total < 1 {
t.Fatalf("page = %#v err = %v", page, err)
}
if _, err := store.LatestReleaseChangeSet(ctx); err != nil {
t.Fatalf("latest: %v", err)
}
if _, err := store.GetReleaseChangeSet(ctx, "rcs-missing"); !errors.Is(err, ErrReleaseChangeSetNotFound) {
t.Fatalf("missing detail err = %v", err)
}
}
@@ -0,0 +1,236 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
// TestR4RollbackLevels freezes docs plan R4: publish failures grade into
// single_item/batch, rollbacks walk rolling_back -> rolled_back /
// rollback_failed, a failed rollback escalates the prior publish to overall,
// and an overall-graded publish can no longer be compensated by plain
// rollback authorization (snapshot restore is R2 territory and must be gated).
func TestR4RollbackLevels(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC)
// The shared review fixture binds to conn-manual and packs its URI hash via
// repeat(<hex>, 64), so indexes must stay single-hex-digit.
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
t.Fatal(err)
}
refs := []string{"tp-r4-a", "tp-r4-b"}
for index, publicationID := range refs {
insertReviewFixture(t, ctx, pool, index+1, publicationID, now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
t.Fatal(err)
}
}
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次", CreatedBy: "engineer-1", PublicationIDs: refs, Now: now.Add(2 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
t.Fatal(err)
}
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
if err != nil {
t.Fatal(err)
}
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
if err != nil || len(plan) != 2 {
t.Fatalf("publish plan=%#v err=%v", plan, err)
}
// ---- single_item grade: first item fails permanently, nothing verified.
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
// Second item also fails so FinishExecution can settle the execution.
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
var level string
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil {
t.Fatal(err)
}
if level != "single_item" {
t.Fatalf("zero-progress failure must grade single_item, got %q", level)
}
// Nothing was ever verified: auto rollback is a no-op (idempotent, no rows).
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil {
t.Fatal(err)
}
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil {
t.Fatal(err)
}
var rollbackRows int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_executions WHERE prior_execution_id=$1`, publish.ID).Scan(&rollbackRows); err != nil {
t.Fatal(err)
}
if rollbackRows != 0 {
t.Fatalf("single_item with zero verified must schedule no rollback, got %d", rollbackRows)
}
// ---- batch grade: reset items, retry, verify one, fail the other.
// (The job is still pending in this store-level fixture; a real worker run
// would have parked it failed. Retry requires a terminal job, so we set it.)
if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, publish.JobID); err != nil {
t.Fatal(err)
}
retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: publish.ID, Now: now.Add(10 * time.Minute)})
if err != nil || retried.State != "pending" {
t.Fatalf("retry publish = %#v err=%v", retried, err)
}
plan, err = store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(11 * time.Minute)})
if err != nil || len(plan) != 2 {
t.Fatalf("retry plan=%#v err=%v", plan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(12 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(12 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, publish.ID, now.Add(13*time.Minute)); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil {
t.Fatal(err)
}
if level != "batch" {
t.Fatalf("partial-progress failure must grade batch, got %q", level)
}
// ---- batch rollback: rolling_back -> rolled_back with restart re-claim.
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(14*time.Minute)); err != nil {
t.Fatal(err)
}
rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10})
if err != nil || rollbacks.Total != 1 {
t.Fatalf("rollbacks=%#v err=%v", rollbacks, err)
}
rollback := rollbacks.Items[0]
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(15 * time.Minute)})
if err != nil || len(rollbackPlan) != 1 {
t.Fatalf("rollback plan=%#v err=%v", rollbackPlan, err)
}
// R4: a claimed rollback runs in the dedicated rolling_back stage.
var stage string
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil {
t.Fatal(err)
}
if stage != "rolling_back" {
t.Fatalf("rollback claim must enter rolling_back, got %q", stage)
}
// Worker restart mid-rollback: the same execution is re-claimable.
if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)}); err != nil {
t.Fatalf("re-claim after restart: %v", err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: rollbackPlan[0].PublicationID, BundleSHA256: "", PreviousProductionRevision: "prod-before", ProductionRevision: "restored-v0", Now: now.Add(17 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil {
t.Fatal(err)
}
if stage != "rolled_back" {
t.Fatalf("successful compensation must land rolled_back, got %q", stage)
}
// ---- second batch with another verified item so we can fail the rollback.
refs2 := []string{"tp-r4-b"}
batch2, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次2", CreatedBy: "engineer-1", PublicationIDs: refs2, DeletePublicationIDs: []string{}, Now: now.Add(19 * time.Minute)})
if err == nil {
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch2.ID, SealedBy: "engineer-2", Now: now.Add(20 * time.Minute)}); err != nil {
t.Fatalf("seal batch2: %v", err)
}
publish2, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(21 * time.Minute)})
if err != nil {
t.Fatalf("authorize batch2: %v", err)
}
plan2, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish2.ID, JobID: publish2.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(22 * time.Minute)})
if err != nil || len(plan2) != 1 {
t.Fatalf("publish2 plan=%#v err=%v", plan2, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish2.ID, PublicationID: plan2[0].PublicationID, BundleSHA256: strings.Repeat("c", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan2[0].ReleaseRevision, Now: now.Add(23 * time.Minute)}); err != nil {
t.Fatal(err)
}
// Finish publish2 successfully so its item stays verified.
if err := store.FinishExecution(ctx, publish2.ID, now.Add(24*time.Minute)); err != nil {
t.Fatal(err)
}
// Authorize a rollback for the completed publish2 and make it fail.
manual, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(25 * time.Minute)})
if err != nil {
t.Fatalf("authorize rollback: %v", err)
}
manualPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: manual.ID, JobID: manual.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(26 * time.Minute)})
_ = manualPlan
if err != nil {
t.Fatalf("start manual rollback: %v", err)
}
// The compensated publication now has a newer publication row; fail the
// rollback item permanently.
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: manual.ID, PublicationID: plan2[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(27 * time.Minute)}); err != nil {
t.Fatalf("fail rollback item: %v", err)
}
if err := store.FinishExecution(ctx, manual.ID, now.Add(28*time.Minute)); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, manual.ID).Scan(&stage); err != nil {
t.Fatal(err)
}
if stage != "rollback_failed" {
t.Fatalf("failed rollback must land rollback_failed, got %q", stage)
}
// Escalation: the prior publish is now 'overall' (needs snapshot restore).
if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish2.ID).Scan(&level); err != nil {
t.Fatal(err)
}
if level != "overall" {
t.Fatalf("failed rollback must escalate prior publish to overall, got %q", level)
}
// Overall-graded publish can no longer be plain-rollback authorized.
if _, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(29 * time.Minute)}); err == nil {
t.Fatal("overall-graded publish must not accept plain rollback authorization")
}
// A permanently failed rollback stays retriable (re-runs compensation).
if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, manual.JobID); err != nil {
t.Fatal(err)
}
retriedRollback, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: manual.ID, Now: now.Add(30 * time.Minute)})
if err != nil || retriedRollback.State != "pending" {
t.Fatalf("retry failed rollback = %#v err=%v", retriedRollback, err)
}
} else {
// Batch creation legitimately refused only when tp-r4-b has no
// publish-eligible state; in this fixture it should succeed.
t.Fatalf("batch2 create: %v", err)
}
}
@@ -0,0 +1,123 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
// TestR5ExecutionManifestSealAndMismatch freezes docs plan R5: finishing an
// execution seals a deterministic manifest of what actually happened, and a
// "success" that does not match the frozen release change set intent is
// downgraded to failed — never reported as completed.
func TestR5ExecutionManifestSealAndMismatch(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, time.September, 12, 13, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual')`); err != nil {
t.Fatal(err)
}
insertReviewFixture(t, ctx, pool, 1, "tp-r5-a", now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
t.Fatal(err)
}
runExecution := func(publicationID, tag string) (executionID, jobID string) {
t.Helper()
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R5批次-" + tag, CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
t.Fatal(err)
}
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
if err != nil {
t.Fatal(err)
}
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
if err != nil || len(plan) != 1 {
t.Fatalf("plan=%#v err=%v", plan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("d", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
return publish.ID, publish.JobID
}
// Happy path (unbound batch): manifest is sealed, deterministic, non-empty.
executionID, _ := runExecution("tp-r5-a", "happy")
if err := store.FinishExecution(ctx, executionID, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
var sealed, state, publicationID, bundleSHA, revision string
if err := pool.QueryRow(ctx, `SELECT e.manifest_sha256, e.state, i.publication_id, i.bundle_sha256, i.production_revision
FROM release_executions e JOIN release_item_executions i ON i.release_execution_id=e.id WHERE e.id=$1`, executionID).Scan(&sealed, &state, &publicationID, &bundleSHA, &revision); err != nil {
t.Fatal(err)
}
if state != "completed" {
t.Fatalf("unbound success must stay completed, got %q", state)
}
if len(sealed) != 64 || sealed != ComputeReleaseExecutionManifest("publish", "", []string{publicationID + "|" + bundleSHA + "|" + revision}) {
t.Fatalf("manifest not sealed/deterministic: %q", sealed)
}
// Re-seal must be a no-op (immutable once set).
if err := store.FinishExecution(ctx, executionID, now.Add(8*time.Minute)); err == nil {
t.Fatal("settled execution must not finish twice")
}
// Mismatch path: simulate a tampered/legacy row where the bound frozen
// change set does NOT contain the publication that was "successfully"
// released — R5 forbids reporting success.
insertReviewFixture(t, ctx, pool, 2, "tp-r5-b", now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-b", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
t.Fatal(err)
}
secondExecution, jobID := runExecution("tp-r5-b", "mismatch")
executionID = secondExecution
if _, err := pool.Exec(ctx, `INSERT INTO release_change_sets(id,name,number,state,manifest_sha256,created_by,acceptance_cutoff) VALUES ('rcs-r5-x','不相干计划集',99,'confirmed',repeat('a',64),'tamper',$1)`, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES ('rcs-r5-x',1,'tp-other','别人','internal','add','ready','absent',true,'')`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE release_executions SET release_change_set_id='rcs-r5-x' WHERE id=$1`, executionID); err != nil {
t.Fatal(err)
}
// Reset to running so FinishExecution can settle again for this scenario.
if _, err := pool.Exec(ctx, `UPDATE release_executions SET state='running', manifest_sha256='', completed_at=NULL WHERE id=$1`, executionID); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, executionID, now.Add(9*time.Minute)); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT state, error_code, manifest_sha256 FROM release_executions WHERE id=$1`, executionID).Scan(&state, &bundleSHA, &sealed); err != nil {
t.Fatal(err)
}
if state != "failed" || bundleSHA != "release_manifest_mismatch" {
t.Fatalf("mismatch must downgrade to failed, got state=%q code=%q", state, bundleSHA)
}
if len(sealed) != 64 {
t.Fatalf("mismatched execution must still seal its actual manifest, got %q", sealed)
}
_ = jobID
}
@@ -0,0 +1,144 @@
package postgres
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
// TestR6ReleaseAuditChain freezes docs plan R6: one execution traces through
// real FK rows to batch -> confirmed release change set -> bound snapshots and
// catalog change sets -> per-item review/acceptance/source identity, and the
// linked evidence cannot be deleted underneath the chain.
func TestR6ReleaseAuditChain(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, time.September, 12, 11, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES
('snap-r6','conn-manual',repeat('e',64),'R6快照',3,2,1,now() - interval '2 hours');
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES
('conn-manual','snap-r6',now() - interval '2 hours');
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,confirmed_at) VALUES
('cs-r6','conn-manual',NULL,'snap-r6','confirmed',1,0,0,0,now() - interval '90 minutes');`); err != nil {
t.Fatal(err)
}
insertReviewFixture(t, ctx, pool, 1, "tp-r6-a", now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r6-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
t.Fatal(err)
}
set, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "R6发布", CreatedBy: "yaozhanghao", Now: now})
if err != nil {
t.Fatal(err)
}
if _, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: set.ID, ConfirmedBy: "boss", Now: now}); err != nil {
t.Fatal(err)
}
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R6批次", CreatedBy: "yaozhanghao", PublicationIDs: []string{"tp-r6-a"}, ReleaseChangeSetID: set.ID, Now: now.Add(time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(2 * time.Minute)}); err != nil {
t.Fatal(err)
}
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(3 * time.Minute)})
if err != nil {
t.Fatal(err)
}
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(4 * time.Minute)})
if err != nil || len(plan) != 1 {
t.Fatalf("plan=%#v err=%v", plan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(5 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, publish.ID, now.Add(6*time.Minute)); err != nil {
t.Fatal(err)
}
chain, err := store.GetReleaseAuditChain(ctx, publish.ID)
if err != nil {
t.Fatal(err)
}
if chain.ExecutionID != publish.ID || chain.Operation != "publish" || chain.State != "completed" {
t.Fatalf("head = %#v", chain)
}
if chain.BatchID != batch.ID || chain.BatchName != "R6批次" || chain.CreatedBy != "yaozhanghao" || chain.SealedBy != "engineer-2" || chain.AuthorizedBy != "release-owner" {
t.Fatalf("batch links = %#v", chain)
}
if chain.ReleaseChangeSetID != set.ID || chain.ChangeSetName != "R6发布" || chain.ManifestSHA256 != set.ManifestSHA256 || chain.ChangeSetConfirmedBy != "boss" {
t.Fatalf("change set links = %#v", chain)
}
if len(chain.Snapshots) != 1 || chain.Snapshots[0].ID != "snap-r6" || chain.Snapshots[0].SourceName != "Manual" || chain.Snapshots[0].FileCount != 1 {
t.Fatalf("snapshot links = %#v", chain.Snapshots)
}
if len(chain.CatalogChangeSets) != 1 || chain.CatalogChangeSets[0].ID != "cs-r6" || chain.CatalogChangeSets[0].State != "confirmed" {
t.Fatalf("catalog links = %#v", chain.CatalogChangeSets)
}
if len(chain.ManifestSHA256) != 64 {
t.Fatalf("manifest = %q", chain.ManifestSHA256)
}
if len(chain.Items) != 1 {
t.Fatalf("items = %#v", chain.Items)
}
item := chain.Items[0]
if item.PublicationID != "tp-r6-a" || item.Action != "upsert" || item.ItemState != "verified" || item.ProductionRevision != plan[0].ReleaseRevision {
t.Fatalf("item = %#v", item)
}
if item.ReviewID == "" || item.AcceptanceRunID == "" || item.SourceItemID == "" || item.SourceVersionID == "" {
t.Fatalf("item evidence missing: %#v", item)
}
// R6 integrity: FK rows protect the evidence — the bound snapshot and the
// confirmed release change set cannot be deleted while the chain exists.
if _, err := pool.Exec(ctx, `DELETE FROM source_catalog_snapshots WHERE id='snap-r6'`); err == nil {
t.Fatal("snapshot delete must be RESTRICTed by the audit chain")
}
if _, err := pool.Exec(ctx, `DELETE FROM release_change_sets WHERE id=$1`, set.ID); err == nil {
t.Fatal("release change set delete must be RESTRICTed by batch/execution bindings")
}
// The catalog chain row links are real FK rows, not just JSONB.
var linkRows int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_set_snapshot_links WHERE change_set_id=$1`, set.ID).Scan(&linkRows); err != nil || linkRows != 1 {
t.Fatalf("snapshot link rows = %d err=%v", linkRows, err)
}
// Rollback chain: authorize a rollback of the completed publish and see it
// appear on the original execution's projection.
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(7 * time.Minute)})
if err != nil {
t.Fatal(err)
}
publishChain, err := store.GetReleaseAuditChain(ctx, publish.ID)
if err != nil || publishChain.RollbackExecutionID != rollback.ID {
t.Fatalf("publish chain rollback = %#v err=%v", publishChain, err)
}
rollbackChain, err := store.GetReleaseAuditChain(ctx, rollback.ID)
if err != nil || rollbackChain.PriorExecutionID != publish.ID || rollbackChain.ReleaseChangeSetID != set.ID {
t.Fatalf("rollback chain = %#v err=%v", rollbackChain, err)
}
if _, err := store.GetReleaseAuditChain(ctx, "rx-unknown"); err != ErrReleaseAuditChainNotFound {
t.Fatalf("unknown chain err = %v", err)
}
}
@@ -0,0 +1,155 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// ErrReleaseAuditChainNotFound is returned for an unknown execution id.
var ErrReleaseAuditChainNotFound = errors.New("release audit chain not found")
const releaseAuditChainHeadSQL = `
SELECT e.id, e.operation, e.state, e.failure_level,
COALESCE(e.release_batch_id,''), COALESCE(b.name,''), COALESCE(b.created_by,''), COALESCE(b.sealed_by,''),
COALESCE(e.authorized_by,''), COALESCE(e.release_change_set_id::text,''),
COALESCE(rcs.name,''), COALESCE(rcs.number,0), COALESCE(rcs.state,''),
COALESCE(rcs.created_by,''), COALESCE(rcs.confirmed_by,''), rcs.confirmed_at,
COALESCE(rcs.manifest_sha256,''), COALESCE(e.prior_execution_id,''),
COALESCE((SELECT rb.id FROM release_executions AS rb
WHERE rb.prior_execution_id = e.id AND rb.operation = 'rollback'
ORDER BY rb.created_at DESC LIMIT 1),'')
FROM release_executions AS e
LEFT JOIN release_batches AS b ON b.id = e.release_batch_id
LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id
WHERE e.id = $1
`
const releaseAuditSnapshotLinksSQL = `
SELECT snap.id, COALESCE(conn.name,''), snap.node_count, snap.file_count, snap.observed_at
FROM release_change_set_snapshot_links AS link
JOIN source_catalog_snapshots AS snap ON snap.id = link.snapshot_id
LEFT JOIN source_connections AS conn ON conn.id = snap.source_connection_id
WHERE link.change_set_id = $1
ORDER BY snap.id
`
const releaseAuditCatalogLinksSQL = `
SELECT cset.id, cset.state, cset.add_count, cset.update_count,
cset.move_count, cset.delete_count
FROM release_change_set_catalog_links AS link
JOIN source_catalog_change_sets AS cset ON cset.id = link.catalog_change_set_id
WHERE link.change_set_id = $1
ORDER BY cset.id
`
const releaseAuditItemsSQL = `
SELECT COALESCE(batch_item.publication_id, item.publication_id),
COALESCE(version_title.title, ''),
COALESCE(item.action, batch_item.action),
COALESCE(item.state, 'pending'),
COALESCE(batch_item.review_id, ''),
COALESCE(batch_item.acceptance_run_id, ''),
COALESCE(version.source_item_id, ''),
COALESCE(batch_item.source_item_version_id, ''),
COALESCE(item.production_revision, ''),
COALESCE(item.error_code, '')
FROM release_item_executions AS item
JOIN test_publications AS publication ON publication.id = item.publication_id
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
JOIN source_items AS version_title ON version_title.id = version.source_item_id
LEFT JOIN release_batch_items AS batch_item
ON batch_item.release_batch_id = $2 AND batch_item.publication_id = item.publication_id
WHERE item.release_execution_id = $1
ORDER BY item.id
`
// GetReleaseAuditChain projects the complete R6 evidence chain for one
// execution: execution -> batch -> release change set -> bound source
// snapshots and catalog change sets -> per-item review/acceptance/source
// identity, plus the compensating rollback row when one exists. Every link is
// read through real foreign keys (migration 30), not denormalized JSON. Only
// safe identifiers, labels, counts and timestamps are selected — never bundle
// paths, credentials, base URLs, or signed object URLs.
func (s *Store) GetReleaseAuditChain(ctx context.Context, executionID string) (governance.ReleaseAuditChain, error) {
if !validShortID(executionID) {
return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound
}
var chain governance.ReleaseAuditChain
err := s.db.QueryRow(ctx, releaseAuditChainHeadSQL, executionID).Scan(
&chain.ExecutionID, &chain.Operation, &chain.State, &chain.FailureLevel,
&chain.BatchID, &chain.BatchName, &chain.CreatedBy, &chain.SealedBy,
&chain.AuthorizedBy, &chain.ReleaseChangeSetID,
&chain.ChangeSetName, &chain.ChangeSetNumber, &chain.ChangeSetState,
&chain.ChangeSetCreatedBy, &chain.ChangeSetConfirmedBy, &chain.ChangeSetConfirmedAt,
&chain.ManifestSHA256, &chain.PriorExecutionID,
&chain.RollbackExecutionID,
)
if errors.Is(err, pgx.ErrNoRows) {
return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound
}
if err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("load release audit chain: %w", err)
}
chain.Snapshots = []governance.AuditSnapshot{}
chain.CatalogChangeSets = []governance.AuditCatalogSet{}
chain.Items = []governance.AuditChainItem{}
if chain.ReleaseChangeSetID != "" {
rows, err := s.db.Query(ctx, releaseAuditSnapshotLinksSQL, chain.ReleaseChangeSetID)
if err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("list release snapshot links: %w", err)
}
for rows.Next() {
var snap governance.AuditSnapshot
if err := rows.Scan(&snap.ID, &snap.SourceName, &snap.NodeCount, &snap.FileCount, &snap.ObservedAt); err != nil {
rows.Close()
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release snapshot link: %w", err)
}
chain.Snapshots = append(chain.Snapshots, snap)
}
if err := rows.Err(); err != nil {
rows.Close()
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release snapshot links: %w", err)
}
rows.Close()
rows, err = s.db.Query(ctx, releaseAuditCatalogLinksSQL, chain.ReleaseChangeSetID)
if err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("list release catalog links: %w", err)
}
for rows.Next() {
var set governance.AuditCatalogSet
if err := rows.Scan(&set.ID, &set.State, &set.AddCount, &set.UpdateCount, &set.MoveCount, &set.DeleteCount); err != nil {
rows.Close()
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release catalog link: %w", err)
}
chain.CatalogChangeSets = append(chain.CatalogChangeSets, set)
}
if err := rows.Err(); err != nil {
rows.Close()
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release catalog links: %w", err)
}
rows.Close()
}
rows, err := s.db.Query(ctx, releaseAuditItemsSQL, executionID, chain.BatchID)
if err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("list release audit items: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.AuditChainItem
if err := rows.Scan(&item.PublicationID, &item.Title, &item.Action, &item.ItemState,
&item.ReviewID, &item.AcceptanceRunID, &item.SourceItemID, &item.SourceVersionID,
&item.ProductionRevision, &item.ErrorCode); err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("scan release audit item: %w", err)
}
chain.Items = append(chain.Items, item)
}
if err := rows.Err(); err != nil {
return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release audit items: %w", err)
}
return chain, nil
}
@@ -0,0 +1,258 @@
package postgres
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
func TestReleaseAuthorizationPersistentIntegration(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
t.Fatal(err)
}
insertReviewFixture(t, ctx, pool, 10, "tp-auth", now)
review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{
PublicationID: "tp-auth", Decision: governance.ReviewApproved,
DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1",
DecisionCode: "approved_for_release", Now: now.Add(time.Minute),
})
if err != nil || review.ID == "" {
t.Fatalf("review=%#v error=%v", review, err)
}
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
Name: "Authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auth"}, Now: now.Add(2 * time.Minute),
})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
t.Fatal(err)
}
request := governance.AuthorizeReleaseBatchRequest{
BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute),
}
first, err := store.AuthorizeReleaseBatch(ctx, request)
if err != nil {
t.Fatal(err)
}
second, err := store.AuthorizeReleaseBatch(ctx, request)
if err != nil {
t.Fatal(err)
}
if first.ID != second.ID || first.JobID != second.JobID || first.ItemCount != 1 {
t.Fatalf("first=%#v second=%#v", first, second)
}
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{
ExecutionID: first.ID, JobID: first.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute),
})
if err != nil || len(plan) != 1 || plan[0].PublicationID != "tp-auth" {
t.Fatalf("publish plan=%#v error=%v", plan, err)
}
if plan[0].ReleaseExecutionID != first.ID || plan[0].SourceRevision != "revision-10" || plan[0].ParseRevision != "parser-v1@index-v1" || plan[0].ReleaseRevision != review.ID {
t.Fatalf("publish revisions=%#v", plan[0])
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{
ExecutionID: first.ID, PublicationID: "tp-auth", BundleSHA256: strings.Repeat("a", 64),
PreviousProductionRevision: "prod-v0", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute),
}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, first.ID, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
currentPlan, err := store.ListReleasePlan(ctx)
if err != nil || len(currentPlan.Items) != 0 {
t.Fatalf("current release plan=%#v error=%v", currentPlan, err)
}
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-10'`); err != nil {
t.Fatal(err)
}
unavailablePlan, err := store.ListReleasePlan(ctx)
if err != nil || len(unavailablePlan.Items) != 1 || unavailablePlan.Items[0].Action != "blocked" || unavailablePlan.Items[0].CanRelease || unavailablePlan.Items[0].Blocker != "source_unavailable" {
t.Fatalf("unavailable release plan=%#v error=%v", unavailablePlan, err)
}
if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-10'`); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-10','blocked','engineer-1','禁止进入生产',$1,$1)`, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
deletePlan, err := store.ListReleasePlan(ctx)
if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != "tp-auth" || deletePlan.Items[0].Blocker != "" {
t.Fatalf("delete release plan=%#v error=%v", deletePlan, err)
}
rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{
PriorExecutionID: first.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(8 * time.Minute),
})
if err != nil {
t.Fatal(err)
}
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{
ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(9 * time.Minute),
})
if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PreviousProductionRevision != "prod-v0" {
t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{
ExecutionID: rollback.ID, PublicationID: "tp-auth", ProductionRevision: "prod-v0", Now: now.Add(10 * time.Minute),
}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, rollback.ID, now.Add(11*time.Minute)); err != nil {
t.Fatal(err)
}
for _, mutation := range []struct {
sql string
args []any
}{
{`UPDATE release_item_executions SET state='failed', error_code='release_verification_failed', completed_at=$2 WHERE release_execution_id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}},
{`UPDATE release_executions SET state='failed', completed_item_count=0, failed_item_count=1, error_code='release_item_failed', completed_at=$2 WHERE id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}},
{`UPDATE jobs SET state='completed', attempt=3, completed_at=$2 WHERE id=$1`, []any{rollback.JobID, now.Add(12 * time.Minute)}},
} {
if _, err := pool.Exec(ctx, mutation.sql, mutation.args...); err != nil {
t.Fatal(err)
}
}
retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: rollback.ID, Now: now.Add(13 * time.Minute)})
if err != nil || retried.State != governance.ReleaseExecutionPending || retried.FailedItemCount != 0 {
t.Fatalf("retried=%#v error=%v", retried, err)
}
var executionState, itemState, jobState string
var attempt int
if err := pool.QueryRow(ctx, `SELECT execution.state, item.state, job.state, job.attempt FROM release_executions execution JOIN release_item_executions item ON item.release_execution_id=execution.id JOIN jobs job ON job.id=execution.job_id WHERE execution.id=$1`, rollback.ID).Scan(&executionState, &itemState, &jobState, &attempt); err != nil {
t.Fatal(err)
}
if executionState != "pending" || itemState != "pending" || jobState != "pending" || attempt != 0 {
t.Fatalf("states=%s/%s/%s attempt=%d", executionState, itemState, jobState, attempt)
}
insertReviewFixture(t, ctx, pool, 11, "tp-stale-auth", now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{
PublicationID: "tp-stale-auth", Decision: governance.ReviewApproved,
DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1",
DecisionCode: "approved_for_release", Now: now.Add(5 * time.Minute),
}); err != nil {
t.Fatal(err)
}
staleBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{
Name: "Stale authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-stale-auth"}, Now: now.Add(6 * time.Minute),
})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: staleBatch.ID, SealedBy: "engineer-2", Now: now.Add(7 * time.Minute)}); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE test_publications SET updated_at = updated_at + INTERVAL '1 second' WHERE id = 'tp-stale-auth'`); err != nil {
t.Fatal(err)
}
_, err = store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{
BatchID: staleBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(8 * time.Minute),
})
if !errors.Is(err, ErrReleaseAuthorizationGateConflict) {
t.Fatalf("stale authorization error=%v", err)
}
var executions, jobs, itemExecutions, completed, rolledBack, pending int
if err := pool.QueryRow(ctx, `SELECT
(SELECT COUNT(*)::INTEGER FROM release_executions),
(SELECT COUNT(*)::INTEGER FROM jobs WHERE type = 'release.publish'),
(SELECT COUNT(*)::INTEGER FROM release_item_executions),
(SELECT COUNT(*)::INTEGER FROM release_executions WHERE state = 'completed'),
(SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'rolled_back'),
(SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'pending')`).Scan(&executions, &jobs, &itemExecutions, &completed, &rolledBack, &pending); err != nil {
t.Fatal(err)
}
if executions != 2 || jobs != 1 || itemExecutions != 2 || completed != 1 || rolledBack != 0 || pending != 1 {
t.Fatalf("executions=%d publishJobs=%d itemExecutions=%d completed=%d rolledBack=%d pending=%d", executions, jobs, itemExecutions, completed, rolledBack, pending)
}
}
func TestFailedPublishSchedulesIdempotentRollbackForVerifiedItemsOnly(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 10, 3, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil {
t.Fatal(err)
}
for index, publicationID := range []string{"tp-auto-a", "tp-auto-b"} {
insertReviewFixture(t, ctx, pool, 8+index, publicationID, now)
if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil {
t.Fatal(err)
}
}
batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Automatic rollback batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auto-a", "tp-auto-b"}, Now: now.Add(2 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil {
t.Fatal(err)
}
publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)})
if err != nil {
t.Fatal(err)
}
plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)})
if err != nil || len(plan) != 2 {
t.Fatalf("publish plan=%#v error=%v", plan, err)
}
if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil {
t.Fatal(err)
}
if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil {
t.Fatal(err)
}
if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil {
t.Fatal(err)
}
rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10})
if err != nil || rollbacks.Total != 1 || len(rollbacks.Items) != 1 || rollbacks.Items[0].ItemCount != 1 || rollbacks.Items[0].AuthorizationCode != "automatic_rollback_after_failure" {
t.Fatalf("rollbacks=%#v error=%v", rollbacks, err)
}
rollback := rollbacks.Items[0]
rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(10 * time.Minute)})
if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PublicationID != plan[0].PublicationID || rollbackPlan[0].PreviousProductionRevision != "prod-before" {
t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err)
}
}
@@ -0,0 +1,450 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"knowledge-governance/internal/governance"
)
const releaseGatePolicyVersion = "release-gate-v1"
var (
ErrInvalidReleaseBatch = errors.New("invalid release batch")
ErrInvalidReleaseBatchFilter = errors.New("invalid release batch filter")
ErrReleaseBatchNotEligible = errors.New("release batch contains ineligible publications")
ErrReleaseBatchGateConflict = errors.New("release batch gate conflict")
ErrReleaseBatchNotFound = errors.New("release batch not found")
ErrReleaseChangeSetUnconfirmed = errors.New("release change set must be confirmed before batching")
)
const releaseGateCTEs = `
acceptance_run_summary AS (
SELECT publication_id, run_id, MAX(observed_at) AS observed_at,
COALESCE(MAX(state) FILTER (WHERE check_type = 'knowledge_exists'), '') AS knowledge_exists,
COALESCE(MAX(state) FILTER (WHERE check_type = 'chunks_present'), '') AS chunks_present,
COALESCE(MAX(state) FILTER (WHERE check_type = 'retrieval_hit'), '') AS retrieval_hit
FROM acceptance_results
GROUP BY publication_id, run_id
), latest_acceptance AS (
SELECT DISTINCT ON (publication_id)
publication_id, run_id, knowledge_exists, chunks_present, retrieval_hit, observed_at
FROM acceptance_run_summary
ORDER BY publication_id, observed_at DESC, run_id DESC
), latest_review AS (
SELECT DISTINCT ON (publication_id)
id, publication_id, decision, decided_visibility, acceptance_run_id,
publication_updated_at, created_at
FROM publication_reviews
ORDER BY publication_id, created_at DESC, id DESC
), release_ready AS (
SELECT publication.id AS publication_id,
publication.source_item_version_id,
publication.updated_at AS publication_updated_at,
review.id AS review_id,
review.acceptance_run_id,
review.decided_visibility
FROM test_publications AS publication
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
JOIN source_items AS source ON source.id = version.source_item_id
JOIN latest_acceptance AS acceptance ON acceptance.publication_id = publication.id
JOIN publication_reviews AS review ON review.id = (
SELECT latest_review.id FROM latest_review WHERE latest_review.publication_id = publication.id
)
WHERE review.decision = 'approved'
AND review.acceptance_run_id = acceptance.run_id
AND review.publication_updated_at = publication.updated_at
AND publication.ingest_state = 'uploaded'
AND publication.processing_state = 'completed'
AND publication.retrieval_state = 'unpublished'
AND version.revision = source.revision
AND source.source_change NOT IN ('deleted', 'unavailable')
AND acceptance.knowledge_exists = 'passed'
AND acceptance.chunks_present = 'passed'
AND acceptance.retrieval_hit IN ('passed', 'manual_required')
)
`
const releaseDeleteReadyCTEs = `
source_truth AS (
SELECT source.id, source.source_change,
COALESCE(item_rule.visibility, directory_rule.visibility, source.visibility) AS desired_policy
FROM source_items AS source
LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id
LEFT JOIN LATERAL (
SELECT rule.visibility
FROM directory_scope_rules AS rule
WHERE rule.source_connection_id = source.source_connection_id
AND (rule.directory_path = '' OR source.normalized_path = rule.directory_path OR source.normalized_path LIKE rule.directory_path || '/%')
ORDER BY LENGTH(rule.directory_path) DESC, rule.directory_path DESC
LIMIT 1
) AS directory_rule ON TRUE
), latest_production AS (
SELECT DISTINCT ON (version.source_item_id, publication.target_scope)
publication.id AS publication_id, version.source_item_id, publication.target_scope,
execution_item.production_revision,
batch_item.review_id, batch_item.source_item_version_id,
batch_item.acceptance_run_id, batch_item.publication_updated_at,
batch_item.decided_visibility
FROM release_item_executions AS execution_item
JOIN release_executions AS execution ON execution.id = execution_item.release_execution_id
JOIN test_publications AS publication ON publication.id = execution_item.publication_id
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
JOIN release_batch_items AS batch_item
ON batch_item.release_batch_id = execution.release_batch_id
AND batch_item.publication_id = publication.id
WHERE execution.state IN ('completed', 'rolled_back')
AND execution_item.state IN ('verified', 'rolled_back')
ORDER BY version.source_item_id, publication.target_scope,
execution.completed_at DESC, execution_item.completed_at DESC, execution_item.id DESC
), release_delete_ready AS (
SELECT production.publication_id, production.source_item_version_id,
production.publication_updated_at, production.review_id,
production.acceptance_run_id, production.decided_visibility,
'delete' AS action
FROM latest_production AS production
JOIN source_truth AS truth ON truth.id = production.source_item_id
WHERE production.production_revision <> 'absent'
AND truth.source_change <> 'unavailable'
AND (
truth.source_change = 'deleted'
OR truth.desired_policy = 'blocked'
OR (production.target_scope = 'internal' AND truth.desired_policy NOT IN ('internal', 'both'))
OR (production.target_scope = 'external' AND truth.desired_policy NOT IN ('external', 'both'))
)
), release_upsert_ready AS (
SELECT ready.*
FROM release_ready AS ready
JOIN test_publications AS publication ON publication.id = ready.publication_id
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
JOIN source_truth AS truth ON truth.id = version.source_item_id
LEFT JOIN latest_production AS production
ON production.source_item_id = version.source_item_id
AND production.target_scope = publication.target_scope
WHERE truth.source_change NOT IN ('deleted', 'unavailable')
AND truth.desired_policy <> 'blocked'
AND publication.target_scope = ready.decided_visibility
AND (
(publication.target_scope = 'internal' AND truth.desired_policy IN ('internal', 'both'))
OR (publication.target_scope = 'external' AND truth.desired_policy IN ('external', 'both'))
)
AND (production.production_revision IS NULL OR production.production_revision = 'absent' OR production.production_revision <> ready.review_id)
)
`
const createReleaseBatchSQL = `
WITH requested AS (
SELECT value AS publication_id, 'upsert' AS action
FROM jsonb_array_elements_text($5::jsonb)
UNION ALL
SELECT value AS publication_id, 'delete' AS action
FROM jsonb_array_elements_text($6::jsonb)
), ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
request_stats AS (
SELECT COUNT(*)::INTEGER AS requested_count,
COUNT(DISTINCT publication_id)::INTEGER AS distinct_count
FROM requested
), binding AS (
-- R1: when a batch cites a release change set, the set must be CONFIRMED
-- and every requested publication must exist in its frozen manifest as a
-- releasable item. Source refreshes can never move this goalpost.
SELECT 1
WHERE $7::text = '' OR EXISTS (
SELECT 1 FROM release_change_sets rcs
WHERE rcs.id = $7 AND rcs.state = 'confirmed'
AND NOT EXISTS (
SELECT 1 FROM requested q
WHERE NOT EXISTS (
SELECT 1 FROM release_change_set_items frozen
WHERE frozen.change_set_id = rcs.id
AND frozen.publication_id = q.publication_id
AND frozen.can_release
)
)
)
), eligible AS (
SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at,
ready.review_id, ready.acceptance_run_id, ready.decided_visibility,
requested.action
FROM release_upsert_ready AS ready
JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'upsert'
UNION ALL
SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at,
ready.review_id, ready.acceptance_run_id, ready.decided_visibility,
ready.action
FROM release_delete_ready AS ready
JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'delete'
), created_batch AS (
INSERT INTO release_batches (
id, name, state, policy_version, created_by, item_count, created_at, release_change_set_id
)
SELECT $1, $2, 'draft', 'release-gate-v1', $3, request_stats.requested_count, $4, NULLIF($7::text, '')
FROM request_stats
WHERE EXISTS (SELECT 1 FROM binding)
AND request_stats.requested_count BETWEEN 1 AND 100
AND request_stats.requested_count = request_stats.distinct_count
AND request_stats.requested_count = (SELECT COUNT(*) FROM eligible)
RETURNING id, name, state, policy_version, created_by, sealed_by,
item_count, created_at, sealed_at, release_change_set_id
), inserted_items AS (
INSERT INTO release_batch_items (
release_batch_id, publication_id, review_id, source_item_version_id,
acceptance_run_id, publication_updated_at, decided_visibility, action, created_at
)
SELECT created_batch.id, eligible.publication_id, eligible.review_id,
eligible.source_item_version_id, eligible.acceptance_run_id,
eligible.publication_updated_at, eligible.decided_visibility, eligible.action, $4
FROM created_batch CROSS JOIN eligible
RETURNING release_batch_id
)
SELECT id, name, state, policy_version, created_by, sealed_by,
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
FROM created_batch
WHERE item_count = (SELECT COUNT(*) FROM inserted_items)
`
const sealReleaseBatchSQL = `
WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
item_stats AS (
SELECT batch.id AS batch_id,
COUNT(item.publication_id)::INTEGER AS total_count,
COUNT(*) FILTER (WHERE
(item.action = 'upsert' AND ready.publication_id IS NOT NULL)
OR (item.action = 'delete' AND delete_ready.publication_id IS NOT NULL)
)::INTEGER AS valid_count
FROM release_batches AS batch
LEFT JOIN release_batch_items AS item ON item.release_batch_id = batch.id
LEFT JOIN release_upsert_ready AS ready
ON ready.publication_id = item.publication_id
AND item.action = 'upsert'
AND ready.review_id = item.review_id
AND ready.acceptance_run_id = item.acceptance_run_id
AND ready.publication_updated_at = item.publication_updated_at
AND ready.decided_visibility = item.decided_visibility
LEFT JOIN release_delete_ready AS delete_ready
ON delete_ready.publication_id = item.publication_id
AND item.action = 'delete'
AND delete_ready.review_id = item.review_id
AND delete_ready.acceptance_run_id = item.acceptance_run_id
AND delete_ready.publication_updated_at = item.publication_updated_at
AND delete_ready.decided_visibility = item.decided_visibility
WHERE batch.id = $1 AND batch.state = 'draft'
GROUP BY batch.id
), sealed AS (
UPDATE release_batches AS batch
SET state = 'sealed', sealed_by = $2, sealed_at = $3
FROM item_stats
WHERE batch.id = item_stats.batch_id
AND batch.policy_version = 'release-gate-v1'
AND item_stats.total_count > 0
AND item_stats.total_count = item_stats.valid_count
AND batch.item_count = item_stats.total_count
RETURNING batch.id, batch.name, batch.state, batch.policy_version,
batch.created_by, batch.sealed_by, batch.item_count,
batch.created_at, batch.sealed_at, COALESCE(batch.release_change_set_id::text,'')
), already_sealed AS (
SELECT id, name, state, policy_version, created_by, sealed_by,
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
FROM release_batches
WHERE id = $1 AND state = 'sealed'
)
SELECT * FROM sealed
UNION ALL
SELECT * FROM already_sealed
LIMIT 1
`
const countReleaseBatchesSQL = `
SELECT COUNT(*)::INTEGER
FROM release_batches
WHERE $1 = '' OR state = $1
`
const listReleaseBatchesSQL = `
SELECT id, name, state, policy_version, created_by, sealed_by,
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
FROM release_batches
WHERE $1 = '' OR state = $1
ORDER BY created_at DESC, id DESC
LIMIT $2 OFFSET $3
`
const getReleaseBatchSQL = `
SELECT id, name, state, policy_version, created_by, sealed_by,
item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'')
FROM release_batches
WHERE id = $1
`
const listReleaseBatchItemsSQL = `
SELECT item.publication_id, source.title, item.decided_visibility, item.action
FROM release_batch_items AS item
JOIN test_publications AS publication ON publication.id = item.publication_id
JOIN source_item_versions AS version ON version.id = publication.source_item_version_id
JOIN source_items AS source ON source.id = version.source_item_id
WHERE item.release_batch_id = $1
ORDER BY item.publication_id
`
func (s *Store) CreateReleaseBatch(ctx context.Context, request governance.CreateReleaseBatchRequest) (governance.ReleaseBatch, error) {
if !validCreateReleaseBatch(request) {
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
}
batchID, err := newOpaqueID("rb-")
if err != nil {
return governance.ReleaseBatch{}, err
}
publicationIDs, err := json.Marshal(append([]string{}, request.PublicationIDs...))
if err != nil {
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
}
deletePublicationIDs, err := json.Marshal(append([]string{}, request.DeletePublicationIDs...))
if err != nil {
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
}
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, createReleaseBatchSQL,
batchID, request.Name, request.CreatedBy, request.Now, publicationIDs, deletePublicationIDs,
strings.TrimSpace(request.ReleaseChangeSetID),
))
if errors.Is(err, pgx.ErrNoRows) {
// R1: distinguish an unconfirmed/non-covering change set from plain
// gate ineligibility so operators get the honest reason.
if request.ReleaseChangeSetID != "" {
var state string
if lookup := s.db.QueryRow(ctx, `SELECT state FROM release_change_sets WHERE id=$1`, request.ReleaseChangeSetID).Scan(&state); lookup != nil || state != "confirmed" {
return governance.ReleaseBatch{}, ErrReleaseChangeSetUnconfirmed
}
}
return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible
}
if err != nil {
return governance.ReleaseBatch{}, fmt.Errorf("create release batch: %w", err)
}
return batch, nil
}
func (s *Store) SealReleaseBatch(ctx context.Context, request governance.SealReleaseBatchRequest) (governance.ReleaseBatch, error) {
if !validShortID(request.BatchID) || !validReviewerRef(request.SealedBy) || request.Now.IsZero() {
return governance.ReleaseBatch{}, ErrInvalidReleaseBatch
}
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, sealReleaseBatchSQL, request.BatchID, request.SealedBy, request.Now))
if errors.Is(err, pgx.ErrNoRows) {
return governance.ReleaseBatch{}, ErrReleaseBatchGateConflict
}
if err != nil {
return governance.ReleaseBatch{}, fmt.Errorf("seal release batch: %w", err)
}
return batch, nil
}
func (s *Store) ListReleaseBatches(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseBatchPage, error) {
if filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 ||
filter.State != "" && filter.State != string(governance.ReleaseBatchDraft) &&
filter.State != string(governance.ReleaseBatchSealed) && filter.State != string(governance.ReleaseBatchCancelled) {
return governance.ReleaseBatchPage{}, ErrInvalidReleaseBatchFilter
}
page := governance.ReleaseBatchPage{Items: []governance.ReleaseBatch{}, Limit: filter.Limit, Offset: filter.Offset}
if err := s.db.QueryRow(ctx, countReleaseBatchesSQL, filter.State).Scan(&page.Total); err != nil {
return governance.ReleaseBatchPage{}, fmt.Errorf("count release batches: %w", err)
}
rows, err := s.db.Query(ctx, listReleaseBatchesSQL, filter.State, filter.Limit, filter.Offset)
if err != nil {
return governance.ReleaseBatchPage{}, fmt.Errorf("list release batches: %w", err)
}
defer rows.Close()
for rows.Next() {
batch, err := scanReleaseBatch(rows)
if err != nil {
return governance.ReleaseBatchPage{}, fmt.Errorf("scan release batch: %w", err)
}
page.Items = append(page.Items, batch)
}
if err := rows.Err(); err != nil {
return governance.ReleaseBatchPage{}, fmt.Errorf("iterate release batches: %w", err)
}
return page, nil
}
func (s *Store) GetReleaseBatch(ctx context.Context, batchID string) (governance.ReleaseBatchDetail, error) {
if !validShortID(batchID) {
return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound
}
batch, err := scanReleaseBatch(s.db.QueryRow(ctx, getReleaseBatchSQL, batchID))
if errors.Is(err, pgx.ErrNoRows) {
return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound
}
if err != nil {
return governance.ReleaseBatchDetail{}, fmt.Errorf("get release batch: %w", err)
}
rows, err := s.db.Query(ctx, listReleaseBatchItemsSQL, batchID)
if err != nil {
return governance.ReleaseBatchDetail{}, fmt.Errorf("list release batch items: %w", err)
}
defer rows.Close()
detail := governance.ReleaseBatchDetail{ReleaseBatch: batch, Items: []governance.ReleaseBatchItemView{}}
for rows.Next() {
var item governance.ReleaseBatchItemView
var visibility string
if err := rows.Scan(&item.PublicationID, &item.Title, &visibility, &item.Action); err != nil {
return governance.ReleaseBatchDetail{}, fmt.Errorf("scan release batch item: %w", err)
}
item.DecidedVisibility = governance.Visibility(visibility)
if (item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) || (item.Action != "upsert" && item.Action != "delete") {
return governance.ReleaseBatchDetail{}, errors.New("invalid release batch item visibility")
}
detail.Items = append(detail.Items, item)
}
if err := rows.Err(); err != nil {
return governance.ReleaseBatchDetail{}, fmt.Errorf("iterate release batch items: %w", err)
}
return detail, nil
}
func validCreateReleaseBatch(request governance.CreateReleaseBatchRequest) bool {
if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 255 ||
strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) ||
request.Now.IsZero() || len(request.PublicationIDs)+len(request.DeletePublicationIDs) < 1 || len(request.PublicationIDs)+len(request.DeletePublicationIDs) > 100 {
return false
}
if request.ReleaseChangeSetID != "" && !validRouteID(request.ReleaseChangeSetID) {
return false
}
seen := make(map[string]struct{}, len(request.PublicationIDs)+len(request.DeletePublicationIDs))
for _, publicationID := range append(append([]string{}, request.PublicationIDs...), request.DeletePublicationIDs...) {
if !validShortID(publicationID) {
return false
}
if _, exists := seen[publicationID]; exists {
return false
}
seen[publicationID] = struct{}{}
}
return true
}
func scanReleaseBatch(row rowScanner) (governance.ReleaseBatch, error) {
var batch governance.ReleaseBatch
var state string
if err := row.Scan(
&batch.ID, &batch.Name, &state, &batch.PolicyVersion,
&batch.CreatedBy, &batch.SealedBy, &batch.ItemCount,
&batch.CreatedAt, &batch.SealedAt, &batch.ReleaseChangeSetID,
); err != nil {
return governance.ReleaseBatch{}, err
}
batch.State = governance.ReleaseBatchState(state)
if !batch.State.Valid() || batch.PolicyVersion != releaseGatePolicyVersion {
return governance.ReleaseBatch{}, errors.New("invalid release batch row")
}
return batch, nil
}
@@ -0,0 +1,228 @@
package postgres
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"knowledge-governance/internal/governance"
)
func TestCreateReleaseBatchAtomicallyLocksOnlyApprovedCurrentItems(t *testing.T) {
now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC)
db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "draft", policy: "release-gate-v1", createdBy: "engineer-1", itemCount: 3, createdAt: now}}
batch, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{
Name: "Synthetic batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-b"}, DeletePublicationIDs: []string{"tp-c"}, Now: now,
})
if err != nil {
t.Fatalf("CreateReleaseBatch() error = %v", err)
}
if batch.ID != "rb-safe" || batch.State != governance.ReleaseBatchDraft || batch.ItemCount != 3 {
t.Fatalf("batch = %#v", batch)
}
sql := strings.ToLower(db.queryRowSQL)
for _, required := range []string{
"jsonb_array_elements_text", "publication_reviews", "decision = 'approved'",
"acceptance_run_id", "publication_updated_at", "knowledge_exists = 'passed'",
"chunks_present = 'passed'", "retrieval_hit in ('passed', 'manual_required')",
"insert into release_batches", "insert into release_batch_items", "count(distinct publication_id)", "returning",
"release_delete_ready", "'delete' as action", "eligible.action",
} {
if !strings.Contains(sql, required) {
t.Errorf("create batch SQL missing %q", required)
}
}
if strings.Contains(sql, "insert into jobs") || strings.Contains(sql, "release.probe") {
t.Fatal("P5 batch creation must not enqueue release jobs")
}
}
func TestCreateReleaseBatchRejectsInvalidOrDuplicateInput(t *testing.T) {
now := time.Now()
requests := []governance.CreateReleaseBatchRequest{
{Name: "", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: now},
{Name: "Batch", CreatedBy: "", PublicationIDs: []string{"tp-a"}, Now: now},
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: nil, Now: now},
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-a"}, Now: now},
{Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, DeletePublicationIDs: []string{"tp-a"}, Now: now},
}
for _, request := range requests {
if _, err := NewStore(&recordingDB{}).CreateReleaseBatch(context.Background(), request); !errors.Is(err, ErrInvalidReleaseBatch) {
t.Fatalf("request=%#v error=%v, want ErrInvalidReleaseBatch", request, err)
}
}
}
func TestCreateReleaseBatchReturnsNotEligibleWhenAnyItemFailsGate(t *testing.T) {
db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}}
_, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{
Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: time.Now(),
})
if !errors.Is(err, ErrReleaseBatchNotEligible) {
t.Fatalf("error=%v, want ErrReleaseBatchNotEligible", err)
}
}
func TestSealReleaseBatchRevalidatesEveryLockedItemWithoutEnqueue(t *testing.T) {
now := time.Date(2026, 9, 6, 9, 10, 0, 0, time.UTC)
db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 2, createdAt: now.Add(-time.Minute), sealedAt: &now}}
batch, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-2", Now: now})
if err != nil {
t.Fatalf("SealReleaseBatch() error = %v", err)
}
if batch.State != governance.ReleaseBatchSealed || batch.SealedBy != "engineer-2" || batch.SealedAt == nil {
t.Fatalf("batch = %#v", batch)
}
sql := strings.ToLower(db.queryRowSQL)
for _, required := range []string{
"join release_batch_items", "join publication_reviews",
"ready.review_id = item.review_id", "ready.acceptance_run_id = item.acceptance_run_id", "ready.publication_updated_at = item.publication_updated_at",
"publication.ingest_state = 'uploaded'", "publication.processing_state = 'completed'",
"publication.retrieval_state = 'unpublished'", "state = 'sealed'", "state = 'draft'",
"release_delete_ready", "item.action = 'upsert'", "item.action = 'delete'",
} {
if !strings.Contains(sql, required) {
t.Errorf("seal SQL missing %q", required)
}
}
if strings.Contains(sql, "insert into jobs") {
t.Fatal("P5 seal must not enqueue a production release")
}
}
func TestSealReleaseBatchReturnsGateConflict(t *testing.T) {
db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}}
_, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-1", Now: time.Now()})
if !errors.Is(err, ErrReleaseBatchGateConflict) {
t.Fatalf("error=%v, want ErrReleaseBatchGateConflict", err)
}
}
func TestListReleaseBatchesAndDetailReturnSafeProjection(t *testing.T) {
now := time.Date(2026, 9, 6, 9, 20, 0, 0, time.UTC)
db := &recordingDB{
row: releaseCountRow{count: 1},
rows: &releaseBatchRows{values: []releaseBatchRow{{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now}}},
}
page, err := NewStore(db).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "sealed", Limit: 20})
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].State != governance.ReleaseBatchSealed {
t.Fatalf("page=%#v error=%v", page, err)
}
joined := strings.ToLower(db.queryRowSQL + "\n" + db.querySQL)
if !strings.Contains(joined, "from release_batches") || !strings.Contains(joined, "state = $1") {
t.Fatalf("batch list SQL=%s", joined)
}
detailDB := &releaseBatchReadDB{
recordingDB: &recordingDB{},
row: releaseBatchRow{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now},
rows: &releaseBatchItemRows{values: []releaseBatchItemRow{{publicationID: "tp-safe", title: "Synthetic", visibility: "internal", action: "delete"}}},
}
detail, err := NewStore(detailDB).GetReleaseBatch(context.Background(), "rb-safe")
if err != nil || len(detail.Items) != 1 || detail.Items[0].Title != "Synthetic" || detail.Items[0].Action != "delete" {
t.Fatalf("detail=%#v error=%v", detail, err)
}
joined = strings.ToLower(detailDB.queryRowSQL + "\n" + detailDB.querySQL)
for _, forbidden := range []string{"knowledge_id", "test_kb_id", "acceptance_run_id", "publication_updated_at", "stable_external_id", "local_spool_ref"} {
if strings.Contains(joined, forbidden) {
t.Errorf("batch read SQL includes forbidden projection %q", forbidden)
}
}
}
func TestListReleaseBatchesRejectsInvalidFilter(t *testing.T) {
_, err := NewStore(&recordingDB{}).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "released", Limit: 20})
if !errors.Is(err, ErrInvalidReleaseBatchFilter) {
t.Fatalf("error=%v, want ErrInvalidReleaseBatchFilter", err)
}
}
type releaseBatchRow struct {
id, name, state, policy, createdBy, sealedBy string
itemCount int
createdAt time.Time
sealedAt *time.Time
}
func (r releaseBatchRow) Scan(dest ...any) error {
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = r.id, r.name, r.state, r.policy
*dest[4].(*string), *dest[5].(*string), *dest[6].(*int) = r.createdBy, r.sealedBy, r.itemCount
*dest[7].(*time.Time), *dest[8].(**time.Time) = r.createdAt, r.sealedAt
return nil
}
type releaseCountRow struct{ count int }
func (r releaseCountRow) Scan(dest ...any) error {
*dest[0].(*int) = r.count
return nil
}
type releaseBatchRows struct {
values []releaseBatchRow
index int
}
func (r *releaseBatchRows) Close() {}
func (r *releaseBatchRows) Err() error { return nil }
func (r *releaseBatchRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
func (r *releaseBatchRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
func (r *releaseBatchRows) RawValues() [][]byte { return nil }
func (r *releaseBatchRows) Conn() *pgx.Conn { return nil }
func (r *releaseBatchRows) Next() bool {
if r.index >= len(r.values) {
return false
}
r.index++
return true
}
func (r *releaseBatchRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) }
func (r *releaseBatchRows) Values() ([]any, error) { return nil, nil }
type releaseBatchItemRow struct{ publicationID, title, visibility, action string }
type releaseBatchItemRows struct {
values []releaseBatchItemRow
index int
}
func (r *releaseBatchItemRows) Close() {}
func (r *releaseBatchItemRows) Err() error { return nil }
func (r *releaseBatchItemRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
func (r *releaseBatchItemRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
func (r *releaseBatchItemRows) RawValues() [][]byte { return nil }
func (r *releaseBatchItemRows) Conn() *pgx.Conn { return nil }
func (r *releaseBatchItemRows) Next() bool {
if r.index >= len(r.values) {
return false
}
r.index++
return true
}
func (r *releaseBatchItemRows) Scan(dest ...any) error {
v := r.values[r.index-1]
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = v.publicationID, v.title, v.visibility, v.action
return nil
}
func (r *releaseBatchItemRows) Values() ([]any, error) { return nil, nil }
type releaseBatchReadDB struct {
*recordingDB
row pgx.Row
rows pgx.Rows
}
func (d *releaseBatchReadDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row {
d.queryRowSQL, d.queryRowArgs = sql, args
return d.row
}
func (d *releaseBatchReadDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) {
d.querySQL, d.queryArgs = sql, args
return d.rows, nil
}
@@ -0,0 +1,290 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"knowledge-governance/internal/governance"
)
// Errors for R1 release ChangeSets.
var (
ErrInvalidReleaseChangeSet = errors.New("invalid release change set request")
ErrReleaseChangeSetNotFound = errors.New("release change set not found")
ErrReleaseChangeSetStateConflict = errors.New("release change set state conflict")
ErrReleaseChangeSetEmpty = errors.New("release plan has no releasable items")
)
const releaseChangeSetColumns = `id,number,name,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at,confirmed_by,COALESCE(confirmed_at::text,'')`
func scanReleaseChangeSet(row pgx.Row) (governance.ReleaseChangeSet, error) {
var item governance.ReleaseChangeSet
var snapshots, changeSets []byte
var confirmedAt string
err := row.Scan(&item.ID, &item.Number, &item.Name, &item.State, &snapshots, &changeSets, &item.AcceptanceCutoff, &item.ProductionBaselineRef,
&item.ItemCount, &item.AddCount, &item.UpdateCount, &item.DeleteCount, &item.BlockedCount, &item.ManifestSHA256, &item.CreatedBy, &item.CreatedAt, &item.ConfirmedBy, &confirmedAt)
if err != nil {
return governance.ReleaseChangeSet{}, err
}
_ = json.Unmarshal(snapshots, &item.SourceSnapshotIDs)
_ = json.Unmarshal(changeSets, &item.SourceChangeSetIDs)
if item.SourceSnapshotIDs == nil {
item.SourceSnapshotIDs = []string{}
}
if item.SourceChangeSetIDs == nil {
item.SourceChangeSetIDs = []string{}
}
if confirmedAt != "" {
if parsed, parseErr := time.Parse(time.RFC3339Nano, confirmedAt); parseErr == nil {
item.ConfirmedAt = &parsed
}
}
return item, nil
}
// ComputeReleaseChangeSetManifest is exported for tests: the manifest seals
// every frozen decision in seq order with an exact delimiter.
func ComputeReleaseChangeSetManifest(items []governance.ReleaseChangeSetItem) string {
lines := make([]string, 0, len(items))
for _, item := range items {
lines = append(lines, strings.Join([]string{
strconv.Itoa(item.Seq), item.PublicationID, item.TargetScope, item.Action,
item.TestState, item.ProductionState, strconv.FormatBool(item.CanRelease), item.Blocker,
}, "|"))
}
digest := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return hex.EncodeToString(digest[:])
}
// GenerateReleaseChangeSet freezes the current live plan into an immutable
// versioned set. Repeated generation with an identical plan reuses the latest
// PLANNED set (dedup), so concurrent generation cannot create duplicates.
func (s *Store) GenerateReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) {
if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 200 ||
strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) || request.Now.IsZero() {
return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet
}
plan, err := s.ListReleasePlan(ctx)
if err != nil {
return governance.ReleaseChangeSet{}, err
}
items := make([]governance.ReleaseChangeSetItem, 0, len(plan.Items))
for index, planItem := range plan.Items {
items = append(items, governance.ReleaseChangeSetItem{Seq: index + 1, PublicationID: planItem.PublicationID, Title: planItem.Title, TargetScope: planItem.TargetScope, Action: planItem.Action, TestState: planItem.TestState, ProductionState: planItem.ProductionState, CanRelease: planItem.CanRelease, Blocker: planItem.Blocker})
}
if len(items) == 0 {
return governance.ReleaseChangeSet{}, ErrReleaseChangeSetEmpty
}
manifest := ComputeReleaseChangeSetManifest(items)
counts := map[string]int{}
for _, item := range items {
counts[item.Action]++
}
blocked := 0
for _, item := range items {
if item.Action == "blocked" {
blocked++
}
}
// Same plan twice (sequentially) reuses the latest PLANNED set instead of
// piling up identical versions. Concurrent racers fall through to the
// unique-number conflict path below.
if existing, lookupErr := s.findPlannedByManifest(ctx, manifest); lookupErr == nil && existing.ManifestSHA256 == manifest {
return existing, nil
} else if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) {
return governance.ReleaseChangeSet{}, lookupErr
}
for attempt := 0; attempt < 3; attempt++ {
result, retry, err := s.tryInsertReleaseChangeSet(ctx, request, items, manifest, counts["add"], counts["update"], counts["delete"], blocked)
if err == nil {
return result, nil
}
if !retry {
return governance.ReleaseChangeSet{}, err
}
// A concurrent identical generation won; return its set (dedup) if the
// manifest matches, otherwise loop to take the next version number.
existing, lookupErr := s.findPlannedByManifest(ctx, manifest)
if lookupErr == nil && existing.ManifestSHA256 == manifest {
return existing, nil
}
if lookupErr != nil && !errors.Is(lookupErr, ErrReleaseChangeSetNotFound) {
return governance.ReleaseChangeSet{}, lookupErr
}
}
return governance.ReleaseChangeSet{}, fmt.Errorf("release change set generation kept conflicting")
}
func (s *Store) tryInsertReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, items []governance.ReleaseChangeSetItem, manifest string, add, update, del, blocked int) (governance.ReleaseChangeSet, bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return governance.ReleaseChangeSet{}, false, err
}
defer func() { _ = tx.Rollback(ctx) }()
id, err := newOpaqueID("rcs-")
if err != nil {
return governance.ReleaseChangeSet{}, false, err
}
set, err := scanReleaseChangeSet(tx.QueryRow(ctx, `
INSERT INTO release_change_sets(id,name,number,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at)
SELECT $1,$2,(SELECT COALESCE(MAX(number),0)+1 FROM release_change_sets),'planned',
COALESCE((SELECT jsonb_agg(snapshot_id ORDER BY snapshot_id) FROM (SELECT DISTINCT snapshot_id FROM source_catalog_heads) s),'[]'::jsonb),
COALESCE((SELECT jsonb_agg(change_set_id ORDER BY change_set_id) FROM (SELECT DISTINCT id AS change_set_id FROM source_catalog_change_sets WHERE state IN ('confirmed','applied')) c),'[]'::jsonb),
COALESCE((SELECT MAX(updated_at) FROM test_publications),(SELECT now())),
(SELECT COALESCE(MAX(completed_at)::text,'') FROM release_executions WHERE state='completed'),
$3,$4,$5,$6,$7,$8,$9,$10
RETURNING `+releaseChangeSetColumns,
id, request.Name, len(items), add, update, del, blocked, manifest, request.CreatedBy, request.Now))
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
return governance.ReleaseChangeSet{}, true, err
}
return governance.ReleaseChangeSet{}, false, fmt.Errorf("create release change set: %w", err)
}
// R6 audit chain: mirror the JSONB binding into real FK rows so the trace
// survives even if someone later edits denormalized display fields.
if _, err := tx.Exec(ctx, `
INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id)
SELECT $1, elem.value
FROM jsonb_array_elements_text($2::jsonb) AS elem(value)
ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceSnapshotIDs)); err != nil {
return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release snapshots: %w", err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id)
SELECT $1, elem.value
FROM jsonb_array_elements_text($2::jsonb) AS elem(value)
ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceChangeSetIDs)); err != nil {
return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release catalog sets: %w", err)
}
for _, item := range items {
if _, err := tx.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`,
set.ID, item.Seq, item.PublicationID, item.Title, item.TargetScope, item.Action, item.TestState, item.ProductionState, item.CanRelease, item.Blocker); err != nil {
return governance.ReleaseChangeSet{}, false, fmt.Errorf("insert release change set item: %w", err)
}
}
if err := tx.Commit(ctx); err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
return governance.ReleaseChangeSet{}, true, err
}
return governance.ReleaseChangeSet{}, false, err
}
return set, false, nil
}
func (s *Store) findPlannedByManifest(ctx context.Context, manifest string) (governance.ReleaseChangeSet, error) {
return scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE manifest_sha256=$1 AND state='planned' ORDER BY number DESC LIMIT 1`, manifest))
}
// ConfirmReleaseChangeSet freezes a planned set for batching. Re-confirming an
// already-confirmed set replays the stored confirmation (immutability).
func (s *Store) ConfirmReleaseChangeSet(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) {
if !validRouteID(request.ChangeSetID) || !validReviewerRef(request.ConfirmedBy) || request.Now.IsZero() {
return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet
}
set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `UPDATE release_change_sets SET state='confirmed',confirmed_by=$2,confirmed_at=$3 WHERE id=$1 AND state='planned' RETURNING `+releaseChangeSetColumns, request.ChangeSetID, request.ConfirmedBy, request.Now))
if errors.Is(err, pgx.ErrNoRows) {
current, lookupErr := s.GetReleaseChangeSet(ctx, request.ChangeSetID)
if lookupErr != nil {
return governance.ReleaseChangeSet{}, lookupErr
}
if current.State == "confirmed" || current.State == "consumed" {
return current.ReleaseChangeSet, nil
}
return governance.ReleaseChangeSet{}, ErrReleaseChangeSetStateConflict
}
if err != nil {
return governance.ReleaseChangeSet{}, fmt.Errorf("confirm release change set: %w", err)
}
return set, nil
}
func (s *Store) GetReleaseChangeSet(ctx context.Context, id string) (governance.ReleaseChangeSetDetail, error) {
set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE id=$1`, id))
if errors.Is(err, pgx.ErrNoRows) {
return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound
}
if err != nil {
return governance.ReleaseChangeSetDetail{}, err
}
detail := governance.ReleaseChangeSetDetail{ReleaseChangeSet: set, Items: []governance.ReleaseChangeSetItem{}}
rows, err := s.db.Query(ctx, `SELECT seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker FROM release_change_set_items WHERE change_set_id=$1 ORDER BY seq`, id)
if err != nil {
return governance.ReleaseChangeSetDetail{}, fmt.Errorf("list release change set items: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.ReleaseChangeSetItem
if err := rows.Scan(&item.Seq, &item.PublicationID, &item.Title, &item.TargetScope, &item.Action, &item.TestState, &item.ProductionState, &item.CanRelease, &item.Blocker); err != nil {
return governance.ReleaseChangeSetDetail{}, err
}
detail.Items = append(detail.Items, item)
}
return detail, rows.Err()
}
func (s *Store) ListReleaseChangeSets(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseChangeSetPage, error) {
limit, offset := filter.Limit, filter.Offset
if limit <= 0 {
limit = 20
}
if offset < 0 {
offset = 0
}
page := governance.ReleaseChangeSetPage{Items: []governance.ReleaseChangeSet{}, Limit: limit, Offset: offset}
if limit > 100 {
return page, errors.New("release change set limit must be between 1 and 100")
}
var total int
if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&total); err != nil {
return page, err
}
page.Total = total
rows, err := s.db.Query(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets ORDER BY number DESC LIMIT $1 OFFSET $2`, limit, offset)
if err != nil {
return page, err
}
defer rows.Close()
for rows.Next() {
item, err := scanReleaseChangeSet(rows)
if err != nil {
return page, err
}
page.Items = append(page.Items, item)
}
return page, rows.Err()
}
func (s *Store) LatestReleaseChangeSet(ctx context.Context) (governance.ReleaseChangeSetDetail, error) {
var id string
if err := s.db.QueryRow(ctx, `SELECT id FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&id); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound
}
return governance.ReleaseChangeSetDetail{}, err
}
return s.GetReleaseChangeSet(ctx, id)
}
func mustMarshalJSON(values []string) []byte {
raw, err := json.Marshal(values)
if err != nil {
return []byte("[]")
}
return raw
}
@@ -0,0 +1,132 @@
package postgres
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"knowledge-governance/internal/governance"
)
func TestListAndGetReleaseExecutionsReturnsAuditableProjection(t *testing.T) {
now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC)
row := releaseExecutionRow{id: "rx-safe", batchID: "rb-safe", operation: "publish", state: "completed", target: "production", authorizedBy: "owner", authorizationCode: "production_release_authorized", jobID: "job-safe", manifestSHA: strings.Repeat("a", 64), itemCount: 1, completedCount: 1, createdAt: now, completedAt: &now}
db := &releaseExecutionReadDB{countRow: releaseExecutionCountRow{count: 1}, row: row, rows: &releaseExecutionRows{values: []releaseExecutionRow{row}}}
store := NewStore(db)
page, err := store.ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "publish", State: "completed", Limit: 20})
if err != nil || page.Total != 1 || len(page.Items) != 1 {
t.Fatalf("page=%#v error=%v", page, err)
}
db.rows = &releaseItemExecutionRows{values: []releaseItemExecutionRow{{publicationID: "tp-safe", title: "Synthetic", action: "delete", state: "verified", previousRevision: "prod-v0", productionRevision: "absent", completedAt: &now}}}
detail, err := store.GetReleaseExecution(context.Background(), "rx-safe")
if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" || detail.Items[0].ProductionRevision != "absent" {
t.Fatalf("detail=%#v error=%v", detail, err)
}
joined := strings.ToLower(strings.Join(db.queries, "\n"))
for _, required := range []string{"from release_executions", "operation = $1", "state = $2", "from release_item_executions", "publication_id", "item.action"} {
if !strings.Contains(joined, required) {
t.Fatalf("release read SQL missing %q", required)
}
}
for _, forbidden := range []string{"knowledge_id", "test_kb_id", "physical_path", "api_key", "secret"} {
if strings.Contains(joined, forbidden) {
t.Fatalf("release read SQL contains forbidden %q", forbidden)
}
}
}
func TestListReleaseExecutionsRejectsInvalidFilter(t *testing.T) {
_, err := NewStore(&recordingDB{}).ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "deploy", Limit: 20})
if !errors.Is(err, ErrInvalidReleaseExecutionFilter) {
t.Fatalf("error=%v", err)
}
}
type releaseExecutionCountRow struct{ count int }
func (r releaseExecutionCountRow) Scan(dest ...any) error { *dest[0].(*int) = r.count; return nil }
type releaseExecutionRows struct {
values []releaseExecutionRow
index int
}
func (r *releaseExecutionRows) Close() {}
func (r *releaseExecutionRows) Err() error { return nil }
func (r *releaseExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
func (r *releaseExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
func (r *releaseExecutionRows) RawValues() [][]byte { return nil }
func (r *releaseExecutionRows) Conn() *pgx.Conn { return nil }
func (r *releaseExecutionRows) Next() bool {
if r.index >= len(r.values) {
return false
}
r.index++
return true
}
func (r *releaseExecutionRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) }
func (r *releaseExecutionRows) Values() ([]any, error) { return nil, nil }
type releaseItemExecutionRow struct {
publicationID, title, action, state, previousRevision, productionRevision, errorCode string
createdAt time.Time
completedAt *time.Time
}
type releaseItemExecutionRows struct {
values []releaseItemExecutionRow
index int
}
func (r *releaseItemExecutionRows) Close() {}
func (r *releaseItemExecutionRows) Err() error { return nil }
func (r *releaseItemExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
func (r *releaseItemExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
func (r *releaseItemExecutionRows) RawValues() [][]byte { return nil }
func (r *releaseItemExecutionRows) Conn() *pgx.Conn { return nil }
func (r *releaseItemExecutionRows) Next() bool {
if r.index >= len(r.values) {
return false
}
r.index++
return true
}
func (r *releaseItemExecutionRows) Scan(dest ...any) error {
value := r.values[r.index-1]
*dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = value.publicationID, value.title, value.action, value.state
*dest[4].(*string), *dest[5].(*string), *dest[6].(*string) = value.previousRevision, value.productionRevision, value.errorCode
*dest[7].(*time.Time), *dest[8].(**time.Time) = value.createdAt, value.completedAt
return nil
}
func (r *releaseItemExecutionRows) Values() ([]any, error) { return nil, nil }
type releaseExecutionReadDB struct {
countRow, row pgx.Row
rows pgx.Rows
queries []string
queryRows int
}
func (d *releaseExecutionReadDB) Begin(context.Context) (pgx.Tx, error) {
return nil, errors.New("transactions are not supported by the reading fake")
}
func (d *releaseExecutionReadDB) QueryRow(_ context.Context, sql string, _ ...any) pgx.Row {
d.queries = append(d.queries, sql)
d.queryRows++
if d.queryRows == 1 {
return d.countRow
}
return d.row
}
func (d *releaseExecutionReadDB) Query(_ context.Context, sql string, _ ...any) (pgx.Rows, error) {
d.queries = append(d.queries, sql)
return d.rows, nil
}
func (d *releaseExecutionReadDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) {
return pgconn.CommandTag{}, nil
}
@@ -0,0 +1,498 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
var ErrReleaseExecutionNotClaimable = errors.New("release execution is not claimable")
const startReleaseExecutionSQL = `
WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `,
execution_gate AS (
SELECT execution.id, execution.operation,
COUNT(item.publication_id)::INTEGER AS item_count,
COUNT(*) FILTER (
WHERE (item.action = 'upsert'
AND ready.publication_id IS NOT NULL
AND ready.review_id = item.review_id
AND ready.acceptance_run_id = item.acceptance_run_id
AND ready.publication_updated_at = item.publication_updated_at
AND ready.decided_visibility = item.decided_visibility)
OR (item.action = 'delete'
AND delete_ready.publication_id IS NOT NULL
AND delete_ready.review_id = item.review_id
AND delete_ready.acceptance_run_id = item.acceptance_run_id
AND delete_ready.publication_updated_at = item.publication_updated_at
AND delete_ready.decided_visibility = item.decided_visibility)
)::INTEGER AS valid_items
FROM release_executions AS execution
JOIN release_batch_items AS item ON item.release_batch_id = execution.release_batch_id
LEFT JOIN release_upsert_ready AS ready ON ready.publication_id = item.publication_id AND item.action = 'upsert'
LEFT JOIN release_delete_ready AS delete_ready ON delete_ready.publication_id = item.publication_id AND item.action = 'delete'
WHERE execution.id = $1
GROUP BY execution.id, execution.operation
), started AS (
UPDATE release_executions AS execution
SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolling_back' ELSE 'running' END,
started_at = COALESCE(started_at, $4::TIMESTAMPTZ)
FROM execution_gate
WHERE execution.id = $1::VARCHAR
AND execution_gate.id = execution.id
AND (execution_gate.operation = 'rollback' OR execution_gate.valid_items = execution_gate.item_count)
AND execution.job_id = $2::VARCHAR
AND execution.operation = $3::VARCHAR
AND execution.state IN ('pending', 'running', 'rolling_back')
AND NOT EXISTS (
SELECT 1
FROM release_batch_items AS gate_item
JOIN test_publications AS gate_publication ON gate_publication.id = gate_item.publication_id
JOIN source_item_versions AS gate_version ON gate_version.id = gate_item.source_item_version_id
WHERE gate_item.release_batch_id = execution.release_batch_id
AND (gate_version.revision = '' OR gate_publication.parser_generation = '' OR gate_publication.index_generation = '')
)
RETURNING execution.id, execution.release_batch_id, execution.operation, execution.prior_execution_id
), seeded AS (
INSERT INTO release_item_executions (
id, release_execution_id, publication_id, action, state, created_at, updated_at
)
SELECT 'ri-' || SUBSTR(MD5(started.id || ':' || item.publication_id), 1, 32),
started.id, item.publication_id, item.action, 'pending', $4::TIMESTAMPTZ, $4::TIMESTAMPTZ
FROM started
JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id
WHERE started.operation = 'publish' OR EXISTS (
SELECT 1
FROM release_item_executions AS prior_seed
WHERE prior_seed.release_execution_id = started.prior_execution_id
AND prior_seed.publication_id = item.publication_id
AND prior_seed.state = 'verified'
)
ON CONFLICT (release_execution_id, publication_id) DO NOTHING
RETURNING publication_id, action
), active_items AS (
SELECT publication_id, action FROM seeded
UNION
SELECT current_item.publication_id, current_item.action
FROM started
JOIN release_item_executions AS current_item
ON current_item.release_execution_id = started.id
WHERE current_item.state = 'pending'
), plan AS (
SELECT started.id AS release_execution_id,
item.publication_id,
active_items.action,
item.source_item_version_id,
current_version.revision AS source_revision,
publication.parser_generation || '@' || publication.index_generation AS parse_revision,
item.review_id AS release_revision,
publication.test_kb_id,
publication.knowledge_id,
item.decided_visibility,
COALESCE(previous.production_revision, '') AS previous_production_revision
FROM started
JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id
JOIN active_items ON active_items.publication_id = item.publication_id
JOIN test_publications AS publication ON publication.id = item.publication_id
JOIN source_item_versions AS current_version ON current_version.id = item.source_item_version_id
LEFT JOIN LATERAL (
SELECT CASE
WHEN started.operation = 'rollback' THEN prior_item.previous_production_revision
ELSE prior_item.production_revision
END AS production_revision
FROM release_item_executions AS prior_item
JOIN release_executions AS prior_execution ON prior_execution.id = prior_item.release_execution_id
JOIN test_publications AS prior_publication ON prior_publication.id = prior_item.publication_id
JOIN source_item_versions AS prior_version ON prior_version.id = prior_publication.source_item_version_id
WHERE prior_item.state = 'verified'
AND prior_execution.operation = 'publish'
AND prior_execution.state IN ('completed', 'failed')
AND (
(started.operation = 'rollback'
AND prior_execution.id = started.prior_execution_id
AND prior_item.publication_id = item.publication_id)
OR
(started.operation = 'publish'
AND prior_version.source_item_id = current_version.source_item_id
AND (
(active_items.action = 'delete' AND prior_item.publication_id = item.publication_id)
OR (active_items.action = 'upsert' AND prior_item.publication_id <> item.publication_id)
))
)
ORDER BY prior_item.completed_at DESC, prior_item.id DESC
LIMIT 1
) AS previous ON TRUE
ORDER BY item.publication_id
)
SELECT COUNT(DISTINCT started.id)::INTEGER,
COALESCE(
JSONB_AGG(JSONB_BUILD_OBJECT(
'release_execution_id', plan.release_execution_id,
'publication_id', plan.publication_id,
'action', plan.action,
'source_item_version_id', plan.source_item_version_id,
'source_revision', plan.source_revision,
'parse_revision', plan.parse_revision,
'release_revision', plan.release_revision,
'test_knowledge_base_id', plan.test_kb_id,
'knowledge_id', plan.knowledge_id,
'decided_visibility', plan.decided_visibility,
'previous_production_revision', plan.previous_production_revision
) ORDER BY plan.publication_id) FILTER (WHERE plan.publication_id IS NOT NULL),
'[]'::JSONB
)
FROM started
LEFT JOIN plan ON TRUE
`
const beginReleaseBatchSQL = `
UPDATE release_executions
SET current_batch = $2,
total_batches = $3
WHERE id = $1
AND state IN ('running', 'rolling_back')
AND (total_batches = 0 OR total_batches = $3)
AND ($2 = current_batch OR $2 = current_batch + 1)
`
const completeReleaseItemSQL = `
UPDATE release_item_executions AS item
SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolled_back' ELSE 'verified' END,
bundle_sha256 = $3,
previous_production_revision = $4,
production_revision = $5,
error_code = '',
updated_at = $6,
completed_at = $6
FROM release_executions AS execution
WHERE item.release_execution_id = $1
AND item.publication_id = $2
AND execution.id = item.release_execution_id
AND execution.state IN ('running', 'rolling_back')
AND item.state IN ('pending', 'staged', 'activated', 'verified', 'rolled_back')
`
const failReleaseItemSQL = `
UPDATE release_item_executions AS item
SET state = 'failed',
error_code = $3,
updated_at = $4,
completed_at = $4
FROM release_executions AS execution
WHERE item.release_execution_id = $1
AND item.publication_id = $2
AND execution.id = item.release_execution_id
AND execution.state IN ('running', 'rolling_back')
AND item.state IN ('pending', 'staged', 'activated', 'failed')
`
const finishReleaseExecutionSQL = `
WITH item_counts AS (
SELECT release_execution_id,
COUNT(*)::INTEGER AS total_items,
COUNT(*) FILTER (WHERE state IN ('verified', 'rolled_back'))::INTEGER AS completed_items,
COUNT(*) FILTER (WHERE state = 'failed')::INTEGER AS failed_items
FROM release_item_executions
WHERE release_execution_id = $1
GROUP BY release_execution_id
)
UPDATE release_executions AS execution
SET state = CASE
WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'rollback_failed'
WHEN execution.operation = 'rollback' THEN 'rolled_back'
WHEN item_counts.failed_items > 0 THEN 'failed'
ELSE 'completed'
END,
completed_item_count = item_counts.completed_items,
failed_item_count = item_counts.failed_items,
-- R4 failure grading on publish: nothing published yet = single_item
-- (compensate that item only); progress existed = batch (stop later
-- batches, compensate this batch's verified items). A failed ROLLBACK
-- escalates the prior publish to overall (snapshot restore required).
failure_level = CASE
WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 AND item_counts.completed_items > 0 THEN 'batch'
WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 THEN 'single_item'
ELSE execution.failure_level
END,
error_code = CASE
WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'release_rollback_failed'
WHEN item_counts.failed_items > 0 THEN 'release_item_failed'
ELSE ''
END,
completed_at = $2
FROM item_counts
WHERE execution.id = item_counts.release_execution_id
AND execution.state IN ('running', 'rolling_back')
AND item_counts.total_items = execution.item_count
AND item_counts.completed_items + item_counts.failed_items = execution.item_count
`
// R4: a permanently failed rollback means compensation alone is no longer
// enough: the prior publish escalates to overall (snapshot restore).
const escalatePriorOverallSQL = `
UPDATE release_executions AS prior
SET failure_level = 'overall'
FROM release_executions AS rollback
WHERE rollback.id = $1
AND rollback.operation = 'rollback'
AND rollback.state = 'rollback_failed'
AND prior.id = rollback.prior_execution_id
AND prior.operation = 'publish'
`
func (s *Store) StartExecution(ctx context.Context, request releaseworkflow.StartExecutionRequest) ([]releaseworkflow.PlanItem, error) {
if !validReleaseExecutionRequest(request) {
return nil, ErrReleaseExecutionNotClaimable
}
var executionCount int
var payload []byte
if err := s.db.QueryRow(ctx, startReleaseExecutionSQL,
request.ExecutionID, request.JobID, request.Operation, request.Now.UTC(),
).Scan(&executionCount, &payload); err != nil {
return nil, fmt.Errorf("start release execution: %w", err)
}
if executionCount != 1 {
return nil, ErrReleaseExecutionNotClaimable
}
var items []releaseworkflow.PlanItem
if err := json.Unmarshal(payload, &items); err != nil {
return nil, errors.New("decode release execution plan")
}
for _, item := range items {
if !validShortID(item.PublicationID) || !validShortID(item.SourceItemVersionID) ||
!validRemoteIdentifier(item.TestKnowledgeBaseID) || !validRemoteIdentifier(item.KnowledgeID) ||
(item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) ||
len(item.PreviousProductionRevision) > 128 {
return nil, errors.New("invalid release execution plan")
}
}
return items, nil
}
func (s *Store) BeginBatch(ctx context.Context, executionID string, batch, total int, now time.Time) error {
if !validShortID(executionID) || batch < 1 || total < batch || total > 100000 || now.IsZero() {
return ErrReleaseExecutionNotClaimable
}
result, err := s.db.Exec(ctx, beginReleaseBatchSQL, executionID, batch, total)
if err != nil {
return fmt.Errorf("begin release batch: %w", err)
}
if result.RowsAffected() != 1 {
return ErrReleaseExecutionNotClaimable
}
return nil
}
func (s *Store) CompleteItem(ctx context.Context, request releaseworkflow.CompleteItemRequest) error {
if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) ||
(request.BundleSHA256 != "" && !validLowerSHA256(request.BundleSHA256)) ||
!validRevision(request.ProductionRevision) || !validOptionalRevision(request.PreviousProductionRevision) || request.Now.IsZero() {
return ErrReleaseExecutionNotClaimable
}
result, err := s.db.Exec(ctx, completeReleaseItemSQL, request.ExecutionID, request.PublicationID,
request.BundleSHA256, request.PreviousProductionRevision, request.ProductionRevision, request.Now.UTC())
if err != nil {
return fmt.Errorf("complete release item: %w", err)
}
if result.RowsAffected() != 1 {
return ErrReleaseExecutionNotClaimable
}
return nil
}
func (s *Store) FailItem(ctx context.Context, request releaseworkflow.FailItemRequest) error {
if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) || !releaseworkflow.ValidErrorCode(request.ErrorCode) || request.Now.IsZero() {
return ErrReleaseExecutionNotClaimable
}
result, err := s.db.Exec(ctx, failReleaseItemSQL, request.ExecutionID, request.PublicationID, request.ErrorCode, request.Now.UTC())
if err != nil {
return fmt.Errorf("fail release item: %w", err)
}
if result.RowsAffected() != 1 {
return ErrReleaseExecutionNotClaimable
}
return nil
}
// ComputeReleaseExecutionManifest seals what ACTUALLY happened: one line per
// item (sorted by publication id) plus the bound release change set digest.
// Publish lines carry the verified bundle digest; rollback lines carry the
// restored revision. The digest is deterministic and re-computable for audits.
func ComputeReleaseExecutionManifest(operation, changeSetManifest string, lines []string) string {
digest := sha256.New()
bound := changeSetManifest
if bound == "" {
bound = "none"
}
fmt.Fprintf(digest, "op:%s\nchangeset:%s\n", operation, bound)
for _, line := range lines {
fmt.Fprintf(digest, "%s\n", line)
}
return hex.EncodeToString(digest.Sum(nil))
}
func (s *Store) FinishExecution(ctx context.Context, executionID string, now time.Time) error {
if !validShortID(executionID) || now.IsZero() {
return ErrReleaseExecutionNotClaimable
}
result, err := s.db.Exec(ctx, finishReleaseExecutionSQL, executionID, now.UTC())
if err != nil {
return fmt.Errorf("finish release execution: %w", err)
}
if result.RowsAffected() != 1 {
return ErrReleaseExecutionNotClaimable
}
// R4: a permanently failed rollback escalates the prior publish to the
// overall level (snapshot restore becomes the only sanctioned path).
if _, err := s.db.Exec(ctx, escalatePriorOverallSQL, executionID); err != nil {
return fmt.Errorf("escalate rollback failure: %w", err)
}
// R5: seal the execution manifest and reconcile it against the confirmed
// release change set. A success that does not match the frozen manifest is
// downgraded to failed (release_manifest_mismatch) — never completed.
if err := s.sealExecutionManifest(ctx, executionID); err != nil {
return err
}
return nil
}
func (s *Store) sealExecutionManifest(ctx context.Context, executionID string) error {
var operation, state, changeSetManifest, changeSetID string
err := s.db.QueryRow(ctx, `
SELECT e.operation, e.state, COALESCE(rcs.manifest_sha256,''), COALESCE(e.release_change_set_id::text,'')
FROM release_executions AS e
LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id
WHERE e.id = $1`, executionID).Scan(&operation, &state, &changeSetManifest, &changeSetID)
if errors.Is(err, pgx.ErrNoRows) {
return ErrReleaseExecutionNotClaimable
}
if err != nil {
return fmt.Errorf("load execution for manifest seal: %w", err)
}
rows, err := s.db.Query(ctx, `
SELECT item.publication_id, item.action, item.state, COALESCE(item.bundle_sha256,''), COALESCE(item.production_revision,'')
FROM release_item_executions AS item
WHERE item.release_execution_id = $1
ORDER BY item.publication_id`, executionID)
if err != nil {
return fmt.Errorf("load execution items for manifest seal: %w", err)
}
type itemRow struct {
publicationID, action, itemState, bundleSHA, revision string
}
items := []itemRow{}
for rows.Next() {
var item itemRow
if err := rows.Scan(&item.publicationID, &item.action, &item.itemState, &item.bundleSHA, &item.revision); err != nil {
rows.Close()
return fmt.Errorf("scan execution item for manifest seal: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
rows.Close()
return fmt.Errorf("iterate execution items: %w", err)
}
rows.Close()
lines := make([]string, 0, len(items))
for _, item := range items {
if operation == "rollback" {
lines = append(lines, fmt.Sprintf("%s|%s", item.publicationID, item.revision))
} else {
lines = append(lines, fmt.Sprintf("%s|%s|%s", item.publicationID, item.bundleSHA, item.revision))
}
}
// Only a terminal execution seals its manifest; mid-batch re-finishes must
// never freeze a partial snapshot (the seal below is first-write-wins).
switch state {
case "completed", "failed", "rolled_back", "rollback_failed":
default:
return nil
}
manifest := ComputeReleaseExecutionManifest(operation, changeSetManifest, lines)
if _, err := s.db.Exec(ctx, `UPDATE release_executions SET manifest_sha256 = $2 WHERE id = $1 AND manifest_sha256 = ''`, executionID, manifest); err != nil {
return fmt.Errorf("seal execution manifest: %w", err)
}
if state != "completed" || changeSetID == "" {
// Failed/rolled-back terminal states are honest audit facts; R4
// overall escalation is already recorded. Nothing further to verify.
return nil
}
// Success must equal the frozen change-set intent for the publications it
// actually touched.
frozen, err := s.changeSetItemActions(ctx, changeSetID)
if err != nil {
return err
}
for _, item := range items {
frozenAction, exists := frozen[item.publicationID]
// The frozen change set records PLAN actions (add/update/delete);
// executions carry EFFECTIVE actions (upsert/delete). Map equivalently.
consistent := exists && item.itemState == "verified" &&
((item.action == "delete" && frozenAction == "delete") ||
(item.action == "upsert" && (frozenAction == "add" || frozenAction == "update")))
if !consistent {
if _, err := s.db.Exec(ctx, `UPDATE release_executions SET state='failed', error_code='release_manifest_mismatch' WHERE id=$1 AND state='completed'`, executionID); err != nil {
return fmt.Errorf("downgrade manifest mismatch: %w", err)
}
return nil
}
}
return nil
}
func (s *Store) changeSetItemActions(ctx context.Context, changeSetID string) (map[string]string, error) {
rows, err := s.db.Query(ctx, `SELECT publication_id, action FROM release_change_set_items WHERE change_set_id = $1`, changeSetID)
if err != nil {
return nil, fmt.Errorf("load release change set items: %w", err)
}
defer rows.Close()
actions := map[string]string{}
for rows.Next() {
var publicationID, action string
if err := rows.Scan(&publicationID, &action); err != nil {
return nil, fmt.Errorf("scan release change set item: %w", err)
}
actions[publicationID] = action
}
return actions, rows.Err()
}
func validReleaseExecutionRequest(request releaseworkflow.StartExecutionRequest) bool {
return validShortID(request.ExecutionID) && validShortID(request.JobID) && request.Operation.Valid() && !request.Now.IsZero()
}
func validRemoteIdentifier(value string) bool {
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\")
}
func validRevision(value string) bool {
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t")
}
func validOptionalRevision(value string) bool {
return value == "" || validRevision(value)
}
func validLowerSHA256(value string) bool {
if len(value) != 64 {
return false
}
for _, char := range value {
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
return false
}
}
return true
}

Some files were not shown because too many files have changed in this diff Show More