feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
@@ -0,0 +1 @@
DROP TABLE IF EXISTS jobs;
@@ -0,0 +1,31 @@
CREATE TABLE IF NOT EXISTS schema_migrations (
version BIGINT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS jobs (
id VARCHAR(36) PRIMARY KEY,
type VARCHAR(100) NOT NULL,
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed')),
priority INTEGER NOT NULL DEFAULT 0,
available_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
lease_owner VARCHAR(255),
lease_until TIMESTAMPTZ,
attempt INTEGER NOT NULL DEFAULT 0 CHECK (attempt >= 0),
max_attempts INTEGER NOT NULL DEFAULT 5 CHECK (max_attempts > 0),
last_error TEXT NOT NULL DEFAULT '',
idempotency_key VARCHAR(255) NOT NULL,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (idempotency_key)
);
CREATE INDEX IF NOT EXISTS idx_jobs_claim
ON jobs (priority DESC, available_at, created_at, id)
WHERE state IN ('pending', 'retry', 'running');
CREATE INDEX IF NOT EXISTS idx_jobs_state_updated
ON jobs (state, updated_at DESC);
@@ -0,0 +1,5 @@
DROP TABLE IF EXISTS test_publications;
DROP TABLE IF EXISTS source_item_versions;
DROP TABLE IF EXISTS content_objects;
DROP TABLE IF EXISTS source_items;
DROP TABLE IF EXISTS source_connections;
@@ -0,0 +1,114 @@
CREATE TABLE IF NOT EXISTS source_connections (
id VARCHAR(36) PRIMARY KEY,
source_type VARCHAR(20) NOT NULL
CHECK (source_type IN ('pp', 'alist', 'dingtalk')),
name VARCHAR(255) NOT NULL,
base_url TEXT NOT NULL DEFAULT '',
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
enabled BOOLEAN NOT NULL DEFAULT FALSE,
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(schedule) = 'object'),
last_health JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(last_health) = 'object'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS source_items (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
stable_external_id TEXT NOT NULL,
canonical_uri_hash CHAR(64) NOT NULL
CHECK (canonical_uri_hash ~ '^[0-9a-f]{64}$'),
parent_external_id TEXT NOT NULL DEFAULT '',
normalized_path TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL,
item_type VARCHAR(20) NOT NULL
CHECK (item_type IN ('document', 'file', 'link', 'video')),
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
mime TEXT NOT NULL DEFAULT '',
revision TEXT NOT NULL DEFAULT '',
updated_at_source TIMESTAMPTZ,
visibility VARCHAR(20) NOT NULL DEFAULT 'internal'
CHECK (visibility IN ('internal', 'external')),
visibility_origin VARCHAR(30) NOT NULL DEFAULT 'manual'
CHECK (visibility_origin IN ('pp_public_tag', 'configured_scope', 'manual')),
last_seen_scan_id VARCHAR(36) NOT NULL DEFAULT '',
source_change VARCHAR(20) NOT NULL DEFAULT 'unchanged'
CHECK (source_change IN ('unchanged', 'added', 'updated', 'deleted', 'unavailable')),
absence_streak INTEGER NOT NULL DEFAULT 0 CHECK (absence_streak >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_connection_id, stable_external_id)
);
CREATE TABLE IF NOT EXISTS content_objects (
id VARCHAR(36) PRIMARY KEY,
sha256 CHAR(64) NOT NULL
CHECK (sha256 ~ '^[0-9a-f]{64}$'),
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
mime TEXT NOT NULL DEFAULT '',
validated_format VARCHAR(50) NOT NULL DEFAULT '',
local_spool_ref TEXT NOT NULL DEFAULT '',
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
first_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (sha256)
);
CREATE TABLE IF NOT EXISTS source_item_versions (
id VARCHAR(36) PRIMARY KEY,
source_item_id VARCHAR(36) NOT NULL
REFERENCES source_items(id) ON DELETE RESTRICT,
revision TEXT NOT NULL,
content_object_id VARCHAR(36)
REFERENCES content_objects(id) ON DELETE RESTRICT,
content_state VARCHAR(20) NOT NULL DEFAULT 'unknown'
CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')),
download_error TEXT NOT NULL DEFAULT '',
raw_record_summary JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(raw_record_summary) = 'object'),
source_updated_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_item_id, revision)
);
CREATE TABLE IF NOT EXISTS test_publications (
id VARCHAR(36) PRIMARY KEY,
source_item_version_id VARCHAR(36) NOT NULL
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
test_kb_id VARCHAR(255) NOT NULL,
knowledge_id VARCHAR(255) NOT NULL DEFAULT '',
ingest_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (ingest_state IN ('pending', 'uploading', 'uploaded', 'failed')),
processing_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')),
retrieval_state VARCHAR(20) NOT NULL DEFAULT 'unpublished'
CHECK (retrieval_state IN ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed')),
parser_generation VARCHAR(255) NOT NULL DEFAULT '',
index_generation VARCHAR(255) NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_item_version_id, test_kb_id)
);
CREATE INDEX IF NOT EXISTS idx_source_items_change
ON source_items (source_connection_id, source_change, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_source_items_visibility
ON source_items (visibility, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_source_item_versions_content
ON source_item_versions (content_object_id)
WHERE content_object_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_test_publications_state
ON test_publications (processing_state, retrieval_state, updated_at DESC);
@@ -0,0 +1,12 @@
DROP TABLE IF EXISTS acceptance_results;
DROP INDEX IF EXISTS idx_test_publications_poll;
ALTER TABLE test_publications DROP COLUMN IF EXISTS superseded_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS supersedes_publication_id;
ALTER TABLE test_publications DROP COLUMN IF EXISTS stalled_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS completed_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS processing_started_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_observed_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_updated_at;
ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_status;
ALTER TABLE test_publications DROP COLUMN IF EXISTS poll_sequence;
@@ -0,0 +1,33 @@
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS poll_sequence INTEGER NOT NULL DEFAULT 0
CHECK (poll_sequence >= 0);
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_status VARCHAR(32) NOT NULL DEFAULT '';
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_updated_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_observed_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS stalled_at TIMESTAMPTZ;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS supersedes_publication_id VARCHAR(36)
REFERENCES test_publications(id) ON DELETE RESTRICT;
ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS superseded_at TIMESTAMPTZ;
CREATE TABLE IF NOT EXISTS acceptance_results (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
run_id VARCHAR(36) NOT NULL,
check_type VARCHAR(32) NOT NULL
CHECK (check_type IN ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches')),
state VARCHAR(20) NOT NULL
CHECK (state IN ('pending', 'passed', 'failed', 'manual_required')),
policy_version VARCHAR(64) NOT NULL,
evidence_summary JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(evidence_summary) = 'object'),
observed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id, run_id, check_type)
);
CREATE INDEX IF NOT EXISTS idx_test_publications_poll
ON test_publications (processing_state, last_observed_at, updated_at);
CREATE INDEX IF NOT EXISTS idx_acceptance_results_publication
ON acceptance_results (publication_id, observed_at DESC);
@@ -0,0 +1,18 @@
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM source_connections
WHERE source_type = 'manual'
) THEN
RAISE EXCEPTION 'cannot downgrade while manual source connections exist';
END IF;
END
$$;
ALTER TABLE source_connections
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
ALTER TABLE source_connections
ADD CONSTRAINT source_connections_source_type_check
CHECK (source_type IN ('pp', 'alist', 'dingtalk'));
@@ -0,0 +1,6 @@
ALTER TABLE source_connections
DROP CONSTRAINT IF EXISTS source_connections_source_type_check;
ALTER TABLE source_connections
ADD CONSTRAINT source_connections_source_type_check
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual'));
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS source_scan_changes;
DROP TABLE IF EXISTS source_scans;
@@ -0,0 +1,43 @@
CREATE TABLE IF NOT EXISTS source_scans (
id VARCHAR(64) PRIMARY KEY,
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
source_type VARCHAR(32) NOT NULL CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
state VARCHAR(16) NOT NULL CHECK (state IN ('pending', 'running', 'completed', 'failed')),
snapshot_sha256 VARCHAR(64) NOT NULL DEFAULT '' CHECK (snapshot_sha256 = '' OR snapshot_sha256 ~ '^[0-9a-f]{64}$'),
observed_items INTEGER NOT NULL DEFAULT 0 CHECK (observed_items >= 0),
unchanged_items INTEGER NOT NULL DEFAULT 0 CHECK (unchanged_items >= 0),
added_items INTEGER NOT NULL DEFAULT 0 CHECK (added_items >= 0),
updated_items INTEGER NOT NULL DEFAULT 0 CHECK (updated_items >= 0),
over_limit_items INTEGER NOT NULL DEFAULT 0 CHECK (over_limit_items >= 0),
missing_items INTEGER NOT NULL DEFAULT 0 CHECK (missing_items >= 0),
planned_uploads INTEGER NOT NULL DEFAULT 0 CHECK (planned_uploads >= 0),
error_code VARCHAR(64) NOT NULL DEFAULT '',
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_scans_one_active
ON source_scans (source_connection_id)
WHERE state IN ('pending', 'running');
CREATE INDEX IF NOT EXISTS idx_source_scans_connection_created
ON source_scans (source_connection_id, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS source_scan_changes (
id VARCHAR(64) PRIMARY KEY,
source_scan_id VARCHAR(64) NOT NULL REFERENCES source_scans(id) ON DELETE RESTRICT,
source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT,
stable_external_id VARCHAR(512) NOT NULL,
source_item_id VARCHAR(64) REFERENCES source_items(id) ON DELETE RESTRICT,
change_type VARCHAR(16) NOT NULL CHECK (change_type IN ('added', 'updated', 'unavailable')),
previous_revision VARCHAR(512) NOT NULL DEFAULT '',
observed_revision VARCHAR(512) NOT NULL DEFAULT '',
observed_size_bytes BIGINT CHECK (observed_size_bytes IS NULL OR observed_size_bytes >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_scan_id, stable_external_id)
);
CREATE INDEX IF NOT EXISTS idx_source_scan_changes_pending
ON source_scan_changes (source_connection_id, change_type, created_at DESC, id DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS release_batch_items;
DROP TABLE IF EXISTS release_batches;
DROP TABLE IF EXISTS publication_reviews;
@@ -0,0 +1,58 @@
CREATE TABLE IF NOT EXISTS publication_reviews (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
decision VARCHAR(16) NOT NULL
CHECK (decision IN ('approved', 'rejected')),
decided_visibility VARCHAR(20) NOT NULL
CHECK (decided_visibility IN ('internal', 'external')),
reviewer_ref VARCHAR(128) NOT NULL,
decision_code VARCHAR(64) NOT NULL
CHECK (decision_code ~ '^[a-z0-9_]+$'),
acceptance_run_id VARCHAR(36) NOT NULL,
publication_updated_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id, acceptance_run_id, publication_updated_at,
reviewer_ref, decision, decided_visibility, decision_code)
);
CREATE INDEX IF NOT EXISTS idx_publication_reviews_latest
ON publication_reviews (publication_id, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS release_batches (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
state VARCHAR(16) NOT NULL DEFAULT 'draft'
CHECK (state IN ('draft', 'sealed', 'cancelled')),
policy_version VARCHAR(64) NOT NULL,
created_by VARCHAR(128) NOT NULL,
sealed_by VARCHAR(128) NOT NULL DEFAULT '',
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
sealed_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_release_batches_state_created
ON release_batches (state, created_at DESC, id DESC);
CREATE TABLE IF NOT EXISTS release_batch_items (
release_batch_id VARCHAR(36) NOT NULL
REFERENCES release_batches(id) ON DELETE RESTRICT,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
review_id VARCHAR(36) NOT NULL
REFERENCES publication_reviews(id) ON DELETE RESTRICT,
source_item_version_id VARCHAR(36) NOT NULL
REFERENCES source_item_versions(id) ON DELETE RESTRICT,
acceptance_run_id VARCHAR(36) NOT NULL,
publication_updated_at TIMESTAMPTZ NOT NULL,
decided_visibility VARCHAR(20) NOT NULL
CHECK (decided_visibility IN ('internal', 'external')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (release_batch_id, publication_id),
UNIQUE (publication_id),
UNIQUE (review_id)
);
CREATE INDEX IF NOT EXISTS idx_release_batch_items_batch
ON release_batch_items (release_batch_id, publication_id);
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS release_item_executions;
DROP TABLE IF EXISTS release_executions;
@@ -0,0 +1,65 @@
CREATE TABLE IF NOT EXISTS release_executions (
id VARCHAR(36) PRIMARY KEY,
release_batch_id VARCHAR(36) NOT NULL
REFERENCES release_batches(id) ON DELETE RESTRICT,
operation VARCHAR(16) NOT NULL
CHECK (operation IN ('publish', 'rollback')),
prior_execution_id VARCHAR(36)
REFERENCES release_executions(id) ON DELETE RESTRICT,
state VARCHAR(16) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'running', 'completed', 'failed')),
target_environment VARCHAR(16) NOT NULL DEFAULT 'production'
CHECK (target_environment = 'production'),
authorized_by VARCHAR(128) NOT NULL,
authorization_code VARCHAR(64) NOT NULL,
job_id VARCHAR(36) NOT NULL UNIQUE
REFERENCES jobs(id) ON DELETE RESTRICT,
manifest_sha256 CHAR(64) NOT NULL DEFAULT ''
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
completed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (completed_item_count >= 0),
failed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (failed_item_count >= 0),
error_code VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
CHECK (operation = 'publish' OR prior_execution_id IS NOT NULL),
CHECK (completed_item_count + failed_item_count <= item_count)
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_publish_batch
ON release_executions (release_batch_id)
WHERE operation = 'publish';
CREATE UNIQUE INDEX IF NOT EXISTS uq_release_rollback_prior
ON release_executions (prior_execution_id)
WHERE operation = 'rollback';
CREATE INDEX IF NOT EXISTS idx_release_executions_batch_created
ON release_executions (release_batch_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_release_executions_state_created
ON release_executions (state, created_at);
CREATE TABLE IF NOT EXISTS release_item_executions (
id VARCHAR(36) PRIMARY KEY,
release_execution_id VARCHAR(36) NOT NULL
REFERENCES release_executions(id) ON DELETE RESTRICT,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
state VARCHAR(24) NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'staged', 'activated', 'verified', 'failed', 'rolled_back')),
bundle_sha256 CHAR(64) NOT NULL DEFAULT ''
CHECK (bundle_sha256 = '' OR bundle_sha256 ~ '^[0-9a-f]{64}$'),
previous_production_revision VARCHAR(128) NOT NULL DEFAULT '',
production_revision VARCHAR(128) NOT NULL DEFAULT '',
error_code VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
UNIQUE (release_execution_id, publication_id)
);
CREATE INDEX IF NOT EXISTS idx_release_item_executions_execution
ON release_item_executions (release_execution_id, created_at);
CREATE INDEX IF NOT EXISTS idx_release_item_executions_publication
ON release_item_executions (publication_id, created_at DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS scope_change_events;
DROP TABLE IF EXISTS source_item_scope_overrides;
DROP TABLE IF EXISTS directory_scope_rules;
@@ -0,0 +1,57 @@
CREATE TABLE IF NOT EXISTS directory_scope_rules (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
directory_path TEXT NOT NULL DEFAULT '',
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external')),
updated_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, directory_path)
);
CREATE TABLE IF NOT EXISTS source_item_scope_overrides (
source_item_id VARCHAR(36) NOT NULL
REFERENCES source_items(id) ON DELETE RESTRICT,
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external')),
updated_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_item_id)
);
CREATE TABLE IF NOT EXISTS scope_change_events (
id VARCHAR(36) PRIMARY KEY,
target_type VARCHAR(16) NOT NULL
CHECK (target_type IN ('directory', 'item')),
source_connection_id VARCHAR(36)
REFERENCES source_connections(id) ON DELETE RESTRICT,
directory_path TEXT NOT NULL DEFAULT '',
source_item_id VARCHAR(36)
REFERENCES source_items(id) ON DELETE RESTRICT,
action VARCHAR(16) NOT NULL
CHECK (action IN ('set', 'clear')),
previous_visibility VARCHAR(20)
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external')),
decided_visibility VARCHAR(20)
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external')),
operator_ref VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CHECK (
(target_type = 'directory' AND source_connection_id IS NOT NULL AND source_item_id IS NULL)
OR
(target_type = 'item' AND source_connection_id IS NULL AND source_item_id IS NOT NULL)
),
CHECK (
(action = 'set' AND decided_visibility IS NOT NULL)
OR
(action = 'clear' AND decided_visibility IS NULL)
)
);
CREATE INDEX IF NOT EXISTS idx_scope_change_events_target
ON scope_change_events (target_type, source_connection_id, source_item_id, created_at DESC);
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS source_catalog_heads;
DROP TABLE IF EXISTS source_catalog_nodes;
DROP TABLE IF EXISTS source_catalog_snapshots;
@@ -0,0 +1,56 @@
CREATE TABLE IF NOT EXISTS source_catalog_snapshots (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
snapshot_sha256 CHAR(64) NOT NULL
CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'),
source_label VARCHAR(255) NOT NULL,
node_count INTEGER NOT NULL CHECK (node_count >= 1),
folder_count INTEGER NOT NULL CHECK (folder_count >= 1),
file_count INTEGER NOT NULL CHECK (file_count >= 0),
observed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (source_connection_id, snapshot_sha256),
UNIQUE (source_connection_id, id)
);
CREATE TABLE IF NOT EXISTS source_catalog_nodes (
snapshot_id VARCHAR(36) NOT NULL
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
node_id VARCHAR(36) NOT NULL,
parent_node_id VARCHAR(36),
node_type VARCHAR(16) NOT NULL
CHECK (node_type IN ('folder', 'file')),
name TEXT NOT NULL CHECK (name <> '' AND length(name) <= 1024),
relative_path TEXT NOT NULL CHECK (length(relative_path) <= 4096),
source_locator_hash CHAR(64) NOT NULL
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
source_item_id VARCHAR(36)
REFERENCES source_items(id) ON DELETE RESTRICT,
size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0),
modified_at TIMESTAMPTZ,
sort_order INTEGER NOT NULL DEFAULT 0,
direct_child_count INTEGER NOT NULL DEFAULT 0 CHECK (direct_child_count >= 0),
subtree_file_count INTEGER NOT NULL DEFAULT 0 CHECK (subtree_file_count >= 0),
subtree_bytes BIGINT NOT NULL DEFAULT 0 CHECK (subtree_bytes >= 0),
PRIMARY KEY (snapshot_id, node_id),
FOREIGN KEY (snapshot_id, parent_node_id)
REFERENCES source_catalog_nodes(snapshot_id, node_id) ON DELETE RESTRICT
DEFERRABLE INITIALLY DEFERRED
);
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_parent
ON source_catalog_nodes (snapshot_id, parent_node_id, node_type, sort_order, name, node_id);
CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_source_item
ON source_catalog_nodes (source_item_id)
WHERE source_item_id IS NOT NULL;
CREATE TABLE IF NOT EXISTS source_catalog_heads (
source_connection_id VARCHAR(36) PRIMARY KEY
REFERENCES source_connections(id) ON DELETE RESTRICT,
snapshot_id VARCHAR(36) NOT NULL,
activated_at TIMESTAMPTZ NOT NULL,
FOREIGN KEY (source_connection_id, snapshot_id)
REFERENCES source_catalog_snapshots(source_connection_id, id) ON DELETE RESTRICT
);
@@ -0,0 +1,36 @@
UPDATE scope_change_events
SET previous_visibility = 'internal'
WHERE previous_visibility IN ('both', 'blocked');
UPDATE scope_change_events
SET decided_visibility = 'internal'
WHERE decided_visibility IN ('both', 'blocked');
UPDATE source_item_scope_overrides
SET visibility = 'internal'
WHERE visibility IN ('both', 'blocked');
UPDATE directory_scope_rules
SET visibility = 'internal'
WHERE visibility IN ('both', 'blocked');
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_decided_visibility_check
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_previous_visibility_check
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external'));
ALTER TABLE source_item_scope_overrides
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
ALTER TABLE source_item_scope_overrides
ADD CONSTRAINT source_item_scope_overrides_visibility_check
CHECK (visibility IN ('internal', 'external'));
ALTER TABLE directory_scope_rules
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
ALTER TABLE directory_scope_rules
ADD CONSTRAINT directory_scope_rules_visibility_check
CHECK (visibility IN ('internal', 'external'));
@@ -0,0 +1,25 @@
-- Visibility policies used by the source catalog. Publication reviews remain
-- scoped to one concrete internal or external test publication.
ALTER TABLE directory_scope_rules
DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check;
ALTER TABLE directory_scope_rules
ADD CONSTRAINT directory_scope_rules_visibility_check
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE source_item_scope_overrides
DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check;
ALTER TABLE source_item_scope_overrides
ADD CONSTRAINT source_item_scope_overrides_visibility_check
CHECK (visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_previous_visibility_check
CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external', 'both', 'blocked'));
ALTER TABLE scope_change_events
DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check;
ALTER TABLE scope_change_events
ADD CONSTRAINT scope_change_events_decided_visibility_check
CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external', 'both', 'blocked'));
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS processing_confirmations;
ALTER TABLE test_publications DROP COLUMN IF EXISTS target_scope;
@@ -0,0 +1,36 @@
ALTER TABLE test_publications
ADD COLUMN IF NOT EXISTS target_scope VARCHAR(16) NOT NULL DEFAULT 'internal'
CHECK (target_scope IN ('internal', 'external'));
UPDATE test_publications AS publication
SET target_scope = COALESCE(
(
SELECT review.decided_visibility
FROM publication_reviews AS review
WHERE review.publication_id = publication.id
AND review.decision = 'approved'
AND review.decided_visibility IN ('internal', 'external')
ORDER BY review.created_at DESC, review.id DESC
LIMIT 1
),
CASE WHEN item_rule.visibility IN ('internal', 'external') THEN item_rule.visibility END,
source.visibility
)
FROM source_item_versions AS version
JOIN source_items AS source ON source.id = version.source_item_id
LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id
WHERE publication.source_item_version_id = version.id;
CREATE TABLE IF NOT EXISTS processing_confirmations (
id VARCHAR(36) PRIMARY KEY,
publication_id VARCHAR(36) NOT NULL
REFERENCES test_publications(id) ON DELETE RESTRICT,
operator_ref VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL,
confirmed_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (publication_id)
);
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_confirmed
ON processing_confirmations (confirmed_at DESC, publication_id);
@@ -0,0 +1,7 @@
ALTER TABLE release_batch_items
ADD CONSTRAINT release_batch_items_publication_id_key UNIQUE (publication_id);
ALTER TABLE release_batch_items
ADD CONSTRAINT release_batch_items_review_id_key UNIQUE (review_id);
ALTER TABLE release_item_executions DROP COLUMN IF EXISTS action;
ALTER TABLE release_batch_items DROP COLUMN IF EXISTS action;
@@ -0,0 +1,12 @@
ALTER TABLE release_batch_items
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
CHECK (action IN ('upsert', 'delete'));
ALTER TABLE release_item_executions
ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert'
CHECK (action IN ('upsert', 'delete'));
ALTER TABLE release_batch_items
DROP CONSTRAINT IF EXISTS release_batch_items_publication_id_key;
ALTER TABLE release_batch_items
DROP CONSTRAINT IF EXISTS release_batch_items_review_id_key;
@@ -0,0 +1 @@
DROP TABLE IF EXISTS ui_setting_revisions;
@@ -0,0 +1,24 @@
CREATE TABLE IF NOT EXISTS ui_setting_revisions (
id VARCHAR(36) PRIMARY KEY,
state VARCHAR(16) NOT NULL CHECK (state IN ('validated', 'active', 'retired', 'failed')),
document_refresh_seconds INTEGER NOT NULL CHECK (document_refresh_seconds IN (15, 30, 60)),
resource_refresh_seconds INTEGER NOT NULL CHECK (resource_refresh_seconds IN (30, 60)),
stalled_after_seconds INTEGER NOT NULL CHECK (stalled_after_seconds IN (1800, 3600)),
finalizing_after_seconds INTEGER NOT NULL CHECK (finalizing_after_seconds IN (7200, 14400)),
updated_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
activated_at TIMESTAMPTZ,
retired_at TIMESTAMPTZ,
error_code VARCHAR(64) NOT NULL DEFAULT '',
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$')
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_ui_setting_active
ON ui_setting_revisions ((state)) WHERE state = 'active';
INSERT INTO ui_setting_revisions (
id, state, document_refresh_seconds, resource_refresh_seconds,
stalled_after_seconds, finalizing_after_seconds, updated_by, activated_at
) VALUES (
'settings-default', 'active', 15, 30, 1800, 7200, 'system:migration', NOW()
) ON CONFLICT (id) DO NOTHING;
@@ -0,0 +1,2 @@
ALTER TABLE jobs
DROP COLUMN IF EXISTS started_at;
@@ -0,0 +1,2 @@
ALTER TABLE jobs
ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS source_catalog_change_items;
DROP TABLE IF EXISTS source_catalog_change_sets;
@@ -0,0 +1,42 @@
CREATE TABLE IF NOT EXISTS source_catalog_change_sets (
id VARCHAR(36) PRIMARY KEY,
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE RESTRICT,
from_snapshot_id VARCHAR(36)
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
to_snapshot_id VARCHAR(36) NOT NULL UNIQUE
REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
state VARCHAR(16) NOT NULL DEFAULT 'planned'
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled')),
add_count INTEGER NOT NULL DEFAULT 0 CHECK (add_count >= 0),
update_count INTEGER NOT NULL DEFAULT 0 CHECK (update_count >= 0),
move_scope_count INTEGER NOT NULL DEFAULT 0 CHECK (move_scope_count >= 0),
delete_count INTEGER NOT NULL DEFAULT 0 CHECK (delete_count >= 0),
no_change_count INTEGER NOT NULL DEFAULT 0 CHECK (no_change_count >= 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
confirmed_at TIMESTAMPTZ,
applied_at TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS source_catalog_change_items (
change_set_id VARCHAR(36) NOT NULL
REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
source_locator_hash CHAR(64) NOT NULL
CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'),
action VARCHAR(16) NOT NULL
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change')),
title TEXT NOT NULL,
old_path TEXT NOT NULL DEFAULT '',
new_path TEXT NOT NULL DEFAULT '',
old_size_bytes BIGINT CHECK (old_size_bytes IS NULL OR old_size_bytes >= 0),
new_size_bytes BIGINT CHECK (new_size_bytes IS NULL OR new_size_bytes >= 0),
old_modified_at TIMESTAMPTZ,
new_modified_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, source_locator_hash)
);
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_sets_source
ON source_catalog_change_sets (source_connection_id, created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_source_catalog_change_items_action
ON source_catalog_change_items (change_set_id, action, title);
@@ -0,0 +1,4 @@
DELETE FROM source_catalog_change_items
WHERE change_set_id IN (SELECT id FROM source_catalog_change_sets WHERE origin='backfill');
DELETE FROM source_catalog_change_sets WHERE origin='backfill';
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS origin;
@@ -0,0 +1,87 @@
ALTER TABLE source_catalog_change_sets
ADD COLUMN IF NOT EXISTS origin VARCHAR(16) NOT NULL DEFAULT 'refresh'
CHECK (origin IN ('refresh', 'backfill'));
WITH current_targets AS (
SELECT head.source_connection_id,head.snapshot_id AS to_snapshot_id,
(
SELECT prior.id
FROM source_catalog_snapshots prior
JOIN source_catalog_snapshots current ON current.id=head.snapshot_id
WHERE prior.source_connection_id=head.source_connection_id
AND prior.id<>head.snapshot_id
AND prior.observed_at<=current.observed_at
ORDER BY prior.observed_at DESC,prior.id DESC
LIMIT 1
) AS from_snapshot_id
FROM source_catalog_heads head
)
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,origin,created_at)
SELECT 'cs-'||md5(target.source_connection_id||'|'||target.to_snapshot_id),
target.source_connection_id,target.from_snapshot_id,target.to_snapshot_id,'planned','backfill',NOW()
FROM current_targets target
ON CONFLICT (to_snapshot_id) DO NOTHING;
WITH change_sets AS (
SELECT id,from_snapshot_id,to_snapshot_id
FROM source_catalog_change_sets
WHERE origin='backfill'
), old_files AS (
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
FROM change_sets change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (change_set.id,node.source_locator_hash)
change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at
FROM change_sets change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id
), compared AS (
SELECT COALESCE(new_files.change_set_id,old_files.change_set_id) AS change_set_id,
COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
COALESCE(new_files.name,old_files.name) AS title,
COALESCE(old_files.relative_path,'') AS old_path,
COALESCE(new_files.relative_path,'') AS new_path,
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
CASE
WHEN old_files.source_locator_hash IS NULL THEN 'add'
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
ELSE 'no_change'
END AS action
FROM old_files FULL OUTER JOIN new_files
ON new_files.change_set_id=old_files.change_set_id
AND new_files.source_locator_hash=old_files.source_locator_hash
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,created_at)
SELECT change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,NOW()
FROM compared
ON CONFLICT (change_set_id,source_locator_hash) DO NOTHING;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT change_set.id AS change_set_id,
COUNT(item.*) FILTER (WHERE item.action='add')::integer AS add_count,
COUNT(item.*) FILTER (WHERE item.action='update')::integer AS update_count,
COUNT(item.*) FILTER (WHERE item.action='move_scope')::integer AS move_scope_count,
COUNT(item.*) FILTER (WHERE item.action='delete')::integer AS delete_count,
COUNT(item.*) FILTER (WHERE item.action='no_change')::integer AS no_change_count
FROM source_catalog_change_sets change_set
LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
WHERE change_set.origin='backfill'
GROUP BY change_set.id
) summary
WHERE change_set.id=summary.change_set_id;
@@ -0,0 +1,5 @@
ALTER TABLE release_executions
DROP CONSTRAINT IF EXISTS release_execution_batch_progress_check,
DROP COLUMN IF EXISTS total_batches,
DROP COLUMN IF EXISTS current_batch,
DROP COLUMN IF EXISTS batch_size;
@@ -0,0 +1,16 @@
ALTER TABLE release_executions
ADD COLUMN IF NOT EXISTS batch_size INTEGER NOT NULL DEFAULT 50
CHECK (batch_size BETWEEN 1 AND 100),
ADD COLUMN IF NOT EXISTS current_batch INTEGER NOT NULL DEFAULT 0
CHECK (current_batch >= 0),
ADD COLUMN IF NOT EXISTS total_batches INTEGER NOT NULL DEFAULT 0
CHECK (total_batches >= 0),
ADD CONSTRAINT release_execution_batch_progress_check
CHECK (current_batch <= total_batches OR total_batches = 0);
UPDATE release_executions
SET total_batches = CASE WHEN item_count = 0 THEN 0 ELSE CEIL(item_count::numeric / batch_size)::integer END,
current_batch = CASE
WHEN state IN ('completed', 'failed') AND item_count > 0 THEN CEIL(item_count::numeric / batch_size)::integer
ELSE current_batch
END;
@@ -0,0 +1,11 @@
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM jobs WHERE state = 'cancelled') THEN
RAISE EXCEPTION 'cannot roll back job cancelled state while cancelled jobs exist';
END IF;
END
$$;
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed'));
@@ -0,0 +1,3 @@
ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_state_check
CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed', 'cancelled'));
@@ -0,0 +1 @@
DROP TABLE IF EXISTS operation_requests;
@@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS operation_requests (
operation_type TEXT NOT NULL CHECK (operation_type <> '' AND char_length(operation_type) <= 100),
operation_id TEXT NOT NULL CHECK (operation_id <> '' AND char_length(operation_id) <= 128),
request_fingerprint_sha256 CHAR(64) NOT NULL CHECK (request_fingerprint_sha256 ~ '^[0-9a-f]{64}$'),
state TEXT NOT NULL DEFAULT 'completed' CHECK (state IN ('completed')),
result JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ NOT NULL,
UNIQUE (operation_type, operation_id)
);
@@ -0,0 +1 @@
ALTER TABLE source_catalog_nodes DROP COLUMN IF EXISTS content_sha256;
@@ -0,0 +1,3 @@
ALTER TABLE source_catalog_nodes
ADD COLUMN IF NOT EXISTS content_sha256 CHAR(64)
CHECK (content_sha256 IS NULL OR content_sha256 ~ '^[0-9a-f]{64}$');
@@ -0,0 +1,2 @@
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS reason_code, DROP COLUMN IF EXISTS new_content_sha256, DROP COLUMN IF EXISTS old_content_sha256;
ALTER TABLE source_items DROP COLUMN IF EXISTS identity_confidence;
@@ -0,0 +1,12 @@
-- P2 stable identity: source items record how strongly their identity is anchored.
ALTER TABLE source_items
ADD COLUMN IF NOT EXISTS identity_confidence VARCHAR(16) NOT NULL DEFAULT 'path'
CHECK (identity_confidence IN ('stable_id','object_id','content_hash','path'));
-- Change items gain content-hash evidence and a reason code so a rename can be
-- classified as a proven move instead of delete+add when the bytes match 1:1.
ALTER TABLE source_catalog_change_items
ADD COLUMN IF NOT EXISTS old_content_sha256 CHAR(64) CHECK (old_content_sha256 IS NULL OR old_content_sha256 ~ '^[0-9a-f]{64}$'),
ADD COLUMN IF NOT EXISTS new_content_sha256 CHAR(64) CHECK (new_content_sha256 IS NULL OR new_content_sha256 ~ '^[0-9a-f]{64}$'),
ADD COLUMN IF NOT EXISTS reason_code VARCHAR(32) NOT NULL DEFAULT 'locator'
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
@@ -0,0 +1 @@
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
@@ -0,0 +1,99 @@
-- P2/S8: canonical snapshot-diff classifier lives in the database so the
-- importer, backfills and future ChangeSet execution share one source of truth.
-- Two sequential statements (not a data-modifying CTE) so the recount sees the rows just written.
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
WITH change_set AS (
SELECT id,from_snapshot_id,to_snapshot_id FROM source_catalog_change_sets
WHERE id=p_change_set_id
), old_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), proven_moves AS (
-- A content hash proves a move only when exactly one old path carries it and
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
SELECT content_sha256 FROM (
SELECT old_files.content_sha256 FROM old_files WHERE old_files.content_sha256 IS NOT NULL
GROUP BY old_files.content_sha256 HAVING COUNT(*)=1
) one_old
JOIN (
SELECT new_files.content_sha256 FROM new_files WHERE new_files.content_sha256 IS NOT NULL
GROUP BY new_files.content_sha256 HAVING COUNT(*)=1
) one_new USING (content_sha256)
), compared AS (
SELECT COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash,
COALESCE(new_files.name,old_files.name) AS title,
COALESCE(old_files.relative_path,'') AS old_path,
COALESCE(new_files.relative_path,'') AS new_path,
old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes,
old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at,
old_files.content_sha256 AS old_content_sha256,new_files.content_sha256 AS new_content_sha256,
CASE
WHEN old_files.source_locator_hash IS NULL THEN 'add'
WHEN new_files.source_locator_hash IS NULL THEN 'delete'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path
OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes
OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update'
ELSE 'no_change'
END AS action,
CASE
WHEN old_files.source_locator_hash IS NULL AND new_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN new_files.source_locator_hash IS NULL AND old_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN old_files.source_locator_hash IS NULL THEN 'locator'
WHEN new_files.source_locator_hash IS NULL THEN 'source_missing'
WHEN old_files.content_sha256 IS DISTINCT FROM new_files.content_sha256
AND (old_files.relative_path<>new_files.relative_path OR old_files.name<>new_files.name)
THEN 'content_hash_conflict'
WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path OR old_files.name IS DISTINCT FROM new_files.name
THEN 'path_change'
WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes THEN 'size_change'
WHEN old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'content_change'
ELSE 'locator'
END AS reason_code
FROM old_files FULL OUTER JOIN new_files USING(source_locator_hash)
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,created_at)
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,NOW()
FROM change_set CROSS JOIN compared
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
reason_code=EXCLUDED.reason_code;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT items.change_set_id,
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
FROM source_catalog_change_items items
WHERE items.change_set_id=p_change_set_id
GROUP BY items.change_set_id
) summary
WHERE change_set.id=summary.change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -0,0 +1,26 @@
-- Revert S8 contract. Change rows written with new vocabulary remain but become
-- unclassifiable only if constraints reject them; we keep the checks permissive
-- first, then drop added columns.
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_version_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_version_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_source_item_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_source_item_id;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_scope;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_scope;
UPDATE source_catalog_change_items SET action='move_scope' WHERE action='move';
UPDATE source_catalog_change_items SET reason_code='path_change' WHERE reason_code='scope_change';
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing'));
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change'));
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS failure_reason;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS cancelled_at;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS applying_at;
UPDATE source_catalog_change_sets SET move_scope_count=move_scope_count+move_count, move_count=0;
ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS move_count;
UPDATE source_catalog_change_sets SET state='confirmed' WHERE state IN ('applying','failed');
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled'));
@@ -0,0 +1,167 @@
-- S8 complete ChangeSet contract: full lifecycle, distinct move vs move_scope,
-- old/new effective scope, and version identity evidence.
ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check;
ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check
CHECK (state IN ('planned', 'confirmed', 'applying', 'applied', 'cancelled', 'failed'));
ALTER TABLE source_catalog_change_sets
ADD COLUMN IF NOT EXISTS move_count INTEGER NOT NULL DEFAULT 0 CHECK (move_count >= 0),
ADD COLUMN IF NOT EXISTS applying_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS cancelled_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS failure_reason TEXT NOT NULL DEFAULT '';
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check
CHECK (action IN ('add', 'update', 'move', 'move_scope', 'delete', 'no_change'));
ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check;
ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check
CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing','scope_change'));
ALTER TABLE source_catalog_change_items
ADD COLUMN IF NOT EXISTS old_scope VARCHAR(16),
ADD COLUMN IF NOT EXISTS new_scope VARCHAR(16),
ADD COLUMN IF NOT EXISTS old_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS new_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS old_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS new_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT;
-- Canonical classifier v2 (single source of truth; importer and backfills call it).
CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
WITH change_set AS (
SELECT id,from_snapshot_id,to_snapshot_id,source_connection_id FROM source_catalog_change_sets
WHERE id=p_change_set_id
), old_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_item_id
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
), new_files AS (
SELECT DISTINCT ON (node.source_locator_hash)
node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_item_id
FROM change_set
JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file'
ORDER BY node.source_locator_hash,node.relative_path,node.node_id
)
-- Old side reports the source-recorded visibility (what the last sync stored);
-- new side reports the currently effective policy (override > item). A policy
-- change between snapshots therefore surfaces as move_scope, not a fake move.
, scoped_old AS (
SELECT old_files.*,
(SELECT item.visibility FROM source_items item WHERE item.id=old_files.source_item_id) AS item_scope
FROM old_files
), scoped_new AS (
SELECT new_files.*,
(SELECT COALESCE(override.visibility,item.visibility)
FROM source_items item
LEFT JOIN source_item_scope_overrides override ON override.source_item_id=item.id
WHERE item.id=new_files.source_item_id) AS item_scope
FROM new_files
), with_rule_scope AS (
SELECT f.*,COALESCE(f.item_scope,(
SELECT rule.visibility FROM directory_scope_rules rule
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
FROM (SELECT * FROM scoped_old) f
), with_rule_scope_new AS (
SELECT f.*,COALESCE(f.item_scope,(
SELECT rule.visibility FROM directory_scope_rules rule
WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set)
AND (rule.directory_path='' OR f.relative_path=rule.directory_path
OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/')
ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope
FROM (SELECT * FROM scoped_new) f
), proven_moves AS (
-- A content hash proves a move only when exactly one old path carries it and
-- exactly one new path does; copies stay ambiguous and are never auto-merged.
SELECT content_sha256 FROM (
SELECT content_sha256 FROM scoped_old WHERE content_sha256 IS NOT NULL
GROUP BY content_sha256 HAVING COUNT(*)=1
) one_old
JOIN (
SELECT content_sha256 FROM scoped_new WHERE content_sha256 IS NOT NULL
GROUP BY content_sha256 HAVING COUNT(*)=1
) one_new USING (content_sha256)
), compared AS (
SELECT COALESCE(n.source_locator_hash,o.source_locator_hash) AS source_locator_hash,
COALESCE(n.name,o.name) AS title,
COALESCE(o.relative_path,'') AS old_path,
COALESCE(n.relative_path,'') AS new_path,
o.size_bytes AS old_size_bytes,n.size_bytes AS new_size_bytes,
o.modified_at AS old_modified_at,n.modified_at AS new_modified_at,
o.content_sha256 AS old_content_sha256,n.content_sha256 AS new_content_sha256,
o.effective_scope AS old_scope,n.effective_scope AS new_scope,
o.source_item_id AS old_source_item_id,n.source_item_id AS new_source_item_id,
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=o.source_item_id AND i.revision=v.revision WHERE v.source_item_id=o.source_item_id LIMIT 1) AS old_version_id,
(SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=n.source_item_id AND i.revision=v.revision WHERE v.source_item_id=n.source_item_id LIMIT 1) AS new_version_id,
CASE
WHEN o.source_locator_hash IS NULL THEN 'add'
WHEN n.source_locator_hash IS NULL THEN 'delete'
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'move_scope'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256
AND (o.relative_path<>n.relative_path OR o.name<>n.name) THEN 'move'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256<>n.content_sha256 THEN 'update'
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'move_scope'
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes OR o.modified_at IS DISTINCT FROM n.modified_at THEN 'update'
ELSE 'no_change'
END AS action,
CASE
WHEN o.source_locator_hash IS NULL AND n.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN n.source_locator_hash IS NULL AND o.content_sha256 IN (SELECT content_sha256 FROM proven_moves)
THEN 'content_hash_move'
WHEN o.source_locator_hash IS NULL THEN 'locator'
WHEN n.source_locator_hash IS NULL THEN 'source_missing'
WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'scope_change'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256 THEN 'content_hash_move'
WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL THEN 'content_hash_conflict'
WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'path_change'
WHEN o.size_bytes IS DISTINCT FROM n.size_bytes THEN 'size_change'
WHEN o.modified_at IS DISTINCT FROM n.modified_at THEN 'content_change'
ELSE 'locator'
END AS reason_code
FROM with_rule_scope o FULL OUTER JOIN with_rule_scope_new n USING(source_locator_hash)
)
INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,
old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,
old_scope,new_scope,old_source_item_id,new_source_item_id,old_version_id,new_version_id,created_at)
SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title,
compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes,
compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,
compared.old_scope,compared.new_scope,compared.old_source_item_id,compared.new_source_item_id,compared.old_version_id,compared.new_version_id,NOW()
FROM change_set CROSS JOIN compared
ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET
action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path,
old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes,
old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at,
old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256,
reason_code=EXCLUDED.reason_code,
old_scope=EXCLUDED.old_scope,new_scope=EXCLUDED.new_scope,
old_source_item_id=EXCLUDED.old_source_item_id,new_source_item_id=EXCLUDED.new_source_item_id,
old_version_id=EXCLUDED.old_version_id,new_version_id=EXCLUDED.new_version_id;
UPDATE source_catalog_change_sets change_set SET
add_count=summary.add_count,
update_count=summary.update_count,
move_count=summary.move_count,
move_scope_count=summary.move_scope_count,
delete_count=summary.delete_count,
no_change_count=summary.no_change_count
FROM (
SELECT items.change_set_id,
COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count,
COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count,
COUNT(*) FILTER (WHERE items.action='move')::integer AS move_count,
COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count,
COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count,
COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count
FROM source_catalog_change_items items
WHERE items.change_set_id=p_change_set_id
GROUP BY items.change_set_id
) summary
WHERE change_set.id=summary.change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -0,0 +1,5 @@
DROP INDEX IF EXISTS idx_source_catalog_change_items_seq;
ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS item_seq;
ALTER TABLE test_publications DROP COLUMN IF EXISTS change_locator_hash, DROP COLUMN IF EXISTS change_set_id;
DROP INDEX IF EXISTS idx_processing_confirmations_change_set;
ALTER TABLE processing_confirmations DROP COLUMN IF EXISTS change_set_id;
@@ -0,0 +1,14 @@
-- P3: confirmations bind to the ChangeSet that produced them; publications record
-- provenance so every upload/removal job can be traced back to change evidence.
ALTER TABLE processing_confirmations
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT;
CREATE INDEX IF NOT EXISTS idx_processing_confirmations_change_set
ON processing_confirmations (change_set_id) WHERE change_set_id IS NOT NULL;
-- Surrogate public handle for selecting change items without leaking locator hashes.
ALTER TABLE source_catalog_change_items ADD COLUMN IF NOT EXISTS item_seq BIGSERIAL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_catalog_change_items_seq ON source_catalog_change_items (item_seq);
ALTER TABLE test_publications
ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
ADD COLUMN IF NOT EXISTS change_locator_hash CHAR(64) CHECK (change_locator_hash IS NULL OR change_locator_hash ~ '^[0-9a-f]{64}$');
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS jobs_processing_identity;
ALTER TABLE jobs DROP COLUMN IF EXISTS business_action, DROP COLUMN IF EXISTS business_target_kb_id, DROP COLUMN IF EXISTS business_version_id, DROP COLUMN IF EXISTS business_change_set_id;
@@ -0,0 +1,13 @@
-- P5 formal processing identity: every confirmed change-set job carries an
-- explicit business identity (publication uniqueness already exists since
-- migration 000002) so a duplicate delivery or worker restart cannot enqueue
-- the same action twice.
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_change_set_id VARCHAR(36);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_version_id VARCHAR(36);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_target_kb_id VARCHAR(255);
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_action VARCHAR(20)
CHECK (business_action IS NULL OR business_action IN ('upload', 'remove'));
CREATE UNIQUE INDEX IF NOT EXISTS jobs_processing_identity
ON jobs (business_change_set_id, business_version_id, business_target_kb_id, business_action)
WHERE business_change_set_id IS NOT NULL;
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS job_retry_operations;
DROP TABLE IF EXISTS job_failures;
ALTER TABLE jobs DROP COLUMN IF EXISTS last_error_code;
@@ -0,0 +1,48 @@
-- P6 DLQ: every terminal job failure keeps its immutable jobs audit row AND
-- gains a dead-letter projection. Requeued failures flip to 'requeued'; the
-- history is append-only. Retrying resets the attempt budget on the SAME job
-- row so the P5 business identity can never be duplicated.
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS last_error_code VARCHAR(64) NOT NULL DEFAULT '';
CREATE TABLE IF NOT EXISTS job_failures (
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
job_id VARCHAR(36) NOT NULL REFERENCES jobs(id),
job_type VARCHAR(100) NOT NULL,
publication_id VARCHAR(64) NOT NULL DEFAULT '',
error_code VARCHAR(64) NOT NULL,
last_error TEXT NOT NULL,
attempt INTEGER NOT NULL CHECK (attempt > 0),
max_attempts_at_failure INTEGER NOT NULL CHECK (max_attempts_at_failure > 0),
business_change_set_id VARCHAR(64) NOT NULL DEFAULT '',
business_version_id VARCHAR(64) NOT NULL DEFAULT '',
business_action VARCHAR(16) NOT NULL DEFAULT '',
business_target_kb_id VARCHAR(64) NOT NULL DEFAULT '',
state VARCHAR(16) NOT NULL DEFAULT 'open'
CHECK (state IN ('open', 'requeued', 'abandoned')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
resolved_at TIMESTAMPTZ,
resolution_actor_ref VARCHAR(255) NOT NULL DEFAULT '',
resolution_reason TEXT NOT NULL DEFAULT '',
UNIQUE (job_id, attempt)
);
CREATE INDEX IF NOT EXISTS idx_job_failures_open
ON job_failures (created_at DESC) WHERE state = 'open';
-- Retry audit: who re-queued which publication and with what reason
-- (batch-shared key). job_id is not stored: it is derivable from the
-- publication and would break the audit row on cross-batch duplicates.
CREATE TABLE IF NOT EXISTS job_retry_operations (
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
batch_key VARCHAR(255) NOT NULL,
publication_id VARCHAR(64) NOT NULL,
actor_ref VARCHAR(255) NOT NULL,
reason TEXT NOT NULL,
outcome VARCHAR(24) NOT NULL
CHECK (outcome IN ('started', 'already_running', 'not_retryable')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (batch_key, publication_id)
);
CREATE INDEX IF NOT EXISTS idx_job_retry_operations_publication
ON job_retry_operations (publication_id, created_at DESC);
@@ -0,0 +1 @@
DROP TABLE IF EXISTS worker_heartbeats;
@@ -0,0 +1,9 @@
-- P10 worker liveness: refreshed whenever a worker touches the queue (claim
-- attempts), read by the API to decide whether the worker process is alive.
CREATE TABLE IF NOT EXISTS worker_heartbeats (
worker_id VARCHAR(128) PRIMARY KEY,
job_types TEXT NOT NULL DEFAULT '',
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_worker_heartbeats_seen ON worker_heartbeats(last_seen_at DESC);
@@ -0,0 +1,3 @@
ALTER TABLE release_batches DROP COLUMN IF EXISTS release_change_set_id;
DROP TABLE IF EXISTS release_change_set_items;
DROP TABLE IF EXISTS release_change_sets;
@@ -0,0 +1,50 @@
-- R1 persistent release ChangeSet: the real-time release plan becomes an
-- immutable, snapshot-bound object. Batches reference CONFIRMED sets; later
-- source refreshes can never mutate an existing set or its manifest.
CREATE TABLE IF NOT EXISTS release_change_sets (
id VARCHAR(40) PRIMARY KEY,
name VARCHAR(200) NOT NULL,
number INTEGER NOT NULL,
state VARCHAR(16) NOT NULL DEFAULT 'planned'
CHECK (state IN ('planned','confirmed','consumed','cancelled')),
source_snapshot_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
source_change_set_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
acceptance_cutoff TIMESTAMPTZ NOT NULL,
production_baseline_ref VARCHAR(64) NOT NULL DEFAULT '',
item_count INTEGER NOT NULL DEFAULT 0,
add_count INTEGER NOT NULL DEFAULT 0,
update_count INTEGER NOT NULL DEFAULT 0,
delete_count INTEGER NOT NULL DEFAULT 0,
blocked_count INTEGER NOT NULL DEFAULT 0,
manifest_sha256 CHAR(64) NOT NULL
CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'),
created_by VARCHAR(120) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
confirmed_by VARCHAR(120) NOT NULL DEFAULT '',
confirmed_at TIMESTAMPTZ
);
CREATE UNIQUE INDEX IF NOT EXISTS release_change_sets_number_key
ON release_change_sets(number);
-- Frozen per-item plan: every field is written once at generation and never
-- updated (delete protection + immutability are enforced by having no UPDATE
-- path and a state machine on the parent).
CREATE TABLE IF NOT EXISTS release_change_set_items (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
seq INTEGER NOT NULL,
publication_id VARCHAR(64) NOT NULL,
title VARCHAR(500) NOT NULL,
target_scope VARCHAR(16) NOT NULL CHECK (target_scope IN ('internal','external')),
action VARCHAR(16) NOT NULL CHECK (action IN ('add','update','delete','blocked','unchanged')),
test_state VARCHAR(16) NOT NULL,
production_state VARCHAR(16) NOT NULL,
can_release BOOLEAN NOT NULL DEFAULT false,
blocker VARCHAR(32) NOT NULL DEFAULT '',
PRIMARY KEY (change_set_id, seq)
);
-- Batches optionally bind to a confirmed change set (formal releases).
ALTER TABLE release_batches
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
REFERENCES release_change_sets(id);
CREATE INDEX IF NOT EXISTS release_batches_change_set ON release_batches(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
@@ -0,0 +1,12 @@
-- Revert R4 policy. Rollback-stage rows are folded back to running/completed so
-- the old CHECK can be restored without data loss surprises in test schemas.
UPDATE release_executions SET state='running' WHERE state='rolling_back';
UPDATE release_executions SET state='completed' WHERE state='rolled_back';
UPDATE release_executions SET state='failed' WHERE state='rollback_failed';
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
ALTER TABLE release_executions
ADD CONSTRAINT release_executions_state_check CHECK (state IN ('pending','running','completed','failed'));
ALTER TABLE release_executions DROP COLUMN IF EXISTS snapshot_verification_id;
ALTER TABLE release_executions DROP COLUMN IF EXISTS restore_trigger;
ALTER TABLE release_executions DROP COLUMN IF EXISTS failure_level;
DROP INDEX IF EXISTS release_executions_rollback_state;
@@ -0,0 +1,22 @@
-- R4 three-level rollback policy:
-- single_item: one publication permanently failed; later items were skipped
-- and (for publish) compensated by an automatic rollback.
-- batch: remaining batches were never started after a permanent failure.
-- overall: restore the entire release from the pre-release Production
-- Snapshot; only a production authorizer may trigger it, and it
-- requires an R2 snapshot verification id as evidence.
-- The rollback execution walks pending -> rolling_back -> rolled_back /
-- rollback_failed so the UI can show "compensating now" distinctly.
ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check;
ALTER TABLE release_executions
ADD CONSTRAINT release_executions_state_check CHECK (state IN (
'pending','running','rolling_back','completed','failed','rolled_back','rollback_failed'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS failure_level VARCHAR(16) NOT NULL DEFAULT ''
CHECK (failure_level IN ('','single_item','batch','overall'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS restore_trigger VARCHAR(32) NOT NULL DEFAULT ''
CHECK (restore_trigger IN ('','auto_rollback_after_failure','snapshot_restore'));
ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS snapshot_verification_id VARCHAR(64) NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS release_executions_rollback_state
ON release_executions(prior_execution_id) WHERE operation = 'rollback';
@@ -0,0 +1,4 @@
DROP TABLE IF EXISTS release_change_set_catalog_links;
DROP TABLE IF EXISTS release_change_set_snapshot_links;
DROP INDEX IF EXISTS release_executions_change_set;
ALTER TABLE release_executions DROP COLUMN IF EXISTS release_change_set_id;
@@ -0,0 +1,46 @@
-- R6 release audit chain: promote the evidence links that used to live only in
-- JSONB into real, immutable foreign keys so any execution can be traced
-- end-to-end (execution -> batch -> release change set -> source snapshots ->
-- source change sets -> review/acceptance -> source item/version).
ALTER TABLE release_executions
ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40)
REFERENCES release_change_sets(id);
CREATE INDEX IF NOT EXISTS release_executions_change_set
ON release_executions(release_change_set_id) WHERE release_change_set_id IS NOT NULL;
-- Backfill the denormalized execution binding from its batch (R1 binding).
UPDATE release_executions AS execution
SET release_change_set_id = batch.release_change_set_id
FROM release_batches AS batch
WHERE execution.release_batch_id = batch.id
AND batch.release_change_set_id IS NOT NULL
AND execution.release_change_set_id IS NULL;
-- Normalized snapshot bindings (display JSONB stays, but FK rows are truth).
CREATE TABLE IF NOT EXISTS release_change_set_snapshot_links (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
snapshot_id VARCHAR(36) NOT NULL REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, snapshot_id)
);
CREATE TABLE IF NOT EXISTS release_change_set_catalog_links (
change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE,
catalog_change_set_id VARCHAR(36) NOT NULL REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (change_set_id, catalog_change_set_id)
);
-- Backfill from the JSONB captured at generation time (skip ids that no longer
-- exist: RESTRICT keeps live rows, deleted ones simply do not link).
INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id)
SELECT rcs.id, elem.value
FROM release_change_sets rcs
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_snapshot_ids) AS elem(value)
JOIN source_catalog_snapshots s ON s.id = elem.value
ON CONFLICT DO NOTHING;
INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id)
SELECT rcs.id, elem.value
FROM release_change_sets rcs
CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_change_set_ids) AS elem(value)
JOIN source_catalog_change_sets c ON c.id = elem.value
ON CONFLICT DO NOTHING;
@@ -0,0 +1,6 @@
-- Revert V1 revisions.
ALTER TABLE source_connections DROP COLUMN IF EXISTS current_revision_id,
DROP COLUMN IF EXISTS updated_by,
DROP COLUMN IF EXISTS last_scheduled_at,
DROP COLUMN IF EXISTS next_refresh_at;
DROP TABLE IF EXISTS source_connection_revisions CASCADE;
@@ -0,0 +1,64 @@
-- V1 source connection revisions: settings edits become validated -> active -> retired
-- versions; source_connections keeps ONE authoritative active snapshot that runtime
-- schedulers and existing FKs read. Secrets stay references only: base_url is visible,
-- secret_ref names where a credential lives and is never resolved through this table.
CREATE TABLE IF NOT EXISTS source_connection_revisions (
id VARCHAR(36) PRIMARY KEY,
connection_id VARCHAR(36) NOT NULL REFERENCES source_connections(id) ON DELETE CASCADE,
revision_number INTEGER NOT NULL CHECK (revision_number > 0),
state VARCHAR(16) NOT NULL
CHECK (state IN ('validated', 'active', 'retired', 'failed')),
source_type VARCHAR(20) NOT NULL
CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')),
name VARCHAR(255) NOT NULL,
base_url TEXT NOT NULL DEFAULT '',
secret_ref VARCHAR(255) NOT NULL DEFAULT '',
enabled BOOLEAN NOT NULL DEFAULT FALSE,
schedule JSONB NOT NULL DEFAULT '{}'::jsonb
CHECK (jsonb_typeof(schedule) = 'object'),
created_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
validated_at TIMESTAMPTZ,
activated_at TIMESTAMPTZ,
retired_at TIMESTAMPTZ,
error_code VARCHAR(64) NOT NULL DEFAULT '',
CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$'),
UNIQUE (connection_id, revision_number)
);
-- Exactly one active revision per connection.
CREATE UNIQUE INDEX IF NOT EXISTS uq_source_connection_active_revision
ON source_connection_revisions (connection_id) WHERE state = 'active';
-- Scheduler state lives next to the active snapshot so the existing FKs and UI
-- keep reading source_connections. next_refresh_at is NULL until the operator
-- sets a refresh interval, and it is ALWAYS derived from the schedule on the
-- active revision (never trusted from clients).
ALTER TABLE source_connections
ADD COLUMN IF NOT EXISTS next_refresh_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS last_scheduled_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS current_revision_id VARCHAR(36) REFERENCES source_connection_revisions(id),
ADD COLUMN IF NOT EXISTS updated_by VARCHAR(128) NOT NULL DEFAULT 'system';
CREATE INDEX IF NOT EXISTS idx_source_connections_due
ON source_connections (next_refresh_at)
WHERE enabled AND next_refresh_at IS NOT NULL;
-- Seed: every existing connection gets revision 1 reflecting its current row,
-- marked active, created by the migration itself.
INSERT INTO source_connection_revisions (
id, connection_id, revision_number, state, source_type, name, base_url,
secret_ref, enabled, schedule, created_by, validated_at, activated_at
)
SELECT
'scr-' || md5(c.id), c.id, 1, 'active', c.source_type, c.name, c.base_url,
c.secret_ref, c.enabled, c.schedule, 'migration-31', NOW(), NOW()
FROM source_connections AS c
ON CONFLICT (connection_id, revision_number) DO NOTHING;
UPDATE source_connections AS c
SET current_revision_id = COALESCE(c.current_revision_id, 'scr-' || md5(c.id))
FROM source_connection_revisions AS r
WHERE r.connection_id = c.id AND r.revision_number = 1
AND c.current_revision_id IS NULL;