From dcdc8d1ddd1e1720ab9fe229fa85b7e9253329ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= <836283459@qq.com> Date: Sun, 13 Sep 2026 12:02:58 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=9F=A5=E8=AF=86=E6=B2=BB=E7=90=86?= =?UTF-8?q?=E5=B9=B3=E5=8F=B0(=E5=90=AB=E5=B7=B2=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E5=8F=91=E5=B8=83/=E5=9B=9E=E6=BB=9A=E5=A5=91=E7=BA=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全栈知识治理平台首次入库（P1-P10 + 发布链路模拟验证闭环）： - 治理核心：源目录/变更集/人工审核/发布批次/发布执行（fail-closed 闸口） - 测试通道：WeKnora 测试 KB 上传/轮询/验收；poll 完成时写入 parser_generation/index_generation，修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071（本地 source=测试角色 / 云端 target=生产角色 模拟验证通过）： * ES 导出/导入 mapping 自适应（.keyword 子字段、可选 tenant_id） * OSS locator 支持嵌套 scheme；chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换；verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id（strict 解码） - 三通道发布+回滚演练实测：publish 4/4 verified，检索命中， rollback 4/4 rolled_back，目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一，120 用例通过 - 门禁：go vet / go test 全包通过，gofmt 干净 --- .dockerignore | 11 + .env.example | 25 + .gitignore | 30 + Dockerfile | 23 + Dockerfile.web | 11 + Makefile | 23 + README.md | 207 +- api/openapi.yaml | 3145 +++++++++++++++++ cmd/api/main.go | 112 + cmd/api/main_test.go | 45 + cmd/pp-ledger-import/main.go | 153 + cmd/pp-ledger-import/main_test.go | 99 + cmd/release-diag/main.go | 313 ++ cmd/release-worker/config.go | 155 + cmd/release-worker/config_test.go | 65 + cmd/release-worker/main.go | 79 + cmd/release-worker/main_test.go | 44 + cmd/release-worker/runtime.go | 222 ++ cmd/release-worker/runtime_test.go | 60 + cmd/test-publication-seed/main.go | 131 + cmd/test-publication-seed/main_test.go | 58 + cmd/test-weknora-e2e-report/main.go | 174 + cmd/test-weknora-e2e-report/main_test.go | 87 + cmd/test-weknora-probe/main.go | 130 + cmd/test-weknora-probe/main_test.go | 37 + cmd/worker/main.go | 156 + cmd/worker/main_test.go | 92 + cmd/worker/scheduler.go | 53 + cmd/worker/scheduler_test.go | 116 + compose.release-sim.yaml | 36 + compose.yaml | 164 + config/release.sim.json | 46 + config/release.v071.example.json | 48 + deploy/nginx.conf | 24 + docs/ACTION_ITEMS_12_TO_END_ACCEPTANCE.md | 84 + docs/DOCUMENT_WORKBENCH_ACCEPTANCE.md | 49 + docs/DOCUMENT_WORKBENCH_V2_ACCEPTANCE.md | 47 + docs/DOMAIN_MODEL.md | 293 ++ docs/P1_ACCEPTANCE_TEST_PLAN.md | 114 + docs/P2_PP_INVENTORY_ACCEPTANCE.md | 84 + docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md | 99 + docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md | 86 + docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md | 164 + docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md | 100 + docs/P42_SOURCE_SCAN_ACCEPTANCE.md | 250 ++ docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md | 320 ++ docs/P6_RELEASE_WORKER_ACCEPTANCE.md | 495 +++ docs/RELEASE_PLANNER_SIMULATION_ACCEPTANCE.md | 13 + docs/SOURCE_CATALOG_ACCEPTANCE.md | 47 + .../SOURCE_CATALOG_LIVE_ROUTING_ACCEPTANCE.md | 44 + ...RCE_LINK_SETTINGS_SIMULATION_ACCEPTANCE.md | 28 + docs/SOURCE_SCOPE_POLICY.md | 38 + docs/STATE_MACHINE.md | 289 ++ docs/WEKNORA_MONITOR_SIMULATION_ACCEPTANCE.md | 29 + docs/previews/monitor-a-compact-table.html | 159 + docs/previews/monitor-b-task-inbox.html | 148 + .../refresh-2026-09-12/a-data-console.html | 165 + .../b-editorial-ledger.html | 135 + .../refresh-2026-09-12/c-current-polish.html | 132 + go.mod | 33 + go.sum | 78 + internal/appconfig/config.go | 247 ++ internal/appconfig/config_test.go | 243 ++ internal/connectors/pp/ledger.go | 141 + internal/connectors/pp/ledger_test.go | 224 ++ internal/connectors/pp/manifest.go | 91 + internal/connectors/pp/overlimit.go | 55 + internal/connectors/pp/overlimit_test.go | 63 + internal/connectors/pp/plan.go | 150 + internal/connectors/pp/plan_test.go | 60 + internal/connectors/pp/reconcile.go | 38 + internal/connectors/pp/reconcile_test.go | 58 + internal/governance/change_set_execution.go | 26 + .../governance/domain_model_contract_test.go | 40 + internal/governance/job_failures.go | 64 + internal/governance/model.go | 285 ++ internal/governance/model_test.go | 57 + internal/governance/monitor.go | 117 + internal/governance/release_audit_chain.go | 63 + internal/governance/release_batch.go | 75 + internal/governance/release_change_set.go | 63 + internal/governance/release_execution.go | 115 + internal/governance/release_plan.go | 25 + internal/governance/review.go | 67 + internal/governance/scope_rule.go | 80 + internal/governance/scope_rule_test.go | 42 + internal/governance/source_asset.go | 45 + internal/governance/source_asset_test.go | 19 + internal/governance/source_catalog.go | 154 + internal/governance/source_connection.go | 84 + .../source_connection_projection_test.go | 34 + internal/governance/source_scan.go | 92 + internal/governance/source_schedule.go | 42 + internal/governance/source_schedule_test.go | 49 + .../governance/state_machine_contract_test.go | 38 + internal/governance/test_publication.go | 63 + internal/governance/ui_settings.go | 51 + internal/httpapi/openapi_contract_test.go | 110 + internal/httpapi/server.go | 1965 ++++++++++ internal/httpapi/server_test.go | 1888 ++++++++++ .../httpapi/ui_settings_integration_test.go | 78 + internal/jobs/demo.go | 35 + internal/jobs/demo_test.go | 18 + internal/jobs/errors.go | 51 + internal/jobs/job.go | 57 + internal/jobs/job_test.go | 21 + internal/jobs/worker.go | 179 + internal/jobs/worker_test.go | 295 ++ internal/monitoring/health.go | 328 ++ internal/monitoring/health_test.go | 84 + internal/monitoring/metrics.go | 171 + internal/monitoring/metrics_test.go | 71 + .../monitoring/p10_health_integration_test.go | 116 + .../monitoring/p9_metrics_integration_test.go | 190 + internal/monitoring/real.go | 320 ++ internal/monitoring/real_test.go | 24 + internal/monitoring/simulation.go | 87 + internal/monitoring/simulation_test.go | 52 + ...4_concurrent_operation_integration_test.go | 104 + .../platform/postgres/catalog_change_sets.go | 71 + .../catalog_change_sets_integration_test.go | 47 + .../postgres/catalog_refresh_progress.go | 27 + internal/platform/postgres/catalog_routing.go | 243 ++ .../catalog_routing_integration_test.go | 158 + ...alog_routing_operation_integration_test.go | 113 + .../platform/postgres/change_set_execution.go | 358 ++ .../integration_isolation_external_test.go | 44 + .../postgres/integration_isolation_test.go | 44 + internal/platform/postgres/job_failures.go | 201 ++ .../platform/postgres/manual_publication.go | 196 + .../postgres/manual_publication_test.go | 60 + internal/platform/postgres/migrate.go | 140 + .../postgres/migrations/000001_jobs.down.sql | 1 + .../postgres/migrations/000001_jobs.up.sql | 31 + .../000002_source_governance.down.sql | 5 + .../000002_source_governance.up.sql | 114 + .../000003_test_orchestration.down.sql | 12 + .../000003_test_orchestration.up.sql | 33 + .../migrations/000004_manual_source.down.sql | 18 + .../migrations/000004_manual_source.up.sql | 6 + .../migrations/000005_source_scans.down.sql | 2 + .../migrations/000005_source_scans.up.sql | 43 + .../000006_review_and_seal.down.sql | 3 + .../migrations/000006_review_and_seal.up.sql | 58 + .../000007_release_executions.down.sql | 2 + .../000007_release_executions.up.sql | 65 + .../migrations/000008_scope_rules.down.sql | 3 + .../migrations/000008_scope_rules.up.sql | 57 + .../migrations/000009_source_catalog.down.sql | 3 + .../migrations/000009_source_catalog.up.sql | 56 + .../000010_catalog_policies.down.sql | 36 + .../migrations/000010_catalog_policies.up.sql | 25 + .../000011_processing_confirmations.down.sql | 2 + .../000011_processing_confirmations.up.sql | 36 + .../000012_release_actions.down.sql | 7 + .../migrations/000012_release_actions.up.sql | 12 + .../000013_ui_system_settings.down.sql | 1 + .../000013_ui_system_settings.up.sql | 24 + ...000014_catalog_refresh_started_at.down.sql | 2 + .../000014_catalog_refresh_started_at.up.sql | 2 + .../000015_catalog_change_sets.down.sql | 2 + .../000015_catalog_change_sets.up.sql | 42 + ...00016_catalog_change_set_backfill.down.sql | 4 + .../000016_catalog_change_set_backfill.up.sql | 87 + .../000017_release_batch_progress.down.sql | 5 + .../000017_release_batch_progress.up.sql | 16 + .../000018_job_cancelled_state.down.sql | 11 + .../000018_job_cancelled_state.up.sql | 3 + .../000019_operation_requests.down.sql | 1 + .../000019_operation_requests.up.sql | 10 + ...020_catalog_snapshot_content_hash.down.sql | 1 + ...00020_catalog_snapshot_content_hash.up.sql | 3 + .../000021_source_identity_evidence.down.sql | 2 + .../000021_source_identity_evidence.up.sql | 12 + .../000022_catalog_change_comparison.down.sql | 1 + .../000022_catalog_change_comparison.up.sql | 99 + .../000023_complete_change_set.down.sql | 26 + .../000023_complete_change_set.up.sql | 167 + .../000024_change_set_confirmations.down.sql | 5 + .../000024_change_set_confirmations.up.sql | 14 + .../000025_processing_job_identity.down.sql | 2 + .../000025_processing_job_identity.up.sql | 13 + .../000026_job_failure_queue.down.sql | 3 + .../000026_job_failure_queue.up.sql | 48 + .../000027_worker_heartbeats.down.sql | 1 + .../000027_worker_heartbeats.up.sql | 9 + .../000028_release_change_sets.down.sql | 3 + .../000028_release_change_sets.up.sql | 50 + .../000029_release_rollback_policy.down.sql | 12 + .../000029_release_rollback_policy.up.sql | 22 + .../000030_release_audit_links.down.sql | 4 + .../000030_release_audit_links.up.sql | 46 + ...00031_source_connection_revisions.down.sql | 6 + .../000031_source_connection_revisions.up.sql | 64 + internal/platform/postgres/migrations_test.go | 633 ++++ .../platform/postgres/operation_requests.go | 180 + .../platform/postgres/operation_wrappers.go | 163 + .../postgres/p2_identity_integration_test.go | 100 + .../postgres/p3_confirm_integration_test.go | 228 ++ .../p5_job_identity_integration_test.go | 71 + .../postgres/p6_dlq_integration_test.go | 179 + internal/platform/postgres/pp_import.go | 173 + .../postgres/pp_import_integration_test.go | 155 + .../postgres/processing_confirmations.go | 137 + .../production_delete_integration_test.go | 126 + .../platform/postgres/publication_removal.go | 65 + internal/platform/postgres/queue_sql.go | 169 + .../r1_release_change_set_integration_test.go | 317 ++ .../r4_rollback_levels_integration_test.go | 236 ++ .../r5_manifest_seal_integration_test.go | 123 + .../r6_audit_chain_integration_test.go | 144 + .../platform/postgres/release_audit_chain.go | 155 + .../release_authorization_integration_test.go | 258 ++ internal/platform/postgres/release_batches.go | 450 +++ .../platform/postgres/release_batches_test.go | 228 ++ .../platform/postgres/release_change_sets.go | 290 ++ .../postgres/release_execution_reads_test.go | 132 + .../release_execution_worker_store.go | 498 +++ .../release_execution_worker_store_test.go | 163 + .../platform/postgres/release_executions.go | 540 +++ .../postgres/release_executions_test.go | 158 + internal/platform/postgres/release_plan.go | 132 + .../platform/postgres/release_plan_test.go | 91 + .../review_and_seal_integration_test.go | 143 + internal/platform/postgres/reviews.go | 253 ++ internal/platform/postgres/reviews_test.go | 169 + .../s2_snapshot_history_integration_test.go | 109 + .../s4_refresh_progress_integration_test.go | 68 + .../s6_scope_policy_integration_test.go | 75 + ...s8_change_set_contract_integration_test.go | 166 + internal/platform/postgres/scope_rules.go | 183 + .../postgres/scope_rules_integration_test.go | 137 + internal/platform/postgres/source_assets.go | 148 + .../platform/postgres/source_assets_test.go | 14 + internal/platform/postgres/source_catalog.go | 307 ++ .../source_catalog_integration_test.go | 80 + .../platform/postgres/source_connections.go | 227 ++ .../source_connections_integration_test.go | 68 + .../postgres/source_connections_test.go | 132 + .../postgres/source_scan_integration_test.go | 116 + .../source_scan_migration_integration_test.go | 99 + internal/platform/postgres/source_scans.go | 438 +++ .../platform/postgres/source_scans_test.go | 395 +++ internal/platform/postgres/source_settings.go | 310 ++ internal/platform/postgres/store.go | 170 + internal/platform/postgres/store_test.go | 295 ++ .../platform/postgres/test_publications.go | 806 +++++ .../test_publications_integration_test.go | 253 ++ .../postgres/test_publications_test.go | 601 ++++ internal/platform/postgres/ui_settings.go | 134 + .../postgres/ui_settings_integration_test.go | 51 + .../platform/postgres/ui_settings_test.go | 65 + .../postgres/v1_migration_seed_test.go | 53 + .../postgres/v1_scheduler_integration_test.go | 218 ++ .../v1_source_settings_integration_test.go | 225 ++ internal/platform/spool/artifact.go | 285 ++ internal/platform/spool/artifact_test.go | 216 ++ internal/platform/weknora/client.go | 368 ++ internal/platform/weknora/client_test.go | 379 ++ internal/platform/weknora/secret.go | 49 + internal/platform/weknora/secret_test.go | 55 + internal/platform/weknora/types.go | 76 + internal/releaseadapter/v071/archive.go | 197 ++ internal/releaseadapter/v071/archive_test.go | 155 + .../releaseadapter/v071/contract_inspector.go | 101 + .../v071/contract_inspector_test.go | 87 + .../releaseadapter/v071/document_export.go | 149 + .../v071/document_export_test.go | 101 + .../releaseadapter/v071/document_target.go | 209 ++ .../v071/document_target_test.go | 173 + internal/releaseadapter/v071/object_export.go | 161 + .../releaseadapter/v071/object_export_test.go | 76 + internal/releaseadapter/v071/object_import.go | 130 + .../releaseadapter/v071/object_import_test.go | 106 + .../releaseadapter/v071/postgres_export.go | 340 ++ .../v071/postgres_export_integration_test.go | 36 + .../v071/postgres_export_test.go | 130 + .../releaseadapter/v071/postgres_import.go | 416 +++ .../v071/postgres_import_integration_test.go | 115 + .../releaseadapter/v071/s3_object_store.go | 141 + .../v071/s3_object_store_integration_test.go | 57 + internal/releaseadapter/v071/search.go | 322 ++ internal/releaseadapter/v071/search_import.go | 179 + .../releaseadapter/v071/search_import_test.go | 220 ++ internal/releaseadapter/v071/search_test.go | 159 + .../v071/three_lane_integration_test.go | 158 + internal/releaseadapter/v071/verifier.go | 141 + internal/releaseadapter/v071/verifier_test.go | 61 + internal/releasebundle/manifest.go | 187 + internal/releasebundle/manifest_test.go | 119 + internal/workflows/catalogrefresh/handler.go | 191 + .../workflows/catalogrefresh/handler_test.go | 126 + internal/workflows/release/handler.go | 299 ++ internal/workflows/release/handler_test.go | 209 ++ .../workflows/release/routing_publisher.go | 65 + .../release/routing_publisher_test.go | 52 + .../workflows/release/sidecar_publisher.go | 162 + .../release/sidecar_publisher_test.go | 217 ++ internal/workflows/sourcescan/file_scanner.go | 194 + internal/workflows/sourcescan/handler.go | 127 + internal/workflows/sourcescan/handler_test.go | 208 ++ .../workflows/testweknora/legacy_verify.go | 196 + .../testweknora/legacy_verify_test.go | 192 + internal/workflows/testweknora/poll.go | 141 + internal/workflows/testweknora/poll_test.go | 188 + internal/workflows/testweknora/remove.go | 117 + internal/workflows/testweknora/remove_test.go | 100 + internal/workflows/testweknora/upload.go | 241 ++ internal/workflows/testweknora/upload_test.go | 354 ++ internal/workflows/testweknora/validate.go | 143 + .../workflows/testweknora/validate_test.go | 118 + .../import_display_metadata.cpython-313.pyc | Bin 0 -> 6678 bytes .../live_catalog_refresh.cpython-313.pyc | Bin 0 -> 16478 bytes .../source_catalog_import.cpython-313.pyc | Bin 0 -> 35808 bytes ...play_metadata.cpython-313-pytest-8.3.4.pyc | Bin 0 -> 3842 bytes ...st_import_display_metadata.cpython-313.pyc | Bin 0 -> 3729 bytes ...resh_progress.cpython-313-pytest-8.3.4.pyc | Bin 0 -> 10838 bytes ...atalog_import.cpython-313-pytest-8.3.4.pyc | Bin 0 -> 18494 bytes ...test_source_catalog_import.cpython-313.pyc | Bin 0 -> 11007 bytes scripts/accept_catalog_controls.py | 32 + scripts/accept_catalog_route_canary.py | 23 + scripts/accept_document_workbench.py | 110 + scripts/accept_document_workbench_v2.py | 141 + scripts/accept_live_catalog_refresh.py | 19 + scripts/accept_monitor_simulation.py | 34 + scripts/accept_refresh_failure_message.py | 18 + scripts/accept_release_planner.py | 31 + scripts/accept_source_catalog.py | 62 + scripts/accept_source_links.py | 28 + scripts/accept_system_settings.py | 27 + scripts/ensure_local_routing_kbs.py | 22 + scripts/import_display_metadata.py | 90 + scripts/live_catalog_refresh.py | 132 + scripts/source_catalog_import.py | 355 ++ scripts/stage_pp_routing_canary.py | 25 + scripts/test_import_display_metadata.py | 39 + scripts/test_live_catalog_refresh_progress.py | 39 + scripts/test_source_catalog_import.py | 142 + secrets/alist_api_token.example | 1 + secrets/pgpass.example | 1 + secrets/postgres_password.example | 1 + secrets/test_weknora_api_key.example | 1 + ...89fd0fb90568d9a45595535b409f67f61bd68a.txt | 5 + spool/manual/e2e-fullchain-20260912.md | 9 + spool/manual/p3-durable-restart-20260905.md | 7 + spool/manual/p3-e2e-20260905.md | 7 + spool/manual/p3-recovery-20260905.md | 7 + spool/manual/p3-restart-20260905.md | 7 + spool/manual/p4-cross-env-20260905.md | 7 + web/index.html | 13 + web/package-lock.json | 2317 ++++++++++++ web/package.json | 24 + web/src/App.test.tsx | 483 +++ web/src/App.tsx | 1576 +++++++++ web/src/DocumentWorkbench.test.tsx | 33 + web/src/DocumentWorkbench.tsx | 160 + web/src/ReleasePlanner.test.tsx | 20 + web/src/ReleasePlanner.tsx | 26 + web/src/ScopeRuleEditor.test.tsx | 18 + web/src/ScopeRuleEditor.tsx | 33 + web/src/SnapshotHistory.test.tsx | 34 + web/src/SnapshotHistory.tsx | 32 + web/src/SourceCatalogBrowser.test.tsx | 127 + web/src/SourceCatalogBrowser.tsx | 111 + web/src/SourceLinkManager.test.tsx | 26 + web/src/SourceLinkManager.tsx | 35 + web/src/SystemSettings.test.tsx | 49 + web/src/SystemSettings.tsx | 222 ++ web/src/WeKnoraMonitor.test.tsx | 65 + web/src/WeKnoraMonitor.tsx | 398 +++ web/src/assets.test.ts | 41 + web/src/assets.ts | 78 + web/src/documentFixture.ts | 6 + web/src/documentWorkbench.css | 96 + web/src/documentWorkbench.test.ts | 65 + web/src/documentWorkbench.ts | 128 + web/src/jobs.test.ts | 48 + web/src/jobs.ts | 36 + web/src/main.tsx | 16 + web/src/monitor.test.ts | 34 + web/src/monitor.ts | 38 + web/src/operationKey.test.ts | 31 + web/src/operationKey.ts | 13 + web/src/releasePlanner.css | 2 + web/src/releasePlanner.test.ts | 18 + web/src/releasePlanner.ts | 16 + web/src/responsiveStyles.test.ts | 12 + web/src/reviewRelease.test.ts | 192 + web/src/reviewRelease.ts | 532 +++ web/src/scopeRules.test.ts | 25 + web/src/scopeRules.ts | 53 + web/src/sourceCatalog.test.ts | 73 + web/src/sourceCatalog.ts | 112 + web/src/sourceCatalogBrowser.css | 31 + web/src/sourceConnections.test.ts | 181 + web/src/sourceConnections.ts | 378 ++ web/src/sourceLinkManager.css | 1 + web/src/styles.css | 1916 ++++++++++ web/src/systemSettings.css | 28 + web/src/testPublications.test.ts | 143 + web/src/testPublications.ts | 181 + web/src/uiSettings.test.ts | 42 + web/src/uiSettings.ts | 84 + web/src/weknoraMonitor.css | 64 + web/src/workspace.test.ts | 51 + web/src/workspace.ts | 63 + web/src/workspaceContext.test.tsx | 15 + web/src/workspaceContext.tsx | 10 + web/tsconfig.json | 21 + web/vite.config.ts | 12 + 410 files changed, 56661 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Dockerfile.web create mode 100644 Makefile create mode 100644 api/openapi.yaml create mode 100644 cmd/api/main.go create mode 100644 cmd/api/main_test.go create mode 100644 cmd/pp-ledger-import/main.go create mode 100644 cmd/pp-ledger-import/main_test.go create mode 100644 cmd/release-diag/main.go create mode 100644 cmd/release-worker/config.go create mode 100644 cmd/release-worker/config_test.go create mode 100644 cmd/release-worker/main.go create mode 100644 cmd/release-worker/main_test.go create mode 100644 cmd/release-worker/runtime.go create mode 100644 cmd/release-worker/runtime_test.go create mode 100644 cmd/test-publication-seed/main.go create mode 100644 cmd/test-publication-seed/main_test.go create mode 100644 cmd/test-weknora-e2e-report/main.go create mode 100644 cmd/test-weknora-e2e-report/main_test.go create mode 100644 cmd/test-weknora-probe/main.go create mode 100644 cmd/test-weknora-probe/main_test.go create mode 100644 cmd/worker/main.go create mode 100644 cmd/worker/main_test.go create mode 100644 cmd/worker/scheduler.go create mode 100644 cmd/worker/scheduler_test.go create mode 100644 compose.release-sim.yaml create mode 100644 compose.yaml create mode 100644 config/release.sim.json create mode 100644 config/release.v071.example.json create mode 100644 deploy/nginx.conf create mode 100644 docs/ACTION_ITEMS_12_TO_END_ACCEPTANCE.md create mode 100644 docs/DOCUMENT_WORKBENCH_ACCEPTANCE.md create mode 100644 docs/DOCUMENT_WORKBENCH_V2_ACCEPTANCE.md create mode 100644 docs/DOMAIN_MODEL.md create mode 100644 docs/P1_ACCEPTANCE_TEST_PLAN.md create mode 100644 docs/P2_PP_INVENTORY_ACCEPTANCE.md create mode 100644 docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md create mode 100644 docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md create mode 100644 docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md create mode 100644 docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md create mode 100644 docs/P42_SOURCE_SCAN_ACCEPTANCE.md create mode 100644 docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md create mode 100644 docs/P6_RELEASE_WORKER_ACCEPTANCE.md create mode 100644 docs/RELEASE_PLANNER_SIMULATION_ACCEPTANCE.md create mode 100644 docs/SOURCE_CATALOG_ACCEPTANCE.md create mode 100644 docs/SOURCE_CATALOG_LIVE_ROUTING_ACCEPTANCE.md create mode 100644 docs/SOURCE_LINK_SETTINGS_SIMULATION_ACCEPTANCE.md create mode 100644 docs/SOURCE_SCOPE_POLICY.md create mode 100644 docs/STATE_MACHINE.md create mode 100644 docs/WEKNORA_MONITOR_SIMULATION_ACCEPTANCE.md create mode 100644 docs/previews/monitor-a-compact-table.html create mode 100644 docs/previews/monitor-b-task-inbox.html create mode 100644 docs/previews/refresh-2026-09-12/a-data-console.html create mode 100644 docs/previews/refresh-2026-09-12/b-editorial-ledger.html create mode 100644 docs/previews/refresh-2026-09-12/c-current-polish.html create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/appconfig/config.go create mode 100644 internal/appconfig/config_test.go create mode 100644 internal/connectors/pp/ledger.go create mode 100644 internal/connectors/pp/ledger_test.go create mode 100644 internal/connectors/pp/manifest.go create mode 100644 internal/connectors/pp/overlimit.go create mode 100644 internal/connectors/pp/overlimit_test.go create mode 100644 internal/connectors/pp/plan.go create mode 100644 internal/connectors/pp/plan_test.go create mode 100644 internal/connectors/pp/reconcile.go create mode 100644 internal/connectors/pp/reconcile_test.go create mode 100644 internal/governance/change_set_execution.go create mode 100644 internal/governance/domain_model_contract_test.go create mode 100644 internal/governance/job_failures.go create mode 100644 internal/governance/model.go create mode 100644 internal/governance/model_test.go create mode 100644 internal/governance/monitor.go create mode 100644 internal/governance/release_audit_chain.go create mode 100644 internal/governance/release_batch.go create mode 100644 internal/governance/release_change_set.go create mode 100644 internal/governance/release_execution.go create mode 100644 internal/governance/release_plan.go create mode 100644 internal/governance/review.go create mode 100644 internal/governance/scope_rule.go create mode 100644 internal/governance/scope_rule_test.go create mode 100644 internal/governance/source_asset.go create mode 100644 internal/governance/source_asset_test.go create mode 100644 internal/governance/source_catalog.go create mode 100644 internal/governance/source_connection.go create mode 100644 internal/governance/source_connection_projection_test.go create mode 100644 internal/governance/source_scan.go create mode 100644 internal/governance/source_schedule.go create mode 100644 internal/governance/source_schedule_test.go create mode 100644 internal/governance/state_machine_contract_test.go create mode 100644 internal/governance/test_publication.go create mode 100644 internal/governance/ui_settings.go create mode 100644 internal/httpapi/openapi_contract_test.go create mode 100644 internal/httpapi/server.go create mode 100644 internal/httpapi/server_test.go create mode 100644 internal/httpapi/ui_settings_integration_test.go create mode 100644 internal/jobs/demo.go create mode 100644 internal/jobs/demo_test.go create mode 100644 internal/jobs/errors.go create mode 100644 internal/jobs/job.go create mode 100644 internal/jobs/job_test.go create mode 100644 internal/jobs/worker.go create mode 100644 internal/jobs/worker_test.go create mode 100644 internal/monitoring/health.go create mode 100644 internal/monitoring/health_test.go create mode 100644 internal/monitoring/metrics.go create mode 100644 internal/monitoring/metrics_test.go create mode 100644 internal/monitoring/p10_health_integration_test.go create mode 100644 internal/monitoring/p9_metrics_integration_test.go create mode 100644 internal/monitoring/real.go create mode 100644 internal/monitoring/real_test.go create mode 100644 internal/monitoring/simulation.go create mode 100644 internal/monitoring/simulation_test.go create mode 100644 internal/platform/postgres/c4_concurrent_operation_integration_test.go create mode 100644 internal/platform/postgres/catalog_change_sets.go create mode 100644 internal/platform/postgres/catalog_change_sets_integration_test.go create mode 100644 internal/platform/postgres/catalog_refresh_progress.go create mode 100644 internal/platform/postgres/catalog_routing.go create mode 100644 internal/platform/postgres/catalog_routing_integration_test.go create mode 100644 internal/platform/postgres/catalog_routing_operation_integration_test.go create mode 100644 internal/platform/postgres/change_set_execution.go create mode 100644 internal/platform/postgres/integration_isolation_external_test.go create mode 100644 internal/platform/postgres/integration_isolation_test.go create mode 100644 internal/platform/postgres/job_failures.go create mode 100644 internal/platform/postgres/manual_publication.go create mode 100644 internal/platform/postgres/manual_publication_test.go create mode 100644 internal/platform/postgres/migrate.go create mode 100644 internal/platform/postgres/migrations/000001_jobs.down.sql create mode 100644 internal/platform/postgres/migrations/000001_jobs.up.sql create mode 100644 internal/platform/postgres/migrations/000002_source_governance.down.sql create mode 100644 internal/platform/postgres/migrations/000002_source_governance.up.sql create mode 100644 internal/platform/postgres/migrations/000003_test_orchestration.down.sql create mode 100644 internal/platform/postgres/migrations/000003_test_orchestration.up.sql create mode 100644 internal/platform/postgres/migrations/000004_manual_source.down.sql create mode 100644 internal/platform/postgres/migrations/000004_manual_source.up.sql create mode 100644 internal/platform/postgres/migrations/000005_source_scans.down.sql create mode 100644 internal/platform/postgres/migrations/000005_source_scans.up.sql create mode 100644 internal/platform/postgres/migrations/000006_review_and_seal.down.sql create mode 100644 internal/platform/postgres/migrations/000006_review_and_seal.up.sql create mode 100644 internal/platform/postgres/migrations/000007_release_executions.down.sql create mode 100644 internal/platform/postgres/migrations/000007_release_executions.up.sql create mode 100644 internal/platform/postgres/migrations/000008_scope_rules.down.sql create mode 100644 internal/platform/postgres/migrations/000008_scope_rules.up.sql create mode 100644 internal/platform/postgres/migrations/000009_source_catalog.down.sql create mode 100644 internal/platform/postgres/migrations/000009_source_catalog.up.sql create mode 100644 internal/platform/postgres/migrations/000010_catalog_policies.down.sql create mode 100644 internal/platform/postgres/migrations/000010_catalog_policies.up.sql create mode 100644 internal/platform/postgres/migrations/000011_processing_confirmations.down.sql create mode 100644 internal/platform/postgres/migrations/000011_processing_confirmations.up.sql create mode 100644 internal/platform/postgres/migrations/000012_release_actions.down.sql create mode 100644 internal/platform/postgres/migrations/000012_release_actions.up.sql create mode 100644 internal/platform/postgres/migrations/000013_ui_system_settings.down.sql create mode 100644 internal/platform/postgres/migrations/000013_ui_system_settings.up.sql create mode 100644 internal/platform/postgres/migrations/000014_catalog_refresh_started_at.down.sql create mode 100644 internal/platform/postgres/migrations/000014_catalog_refresh_started_at.up.sql create mode 100644 internal/platform/postgres/migrations/000015_catalog_change_sets.down.sql create mode 100644 internal/platform/postgres/migrations/000015_catalog_change_sets.up.sql create mode 100644 internal/platform/postgres/migrations/000016_catalog_change_set_backfill.down.sql create mode 100644 internal/platform/postgres/migrations/000016_catalog_change_set_backfill.up.sql create mode 100644 internal/platform/postgres/migrations/000017_release_batch_progress.down.sql create mode 100644 internal/platform/postgres/migrations/000017_release_batch_progress.up.sql create mode 100644 internal/platform/postgres/migrations/000018_job_cancelled_state.down.sql create mode 100644 internal/platform/postgres/migrations/000018_job_cancelled_state.up.sql create mode 100644 internal/platform/postgres/migrations/000019_operation_requests.down.sql create mode 100644 internal/platform/postgres/migrations/000019_operation_requests.up.sql create mode 100644 internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.down.sql create mode 100644 internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.up.sql create mode 100644 internal/platform/postgres/migrations/000021_source_identity_evidence.down.sql create mode 100644 internal/platform/postgres/migrations/000021_source_identity_evidence.up.sql create mode 100644 internal/platform/postgres/migrations/000022_catalog_change_comparison.down.sql create mode 100644 internal/platform/postgres/migrations/000022_catalog_change_comparison.up.sql create mode 100644 internal/platform/postgres/migrations/000023_complete_change_set.down.sql create mode 100644 internal/platform/postgres/migrations/000023_complete_change_set.up.sql create mode 100644 internal/platform/postgres/migrations/000024_change_set_confirmations.down.sql create mode 100644 internal/platform/postgres/migrations/000024_change_set_confirmations.up.sql create mode 100644 internal/platform/postgres/migrations/000025_processing_job_identity.down.sql create mode 100644 internal/platform/postgres/migrations/000025_processing_job_identity.up.sql create mode 100644 internal/platform/postgres/migrations/000026_job_failure_queue.down.sql create mode 100644 internal/platform/postgres/migrations/000026_job_failure_queue.up.sql create mode 100644 internal/platform/postgres/migrations/000027_worker_heartbeats.down.sql create mode 100644 internal/platform/postgres/migrations/000027_worker_heartbeats.up.sql create mode 100644 internal/platform/postgres/migrations/000028_release_change_sets.down.sql create mode 100644 internal/platform/postgres/migrations/000028_release_change_sets.up.sql create mode 100644 internal/platform/postgres/migrations/000029_release_rollback_policy.down.sql create mode 100644 internal/platform/postgres/migrations/000029_release_rollback_policy.up.sql create mode 100644 internal/platform/postgres/migrations/000030_release_audit_links.down.sql create mode 100644 internal/platform/postgres/migrations/000030_release_audit_links.up.sql create mode 100644 internal/platform/postgres/migrations/000031_source_connection_revisions.down.sql create mode 100644 internal/platform/postgres/migrations/000031_source_connection_revisions.up.sql create mode 100644 internal/platform/postgres/migrations_test.go create mode 100644 internal/platform/postgres/operation_requests.go create mode 100644 internal/platform/postgres/operation_wrappers.go create mode 100644 internal/platform/postgres/p2_identity_integration_test.go create mode 100644 internal/platform/postgres/p3_confirm_integration_test.go create mode 100644 internal/platform/postgres/p5_job_identity_integration_test.go create mode 100644 internal/platform/postgres/p6_dlq_integration_test.go create mode 100644 internal/platform/postgres/pp_import.go create mode 100644 internal/platform/postgres/pp_import_integration_test.go create mode 100644 internal/platform/postgres/processing_confirmations.go create mode 100644 internal/platform/postgres/production_delete_integration_test.go create mode 100644 internal/platform/postgres/publication_removal.go create mode 100644 internal/platform/postgres/queue_sql.go create mode 100644 internal/platform/postgres/r1_release_change_set_integration_test.go create mode 100644 internal/platform/postgres/r4_rollback_levels_integration_test.go create mode 100644 internal/platform/postgres/r5_manifest_seal_integration_test.go create mode 100644 internal/platform/postgres/r6_audit_chain_integration_test.go create mode 100644 internal/platform/postgres/release_audit_chain.go create mode 100644 internal/platform/postgres/release_authorization_integration_test.go create mode 100644 internal/platform/postgres/release_batches.go create mode 100644 internal/platform/postgres/release_batches_test.go create mode 100644 internal/platform/postgres/release_change_sets.go create mode 100644 internal/platform/postgres/release_execution_reads_test.go create mode 100644 internal/platform/postgres/release_execution_worker_store.go create mode 100644 internal/platform/postgres/release_execution_worker_store_test.go create mode 100644 internal/platform/postgres/release_executions.go create mode 100644 internal/platform/postgres/release_executions_test.go create mode 100644 internal/platform/postgres/release_plan.go create mode 100644 internal/platform/postgres/release_plan_test.go create mode 100644 internal/platform/postgres/review_and_seal_integration_test.go create mode 100644 internal/platform/postgres/reviews.go create mode 100644 internal/platform/postgres/reviews_test.go create mode 100644 internal/platform/postgres/s2_snapshot_history_integration_test.go create mode 100644 internal/platform/postgres/s4_refresh_progress_integration_test.go create mode 100644 internal/platform/postgres/s6_scope_policy_integration_test.go create mode 100644 internal/platform/postgres/s8_change_set_contract_integration_test.go create mode 100644 internal/platform/postgres/scope_rules.go create mode 100644 internal/platform/postgres/scope_rules_integration_test.go create mode 100644 internal/platform/postgres/source_assets.go create mode 100644 internal/platform/postgres/source_assets_test.go create mode 100644 internal/platform/postgres/source_catalog.go create mode 100644 internal/platform/postgres/source_catalog_integration_test.go create mode 100644 internal/platform/postgres/source_connections.go create mode 100644 internal/platform/postgres/source_connections_integration_test.go create mode 100644 internal/platform/postgres/source_connections_test.go create mode 100644 internal/platform/postgres/source_scan_integration_test.go create mode 100644 internal/platform/postgres/source_scan_migration_integration_test.go create mode 100644 internal/platform/postgres/source_scans.go create mode 100644 internal/platform/postgres/source_scans_test.go create mode 100644 internal/platform/postgres/source_settings.go create mode 100644 internal/platform/postgres/store.go create mode 100644 internal/platform/postgres/store_test.go create mode 100644 internal/platform/postgres/test_publications.go create mode 100644 internal/platform/postgres/test_publications_integration_test.go create mode 100644 internal/platform/postgres/test_publications_test.go create mode 100644 internal/platform/postgres/ui_settings.go create mode 100644 internal/platform/postgres/ui_settings_integration_test.go create mode 100644 internal/platform/postgres/ui_settings_test.go create mode 100644 internal/platform/postgres/v1_migration_seed_test.go create mode 100644 internal/platform/postgres/v1_scheduler_integration_test.go create mode 100644 internal/platform/postgres/v1_source_settings_integration_test.go create mode 100644 internal/platform/spool/artifact.go create mode 100644 internal/platform/spool/artifact_test.go create mode 100644 internal/platform/weknora/client.go create mode 100644 internal/platform/weknora/client_test.go create mode 100644 internal/platform/weknora/secret.go create mode 100644 internal/platform/weknora/secret_test.go create mode 100644 internal/platform/weknora/types.go create mode 100644 internal/releaseadapter/v071/archive.go create mode 100644 internal/releaseadapter/v071/archive_test.go create mode 100644 internal/releaseadapter/v071/contract_inspector.go create mode 100644 internal/releaseadapter/v071/contract_inspector_test.go create mode 100644 internal/releaseadapter/v071/document_export.go create mode 100644 internal/releaseadapter/v071/document_export_test.go create mode 100644 internal/releaseadapter/v071/document_target.go create mode 100644 internal/releaseadapter/v071/document_target_test.go create mode 100644 internal/releaseadapter/v071/object_export.go create mode 100644 internal/releaseadapter/v071/object_export_test.go create mode 100644 internal/releaseadapter/v071/object_import.go create mode 100644 internal/releaseadapter/v071/object_import_test.go create mode 100644 internal/releaseadapter/v071/postgres_export.go create mode 100644 internal/releaseadapter/v071/postgres_export_integration_test.go create mode 100644 internal/releaseadapter/v071/postgres_export_test.go create mode 100644 internal/releaseadapter/v071/postgres_import.go create mode 100644 internal/releaseadapter/v071/postgres_import_integration_test.go create mode 100644 internal/releaseadapter/v071/s3_object_store.go create mode 100644 internal/releaseadapter/v071/s3_object_store_integration_test.go create mode 100644 internal/releaseadapter/v071/search.go create mode 100644 internal/releaseadapter/v071/search_import.go create mode 100644 internal/releaseadapter/v071/search_import_test.go create mode 100644 internal/releaseadapter/v071/search_test.go create mode 100644 internal/releaseadapter/v071/three_lane_integration_test.go create mode 100644 internal/releaseadapter/v071/verifier.go create mode 100644 internal/releaseadapter/v071/verifier_test.go create mode 100644 internal/releasebundle/manifest.go create mode 100644 internal/releasebundle/manifest_test.go create mode 100644 internal/workflows/catalogrefresh/handler.go create mode 100644 internal/workflows/catalogrefresh/handler_test.go create mode 100644 internal/workflows/release/handler.go create mode 100644 internal/workflows/release/handler_test.go create mode 100644 internal/workflows/release/routing_publisher.go create mode 100644 internal/workflows/release/routing_publisher_test.go create mode 100644 internal/workflows/release/sidecar_publisher.go create mode 100644 internal/workflows/release/sidecar_publisher_test.go create mode 100644 internal/workflows/sourcescan/file_scanner.go create mode 100644 internal/workflows/sourcescan/handler.go create mode 100644 internal/workflows/sourcescan/handler_test.go create mode 100644 internal/workflows/testweknora/legacy_verify.go create mode 100644 internal/workflows/testweknora/legacy_verify_test.go create mode 100644 internal/workflows/testweknora/poll.go create mode 100644 internal/workflows/testweknora/poll_test.go create mode 100644 internal/workflows/testweknora/remove.go create mode 100644 internal/workflows/testweknora/remove_test.go create mode 100644 internal/workflows/testweknora/upload.go create mode 100644 internal/workflows/testweknora/upload_test.go create mode 100644 internal/workflows/testweknora/validate.go create mode 100644 internal/workflows/testweknora/validate_test.go create mode 100644 scripts/__pycache__/import_display_metadata.cpython-313.pyc create mode 100644 scripts/__pycache__/live_catalog_refresh.cpython-313.pyc create mode 100644 scripts/__pycache__/source_catalog_import.cpython-313.pyc create mode 100644 scripts/__pycache__/test_import_display_metadata.cpython-313-pytest-8.3.4.pyc create mode 100644 scripts/__pycache__/test_import_display_metadata.cpython-313.pyc create mode 100644 scripts/__pycache__/test_live_catalog_refresh_progress.cpython-313-pytest-8.3.4.pyc create mode 100644 scripts/__pycache__/test_source_catalog_import.cpython-313-pytest-8.3.4.pyc create mode 100644 scripts/__pycache__/test_source_catalog_import.cpython-313.pyc create mode 100644 scripts/accept_catalog_controls.py create mode 100644 scripts/accept_catalog_route_canary.py create mode 100644 scripts/accept_document_workbench.py create mode 100644 scripts/accept_document_workbench_v2.py create mode 100644 scripts/accept_live_catalog_refresh.py create mode 100644 scripts/accept_monitor_simulation.py create mode 100644 scripts/accept_refresh_failure_message.py create mode 100644 scripts/accept_release_planner.py create mode 100644 scripts/accept_source_catalog.py create mode 100644 scripts/accept_source_links.py create mode 100644 scripts/accept_system_settings.py create mode 100644 scripts/ensure_local_routing_kbs.py create mode 100644 scripts/import_display_metadata.py create mode 100644 scripts/live_catalog_refresh.py create mode 100644 scripts/source_catalog_import.py create mode 100644 scripts/stage_pp_routing_canary.py create mode 100644 scripts/test_import_display_metadata.py create mode 100644 scripts/test_live_catalog_refresh_progress.py create mode 100644 scripts/test_source_catalog_import.py create mode 100644 secrets/alist_api_token.example create mode 100644 secrets/pgpass.example create mode 100644 secrets/postgres_password.example create mode 100644 secrets/test_weknora_api_key.example create mode 100644 spool/catalog-route/7493b15ba5d0bd18f49f462f0989fd0fb90568d9a45595535b409f67f61bd68a.txt create mode 100644 spool/manual/e2e-fullchain-20260912.md create mode 100644 spool/manual/p3-durable-restart-20260905.md create mode 100644 spool/manual/p3-e2e-20260905.md create mode 100644 spool/manual/p3-recovery-20260905.md create mode 100644 spool/manual/p3-restart-20260905.md create mode 100644 spool/manual/p4-cross-env-20260905.md create mode 100644 web/index.html create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/src/App.test.tsx create mode 100644 web/src/App.tsx create mode 100644 web/src/DocumentWorkbench.test.tsx create mode 100644 web/src/DocumentWorkbench.tsx create mode 100644 web/src/ReleasePlanner.test.tsx create mode 100644 web/src/ReleasePlanner.tsx create mode 100644 web/src/ScopeRuleEditor.test.tsx create mode 100644 web/src/ScopeRuleEditor.tsx create mode 100644 web/src/SnapshotHistory.test.tsx create mode 100644 web/src/SnapshotHistory.tsx create mode 100644 web/src/SourceCatalogBrowser.test.tsx create mode 100644 web/src/SourceCatalogBrowser.tsx create mode 100644 web/src/SourceLinkManager.test.tsx create mode 100644 web/src/SourceLinkManager.tsx create mode 100644 web/src/SystemSettings.test.tsx create mode 100644 web/src/SystemSettings.tsx create mode 100644 web/src/WeKnoraMonitor.test.tsx create mode 100644 web/src/WeKnoraMonitor.tsx create mode 100644 web/src/assets.test.ts create mode 100644 web/src/assets.ts create mode 100644 web/src/documentFixture.ts create mode 100644 web/src/documentWorkbench.css create mode 100644 web/src/documentWorkbench.test.ts create mode 100644 web/src/documentWorkbench.ts create mode 100644 web/src/jobs.test.ts create mode 100644 web/src/jobs.ts create mode 100644 web/src/main.tsx create mode 100644 web/src/monitor.test.ts create mode 100644 web/src/monitor.ts create mode 100644 web/src/operationKey.test.ts create mode 100644 web/src/operationKey.ts create mode 100644 web/src/releasePlanner.css create mode 100644 web/src/releasePlanner.test.ts create mode 100644 web/src/releasePlanner.ts create mode 100644 web/src/responsiveStyles.test.ts create mode 100644 web/src/reviewRelease.test.ts create mode 100644 web/src/reviewRelease.ts create mode 100644 web/src/scopeRules.test.ts create mode 100644 web/src/scopeRules.ts create mode 100644 web/src/sourceCatalog.test.ts create mode 100644 web/src/sourceCatalog.ts create mode 100644 web/src/sourceCatalogBrowser.css create mode 100644 web/src/sourceConnections.test.ts create mode 100644 web/src/sourceConnections.ts create mode 100644 web/src/sourceLinkManager.css create mode 100644 web/src/styles.css create mode 100644 web/src/systemSettings.css create mode 100644 web/src/testPublications.test.ts create mode 100644 web/src/testPublications.ts create mode 100644 web/src/uiSettings.test.ts create mode 100644 web/src/uiSettings.ts create mode 100644 web/src/weknoraMonitor.css create mode 100644 web/src/workspace.test.ts create mode 100644 web/src/workspace.ts create mode 100644 web/src/workspaceContext.test.tsx create mode 100644 web/src/workspaceContext.tsx create mode 100644 web/tsconfig.json create mode 100644 web/vite.config.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..84bc799 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.gitignore +.env +.env.* +!.env.example +bin +coverage.out +secrets +web/dist +web/node_modules +web/tsconfig.tsbuildinfo diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f9229a7 --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +APP_ADDR=:8080 +DATABASE_URL=postgres://governance@postgres:5432/governance?sslmode=disable +PGPASSFILE=/run/secrets/pgpass +WORKER_ID=worker-1 +WORKER_POLL_INTERVAL=1s +WORKER_LEASE_DURATION=30s +WORKER_RETRY_DELAY=5s +RELEASE_WORKER_ENABLED=false +PRODUCTION_RELEASE_REQUESTS_ENABLED=false +PP_SCAN_ENABLED=false +PP_SCAN_HOST_DATA_ROOT=/home/zheromiku/dobot-portal-sync/data +PP_SCAN_CONTAINER_DATA_ROOT=/home/zheromiku/dobot-portal-sync/data +TEST_WEKNORA_ENABLED=false +TEST_WEKNORA_ALLOW_DOCKER_HTTP=false +TEST_WEKNORA_BASE_URL= +MINERU_BASE_URL= +TEST_WEKNORA_KB_ID= +TEST_WEKNORA_API_KEY_FILE=/run/secrets/test_weknora_api_key +GOVERNANCE_SPOOL_ROOT=/var/lib/governance/spool +TEST_WEKNORA_HTTP_TIMEOUT=60s +TEST_WEKNORA_POLL_INITIAL=10s +TEST_WEKNORA_POLL_MAX=60s +TEST_WEKNORA_STALLED_AFTER=30m +TEST_WEKNORA_MAX_UPLOAD_BYTES=52428800 +TEST_ACCEPTANCE_POLICY_VERSION=structural-v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..abf104d --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Go +/bin/ +/coverage.out + +# Web +/web/node_modules/ +/web/dist/ +/web/tsconfig.tsbuildinfo + +# Local runtime +.env +.env.* +!.env.example +/tmp/ +/data/ +/secrets/* +!/secrets/*.example + +# Editors and OS +.idea/ +.vscode/ +.DS_Store + +# Local artifacts (binaries, staging scripts, backups) +/release-diag +/worker +/apply_s5_patch.py +/tmp_* +.style-backup-*/ +.hermes/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1a69c50 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/worker ./cmd/worker \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/release-worker ./cmd/release-worker \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-publication-seed ./cmd/test-publication-seed \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-weknora-e2e-report ./cmd/test-weknora-e2e-report \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-weknora-probe ./cmd/test-weknora-probe + +FROM alpine:3.22 +ENV PYTHONDONTWRITEBYTECODE=1 +RUN apk add --no-cache ca-certificates python3 postgresql-client \ + && addgroup -g 65532 governance \ + && adduser -D -H -u 65532 -G governance governance +WORKDIR /app +COPY --from=build /out/ /app/ +COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py /app/scripts/ +USER 65532:65532 +CMD ["/app/api"] diff --git a/Dockerfile.web b/Dockerfile.web new file mode 100644 index 0000000..1a1d4c8 --- /dev/null +++ b/Dockerfile.web @@ -0,0 +1,11 @@ +FROM node:22-alpine AS build +WORKDIR /src +COPY web/package.json web/package-lock.json ./ +RUN npm ci --no-audit --no-fund +COPY web/ ./ +RUN npm run build + +FROM nginxinc/nginx-unprivileged:1.29-alpine +COPY deploy/nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build /src/dist /usr/share/nginx/html +EXPOSE 8080 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f63af64 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +GO ?= go + +.PHONY: test vet build web-test web-typecheck web-build verify + +test: + $(GO) test ./... + +vet: + $(GO) vet ./... + +build: + $(GO) build ./cmd/... + +web-test: + npm --prefix web test + +web-typecheck: + npm --prefix web run typecheck + +web-build: + npm --prefix web run build + +verify: test vet build web-test web-typecheck web-build diff --git a/README.md b/README.md index 219af4b..3332bc0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,207 @@ -# knowledge-governance +# 知识同步治理平台 +独立的知识治理控制平面。当前包含P1持久任务队列、P2统一来源与PP历史接管、P3本地/测试WeKnora编排验收、P4来源只读扫描、P5知识工程师审核与封批门禁、默认关闭的P6三通道生产发布与回滚Worker，以及P7持久快照ChangeSet、真实运行监控和每50项一批的发布进度。 + +## 安全边界 + +- 本工程不修改 WeKnora、PP、AList、钉钉或 DobotAI 源码。 +- PP接管与核验只读取指定的本地账本、manifest和测试WeKnora；不请求PP写接口，不修改来源文件。 +- 普通Worker领取演示任务、`test.upload/test.poll/test.validate`和显式启用的`source.scan.pp`；只有普通Worker可挂载测试WeKnora Key和只读PP快照。 +- Release Worker默认`RELEASE_WORKER_ENABLED=false`，禁用时不读取release config/生产secret，不连接生产数据面或治理数据库。 +- P5封批只生成治理侧`sealed`记录，不创建生产发布Job；5条真实候选仍为待人工审核。 +- P6生产请求另受`PRODUCTION_RELEASE_REQUESTS_ENABLED=false`保护；开启时还必须有服务端固定actor和0600 Bearer token文件。当前未授权、未配置、未发布生产。 +- 凭据文件位于`secrets/`且被Git忽略；仓库只保留示例文件。 + +## 目录 + +```text +cmd/api HTTP API +cmd/worker 普通任务Worker +cmd/release-worker 默认关闭的发布Worker +cmd/pp-ledger-import PP历史账本一次性导入工具 +internal/jobs 任务模型、租约和Worker循环 +internal/governance 统一来源治理领域模型 +internal/platform PostgreSQL仓库与Migration +internal/releasebundle v0.7.1发布bundle契约与摘要 +internal/releaseadapter v0.7.1 PostgreSQL/ES/S3 Sidecar +internal/httpapi HTTP路由 +web React TypeScript状态页 +deploy Nginx配置 +docs 验收计划 +``` + +## P2.1来源数据底座 + +Migration 2创建五张空表： + +```text +source_connections 来源连接配置，只保存secret引用名 +source_items 来源内稳定对象身份与变化状态 +content_objects 按完整SHA-256唯一的内容对象 +source_item_versions 来源对象版本与内容对象关联 +test_publications 测试知识库导入、处理、检索状态 +``` + +`source_change`、`ingest_state`、`processing_state`和`retrieval_state`相互独立。只有同一来源连接内的`stable_external_id`唯一；只有完整小写SHA-256相同才能共享内容对象。 + +P2.1不会自动创建来源连接、读取PP账本或上传文档。完整验收步骤见`docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md`。 + +## P2.2 PP历史账本接管 + +一次性工具将PP append-only账本折叠为当前终态： + +```text +1093 条历史事件 +→ 559 个来源对象和版本 +→ 530 个SHA内容对象 +→ 530 条历史测试发布 +→ 29 个content_alias共享canonical内容和Knowledge ID +``` + +75条旧`completed`记录缺少SHA时，只允许从locator hash匹配的本地manifest恢复，并重新校验文件大小和实际SHA-256。单文件继续遵守旧连接器50MiB上限。连接器默认关闭，历史发布的`retrieval_state`保持`unpublished`，等待未来对目标WeKnora逐条只读核验。 + +工具只执行幂等INSERT，不执行UPDATE或DELETE。完整验收步骤见`docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md`。 + +## P2.3 PP完整清单与知识资产页 + +P2.3将manifest中未进入旧账本的54个超限来源纳入治理视图： + +```text +559 个历史终态 + 54 个over_limit阻断项 = 613 个来源对象 +530 个内容对象 +613 个来源版本 +530 条历史测试发布 +历史已同步内容计划重新上传数 = 0 +``` + +超限版本没有虚假ContentObject或TestPublication，保持`content_state=over_limit`和阻断状态。`GET /api/v1/source-items`提供只读分页清单，支持`state=completed|content_alias|over_limit`、`limit`和`offset`；响应不包含来源URL、URI哈希、raw记录或Knowledge ID。Web“知识资产”模块读取该真实API。完整验收见`docs/P2_PP_INVENTORY_ACCEPTANCE.md`。 + +## P3 WeKnora测试编排 + +普通Worker从只读Spool流式上传到测试WeKnora，持久保存远端身份，按单调`poll_sequence`轮询，并在完成后执行Knowledge/Chunk验收。API和Web只返回安全投影；人工重试只允许失败或停滞且仍有已验证制品的Publication。530条PP历史Knowledge已在测试服务器完成在线只读核验，没有重新上传。 + +本地、测试服务器、失败恢复、Worker重启和浏览器验收见`docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md`。 + +## P4.1 来源连接只读工作台 + +`GET /api/v1/source-connections`返回来源ID、类型、名称、启用状态、是否配置调度、健康枚举、对象数和更新时间。查询与响应均不包含`base_url`、`secret_ref`或原始健康错误正文。Web“来源管理”模块展示真实PostgreSQL数据，保留左侧导航和顶部多标签。 + +P4.1完整验收见`docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md`。 + +## P4.2 PP只读扫描与变化入账 + +普通Worker在`PP_SCAN_ENABLED=true`且三个安全路径完整时注册`source.scan.pp` Handler。Compose仅给普通Worker只读挂载PP数据目录；API、Web和Release Worker均不挂载。Scanner拒绝越界、软链接、非普通文件和不完整manifest；54个历史`over_limit`项允许其制品未下载，但仍保持阻断。 + +Migration 5新增`source_scans`和`source_scan_changes`。触发Scan与创建Job原子执行，每个连接最多一个活动Scan；完成时聚合、变化与来源健康原子写入；失败只保存固定错误码。安全API支持触发、扫描历史和连接作用域详情，Web展示真实聚合与变化记录，不返回来源ID/revision/SHA/路径/URL或凭据。 + +真实PP快照只读扫描结果： + +```text +observed=613 +unchanged=559 +over_limit=54 +added=0 +updated=0 +missing=0 +planned_uploads=0 +``` + +过期`running`租约已由常驻Worker真实重领并以`attempt=2`完成；并发第二次触发返回`409`。完整迁移、备份、恢复、API和Firefox验收见`docs/P42_SOURCE_SCAN_ACCEPTANCE.md`。 + +## P5 知识工程师审核与封批 + +Migration 6新增追加式`publication_reviews`、`release_batches`和`release_batch_items`。审核决定绑定最新Acceptance run与Publication时间戳；批准仅允许完整结构门禁通过的候选，`retrieval_hit=manual_required`保持人工判断。创建draft和sealed封批均原子验证整个项目集合，任一项目失效则整批拒绝。 + +“发布管理”模块展示真实审核候选、固定门禁、决定可见性、草稿批次和封批记录，并明确显示“生产门禁保持关闭”“Release Worker保持关闭”“尚未发布”。当前535条Publication中，530条无逐条Acceptance的PP legacy不自动进入人工队列；5条manual合成验收项显示为ready/pending，持久库没有代替知识工程师创建Review或Batch。 + +P5 Store与HTTP写链路已在disposable PostgreSQL和临时API中验证`pending→approved→draft→sealed`，完成后临时数据库与临时密钥文件均清理；持久环境只应用空Migration 6。完整验收见`docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md`。 + +## P6 生产发布Worker与回滚 + +Migration 7新增`release_executions`与`release_item_executions`。只有受保护的生产授权请求才能把P5 sealed batch原子转换为`release.publish` Job；rollback只允许针对完整成功的publish execution。终态失败可在同一Bearer门禁下重排，仅重置failed items并重新验证sealed snapshot。API和Release Worker使用两个独立默认关闭的门禁，UI只展示执行审计，不提供一键生产授权按钮。 + +Release Worker按Knowledge Base精确路由，固定WeKnora `v0.7.1 / migration 74 / elasticsearch_v8 / 1024维`契约；同一release package协调复制以下内容且不触发生产解析/Embedding： + +```text +PostgreSQL: knowledges/chunks/knowledge_tags/relations/resources/bindings +对象存储: referenced assets，staging→SHA校验→相同relative key +Elasticsearch: chunk text+vectors，先upsert新文档再清理同Knowledge旧_id +``` + +每个release revision写入0700目录中的0600 bundle并重验整包SHA。失败恢复旧bundle；首次发布无旧revision时禁用Knowledge/Chunk/ES，不删除行、文档或对象。生产WeKnora官方API最终验证Knowledge、Chunk数和knowledge-scoped检索。 + +本地真实验收使用完整v0.7.1 75个migration的source/target PostgreSQL、真实MinIO、状态化ES协议服务，已通过`publish v1 → publish v2 → rollback v1`以及PG disable、S3 Open/Put/Copy、Worker受限配置装配。模板见`config/release.v071.example.json`，完整边界与结果见`docs/P6_RELEASE_WORKER_ACCEPTANCE.md`。 + +当前生产状态保持： + +```text +PRODUCTION_RELEASE_REQUESTS_ENABLED=false +RELEASE_WORKER_ENABLED=false +release executions/items/jobs=0/0/0 +``` + +## 本地验证 + +要求Go 1.26、Node.js 22和npm。 + +```bash +make test +make vet +make build +make web-test +make web-typecheck +make web-build +``` + +## Docker Compose（由操作者执行） + +先创建仅限本机的密钥文件： + +```bash +cp secrets/postgres_password.example secrets/postgres_password +cp secrets/pgpass.example secrets/pgpass +# 将两个文件中的CHANGE_ME_LOCAL_ONLY替换为同一随机本地密码 +chmod 600 secrets/postgres_password secrets/pgpass +docker compose up --build -d +``` + +Compose默认让三个Go进程以`1000:1000`运行，以读取宿主机Owner为1000且权限为600的密钥文件。若操作者UID/GID不同，启动前设置`APP_UID`和`APP_GID`为密钥文件Owner。 + +访问： + +- 状态页：`http://127.0.0.1:8088/` +- 同源健康检查：`http://127.0.0.1:8088/health` + +停止时保留数据库： + +```bash +docker compose down +``` + +不要执行`docker compose down -v`，除非明确决定删除本地数据库数据。 + +## P1 API + +创建普通演示任务： + +```bash +curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \ + -H 'Content-Type: application/json' \ + -d '{"type":"demo","payload":{},"idempotency_key":"demo:001"}' +``` + +创建可用于租约恢复验收的30秒任务： + +```bash +curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \ + -H 'Content-Type: application/json' \ + -d '{"type":"demo.sleep","payload":{"sleep_ms":30000},"idempotency_key":"demo:sleep:001"}' +``` + +查看最近任务： + +```bash +curl -fsS 'http://127.0.0.1:8088/api/v1/jobs?limit=50' +``` + +完整人工步骤见`docs/P1_ACCEPTANCE_TEST_PLAN.md`。 diff --git a/api/openapi.yaml b/api/openapi.yaml new file mode 100644 index 0000000..c9ec89e --- /dev/null +++ b/api/openapi.yaml @@ -0,0 +1,3145 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Knowledge Governance API", + "version": "v1", + "description": "独立知识治理控制平面API。所有响应使用安全投影；不返回来源凭据、签名下载地址、原始内容、内部目标连接或生产密钥。" + }, + "servers": [ + { + "url": "/", + "description": "当前治理控制平面" + } + ], + "tags": [ + { + "name": "Health" + }, + { + "name": "Jobs" + }, + { + "name": "Source catalog" + }, + { + "name": "Monitoring" + }, + { + "name": "Scope" + }, + { + "name": "Source connections" + }, + { + "name": "Review" + }, + { + "name": "Release" + }, + { + "name": "Test processing" + }, + { + "name": "Settings" + } + ], + "paths": { + "/health": { + "get": { + "tags": [ + "Health" + ], + "operationId": "getHealth", + "summary": "读取服务健康状态", + "responses": { + "200": { + "description": "服务可响应", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Health" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/jobs": { + "get": { + "tags": [ + "Jobs" + ], + "operationId": "listJobs", + "summary": "列出最近治理任务", + "parameters": [ + { + "$ref": "#/components/parameters/Limit" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "post": { + "tags": [ + "Jobs" + ], + "operationId": "createJob", + "summary": "创建幂等后台任务", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobCreateRequest" + } + } + } + }, + "responses": { + "201": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-items": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "listSourceItems", + "summary": "列出安全来源文件投影", + "parameters": [ + { + "$ref": "#/components/parameters/State" + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/sources": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "listCatalogSources", + "summary": "列出当前可浏览来源快照", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/nodes": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "listCatalogNodes", + "summary": "按父目录分页列出当前快照直接子项", + "parameters": [ + { + "$ref": "#/components/parameters/ConnectionID" + }, + { + "name": "parent_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "maxLength": 36 + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/change-sets/latest": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "getLatestCatalogChangeSet", + "summary": "读取某来源最新持久ChangeSet", + "parameters": [ + { + "$ref": "#/components/parameters/ConnectionID" + }, + { + "name": "action", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "add", + "update", + "move", + "move_scope", + "delete", + "no_change" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/routing-capabilities": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "getCatalogRoutingCapabilities", + "summary": "读取对内和对外测试路由是否已配置", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/route": { + "post": { + "tags": [ + "Source catalog" + ], + "operationId": "setCatalogPolicy", + "summary": "写入目录或文件的期望处理策略", + "description": "当前接口只修改治理策略和审计记录，不代表已经上传、解析或发布。调用方必须为每次用户动作生成新的 Idempotency-Key；网络重试复用原键会返回原结果且 operation=replayed，同键不同请求返回409。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CatalogRouteRequest" + } + } + } + }, + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/sources/{id}/refresh": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Source catalog" + ], + "operationId": "refreshCatalogSource", + "summary": "创建来源目录刷新任务", + "description": "调用方必须为每次用户动作生成新的 Idempotency-Key；网络重试复用原键会返回同一刷新任务且 operation=replayed。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/monitor/overview": { + "get": { + "tags": [ + "Monitoring" + ], + "operationId": "getMonitorOverview", + "summary": "读取主机、服务和知识库处理概览", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/monitor/knowledge-bases/{id}/files": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Monitoring" + ], + "operationId": "listKnowledgeBaseFiles", + "summary": "按处理状态分页列出知识库文件", + "parameters": [ + { + "name": "status", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "pending", + "processing", + "finalizing", + "completed", + "failed", + "cancelled" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/scope-rules": { + "get": { + "tags": [ + "Scope" + ], + "operationId": "listScopeRules", + "summary": "读取来源范围规则", + "parameters": [ + { + "$ref": "#/components/parameters/ConnectionID" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "put": { + "tags": [ + "Scope" + ], + "operationId": "setScopeRule", + "summary": "创建或更新范围规则", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScopeRuleMutation" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "delete": { + "tags": [ + "Scope" + ], + "operationId": "clearScopeRule", + "summary": "清除范围规则", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScopeRuleMutation" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-connections": { + "get": { + "tags": [ + "Source connections" + ], + "operationId": "listSourceConnections", + "summary": "列出安全来源连接投影", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-connections/{id}/scan": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Source connections" + ], + "operationId": "triggerSourceScan", + "summary": "创建旧PP来源扫描任务", + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/source-connections/{id}/scans": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Source connections" + ], + "operationId": "listSourceScans", + "summary": "分页读取来源扫描记录", + "parameters": [ + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-connections/{id}/scans/{scan_id}": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "name": "scan_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64 + } + } + ], + "get": { + "tags": [ + "Source connections" + ], + "operationId": "getSourceScan", + "summary": "读取单次扫描及其安全变更投影", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/review-candidates": { + "get": { + "tags": [ + "Review" + ], + "operationId": "listReviewCandidates", + "summary": "分页列出可审核测试知识", + "parameters": [ + { + "name": "review_state", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "pending", + "approved", + "rejected", + "not_required" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/review-candidates/{id}/decision": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Review" + ], + "operationId": "reviewCandidate", + "summary": "提交人工审核决定", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReviewDecisionRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-plan": { + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleasePlan", + "summary": "预览当前测试到生产的发布计划", + "description": "当前是实时预览，不是不可变Release ChangeSet；R1会替换为持久发布ChangeSet。", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-batches": { + "get": { + "tags": [ + "Release" + ], + "operationId": "listReleaseBatches", + "summary": "分页列出草稿或已封存发布批次", + "parameters": [ + { + "name": "state", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "draft", + "sealed", + "cancelled" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "post": { + "tags": [ + "Release" + ], + "operationId": "createReleaseBatch", + "summary": "创建草稿发布批次", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReleaseBatchCreateRequest" + } + } + } + }, + "responses": { + "201": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-batches/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleaseBatch", + "summary": "读取发布批次详情", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-batches/{id}/seal": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Release" + ], + "operationId": "sealReleaseBatch", + "summary": "封存草稿发布批次，不触发生产发布", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReleaseBatchSealRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-batches/{id}/authorize-production": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Release" + ], + "operationId": "authorizeProductionRelease", + "summary": "受门禁保护的生产发布授权", + "description": "默认关闭。调用方不得提交审核人身份；服务端从受限配置读取固定授权身份。", + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-executions": { + "get": { + "tags": [ + "Release" + ], + "operationId": "listReleaseExecutions", + "summary": "分页读取生产发布和回滚审计", + "parameters": [ + { + "name": "operation", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "publish", + "rollback" + ] + } + }, + { + "name": "state", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "pending", + "running", + "rolling_back", + "completed", + "failed", + "rolled_back", + "rollback_failed" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-executions/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleaseExecution", + "summary": "读取单次生产发布或回滚详情", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-executions/{id}/authorize-rollback": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Release" + ], + "operationId": "authorizeProductionRollback", + "summary": "受门禁保护的生产回滚授权", + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-executions/{id}/retry": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Release" + ], + "operationId": "retryProductionRelease", + "summary": "受门禁保护地重试失败发布", + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/release-capabilities": { + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleaseCapabilities", + "summary": "读取不含敏感信息的生产门禁能力", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/ui-settings": { + "get": { + "tags": [ + "Settings" + ], + "operationId": "getActiveUISettings", + "summary": "读取当前活动的安全刷新和告警设置", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/ui-settings/candidates": { + "post": { + "tags": [ + "Settings" + ], + "operationId": "createUISettingsCandidate", + "summary": "创建待验证的安全设置版本", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UISettingsCandidateRequest" + } + } + } + }, + "responses": { + "201": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/ui-settings/{id}/activate": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Settings" + ], + "operationId": "activateUISettings", + "summary": "验证并原子启用设置版本", + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/ID" + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ] + } + }, + "/api/v1/test-publications": { + "get": { + "tags": [ + "Test processing" + ], + "operationId": "listTestPublications", + "summary": "按处理状态分页读取测试发布记录", + "parameters": [ + { + "name": "processing_state", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "pending", + "processing", + "finalizing", + "completed", + "failed", + "stalled" + ] + } + }, + { + "in": "query", + "name": "target_scope", + "description": "Filter the test queue by routing target: internal or external (P4).", + "required": false, + "schema": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/test-publications/start": { + "post": { + "tags": [ + "Test processing" + ], + "operationId": "startTestPublications", + "summary": "人工确认后启动测试解析任务", + "description": "当前按TestPublication确认；P3会把该操作改为按Catalog ChangeSet确认。网络重试复用原 Idempotency-Key。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StartTestPublicationsRequest" + } + } + } + }, + "responses": { + "202": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/test-publications/{id}/acceptance": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Test processing" + ], + "operationId": "listTestPublicationAcceptance", + "summary": "读取测试发布的安全验收记录", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/test-publications/{id}/retry": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Test processing" + ], + "operationId": "retryTestPublication", + "summary": "重新排队一份可重试测试发布", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/snapshots": { + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "listCatalogSnapshots", + "summary": "按来源分页读取历史快照清单", + "description": "Head当前生效快照排在最前；只返回安全投影，不返回snapshot hash或locator hash。", + "parameters": [ + { + "$ref": "#/components/parameters/ConnectionID" + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/snapshots/{id}/nodes": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "get": { + "tags": [ + "Source catalog" + ], + "operationId": "listCatalogSnapshotNodes", + "summary": "只读分页查看历史快照中的目录节点", + "description": "该接口只读，绝不切换当前Head。content_sha256仅在来源实际提供时返回；未知即null，不伪造。", + "parameters": [ + { + "$ref": "#/components/parameters/ConnectionID" + }, + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-catalog/change-sets/{id}/confirm": { + "parameters": [ + { + "$ref": "#/components/parameters/ID" + } + ], + "post": { + "tags": [ + "Source catalog" + ], + "operationId": "confirmCatalogChangeSet", + "summary": "确认 ChangeSet 中选中的变更项并创建解析任务", + "description": "仅执行选中的 item_seqs；NO_CHANGE 永不入队；禁止/删除项形成测试库移除任务。重复确认返回第一次结果 operation=replayed；同键不同请求 409。绝不返回内部 locator hash。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConfirmChangeSetRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/job-failures": { + "get": { + "summary": "分页列出解析死信（DLQ）投影", + "description": "终端失败尝试的只读投影；原 jobs 与发布审计行永不移动或删除。", + "parameters": [ + { + "name": "state", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "open", + "requeued", + "abandoned" + ] + }, + "description": "默认 open" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50 + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0 + } + } + ], + "responses": { + "200": { + "description": "死信投影分页", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "job_id": { + "type": "string" + }, + "job_type": { + "type": "string" + }, + "publication_id": { + "type": "string" + }, + "source_item_id": { + "type": "string" + }, + "error_code": { + "type": "string" + }, + "attempt": { + "type": "integer" + }, + "max_attempts_at_failure": { + "type": "integer" + }, + "change_set_id": { + "type": "string" + }, + "action": { + "type": "string" + }, + "target_kb_id": { + "type": "string" + }, + "state": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "total": { + "type": "integer" + }, + "limit": { + "type": "integer" + }, + "offset": { + "type": "integer" + } + } + } + }, + "required": [ + "data" + ] + } + } + } + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + } + }, + "operationId": "listJobFailures" + } + }, + "/api/v1/test-publications/retry-batch": { + "post": { + "summary": "服务端批量重试失败发布", + "description": "一次事务内逐项返回 started/already_running/not_retryable；批量操作人、原因与幂等信封原子记录。最多100项，前端不得循环调用单项接口。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "publication_ids": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "type": "string" + } + }, + "operator_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "publication_ids", + "operator_ref", + "reason" + ] + } + } + } + }, + "responses": { + "200": { + "description": "批量重试结果（含逐项outcome）", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "batch_key": { + "type": "string" + }, + "total": { + "type": "integer" + }, + "started": { + "type": "integer" + }, + "already_running": { + "type": "integer" + }, + "not_retryable": { + "type": "integer" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "publication_id": { + "type": "string" + }, + "result": { + "type": "string", + "enum": [ + "started", + "already_running", + "not_retryable" + ] + }, + "detail": { + "type": "string" + } + } + } + } + }, + "required": [ + "batch_key", + "total", + "started", + "already_running", + "not_retryable", + "outcomes" + ] + }, + "operation": { + "type": "string", + "enum": [ + "created", + "replayed" + ] + } + }, + "required": [ + "data", + "operation" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "operationId": "retryPublicationsBatch" + } + }, + "/api/v1/monitor/metrics": { + "get": { + "operationId": "getMonitorMetrics", + "summary": "真实运行指标（Worker/队列/吞吐/成功率/耗时分布）", + "description": "冻结1小时与24小时统计窗口；样本不足的比率与分位数返回null而非0；活跃Worker按PostgreSQL新鲜租约去重计数，不按容器数猜测。模拟模式声明worker_source=simulation。", + "parameters": [], + "responses": { + "200": { + "description": "运行指标快照", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "object", + "properties": { + "observed_at": { + "type": "string", + "format": "date-time" + }, + "active_workers": { + "type": "integer" + }, + "worker_source": { + "type": "string", + "enum": [ + "pg_lease", + "simulation" + ] + }, + "queue": { + "type": "object", + "properties": { + "pending": { + "type": "integer" + }, + "running": { + "type": "integer" + }, + "retry": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "oldest_pending_age_seconds": { + "type": [ + "number", + "null" + ] + } + } + }, + "short_window": { + "$ref": "#/components/schemas/JobWindowMetrics" + }, + "long_window": { + "$ref": "#/components/schemas/JobWindowMetrics" + } + } + } + } + } + } + } + }, + "503": { + "description": "指标暂不可用", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/livez": { + "get": { + "operationId": "getLivez", + "summary": "存活探针（Liveness）", + "description": "仅回答“本进程能否服务HTTP”。绝不依赖PostgreSQL、WeKnora、MinerU或Worker：第三方故障时本端点必须保持200。", + "responses": { + "200": { + "description": "进程存活", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "observed_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "status", + "observed_at" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "tags": [ + "Health" + ] + } + }, + "/readyz": { + "get": { + "operationId": "getReadyz", + "summary": "就绪探针（Readiness）", + "description": "逐服务独立评估Liveness与Readiness（P10）。只有“已部署且已配置但当前不可工作”的服务阻塞就绪并返回503；未部署=not_applicable、缺配置=not_configured 仅警告不阻塞；结果绝不泄漏URL、密钥或主机。", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "503": { + "description": "存在已配置依赖当前不可接单", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "blocked": { + "type": "array", + "items": { + "type": "string" + } + }, + "not_configured": { + "type": "array", + "items": { + "type": "string" + } + }, + "observed_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "status", + "blocked", + "not_configured", + "observed_at" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + }, + "tags": [ + "Health" + ] + } + }, + "/api/v1/release-change-sets": { + "get": { + "tags": [ + "Release" + ], + "operationId": "listReleaseChangeSets", + "summary": "分页列出版本化发布计划集", + "parameters": [ + { + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + }, + "description": "每页数量，默认20上限100" + }, + { + "in": "query", + "name": "offset", + "schema": { + "type": "integer" + }, + "description": "偏移" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "post": { + "tags": [ + "Release" + ], + "operationId": "generateReleaseChangeSet", + "summary": "把当前实时发布计划固化为不可变版本化ChangeSet", + "description": "生成即冻结：绑定当前head快照集与确认过的目录ChangeSet集，按精确字段顺序与分隔符计算清单摘要。相同计划并发去重（唯一索引冲突则复用最新planned或取下一版本）。来源刷新绝不可能改变已固化清单。", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "created_by": { + "type": "string" + } + }, + "required": [ + "name", + "created_by" + ] + } + } + } + }, + "responses": { + "201": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-change-sets/latest": { + "get": { + "tags": [ + "Release" + ], + "operationId": "getLatestReleaseChangeSet", + "summary": "最新发布计划集（含冻结逐项清单）", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-change-sets/{id}": { + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleaseChangeSet", + "summary": "按ID读取发布计划集与清单", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-change-sets/{id}/confirm": { + "post": { + "tags": [ + "Release" + ], + "operationId": "confirmReleaseChangeSet", + "summary": "冻结planned发布计划集供正式批次引用", + "description": "planned转confirmed幂等重放；非法状态迁移返回409。确认后清单与摘要永不改变。", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "confirmed_by": { + "type": "string" + } + }, + "required": [ + "confirmed_by" + ] + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/OperationEnvelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/release-executions/{id}/audit-chain": { + "get": { + "tags": [ + "Release" + ], + "operationId": "getReleaseAuditChain", + "summary": "发布审计链（R6 只读投影）", + "description": "一次执行可追溯到批次、已确认发布计划集、绑定的来源快照与目录变更集、逐项审核/验收/来源身份，以及补偿回滚行。所有链路通过真实外键读取（迁移30），只返回安全标识、名称、计数与时间戳，绝不返回Bundle内部路径、凭据、Base URL或对象签名URL。", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "完整证据链", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "object" + } + }, + "required": [ + "data" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-connections/{id}/settings/revisions": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + }, + "description": "来源连接 ID" + } + ], + "get": { + "operationId": "listSourceSettingsRevisions", + "summary": "来源设置修订历史（V1）", + "description": "validated→active→retired 完整历史；仅安全投影：credential_ref 是凭据引用名，真实凭据不可回读。", + "tags": [ + "governance" + ], + "responses": { + "200": { + "description": "修订列表", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "revision_number": { + "type": "integer" + }, + "state": { + "enum": [ + "validated", + "active", + "retired", + "failed" + ] + }, + "name": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "credential_ref": { + "type": "string", + "description": "引用名，非密钥本身" + }, + "enabled": { + "type": "boolean" + }, + "refresh_interval_seconds": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "validated_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "activated_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "retired_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + } + }, + "operation": { + "enum": [ + "created", + "replayed" + ] + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + }, + "post": { + "operationId": "createSourceSettingsRevision", + "summary": "创建来源设置草稿修订（V1）", + "description": "校验通过落为 validated 修订，不影响运行态；需要 X-Governance-Action: source-settings 与 Idempotency-Key。", + "tags": [ + "governance" + ], + "parameters": [ + { + "in": "header", + "name": "Idempotency-Key", + "required": true, + "schema": { + "type": "string", + "maxLength": 128 + } + }, + { + "in": "header", + "name": "X-Governance-Action", + "required": true, + "schema": { + "type": "string", + "enum": [ + "source-settings" + ] + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "source_type", + "name", + "operator", + "schedule" + ], + "additionalProperties": false, + "properties": { + "source_type": { + "enum": [ + "pp", + "alist", + "dingtalk", + "manual" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255 + }, + "endpoint": { + "type": "string", + "maxLength": 2048 + }, + "credential_ref": { + "type": "string", + "maxLength": 255, + "description": "密钥引用名；真实凭据永不入库、永不回显" + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_interval_seconds": { + "type": "integer", + "nullable": true, + "description": "缺省/0=仅手动；非零最低300秒" + } + } + }, + "operator": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + } + } + } + }, + "responses": { + "201": { + "description": "修订已创建", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } + }, + "operation": { + "enum": [ + "created", + "replayed" + ] + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, + "/api/v1/source-connections/{id}/settings/revisions/{revision_id}/activate": { + "post": { + "operationId": "activateSourceSettingsRevision", + "summary": "激活来源设置修订（V1）", + "description": "原子切换：退休旧 active、同步运行态快照、由服务端按周期推导下次刷新时间；停用立即清空到期时间（停止新扫描与新任务），历史全部保留。", + "tags": [ + "governance" + ], + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "revision_id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "header", + "name": "Idempotency-Key", + "required": true, + "schema": { + "type": "string", + "maxLength": 128 + } + }, + { + "in": "header", + "name": "X-Governance-Action", + "required": true, + "schema": { + "type": "string", + "enum": [ + "source-settings" + ] + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "maxProperties": 0 + } + } + } + }, + "responses": { + "200": { + "description": "已激活", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "object", + "properties": { + "connection_id": { + "type": "string" + }, + "revision_id": { + "type": "string" + } + } + }, + "operation": { + "enum": [ + "created", + "replayed" + ] + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + } + }, + "components": { + "parameters": { + "ID": { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64 + } + }, + "IdempotencyKey": { + "name": "Idempotency-Key", + "in": "header", + "required": true, + "description": "每次用户动作使用一个新键；仅网络重试复用同一键。不得包含凭据或业务内容。", + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "ConnectionID": { + "name": "connection_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "maxLength": 64 + } + }, + "Limit": { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 20 + } + }, + "Offset": { + "name": "offset", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0, + "maximum": 1000000, + "default": 0 + } + }, + "State": { + "name": "state", + "in": "query", + "schema": { + "type": "string", + "maxLength": 32 + } + } + }, + "responses": { + "Envelope": { + "description": "安全数据响应", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope" + } + } + } + }, + "OperationEnvelope": { + "description": "可安全重试的目录操作响应；operation=created 表示首次执行，replayed 表示返回原操作结果。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperationEnvelope" + } + } + } + }, + "BadRequest": { + "description": "请求参数或状态非法", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "Forbidden": { + "description": "安全门禁或权限拒绝", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "NotFound": { + "description": "资源不存在", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "Conflict": { + "description": "资源已变更、状态冲突或重复动作", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "InternalError": { + "description": "安全的服务端错误码", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "IdempotencyConflict": { + "description": "资源已变更、状态冲突或重复动作", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + }, + "schemas": { + "Health": { + "type": "object", + "additionalProperties": false, + "required": [ + "status" + ], + "properties": { + "status": { + "type": "string", + "const": "ok" + } + } + }, + "Envelope": { + "type": "object", + "additionalProperties": false, + "required": [ + "data" + ], + "properties": { + "data": { + "description": "端点定义的安全投影" + } + } + }, + "OperationEnvelope": { + "type": "object", + "additionalProperties": false, + "required": [ + "data", + "operation" + ], + "properties": { + "data": { + "description": "端点定义的安全投影" + }, + "operation": { + "type": "string", + "enum": [ + "created", + "replayed" + ] + } + } + }, + "Error": { + "type": "object", + "additionalProperties": false, + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string", + "maxLength": 512 + } + } + }, + "JobCreateRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "idempotency_key" + ], + "properties": { + "type": { + "type": "string", + "maxLength": 100 + }, + "payload": { + "type": "object" + }, + "priority": { + "type": "integer" + }, + "max_attempts": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "idempotency_key": { + "type": "string", + "minLength": 1, + "maxLength": 255 + } + } + }, + "CatalogRouteRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "connection_id", + "node_ids", + "operator_ref", + "reason" + ], + "properties": { + "connection_id": { + "type": "string", + "maxLength": 64 + }, + "node_ids": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "type": "string", + "maxLength": 64 + } + }, + "policy": { + "type": "string", + "enum": [ + "internal", + "external", + "both", + "blocked" + ] + }, + "visibility": { + "type": "string", + "enum": [ + "internal", + "external", + "both", + "blocked" + ] + }, + "operator_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + } + }, + "ScopeRuleMutation": { + "type": "object", + "additionalProperties": false, + "required": [ + "target_type", + "operator_ref", + "reason" + ], + "properties": { + "target_type": { + "type": "string", + "enum": [ + "directory", + "item" + ] + }, + "connection_id": { + "type": "string", + "maxLength": 64 + }, + "directory_path": { + "type": "string", + "maxLength": 4096 + }, + "source_item_id": { + "type": "string", + "maxLength": 64 + }, + "visibility": { + "type": "string", + "enum": [ + "internal", + "external", + "both", + "blocked" + ] + }, + "operator_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + } + }, + "ReviewDecisionRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "decision", + "decided_visibility", + "reviewer_ref", + "decision_code" + ], + "properties": { + "decision": { + "type": "string", + "enum": [ + "approved", + "rejected" + ] + }, + "decided_visibility": { + "type": "string", + "enum": [ + "internal", + "external" + ] + }, + "reviewer_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "decision_code": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + } + }, + "ReleaseBatchCreateRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "created_by" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255 + }, + "created_by": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "publication_ids": { + "type": "array", + "maxItems": 100, + "items": { + "type": "string", + "maxLength": 64 + } + }, + "delete_publication_ids": { + "type": "array", + "maxItems": 100, + "items": { + "type": "string", + "maxLength": 64 + } + }, + "release_change_set_id": { + "type": "string", + "description": "R1正式批次绑定的confirmed发布计划集ID；绑定时所有请求项必须在该集冻结清单内且可发布，否则422" + } + } + }, + "ReleaseBatchSealRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "sealed_by" + ], + "properties": { + "sealed_by": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "UISettingsCandidateRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "document_refresh_seconds", + "resource_refresh_seconds", + "stalled_after_seconds", + "finalizing_after_seconds", + "updated_by" + ], + "properties": { + "document_refresh_seconds": { + "type": "integer", + "enum": [ + 15, + 30, + 60 + ] + }, + "resource_refresh_seconds": { + "type": "integer", + "enum": [ + 30, + 60 + ] + }, + "stalled_after_seconds": { + "type": "integer", + "enum": [ + 1800, + 3600 + ] + }, + "finalizing_after_seconds": { + "type": "integer", + "enum": [ + 7200, + 14400 + ] + }, + "updated_by": { + "type": "string", + "minLength": 2, + "maxLength": 128 + } + } + }, + "StartTestPublicationsRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "publication_ids", + "operator_ref", + "reason" + ], + "properties": { + "publication_ids": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "type": "string", + "maxLength": 64 + } + }, + "operator_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + } + }, + "ConfirmChangeSetRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "item_seqs", + "operator_ref", + "reason" + ], + "properties": { + "item_seqs": { + "type": "array", + "minItems": 1, + "maxItems": 200, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "operator_ref": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + } + }, + "JobWindowMetrics": { + "type": "object", + "properties": { + "window_seconds": { + "type": "integer" + }, + "jobs_started": { + "type": "integer" + }, + "jobs_completed": { + "type": "integer" + }, + "jobs_failed": { + "type": "integer" + }, + "success_rate": { + "type": [ + "number", + "null" + ] + }, + "failure_rate": { + "type": [ + "number", + "null" + ] + }, + "duration_avg_seconds": { + "type": [ + "number", + "null" + ] + }, + "duration_p50_seconds": { + "type": [ + "number", + "null" + ] + }, + "duration_p95_seconds": { + "type": [ + "number", + "null" + ] + } + } + } + } + } +} diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 0000000..0b305ae --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "errors" + "log" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/httpapi" + "knowledge-governance/internal/monitoring" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, getenv func(string) string) error { + config, err := appconfig.Load(getenv) + if err != nil { + return err + } + pool, err := pgxpool.New(ctx, config.DatabaseURL) + if err != nil { + return err + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return err + } + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return err + } + releaseAuthorizationToken := "" + if config.ProductionReleaseRequestsEnabled { + releaseAuthorizationToken, err = weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile) + if err != nil { + return errors.New("read production release authorization token") + } + } + // P10: with a mounted API key the probe verifies a real knowledge-base read + // (readiness); without one it degrades to the liveness-only HTTP check. + var weknoraReadiness func(context.Context) error + if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" { + if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil { + probeTimeout := config.TestWeKnoraHTTPTimeout + if probeTimeout > 10*time.Second || probeTimeout <= 0 { + probeTimeout = 10 * time.Second + } + if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil { + routingKB := config.InternalRoutingKnowledgeBaseID + weknoraReadiness = func(probeCtx context.Context) error { + _, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1) + return err + } + } + } + } + monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter}) + if err != nil { + return err + } + handler, err := httpapi.NewHandler(httpapi.Config{ + Store: postgresstore.NewStore(pool), + Monitor: monitorProvider, + InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID, + ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID, + ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled, + ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified, + ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef, + ProductionReleaseAuthorizationToken: releaseAuthorizationToken, + }) + if err != nil { + return err + } + server := &http.Server{ + Addr: config.AppAddr, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + } + + errCh := make(chan error, 1) + go func() { errCh <- server.ListenAndServe() }() + log.Printf("governance API listening on %s", config.AppAddr) + select { + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return server.Shutdown(shutdownCtx) + } +} diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go new file mode 100644 index 0000000..58076be --- /dev/null +++ b/cmd/api/main_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "os" + "strings" + "testing" +) + +func TestRunRequiresDatabaseURL(t *testing.T) { + err := run(context.Background(), func(string) string { return "" }) + if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") { + t.Fatalf("run() error = %v, want DATABASE_URL error", err) + } +} + +func TestAPIPropagatesProductionReleaseRequestGate(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + normalized := strings.Join(strings.Fields(string(source)), " ") + for _, required := range []string{ + "if config.ProductionReleaseRequestsEnabled {", + "weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)", + "ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled", + "ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified", + "ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef", + "ProductionReleaseAuthorizationToken: releaseAuthorizationToken", + } { + if !strings.Contains(normalized, required) { + t.Fatalf("API production authorization wiring missing %q", required) + } + } +} + +func TestComposeDefaultsProductionBackupRestoreGateToFalse(t *testing.T) { + source, err := os.ReadFile("../../compose.yaml") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(source), `PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false}`) { + t.Fatal("compose must pass the production backup restore gate with a false default") + } +} diff --git a/cmd/pp-ledger-import/main.go b/cmd/pp-ledger-import/main.go new file mode 100644 index 0000000..83d2836 --- /dev/null +++ b/cmd/pp-ledger-import/main.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "os" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + ppconnector "knowledge-governance/internal/connectors/pp" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +func main() { + if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error { + flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError) + flags.SetOutput(io.Discard) + ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL") + manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON") + dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access") + if err := flags.Parse(args); err != nil { + return fmt.Errorf("parse arguments: %w", err) + } + if *ledgerPath == "" || *manifestPath == "" { + return fmt.Errorf("-ledger and -manifest are required") + } + + ledger, err := os.Open(*ledgerPath) + if err != nil { + return fmt.Errorf("open PP ledger: %w", err) + } + snapshot, foldErr := ppconnector.FoldLedger(ledger) + closeErr := ledger.Close() + if foldErr != nil { + return foldErr + } + if closeErr != nil { + return fmt.Errorf("close PP ledger: %w", closeErr) + } + + manifest, err := os.Open(*manifestPath) + if err != nil { + return fmt.Errorf("open PP download manifest: %w", err) + } + recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest) + closeErr = manifest.Close() + if recoveryErr != nil { + return recoveryErr + } + if closeErr != nil { + return fmt.Errorf("close PP download manifest: %w", closeErr) + } + overLimitManifest, err := os.Open(*manifestPath) + if err != nil { + return fmt.Errorf("reopen PP download manifest: %w", err) + } + complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest) + closeErr = overLimitManifest.Close() + if overLimitErr != nil { + return overLimitErr + } + if closeErr != nil { + return fmt.Errorf("close PP download manifest: %w", closeErr) + } + if err := ppconnector.ValidateSnapshot(complete); err != nil { + return fmt.Errorf("validate PP snapshot: %w", err) + } + plan, err := ppconnector.BuildImportPlan(complete) + if err != nil { + return fmt.Errorf("build PP import plan: %w", err) + } + + completed := 0 + aliases := 0 + for _, item := range complete.Items { + switch item.Status { + case ppconnector.StatusCompleted: + completed++ + case ppconnector.StatusContentAlias: + aliases++ + } + } + _, _ = fmt.Fprintf(stdout, + "events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n", + complete.EventCount, + len(plan.SourceItems), + completed, + aliases, + overLimit.Added, + recovery.Recovered, + recovery.VerifiedBytes, + len(plan.ContentObjects), + len(plan.Versions), + len(plan.Publications), + ) + if *dryRun { + _, _ = fmt.Fprintln(stdout, "dry_run=true") + return nil + } + + databaseURL := strings.TrimSpace(getenv("DATABASE_URL")) + if databaseURL == "" { + return fmt.Errorf("DATABASE_URL is required unless -dry-run is used") + } + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return fmt.Errorf("open governance database: %w", err) + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return fmt.Errorf("ping governance database: %w", err) + } + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return err + } + report, err := postgresstore.ImportPPLegacy(ctx, pool, plan) + if err != nil { + return err + } + _, _ = fmt.Fprintf(stdout, + "inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n", + report.Connections, + report.SourceItems, + report.ContentObjects, + report.Versions, + report.Publications, + ) + current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID) + if err != nil { + return err + } + reconciliation, err := ppconnector.PlanReconciliation(complete, current) + if err != nil { + return err + } + _, _ = fmt.Fprintf(stdout, + "reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n", + reconciliation.Unchanged, + reconciliation.Adopted, + reconciliation.OverLimit, + reconciliation.UploadRequired, + ) + return nil +} diff --git a/cmd/pp-ledger-import/main_test.go b/cmd/pp-ledger-import/main_test.go new file mode 100644 index 0000000..7955ffe --- /dev/null +++ b/cmd/pp-ledger-import/main_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) { + directory := t.TempDir() + ledgerPath := filepath.Join(directory, "state.jsonl") + manifestPath := filepath.Join(directory, "manifest.json") + sourceHash := strings.Repeat("a", 64) + contentHash := strings.Repeat("c", 64) + ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatalf("write ledger fixture: %v", err) + } + if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil { + t.Fatalf("write manifest fixture: %v", err) + } + var stdout bytes.Buffer + + err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout) + if err != nil { + t.Fatalf("run() error = %v", err) + } + output := stdout.String() + for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} { + if !strings.Contains(output, required) { + t.Fatalf("run() output missing %q: %q", required, output) + } + } + for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} { + if strings.Contains(output, forbidden) { + t.Fatalf("run() output disclosed %q", forbidden) + } + } +} + +func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) { + directory := t.TempDir() + ledgerPath := filepath.Join(directory, "state.jsonl") + manifestPath := filepath.Join(directory, "manifest.json") + sha := strings.Repeat("c", 64) + ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}` + manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + + var output bytes.Buffer + err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output) + if err != nil { + t.Fatalf("run() error = %v", err) + } + text := output.String() + if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") { + t.Fatalf("dry-run output = %q", text) + } +} + +func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) { + databaseURL := os.Getenv("TEST_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + directory := t.TempDir() + ledgerPath := filepath.Join(directory, "state.jsonl") + manifestPath := filepath.Join(directory, "manifest.json") + sha := strings.Repeat("f", 64) + ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil { + t.Fatal(err) + } + + var output bytes.Buffer + err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string { + if key == "DATABASE_URL" { + return databaseURL + } + return "" + }, &output) + if err != nil { + t.Fatalf("run() error = %v", err) + } + if !strings.Contains(output.String(), "planned_uploads=0\n") { + t.Fatalf("run() output = %q", output.String()) + } +} diff --git a/cmd/release-diag/main.go b/cmd/release-diag/main.go new file mode 100644 index 0000000..2107982 --- /dev/null +++ b/cmd/release-diag/main.go @@ -0,0 +1,313 @@ +// Command release-diag replays the read-only steps of the release export path +// (source PG snapshot, source ES snapshot, source object read) and additionally +// lists candidate OSS key prefixes, to pinpoint a failing export step with its +// real underlying error. Credentials are read from files only and never printed. +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/url" + "os" + "time" + "unicode/utf8" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/platform/weknora" + "knowledge-governance/internal/releaseadapter/v071" +) + +type objectCfg struct { + Provider string `json:"provider"` + StorageBackendID string `json:"storage_backend_id"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Bucket string `json:"bucket"` + AccessKeyFile string `json:"access_key_file"` + SecretKeyFile string `json:"secret_key_file"` +} + +type endpointCfg struct { + DatabaseURLFile string `json:"database_url_file"` + Elasticsearch struct { + BaseURL string `json:"base_url"` + Index string `json:"index"` + } `json:"elasticsearch"` + Object objectCfg `json:"object"` + WeKnora struct { + BaseURL string `json:"base_url"` + APIKeyFile string `json:"api_key_file"` + } `json:"weknora"` +} + +type config struct { + Routes []struct { + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + EmbeddingModel string `json:"embedding_model"` + Source endpointCfg `json:"source"` + Target endpointCfg `json:"target"` + } `json:"routes"` +} + +func secret(path string) string { + data, err := os.ReadFile(path) + if err != nil { + fmt.Println("FATAL read secret:", err) + os.Exit(1) + } + return string(data) +} + +func main() { + configPath := flag.String("config", "", "release config json") + knowledgeID := flag.String("knowledge", "", "knowledge id to diagnose") + flag.Parse() + raw, err := os.ReadFile(*configPath) + if err != nil { + panic(err) + } + var cfg config + if err := json.Unmarshal(raw, &cfg); err != nil || len(cfg.Routes) != 1 { + panic(err) + } + route := cfg.Routes[0] + fmt.Println("route:", route.KnowledgeBaseID, "tenant:", route.TenantID) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + pool, err := pgxpool.New(ctx, secret(route.Source.DatabaseURLFile)) + if err != nil { + panic(err) + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + fmt.Println("STEP source PG connect: FAIL", err) + return + } + fmt.Println("STEP source PG connect: ok") + + postgres, err := v071.NewPostgresExporter(pool) + if err != nil { + panic(err) + } + snapshot, err := postgres.Export(ctx, v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}) + if err != nil { + fmt.Println("STEP postgres export: FAIL", err) + return + } + names := make([]string, 0, len(snapshot.Tables)) + for _, table := range snapshot.Tables { + names = append(names, fmt.Sprintf("%s:%d", table.Name, table.RowCount)) + } + fmt.Println("STEP postgres export: ok tables", names) + + search, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, PageSize: 500}) + if err != nil { + panic(err) + } + searchSnapshot, err := search.Export(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}) + if err != nil { + fmt.Println("STEP elasticsearch export: FAIL", err) + return + } + fmt.Printf("STEP elasticsearch export: ok documents=%d\n", len(searchSnapshot.Documents)) + + store, err := newStore(ctx, route.Source.Object) + if err != nil { + fmt.Println("STEP object store build: FAIL", err) + return + } + exporter, err := v071.NewObjectExporter(store, v071.ObjectExporterLimits{}) + if err != nil { + panic(err) + } + objectSnapshot, err := exporter.Export(ctx, snapshot) + if err != nil { + fmt.Println("STEP object export: FAIL", err) + } else { + fmt.Printf("STEP object export: ok objects=%d bytes=%d\n", len(objectSnapshot.Objects), objectSnapshot.TotalBytes) + } + + client, err := newRawClient(ctx, route.Source.Object) + if err != nil { + panic(err) + } + for _, prefix := range []string{"weknora/", "oss://", ""} { + listed, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(route.Source.Object.Bucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(3)}) + if err != nil { + fmt.Printf("STEP list prefix %q: FAIL %v\n", prefix, err) + continue + } + fmt.Printf("STEP list prefix %q: %d objects\n", prefix, len(listed.Contents)) + for _, object := range listed.Contents { + fmt.Println(" ", aws.ToString(object.Key)) + } + } + + knowledge := snapshot.Table("knowledges") + for _, row := range knowledge.Rows { + var typed struct { + Source string `json:"source"` + } + _ = json.Unmarshal(row, &typed) + fmt.Println("knowledge.source:", typed.Source) + } + for _, raw := range snapshot.Table("resources").Rows { + var typed struct { + PhysicalPath string `json:"physical_path"` + } + _ = json.Unmarshal(raw, &typed) + fmt.Println("resource physical_path:", typed.PhysicalPath) + } + + fmt.Println("STEP target object stage probe:") + targetStore, err := newStore(ctx, route.Target.Object) + if err != nil { + fmt.Println(" target store build FAIL", err) + return + } + probePath := ".governance-release-diag/probe.txt" + payload := []byte("release-sim-probe") + if err := targetStore.Put(ctx, route.Target.Object.Provider, probePath, bytes.NewReader(payload), int64(len(payload))); err != nil { + fmt.Println(" target Put FAIL:", err) + } else { + size, digest, err := targetStore.Stat(ctx, route.Target.Object.Provider, probePath) + fmt.Println(" target Put ok; Stat:", size, digest[:8], "err:", err) + } + + fmt.Println("STEP real ObjectImporter Stage/Promote with exported snapshot:") + importer, err := v071.NewObjectImporter(targetStore, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider}) + if err != nil { + fmt.Println(" importer build FAIL", err) + return + } + stage, err := importer.Stage(ctx, "diag-stage", objectSnapshot) + if err != nil { + fmt.Println(" Stage FAIL:", err) + return + } + fmt.Println(" Stage ok objects:", len(stage.Objects)) + if err := importer.Promote(ctx, stage); err != nil { + fmt.Println(" Promote FAIL:", err) + return + } + fmt.Println(" Promote ok") + + fmt.Println("STEP target Postgres replace:") + targetPool, err := pgxpool.New(ctx, secret(route.Target.DatabaseURLFile)) + if err != nil { + fmt.Println(" target pool FAIL", err) + return + } + defer targetPool.Close() + if err := targetPool.Ping(ctx); err != nil { + fmt.Println(" target PG connect FAIL", err) + return + } + pgImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider}) + if err != nil { + fmt.Println(" importer config FAIL", err) + return + } + identity := v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID} + if err := pgImporter.Replace(ctx, identity, snapshot); err != nil { + fmt.Println(" Replace FAIL:", err) + return + } + fmt.Println(" Replace ok") + + fmt.Println("STEP target ES replace:") + targetSearch, err := v071.NewSearchImporter(v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, PageSize: 500}) + if err != nil { + fmt.Println(" importer build FAIL", err) + return + } + if _, err := targetSearch.Replace(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}, searchSnapshot); err != nil { + fmt.Println(" ES Replace FAIL:", err) + return + } + fmt.Println(" ES Replace ok") + + fmt.Println("STEP target WeKnora verification replay:") + wknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: secret(route.Target.WeKnora.APIKeyFile), Timeout: 30 * time.Second, AllowDockerHTTP: true}) + if err != nil { + fmt.Println(" client build FAIL", err) + return + } + knowledgeRemote, err := wknoraClient.GetKnowledge(ctx, *knowledgeID) + fmt.Println(" GetKnowledge err:", err, "id:", knowledgeRemote.ID, "kb:", knowledgeRemote.KnowledgeBaseID, "parse:", knowledgeRemote.ParseStatus) + chunksRemote, total, err := wknoraClient.ListChunks(ctx, *knowledgeID, 1, 5, nil) + fmt.Println(" ListChunks err:", err, "total:", total, "page:", len(chunksRemote)) + probe := "" + for _, document := range searchSnapshot.Documents { + var fields struct { + Content string `json:"content"` + } + _ = json.Unmarshal(document.Source, &fields) + if utf8.RuneCountInString(fields.Content) >= 12 { + probe = string([]rune(fields.Content)[:min(48, utf8.RuneCountInString(fields.Content))]) + break + } + } + results, err := wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: probe, KnowledgeIDs: []string{*knowledgeID}}) + fmt.Println(" SearchKnowledge err:", err, "probe_len:", len(probe), "results:", len(results)) + for index, result := range results { + if index >= 3 { + break + } + fmt.Printf(" hit knowledge=%s score=%.3f type=%s\n", result.KnowledgeID, result.Score, result.ChunkType) + } + if len(results) == 0 && probe != "" { + results, err = wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: "CRA 620mm", KnowledgeIDs: []string{*knowledgeID}}) + fmt.Println(" short-probe SearchKnowledge err:", err, "results:", len(results)) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func newStore(ctx context.Context, object objectCfg) (*v071.S3ObjectStore, error) { + parsed, err := url.Parse(object.Endpoint) + if err != nil { + return nil, err + } + return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{ + Provider: object.Provider, Endpoint: object.Endpoint, Region: object.Region, + AccessKeyID: secret(object.AccessKeyFile), SecretAccessKey: secret(object.SecretKeyFile), + Bucket: object.Bucket, Secure: parsed.Scheme == "https", + }) +} + +func newRawClient(ctx context.Context, object objectCfg) (*s3.Client, error) { + parsed, err := url.Parse(object.Endpoint) + if err != nil { + return nil, err + } + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(object.Region), awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(secret(object.AccessKeyFile), secret(object.SecretKeyFile), ""))) + if err != nil { + return nil, err + } + return s3.NewFromConfig(awsCfg, func(options *s3.Options) { + options.BaseEndpoint = aws.String(object.Endpoint) + options.UsePathStyle = false + _ = parsed + }), nil +} + +var _ = io.Discard diff --git a/cmd/release-worker/config.go b/cmd/release-worker/config.go new file mode 100644 index 0000000..343b2a0 --- /dev/null +++ b/cmd/release-worker/config.go @@ -0,0 +1,155 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/url" + "os" + "path/filepath" + "strings" + "syscall" +) + +const maxReleaseConfigBytes = 1024 * 1024 + +type releaseRuntimeConfig struct { + ArchiveRoot string `json:"archive_root"` + Routes []releaseRouteConfig `json:"routes"` +} + +type releaseRouteConfig struct { + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + EmbeddingModel string `json:"embedding_model"` + Source releaseEndpointConfig `json:"source"` + Target releaseEndpointConfig `json:"target"` +} + +type releaseEndpointConfig struct { + DatabaseURLFile string `json:"database_url_file"` + Elasticsearch releaseElasticsearchConfig `json:"elasticsearch"` + Object releaseObjectConfig `json:"object"` + WeKnora releaseWeKnoraConfig `json:"weknora"` +} + +type releaseElasticsearchConfig struct { + BaseURL string `json:"base_url"` + Index string `json:"index"` + TokenFile string `json:"token_file"` +} + +type releaseObjectConfig struct { + Provider string `json:"provider"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Bucket string `json:"bucket"` + StorageBackendID string `json:"storage_backend_id"` + AccessKeyFile string `json:"access_key_file"` + SecretKeyFile string `json:"secret_key_file"` +} + +type releaseWeKnoraConfig struct { + BaseURL string `json:"base_url"` + APIKeyFile string `json:"api_key_file"` + AllowDockerHTTP bool `json:"allow_docker_http"` +} + +func loadReleaseRuntimeConfig(path string) (releaseRuntimeConfig, error) { + if !filepath.IsAbs(path) { + return releaseRuntimeConfig{}, errors.New("release config path must be absolute") + } + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return releaseRuntimeConfig{}, errors.New("open release config") + } + file := os.NewFile(uintptr(fd), "release-config") + if file == nil { + syscall.Close(fd) + return releaseRuntimeConfig{}, errors.New("open release config") + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() <= 0 || info.Size() > maxReleaseConfigBytes { + return releaseRuntimeConfig{}, errors.New("release config must be a non-empty regular 0600 file") + } + payload, err := io.ReadAll(io.LimitReader(file, maxReleaseConfigBytes+1)) + if err != nil || len(payload) > maxReleaseConfigBytes { + return releaseRuntimeConfig{}, errors.New("read release config") + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var config releaseRuntimeConfig + if err := decoder.Decode(&config); err != nil { + return releaseRuntimeConfig{}, errors.New("decode release config") + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return releaseRuntimeConfig{}, errors.New("release config contains trailing data") + } + if err := validateReleaseRuntimeConfig(config); err != nil { + return releaseRuntimeConfig{}, err + } + return config, nil +} + +func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error { + if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 { + return errors.New("release archive root and routes are required") + } + seen := map[string]bool{} + for _, route := range config.Routes { + if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] { + return errors.New("invalid or duplicate release route") + } + seen[route.KnowledgeBaseID] = true + if !validReleaseEndpoint(route.Source, false) || !validReleaseEndpoint(route.Target, true) { + return errors.New("invalid release route endpoint") + } + if route.Source.DatabaseURLFile == route.Target.DatabaseURLFile || + route.Source.Elasticsearch.BaseURL == route.Target.Elasticsearch.BaseURL || + (route.Source.Object.Endpoint == route.Target.Object.Endpoint && route.Source.Object.Bucket == route.Target.Object.Bucket) { + return errors.New("release source and target must be distinct") + } + } + return nil +} + +func validReleaseEndpoint(endpoint releaseEndpointConfig, target bool) bool { + if !absoluteFile(endpoint.DatabaseURLFile) || !validHTTPURL(endpoint.Elasticsearch.BaseURL) || !validConfigIdentifier(endpoint.Elasticsearch.Index) || + (endpoint.Elasticsearch.TokenFile != "" && !absoluteFile(endpoint.Elasticsearch.TokenFile)) || + !validObjectEndpoint(endpoint.Object) { + return false + } + if target { + return validConfigIdentifier(endpoint.Object.StorageBackendID) && validHTTPURL(endpoint.WeKnora.BaseURL) && absoluteFile(endpoint.WeKnora.APIKeyFile) + } + return endpoint.WeKnora.BaseURL == "" && endpoint.WeKnora.APIKeyFile == "" +} + +func validObjectEndpoint(object releaseObjectConfig) bool { + return (object.Provider == "minio" || object.Provider == "s3" || object.Provider == "oss") && validHTTPURL(object.Endpoint) && + validConfigIdentifier(object.Bucket) && absoluteFile(object.AccessKeyFile) && absoluteFile(object.SecretKeyFile) +} + +func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path } + +func validHTTPURL(value string) bool { + parsed, err := url.Parse(value) + return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" +} + +func validConfigIdentifier(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > 128 { + return false + } + for _, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' { + continue + } + return false + } + return true +} diff --git a/cmd/release-worker/config_test.go b/cmd/release-worker/config_test.go new file mode 100644 index 0000000..daad37d --- /dev/null +++ b/cmd/release-worker/config_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadReleaseRuntimeConfigValidatesRestrictedMultiKBConfig(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "release.json") + payload := validReleaseConfigJSON(root) + if err := os.WriteFile(path, []byte(payload), 0o600); err != nil { + t.Fatal(err) + } + config, err := loadReleaseRuntimeConfig(path) + if err != nil { + t.Fatal(err) + } + if len(config.Routes) != 1 || config.Routes[0].KnowledgeBaseID != "kb-safe" || config.Routes[0].Target.Object.StorageBackendID != "target-backend" { + t.Fatalf("config=%#v", config) + } +} + +func TestLoadReleaseRuntimeConfigRejectsUnsafeFilesAndUnknownFields(t *testing.T) { + root := t.TempDir() + payload := validReleaseConfigJSON(root) + for _, test := range []struct { + name string + setup func(string) error + }{ + {name: "overbroad", setup: func(path string) error { return os.WriteFile(path, []byte(payload), 0o644) }}, + {name: "unknown field", setup: func(path string) error { + return os.WriteFile(path, []byte(strings.Replace(payload, `"archive_root":`, `"unknown":true,"archive_root":`, 1)), 0o600) + }}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json") + if err := test.setup(path); err != nil { + t.Fatal(err) + } + if _, err := loadReleaseRuntimeConfig(path); err == nil { + t.Fatal("unsafe config accepted") + } + }) + } + real := filepath.Join(root, "real.json") + _ = os.WriteFile(real, []byte(payload), 0o600) + link := filepath.Join(root, "link.json") + _ = os.Symlink(real, link) + if _, err := loadReleaseRuntimeConfig(link); err == nil { + t.Fatal("symlink accepted") + } +} + +func validReleaseConfigJSON(root string) string { + return `{ + "archive_root":"` + filepath.ToSlash(filepath.Join(root, "archive")) + `", + "routes":[{ + "tenant_id":10000,"knowledge_base_id":"kb-safe","embedding_model":"bge-m3", + "source":{"database_url_file":"/run/secrets/source-db","elasticsearch":{"base_url":"http://source-es:9200","index":"chunks","token_file":"/run/secrets/source-es"},"object":{"provider":"minio","endpoint":"http://source-minio:9000","region":"us-east-1","bucket":"source","access_key_file":"/run/secrets/source-ak","secret_key_file":"/run/secrets/source-sk"}}, + "target":{"database_url_file":"/run/secrets/target-db","elasticsearch":{"base_url":"https://target-es.example","index":"chunks","token_file":"/run/secrets/target-es"},"object":{"provider":"oss","endpoint":"https://oss.example","region":"cn-shenzhen","bucket":"target","storage_backend_id":"target-backend","access_key_file":"/run/secrets/target-ak","secret_key_file":"/run/secrets/target-sk"},"weknora":{"base_url":"https://weknora.example","api_key_file":"/run/secrets/target-weknora"}} + }]}` +} diff --git a/cmd/release-worker/main.go b/cmd/release-worker/main.go new file mode 100644 index 0000000..3a0abef --- /dev/null +++ b/cmd/release-worker/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, getenv func(string) string) error { + config, err := appconfig.Load(getenv) + if err != nil { + return err + } + if !config.ReleaseWorkerEnabled { + log.Print("release worker disabled; no production credentials are loaded") + <-ctx.Done() + return nil + } + runtimeConfig, err := loadReleaseRuntimeConfig(config.ReleaseConfigFile) + if err != nil { + return err + } + releaseRuntime, err := buildReleaseRuntime(ctx, runtimeConfig) + if err != nil { + return err + } + defer releaseRuntime.Close() + + pool, err := pgxpool.New(ctx, config.DatabaseURL) + if err != nil { + return err + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return err + } + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return err + } + store := postgresstore.NewStore(pool) + releaseHandler, err := releaseworkflow.NewHandler(store, releaseRuntime.publisher, time.Now) + if err != nil { + return err + } + worker, err := jobs.NewWorker(jobs.WorkerConfig{ + Queue: store, + WorkerID: config.WorkerID, + LeaseDuration: config.LeaseDuration, + RetryDelay: config.RetryDelay, + Handlers: map[string]jobs.Handler{ + "release.publish": releaseHandler, + "release.rollback": releaseHandler, + }, + OnError: func(err error) { log.Printf("release worker error: %v", err) }, + }) + if err != nil { + return err + } + log.Printf("release worker %s started with %d release route(s)", config.WorkerID, len(runtimeConfig.Routes)) + return worker.Run(ctx, config.PollInterval) +} diff --git a/cmd/release-worker/main_test.go b/cmd/release-worker/main_test.go new file mode 100644 index 0000000..77ebf13 --- /dev/null +++ b/cmd/release-worker/main_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "os" + "strings" + "testing" +) + +func TestRunReturnsWithoutDatabaseWhenReleaseWorkerDisabled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + values := map[string]string{ + "DATABASE_URL": "postgres://must-not-be-contacted.invalid/governance", + "RELEASE_WORKER_ENABLED": "false", + } + if err := run(ctx, func(key string) string { return values[key] }); err != nil { + t.Fatalf("run() error = %v", err) + } +} + +func TestEnabledReleaseWorkerLoadsRestrictedConfigBeforeDatabaseConnections(t *testing.T) { + values := map[string]string{"DATABASE_URL": "not-a-database-url", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/does/not/exist/release.json"} + err := run(context.Background(), func(key string) string { return values[key] }) + if err == nil || !strings.Contains(err.Error(), "release config") || strings.Contains(err.Error(), "database") { + t.Fatalf("error=%v", err) + } +} + +func TestReleaseWorkerRegistersOnlyPublishAndRollbackHandlers(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + text := string(source) + for _, required := range []string{`"release.publish"`, `"release.rollback"`} { + if !strings.Contains(text, required) { + t.Fatalf("missing handler %s", required) + } + } + if strings.Contains(text, `"release.probe"`) { + t.Fatal("probe-only release handler remains") + } +} diff --git a/cmd/release-worker/runtime.go b/cmd/release-worker/runtime.go new file mode 100644 index 0000000..970a9d6 --- /dev/null +++ b/cmd/release-worker/runtime.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "errors" + "net/url" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/platform/weknora" + v071 "knowledge-governance/internal/releaseadapter/v071" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type releaseRuntime struct { + publisher *releaseworkflow.RoutingPublisher + pools []*pgxpool.Pool +} + +func (runtime *releaseRuntime) Close() { + for index := len(runtime.pools) - 1; index >= 0; index-- { + runtime.pools[index].Close() + } +} + +func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) { + runtime := &releaseRuntime{} + publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes)) + for _, route := range config.Routes { + publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route) + if err != nil { + runtime.Close() + return nil, err + } + runtime.pools = append(runtime.pools, pools...) + publishers[route.KnowledgeBaseID] = publisher + } + router, err := releaseworkflow.NewRoutingPublisher(publishers) + if err != nil { + runtime.Close() + return nil, errors.New("create release publisher routes") + } + runtime.publisher = router + return runtime, nil +} + +func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) { + sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile) + if err != nil { + return nil, nil, errors.New("read source database credential") + } + targetDatabaseURL, err := readRestrictedValue(route.Target.DatabaseURLFile) + if err != nil { + return nil, nil, errors.New("read target database credential") + } + if err := validateDistinctDatabaseURLs(sourceDatabaseURL, targetDatabaseURL); err != nil { + return nil, nil, err + } + sourcePool, err := openReleasePool(ctx, sourceDatabaseURL) + if err != nil { + return nil, nil, errors.New("connect release source database") + } + pools := []*pgxpool.Pool{sourcePool} + targetPool, err := openReleasePool(ctx, targetDatabaseURL) + if err != nil { + sourcePool.Close() + return nil, nil, errors.New("connect release target database") + } + pools = append(pools, targetPool) + cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) { + for index := len(pools) - 1; index >= 0; index-- { + pools[index].Close() + } + return nil, nil, err + } + + sourceSearchToken, err := readOptionalRestrictedValue(route.Source.Elasticsearch.TokenFile) + if err != nil { + return cleanup(errors.New("read source Elasticsearch credential")) + } + targetSearchToken, err := readOptionalRestrictedValue(route.Target.Elasticsearch.TokenFile) + if err != nil { + return cleanup(errors.New("read target Elasticsearch credential")) + } + sourceSearch, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, AuthorizationToken: sourceSearchToken, PageSize: 500}) + if err != nil { + return cleanup(errors.New("create source Elasticsearch exporter")) + } + targetSearchConfig := v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, AuthorizationToken: targetSearchToken, PageSize: 500} + targetSearchInspector, err := v071.NewSearchExporter(targetSearchConfig) + if err != nil { + return cleanup(errors.New("create target Elasticsearch inspector")) + } + targetSearch, err := v071.NewSearchImporter(targetSearchConfig) + if err != nil { + return cleanup(errors.New("create target Elasticsearch importer")) + } + + sourceObject, err := buildS3ObjectStore(route.Source.Object) + if err != nil { + return cleanup(errors.New("create source object reader")) + } + targetObject, err := buildS3ObjectStore(route.Target.Object) + if err != nil { + return cleanup(errors.New("create target object store")) + } + objectExporter, err := v071.NewObjectExporter(sourceObject, v071.ObjectExporterLimits{}) + if err != nil { + return cleanup(errors.New("create source object exporter")) + } + objectImporter, err := v071.NewObjectImporter(targetObject, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider}) + if err != nil { + return cleanup(errors.New("create target object importer")) + } + + sourceContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: sourcePool, Mapping: sourceSearch, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Source.Elasticsearch.Index}) + if err != nil { + return cleanup(errors.New("create source release contract inspector")) + } + targetContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: targetPool, Mapping: targetSearchInspector, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Target.Elasticsearch.Index}) + if err != nil { + return cleanup(errors.New("create target release contract inspector")) + } + postgresExporter, err := v071.NewPostgresExporter(sourcePool) + if err != nil { + return cleanup(errors.New("create source PostgreSQL exporter")) + } + postgresImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider}) + if err != nil { + return cleanup(errors.New("create target PostgreSQL importer")) + } + documentExporter, err := v071.NewDocumentExporter(v071.DocumentExporterConfig{Contract: sourceContract, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: route.TenantID, EmbeddingModel: route.EmbeddingModel, Now: time.Now}) + if err != nil { + return cleanup(errors.New("create v0.7.1 document exporter")) + } + + archive, err := v071.NewBundleArchive(filepath.Join(archiveRoot, route.KnowledgeBaseID), 1024*1024*1024) + if err != nil { + return cleanup(errors.New("open release bundle archive")) + } + apiKey, err := readRestrictedValue(route.Target.WeKnora.APIKeyFile) + if err != nil { + return cleanup(errors.New("read target WeKnora credential")) + } + weknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: apiKey, Timeout: 60 * time.Second, AllowDockerHTTP: route.Target.WeKnora.AllowDockerHTTP}) + if err != nil { + return cleanup(errors.New("create target WeKnora verification client")) + } + verifier, err := v071.NewWeKnoraVerifier(weknoraClient) + if err != nil { + return cleanup(errors.New("create target WeKnora verifier")) + } + documentTarget, err := v071.NewDocumentTarget(v071.DocumentTargetConfig{Archive: archive, Contract: targetContract, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier}) + if err != nil { + return cleanup(errors.New("create v0.7.1 document target")) + } + publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget) + if err != nil { + return cleanup(errors.New("create v0.7.1 release publisher")) + } + return publisher, pools, nil +} + +func buildS3ObjectStore(config releaseObjectConfig) (*v071.S3ObjectStore, error) { + accessKey, err := readRestrictedValue(config.AccessKeyFile) + if err != nil { + return nil, err + } + secretKey, err := readRestrictedValue(config.SecretKeyFile) + if err != nil { + return nil, err + } + parsed, err := url.Parse(config.Endpoint) + if err != nil { + return nil, errors.New("invalid object endpoint") + } + return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: config.Provider, Endpoint: config.Endpoint, Region: config.Region, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: config.Bucket, Secure: parsed.Scheme == "https", ForcePathStyle: config.Provider == "minio"}) +} + +func readRestrictedValue(path string) (string, error) { + return weknora.ReadAPIKeyFile(path) +} + +func readOptionalRestrictedValue(path string) (string, error) { + if path == "" { + return "", nil + } + return readRestrictedValue(path) +} + +func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) { + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, err + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, err + } + return pool, nil +} + +func validateDistinctDatabaseURLs(sourceURL, targetURL string) error { + source, err := pgxpool.ParseConfig(sourceURL) + if err != nil { + return errors.New("invalid source database credential") + } + target, err := pgxpool.ParseConfig(targetURL) + if err != nil { + return errors.New("invalid target database credential") + } + sourceIdentity := strings.ToLower(source.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(source.ConnConfig.Port), 10) + "|" + source.ConnConfig.Database + targetIdentity := strings.ToLower(target.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(target.ConnConfig.Port), 10) + "|" + target.ConnConfig.Database + if sourceIdentity == targetIdentity { + return errors.New("release source and target database must be distinct") + } + return nil +} diff --git a/cmd/release-worker/runtime_test.go b/cmd/release-worker/runtime_test.go new file mode 100644 index 0000000..672ae00 --- /dev/null +++ b/cmd/release-worker/runtime_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestReleaseRuntimeRejectsSameSourceAndTargetDatabaseValues(t *testing.T) { + if err := validateDistinctDatabaseURLs("postgres://user@same/db?sslmode=require", "postgres://user@same/db?sslmode=require"); err == nil { + t.Fatal("same database URL accepted") + } + if err := validateDistinctDatabaseURLs("postgres://user@source/db?sslmode=require", "postgres://user@target/db?sslmode=require"); err != nil { + t.Fatal(err) + } +} + +func TestBuildReleaseRuntimeWithRestrictedFilesIntegration(t *testing.T) { + sourceDatabaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL") + targetDatabaseURL := os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL") + minioEndpoint := os.Getenv("TEST_MINIO_ENDPOINT") + if sourceDatabaseURL == "" || targetDatabaseURL == "" || minioEndpoint == "" { + t.Skip("release runtime integration environment is not set") + } + esSource := httptest.NewServer(http.NotFoundHandler()) + defer esSource.Close() + esTarget := httptest.NewServer(http.NotFoundHandler()) + defer esTarget.Close() + weknoraTarget := httptest.NewServer(http.NotFoundHandler()) + defer weknoraTarget.Close() + root := t.TempDir() + archiveRoot := filepath.Join(root, "archive") + archiveRoute := filepath.Join(archiveRoot, "kb-release-safe") + if err := os.MkdirAll(archiveRoute, 0o700); err != nil { + t.Fatal(err) + } + secret := func(name, value string) string { + path := filepath.Join(root, name) + if err := os.WriteFile(path, []byte(value+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return path + } + route := releaseRouteConfig{ + TenantID: 10000, KnowledgeBaseID: "kb-release-safe", EmbeddingModel: "bge-m3", + Source: releaseEndpointConfig{DatabaseURLFile: secret("source-db", sourceDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esSource.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-source", AccessKeyFile: secret("source-ak", "release-test"), SecretKeyFile: secret("source-sk", "release-test-secret")}}, + Target: releaseEndpointConfig{DatabaseURLFile: secret("target-db", targetDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esTarget.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-target", StorageBackendID: "target-backend", AccessKeyFile: secret("target-ak", "release-test"), SecretKeyFile: secret("target-sk", "release-test-secret")}, WeKnora: releaseWeKnoraConfig{BaseURL: weknoraTarget.URL, APIKeyFile: secret("target-weknora", "test-api-key"), AllowDockerHTTP: true}}, + } + runtime, err := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{ArchiveRoot: archiveRoot, Routes: []releaseRouteConfig{route}}) + if err != nil { + t.Fatal(err) + } + runtime.Close() + if runtime.publisher == nil || len(runtime.pools) != 2 { + t.Fatalf("runtime=%#v", runtime) + } +} diff --git a/cmd/test-publication-seed/main.go b/cmd/test-publication-seed/main.go new file mode 100644 index 0000000..1e70d69 --- /dev/null +++ b/cmd/test-publication-seed/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/spool" +) + +type manualSeeder interface { + SeedManualPublication(context.Context, postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error) +} + +type seedConfig struct { + SpoolRoot string + SpoolRef string + Format string + Title string + TestKBID string + MaxBytes int64 + Now time.Time +} + +func main() { + if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error { + flags := flag.NewFlagSet("test-publication-seed", flag.ContinueOnError) + flags.SetOutput(io.Discard) + spoolRef := flags.String("spool-ref", "", "opaque relative spool reference") + format := flags.String("format", "md", "validated artifact format") + title := flags.String("title", "", "safe display title") + if err := flags.Parse(args); err != nil { + return errors.New("invalid seed arguments") + } + databaseURL := getenv("DATABASE_URL") + spoolRoot := getenv("GOVERNANCE_SPOOL_ROOT") + if spoolRoot == "" { + spoolRoot = "/var/lib/governance/spool" + } + testKBID := getenv("TEST_WEKNORA_KB_ID") + maxBytes := int64(50 * 1024 * 1024) + if raw := getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"); raw != "" { + parsed, err := strconv.ParseInt(raw, 10, 64) + if err != nil || parsed <= 0 || parsed > 50*1024*1024 { + return errors.New("invalid TEST_WEKNORA_MAX_UPLOAD_BYTES") + } + maxBytes = parsed + } + if databaseURL == "" || testKBID == "" || *spoolRef == "" || *title == "" { + return errors.New("DATABASE_URL, TEST_WEKNORA_KB_ID, --spool-ref and --title are required") + } + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return errors.New("open governance database") + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return errors.New("connect governance database") + } + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return errors.New("apply governance migrations") + } + result, err := seedFromSpool(ctx, postgresstore.NewStore(pool), seedConfig{ + SpoolRoot: spoolRoot, + SpoolRef: *spoolRef, + Format: *format, + Title: *title, + TestKBID: testKBID, + MaxBytes: maxBytes, + Now: time.Now().UTC(), + }) + if err != nil { + return err + } + return json.NewEncoder(output).Encode(result) +} + +func seedFromSpool(ctx context.Context, store manualSeeder, config seedConfig) (postgresstore.ManualPublicationSeedResult, error) { + if store == nil { + return postgresstore.ManualPublicationSeedResult{}, errors.New("manual publication store is required") + } + expectation, err := spool.InspectVerified(config.SpoolRoot, config.SpoolRef, config.Format, config.MaxBytes) + if err != nil { + return postgresstore.ManualPublicationSeedResult{}, errors.New("inspect manual spool artifact") + } + mime, ok := mimeForFormat(config.Format) + if !ok { + return postgresstore.ManualPublicationSeedResult{}, errors.New("unsupported manual publication format") + } + result, err := store.SeedManualPublication(ctx, postgresstore.ManualPublicationSeedRequest{ + TestKBID: config.TestKBID, + Title: config.Title, + SHA256: expectation.SHA256, + SizeBytes: expectation.SizeBytes, + MIME: mime, + ValidatedFormat: expectation.Format, + LocalSpoolRef: expectation.RelativeRef, + Now: config.Now, + }) + if err != nil { + return postgresstore.ManualPublicationSeedResult{}, fmt.Errorf("seed manual publication: %w", err) + } + return result, nil +} + +func mimeForFormat(format string) (string, bool) { + mimes := map[string]string{ + "md": "text/markdown", "txt": "text/plain", "csv": "text/csv", "json": "application/json", + "pdf": "application/pdf", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "doc": "application/msword", "xls": "application/vnd.ms-excel", "ppt": "application/vnd.ms-powerpoint", + } + value, ok := mimes[format] + return value, ok +} diff --git a/cmd/test-publication-seed/main_test.go b/cmd/test-publication-seed/main_test.go new file mode 100644 index 0000000..f8d934a --- /dev/null +++ b/cmd/test-publication-seed/main_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" + + postgresstore "knowledge-governance/internal/platform/postgres" +) + +func TestSeedFromSpoolPassesVerifiedArtifactProjection(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("# Governance fixture\n\nThe unique marker is MIKU-P3-5927.\n") + ref := filepath.Join("manual", "fixture.md") + if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil { + t.Fatal(err) + } + store := &manualSeederStub{result: postgresstore.ManualPublicationSeedResult{PublicationID: "tp-1", JobID: "job-1"}} + now := time.Date(2026, 9, 5, 6, 0, 0, 0, time.UTC) + + result, err := seedFromSpool(context.Background(), store, seedConfig{ + SpoolRoot: root, + SpoolRef: ref, + Format: "md", + Title: "Governance fixture", + TestKBID: "kb-1", + MaxBytes: 1024, + Now: now, + }) + if err != nil { + t.Fatalf("seedFromSpool() error = %v", err) + } + digest := sha256.Sum256(body) + if result.PublicationID != "tp-1" || store.request.SHA256 != hex.EncodeToString(digest[:]) || store.request.SizeBytes != int64(len(body)) { + t.Fatalf("result/request = %#v %#v", result, store.request) + } + if store.request.LocalSpoolRef != ref || store.request.ValidatedFormat != "md" || store.request.MIME != "text/markdown" || store.request.Now != now { + t.Fatalf("request = %#v", store.request) + } +} + +type manualSeederStub struct { + request postgresstore.ManualPublicationSeedRequest + result postgresstore.ManualPublicationSeedResult + err error +} + +func (store *manualSeederStub) SeedManualPublication(_ context.Context, request postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error) { + store.request = request + return store.result, store.err +} diff --git a/cmd/test-weknora-e2e-report/main.go b/cmd/test-weknora-e2e-report/main.go new file mode 100644 index 0000000..d91bf24 --- /dev/null +++ b/cmd/test-weknora-e2e-report/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "io" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +type reportStore interface { + LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) + ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) +} + +type reportClient interface { + GetKnowledge(context.Context, string) (weknora.Knowledge, error) + ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) + SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) +} + +type e2eReport struct { + PublicationID string `json:"publication_id"` + RemoteStatus string `json:"remote_status"` + ChunkTotal int64 `json:"chunk_total"` + ChunkItemsRead int64 `json:"chunk_items_read"` + AllChunksMatch bool `json:"all_chunks_match"` + SearchResultCount int `json:"search_result_count"` + TargetSearchHits int `json:"target_search_hits"` + MarkerFound bool `json:"marker_found"` + MaxScore float64 `json:"max_score"` + StructuralPassed bool `json:"structural_passed"` + SemanticState string `json:"semantic_state"` +} + +func main() { + if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error { + flags := flag.NewFlagSet("test-weknora-e2e-report", flag.ContinueOnError) + flags.SetOutput(io.Discard) + publicationID := flags.String("publication-id", "", "governance publication ID") + query := flags.String("query", "", "synthetic scoped retrieval query") + expectedMarker := flags.String("expected-marker", "", "synthetic marker expected in a scoped result") + if err := flags.Parse(args); err != nil { + return errors.New("invalid report arguments") + } + if *publicationID == "" || *query == "" || *expectedMarker == "" { + return errors.New("--publication-id, --query and --expected-marker are required") + } + databaseURL := getenv("DATABASE_URL") + baseURL := getenv("TEST_WEKNORA_BASE_URL") + keyFile := getenv("TEST_WEKNORA_API_KEY_FILE") + if databaseURL == "" || baseURL == "" || keyFile == "" { + return errors.New("database and test WeKnora configuration are required") + } + allowDockerHTTP := false + if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" { + parsed, err := strconv.ParseBool(raw) + if err != nil { + return errors.New("invalid Docker HTTP setting") + } + allowDockerHTTP = parsed + } + apiKey, err := weknora.ReadAPIKeyFile(keyFile) + if err != nil { + return errors.New("read test WeKnora API key") + } + client, err := weknora.NewClient(weknora.ClientConfig{ + BaseURL: baseURL, + APIKey: apiKey, + Timeout: 60 * time.Second, + AllowDockerHTTP: allowDockerHTTP, + }) + if err != nil { + return errors.New("create test WeKnora client") + } + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return errors.New("open governance database") + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return errors.New("connect governance database") + } + report, err := verifyPublication(ctx, postgresstore.NewStore(pool), client, *publicationID, *query, *expectedMarker) + if err != nil { + return err + } + return json.NewEncoder(output).Encode(report) +} + +func verifyPublication(ctx context.Context, store reportStore, client reportClient, publicationID, query, expectedMarker string) (e2eReport, error) { + publication, err := store.LoadPublicationForValidation(ctx, publicationID) + if err != nil { + return e2eReport{}, errors.New("load completed publication") + } + knowledge, err := client.GetKnowledge(ctx, publication.KnowledgeID) + if err != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID { + return e2eReport{}, errors.New("verify remote knowledge identity") + } + report := e2eReport{ + PublicationID: publication.ID, + RemoteStatus: knowledge.ParseStatus, + AllChunksMatch: true, + } + for page := 1; page <= 100; page++ { + chunks, total, err := client.ListChunks(ctx, publication.KnowledgeID, page, 100, []string{"text", "image_ocr"}) + if err != nil { + return e2eReport{}, errors.New("read remote chunks") + } + report.ChunkTotal = total + for _, chunk := range chunks { + report.ChunkItemsRead++ + if chunk.KnowledgeID != publication.KnowledgeID { + report.AllChunksMatch = false + } + } + if report.ChunkItemsRead >= total || len(chunks) == 0 { + break + } + if page == 100 { + return e2eReport{}, errors.New("remote chunk pagination limit exceeded") + } + } + results, err := client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{publication.KnowledgeID}}) + if err != nil { + return e2eReport{}, errors.New("run scoped retrieval probe") + } + report.SearchResultCount = len(results) + for _, result := range results { + if result.KnowledgeID == publication.KnowledgeID { + report.TargetSearchHits++ + if strings.Contains(result.Content, expectedMarker) { + report.MarkerFound = true + } + } + if result.Score > report.MaxScore { + report.MaxScore = result.Score + } + } + acceptance, err := store.ListAcceptanceResults(ctx, publication.ID) + if err != nil { + return e2eReport{}, errors.New("read governance acceptance results") + } + structural := map[string]bool{"knowledge_exists": false, "chunks_present": false} + for _, record := range acceptance { + if _, ok := structural[record.CheckType]; ok && record.State == string(governance.AcceptancePassed) { + structural[record.CheckType] = true + } + if record.CheckType == "retrieval_hit" { + report.SemanticState = record.State + } + } + report.StructuralPassed = structural["knowledge_exists"] && structural["chunks_present"] + if report.RemoteStatus != "completed" || report.ChunkTotal <= 0 || report.ChunkItemsRead != report.ChunkTotal || !report.AllChunksMatch || report.TargetSearchHits == 0 || !report.MarkerFound { + return e2eReport{}, errors.New("end-to-end verification failed") + } + return report, nil +} diff --git a/cmd/test-weknora-e2e-report/main_test.go b/cmd/test-weknora-e2e-report/main_test.go new file mode 100644 index 0000000..09f8cd8 --- /dev/null +++ b/cmd/test-weknora-e2e-report/main_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +func TestVerifyPublicationReturnsSanitizedEvidence(t *testing.T) { + store := &reportStoreStub{ + publication: postgresstore.PublicationForValidation{ID: "tp-1", TestKBID: "kb-1", KnowledgeID: "knowledge-secret-id"}, + acceptance: []governance.AcceptanceRecord{ + {CheckType: "knowledge_exists", State: string(governance.AcceptancePassed)}, + {CheckType: "chunks_present", State: string(governance.AcceptancePassed)}, + {CheckType: "retrieval_hit", State: string(governance.AcceptanceManualRequired)}, + }, + } + client := &reportClientStub{ + knowledge: weknora.Knowledge{ID: "knowledge-secret-id", KnowledgeBaseID: "kb-1", ParseStatus: "completed"}, + chunks: []weknora.Chunk{ + {ID: "chunk-1", KnowledgeID: "knowledge-secret-id", ChunkType: "text"}, + {ID: "chunk-2", KnowledgeID: "knowledge-secret-id", ChunkType: "text"}, + }, + search: []weknora.SearchResult{{KnowledgeID: "knowledge-secret-id", Content: "answer contains MIKU-GOV-P3-5927", Score: 0.02}}, + } + + report, err := verifyPublication(context.Background(), store, client, "tp-1", "unique marker", "MIKU-GOV-P3-5927") + if err != nil { + t.Fatalf("verifyPublication() error = %v", err) + } + if report.PublicationID != "tp-1" || report.RemoteStatus != "completed" || report.ChunkTotal != 2 || !report.AllChunksMatch || report.SearchResultCount != 1 || report.TargetSearchHits != 1 || !report.MarkerFound || !report.StructuralPassed || report.SemanticState != "manual_required" { + t.Fatalf("report = %#v", report) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"knowledge-secret-id", "chunk-1", "unique marker", "MIKU-GOV-P3-5927"} { + if string(encoded) == forbidden || contains(string(encoded), forbidden) { + t.Fatalf("report leaked %q: %s", forbidden, encoded) + } + } +} + +func contains(value, fragment string) bool { + for index := 0; index+len(fragment) <= len(value); index++ { + if value[index:index+len(fragment)] == fragment { + return true + } + } + return false +} + +type reportStoreStub struct { + publication postgresstore.PublicationForValidation + acceptance []governance.AcceptanceRecord +} + +func (store *reportStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) { + return store.publication, nil +} + +func (store *reportStoreStub) ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) { + return store.acceptance, nil +} + +type reportClientStub struct { + knowledge weknora.Knowledge + chunks []weknora.Chunk + search []weknora.SearchResult +} + +func (client *reportClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + return client.knowledge, nil +} + +func (client *reportClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) { + return client.chunks, int64(len(client.chunks)), nil +} + +func (client *reportClientStub) SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) { + return client.search, nil +} diff --git a/cmd/test-weknora-probe/main.go b/cmd/test-weknora-probe/main.go new file mode 100644 index 0000000..5baebf0 --- /dev/null +++ b/cmd/test-weknora-probe/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" + "knowledge-governance/internal/workflows/testweknora" +) + +type probeConfig struct { + DatabaseURL string + Concurrency int + Online bool + BaseURL string + APIKeyFile string + ChunkSampleEvery int +} + +func main() { + if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error { + config, err := parseProbeConfig(args, getenv) + if err != nil { + return err + } + + pool, err := pgxpool.New(ctx, config.DatabaseURL) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return fmt.Errorf("ping database: %w", err) + } + store := postgresstore.NewStore(pool) + legacy, err := store.ListLegacyVerificationCandidates(ctx) + if err != nil { + return err + } + candidates := make([]testweknora.LegacyCandidate, 0, len(legacy)) + for _, candidate := range legacy { + candidates = append(candidates, testweknora.LegacyCandidate{ + ID: candidate.ID, TestKBID: candidate.TestKBID, KnowledgeID: candidate.KnowledgeID, + }) + } + if !config.Online { + _, _ = io.WriteString(stdout, testweknora.FormatDryRunReport(candidates, config.Concurrency)) + return nil + } + apiKey, err := weknora.ReadAPIKeyFile(config.APIKeyFile) + if err != nil { + return fmt.Errorf("read test WeKnora API key: %w", err) + } + client, err := weknora.NewClient(weknora.ClientConfig{ + BaseURL: config.BaseURL, + APIKey: apiKey, + Timeout: 30 * time.Second, + }) + if err != nil { + return fmt.Errorf("create test WeKnora client: %w", err) + } + report, err := testweknora.VerifyLegacyConcurrent(ctx, client, candidates, config.ChunkSampleEvery, config.Concurrency) + if err != nil { + return fmt.Errorf("verify historical test publications: %w", err) + } + _, _ = io.WriteString(stdout, testweknora.FormatLegacyVerificationReport(report)) + return nil +} + +func parseProbeConfig(args []string, getenv func(string) string) (probeConfig, error) { + flags := flag.NewFlagSet("test-weknora-probe", flag.ContinueOnError) + flags.SetOutput(io.Discard) + databaseURL := flags.String("database-url", "", "PostgreSQL connection URL (or DATABASE_URL)") + concurrency := flags.Int("concurrency", 4, "planned read-only verify concurrency for the online phase") + online := flags.Bool("online", false, "run authenticated read-only verification") + baseURL := flags.String("base-url", "", "HTTPS WeKnora base URL") + apiKeyFile := flags.String("api-key-file", "", "mode-0600 WeKnora API key file") + chunkSampleEvery := flags.Int("chunk-sample-every", 25, "check one completed Knowledge chunk count every N items; 0 disables") + if err := flags.Parse(args); err != nil { + return probeConfig{}, fmt.Errorf("parse arguments: %w", err) + } + if *databaseURL == "" { + *databaseURL = getenv("DATABASE_URL") + } + if *databaseURL == "" { + return probeConfig{}, fmt.Errorf("-database-url or DATABASE_URL is required") + } + if *concurrency < 1 || *concurrency > 16 || *chunkSampleEvery < 0 { + return probeConfig{}, fmt.Errorf("concurrency must be 1..16 and chunk-sample-every must be non-negative") + } + if *baseURL == "" { + *baseURL = getenv("TEST_WEKNORA_BASE_URL") + } + if *apiKeyFile == "" { + *apiKeyFile = getenv("TEST_WEKNORA_API_KEY_FILE") + } + if *online { + parsed, err := url.Parse(strings.TrimSpace(*baseURL)) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil { + return probeConfig{}, fmt.Errorf("online verification requires an HTTPS base-url") + } + if !filepath.IsAbs(*apiKeyFile) { + return probeConfig{}, fmt.Errorf("online verification requires an absolute api-key-file") + } + } + return probeConfig{ + DatabaseURL: *databaseURL, + Concurrency: *concurrency, + Online: *online, + BaseURL: strings.TrimRight(strings.TrimSpace(*baseURL), "/"), + APIKeyFile: *apiKeyFile, + ChunkSampleEvery: *chunkSampleEvery, + }, nil +} diff --git a/cmd/test-weknora-probe/main_test.go b/cmd/test-weknora-probe/main_test.go new file mode 100644 index 0000000..8302e92 --- /dev/null +++ b/cmd/test-weknora-probe/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "io" + "strings" + "testing" +) + +func TestRunRequiresDatabaseURL(t *testing.T) { + err := run(context.Background(), nil, func(string) string { return "" }, io.Discard) + if err == nil || !strings.Contains(err.Error(), "database-url") { + t.Fatalf("run() error = %v, want database-url requirement", err) + } +} + +func TestParseProbeConfigSupportsExplicitOnlineMode(t *testing.T) { + config, err := parseProbeConfig([]string{ + "--online", + "--base-url", "https://weknora-test.example", + "--api-key-file", "/run/secrets/test_weknora_api_key", + "--concurrency", "4", + "--chunk-sample-every", "25", + }, func(key string) string { + if key == "DATABASE_URL" { + return "postgres://governance@example/governance" + } + return "" + }) + if err != nil { + t.Fatalf("parseProbeConfig() error = %v", err) + } + if !config.Online || config.DatabaseURL == "" || config.BaseURL != "https://weknora-test.example" || + config.APIKeyFile != "/run/secrets/test_weknora_api_key" || config.Concurrency != 4 || config.ChunkSampleEvery != 25 { + t.Fatalf("config = %#v", config) + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..9a6c4ac --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" + catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh" + "knowledge-governance/internal/workflows/sourcescan" + "knowledge-governance/internal/workflows/testweknora" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, getenv func(string) string) error { + config, err := appconfig.Load(getenv) + if err != nil { + return err + } + pool, err := pgxpool.New(ctx, config.DatabaseURL) + if err != nil { + return err + } + defer pool.Close() + if err := pool.Ping(ctx); err != nil { + return err + } + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return err + } + store := postgresstore.NewStore(pool) + handlers, err := workerHandlers(store, config) + if err != nil { + return err + } + worker, err := jobs.NewWorker(jobs.WorkerConfig{ + Queue: store, + WorkerID: config.WorkerID, + LeaseDuration: config.LeaseDuration, + RetryDelay: config.RetryDelay, + Handlers: handlers, + OnError: func(err error) { log.Printf("worker error: %v", err) }, + }) + if err != nil { + return err + } + go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) { + log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err) + }) + log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval) + return worker.Run(ctx, config.PollInterval) +} + +func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) { + handlers := map[string]jobs.Handler{ + "demo": func(_ context.Context, job jobs.Job) error { + log.Printf("completed demo job id=%s", job.ID) + return nil + }, + "demo.sleep": jobs.DemoSleepHandler, + } + catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py") + if err != nil { + return nil, fmt.Errorf("create catalog refresh runner: %w", err) + } + catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner}) + if err != nil { + return nil, fmt.Errorf("create catalog refresh handler: %w", err) + } + handlers["source.catalog.refresh"] = catalogHandler.Handle + if config.PPScanEnabled { + scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{ + AllowedRoot: config.PPScanAllowedRoot, + LedgerPath: config.PPScanLedgerPath, + ManifestPath: config.PPScanManifestPath, + }) + if err != nil { + return nil, fmt.Errorf("create PP source scanner: %w", err) + } + handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner}) + if err != nil { + return nil, fmt.Errorf("create PP source scan handler: %w", err) + } + handlers["source.scan.pp"] = handler.Handle + } + if !config.TestWeKnoraEnabled { + return handlers, nil + } + apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) + if err != nil { + return nil, fmt.Errorf("read test WeKnora API key: %w", err) + } + client, err := weknora.NewClient(weknora.ClientConfig{ + BaseURL: config.TestWeKnoraBaseURL, + APIKey: apiKey, + Timeout: config.TestWeKnoraHTTPTimeout, + AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP, + }) + if err != nil { + return nil, fmt.Errorf("create test WeKnora client: %w", err) + } + upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{ + Store: store, + Client: client, + SpoolRoot: config.GovernanceSpoolRoot, + MaxUploadBytes: config.TestWeKnoraMaxUploadBytes, + FirstPollDelay: config.TestWeKnoraPollInitial, + }) + if err != nil { + return nil, fmt.Errorf("create test upload handler: %w", err) + } + poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{ + Store: store, + Client: client, + PollDelay: config.TestWeKnoraPollInitial, + StallAfter: config.TestWeKnoraStalledAfter, + }) + if err != nil { + return nil, fmt.Errorf("create test poll handler: %w", err) + } + validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{ + Store: store, + Client: client, + PolicyVersion: config.TestAcceptancePolicyVersion, + }) + if err != nil { + return nil, fmt.Errorf("create test validate handler: %w", err) + } + remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{ + Store: store, + Client: client, + }) + if err != nil { + return nil, fmt.Errorf("create test removal handler: %w", err) + } + handlers["test.upload"] = upload.Handle + handlers["test.poll"] = poll.Handle + handlers["test.validate"] = validate.Handle + handlers["test.remove"] = remove.Handle + return handlers, nil +} diff --git a/cmd/worker/main_test.go b/cmd/worker/main_test.go new file mode 100644 index 0000000..d753ef9 --- /dev/null +++ b/cmd/worker/main_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "knowledge-governance/internal/appconfig" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +func TestRunRequiresDatabaseURL(t *testing.T) { + err := run(context.Background(), func(string) string { return "" }) + if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") { + t.Fatalf("run() error = %v, want DATABASE_URL error", err) + } +} + +func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) { + handlers, err := workerHandlers(nil, appconfig.Config{}) + if err != nil { + t.Fatalf("workerHandlers() error = %v", err) + } + if handlers["demo"] == nil || handlers["demo.sleep"] == nil { + t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers) + } + if handlers["release.probe"] != nil { + t.Fatal("ordinary worker must not register release.probe") + } + if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil { + t.Fatal("ordinary worker must not register test WeKnora jobs when disabled") + } + if handlers["source.scan.pp"] != nil { + t.Fatal("ordinary worker must not register PP scan jobs when disabled") + } +} + +func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) { + root := t.TempDir() + config := appconfig.Config{ + PPScanEnabled: true, + PPScanAllowedRoot: root, + PPScanLedgerPath: filepath.Join(root, "state.jsonl"), + PPScanManifestPath: filepath.Join(root, "manifest.json"), + } + handlers, err := workerHandlers(postgresstore.NewStore(nil), config) + if err != nil { + t.Fatalf("workerHandlers() error = %v", err) + } + if handlers["source.scan.pp"] == nil { + t.Fatal("source.scan.pp handler is not registered") + } +} + +func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) { + config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"} + _, err := workerHandlers(nil, config) + if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") { + t.Fatalf("workerHandlers() error = %v", err) + } +} + +func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) { + keyFile := filepath.Join(t.TempDir(), "api-key") + if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil { + t.Fatal(err) + } + config := appconfig.Config{ + TestWeKnoraEnabled: true, + TestWeKnoraAllowDockerHTTP: true, + TestWeKnoraBaseURL: "http://app:8080", + TestWeKnoraAPIKeyFile: keyFile, + GovernanceSpoolRoot: t.TempDir(), + TestWeKnoraHTTPTimeout: time.Second, + TestWeKnoraPollInitial: time.Second, + TestWeKnoraStalledAfter: time.Minute, + TestWeKnoraMaxUploadBytes: 1024, + TestAcceptancePolicyVersion: "structural-v1", + } + handlers, err := workerHandlers(postgresstore.NewStore(nil), config) + if err != nil { + t.Fatalf("workerHandlers() error = %v", err) + } + for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} { + if handlers[jobType] == nil { + t.Fatalf("handler %q is not registered", jobType) + } + } +} diff --git a/cmd/worker/scheduler.go b/cmd/worker/scheduler.go new file mode 100644 index 0000000..5d39d5a --- /dev/null +++ b/cmd/worker/scheduler.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "errors" + "log" + "time" +) + +// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store +// satisfies it and tests can fake it without a database. +type dueSourceClaimer interface { + ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error) +} + +// runSourceRefreshTick performs one dispatcher pass and logs only when work +// moved, so an idle system does not spam the worker log. +func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error { + claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now) + if err != nil { + return err + } + if claimed > 0 || enqueued > 0 { + log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued) + } + return nil +} + +// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is +// deliberately delayed by one interval: startup already races ApplyMigrations +// and handler registration, and a due backlog enqueue adds nothing on the +// second the process comes up (the worker will drain whatever is queued anyway +// once sources were scheduled before a restart). +func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) { + if claimer == nil || interval <= 0 { + return + } + if onError == nil { + onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) } + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case tick := <-ticker.C: + if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) { + onError(err) + } + } + } +} diff --git a/cmd/worker/scheduler_test.go b/cmd/worker/scheduler_test.go new file mode 100644 index 0000000..c6ae82a --- /dev/null +++ b/cmd/worker/scheduler_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type fakeClaimer struct { + mu sync.Mutex + calls []time.Time + claimed [2]int + err error +} + +func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, now) + if f.err != nil { + return 0, 0, f.err + } + return f.claimed[0], f.claimed[1], nil +} + +func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) { + claim := &fakeClaimer{claimed: [2]int{3, 2}} + if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil { + t.Fatal(err) + } + if len(claim.calls) != 1 { + t.Fatalf("calls=%d want 1", len(claim.calls)) + } + sentinel := errors.New("boom") + failing := &fakeClaimer{err: sentinel} + if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) { + t.Fatalf("error=%v want sentinel", err) + } +} + +func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) { + claim := &fakeClaimer{} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil) + }() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + claim.mu.Lock() + count := len(claim.calls) + claim.mu.Unlock() + if count >= 2 { + break + } + time.Sleep(5 * time.Millisecond) + } + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("scheduler did not stop on context cancel") + } + claim.mu.Lock() + defer claim.mu.Unlock() + if len(claim.calls) < 2 { + t.Fatalf("calls=%d want at least 2", len(claim.calls)) + } + for _, stamp := range claim.calls { + if stamp.Location() != time.UTC { + t.Fatalf("tick stamp %v not UTC", stamp) + } + } +} + +func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) { + claim := &fakeClaimer{err: errors.New("database is offline")} + var errorsSeen int + var mu sync.Mutex + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) { + mu.Lock() + errorsSeen++ + mu.Unlock() + }) + }() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + mu.Lock() + seen := errorsSeen + mu.Unlock() + if seen >= 2 { + break + } + time.Sleep(5 * time.Millisecond) + } + cancel() + <-done + mu.Lock() + defer mu.Unlock() + if errorsSeen < 2 { + t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen) + } +} + +func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) { + // guards against misconfiguration killing the process + runSourceRefreshScheduler(context.Background(), nil, time.Second, nil) + runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil) +} diff --git a/compose.release-sim.yaml b/compose.release-sim.yaml new file mode 100644 index 0000000..4431ab6 --- /dev/null +++ b/compose.release-sim.yaml @@ -0,0 +1,36 @@ +# 本地模拟生产发布专用 override（target=测试服务器 weknora-test-v071） +# 用法：docker compose -f compose.yaml -f compose.release-sim.yaml up -d +# 真实生产严禁使用本文件（真实生产 = 未来独立授权 + 独立配置） +services: + api: + environment: + PRODUCTION_RELEASE_REQUESTS_ENABLED: "true" + PRODUCTION_BACKUP_RESTORE_VERIFIED: "true" + PRODUCTION_RELEASE_AUTHORIZER_REF: yaozhanghao + PRODUCTION_RELEASE_AUTH_TOKEN_FILE: /run/release-secrets/production_release_auth_token + volumes: + - ./secrets/release-sim:/run/release-secrets:ro + + release-worker: + environment: + RELEASE_WORKER_ENABLED: "true" + RELEASE_CONFIG_FILE: /run/release-sim/release.sim.json + RELEASE_DEBUG_ITEM_ERRORS: "true" + PRODUCTION_RELEASE_AUTHORIZER_REF: yaozhanghao + extra_hosts: + - "host.docker.internal:host-gateway" + - "prod-pg:host-gateway" + - "prod-es:host-gateway" + - "prod-weknora:host-gateway" + networks: + - default + - weknora-local + volumes: + - ./secrets/release-sim:/run/release-secrets:ro + - ./config/release.sim.json:/run/release-sim/release.sim.json:ro + - release-bundles:/var/lib/knowledge-governance/release-bundles + read_only: false + +volumes: + release-bundles: + name: knowledge-governance-release-bundles diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..6cdebe1 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,164 @@ +name: knowledge-governance-p1 + +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_DB: governance + POSTGRES_USER: governance + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - governance-pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U governance -d governance"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + api: + build: + context: . + dockerfile: Dockerfile + command: ["/app/api"] + user: "${APP_UID:-1000}:${APP_GID:-1000}" + environment: + <<: &app-environment + DATABASE_URL: postgres://governance@postgres:5432/governance?sslmode=disable + PGPASSFILE: /run/secrets/pgpass + APP_ADDR: :8080 + WORKER_POLL_INTERVAL: 1s + WORKER_LEASE_DURATION: 30s + WORKER_RETRY_DELAY: 5s + TEST_WEKNORA_BASE_URL: "${TEST_WEKNORA_BASE_URL:-}" + MINERU_BASE_URL: "${MINERU_BASE_URL:-}" + TEST_INTERNAL_WEKNORA_KB_ID: "${TEST_INTERNAL_WEKNORA_KB_ID:-1b52c140-2e18-4676-afc5-4c7d8d190a8c}" + TEST_EXTERNAL_WEKNORA_KB_ID: "${TEST_EXTERNAL_WEKNORA_KB_ID:-fd7db6aa-763a-46f1-84d2-ecc7c952a5ea}" + # P10 readiness probe reads the routing KB with the same scoped key. + TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key + TEST_WEKNORA_ALLOW_DOCKER_HTTP: "${TEST_WEKNORA_ALLOW_DOCKER_HTTP:-false}" + WORKER_LIVENESS_STALE_AFTER: "${WORKER_LIVENESS_STALE_AFTER:-2m}" + PRODUCTION_RELEASE_REQUESTS_ENABLED: ${PRODUCTION_RELEASE_REQUESTS_ENABLED:-false} + PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false} + secrets: + - pgpass + - test_weknora_api_key + depends_on: + postgres: + condition: service_healthy + healthcheck: + # P10: liveness only — a third-party outage must never restart the API. + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/livez"] + interval: 5s + timeout: 3s + retries: 20 + read_only: true + tmpfs: + - /tmp + networks: + - default + - weknora-local + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile + command: ["/app/worker"] + user: "${APP_UID:-1000}:${APP_GID:-1000}" + environment: + <<: *app-environment + WORKER_ID: worker-1 + PP_SCAN_ENABLED: "${PP_SCAN_ENABLED:-false}" + PP_SCAN_ALLOWED_ROOT: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}" + PP_SCAN_LEDGER_PATH: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}/public-weknora-sync/state.jsonl" + PP_SCAN_MANIFEST_PATH: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}/public-documents/manifest.json" + TEST_WEKNORA_ENABLED: "${TEST_WEKNORA_ENABLED:-false}" + TEST_WEKNORA_ALLOW_DOCKER_HTTP: "${TEST_WEKNORA_ALLOW_DOCKER_HTTP:-false}" + TEST_WEKNORA_KB_ID: "${TEST_WEKNORA_KB_ID:-}" + TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key + GOVERNANCE_SPOOL_ROOT: /var/lib/governance/spool + TEST_WEKNORA_HTTP_TIMEOUT: "${TEST_WEKNORA_HTTP_TIMEOUT:-60s}" + TEST_WEKNORA_POLL_INITIAL: "${TEST_WEKNORA_POLL_INITIAL:-10s}" + TEST_WEKNORA_POLL_MAX: "${TEST_WEKNORA_POLL_MAX:-60s}" + TEST_WEKNORA_STALLED_AFTER: "${TEST_WEKNORA_STALLED_AFTER:-30m}" + TEST_WEKNORA_MAX_UPLOAD_BYTES: "${TEST_WEKNORA_MAX_UPLOAD_BYTES:-52428800}" + TEST_ACCEPTANCE_POLICY_VERSION: "${TEST_ACCEPTANCE_POLICY_VERSION:-structural-v1}" + secrets: + - pgpass + - test_weknora_api_key + volumes: + - ./spool:/var/lib/governance/spool:ro + - "${PP_SCAN_HOST_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:ro" + - /home/zheromiku/dingtalk_kb_download:/sources/dingtalk:ro + - /home/zheromiku/tmp/alist:/sources/alist:ro + - /home/zheromiku/dobot-portal-sync:/sources/pp:ro + - ./secrets/dingtalk_capture.json:/run/source-secrets/dingtalk_capture.json:ro + - ./secrets/alist_connections.xlsx:/run/source-secrets/alist_connections.xlsx:ro + - ./secrets/alist_api_token:/run/source-secrets/alist_api_token:ro + - ./secrets/pp_credentials.json:/run/source-secrets/pp_credentials.json:ro + networks: + - default + - weknora-local + depends_on: + postgres: + condition: service_healthy + read_only: true + tmpfs: + - /tmp + restart: unless-stopped + + release-worker: + build: + context: . + dockerfile: Dockerfile + command: ["/app/release-worker"] + user: "${APP_UID:-1000}:${APP_GID:-1000}" + environment: + <<: *app-environment + WORKER_ID: release-worker-1 + RELEASE_WORKER_ENABLED: "false" + RELEASE_CONFIG_FILE: "" + secrets: + - pgpass + depends_on: + postgres: + condition: service_healthy + read_only: true + tmpfs: + - /tmp + restart: unless-stopped + + web: + build: + context: . + dockerfile: Dockerfile.web + ports: + - "8088:8080" + depends_on: + api: + condition: service_healthy + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + - /var/run + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + pgpass: + file: ./secrets/pgpass + test_weknora_api_key: + file: ./secrets/test_weknora_api_key + +volumes: + governance-pgdata: + +networks: + weknora-local: + external: true + name: weknora071full-network diff --git a/config/release.sim.json b/config/release.sim.json new file mode 100644 index 0000000..a055609 --- /dev/null +++ b/config/release.sim.json @@ -0,0 +1,46 @@ +{ + "archive_root": "/var/lib/knowledge-governance/release-bundles", + "routes": [ + { + "tenant_id": 10000, + "knowledge_base_id": "c12ff4d5-dd87-423b-ab0f-af745b9d54a7", + "embedding_model": "text-embedding-v4", + "source": { + "database_url_file": "/run/release-secrets/source_database_url", + "elasticsearch": { + "base_url": "http://elasticsearch:9200", + "index": "weknora_full_chunks" + }, + "object": { + "provider": "oss", + "endpoint": "https://oss-cn-shenzhen.aliyuncs.com", + "region": "cn-shenzhen", + "bucket": "yzh-weknora-test", + "access_key_file": "/run/release-secrets/source_object_access_key", + "secret_key_file": "/run/release-secrets/source_object_secret_key" + } + }, + "target": { + "database_url_file": "/run/release-secrets/target_database_url", + "elasticsearch": { + "base_url": "http://prod-es:29200", + "index": "weknora_test_chunks" + }, + "object": { + "provider": "oss", + "endpoint": "https://oss-cn-shenzhen.aliyuncs.com", + "region": "cn-shenzhen", + "bucket": "dobot-weknora-test", + "storage_backend_id": "6129cfd1-a26c-41dd-88cb-e6a800daa3bb", + "access_key_file": "/run/release-secrets/target_object_access_key", + "secret_key_file": "/run/release-secrets/target_object_secret_key" + }, + "weknora": { + "base_url": "http://prod-weknora:28080", + "api_key_file": "/run/release-secrets/target_weknora_api_key", + "allow_docker_http": true + } + } + } + ] +} diff --git a/config/release.v071.example.json b/config/release.v071.example.json new file mode 100644 index 0000000..d0d43e9 --- /dev/null +++ b/config/release.v071.example.json @@ -0,0 +1,48 @@ +{ + "archive_root": "/var/lib/knowledge-governance/release-bundles", + "routes": [ + { + "tenant_id": 10000, + "knowledge_base_id": "replace-with-identical-test-and-production-kb-id", + "embedding_model": "bge-m3", + "source": { + "database_url_file": "/run/release-secrets/source_database_url", + "elasticsearch": { + "base_url": "http://source-elasticsearch:9200", + "index": "chunks", + "token_file": "/run/release-secrets/source_elasticsearch_token" + }, + "object": { + "provider": "minio", + "endpoint": "http://source-object-storage:9000", + "region": "us-east-1", + "bucket": "source-weknora", + "access_key_file": "/run/release-secrets/source_object_access_key", + "secret_key_file": "/run/release-secrets/source_object_secret_key" + } + }, + "target": { + "database_url_file": "/run/release-secrets/target_database_url", + "elasticsearch": { + "base_url": "https://target-elasticsearch.example", + "index": "chunks", + "token_file": "/run/release-secrets/target_elasticsearch_token" + }, + "object": { + "provider": "oss", + "endpoint": "https://target-object-storage.example", + "region": "cn-shenzhen", + "bucket": "target-weknora", + "storage_backend_id": "replace-with-production-storage-backend-id", + "access_key_file": "/run/release-secrets/target_object_access_key", + "secret_key_file": "/run/release-secrets/target_object_secret_key" + }, + "weknora": { + "base_url": "https://production-weknora.example", + "api_key_file": "/run/release-secrets/target_weknora_api_key", + "allow_docker_http": false + } + } + } + ] +} diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 0000000..a25c096 --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,24 @@ +server { + listen 8080; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://api:8080; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /health { + proxy_pass http://api:8080/health; + proxy_set_header Host $host; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/docs/ACTION_ITEMS_12_TO_END_ACCEPTANCE.md b/docs/ACTION_ITEMS_12_TO_END_ACCEPTANCE.md new file mode 100644 index 0000000..19ae64b --- /dev/null +++ b/docs/ACTION_ITEMS_12_TO_END_ACCEPTANCE.md @@ -0,0 +1,84 @@ +# 工程实施行动清单：第12项至结束验收 + +## 范围 + +本轮从“来源与目录”的第8项（全局第12项：来源快照差异 ChangeSet）继续到清单结束。来源与目录现有浏览和标注界面保持不变；解析监控、发布管理、系统设置继续完善。生产发布请求与 Release Worker 保持关闭，本验收不代表生产上线。 + +## 第12项：来源快照 ChangeSet + +- Migration 15 新增 `source_catalog_change_sets` 与 `source_catalog_change_items`。 +- 每次快照激活前持久计算 `add / update / move_scope / delete / no_change`。 +- Migration 16 为现有7个来源回填最新快照差异。 +- 提供只读安全 API：`GET /api/v1/source-catalog/change-sets/latest`。 +- 解析监控页展示各来源最新持久 ChangeSet 和变更文件预览，不返回 locator hash、snapshot hash 或凭据。 +- 真实读回：国内工业新增80、更新3、删除18、未变化4859。 + +## 第13–22项：解析链路与监控 + +已验证： + +- Spool 文件上传前计算并校验完整 SHA-256、字节数和格式；流式上传后再次校验。 +- `source_items` 与 `source_item_versions` 分离文件身份和版本；`content_objects.sha256` 全局唯一。 +- 标注只创建计划；`processing_confirmations` 人工确认后才创建 `test.upload` Job。 +- `target_scope` 明确对内/对外；禁止导入进入 `test.remove` 并读回确认。 +- Publication、Job、轮询序列和人工重试均有幂等键或唯一约束。 +- Worker 支持租约、心跳、退避重试、最大尝试次数和失败终态；页面支持单个/批量重试。 +- WeKnora 上传、轮询、最终 completed、Knowledge/Chunk 验收已经接通；当前测试编排开关为启用状态。 +- 解析监控已切换为真实 Provider：主机 CPU、内存、磁盘和可见 GPU；PostgreSQL、任务队列与 WeKnora 健康探测；未配置的 MinerU、OSS、Redis/消息队列明确显示未配置/未部署，不再模拟正常。 + +## 第23–28项：发布管理 + +已验证： + +- `GET /api/v1/release-plan` 根据来源最终策略、测试终态和最近生产状态生成 add/update/delete/blocked/unchanged。 +- 发布前 bundle 保存完整 manifest、三通道摘要和 SHA-256；生产恢复验证未通过时授权门禁阻断。 +- Migration 17 新增 `batch_size/current_batch/total_batches`；发布默认每50项一批并持久显示批次进度。 +- 一项永久失败后，当前项之后和后续批次停止；未执行项记录 `release_stopped_after_failure`。 +- 已成功项才进入自动回滚范围；回滚结果区分 restored/rolled_back/failed。 +- PostgreSQL、Elasticsearch、对象存储完成后均读回验证；不以 HTTP 200 代替一致性验收。 +- 发布计划、审核、封批、执行历史、逐项结果和固定错误码均可从页面查看。 +- 修复发布计划安全投影省略空 `publication_id/blocker` 导致前端拒绝整个计划的问题。 + +## 第29项：系统设置 + +- 来源列表从真实来源配置动态生成，不写死具体来源名称。 +- Test/Production WeKnora、OSS/备份、刷新与告警参数分区显示。 +- AList 显示直连 API Token；密钥值不回传、不写任务载荷、不进入页面。 +- 页面仅允许修改低风险刷新/告警参数；端点、凭据、生产开关继续由管理员受限配置管理。 +- 系统设置使用真实服务健康结果，不再混用解析监控模拟数据。 + +## 第30项：本地故障演练 + +通过的自动化场景： + +- 目录标注重复确认不会重复创建上传任务。 +- SHA/大小/格式不一致时上传前失败；流式过程中文件改变时失败。 +- 上传完成后进入轮询；WeKnora completed 后创建验收任务，不继续无效轮询。 +- 禁止导入调用测试删除并读回确认。 +- Worker 失败按固定延迟重试，达到最大次数进入失败终态。 +- 105项发布计划分成3批（50/50/5）。 +- 永久失败立即停止后续项目和批次，并只为已验证项目创建一次自动回滚。 +- Migration 17→16→17 往返和 Migration 11 历史范围回填均在独立临时数据库通过。 +- 解析监控、发布管理、系统设置在 1440/1024/768 三种宽度无页面级横向溢出。 + +## 当前真实运行状态 + +- schema migration：17。 +- Monitor：`mode=real`、`simulated=false`。 +- PostgreSQL、任务队列、WeKnora API：真实健康。 +- Test Publications：537条，均处于 completed。 +- Release Plan：新增4、阻断614。 +- Production release executions：0。 +- `PRODUCTION_RELEASE_REQUESTS_ENABLED=false`。 +- `RELEASE_WORKER_ENABLED=false`，进程确认未加载生产凭据。 + +## 验证命令结果 + +- Go：`go test ./... && go vet ./... && go build ./cmd/...` 通过。 +- Web：22个测试文件、90项测试通过；TypeScript检查和生产构建通过。 +- PostgreSQL：ChangeSet、标注/确认幂等、发布授权/自动回滚、Migration往返均在一次性临时数据库通过。 +- Browser：三个剩余页面真实内容加载成功，响应式检查通过。 + +## 仍需生产负责人单独确认的外部验收 + +生产发布和真实生产回滚没有执行，也不应由本轮开发授权隐含开启。上线前仍需生产负责人提供独立授权，并完成：生产三通道备份恢复验证、代表性小批发布、故障注入和真实回滚读回。当前页面与后端会在该门禁未满足时持续阻断生产操作。 diff --git a/docs/DOCUMENT_WORKBENCH_ACCEPTANCE.md b/docs/DOCUMENT_WORKBENCH_ACCEPTANCE.md new file mode 100644 index 0000000..f22ae5c --- /dev/null +++ b/docs/DOCUMENT_WORKBENCH_ACCEPTANCE.md @@ -0,0 +1,49 @@ +# 文档工作台第一版：已交付与未完成边界 + +## 打开方式 + +本地治理平台：http://100.92.49.78:8088/ （Tailscale 内访问） + +首页“来源与目录”：选择来源/分类，在文件表中搜索、筛选对内/对外和处理状态；点击文件名称打开详情。“解析监控”每15秒刷新当前记录状态；“待处理文件”只列超限、失败、来源异常。“发布管理”保留审核和生产门禁，流程说明默认折叠。“来源连接”显示连接和本机快照检查，历史审计默认折叠。 + +文件表区域内滚动，表头固定；不是无限拉长整页。顶部标签保持单例、可关闭、刷新恢复，首页固定。 + +## 真实数据与来源边界 + +- 治理清单618份：PP历史文档613份，人工隔离验收文档5份。 +- 618份均有名称；547份能匹配现存PP分类，PP其余66份标记分类待补齐。 +- 现有快照只证明分类，不证明完整原始目录/父子关系。页面明确标注“PP分类 · 不代表完整原始目录”。 +- 对外613、对内5，均展示现有记录，不自动改变可见性；解析完成不代表生产已上线。 +- 已解析535；超限54，列入待处理。其他状态保持后端原值映射，不凑百分比。 +- 钉钉/AList目录尚未接通，不展示虚构目录；多个来源以连接ID而非来源类型隔离筛选。 +- PP“检查快照变化”只检查已保存的本机快照，不是实时抓取PP网站。 + +## 本轮改动范围 + +- SourceAsset API补充安全展示字段（文件名、相对分类/目录、连接、可见性依据、摄取状态、失败码、处理记录和受控重试能力）。拒绝展示绝对主机路径、URL和越界目录。 +- `scripts/import_display_metadata.py`：读取既有只读manifest和inventory；按已有来源身份匹配，只补空白title/normalized_path。默认事务回滚预览，`--apply`才提交。没有源文件写入、下载、上传、版本/范围/发布变更。重复dry-run更新0行。 +- 前端完整分页加载、去重及总数校验后建立来源分类清单，不能将第一页冒充全部数据。加载失败不能显示成功的零数量。 +- 复用后端已存在的失败重试门禁；未对真实文件触发重试。 +- 发布批次不再要求审核完整候选页，只允许选择已批准且就绪项。生产发布仍独立关闭。 + +## 实测证据 + +- Web：45/45 tests；TypeScript、Vite build通过。 +- Go：`GOPROXY=off go test ./...`通过；相关包vet通过。依赖外部配置的集成测试默认仍跳过，不能把全量命令扩称所有远端E2E通过。 +- 独立PostgreSQL测试库`governance_document_ui_test`：本轮SourceAsset真实查询测试通过；测试库保留，未删除数据。 +- `scripts/accept_document_workbench.py`：真实Firefox搜索、分类筛选、范围筛选、文件详情、自动刷新、标签恢复、默认折叠说明、列表区域滚动通过；1440/1024/768宽度无页面横向溢出。脚本不执行审核/上传/重试/生产写入。 +- 结果：`tmp/document-ui-acceptance.json`；截图：`tmp/DOCUMENT_WORKBENCH_FIREFOX.png`；完整清单核对：`tmp/document-ui-catalog.json`。 +- 本机Web HTTP 200，Tailscale入口HTTP 200；API healthy，API/Web运行镜像匹配、零重启。 +- 生产请求开关false，Release Worker开关false、配置路径空。 +- 数据保持source_items/content_objects/versions/test_publications = 618/535/618/535；活动任务0。 +- 仅重建API/Web，不重建PostgreSQL及两个Worker、不修改WeKnora、不连接生产，不提交或push。 +- 回退镜像：`knowledge-governance-api:before-document-ui`、`knowledge-governance-web:before-document-ui`。补齐元数据前已生成0600原生数据库备份并检查dump目录。 + +## 尚未完成（不得称为全部功能完成） + +1. 来源/目录级对内外规则编辑、继承与单独覆盖、批量设置、规则变更审计和下游权限生效验证。当前范围只可查看/筛选，不是权限管理器。 +2. 钉钉及AList真实目录接入；PP完整原始层级仍需可靠来源元数据。 +3. 原文预览、内容差异、逐文件审核核对入口。 +4. 页面化配置检查计划、目标知识库及通知方式；目前仍由管理员配置。 + +以上未完成项是独立实施和验收边界，本轮不以换标题或隐藏技术细节冒充完成。 diff --git a/docs/DOCUMENT_WORKBENCH_V2_ACCEPTANCE.md b/docs/DOCUMENT_WORKBENCH_V2_ACCEPTANCE.md new file mode 100644 index 0000000..ef4081b --- /dev/null +++ b/docs/DOCUMENT_WORKBENCH_V2_ACCEPTANCE.md @@ -0,0 +1,47 @@ +# 文档工作台 V2 验收 + +入口：http://100.92.49.78:8088/（Tailscale 内） + +## 已实现 + +- 日常首页默认只显示613份PP业务文档；5份隔离验收文档默认隐藏，可手动显示。 +- 来源/分类 → 文件清单为主操作路径；顶部显示可点击的来源、分类/目录、文件上下文。 +- 来源与可靠分类可设置对内/对外；规则作用于当前目录及子目录。分类待补齐不允许伪装成独立目录规则。 +- 单文件可单独覆盖目录规则；清除覆盖后回退至最近目录规则，再回退至来源原始标记。 +- 每次设置或清除都要求操作人和判断原因，并与当前规则在同一数据库事务中记录审计。 +- 范围规则只更新治理判断，不修改PP/钉钉/AList源文件，也不自动发布生产。 +- 解析监控按来源显示：已就绪、独立解析、内容复用、处理中、需处理、等待；不显示虚构百分比。 +- 文件详情显示取得文件、上传测试库、解析与索引、测试检索四段真实状态，明确原文与版本差异尚未接通。 + +## 数据口径 + +- 全部记录：618；默认业务清单：613；隐藏验收文档：5。 +- PP：独立解析530、内容复用29、需处理54、处理中0、等待0；已就绪559/613。 +- 当前原始范围：对外613、对内5。 +- Migration 8仅新增`directory_scope_rules`、`source_item_scope_overrides`、`scope_change_events`。 + +## 自动化与运行验收 + +- Web：12个测试文件、54项测试全部通过；TypeScript和Vite构建通过。 +- Go：`GOPROXY=off go test ./...`全部通过；范围规则相关包vet通过。 +- 独立数据库：文件覆盖 → 目录继承 → 来源默认的真实PostgreSQL链路通过。 +- 真实Firefox：目录规则设置、5份文件生效、单文件覆盖、清除覆盖恢复继承、清除目录恢复来源标记、顶部上下文、处理链、默认隐藏测试文档、1440/1024/768无页面横向溢出均通过。 +- 三轮浏览器写回验收共留下12条审计事件（目录/文件各3次set和3次clear）；当前有效目录规则0、文件覆盖0，没有遗留验收设置。 +- 最终数据保持：source_items/content_objects/versions/test_publications = 618/535/618/535；活动任务0。 +- API、PostgreSQL健康；所有服务运行、零重启且镜像匹配；本机和Tailscale入口HTTP 200。 +- 生产请求开关false；Release Worker开关false、配置路径空。 + +证据： + +- `tmp/document-workbench-v2-acceptance.json` +- `tmp/DOCUMENT_WORKBENCH_V2_FIREFOX.png` +- `tmp/DOCUMENT_MONITOR_V2_FIREFOX.png` +- `tmp/backups/pre-scope-rules-20260908T170120.dump`（PostgreSQL custom dump，290185字节，0600，目录可读） +- 回退镜像：`knowledge-governance-api:before-scope-v2`、`knowledge-governance-web:before-scope-v2` + +## 仍未伪装成已完成的能力 + +- PP目前只有分类快照，不是完整原始目录树；钉钉/AList目录仍未接通。 +- 原文预览和版本差异仍未接通。 +- 本轮范围规则会影响后续人工审核时看到的治理范围，但不会绕过审核或自动改变既有生产知识；生产生效仍需独立发布和验证。 +- 未提交、未push，未连接或写入生产WeKnora。 diff --git a/docs/DOMAIN_MODEL.md b/docs/DOMAIN_MODEL.md new file mode 100644 index 0000000..1e5eb6f --- /dev/null +++ b/docs/DOMAIN_MODEL.md @@ -0,0 +1,293 @@ +# 知识治理平台领域对象契约 + +## 目的 + +本文件是知识治理平台当前实现的唯一领域对象映射。它将产品术语映射到现有Go模型、PostgreSQL表和外部边界；后续模块不得另造一套身份、时间或状态语义。 + +本文件不把“代码存在”“容器健康”写成已完成能力。尚未存在的对象或关联明确列在 [Current implementation gaps](#current-implementation-gaps)。 + +## 统一主链 + +```text +SourceConnection → SourceItem → SourceItemVersion → ContentObject +SourceConnection → SourceCatalogSnapshot → SourceCatalogNode +SourceCatalogChangeSet → SourceCatalogChangeItem +SourceItemVersion → TestPublication → AcceptanceResult +ReleaseBatch → ReleaseExecution → ReleaseItemExecution +``` + +完整目标主链将在后续工作包补齐为： + +```text +SourceConnection +→ SourceItem / SourceItemVersion / ContentObject +→ SourceCatalogSnapshot +→ SourceCatalogChangeSet +→ ProcessingExecution / Job +→ TestPublication / AcceptanceResult +→ ReleaseChangeSet +→ ProductionSnapshot +→ ReleaseBatch / ReleaseExecution +``` + +## 全局身份与时间规则 + +| 规则 | 契约 | +| --- | --- | +| ID | 所有主对象使用服务端生成的不透明ID；页面和Job Payload只能传递ID，不能传递受保护来源URL、Token或Cookie。 | +| 时间 | `created_at`表示治理对象创建；`updated_at`表示本对象最后治理状态变化；来源侧时间必须单独使用`updated_at_source`或`source_updated_at`。 | +| 内容Hash | 只有`content_objects.sha256`是文件内容SHA-256。`canonical_uri_hash`、`source_locator_hash`和`snapshot_sha256`绝不能作为内容Hash展示或用于内容去重。 | +| 删除 | 来源不可用不等于来源删除。只有权威来源删除事件或经人工确认的删除ChangeSet才可进入目标移除流程。 | +| 目标写入 | 普通治理层不直接写WeKnora数据库；测试写入通过官方API/受控适配器；生产发布由独立Release Worker和门禁处理。 | + +## Source + +**产品含义：**一个独立配置、独立健康状态、独立刷新策略的资料来源。 + +**当前实现：** + +| 项目 | 当前映射 | +| --- | --- | +| Go模型 | `governance.SourceConnection` | +| 数据表 | `source_connections` | +| 主键 | `source_connections.id` | +| 类型 | `source_type`：当前为`pp`、`alist`、`dingtalk`；`manual`是Go层支持的扩展类型。 | +| 时间 | `created_at`、`updated_at` | +| 可变配置 | `enabled`、`schedule`、`last_health` | +| 敏感字段 | `base_url`、`secret_ref`；不得返回前端或日志。 | + +**不可变/关系约束：** + +```text +一个SourceConnection可以有多个SourceItem。 +一个SourceConnection可以有多个SourceCatalogSnapshot。 +SourceItem、Snapshot、ChangeSet不得跨SourceConnection关联。 +``` + +## SourceFile + +**产品含义：**来源中的一份可治理资料，以及该资料的每个来源版本和内容对象。 + +SourceFile不是一个单表对象，当前由三个对象组成： + +| 逻辑层 | Go模型 | 数据表 | 稳定身份 | +| --- | --- | --- | --- | +| 文件身份 | `governance.SourceItem` | `source_items` | `(source_connection_id, stable_external_id)` | +| 文件版本 | `governance.SourceItemVersion` | `source_item_versions` | `(source_item_id, revision)` | +| 内容对象 | `governance.ContentObject` | `content_objects` | 全局`sha256` | + +**关键字段：** + +```text +SourceItem: +- source_connection_id +- stable_external_id +- canonical_uri_hash +- normalized_path +- revision +- updated_at_source +- source_change + +SourceItemVersion: +- source_item_id +- revision +- content_object_id（内容尚未下载或验证前允许为空） +- content_state +- source_updated_at + +ContentObject: +- sha256 +- size_bytes +- mime +- validated_format +- local_spool_ref +- content_state +``` + +**关系规则：** + +```text +路径变化不必然产生新SourceItem。 +内容Hash变化产生新的SourceItemVersion或新的内容关联。 +同一内容Hash可以被多个SourceItemVersion复用，但每个版本仍保留自己的来源身份和来源时间。 +TestPublication必须引用一个SourceItemVersion，不能直接引用路径或ContentObject。 +``` + +## Snapshot + +**产品含义：**一次来源目录扫描获得的不可变目录事实；它记录当时看到的文件/目录，而不是文件正文副本。 + +**当前实现：** + +| 项目 | 当前映射 | +| --- | --- | +| 快照表 | `source_catalog_snapshots` | +| 节点表 | `source_catalog_nodes` | +| 当前指针 | `source_catalog_heads` | +| 快照主键 | `source_catalog_snapshots.id` | +| 来源约束 | `(source_connection_id, snapshot_sha256)`唯一 | +| 时间 | `observed_at`、`created_at`；Head使用`activated_at` | +| 目录节点 | `node_type=folder` | +| 文件节点 | `node_type=file` | + +**Snapshot规则：** + +```text +SourceConnection → SourceCatalogSnapshot → SourceCatalogNode +Head只指向一个当前快照；切换Head不得修改旧快照。 +节点的source_locator_hash是来源定位摘要，不是内容Hash。 +节点可选关联source_item_id；未关联节点仍可作为目录事实展示。 +``` + +**操作记录补充：**`source_scans`和`source_scan_changes`是旧扫描运行记录，不能替代当前不可变目录快照。 + +## ChangeSet + +**产品含义：**两个不可变来源Snapshot之间的可审查差异计划。 + +**当前实现：** + +| 项目 | 当前映射 | +| --- | --- | +| 主表 | `source_catalog_change_sets` | +| 条目表 | `source_catalog_change_items` | +| Go投影 | `governance.CatalogChangeSet`、`CatalogChangeItem` | +| 主键 | `source_catalog_change_sets.id` | +| 快照关系 | `from_snapshot_id`、`to_snapshot_id` | +| 动作 | `add`、`update`、`move_scope`、`delete`、`no_change` | +| 时间 | `created_at`、`confirmed_at`、`applied_at` | +| 当前状态字段 | `planned`、`confirmed`、`applied`、`cancelled` | + +**ChangeSet规则：** + +```text +一个ChangeSet只能属于一个SourceConnection。 +from_snapshot_id和to_snapshot_id必须属于同一SourceConnection。 +to_snapshot_id当前唯一，因此一份激活快照最多对应一份目录ChangeSet。 +ChangeItem当前使用(change_set_id, source_locator_hash)唯一。 +ChangeSet只表达比较结果；它本身不等于已经上传、解析或删除。 +``` + +**后续目标：**S8/P3/P5会补充SourceItemVersion、内容Hash、旧新范围、确认执行和Job关联。 + +## Job + +**产品含义：**可租约、可重试、可恢复的异步工作项；Job只保存安全业务ID和进度，不保存敏感连接数据。 + +**当前实现：** + +| 项目 | 当前映射 | +| --- | --- | +| Go模型 | `jobs.Job`（`internal/jobs`） | +| 数据表 | `jobs` | +| 主键 | `jobs.id` | +| 幂等 | `idempotency_key`唯一 | +| 调度 | `priority`、`available_at` | +| 租约 | `lease_owner`、`lease_until` | +| 尝试 | `attempt`、`max_attempts` | +| 时间 | `created_at`、`updated_at`、`started_at`、`completed_at` | + +**当前Job类型示例：** + +```text +source.catalog.refresh +test.upload +test.poll +test.validate +test.remove +release.publish +release.rollback +``` + +**Job规则：** + +```text +Job完成只表示本地处理器完成；对于WeKnora解析，必须继续由test.poll确认远端终态。 +Job payload不允许保存Token、Cookie、签名URL、原始文件内容或生产凭据。 +ReleaseExecution通过job_id外键绑定发布Job。 +``` + +## Release + +**产品含义：**把经过测试验收的知识，以可追溯、可验证、可回滚方式发布到生产。 + +当前Release由批次、执行和执行项组成： + +| 逻辑层 | Go模型 | 数据表 | 主键 | +| --- | --- | --- | --- | +| 发布批次 | `governance.ReleaseBatch` | `release_batches` | `id` | +| 批次条目 | `ReleaseBatchItemView`投影 | `release_batch_items` | `(release_batch_id, publication_id)` | +| 发布执行 | `governance.ReleaseExecution` | `release_executions` | `id` | +| 执行条目 | `ReleaseItemExecutionView`投影 | `release_item_executions` | `id` | + +**Release关系：** + +```text +ReleaseBatch → ReleaseExecution → ReleaseItemExecution +ReleaseExecution → Job（通过job_id） +ReleaseItemExecution → TestPublication（通过publication_id） +Rollback ReleaseExecution → prior publish ReleaseExecution +``` + +**Release规则：** + +```text +ReleaseBatch sealed后内容不可变。 +一个批次最多有一个publish execution。 +一个publish execution最多有一个rollback execution。 +ReleaseItemExecution记录生产前后revision和安全错误码。 +只有独立生产门禁、备份恢复验证和Release Worker均满足时才可以请求生产执行。 +``` + +## TestPublication和验收（辅助对象） + +TestPublication不是原始SourceFile，也不是生产Release；它表示某个SourceItemVersion在指定测试知识库中的处理实例。 + +```text +SourceItemVersion → TestPublication → AcceptanceResult +``` + +| 项目 | 当前映射 | +| --- | --- | +| 测试发布 | `governance.Publication` / API投影`TestPublication` / 表`test_publications` | +| 验收结果 | `governance.AcceptanceResult` / 表`acceptance_results` | +| 身份 | `(source_item_version_id, test_kb_id)`唯一 | +| 处理维度 | ingest_state、processing_state、retrieval_state | +| 版本维度 | parser_generation、index_generation | + +## 当前实现缺口 + +### Current implementation gaps + +以下缺口是已知事实，不能在任何页面、报告或测试中标为“已完成”： + +1. `SourceCatalogNode`尚不能为所有来源绑定`SourceItem`和`SourceItemVersion`。 +2. 目录Snapshot节点尚未持久保存文件内容SHA-256；`snapshot_sha256`不是文件内容Hash。 +3. `SourceCatalogChangeItem`尚未保存旧新内容Hash、旧新范围和SourceItemVersion关联。 +4. ChangeSet确认尚未直接创建Processing Job；当前Processing Confirmation仅绑定TestPublication。 +5. Job尚未通过统一外键绑定ChangeSet、文件版本、目标和动作；该工作属于P3/P5。 +6. `ReleaseChangeSet`和`ProductionSnapshot`尚未实现；当前Release Plan是实时计算，不能冒充不可变发布ChangeSet。 +7. 当前ReleaseExecution没有来源Snapshot、来源ChangeSet、发布ChangeSet和生产Snapshot的完整外键链；该工作属于R1、R2、R6。 +8. Source配置、SourceFile身份和内容Hash的运行时证据仍需后续S2、P1、P2补齐。 + +## C1验收边界 + +C1只完成以下内容： + +```text +冻结当前对象术语和真实映射 +明确身份、时间和关系规则 +标出当前不存在的对象和关联 +用测试防止领域契约文档丢失关键对象 +``` + +C1不完成： + +```text +统一状态机（C2） +OpenAPI（C3） +写操作幂等（C4） +内容Hash闭环（P1） +ChangeSet执行（P3） +发布ChangeSet或Production Snapshot（R1/R2） +``` diff --git a/docs/P1_ACCEPTANCE_TEST_PLAN.md b/docs/P1_ACCEPTANCE_TEST_PLAN.md new file mode 100644 index 0000000..0798159 --- /dev/null +++ b/docs/P1_ACCEPTANCE_TEST_PLAN.md @@ -0,0 +1,114 @@ +# P1 独立验收计划 + +## 1. 范围 + +验证独立治理平台基础骨架：PostgreSQL持久队列、API、普通Worker、默认关闭的Release Worker、React状态页，以及Worker异常中断后的租约恢复。 + +明确不验收：PP/AList/钉钉Connector、测试WeKnora导入、生产WeKnora发布、PG/ES/OSS跨环境复制。 + +## 2. 前置条件 + +- 在`/home/zheromiku/knowledge-governance`执行。 +- Go 1.26、Node.js 22、npm和Docker Compose可用。 +- 不使用生产凭据，不连接测试或生产环境。 +- `secrets/postgres_password`和`secrets/pgpass`使用同一个随机本地密码，权限为600。 + +## 3. 自动化检查 + +```bash +make test +make vet +make build +make web-test +make web-typecheck +make web-build +``` + +通过标准：全部退出码为0。 + +## 4. 启动与健康检查 + +由操作者执行： + +```bash +docker compose up --build -d +docker compose ps +curl -fsS http://127.0.0.1:8088/health +``` + +通过标准：PostgreSQL和API健康；Web可访问；健康响应为`{"status":"ok"}`。 + +## 5. 幂等入队 + +连续两次发送完全相同的请求： + +```bash +for n in 1 2; do + curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \ + -H 'Content-Type: application/json' \ + -d '{"type":"demo","payload":{"round":1},"idempotency_key":"acceptance:idempotent:001"}' +done +``` + +通过标准：两次响应中的任务ID相同；列表中该幂等键只有一个任务；任务最终为`completed`。 + +## 6. Worker租约恢复 + +1. 创建60秒本地任务： + +```bash +curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \ + -H 'Content-Type: application/json' \ + -d '{"type":"demo.sleep","payload":{"sleep_ms":60000},"idempotency_key":"acceptance:lease:001"}' +``` + +2. 在状态页或列表API确认任务进入`running`且`attempt=1`。 +3. 强制终止普通Worker： + +```bash +docker compose kill -s KILL worker +``` + +4. 重新启动Worker： + +```bash +docker compose up -d worker +``` + +5. 等待原租约到期（最多35秒）并持续查询： + +```bash +curl -fsS 'http://127.0.0.1:8088/api/v1/jobs?limit=50' +``` + +通过标准：同一个任务被重新领取，`attempt=2`；继续等待后进入`completed`。不得产生第二条同幂等键任务。 + +## 7. Release Worker默认关闭 + +```bash +docker compose logs --no-color release-worker +``` + +通过标准：日志包含`release worker disabled`；没有`release worker started in probe-only mode`；没有任何测试或生产外部连接配置。 + +## 8. 状态页 + +打开`http://127.0.0.1:8088/`： + +- 页面标题为“知识同步治理平台”。 +- 显示任务总数、运行中、待重试、失败、已完成。 +- 显示最近50条任务及尝试次数。 +- 页面明确显示“Release Worker 默认关闭”。 +- 桌面端无页面级横向溢出；窄屏仅任务表自身可横向滚动。 + +## 9. 停止与保留数据 + +```bash +docker compose down +``` + +通过标准：容器停止，`governance-pgdata`卷保留。未经单独确认不得执行`docker compose down -v`。 + +## 10. 验收记录 + +记录每一步的时间、退出码、任务ID和最终状态。任何步骤失败时停止扩展，不连接WeKnora或来源系统，以失败证据定位P1本身。 diff --git a/docs/P2_PP_INVENTORY_ACCEPTANCE.md b/docs/P2_PP_INVENTORY_ACCEPTANCE.md new file mode 100644 index 0000000..7c49a29 --- /dev/null +++ b/docs/P2_PP_INVENTORY_ACCEPTANCE.md @@ -0,0 +1,84 @@ +# P2.3 PP完整来源清单验收 + +## 交付范围 + +- 将旧manifest中54个超过50 MiB、未进入旧账本终态的来源建模为阻断对象； +- 形成613个PP来源对象的统一治理快照； +- reconciliation以内容SHA revision为依据，同URL内容变化必须计划上传； +- 历史已同步快照与治理数据库一致时，计划上传数必须为0； +- 提供只读分页/状态筛选API和真实知识资产Web页面。 + +不在本阶段执行：来源网络访问、文件下载或上传、测试/生产WeKnora连接、生产发布、AList写操作、钉钉授权、commit或push。 + +## 数据契约 + +```text +source_items=613 +completed=530 +content_alias=29 +over_limit=54 +content_objects=530 +source_item_versions=613 +test_publications=530 +planned_uploads=0 +``` + +54个over_limit版本必须同时满足： + +```text +content_state=over_limit +content_object_id IS NULL +download_error=legacy_50_mib_limit +关联TestPublication数量=0 +``` + +## Reconciliation契约 + +- stable source key相同且SHA revision相同：unchanged； +- stable source key不存在：adopted； +- stable source key相同但SHA revision变化：upload_required； +- over_limit：始终计为阻断，不计划上传； +- 完整真实快照：unchanged=559、over_limit=54、upload_required=0。 + +## 只读API + +```text +GET /api/v1/source-items?state=&limit=50&offset=0 +GET /api/v1/source-items?state=over_limit&limit=100&offset=0 +``` + +`state`只接受`completed`、`content_alias`、`over_limit`或空值；`limit`范围1..100；`offset`范围0..1000000。响应只投影安全治理字段，不返回stable external ID、URI hash、raw_record_summary、Knowledge ID、来源URL、签名URL或凭据。 + +## UI验收 + +“知识资产”页必须读取真实API，并显示： + +- 来源对象613； +- 历史已同步530； +- 内容别名29； +- 超过上限54； +- 状态筛选和服务端分页； +- over_limit项显示“超过旧同步上限”和“阻断”。 + +桌面宽度1440px必须无页面级横向溢出。多标签工作台、左侧导航和刷新恢复行为保持不变。 + +## 数据库升级门禁 + +1. 在唯一临时数据库导入真实613项两次； +2. 首次新增`1/613/530/613/530`； +3. 第二次新增全0； +4. 验证54个阻断版本约束； +5. 当前库升级前创建非空、mode 600且通过`pg_restore --list`的自定义格式备份； +6. 当前库只增量新增54个SourceItem和54个Version； +7. 不重启或删除PostgreSQL卷。 + +## 自动门禁 + +```bash +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1 +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./... +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/... +npm --prefix web test -- --run +npm --prefix web run typecheck +npm --prefix web run build +``` diff --git a/docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md b/docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md new file mode 100644 index 0000000..ee7996f --- /dev/null +++ b/docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md @@ -0,0 +1,99 @@ +# P2.2 PP历史账本离线接管验收 + +## 范围 + +本阶段将旧PP同步器的append-only本地账本接入治理数据库： + +- 不请求PP、AList、钉钉或WeKnora网络API； +- 不修改旧同步项目及其账本、manifest或下载文件； +- 不下载、上传、移动或删除源文件； +- 不启用新PP定时Connector； +- 不将离线历史记录宣称为当前WeKnora已验证状态。 + +## 折叠与恢复契约 + +1. 按旧脚本的`source_url_hash`稳定键折叠； +2. 只接受`completed`和带Knowledge ID的`content_alias`终态； +3. 单次账本只能包含一个KB； +4. 所有终态字段必须完整，source locator和SHA必须是64位小写十六进制； +5. 任何仅有中间状态、没有终态的source key都会阻断整批导入； +6. 缺SHA终态只能从匹配locator hash的manifest恢复； +7. 恢复前验证账本大小、manifest大小、实际文件大小和实际SHA； +8. 超过旧连接器50MiB单文件限制时阻断； +9. alias必须指向同一快照内的completed canonical，并且SHA、Knowledge ID一致； +10. 输出只含聚合数字，不输出KB ID、source ID、Knowledge ID、URL或本地文件路径。 + +## 预期真实账本口径 + +```text +events=1093 +source_items=559 +completed=530 +content_aliases=29 +recovered_sha256=75 +verified_bytes=871524116 +content_objects=530 +versions=559 +publications=530 +``` + +`recovered_sha256=75`必须建立在75个文件逐字节重算全部匹配的前提上。 + +## 数据库映射 + +```text +1 source_connection enabled=false，secret_ref/base_url为空 +559 source_items external + pp_public_tag + unchanged +530 content_objects verified，完整SHA唯一，不保存local_spool_ref +559 versions raw摘要只保留legacy_status、source_ids和alias canonical hash +530 publications uploaded + completed + unpublished +``` + +29个alias只建立自己的SourceItem和Version，共享canonical ContentObject，不重复创建TestPublication。 + +## 幂等和事务 + +- 整批导入在单个PostgreSQL事务内完成； +- 只允许`INSERT ... ON CONFLICT DO NOTHING`； +- 禁止UPDATE和DELETE； +- 第一次真实导入期望新增`1/559/530/559/530`； +- 第二次相同导入期望新增`0/0/0/0/0`； +- 任一步失败必须回滚整批。 + +## 自动验证 + +在项目根目录运行： + +```bash +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1 +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./... +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/... +``` + +真实数据库集成测试必须在唯一临时数据库中设置`TEST_DATABASE_URL`后运行；普通无环境测试显示SKIP不等于集成测试通过。 + +## 本地库读回验收 + +必须验证： + +```text +schema_migrations=1,2 +jobs=2 +source_connections=1 +source_items=559 +content_objects=530 +source_item_versions=559 +test_publications=530 +content_alias versions=29 +unique publication knowledge IDs=530 +forbidden raw fields=0 +``` + +530条历史Publication的`retrieval_state`必须是`unpublished`。只有未来通过目标WeKnora只读API确认对象存在、状态和检索行为后，才能改为`enabled`。 + +## 备份与运行边界 + +- 导入前创建PostgreSQL原生自定义格式备份； +- 备份必须非空、权限600并通过`pg_restore --list`； +- 导入只使用短期容器，不重启Compose五个长期服务； +- 不执行`docker compose down -v`、卷删除、commit、push或外部部署。 diff --git a/docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md b/docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md new file mode 100644 index 0000000..d02ce53 --- /dev/null +++ b/docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md @@ -0,0 +1,86 @@ +# P2.1 来源数据底座验收 + +## 范围 + +本阶段只验证统一来源领域模型和PostgreSQL Migration 2： + +- 来源连接、来源对象、内容对象、来源版本、测试发布五层身份分离； +- 来源内`stable_external_id`唯一； +- 全局内容对象仅按完整64位小写十六进制SHA-256去重； +- 来源变化、内容获取、测试导入、处理和检索状态分开保存； +- 新表初始为空，不读取或导入PP、AList、钉钉数据； +- 不连接测试或生产WeKnora。 + +## 静态与单元验证 + +在项目根目录执行： + +```bash +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1 +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./... +/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/... +``` + +在`web/`执行： + +```bash +npm test -- --run +npm run typecheck +npm run build +``` + +全部命令必须返回exit code 0。 + +## 一次性数据库验收 + +在唯一命名的临时数据库中依次验证： + +1. 执行Migration 1和Migration 2； +2. 再执行一次Migration 2，确认`IF NOT EXISTS`允许安全重跑； +3. 插入一条完整的来源→版本→内容→测试发布链； +4. 同一来源内重复`stable_external_id`必须被拒绝； +5. 不同来源可使用相同`stable_external_id`； +6. 重复SHA-256必须被拒绝； +7. 非64位、小写十六进制之外的SHA-256必须被拒绝； +8. 未声明状态值必须被拒绝； +9. 执行Migration 2 down，五张P2表必须按外键依赖反序删除； +10. P1的`jobs`表必须保留； +11. 删除临时数据库。 + +禁止在当前持久本地数据库中执行down验收。 + +## 当前本地数据库只读验收 + +```bash +docker compose exec -T postgres psql -U governance -d governance -At -c \ + "SELECT string_agg(version::text, ',' ORDER BY version) FROM schema_migrations;" +``` + +期望： + +```text +1,2 +``` + +```bash +docker compose exec -T postgres psql -U governance -d governance -At -c \ + "SELECT COUNT(*) FROM pg_tables WHERE schemaname='public' AND tablename IN ('source_connections','source_items','content_objects','source_item_versions','test_publications');" +``` + +期望：`5`。 + +```bash +docker compose exec -T postgres psql -U governance -d governance -At -c \ + "SELECT (SELECT COUNT(*) FROM source_connections),(SELECT COUNT(*) FROM source_items),(SELECT COUNT(*) FROM content_objects),(SELECT COUNT(*) FROM source_item_versions),(SELECT COUNT(*) FROM test_publications);" +``` + +在正式导入来源前期望：`0|0|0|0|0`。 + +## 数据保留与运行状态 + +- Migration前必须创建非空PostgreSQL原生备份，并通过`pg_restore --list`校验； +- 只替换API容器，PostgreSQL、普通Worker、Release Worker和Web容器ID不得变化； +- Migration后原有`jobs`数量和代表任务状态必须保持； +- `/health`必须返回`{"status":"ok"}`； +- API启动日志不得包含Migration错误； +- 不执行`docker compose down -v`、卷删除、提交、推送或外部部署。 diff --git a/docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md b/docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md new file mode 100644 index 0000000..3f15027 --- /dev/null +++ b/docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md @@ -0,0 +1,164 @@ +# P3 本地 WeKnora 与测试服务器互测验收 + +## 范围 + +本阶段验证知识治理平台从安全 Spool 制品到 WeKnora 的真实编排链路： + +- 本地 WeKnora 上传、轮询、Chunk 与限定检索； +- 上传失败终态、人工重试和持久 Worker 恢复； +- 测试服务器 HTTPS 互测； +- 530 条 PP 历史 Publication 在线只读核验； +- API/UI 安全投影及运行资源； +- 不启用生产 Release Worker，不执行生产发布。 + +## 本地真实 E2E + +使用 4 份明确无敏感内容的 Markdown 合成制品。每份制品均执行： + +```text +spool inspection +→ atomic manual source/content/version/publication/job seed +→ test.upload +→ test.poll +→ test.validate +→ scoped retrieval probe +``` + +验收结果： + +- 4/4 Publication `completed`； +- 每份文档 Chunk 总数为 1，读取 1/1，Knowledge 归属一致； +- 限定 Knowledge 检索均返回目标结果； +- 合成 marker 均在内存中验证命中，报告只保存布尔值； +- `knowledge_exists=passed`； +- `chunks_present=passed`； +- 检索阈值未批准，`retrieval_hit=manual_required`。 + +## 技术失败与人工重试 + +真实故障注入为暂时移走 Spool 制品，不删除文件： + +1. Worker 将 Publication 写为 `ingest_state=failed`、`processing_state=failed`； +2. 只保存固定错误码 `upload_artifact_invalid`； +3. API 安全投影返回 `can_retry=true`； +4. 恢复制品后调用正式 Retry API，返回 `retry_queued`； +5. 同一 Publication 最终完成，`retry_count=1`； +6. 原失败 Job 与重试 Job均为 `completed/attempt=1`。 + +同时兼容历史不一致状态 `ingest=failed, processing=pending`，允许通过正式 Retry API 自愈；普通 pending 或 completed Publication 仍不可重试。 + +## 持久 Worker 恢复 + +在 Job 已持久化后让 Worker 多次启动失败，再恢复正确配置： + +- Job 未丢失； +- Worker 恢复后自动消费； +- upload、poll、validate 均为 `completed/attempt=1`； +- 未重复上传； +- 恢复后的 Worker 为 `running/restarts=0`。 + +## 测试服务器互测 + +测试服务器使用 HTTPS `weknora-test.dobot.cn`，并创建隔离 KB `governance-cross-env-test`。未写入既有对内、对外或历史 E2E KB。 + +互测结果： + +- 合成 Markdown 上传成功； +- remote status `completed`； +- Chunk 1/1，归属一致； +- 限定检索结果 2，目标结果 2； +- `marker_found=true`； +- 最高检索分 `0.7180294855048308`； +- 结构验收通过； +- 语义验收保持 `manual_required`。 + +## MinerU 阻塞与恢复 + +首次互测时，测试 WeKnora 的 default Asynq 队列为： + +```text +active=10 +pending=208 +archived=186 +``` + +测试服务器硬件空闲，但云 MinerU 健康请求超时。GPU 主机日志确认根因为： + +```text +OSError: [Errno 24] Too many open files +``` + +systemd hard limit 为 524288，但进程 soft limit 只有 1024。仅在 MinerU `processing_tasks=0`、`queued_tasks=0` 后执行： + +- 备份 `/etc/systemd/system/mineru.service`； +- 在 `[Service]` 设置 `LimitNOFILE=65536`； +- daemon-reload 并重启 MinerU； +- 验证 soft/hard 均为 65536； +- 验证 port 8000 listener PID 与 systemd MainPID 一致； +- 验证 MinerU v3.4.2 health 为 healthy；修复瞬间回读并发为 8； +- 验证 A10 已加载模型。 + +恢复后测试 WeKnora 的 pending/active 队列归零，隔离互测文档完成。历史 archived 数为 403；本阶段没有删除、重提或改写这些历史失败任务。 + +最终状态复核时，MinerU unit 已由另一个独立操作改为并发 2 并显式重启；本阶段没有覆盖该并发变更。`LimitNOFILESoft=65536` 继续生效，health 保持 healthy。 + +## 530 条历史 Publication 在线只读核验 + +候选查询严格限定为具有 Publication 的 PP 来源，排除所有 manual/future 来源。Dry-run 精确得到 530 条。 + +使用 API Key 文件、HTTPS、并发 4，每 25 个 completed 抽查 1 个 Chunk： + +```text +total=530 +existing=530 +missing=0 +wrong_kb=0 +pending=0 +processing=0 +finalizing=0 +completed=530 +failed=0 +chunk_checked=21 +chunk_empty=0 +http_errors=0 +``` + +核验只调用 GET；未重新上传 530 条历史 Knowledge。 + +## API、UI 与密钥边界 + +- API/Web 只展示安全 Publication 投影； +- 页面未出现 Knowledge ID、KB ID、SHA、Spool 路径、API Key或对象存储地址； +- 测试 WeKnora Key 只以普通 0600 文件提供给普通 Worker； +- API、Web、Release Worker 不挂载测试 Key； +- Spool 在 Worker 中只读； +- `RELEASE_WORKER_ENABLED=false`； +- Firefox 1440×900 验收通过左导航、顶部多标签、任务页、统一来源清单、手工验收标签及无横向溢出。 + +截图：`tmp/P3_LOCAL_E2E_FIREFOX.png` + +## 自动化门禁 + +```text +Go tests: 16 packages passed +Go vet: passed +Go race (testweknora): passed +Web tests: 24/24 passed +TypeScript typecheck: passed +Vite production build: passed +PostgreSQL isolated integration tests: passed +Compose config: passed +``` + +## 数据与发布边界 + +- Migration 1–4 已应用； +- 最终治理数据为 618 个 Source Item、535 个 Content、618 个 Version、535 个 Publication 和 69 个 Job； +- 535/535 Publication completed，69/69 Job completed； +- 5 个 manual E2E 对象对应 15 条验收记录； +- 54 个 `over_limit` 项继续阻断； +- 不删除 WeKnora Knowledge、源文件、PostgreSQL 数据或命名卷； +- 不执行 `docker compose down -v`； +- 未连接生产 WeKnora； +- 未启用生产发布； +- 未 commit、未 push。 diff --git a/docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md b/docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md new file mode 100644 index 0000000..f1db41c --- /dev/null +++ b/docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md @@ -0,0 +1,100 @@ +# P4.1 来源连接只读工作台验收 + +## 范围 + +本切片将“来源管理”从占位页接入真实 PostgreSQL，只提供安全只读视图： + +- 列出已登记来源连接； +- 展示启用状态、是否配置调度、健康枚举、对象数和更新时间； +- 汇总总数、启用、停用、健康、降级、异常和未知； +- 不展示或读取到 API 投影中的连接地址、secret引用和原始健康错误正文； +- 不创建、修改、启用或删除来源连接； +- 不调用 PP、AList 或钉钉写接口。 + +## API + +```text +GET /api/v1/source-connections +``` + +安全字段： + +```text +id +source_type +name +enabled +schedule_configured +health_status +item_count +updated_at +``` + +明确省略： + +```text +base_url +secret_ref +schedule原始JSON +last_health原始JSON及错误正文 +API Key、Token、Cookie +``` + +## 当前真实数据 + +```text +connections=2 +enabled=0 +disabled=2 +healthy=0 +degraded=0 +unhealthy=0 +unknown=2 +``` + +来源对象数： + +```text +manual=5 +pp=613 +``` + +## 自动化验证 + +```text +Store安全投影单元测试：passed +HTTP API安全投影测试：passed +前端严格解析与额外字段剥离：passed +一次性PostgreSQL集成测试：passed (0.121s) +Go全量测试：passed +Go vet：passed +Web测试：26/26 passed +TypeScript typecheck：passed +Vite production build：passed +Compose config：passed +``` + +一次性 PostgreSQL 测试实际插入包含敏感 `base_url`、`secret_ref` 和健康 detail 的连接，再确认序列化投影不包含这些内容。临时数据库在测试后删除，持久治理库没有写入测试记录。 + +## 浏览器验收 + +Firefox headless 1440×900： + +- 左侧导航“来源管理”高亮； +- 顶部保留“工作台/来源管理”多标签； +- 两条真实连接记录可见； +- 状态、调度、健康、对象数易读； +- 来源占位文案已移除； +- 页面敏感字段泄漏为 0； +- 紧凑表格修复后“更新时间”标题及两行时间完整可见； +- 无横向裁切、遮挡或乱码。 + +截图：`tmp/P41_SOURCE_CONNECTIONS_FIREFOX.png` + +## 运行边界 + +- 只重建 API 与 Web； +- PostgreSQL、普通 Worker、Release Worker 未重建； +- Release Worker 继续保持关闭； +- 未连接生产； +- 未 commit、未 push。 diff --git a/docs/P42_SOURCE_SCAN_ACCEPTANCE.md b/docs/P42_SOURCE_SCAN_ACCEPTANCE.md new file mode 100644 index 0000000..efabcc1 --- /dev/null +++ b/docs/P42_SOURCE_SCAN_ACCEPTANCE.md @@ -0,0 +1,250 @@ +# P4.2 来源只读扫描与变化入账验收 + +## 范围 + +本切片将 PP 本地历史快照接入普通 Worker 的持久扫描链路： + +```text +来源管理手工触发 +→ 原子创建 source_scan + source.scan.pp Job +→ 普通 Worker 只读 ledger/manifest/制品元数据 +→ 与 613 条治理 Source Item 对账 +→ 原子保存聚合、变化记录与连接健康 +→ API/Web 展示扫描历史和变化结果 +``` + +本切片不上传、移动、重命名或删除 PP 文件，不调用 PP 网络写接口，不删除治理对象或 WeKnora Knowledge。`missing` 只记录为 `unavailable`，54 个 `over_limit` 项继续阻断，生产 Release Worker 保持关闭。 + +## 安全配置与装配 + +- `PP_SCAN_ENABLED` 默认 `false`；仅显式启用且 allowed root、ledger、manifest 三个路径完整时注册 `source.scan.pp` Handler。 +- allowed root 必须是绝对规范路径；ledger 和 manifest 必须位于其内。 +- Scanner 拒绝越界、软链接、非普通文件、缺少 `local_path`、打开期间文件身份变化和超过字节上限的元数据文件。 +- `over_limit` 记录允许其根目录内的制品不存在，因为它从未被下载；若制品存在仍必须通过普通文件和非软链接校验。 +- Compose 只给普通 Worker 挂载 PP 数据目录，挂载模式为只读；API、Web、Release Worker均没有该挂载。 +- 实际容器探针：ledger/manifest 可读，向 manifest 追加写入被只读文件系统拒绝。 +- 本地测试 WeKnora Key 仍只挂载给普通 Worker；本切片不调用 WeKnora 上传或删除 API。 + +## Migration 5 + +Migration 5 新增： + +```text +source_scans +source_scan_changes +idx_source_scans_one_active +idx_source_scans_connection_created +idx_source_scan_changes_pending +``` + +关键约束： + +- 每个来源连接最多一个 `pending/running` Scan； +- Scan 状态限定为 `pending/running/completed/failed`； +- Change 类型限定为 `added/updated/unavailable`； +- `snapshot_sha256` 只在内部审计，不进入 API； +- down migration 先删除依赖的 changes 表，再删除 scans 表。 + +一次性 PostgreSQL 执行了： + +```text +up to 5 → down to 4 → up to 5 +result: passed +``` + +持久库迁移前逻辑备份： + +```text +tmp/backups/governance-pre-migration5-20260906T011223+0800.dump +bytes=242987 +mode=0600 +pg_restore catalog=valid +``` + +持久库应用后： + +```text +schema_migrations=1,2,3,4,5 +source_items=618 +content_objects=535 +source_item_versions=618 +test_publications=535 +``` + +旧业务数据计数未因 Migration 5 改变。 + +## API + +```text +POST /api/v1/source-connections/{id}/scan +GET /api/v1/source-connections/{id}/scans?limit=20&offset=0 +GET /api/v1/source-connections/{id}/scans/{scan_id} +``` + +触发语义： + +- 成功返回 `202`； +- 同一连接已有活动扫描返回 `409`； +- 不存在或不可扫描的连接返回 `404`； +- 创建 Scan 和 Job 在同一条 PostgreSQL data-modifying CTE 中完成。 + +安全只读投影允许： + +```text +Scan: id/state/聚合计数/固定错误码/创建完成时间 +Change: id/source_item_id(可空)/change_type/observed_size_bytes(可空)/created_at +``` + +明确省略： + +```text +snapshot_sha256 +stable_external_id +previous_revision +observed_revision +allowed root、ledger/manifest/local_path +来源 URL、正文、base_url、secret_ref、Token/API Key/Cookie +``` + +跨连接读取 Scan 详情返回 `404`。前端再次使用 allowlist 重建对象，不透传服务端额外字段。 + +## 真实 PP 只读扫描 + +真实输入只读基线： + +```text +ledger events=1093 +manifest/source items=613 +existing artifacts=559 +historical over_limit without downloaded artifact=54 +``` + +第一次真实扫描按安全规则失败： + +```text +state=failed +error_code=pp_snapshot_invalid +``` + +根因是初版 Scanner 把 54 个历史 `over_limit` 项未下载的制品也要求为已存在普通文件。修复后规则为：`over_limit` 缺失制品是合法阻断状态，但路径仍必须声明且位于 allowed root 内；其他状态缺失仍 fail-closed。 + +修复后的多次真实扫描均一致： + +```text +observed=613 +unchanged=559 +added=0 +updated=0 +over_limit=54 +missing=0 +planned_uploads=0 +changes=0 +``` + +PP ledger、manifest 和来源制品未被修改；没有生成上传计划。 + +## 调度、并发与恢复 + +### 单活动扫描 + +Worker 停止时连续触发同一连接： + +```text +first=202 +second=409 +active_scans=1 +``` + +恢复 Worker 后唯一活动扫描完成。 + +### pending 恢复 + +```text +Worker stopped +→ POST creates pending Scan/Job +→ Worker started +→ Scan completed, Job completed +``` + +### 过期 running 租约重领 + +真实持久队列构造“旧 Worker 已领取但完成前崩溃”的状态： + +```text +before: scan=running job=running attempt=1 lease_expired=true +restart persistent Worker +after: scan=completed job=completed attempt=2 +result: observed=613 unchanged=559 over_limit=54 +``` + +Handler 对 `running` Scan 可恢复；若 Scan 已经 `completed` 而 Job 尚未完成，则重领为幂等 no-op，不重复扫描或写 Change。 + +最终状态： + +```text +source_scans=6 +completed=5 +failed_fixed_code=1 +active_scans=0 +active_source_scan_jobs=0 +``` + +## Web 与 Firefox + +“来源管理”保留左侧导航和顶部标签，并新增： + +- 来源连接的最近扫描聚合； +- “立即扫描”按钮及活动态禁用； +- 最近 20 条扫描历史； +- 单次扫描变化记录安全详情； +- 零变化时明确显示“本次扫描未发现新增、更新或不可用变化”。 + +Firefox headless 1440×1200 验收： + +```text +history_rows=6 +completed=5 +failed=1 +detail_selection=passed +empty_changes_message=visible +safe_projection_leaks=0 +root_overflow=false +table_overflow=false +``` + +截图：`tmp/P42_SOURCE_SCAN_HISTORY_FIREFOX.png` + +视觉复核确认：导航、顶部标签、5 条完成/1 条失败、观察数、上限阻断和变化面板均清晰；无横向裁切、遮挡或乱码。 + +## 自动化验证 + +```text +PP配置安全默认和路径校验：passed +Worker Handler装配测试：passed +File Scanner边界与over_limit回归：passed +Store扫描触发/完成/失败/恢复/安全读取：passed +HTTP触发/历史/详情/404/409安全投影：passed +前端allowlist解析、请求和页面渲染：passed +一次性PostgreSQL真实扫描与变化读取：passed +Migration 5 up/down/up：passed +Web tests：31/31 passed +TypeScript typecheck：passed +Vite production build：passed +Compose config：passed +Compose全镜像build：passed +Go全仓库test：passed +Go vet：passed +Go cmd build：passed +gofmt差异：0 +Firefox真实页面：passed +``` + +## 运行边界 + +- 本地治理平台与本地测试 WeKnora 范围内完成； +- PP 来源只读，未调用来源写/删 API； +- 未删除 PostgreSQL 或 WeKnora 卷； +- 未连接或修改生产环境； +- `RELEASE_WORKER_ENABLED=false`； +- 54 个 `over_limit` 项继续阻断； +- 未 commit、未 push。 diff --git a/docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md b/docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md new file mode 100644 index 0000000..0dfce0c --- /dev/null +++ b/docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md @@ -0,0 +1,320 @@ +# P5 知识工程师审核、封批与发布前门禁验收 + +## 范围 + +P5 在测试发布完成后增加人工控制平面： + +```text +当前 Test Publication + 最新 Acceptance run +→ 固定门禁计算 +→ 知识工程师批准或驳回 +→ 已批准项目创建 draft 批次 +→ 封批时再次验证 Publication / Review / Acceptance +→ sealed（尚未发布） +``` + +P5 不实现生产复制，不创建 `release.*` Job，不启用 Release Worker。 + +## 数据模型 + +Migration 6 新增： + +```text +publication_reviews +release_batches +release_batch_items +``` + +设计约束： + +- `publication_reviews` 是追加式决定，不覆盖历史决定； +- 决定只允许 `approved` / `rejected`； +- 可见性只允许 `internal` / `external`； +- 使用固定 `decision_code`，不保存自由文本或源正文； +- 批准绑定当时最新 `acceptance_run_id` 和 Publication `updated_at`； +- 完全相同的决定通过唯一键幂等； +- `release_batch_items` 锁定 Publication、Review、Source Item Version、Acceptance run、可见性和 Publication 时间戳； +- 同一 Publication 只能进入一个批次； +- 批次状态只允许 `draft` / `sealed` / `cancelled`； +- 策略版本固定为 `release-gate-v1`； +- Migration down 顺序为 item → batch → review； +- 没有历史自动回填，没有把 530 条 PP legacy 项标为“已人工审核”。 + +Migration round-trip 在 disposable PostgreSQL 验证： + +```text +up 6 → down 5 → up 6 +``` + +回滚 6 时，P4 的 `source_scans` / `source_scan_changes` 保持存在。 + +## 审核候选门禁 + +审核队列只纳入至少有一个 Acceptance run 的 Publication。当前持久数据： + +```text +Test Publications 535 +PP legacy（无逐条 Acceptance） 530 +manual 合成验收项 5 +审核候选 5 +``` + +固定 gate code： + +```text +ready +publication_not_ready +source_not_current +acceptance_failed +acceptance_incomplete +``` + +批准要求： + +```text +ingest_state = uploaded +processing_state = completed +retrieval_state = unpublished +当前 Source Item revision = Publication 使用的 Source Item Version revision +source_change 不得为 unavailable / deleted +knowledge_exists = passed +chunks_present = passed +retrieval_hit = passed 或 manual_required +``` + +`retrieval_hit=manual_required` 没有被自动改成 passed；知识工程师点击批准时明确承担人工判断。 + +驳回固定 code： + +```text +needs_correction +classification_changed +acceptance_rejected +``` + +批准固定 code： + +```text +approved_for_release +``` + +## 封批门禁 + +创建 draft 时必须： + +- 请求至少 1 项、最多 100 项； +- Publication ID 不重复； +- 每项均有当前有效的最新批准； +- 批准仍绑定当前 Acceptance run 和 Publication `updated_at`； +- 每项仍满足完整 release gate； +- 任一项目不合格则整条 SQL 不创建批次或部分 item； +- 唯一冲突导致整条语句回滚。 + +封批时再次验证： + +- batch 当前为 `draft`； +- batch 至少有 1 项； +- 每个锁定的 Review、Acceptance run、Publication timestamp、可见性仍匹配； +- 当前 Publication 和 Source revision 仍满足门禁； +- 所有 item 都有效后才原子变为 `sealed`； +- 已 sealed 请求幂等返回现有 sealed 记录； +- 不创建 Job。 + +P6 在真正发布前仍需再次验证封批快照；P5 的 sealed 只表示“人工封批完成”，不表示生产已发布。 + +## API + +```text +GET /api/v1/review-candidates +POST /api/v1/review-candidates/{publication_id}/decision + +GET /api/v1/release-batches +POST /api/v1/release-batches +GET /api/v1/release-batches/{batch_id} +POST /api/v1/release-batches/{batch_id}/seal +``` + +请求体使用 strict JSON decoder： + +- 最大 64 KiB； +- 未知字段拒绝； +- 尾随 JSON 拒绝； +- 非法输入为 400； +- 不满足审核/创建条件为 422； +- 封批快照失效为 409； +- 不回显 Store 原始错误。 + +安全响应不包含： + +```text +knowledge_id +test_kb_id +acceptance_run_id +publication_updated_at +stable_external_id +canonical_uri_hash +local_spool_ref +raw_record_summary +base_url +secret_ref +API Key / Token +``` + +审核 API 的 `reviewer_ref` 当前是本地治理工作台的调用方提供标识，不等同于企业 SSO 强身份。P6 生产启用前仍需将控制面置于经认证/授权的管理员入口；当前 Release Worker关闭，因此该限制不会形成生产发布通路。 + +## Web 工作台 + +“发布管理”由占位页升级为真实 P5 工作台： + +- 左侧导航 + 顶部可关闭标签保持不变； +- 展示审核候选、门禁就绪、待审核和已选数量； +- 每条显示对象、分块、检索验收； +- `manual_required` 显示为“需人工确认”； +- 人工选择 `internal` / `external`； +- 工程师标识为空时批准、驳回和封批按钮禁用； +- 只有已批准且当前 gate ready 的项目可勾选入批； +- 创建 draft 后仍需单独点击“封批”； +- 页面固定显示“生产门禁保持关闭”“Release Worker保持关闭”“尚未发布”； +- 不在浏览器中录入或显示 API Key、数据库密码或 WeKnora凭据。 + +## 测试与真实证据 + +### 自动化测试 + +```text +Go全仓库test passed +Go vet passed +Go cmd build passed +Web tests 35/35 passed +TypeScript typecheck passed +Vite production build passed +Compose config passed +Migration 6 contract passed +Migration 6 up/down/up passed（disposable PostgreSQL） +``` + +### Disposable PostgreSQL Store E2E + +验证： + +```text +ready → approved → draft → sealed +无批准 → create batch 拒绝 +批准后 Publication updated_at 改变 → seal 拒绝 +sealed batch list/detail → 读回 1 item +P5 操作新增 Jobs → 0 +``` + +### Disposable API HTTP E2E + +使用独立数据库、临时 API 端口和无敏感合成记录： + +```text +pending → POST approved +approved → POST draft + draft → POST sealed +GET detail items=1 +safe projection leaks=0 +Jobs=0 +``` + +该测试完成后： + +```text +临时 API stopped +临时数据库 deleted +临时 pgpass deleted +持久库人工决定 0 +持久库批次 0 +持久库批次项目 0 +``` + +### 持久本地环境 + +应用 Migration 6 前备份： + +```text +tmp/backups/governance-pre-migration6-20260906T030213+0800.dump +bytes=252653 +mode=0600 +pg_restore catalog=valid +``` + +应用后： + +```text +schema_migrations 1,2,3,4,5,6 +publication_reviews 0 +release_batches 0 +release_batch_items 0 +source_items 618 +content_objects 535 +source_item_versions 618 +test_publications 535 +acceptance_results 15 +``` + +真实审核队列： + +```text +candidates 5 +pending 5 +gate ready 5 +retrieval manual_required 5 +``` + +未替知识工程师批准任何持久候选。 + +### Firefox + +脚本： + +```text +tmp/p5_review_gate_browser_accept.py +``` + +截图： + +```text +tmp/P5_REVIEW_GATE_TOP_FIREFOX.png +tmp/P5_REVIEW_GATE_FIREFOX.png +``` + +验收： + +```text +review rows 5 +ready / pending 5 / 5 +actor required true +批准/驳回填写actor后可用 true +mutation clicks 0 +batch rows 0 +创建批次按钮 disabled +safe projection leaks 0 +root horizontal overflow false +table horizontal overflow false +``` + +截图视觉检查未发现横向裁切、遮挡、乱码或阻断问题。 + +## 已发现并修复的问题 + +1. PostgreSQL 审核参数同时参与比较与 VARCHAR 插入时类型推断冲突；为参数加显式 cast。 +2. data-modifying CTE 同语句从基表读取新 batch 看不到新 snapshot；改为直接从 `created_batch RETURNING` 返回。 +3. 前端 Page parser 使用 object spread 意外保留 `rawItems`，可能泄漏服务端额外字段；改为显式逐字段构造 Page。 +4. disposable API 的 pgpass 原文件只匹配持久数据库名；使用单次 0600 临时 pgpass 精确匹配 disposable DB，测试后删除，没有改动原密钥文件。 + +## 最终边界 + +```text +业务人工审核决定 0 +业务封批 0 +生产发布 未执行 +生产 WeKnora 未连接 +Release Worker false +release Jobs 0 +PP 源文件 未修改 +卷 保留 +commit / push 未执行 +凭据输出 0 +``` diff --git a/docs/P6_RELEASE_WORKER_ACCEPTANCE.md b/docs/P6_RELEASE_WORKER_ACCEPTANCE.md new file mode 100644 index 0000000..4dcd85d --- /dev/null +++ b/docs/P6_RELEASE_WORKER_ACCEPTANCE.md @@ -0,0 +1,495 @@ +# P6 生产发布 Worker 与回滚验收 + +## 结论 + +P6 的发布控制面、v0.7.1 三通道 Sidecar、Release Worker、归档、补偿和回滚代码已经完成，并通过本地真实依赖 E2E。 + +生产发布仍未授权、未配置、未启动、未执行： + +```text +PRODUCTION_RELEASE_REQUESTS_ENABLED=false +RELEASE_WORKER_ENABLED=false +persistent release_executions=0 +persistent release_item_executions=0 +production release jobs=0 +``` + +因此本文的“通过”表示实现和隔离验收通过，不表示生产已发布。 + +## 范围 + +```text +P5 sealed batch +→ 受保护的生产授权请求 +→ ReleaseExecution + release.publish Job 原子入账 +→ Release Worker 按 KB 路由 +→ 从测试 WeKnora 导出经过验收的同一 Knowledge +→ PostgreSQL / Elasticsearch / 对象存储三通道发布 +→ 生产 WeKnora 官方 API 验收 +→ 成功记录 production revision +``` + +回滚： + +```text +已完成的 publish execution +→ 受保护的 rollback 授权 +→ release.rollback Job +→ 载入旧 0600 release bundle +→ 三通道恢复 +→ 官方 API 验收 +``` + +首次发布没有旧 revision 时，失败补偿和回滚采用禁用而不是删除： + +```text +PostgreSQL Knowledge enable_status=disabled +PostgreSQL Chunks is_enabled=false +Elasticsearch 文档保留但 is_enabled=false +对象保留 +官方 knowledge-scoped Search 必须零命中 +``` + +## 人工授权与双门禁 + +发布请求门禁： + +```text +PRODUCTION_RELEASE_REQUESTS_ENABLED=false +``` + +执行门禁： + +```text +RELEASE_WORKER_ENABLED=false +``` + +两个门禁默认都关闭且职责不同： + +- 请求门禁关闭时，授权 HTTP API 返回 403，Store 不会被调用。 +- 执行门禁关闭时，Release Worker 不加载 release config、不读取生产 secret、不连接生产数据面。 +- P5 的 `sealed` 仍不创建 Job。 +- 只有受保护的生产授权请求才会原子创建 ReleaseExecution 和 Job。 + +授权 HTTP API 不接受调用方提交的审核人身份。审核人来自服务端固定配置： + +```text +PRODUCTION_RELEASE_AUTHORIZER_REF +``` + +门禁开启时还必须提供普通、非软链接、权限 0600 的 Bearer token 文件： + +```text +PRODUCTION_RELEASE_AUTH_TOKEN_FILE +``` + +缺少固定 actor 或 token 文件时 API 拒绝启动。UI 不录入、不读取、不显示 token，并且没有“一键生产发布”按钮。 + +固定确认语句： + +```text +publish-to-production +rollback-production-release +retry-production-release +``` + +## API + +```text +POST /api/v1/release-batches/{batch_id}/authorize-production +POST /api/v1/release-executions/{execution_id}/authorize-rollback +POST /api/v1/release-executions/{execution_id}/retry +GET /api/v1/release-capabilities +GET /api/v1/release-executions +GET /api/v1/release-executions/{execution_id} +``` + +授权响应和只读审计响应不返回： + +- authorization token +- authorization_code +- bundle SHA +- WeKnora API key +- Elasticsearch / PostgreSQL / 对象存储凭据或 URL +- 测试 KB 内部连接信息 + +审计响应显示： + +- publish / rollback +- pending / running / completed / failed +- 固定授权人 +- 总项数、成功数、失败数 +- 每项固定错误码 +- production revision / previous production revision +- 开始和完成时间 + +## Migration 7 + +新增： + +```text +release_executions +release_item_executions +``` + +约束： + +- operation 只允许 `publish` / `rollback`。 +- target_environment 固定为 `production`。 +- state 使用固定枚举。 +- publish execution 每个 sealed batch 唯一。 +- rollback execution 每个 prior publish execution 唯一。 +- Job 外键、逐项 Publication 外键和计数约束保留审计关联。 +- Migration 7 不回填历史 release execution，也不自动创建 Job。 + +往返验收： + +```text +up 7 → down 6 → up 7 +``` + +Migration 7 应用前备份： + +```text +tmp/backups/governance-pre-migration7-20260906T133742+0800.dump +bytes=263001 +mode=0600 +pg_restore catalog=valid +``` + +## Release Worker 状态机 + +批次 Job 在 Worker 中按每批最多50项展开，并持久记录当前批次/总批次；批内逐项执行： + +```text +pending execution +→ running +→ 为 batch items 幂等建立 release_item_executions +→ 每个 item 执行 publish 或 rollback +→ verified / rolled_back / failed +→ execution completed 或 failed +``` + +行为： + +- 一项永久失败会立即停止本项之后的同批项目与全部后续批次；未执行项记录固定`release_stopped_after_failure`，随后仅对已验证成功项安排自动回滚。 +- 可重试故障在重试次数耗尽前保留重试语义，不把尚未终态的项目误标为永久失败。 +- 可重试失败在 Job 还有 attempt 时返回固定 `release_retryable`。 +- 最后一次 attempt 仍失败时，逐项写固定错误码并收口，不留下永远 running 的 Execution。 +- 外部响应正文、对象 key、路径、凭据和错误正文不写入治理库。 +- Worker 重启后跳过已经 verified / rolled_back 的项。 +- 终态failed execution可由同一Bearer门禁和固定`retry-production-release`确认重排；只重置failed items，verified / rolled_back项保持完成，Job从completed/failed回pending并从attempt 0重新开始。 +- publish重排前重新验证P5 sealed snapshot；过期审核或Publication变化时返回冲突。 +- 未知 KB 不会落入默认适配器，而是 `release_contract_mismatch`。 + +## v0.7.1 release bundle 契约 + +固定： + +```text +format weknora-release-bundle/v1 +WeKnora v0.7.1 +migration 74 +retriever elasticsearch_v8 +embedding dimension 1024 +``` + +每个 bundle 绑定： + +- ReleaseExecution ID +- Publication ID +- SourceItemVersion ID +- source revision +- parse revision（parser generation + index generation） +- release revision（封批时锁定的 review ID） +- tenant / KB / Knowledge ID +- embedding model identity +- Elasticsearch mapping SHA-256 +- 各 PostgreSQL 表行数和 canonical SHA-256 +- ES 文档数和 canonical SHA-256 +- 资产数量、总字节和 canonical SHA-256 +- 整包 SHA-256 + +任何 revision、模型、维度、KB、tenant、mapping 或 bundle digest 不一致都在目标写入前 fail-closed。 + +## PostgreSQL 通道 + +只允许以下 v0.7.1 内容表： + +```text +knowledges +chunks +knowledge_tags +knowledge_tag_relations +resources +resource_bindings +``` + +没有读取或复制： + +- users / sessions / messages +- tenants / memberships / permissions +- API keys / model provider credentials +- vector store connection credentials +- Jobs / audit logs / chat history + +导出范围同时绑定： + +```text +tenant_id + knowledge_base_id + knowledge_id +``` + +目标导入： + +- 必须是 migration 74 且 dirty=false。 +- 使用 per-Knowledge PostgreSQL advisory transaction lock。 +- 单 Knowledge 单事务替换。 +- ID 在其他 tenant / KB 已存在时拒绝。 +- Tag 定义和 relation 一起复制；共享 Tag 不删除。 +- Resource stable handle 保留。 +- production storage_backend_id、provider、storage URI 和 location_hash 按目标配置重写。 +- Chunk seq_id 冲突时整个事务失败，不静默改 ID。 +- 不调用 WeKnora 解析和 Embedding。 + +## Elasticsearch 通道 + +源导出： + +- tenant / KB / Knowledge 三重 term filter。 +- 原样保存 `_id`、chunk文本、向量和其他 `_source` 字段。 +- mapping 只计算 `mappings` canonical SHA，不把索引名混入指纹。 +- 导出后按 `_id` canonical排序。 +- 单 Knowledge 超过 10000 个 ES 文档时 fail-closed，不截断。 + +目标替换： + +1. 先 bulk upsert 新文档，`refresh=true`。 +2. 再只删除同 tenant / KB / Knowledge 下、不属于新 release 的旧 `_id`。 +3. 读回数量和 canonical SHA。 +4. 失败时自动恢复 prior SearchSnapshot。 + +检索禁用使用 knowledge-scoped `_update_by_query` 将 `is_enabled=false`，不删除 ES 文档，并读回确认文档仍在且全部禁用。 + +## 对象存储通道 + +支持 S3-compatible MinIO / S3 / OSS： + +- secret 仅从 0600 文件读取。 +- 不在 URL 中携带 token。 +- 支持 v0.7.1 `storage:///`。 +- 拒绝 traversal、query、fragment、控制字符和越界 locator。 +- 导出时核对 resource size / content SHA。 +- staging：`.governance-release-stage//`。 +- staging 写入后做完整流式 SHA-256。 +- Copy 到 final 相同 relative key 后再次完整流式 SHA-256。 +- ObjectStore 接口没有 Delete 方法。 +- 回滚重传旧 bundle 中的资产；不删除新资产。 + +## Release bundle 归档 + +每个 KB 使用独立目录： + +```text +// +``` + +要求： + +- archive root 和 KB 子目录是真实 0700 目录。 +- 不允许目录或文件软链接。 +- bundle 文件为普通 0600 文件。 +- 原子写、文件 fsync、目录 fsync。 +- 相同 release revision + 相同 SHA 可幂等重写。 +- 相同 revision + 不同 SHA 为碰撞并拒绝。 +- 加载时重验 manifest、PG、ES、资产和整包 SHA。 +- 默认每 bundle 最大 1 GiB。 + +Bundle payload 可能包含对象 locator 和资产内容，因此不能进入 Web/API/普通日志；只有摘要进入治理库。 + +## Release Worker 配置 + +模板： + +```text +config/release.v071.example.json +``` + +Runtime 配置文件必须： + +- 使用绝对路径； +- 是普通文件； +- 不是软链接； +- 权限为 0600； +- JSON 不允许未知字段或尾随数据； +- 每个 KB 只允许一个 route； +- source 与 target 的数据库值、ES endpoint、对象 endpoint+bucket 必须不同； +- 所有数据库URL、ES token、对象AK/SK、WeKnora API key 只通过各自 0600 文件引用。 + +模板不能直接启用生产。启用前必须根据真实数据面生成独立受限配置和 Compose override，并由人工授权。 + +## 本地真实 E2E + +### 精确 schema + +使用： + +```text +ParadeDB PostgreSQL 17 +WeKnora v0.7.1 migrations 000000..000074 +source DB + target DB +``` + +结果： + +```text +source migration=74 +目标 migration=74 +合成 source: + knowledges=1 + chunks=1 + knowledge_tags=1 + knowledge_tag_relations=1 + resources=2 + resource_bindings=1 +``` + +通过： + +```text +TestPostgresExporterV071Integration +TestPostgresImporterV071ReplacesAndRestoresDocument +``` + +覆盖： + +```text +old → new → restore old +disable keeps 1 Knowledge + 1 Chunk +Knowledge=disabled +chunk enabled count=0 +``` + +### 对象存储 + +使用本机已有的 disposable MinIO，无卷、loopback随机端口、合成凭据和合成bucket。 + +通过： + +```text +TestS3ObjectStoreIntegration +``` + +覆盖： + +```text +Open storage:// source +Put staging +streaming SHA Stat +Copy final +streaming SHA Stat +``` + +### 三通道发布与回滚 + +通过： + +```text +TestThreeLaneReleaseAndRollbackV071Integration +``` + +链路： + +```text +publish review-release-1 +→ publish review-release-2 +→ rollback review-release-1 +→ target PG chunks=1 +→ target ES has es-release +→ target two assets non-empty +``` + +说明： + +- PostgreSQL 和 MinIO 为真实 disposable 服务。 +- Elasticsearch 为状态化 HTTP contract server，真实执行 mapping/search/bulk/delete-by-query/update-by-query 协议和 digest 回读。 +- WeKnora verifier 使用现有官方 client 接口 contract stub，真实执行 GetKnowledge/ListChunks/SearchKnowledge 的判断逻辑；没有连接生产 WeKnora。 +- 所有 disposable 容器最终均已停止并由 `--rm` 删除。 + +### Worker装配 + +通过： + +```text +TestBuildReleaseRuntimeWithRestrictedFilesIntegration +``` + +验证： + +```text +两个真实 v0.7.1 DB连接 +source/target secret文件0600 +独立ES endpoint +独立对象bucket +独立官方WeKnora endpoint +KB archive 0700 +RoutingPublisher构建 +关闭时两个DB pool均释放 +``` + +## Web验收 + +Firefox Selenium只读验收： + +```text +review_candidates=5 +release_batches=0 +release_executions=0 +production_request_gate=false +unauthorized_release_controls=0 +``` + +截图： + +```text +tmp/P6_RELEASE_AUDIT_FIREFOX.png +``` + +可见： + +- 左侧发布管理导航和顶部标签。 +- P5审核候选与封批记录。 +- P6生产执行审计。 +- 生产请求门禁关闭。 +- Release Worker保持关闭。 +- 当前无生产执行记录。 +- 页面没有生产授权按钮。 + +## 当前持久治理库 + +```text +schema migrations=1,2,3,4,5,6,7 +publication_reviews=0 +release_batches=0 +release_batch_items=0 +release_executions=0 +release_item_executions=0 +release publish/rollback jobs=0 +``` + +现有 5 条候选仍为知识工程师业务待审，未由测试代替人工批准。 + +## 生产前仍需人工完成 + +以下不属于本地自动验收，也未执行： + +1. 明确批准生产发布窗口。 +2. 对测试和生产各选一份同ID真实Knowledge，做只读PG/ES/OSS依赖盘点。 +3. 确认两端均是WeKnora v0.7.1 migration 74、1024维、同模型和同mapping。 +4. 为source创建只读数据面凭据。 +5. 为target创建仅允许约定KB内容表、ES索引和对象bucket的最小写权限凭据。 +6. 创建0600 release config和secret文件，并让Release Worker独占挂载。 +7. 先做1份无敏感canary。 +8. 官方API读取、Chunks、检索、资产和审计全部通过后再扩批。 +9. 保留Release Worker关闭状态，直到上述材料由人工核对。 + +不得因为本文完成而自动执行生产发布、创建生产Job、连接生产数据面、删除任何WeKnora Knowledge或对象。 diff --git a/docs/RELEASE_PLANNER_SIMULATION_ACCEPTANCE.md b/docs/RELEASE_PLANNER_SIMULATION_ACCEPTANCE.md new file mode 100644 index 0000000..e7bd29f --- /dev/null +++ b/docs/RELEASE_PLANNER_SIMULATION_ACCEPTANCE.md @@ -0,0 +1,13 @@ +# 发布准备模拟版验收 + +入口：http://100.92.49.78:8088/ → 发布管理 + +页面将旧8步流程收敛为：选择文档→检查数据闭包→锁定清单→生产负责人授权。 + +当前候选及数量均明确标为本地模拟。规则引擎已验证：切片/向量/文件/Wiki/图谱均可独立验证时推荐文档级；Wiki或图谱共享时强制整知识库且不能跨KB；解析失败、切片缺失、向量数不一致、文件缺失、Wiki/图谱缺失时阻断。每行显示闭包完整、共享需整库或闭包不完整。 + +生产备份到OSS尚未配置，生产授权关闭，Release Worker关闭；只能生成浏览器内模拟清单，不创建治理批次、Job或生产任务。真实适配器目前仅支持Knowledge/Chunk/标签/资源/对象和Elasticsearch文档；Wiki实际导出、Neo4j图谱、整知识库导出和OSS生产备份尚未完成，不能将模拟闭包称为真实可发布。 + +验证：发布规划纯规则测试3项、页面测试2项通过；真实Firefox完成文档级选择、模拟清单、共享图谱升级整知识库、清空和失败项禁选；1440/1024/768无页面级横向溢出；外部写入0；生产能力开关false。 + +证据：`tmp/release-planner-simulation-acceptance.json`、`tmp/RELEASE_PLANNER_SIMULATION_FIREFOX.png`。 diff --git a/docs/SOURCE_CATALOG_ACCEPTANCE.md b/docs/SOURCE_CATALOG_ACCEPTANCE.md new file mode 100644 index 0000000..27b42df --- /dev/null +++ b/docs/SOURCE_CATALOG_ACCEPTANCE.md @@ -0,0 +1,47 @@ +# 真实多层来源目录验收 + +入口：http://100.92.49.78:8088/（Tailscale内） + +## 交付范围 + +- 首页选择来源后默认只显示第一层文件夹；根目录文件收起并显示数量，可主动展开。 +- 点击文件夹只请求当前目录的直接子项，不把全量目录树塞进浏览器。 +- 顶部工作标签显示真实来源/目录路径，可逐级返回。 +- 文件夹优先于文件，显示递归文件数、大小、治理范围、已导入/已解析、处理中和失败数。 +- 当前为本地只读快照，页面明确标注，不冒充实时来源连接。 +- 来源文件名中的控制字符转换为可见符号，目录身份不变；不读取或投影宿主机`local_path`。 + +## 真实来源 + +- 钉钉“公司级技术支持知识库”：4548个节点、1079个实际文件夹、3468份文件；第一层8个目录，另有5份根目录文件默认收起。 +- 钉钉“技术诊断部知识沉淀”：2173个节点、404个实际文件夹、1768份文件。 +- AList“国内工业”：6569个节点、1688个实际文件夹、4880份文件；第一层严格为01公司资料、02产品资料、03行业洞察、04销售赋能、05流程制度、06通用模板。 +- AList同时导入海外工业、国内教育、海外教育三个完整只读快照。 +- PP资料下载中心：以真实`potalTags.tagType`构造8个第一层入口；613份已治理公开文档去重计数。597份进入至少一个标签目录，16份没有这8类标签，作为根目录文件默认收起。一个文件可出现在多个标签目录，但总数只计算一次。 + +总计：7个来源、38986个目录节点、9880个含来源根节点的文件夹节点、28796份跨来源文件。 + +## 数据与安全 + +Migration 9新增不可变目录快照、节点和当前快照指针。导入流程为：构建安全快照→完整校验→单事务插入→切换当前指针。没有删除旧快照，没有修改来源，没有调用钉钉/PP/AList写入、上传、移动、重命名或删除接口。 + +目录API仅投影治理连接ID、节点ID、名称、类型和聚合状态；不返回来源外部ID、来源定位哈希、宿主机路径或快照指纹。 + +## 验证 + +- Python目录构建测试：7/7通过；完整真实快照在独立PostgreSQL库导入，数据库38986节点与构建结果一致。 +- Go全项目测试、相关vet通过；目录API独立PostgreSQL测试验证直接子项、文件夹优先、面包屑、状态聚合及未知节点404。 +- Web：14个测试文件、58项测试通过；TypeScript、Vite构建通过。 +- Firefox真实验收：公司级知识库3层导航、PP三层导航、AList两层导航；首层目录数8/8/6；根目录文件5/16默认收起；DOM与API直接子项完全一致；1440/1024/768无页面级横向溢出；写请求0。 +- 视觉复核：顶部路径无重复、范围标签不拆字、递归文件数口径明确、单页不显示无效分页。 +- Migration=9；原source_items/content_objects/versions/test_publications保持618/535/618/535；活动任务0；范围规则0；文件覆盖0。 +- API、数据库健康；所有服务运行、零重启且运行镜像匹配；本机和Tailscale入口HTTP 200。 +- 生产发布请求仍关闭，Release Worker仍关闭。 + +证据： +- `tmp/source-catalog-acceptance.json` +- `tmp/SOURCE_CATALOG_FIREFOX.png` +- `tmp/backups/pre-source-catalog-20260908T223638.dump`（299195字节，0600，pg_restore目录检查通过） +- 回退镜像：`knowledge-governance-api:before-source-catalog`、`knowledge-governance-web:before-source-catalog` + +未包含：来源实时登录/刷新、钉钉/AList文件到治理`source_items`的全量映射、原文预览。这些属于后续已确认的“来源链接”和解析/导入阶段，不能以目录快照冒充。 diff --git a/docs/SOURCE_CATALOG_LIVE_ROUTING_ACCEPTANCE.md b/docs/SOURCE_CATALOG_LIVE_ROUTING_ACCEPTANCE.md new file mode 100644 index 0000000..33ec98d --- /dev/null +++ b/docs/SOURCE_CATALOG_LIVE_ROUTING_ACCEPTANCE.md @@ -0,0 +1,44 @@ +# 来源目录实时刷新与分流验收 + +## 页面功能 + +- 进入子目录后显示“返回上一级”；顶部来源/目录标签仍可逐级返回。 +- 右下角显示当前来源最近一次快照检查/激活时间。 +- 每个文件显示不选择/对内/对外；有安全本地制品的文件可单条标注并立即进入对应本地WeKnora解析队列。 +- 当前页支持勾选、全选可入队文件、批量标注对内/对外。零选择按钮显式灰色禁用。 +- 目录元数据存在但没有可验证文件制品时，分流控件禁用并显示“只有目录元数据，尚无可上传文件”；不会假装入队。 +- 分流结果显示标注数、新入队、已在队列及暂不可入队数量。 + +## 本地WeKnora目标 + +新增并读回验证两个隔离本地知识库： +- `governance-internal-local`：对内解析队列。 +- `governance-external-local`：对外解析队列。 + +API只返回两个范围是否启用，不把目标知识库ID交给浏览器。文件路由在单个PostgreSQL事务中写人工范围覆盖和审计、创建目标Publication及`test.upload` Job。重复标注同一文件/目标返回already_queued，不重复上传。 + +从PP公开Manifest选择最小的626字节TXT，按Manifest字节数和SHA-256复核后复制到治理Spool，并补齐一个content_object的受控spool引用。该文件分别通过对外和对内队列真实上传到本地WeKnora，两个Knowledge均达到`parse_status=completed`。测试对象保留，未调用删除API。 + +## 实时快照 + +点击“实时刷新目录”创建持久`source.catalog.refresh` Job。普通Worker运行只读扫描器，Job payload只保存来源治理ID、类型、名称、阶段、进度和计数，不保存URL、Token、Cookie或捕获请求。网页轮询Job，可以切换页面；完成后自动读取当前快照。 + +阶段：等待后台刷新→验证登录→扫描目录→校验父子关系和数量→写不可变快照/切换当前指针→完成。失败不会切换当前快照。 + +真实验收： +- 钉钉公司级技术支持知识库：浏览器捕获会话有效；扫描1080个文件夹；新快照4524节点、1079目录、3444文件；原快照3468文件，证明读取的是实时来源变化；新快照原子激活，旧快照保留。 +- PP：当前凭据文件可读取且网络可达，但登录返回无Token；网页显示“PP登录失败，请在来源链接更新凭据；旧快照仍然保留”。 +- AList直连：保留各来源原地址，仅向`POST /api/fs/list`添加`x-api-token`；拒绝所有重定向，不依赖钉钉SSO Cookie。2026-09-10 实测四个来源均完成只读全量快照并原子激活：国内工业1718目录/4942文件、海外工业1486目录/4463文件、国内教育3566目录/9067文件、海外教育1742目录/5015文件。 + +Worker只读挂载源脚本和来源目录；DingTalk/PP/AList凭据副本位于治理`secrets/`、权限0600，仅普通Worker挂载，API/Web/Release Worker不挂载。AList直连令牌从`secrets/alist_api_token`读取，不进入来源地址表、任务载荷、数据库或日志。 + +## 验证证据 + +- Python测试12项、Web 77项、Go全项目测试和相关vet通过。 +- 独立PostgreSQL验证：文件标注、人工范围审计、目标Publication、Job入队、重复幂等和未映射阻断。 +- 浏览器：返回、快照时间、实时任务完成卡、单条选择、当前页批量、幂等结果、灰态、1440/1024/768无页面级横向溢出。 +- `tmp/catalog-routing-refresh-acceptance.json` +- `tmp/CATALOG_ROUTING_REFRESH_FIREFOX.png` +- `tmp/backups/pre-live-refresh-routing-20260909T170939.dump`（3607159字节、0600、pg_restore目录检查通过） + +生产发布请求和Release Worker保持关闭；未连接或写入生产WeKnora、生产数据库或OSS。 diff --git a/docs/SOURCE_LINK_SETTINGS_SIMULATION_ACCEPTANCE.md b/docs/SOURCE_LINK_SETTINGS_SIMULATION_ACCEPTANCE.md new file mode 100644 index 0000000..ce1802a --- /dev/null +++ b/docs/SOURCE_LINK_SETTINGS_SIMULATION_ACCEPTANCE.md @@ -0,0 +1,28 @@ +# 来源链接与系统设置模拟版验收 + +入口：http://100.92.49.78:8088/ + +## 来源链接 + +使用真实7个来源名称、类型、文件/目录快照数量。钉钉展示专用浏览器扫码登录流程，PP展示公开访问，AList展示只读Token；AList站点地址与起始目录分离。模拟验证包含网络、认证、根目录、第一层目录、只读边界五项。 + +当前本地模式不连接登录服务、不保存或接收真实Key；Key输入禁用。页面明确不下载文件内容，只读取目录和元数据，并禁止上传、移动、重命名、删除。视觉复核确认模拟状态不冒充实时连接。 + +## 系统设置 + +六区：当前环境、WeKnora环境、解析器与模型、发布目标、OSS备份、监控与安全。文档工程师仅能在浏览器内调整模拟刷新/停滞参数；管理员端点、凭据、映射、OSS和生产开关全部锁定。测试/生产WeKnora分开；生产写入、生产发布请求均关闭；未接入的Provider明确标待配置或未接入。 + +## 验证 + +- 当前全Web：20个测试文件、72项测试通过；TypeScript和Vite构建通过。 +- 来源链接Firefox：7个来源、钉钉/PP/AList三类流程、Key禁用、五项模拟验证、1440/1024/768无页面级横向溢出；外部写入0。 +- 系统设置Firefox：六区、四项安全模拟设置、管理员按钮禁用、密码输入0、生产开关false、1440/1024/768无页面级横向溢出；外部写入0。 +- 视觉复核：两页均为文档/内容治理页面，不是开发控制台；无严重遮挡或横向溢出。 + +证据： +- `tmp/source-link-simulation-acceptance.json` +- `tmp/SOURCE_LINK_SIMULATION_FIREFOX.png` +- `tmp/system-settings-simulation-acceptance.json` +- `tmp/SYSTEM_SETTINGS_SIMULATION_FIREFOX.png` + +真实模式待部署测试服务器后实施：管理员认证、HTTPS、CSRF、Secret单向替换、钉钉扫码会话、AList真实只读Token验证、PP实时目录、WeKnora/模型/MinerU真实Provider、生产目标及OSS备份恢复验证。当前模拟页不能作为真实连接或生产可用证据。 diff --git a/docs/SOURCE_SCOPE_POLICY.md b/docs/SOURCE_SCOPE_POLICY.md new file mode 100644 index 0000000..3609e4f --- /dev/null +++ b/docs/SOURCE_SCOPE_POLICY.md @@ -0,0 +1,38 @@ +# 来源目录批量标注口径（S6 冻结） + +本文件冻结“来源与目录”页批量标注（处理策略）的产品口径。后续任何页面、API 或测试改动与本文件冲突时，必须先修改本文件并经主人确认，不能悄悄改变行为。 + +## 冻结规则 + +```text +1. 文件和目录均可勾选。 +2. 批量选择仅提供“全选当前页”；不提供“递归全选当前目录”。 +3. 目录被单独勾选并标注后，策略递归作用于全部后代文件。 +4. 顶部批量策略按钮仅提供三个：对内、对外、禁止导入。 +5. 历史“对内 + 对外”（both）数据只读展示为“对内 + 对外”， + 顶部不提供该批量入口（P3/后续如要开放须先修订本文件）。 +6. “未选择/未设置”仅在状态列显示，不生成任何专属批量按钮； + 批量按钮只在存在勾选节点时可用。 +``` + +## 语义边界 + +- “标注”只写治理策略（`directory_scope_rules` / `source_item_scope_overrides`）与审计事件， + 不代表已上传、解析或发布。 +- 目录标注写入目录级规则并递归生效；文件级覆盖（override）优先于最近的目录规则， + 目录规则优先于来源默认可见性。 +- “禁止导入”表示从测试知识库移除（retrieval 进入 disabling）， + 不删除来源文件，也不操作生产知识库。 +- 每次用户动作使用一个新的 `Idempotency-Key`；网络重试复用原键（见 OpenAPI C4 契约）。 + +## 验收口径 + +1. UI：`catalog-policy-actions` 区域内恰好 3 个策略按钮（对内/对外/禁止导入）； + 页面不出现“递归全选当前目录”入口；不出现可选择的“对内 + 对外”批量控件。 +2. Store 集成：标注一级目录后，其二级目录子项与深层文件的 effective visibility + 均递归变为该策略；进入子目录、返回父目录、刷新读回结果一致。 +3. 历史 both 行在文件表中仍显示“对内 + 对外”，只读。 + +对应测试： +- `web/src/SourceCatalogBrowser.test.tsx` +- `internal/platform/postgres/s6_scope_policy_integration_test.go` diff --git a/docs/STATE_MACHINE.md b/docs/STATE_MACHINE.md new file mode 100644 index 0000000..23ca5f6 --- /dev/null +++ b/docs/STATE_MACHINE.md @@ -0,0 +1,289 @@ +# 知识治理平台状态机与术语契约 + +## 目的 + +本文件冻结治理平台的状态维度、存储枚举、API枚举、页面中文和终态含义。一个对象不得用单一`status`混合来源变化、人工审核、上传、解析、检索和发布状态。 + +原则： + +```text +数据库/API使用稳定英文枚举。 +页面使用中文业务文案。 +completed是当前统一的“成功完成”内部终态；页面可显示“成功”或“已完成”。 +unknown、空字符串、未配置不能伪装成healthy或completed。 +未来状态先写在“保留值”，未对应迁移和实现前不得持久化。 +``` + +## State dimensions + +| 维度 | 负责回答的问题 | 当前主要对象 | +| --- | --- | --- | +| Scope policy | 资料希望进入哪里？ | 目录规则、文件覆盖规则 | +| Source change | 来源事实发生了什么？ | SourceItem、CatalogChangeItem | +| Review | 人工是否批准？ | PublicationReview | +| Content | 原始内容是否取得和验证？ | ContentObject、SourceItemVersion | +| Job | 后台执行器是否仍在处理？ | jobs | +| Ingest | 是否已提交到测试目标？ | TestPublication | +| Processing | WeKnora解析/索引是否完成？ | TestPublication | +| Retrieval | 是否允许被检索？ | TestPublication/目标系统 | +| Acceptance | 测试结果是否通过？ | AcceptanceResult | +| Release | 生产发布批次/执行进行到哪里？ | ReleaseBatch、ReleaseExecution | +| Settings | 设置版本能否使用？ | UISettingsRevision | + +### Job + +Current storage/API values: + +```text +pending | running | retry | completed | failed | cancelled +``` + +| 值 | 页面中文 | 含义 | 是否终态 | +| --- | --- | --- | --- | +| `pending` | 等待执行 | 已创建，等待Worker领取 | 否 | +| `running` | 执行中 | 已被Worker持有有效租约 | 否 | +| `retry` | 等待重试 | 上次失败但仍可按退避重新执行 | 否 | +| `completed` | 已完成 | 本地Job处理器完成；不自动等于目标系统最终成功 | 是 | +| `failed` | 执行失败 | 达到最大尝试次数或遇到不可重试错误 | 是 | +| `cancelled` | 已取消 | 被明确取消，不得再被Worker领取 | 是 | + +规则： + +```text +Worker只领取pending、retry或过期running任务。 +cancelled任务不可以被重新领取。 +completed仅代表该Job完成；test.upload完成后仍必须由test.poll确认Processing终态。 +``` + +### Scope policy + +Current user-facing desired-policy vocabulary: + +```text +unset | internal | external | blocked +``` + +| 值 | 页面中文 | 当前持久化解释 | +| --- | --- | --- | +| `unset` | 未设置 | 没有显式目录/文件覆盖规则；当前有效范围由来源默认或历史数据决定。它不是可写的`Visibility`数据库值。 | +| `internal` | 对内 | 期望进入对内测试知识库。 | +| `external` | 对外 | 期望进入对外测试知识库，后续仍需人工审核。 | +| `blocked` | 禁止导入 | 期望不再导入；不得因此自动删除生产知识。 | + +historical compatibility value: both + +`both`是历史兼容持久化值，页面只读显示“对内 + 对外”。目录页顶部不提供新增`both`操作。`mixed`和`unknown`是目录汇总显示值，不是文件期望策略。 + +### Source change + +| 存储/API值 | 页面中文 | 含义 | +| --- | --- | --- | +| `unchanged` | 无变化 | 与基线相比无来源差异。 | +| `added` | 新增 | 来源新发现。 | +| `updated` | 已更新 | 现有来源身份出现新修订或新内容证据。 | +| `deleted` | 已删除 | 仅限权威来源明确删除。 | +| `unavailable` | 来源不可用 | 权限、接口或扫描失败；绝不是删除信号。 | + +Catalog ChangeSet使用不同的动作值：`add`、`update`、`move_scope`、`delete`、`no_change`。它是比较计划动作，不直接等同于`SourceItem.source_change`。 + +### Review + +```text +pending | approved | rejected | not_required +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `pending` | 待审核 | 必须等待文档工程师判断。 | +| `approved` | 已批准 | 可以进入后续封批条件判断，不代表生产已发布。 | +| `rejected` | 已驳回 | 不得自动重新入批。 | +| `not_required` | 无需审核 | 仅适用于明确规则允许的情形。 | + +### Content + +```text +unknown | downloading | verified | invalid | over_limit | failed +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `unknown` | 内容待获取 | 尚未取得可验证字节。 | +| `downloading` | 正在获取 | 正在受控下载/导出。 | +| `verified` | 内容已校验 | 大小、格式、内容SHA-256已验证。 | +| `invalid` | 内容无效 | 格式或文件魔数不符。 | +| `over_limit` | 超出限制 | 内容存在但超出当前导入策略。 | +| `failed` | 获取失败 | 内容获取或验证失败。 | + +### Ingest + +```text +pending | uploading | uploaded | failed +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `pending` | 等待上传 | 尚未提交目标。 | +| `uploading` | 上传中 | 已开始向测试目标提交。 | +| `uploaded` | 已上传 | 目标已接收对象；不代表解析完成。 | +| `failed` | 上传失败 | 上传步骤失败。 | + +### Processing + +```text +pending | processing | finalizing | completed | failed | stalled +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `pending` | 等待解析 | 目标尚未开始处理。 | +| `processing` | 解析中 | 正在解析、切片或向量化。 | +| `finalizing` | 索引与增强中 | 基础解析可能完成，索引/Wiki/图谱仍未完成。 | +| `completed` | 解析完成 | 远端最终处理完成；仍需独立Acceptance。 | +| `failed` | 解析失败 | 远端报告失败。 | +| `stalled` | 处理停滞 | 超过配置阈值未见进展。 | + +### Retrieval + +```text +unpublished | enabled | disabling | disabled | enabling | failed +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `unpublished` | 未发布 | 尚未允许检索。 | +| `enabled` | 已启用检索 | 已读回确认允许检索。 | +| `disabling` | 正在停用检索 | 请求已发出，尚未确认。 | +| `disabled` | 已停止检索 | 已读回确认停止检索，可恢复。 | +| `enabling` | 正在恢复检索 | 请求已发出，尚未确认。 | +| `failed` | 检索状态异常 | 停用/恢复失败或读回失败。 | + +### Acceptance + +```text +pending | passed | failed | manual_required +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `pending` | 等待验收 | 尚未检查。 | +| `passed` | 验收通过 | 指定检查通过。 | +| `failed` | 验收失败 | 指定检查失败。 | +| `manual_required` | 需人工确认 | 自动检查不足以得出结论。 | + +### Release + +当前批次状态： + +```text +draft | sealed | cancelled +``` + +当前执行状态： + +```text +pending | running | completed | failed +``` + +当前执行条目状态： + +```text +pending | staged | activated | verified | failed | rolled_back +``` + +规则： + +```text +sealed只表示发布清单冻结，不表示已发布。 +ReleaseExecution.completed必须建立在逐项验证成功之上。 +ReleaseExecution.failed不自动表示整体生产已恢复；需按R4回滚策略判断。 +``` + +### Settings + +```text +validated | active | retired | failed +``` + +| 值 | 页面中文 | 含义 | +| --- | --- | --- | +| `validated` | 已验证待启用 | 配置校验通过，尚未成为运行版本。 | +| `active` | 当前活动版本 | 运行时读取的唯一设置版本。 | +| `retired` | 已退役 | 被后续活动版本替代，保留审计。 | +| `failed` | 验证失败 | 配置未通过，不得启用。 | + +## Transition rules + +### Job transitions + +```text +pending → running +running → completed | retry | failed | cancelled +retry → running | cancelled +pending → cancelled +``` + +`completed`、`failed`、`cancelled`均为终态。后续“重试”必须创建或重置受控Attempt，不能把已完成任务静默改回pending。 + +### Scope policy transitions + +```text +unset → internal | external | blocked +internal | external | blocked → internal | external | blocked +both只能保留和读取，不由当前目录页新增。 +``` + +Scope改变只更新期望状态；S7完成前不得把它视为立即上传或删除命令。 + +### Processing transitions + +```text +pending → processing → finalizing → completed +pending | processing | finalizing → failed | stalled +stalled → processing（仅由确认重试后的远端进展恢复） +``` + +### Release transitions + +当前实现只允许： + +```text +pending → running → completed | failed +``` + +## Reserved future values + +以下值已在产品术语中保留，但当前数据库、API和UI不得写入： + +```text +rolling_back +rolled_back +rollback_failed +snapshotting +verifying +applying +``` + +`rolling_back` must not be persisted before R4。 + +R1、R2、R4完成后，Release状态机会扩展这些值，并新增Production Snapshot与Release ChangeSet关联。 + +## C2验收边界 + +C2完成： + +```text +分维度状态术语冻结 +completed与页面“成功”含义统一 +Job cancelled枚举、数据库约束和终态语义补齐 +历史both策略的兼容边界明确 +未来发布状态不提前持久化 +``` + +C2不完成： + +```text +取消Job的用户界面/API（后续按业务需求另立工作包） +SourceFile内容Hash、ChangeSet确认、生产快照和三级回滚 +Scope策略的实际执行链（S7/P3） +Release rolling_back实际状态（R4） +``` diff --git a/docs/WEKNORA_MONITOR_SIMULATION_ACCEPTANCE.md b/docs/WEKNORA_MONITOR_SIMULATION_ACCEPTANCE.md new file mode 100644 index 0000000..402f041 --- /dev/null +++ b/docs/WEKNORA_MONITOR_SIMULATION_ACCEPTANCE.md @@ -0,0 +1,29 @@ +# 本地模拟WeKnora监控验收 + +入口：http://100.92.49.78:8088/ → 解析监控 + +当前模式严格标记为`simulation`。页面黄色提示、资源卡和文件名均显示模拟属性，不代表测试服务器真实状态。 + +## 模拟内容 + +- CPU 42%；内存18.6/32.0GB。 +- WeKnora API、Embedding模型、LLM模型、MinerU连通卡；均为模拟探测及模拟延迟。 +- 公司级技术支持知识库：已导入5750；completed 4530、pending 450、finalizing 714、processing 50、failed 6，状态合计5750。 +- 对内已解析/已导入3300/4200；对外1230/1550。 +- 五种状态各有一份明确以`[模拟]`开头的文件样例，可按状态筛选。 +- finalizing明确表示基础解析/切片/向量通常完成，Wiki/摘要/图谱仍在生成，不能算全部完成。 + +## 验证 + +- Go simulation provider与HTTP安全投影测试通过；响应不包含URL、Key、Authorization或Secret。 +- Web监控客户端和页面测试通过；当前全Web为16个测试文件、62项测试通过；TypeScript和Vite构建通过。 +- Firefox验证：模拟标识、资源卡、状态合计、对内外计数、finalizing与failed筛选、文件名和错误码均通过；写请求0；1440/1024/768无页面级横向溢出。 +- 视觉复核无严重遮挡或横向溢出。 + +证据： +- `tmp/weknora-monitor-simulation-acceptance.json` +- `tmp/WEKNORA_MONITOR_SIMULATION_FIREFOX.png` + +## 后续真实模式边界 + +治理平台部署到测试服务器后，用同一MonitorProvider接口替换模拟Provider：本机资源读取真实CPU/内存，WeKnora使用认证API汇总Knowledge parse_status，模型使用有时间戳的低频实际探测，MinerU同时检查服务自身和WeKnora到MinerU的路径。切换前模拟数据不能称为真实运行状态。 diff --git a/docs/previews/monitor-a-compact-table.html b/docs/previews/monitor-a-compact-table.html new file mode 100644 index 0000000..37798db --- /dev/null +++ b/docs/previews/monitor-a-compact-table.html @@ -0,0 +1,159 @@ + + + + + +解析监控 · 方向A 紧凑表格版 [预览稿·模拟数据] + + + +
+

解析监控

+ 快照 2026-09-12 08:00 · 下次自动 08:05 + 活跃 Worker 2 · 等待 3 · 运行 4 · 失败 2 + + 预览稿 · 模拟数据 · 方向A +
+
+ +
+

快照差异 → 解析队列 → 死信 → 资源（单页主线，表格为主）

+
+ 待确认 3处理中 4 + 成功 4530失败 2 +
+ +
+
+ +
+

1 · 快照差异（待确认置顶）

国内工业 · from 09-08 → to 09-12操作人
+ + + + + + + + + +
变更文件路径（旧 → 新）目标证据操作
新增CR20B用户手册V2.pdf— → 01｜产品资料/CR20B/对内内容Hash 8e3f…（来源提供）
更新DobotStudio固件说明.docx同一文件，版本 r12 → r13对内对外内容Hash变化（真更新）
范围变化越疆售后FAQ.xlsx对内 → 对内+对外对外Hash未变，仅标注
—移动安装指南.pdf01｜产品资料/ → 快速入门/安装指南.pdf对内同名同Hash → 自动改名，不重解析已自动处理
—删除旧手册.pdf02｜已下架资料/旧手册.pdf →—来源缺失等待移除确认
+
+ +
+

2 · 解析队列（失败/停滞置顶）

共 4536 · 1小时：成功 88.9% · p95 30分钟
+ + + + + + + + + +
状态阶段链文件目标错误码已停留重试
失败✓✓✓✗···· 文档解析CR20A.pdf对外validation_failed—1
失败✓✓✓✓✓✗· 索引增强MG400碰撞检测笔记.pdf对内remote_permanent—5/5 耗尽 · 死信
—解析中✓✓✓✗···· 文档解析XVision标定手册.pdf对内—6分钟0
—等待等待接单NeoSs常见问题.docx对外—2分钟0
—成功✓✓✓✓✓✓✓ 完成CR5说明书V3.pdf对内—完成2
+
+ +
+

3 · 失败队列 DLQ（2条开放死信）

仅投影；原始任务与审计不可变。重试=新尝试
+ + + + + + +
错误码类型文件尝试进入死信
remote_permanent上传解析MG400碰撞检测笔记.pdf5/509-11 21:04
artifact_invalid上传解析测试固件包.zip.bin1/109-12 02:11文件校验永久失败，需来源修复
+
+ +
+
10s 100%
+
1小时 · 成功 88.9% 失败 11.1%
+
24小时 · 成功 96.4%
+
耗时 p50 30秒 · p95 30分钟 · 均值 6.9分钟
+
CPU 42% 内存 18.6/32GB
+
WeKnora 可达
+
MinerU 容量已满(429)
+
PostgreSQL 可接单
+
+
+ +
+ +

CR20A.pdf

+

对内+对外 · kb-external · 版本 r7 · 发布 tp-8841

+
validation_failed：解析服务返回 422，切片数量=0。建议：确认 MinerU 接单容量后批量重试。
+
    +
  • 下载完成
    • +
  • 内容Hash校验 8e3f…d41a
    • +
  • 上传 WeKnora
    • +
  • 文档解析
    • +
  • 索引与增强（未执行）
    • +
+
+
内容Hash
8e3f19…d41a（来源提供）
+
文件身份
来源稳定ID ext-9931 · 置信度 official_id
+
最近尝试
job-3f9a… · 第2次 · 164秒
+
死信
否（仍可重试）
+
+

+
+ + + + diff --git a/docs/previews/monitor-b-task-inbox.html b/docs/previews/monitor-b-task-inbox.html new file mode 100644 index 0000000..9ee7ff1 --- /dev/null +++ b/docs/previews/monitor-b-task-inbox.html @@ -0,0 +1,148 @@ + + + + + +解析监控 · 方向B 任务收件箱版 [预览稿·模拟数据] + + + +
+

解析监控 · 收件箱

+ 快照 2026-09-12 08:00 + 活跃 Worker 2 · 队列 等待3/运行4/失败2 + + 预览稿 · 模拟数据 · 方向B +
+
+ + +
+
+

1 个任务需要你出手

操作人
+ +
+ +
+

CR20A.pdf 对外 更新

+
为什么失败：文档解析返回 422（切片数量=0）· 错误码 validation_failed · 第2次尝试 · 停留164秒。同类文件今天已成功 12 个，可能是该文件本身问题。
+
下载 ✓→Hash ✓→OSS ✓→文档解析 ✗→上传→索引→完成
+
建议：先在 WeKnora 手动解析该文件确认格式，再决定是否重试。
+
+
+
+ +
+ +
+

MG400碰撞检测笔记.pdf 对内 新增

+
为什么失败：索引增强被远端永久拒绝 remote_permanent（5/5 次尝试耗尽，已进入死信）。原因多为 MinerU 容量持续 429。
+
下载 ✓→上传 ✓→解析 ✓→索引 ✗ 429×5
+
资源状态：MinerU 当前容量已满 —— 恢复接单后此任务可一键重新入队。
+
+
+
+
+ +
+

等待我确认的差异（1）

国内工业 09-08 → 09-12 · 共 5 项变化，3 项自动可处理
+
+ +
+

越疆售后FAQ.xlsx 对内→对外 范围变化

+
内容Hash未变，仅标注范围升级；确认后进入发布流程。历史审核：09-02 姚工 标记对内。
+
+
+
+
+ +
+

安装指南.pdf 移动（改名/换目录）

+
同名同Hash证据充分 → 自动更新路径，不重解析。无需操作。
+
+
已自动处理
+
+
+ +
+

旧手册.pdf 删除

+
来源侧消失；将随本批差异确认后从测试知识库移除（不删来源文件）。
+
+
随批次确认
+
+
+ +
+

处理中（4）

无需操作，仅供查看
+

XVision标定手册.pdf

下载 ✓→上传 ✓→文档解析中 6分钟
+
其余 3 个任务正常排队/处理中…
+
+ +
+
10s 100%
+
1小时·成功 88.9%
+
24小时·成功 96.4%
+
p50 30秒 · p95 30分钟
+
CPU 42% · 内存 18.6/32GB
+
WeKnora 可达
+
MinerU 429 容量满
+
PostgreSQL 可接单
+
+
+
+ + diff --git a/docs/previews/refresh-2026-09-12/a-data-console.html b/docs/previews/refresh-2026-09-12/a-data-console.html new file mode 100644 index 0000000..874ed9d --- /dev/null +++ b/docs/previews/refresh-2026-09-12/a-data-console.html @@ -0,0 +1,165 @@ + + + + +方向A · Data-console（解析监控） + + + + +
+ +
+
+
解析监控 / 需人工处理
+
1h成功 88.9%24h成功 96.4%Worker 2队列 待 3 / 跑 4
+
+
+

需人工处理

+

失败原因、处理链定位与建议直接放在卡片上。勾选后可批量重试，死信重新入队会保留原任务与全部审计行。

+ +
+
失败
2
+2 · 今日
+
停滞
1
45分钟
+
处理中
4
—
+
已完成
48
+6 · 今日
+
死信
1
可重入队
+
+ +

失败任务

3 条 · 按失败阶段排序
+ +
+
+
+
+

CR20A 电气规格说明书.pdf 对外 更新

+
错误码 remote_parse_failed · 第 2 次尝试 · 已停留 71.1小时 · 失败于 文档解析
+
下载 ✓Hash ✓OSS ✓文档解析 ✗WeKnora 上传索引完成
+
建议先在 WeKnora 手动解析该文件确认格式，再决定是否重试。 cs-1 · tp-failed
+
+
+
+
+
+
+
+
+

伺服驱动器调试手册.docx 对内 新增

+
错误码 mineru_timeout · 第 1 次尝试 · 已停留 45分钟 · 失败于 索引 / 增强
+
下载 ✓Hash ✓OSS ✓文档解析 ✓WeKnora 上传 ✓索引 ✗完成
+
多为 MinerU 容量或超时问题，确认服务恢复接单后重试。 cs-3 · tp-044
+
+
+
+
+ +

死信（尝试耗尽）

1 条 · 不可自动重试
+
    +
  • 对外知识库 · test.upload release_manifest_mismatch
    cs-2 · CR1A 固件变更记录.pdf · 尝试 5/5 · 2026-09-10 14:22
    • +
+ +
+
MinerU 可接单
+
WeKnora 测试 可接单
+
PostgreSQL 12ms
+
队列 3 待 / 4 跑 / 0 重试
+
CPU 42% · 内存 18.6/32.0GB
+
+
+
+
+ + diff --git a/docs/previews/refresh-2026-09-12/b-editorial-ledger.html b/docs/previews/refresh-2026-09-12/b-editorial-ledger.html new file mode 100644 index 0000000..dbcc0de --- /dev/null +++ b/docs/previews/refresh-2026-09-12/b-editorial-ledger.html @@ -0,0 +1,135 @@ + + + + +方向B · Editorial ledger（解析监控） + + + + +
+ +
+
+
+
+

解析监控 · 需人工处理

+

失败任务以条目登记：缘由、链路断点、处置建议依次列出。勾选后由操作员署名批量重试，死信须逐条复核后方可重新入队。

+
账期 2026-09-121 小时成功 88.9%24 小时成功 96.4%Worker 2
+
+ +
+
2
失败
+
1
停滞
+
4
处理中
+
48
已完成
+
1
死信
+
+ +

失败条目 贰条 · 按断点阶段登记

+ +
+
一
+
+

CR20A 电气规格说明书.pdf对外更新

+
错误码 remote_parse_failed，第二次尝试，已停留七十一小时。建议先在 WeKnora 手动解析该文件确认格式，再决定是否重试。
+
下载 ✓·Hash ✓·OSS ✓·文档解析 ✗·WeKnora 上传·索引·完成
+
+
cs-1 · tp-failed
+
+ +
+
二
+
+

伺服驱动器调试手册.docx对内新增

+
错误码 mineru_timeout，第一次尝试，已停留四十五分钟。多为 MinerU 容量或超时问题，确认服务恢复接单后重试。
+
下载 ✓·Hash ✓·OSS ✓·文档解析 ✓·WeKnora 上传 ✓·索引 ✗·完成
+
+
cs-3 · tp-044
+
+ +

死信登记 壹条 · 尝试耗尽

+
+

对外知识库 · test.upload · release_manifest_mismatch

+

发布结果与冻结清单不一致，已降级为失败。原任务与审计行保留，重新入队将创建全新尝试。来源 cs-2 · CR1A 固件变更记录.pdf · 尝试 5/5 · 2026-09-10 14:22

+
+ +
+ MinerU 可接单 + WeKnora 测试 可接单 + PostgreSQL 12ms + 队列 3 待 / 4 跑 + CPU 42% · 内存 18.6/32.0GB +
+
+
+
+ + diff --git a/docs/previews/refresh-2026-09-12/c-current-polish.html b/docs/previews/refresh-2026-09-12/c-current-polish.html new file mode 100644 index 0000000..680fe24 --- /dev/null +++ b/docs/previews/refresh-2026-09-12/c-current-polish.html @@ -0,0 +1,132 @@ + + + + +方向C · 现行轻打磨（解析监控） + + + + +
+ +
+
+

需人工处理

+

失败原因、处理链定位与建议直接放在卡片上。勾选后可批量重试，死信重新入队会保留原任务与全部审计行。

+ +
+
失败
2
+
停滞
1
+
处理中
4
+
已完成
48
+
死信
1
+
+ +
+

失败任务

3 条 · 按失败阶段排序
+
+
+
+

CR20A 电气规格说明书.pdf 对外 更新

+
错误码 remote_parse_failed · 第 2 次尝试 · 已停留 71.1小时 · 失败于 文档解析
+
下载 ✓Hash ✓OSS ✓文档解析 ✗WeKnora 上传索引完成
+
建议先在 WeKnora 手动解析该文件确认格式，再决定是否重试。 cs-1 · tp-failed
+
+
+
+
+
+
+

伺服驱动器调试手册.docx 对内 新增

+
错误码 mineru_timeout · 第 1 次尝试 · 已停留 45分钟 · 失败于 索引 / 增强
+
下载 ✓Hash ✓OSS ✓文档解析 ✓WeKnora 上传 ✓索引 ✗完成
+
多为 MinerU 容量或超时问题，确认服务恢复接单后重试。 cs-3 · tp-044
+
+
+
+
+ +
+
MinerU 可接单
+
WeKnora 测试 可接单
+
PostgreSQL 12ms
+
队列 3 待 / 4 跑 / 0 重试
+
CPU 42% · 内存 18.6/32.0GB
+
+
+
+
+ + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7d48916 --- /dev/null +++ b/go.mod @@ -0,0 +1,33 @@ +module knowledge-governance + +go 1.26.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.0 + github.com/aws/aws-sdk-go-v2/config v1.33.4 + github.com/aws/aws-sdk-go-v2/credentials v1.20.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/jackc/pgx/v5 v5.9.2 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 // indirect + github.com/aws/smithy-go v1.28.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/text v0.29.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1a07b91 --- /dev/null +++ b/go.sum @@ -0,0 +1,78 @@ +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ= +github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0= +github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/credentials v1.20.4 h1:hTvrJJseKbvw32kmiE0G+u/9ZqpqscjDrTigHIXP2qs= +github.com/aws/aws-sdk-go-v2/credentials v1.20.4/go.mod h1:gWp9O1ZBWwpcIrgV+mVHk4gZUurAEDkgypu/OXOlIaw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0= +github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 h1:khXV3+K5D3f4e8xtplaRdSFn1bEg3gj5EBHQvbCOZbQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.50.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go new file mode 100644 index 0000000..6f960fa --- /dev/null +++ b/internal/appconfig/config.go @@ -0,0 +1,247 @@ +package appconfig + +import ( + "errors" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" +) + +type Config struct { + DatabaseURL string + AppAddr string + WorkerID string + PollInterval time.Duration + LeaseDuration time.Duration + RetryDelay time.Duration + SourceRefreshSchedulerInterval time.Duration + ReleaseWorkerEnabled bool + ReleaseConfigFile string + ProductionReleaseRequestsEnabled bool + ProductionBackupRestoreVerified bool + ProductionReleaseAuthorizerRef string + ProductionReleaseAuthTokenFile string + PPScanEnabled bool + PPScanAllowedRoot string + PPScanLedgerPath string + PPScanManifestPath string + TestWeKnoraEnabled bool + TestWeKnoraAllowDockerHTTP bool + TestWeKnoraBaseURL string + MinerUBaseURL string + TestWeKnoraKBID string + InternalRoutingKnowledgeBaseID string + ExternalRoutingKnowledgeBaseID string + TestWeKnoraAPIKeyFile string + GovernanceSpoolRoot string + WorkerLivenessStaleAfter time.Duration + TestWeKnoraHTTPTimeout time.Duration + TestWeKnoraPollInitial time.Duration + TestWeKnoraPollMax time.Duration + TestWeKnoraStalledAfter time.Duration + TestWeKnoraMaxUploadBytes int64 + TestAcceptancePolicyVersion string +} + +func Load(getenv func(string) string) (Config, error) { + config := Config{ + DatabaseURL: getenv("DATABASE_URL"), + AppAddr: valueOrDefault(getenv("APP_ADDR"), ":8080"), + WorkerID: valueOrDefault(getenv("WORKER_ID"), "worker-1"), + PollInterval: time.Second, + LeaseDuration: 30 * time.Second, + RetryDelay: 5 * time.Second, + SourceRefreshSchedulerInterval: 30 * time.Second, + ReleaseConfigFile: strings.TrimSpace(getenv("RELEASE_CONFIG_FILE")), + ProductionReleaseAuthorizerRef: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTHORIZER_REF")), + ProductionReleaseAuthTokenFile: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTH_TOKEN_FILE")), + PPScanAllowedRoot: strings.TrimSpace(getenv("PP_SCAN_ALLOWED_ROOT")), + PPScanLedgerPath: strings.TrimSpace(getenv("PP_SCAN_LEDGER_PATH")), + PPScanManifestPath: strings.TrimSpace(getenv("PP_SCAN_MANIFEST_PATH")), + TestWeKnoraBaseURL: strings.TrimSpace(getenv("TEST_WEKNORA_BASE_URL")), + MinerUBaseURL: strings.TrimSpace(getenv("MINERU_BASE_URL")), + TestWeKnoraKBID: strings.TrimSpace(getenv("TEST_WEKNORA_KB_ID")), + InternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_INTERNAL_WEKNORA_KB_ID")), + ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")), + TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"), + GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"), + WorkerLivenessStaleAfter: 2 * time.Minute, + TestWeKnoraHTTPTimeout: 60 * time.Second, + TestWeKnoraPollInitial: 10 * time.Second, + TestWeKnoraPollMax: time.Minute, + TestWeKnoraStalledAfter: 30 * time.Minute, + TestWeKnoraMaxUploadBytes: 50 * 1024 * 1024, + TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"), + } + if config.DatabaseURL == "" { + return Config{}, errors.New("DATABASE_URL is required") + } + var err error + if config.PollInterval, err = durationOrDefault(getenv("WORKER_POLL_INTERVAL"), config.PollInterval); err != nil { + return Config{}, fmt.Errorf("WORKER_POLL_INTERVAL: %w", err) + } + if config.LeaseDuration, err = durationOrDefault(getenv("WORKER_LEASE_DURATION"), config.LeaseDuration); err != nil { + return Config{}, fmt.Errorf("WORKER_LEASE_DURATION: %w", err) + } + if config.RetryDelay, err = durationOrDefault(getenv("WORKER_RETRY_DELAY"), config.RetryDelay); err != nil { + return Config{}, fmt.Errorf("WORKER_RETRY_DELAY: %w", err) + } + if config.SourceRefreshSchedulerInterval, err = durationOrDefault(getenv("SOURCE_REFRESH_SCHEDULER_INTERVAL"), config.SourceRefreshSchedulerInterval); err != nil { + return Config{}, fmt.Errorf("SOURCE_REFRESH_SCHEDULER_INTERVAL: %w", err) + } + if config.SourceRefreshSchedulerInterval < 5*time.Second { + return Config{}, errors.New("SOURCE_REFRESH_SCHEDULER_INTERVAL must be at least 5s") + } + if raw := getenv("RELEASE_WORKER_ENABLED"); raw != "" { + config.ReleaseWorkerEnabled, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("RELEASE_WORKER_ENABLED: %w", err) + } + } + if config.ReleaseWorkerEnabled && (!filepath.IsAbs(config.ReleaseConfigFile) || filepath.Clean(config.ReleaseConfigFile) != config.ReleaseConfigFile) { + return Config{}, errors.New("RELEASE_CONFIG_FILE must be an absolute path when release worker is enabled") + } + if raw := getenv("PRODUCTION_RELEASE_REQUESTS_ENABLED"); raw != "" { + config.ProductionReleaseRequestsEnabled, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("PRODUCTION_RELEASE_REQUESTS_ENABLED: %w", err) + } + } + if raw := getenv("PRODUCTION_BACKUP_RESTORE_VERIFIED"); raw != "" { + config.ProductionBackupRestoreVerified, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("PRODUCTION_BACKUP_RESTORE_VERIFIED: %w", err) + } + } + if raw := getenv("PP_SCAN_ENABLED"); raw != "" { + config.PPScanEnabled, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("PP_SCAN_ENABLED: %w", err) + } + } + if raw := getenv("TEST_WEKNORA_ENABLED"); raw != "" { + config.TestWeKnoraEnabled, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_ENABLED: %w", err) + } + } + if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" { + config.TestWeKnoraAllowDockerHTTP, err = strconv.ParseBool(raw) + if err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_ALLOW_DOCKER_HTTP: %w", err) + } + } + if config.WorkerLivenessStaleAfter, err = durationOrDefault(getenv("WORKER_LIVENESS_STALE_AFTER"), config.WorkerLivenessStaleAfter); err != nil { + return Config{}, fmt.Errorf("WORKER_LIVENESS_STALE_AFTER: %w", err) + } + if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err) + } + if config.TestWeKnoraPollInitial, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_INITIAL"), config.TestWeKnoraPollInitial); err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_INITIAL: %w", err) + } + if config.TestWeKnoraPollMax, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_MAX"), config.TestWeKnoraPollMax); err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_MAX: %w", err) + } + if config.TestWeKnoraStalledAfter, err = durationOrDefault(getenv("TEST_WEKNORA_STALLED_AFTER"), config.TestWeKnoraStalledAfter); err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_STALLED_AFTER: %w", err) + } + if config.TestWeKnoraMaxUploadBytes, err = positiveInt64OrDefault(getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"), config.TestWeKnoraMaxUploadBytes); err != nil { + return Config{}, fmt.Errorf("TEST_WEKNORA_MAX_UPLOAD_BYTES: %w", err) + } + if config.TestWeKnoraMaxUploadBytes > 50*1024*1024 { + return Config{}, errors.New("TEST_WEKNORA_MAX_UPLOAD_BYTES must not exceed 50 MiB") + } + if config.TestWeKnoraPollMax < config.TestWeKnoraPollInitial { + return Config{}, errors.New("TEST_WEKNORA_POLL_MAX must not be less than TEST_WEKNORA_POLL_INITIAL") + } + if config.TestWeKnoraStalledAfter <= config.TestWeKnoraPollMax { + return Config{}, errors.New("TEST_WEKNORA_STALLED_AFTER must be greater than TEST_WEKNORA_POLL_MAX") + } + config.TestAcceptancePolicyVersion = strings.TrimSpace(config.TestAcceptancePolicyVersion) + if config.TestAcceptancePolicyVersion == "" { + return Config{}, errors.New("TEST_ACCEPTANCE_POLICY_VERSION must not be empty") + } + if config.ProductionReleaseRequestsEnabled { + if config.ProductionReleaseAuthorizerRef == "" || len(config.ProductionReleaseAuthorizerRef) > 128 || strings.ContainsAny(config.ProductionReleaseAuthorizerRef, "\x00\r\n ") { + return Config{}, errors.New("PRODUCTION_RELEASE_AUTHORIZER_REF is required and must be safe when production release requests are enabled") + } + if config.ProductionReleaseAuthTokenFile == "" || !filepath.IsAbs(config.ProductionReleaseAuthTokenFile) { + return Config{}, errors.New("PRODUCTION_RELEASE_AUTH_TOKEN_FILE must be an absolute path when production release requests are enabled") + } + } + if config.PPScanEnabled { + for key, value := range map[string]string{ + "PP_SCAN_ALLOWED_ROOT": config.PPScanAllowedRoot, + "PP_SCAN_LEDGER_PATH": config.PPScanLedgerPath, + "PP_SCAN_MANIFEST_PATH": config.PPScanManifestPath, + } { + if value == "" { + return Config{}, fmt.Errorf("%s is required when PP scan is enabled", key) + } + if !filepath.IsAbs(value) { + return Config{}, fmt.Errorf("%s must be absolute", key) + } + } + } + if config.TestWeKnoraEnabled { + if config.TestWeKnoraBaseURL == "" { + return Config{}, errors.New("TEST_WEKNORA_BASE_URL is required when test WeKnora is enabled") + } + if config.TestWeKnoraKBID == "" { + return Config{}, errors.New("TEST_WEKNORA_KB_ID is required when test WeKnora is enabled") + } + if strings.TrimSpace(config.TestWeKnoraAPIKeyFile) == "" { + return Config{}, errors.New("TEST_WEKNORA_API_KEY_FILE is required when test WeKnora is enabled") + } + if strings.TrimSpace(config.GovernanceSpoolRoot) == "" { + return Config{}, errors.New("GOVERNANCE_SPOOL_ROOT is required when test WeKnora is enabled") + } + } + if (config.InternalRoutingKnowledgeBaseID == "") != (config.ExternalRoutingKnowledgeBaseID == "") { + return Config{}, errors.New("both internal and external routing knowledge bases must be configured together") + } + for _, knowledgeBaseID := range []string{config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID} { + if len(knowledgeBaseID) > 128 || strings.ContainsAny(knowledgeBaseID, "\x00\r\n /?#\\") { + return Config{}, errors.New("routing knowledge base ID is invalid") + } + } + return config, nil +} + +func valueOrDefault(value, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func durationOrDefault(value string, fallback time.Duration) (time.Duration, error) { + if value == "" { + return fallback, nil + } + parsed, err := time.ParseDuration(value) + if err != nil { + return 0, err + } + if parsed <= 0 { + return 0, errors.New("must be positive") + } + return parsed, nil +} + +func positiveInt64OrDefault(value string, fallback int64) (int64, error) { + if value == "" { + return fallback, nil + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, err + } + if parsed <= 0 { + return 0, errors.New("must be positive") + } + return parsed, nil +} diff --git a/internal/appconfig/config_test.go b/internal/appconfig/config_test.go new file mode 100644 index 0000000..e6ede9e --- /dev/null +++ b/internal/appconfig/config_test.go @@ -0,0 +1,243 @@ +package appconfig + +import ( + "testing" + "time" +) + +func TestLoadUsesSafeDefaults(t *testing.T) { + values := map[string]string{"DATABASE_URL": "postgres://example"} + config, err := Load(func(key string) string { return values[key] }) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if config.AppAddr != ":8080" || config.WorkerID != "worker-1" { + t.Fatalf("address/worker defaults = %#v", config) + } + if config.PollInterval != time.Second || config.LeaseDuration != 30*time.Second || config.RetryDelay != 5*time.Second { + t.Fatalf("duration defaults = %#v", config) + } + if config.ReleaseWorkerEnabled { + t.Fatal("release worker must be disabled by default") + } + if config.ReleaseConfigFile != "" { + t.Fatalf("release config default = %q", config.ReleaseConfigFile) + } + if config.ProductionReleaseRequestsEnabled { + t.Fatal("production release requests must be disabled by default") + } + if config.TestWeKnoraEnabled { + t.Fatal("test WeKnora must be disabled by default") + } + if config.PPScanEnabled { + t.Fatal("PP source scan must be disabled by default") + } + if config.PPScanAllowedRoot != "" || config.PPScanLedgerPath != "" || config.PPScanManifestPath != "" { + t.Fatalf("PP scan paths must be empty by default: %#v", config) + } + if config.TestWeKnoraAllowDockerHTTP { + t.Fatal("test WeKnora Docker HTTP must be disabled by default") + } + if config.TestWeKnoraAPIKeyFile != "/run/secrets/test_weknora_api_key" { + t.Fatalf("test API key file = %q", config.TestWeKnoraAPIKeyFile) + } + if config.GovernanceSpoolRoot != "/var/lib/governance/spool" { + t.Fatalf("spool root = %q", config.GovernanceSpoolRoot) + } + if config.TestWeKnoraHTTPTimeout != 60*time.Second || config.TestWeKnoraPollInitial != 10*time.Second || config.TestWeKnoraPollMax != time.Minute || config.TestWeKnoraStalledAfter != 30*time.Minute { + t.Fatalf("test WeKnora timing defaults = %#v", config) + } + if config.TestWeKnoraMaxUploadBytes != 50*1024*1024 || config.TestAcceptancePolicyVersion != "structural-v1" { + t.Fatalf("test WeKnora safety defaults = %#v", config) + } +} + +func TestLoadRequiresDatabaseURL(t *testing.T) { + if _, err := Load(func(string) string { return "" }); err == nil { + t.Fatal("Load() error = nil, want missing DATABASE_URL error") + } +} + +func TestLoadParsesProductionReleaseRequestGate(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://example", + "PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", + "PRODUCTION_BACKUP_RESTORE_VERIFIED": "true", + "PRODUCTION_RELEASE_AUTHORIZER_REF": "release-owner", + "PRODUCTION_RELEASE_AUTH_TOKEN_FILE": "/run/secrets/production_release_auth_token", + } + config, err := Load(func(key string) string { return values[key] }) + if err != nil || !config.ProductionReleaseRequestsEnabled || !config.ProductionBackupRestoreVerified || config.ProductionReleaseAuthorizerRef != "release-owner" || config.ProductionReleaseAuthTokenFile != "/run/secrets/production_release_auth_token" { + t.Fatalf("config=%#v error=%v", config, err) + } + values["PRODUCTION_RELEASE_REQUESTS_ENABLED"] = "not-a-bool" + if _, err := Load(func(key string) string { return values[key] }); err == nil { + t.Fatal("invalid production release request gate must be rejected") + } + values["PRODUCTION_RELEASE_REQUESTS_ENABLED"] = "true" + values["PRODUCTION_BACKUP_RESTORE_VERIFIED"] = "not-a-bool" + if _, err := Load(func(key string) string { return values[key] }); err == nil { + t.Fatal("invalid production backup restore gate must be rejected") + } +} + +func TestLoadRequiresProductionReleaseAuthorizationConfigWhenGateEnabled(t *testing.T) { + base := map[string]string{ + "DATABASE_URL": "postgres://example", + "PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", + "PRODUCTION_RELEASE_AUTHORIZER_REF": "release-owner", + "PRODUCTION_RELEASE_AUTH_TOKEN_FILE": "/run/secrets/production_release_auth_token", + } + for _, missing := range []string{"PRODUCTION_RELEASE_AUTHORIZER_REF", "PRODUCTION_RELEASE_AUTH_TOKEN_FILE"} { + values := make(map[string]string, len(base)) + for key, value := range base { + values[key] = value + } + delete(values, missing) + if _, err := Load(func(key string) string { return values[key] }); err == nil { + t.Fatalf("missing %s accepted", missing) + } + } + base["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"] = "relative/token" + if _, err := Load(func(key string) string { return base[key] }); err == nil { + t.Fatal("relative production release token file accepted") + } +} + +func TestLoadRequiresReleaseConfigFileWhenReleaseWorkerEnabled(t *testing.T) { + if _, err := Load(func(key string) string { + values := map[string]string{"DATABASE_URL": "postgres://example", "RELEASE_WORKER_ENABLED": "true"} + return values[key] + }); err == nil { + t.Fatal("enabled release worker without config file accepted") + } + config, err := Load(func(key string) string { + values := map[string]string{"DATABASE_URL": "postgres://example", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/run/secrets/release-config"} + return values[key] + }) + if err != nil || config.ReleaseConfigFile != "/run/secrets/release-config" { + t.Fatalf("config=%#v err=%v", config, err) + } +} + +func TestLoadRequiresAllPPScanPathsWhenEnabled(t *testing.T) { + base := map[string]string{ + "DATABASE_URL": "postgres://example", + "PP_SCAN_ENABLED": "true", + "PP_SCAN_ALLOWED_ROOT": "/source", + "PP_SCAN_LEDGER_PATH": "/source/state.jsonl", + "PP_SCAN_MANIFEST_PATH": "/source/manifest.json", + } + for _, missing := range []string{"PP_SCAN_ALLOWED_ROOT", "PP_SCAN_LEDGER_PATH", "PP_SCAN_MANIFEST_PATH"} { + values := make(map[string]string, len(base)) + for key, value := range base { + values[key] = value + } + delete(values, missing) + if _, err := Load(func(key string) string { return values[key] }); err == nil { + t.Fatalf("Load() error = nil, want missing %s rejected", missing) + } + } +} + +func TestLoadRequiresTestWeKnoraEndpointAndKBWhenEnabled(t *testing.T) { + base := map[string]string{ + "DATABASE_URL": "postgres://example", + "TEST_WEKNORA_ENABLED": "true", + } + for _, key := range []string{"TEST_WEKNORA_BASE_URL", "TEST_WEKNORA_KB_ID"} { + values := make(map[string]string, len(base)+1) + for name, value := range base { + values[name] = value + } + if key == "TEST_WEKNORA_BASE_URL" { + values["TEST_WEKNORA_KB_ID"] = "kb-test" + } else { + values["TEST_WEKNORA_BASE_URL"] = "https://weknora.test.example" + } + if _, err := Load(func(name string) string { return values[name] }); err == nil { + t.Fatalf("Load() error = nil, want missing %s error", key) + } + } +} + +func TestLoadParsesTestWeKnoraSettings(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://example", + "TEST_WEKNORA_ENABLED": "true", + "TEST_WEKNORA_ALLOW_DOCKER_HTTP": "true", + "TEST_WEKNORA_BASE_URL": "https://weknora.test.example", + "TEST_WEKNORA_KB_ID": "kb-test", + "TEST_WEKNORA_API_KEY_FILE": "/run/secrets/custom_key", + "GOVERNANCE_SPOOL_ROOT": "/srv/spool", + "TEST_WEKNORA_HTTP_TIMEOUT": "45s", + "TEST_WEKNORA_POLL_INITIAL": "5s", + "TEST_WEKNORA_POLL_MAX": "20s", + "TEST_WEKNORA_STALLED_AFTER": "15m", + "TEST_WEKNORA_MAX_UPLOAD_BYTES": "1048576", + "TEST_ACCEPTANCE_POLICY_VERSION": "structural-v2", + } + config, err := Load(func(name string) string { return values[name] }) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !config.TestWeKnoraEnabled || !config.TestWeKnoraAllowDockerHTTP || config.TestWeKnoraBaseURL != values["TEST_WEKNORA_BASE_URL"] || config.TestWeKnoraKBID != "kb-test" { + t.Fatalf("test WeKnora identity = %#v", config) + } + if config.TestWeKnoraAPIKeyFile != "/run/secrets/custom_key" || config.GovernanceSpoolRoot != "/srv/spool" { + t.Fatalf("test WeKnora file settings = %#v", config) + } + if config.TestWeKnoraHTTPTimeout != 45*time.Second || config.TestWeKnoraPollInitial != 5*time.Second || config.TestWeKnoraPollMax != 20*time.Second || config.TestWeKnoraStalledAfter != 15*time.Minute { + t.Fatalf("test WeKnora timing settings = %#v", config) + } + if config.TestWeKnoraMaxUploadBytes != 1048576 || config.TestAcceptancePolicyVersion != "structural-v2" { + t.Fatalf("test WeKnora safety settings = %#v", config) + } +} + +func TestLoadRejectsUnsafeTestWeKnoraSettings(t *testing.T) { + tests := []struct { + name string + key string + value string + }{ + {name: "poll max before initial", key: "TEST_WEKNORA_POLL_MAX", value: "5s"}, + {name: "stalled before poll max", key: "TEST_WEKNORA_STALLED_AFTER", value: "30s"}, + {name: "upload limit above 50 MiB", key: "TEST_WEKNORA_MAX_UPLOAD_BYTES", value: "52428801"}, + {name: "empty policy", key: "TEST_ACCEPTANCE_POLICY_VERSION", value: " "}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://example", + "TEST_WEKNORA_ENABLED": "true", + "TEST_WEKNORA_BASE_URL": "https://weknora.test.example", + "TEST_WEKNORA_KB_ID": "kb-test", + "TEST_WEKNORA_POLL_INITIAL": "10s", + "TEST_WEKNORA_POLL_MAX": "60s", + "TEST_WEKNORA_STALLED_AFTER": "30m", + "TEST_ACCEPTANCE_POLICY_VERSION": "structural-v1", + } + values[tt.key] = tt.value + if _, err := Load(func(name string) string { return values[name] }); err == nil { + t.Fatalf("Load() error = nil, want unsafe %s rejected", tt.key) + } + }) + } +} + +func TestLoadRequiresBothCatalogRoutingKnowledgeBases(t *testing.T) { + values := map[string]string{ + "DATABASE_URL": "postgres://example", + "TEST_INTERNAL_WEKNORA_KB_ID": "kb-internal", + "TEST_EXTERNAL_WEKNORA_KB_ID": "kb-external", + } + config, err := Load(func(key string) string { return values[key] }) + if err != nil || config.InternalRoutingKnowledgeBaseID != "kb-internal" || config.ExternalRoutingKnowledgeBaseID != "kb-external" { + t.Fatalf("config=%#v err=%v", config, err) + } + delete(values, "TEST_EXTERNAL_WEKNORA_KB_ID") + if _, err = Load(func(key string) string { return values[key] }); err == nil { + t.Fatal("one-sided routing target must fail closed") + } +} diff --git a/internal/connectors/pp/ledger.go b/internal/connectors/pp/ledger.go new file mode 100644 index 0000000..34960cd --- /dev/null +++ b/internal/connectors/pp/ledger.go @@ -0,0 +1,141 @@ +package pp + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "sort" + + "knowledge-governance/internal/governance" +) + +type LegacyStatus string + +const ( + StatusCompleted LegacyStatus = "completed" + StatusContentAlias LegacyStatus = "content_alias" + StatusOverLimit LegacyStatus = "over_limit" +) + +type Item struct { + KBID string `json:"kb_id"` + SourceURLHash string `json:"source_url_hash"` + SourceIDs []string `json:"source_ids"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Status LegacyStatus `json:"status"` + KnowledgeID string `json:"knowledge_id"` + CanonicalSourceURLHash string `json:"canonical_source_url_hash"` +} + +type Snapshot struct { + KBID string + EventCount int + UnresolvedCount int + Items []Item +} + +func FoldLedger(reader io.Reader) (Snapshot, error) { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + terminal := make(map[string]Item) + seen := make(map[string]struct{}) + result := Snapshot{} + for scanner.Scan() { + result.EventCount++ + var item Item + if err := json.Unmarshal(scanner.Bytes(), &item); err != nil { + return Snapshot{}, fmt.Errorf("decode ledger line %d: %w", result.EventCount, err) + } + if result.KBID == "" { + result.KBID = item.KBID + } else if item.KBID != result.KBID { + return Snapshot{}, fmt.Errorf("ledger contains multiple knowledge bases at line %d", result.EventCount) + } + if item.SourceURLHash != "" { + seen[item.SourceURLHash] = struct{}{} + } + if item.Status == StatusCompleted || (item.Status == StatusContentAlias && item.KnowledgeID != "") { + if err := validateTerminalItem(item); err != nil { + return Snapshot{}, fmt.Errorf("invalid terminal ledger record at line %d: %w", result.EventCount, err) + } + terminal[item.SourceURLHash] = item + } + } + if err := scanner.Err(); err != nil { + return Snapshot{}, fmt.Errorf("read ledger: %w", err) + } + result.Items = make([]Item, 0, len(terminal)) + for _, item := range terminal { + result.Items = append(result.Items, item) + } + result.UnresolvedCount = len(seen) - len(terminal) + sort.Slice(result.Items, func(i, j int) bool { + return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash + }) + return result, nil +} + +func validateTerminalItem(item Item) error { + if item.KBID == "" { + return fmt.Errorf("kb_id is required") + } + if !governance.ValidSHA256(item.SourceURLHash) { + return fmt.Errorf("source_url_hash must be a canonical hash") + } + if len(item.SourceIDs) == 0 { + return fmt.Errorf("source_ids are required") + } + for _, sourceID := range item.SourceIDs { + if sourceID == "" { + return fmt.Errorf("source_ids cannot contain an empty value") + } + } + if item.Bytes < 0 { + return fmt.Errorf("bytes cannot be negative") + } + if item.SHA256 != "" && !governance.ValidSHA256(item.SHA256) { + return fmt.Errorf("sha256 must be empty or a canonical hash") + } + if item.KnowledgeID == "" { + return fmt.Errorf("knowledge_id is required") + } + if item.Status == StatusContentAlias && !governance.ValidSHA256(item.CanonicalSourceURLHash) { + return fmt.Errorf("canonical_source_url_hash must be a canonical hash") + } + return nil +} + +func ValidateSnapshot(snapshot Snapshot) error { + if snapshot.UnresolvedCount != 0 { + return fmt.Errorf("snapshot contains %d unresolved source keys", snapshot.UnresolvedCount) + } + bySourceHash := make(map[string]Item, len(snapshot.Items)) + for _, item := range snapshot.Items { + if item.Status == StatusOverLimit { + if item.Bytes <= LegacyMaxBytes { + return fmt.Errorf("over-limit item does not exceed the legacy size limit") + } + bySourceHash[item.SourceURLHash] = item + continue + } + if !governance.ValidSHA256(item.SHA256) { + return fmt.Errorf("snapshot item is missing a verified SHA-256") + } + bySourceHash[item.SourceURLHash] = item + } + for _, item := range snapshot.Items { + if item.Status != StatusContentAlias { + continue + } + canonical, ok := bySourceHash[item.CanonicalSourceURLHash] + if !ok || canonical.Status != StatusCompleted { + return fmt.Errorf("alias does not reference a canonical completed item") + } + if item.SHA256 != canonical.SHA256 || item.KnowledgeID != canonical.KnowledgeID { + return fmt.Errorf("alias metadata does not match canonical completed item") + } + } + return nil +} diff --git a/internal/connectors/pp/ledger_test.go b/internal/connectors/pp/ledger_test.go new file mode 100644 index 0000000..08acb8c --- /dev/null +++ b/internal/connectors/pp/ledger_test.go @@ -0,0 +1,224 @@ +package pp + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestFoldLedgerKeepsLatestTerminalRecordPerSourceURLHash(t *testing.T) { + sourceA := strings.Repeat("a", 64) + sourceB := strings.Repeat("b", 64) + sha := strings.Repeat("c", 64) + ledger := strings.Join([]string{ + `{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-old"}`, + `{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`, + `{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"parse_timeout","knowledge_id":"knowledge-1"}`, + `{"kb_id":"kb-1","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1","canonical_source_url_hash":"` + sourceA + `"}`, + }, "\n") + + snapshot, err := FoldLedger(strings.NewReader(ledger)) + if err != nil { + t.Fatalf("FoldLedger() error = %v", err) + } + if snapshot.EventCount != 4 || snapshot.KBID != "kb-1" || len(snapshot.Items) != 2 { + t.Fatalf("FoldLedger() snapshot = %#v", snapshot) + } + if snapshot.Items[0].SourceURLHash != sourceA || snapshot.Items[0].Status != StatusCompleted || snapshot.Items[0].KnowledgeID != "knowledge-1" { + t.Fatalf("completed item = %#v", snapshot.Items[0]) + } + if snapshot.Items[1].SourceURLHash != sourceB || snapshot.Items[1].Status != StatusContentAlias || snapshot.Items[1].CanonicalSourceURLHash != sourceA { + t.Fatalf("alias item = %#v", snapshot.Items[1]) + } +} + +func TestFoldLedgerReportsUnresolvedSourceKeys(t *testing.T) { + source := strings.Repeat("a", 64) + sha := strings.Repeat("c", 64) + ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-1"}` + + snapshot, err := FoldLedger(strings.NewReader(ledger)) + if err != nil { + t.Fatalf("FoldLedger() error = %v", err) + } + if snapshot.UnresolvedCount != 1 { + t.Fatalf("UnresolvedCount = %d, want 1", snapshot.UnresolvedCount) + } +} + +func TestValidateSnapshotRejectsUnresolvedSourceKeys(t *testing.T) { + err := ValidateSnapshot(Snapshot{KBID: "kb-1", UnresolvedCount: 1}) + if err == nil || !strings.Contains(err.Error(), "unresolved") { + t.Fatalf("ValidateSnapshot() error = %v, want unresolved source error", err) + } +} + +func TestFoldLedgerRejectsMixedKnowledgeBases(t *testing.T) { + sourceA := strings.Repeat("a", 64) + sourceB := strings.Repeat("b", 64) + sha := strings.Repeat("c", 64) + ledger := strings.Join([]string{ + `{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`, + `{"kb_id":"kb-2","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-2"}`, + }, "\n") + + _, err := FoldLedger(strings.NewReader(ledger)) + if err == nil || !strings.Contains(err.Error(), "multiple knowledge bases") { + t.Fatalf("FoldLedger() error = %v, want mixed knowledge base error", err) + } +} + +func TestFoldLedgerRejectsInvalidTerminalSourceHash(t *testing.T) { + sha := strings.Repeat("c", 64) + ledger := `{"kb_id":"kb-1","source_url_hash":"not-a-hash","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}` + + _, err := FoldLedger(strings.NewReader(ledger)) + if err == nil || !strings.Contains(err.Error(), "source_url_hash") { + t.Fatalf("FoldLedger() error = %v, want source hash validation error", err) + } +} + +func TestFoldLedgerRejectsIncompleteTerminalRecord(t *testing.T) { + source := strings.Repeat("a", 64) + sha := strings.Repeat("c", 64) + tests := []struct { + name string + ledger string + }{ + {"missing knowledge base", `{"source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`}, + {"missing source IDs", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":[],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`}, + {"negative bytes", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":-1,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`}, + {"invalid optional SHA", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"bad","status":"completed","knowledge_id":"knowledge-1"}`}, + {"missing knowledge ID", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed"}`}, + {"missing alias canonical hash", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1"}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := FoldLedger(strings.NewReader(tt.ledger)); err == nil { + t.Fatal("FoldLedger() error = nil, want terminal record validation error") + } + }) + } +} + +func TestRecoverMissingSHA256RequiresVerifiedManifestArtifact(t *testing.T) { + content := []byte("verified PP artifact") + contentDigest := sha256.Sum256(content) + contentHash := hex.EncodeToString(contentDigest[:]) + url := "https://source.invalid/document" + urlDigest := sha256.Sum256([]byte(url)) + sourceHash := hex.EncodeToString(urlDigest[:]) + path := filepath.Join(t.TempDir(), "document.bin") + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + manifest := `{"results":[{"url":"` + url + `","local_path":"` + path + `","bytes":20,"sha256":"` + contentHash + `"}]}` + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: sourceHash, + SourceIDs: []string{"pp-1"}, + Bytes: 20, + Status: StatusCompleted, + KnowledgeID: "knowledge-1", + }}} + + recovered, report, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest)) + if err != nil { + t.Fatalf("RecoverMissingSHA256() error = %v", err) + } + if report.Recovered != 1 || report.VerifiedBytes != int64(len(content)) { + t.Fatalf("RecoverMissingSHA256() report = %#v", report) + } + if recovered.Items[0].SHA256 != contentHash { + t.Fatalf("recovered SHA256 = %q, want %q", recovered.Items[0].SHA256, contentHash) + } +} + +func TestRecoverMissingSHA256RejectsArtifactOverLegacyLimit(t *testing.T) { + url := "https://source.invalid/oversized" + urlDigest := sha256.Sum256([]byte(url)) + sourceHash := hex.EncodeToString(urlDigest[:]) + contentHash := strings.Repeat("c", 64) + oversized := int64(50*1024*1024 + 1) + manifest := `{"results":[{"url":"` + url + `","local_path":"/not-opened","bytes":` + "52428801" + `,"sha256":"` + contentHash + `"}]}` + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: sourceHash, + SourceIDs: []string{"pp-1"}, + Bytes: oversized, + Status: StatusCompleted, + KnowledgeID: "knowledge-1", + }}} + + _, _, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest)) + if err == nil || !strings.Contains(err.Error(), "50 MiB") { + t.Fatalf("RecoverMissingSHA256() error = %v, want legacy size limit error", err) + } +} + +func TestValidateSnapshotRejectsAliasWithoutCanonicalCompletedItem(t *testing.T) { + sourceAlias := strings.Repeat("b", 64) + missingCanonical := strings.Repeat("a", 64) + sha := strings.Repeat("c", 64) + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: sourceAlias, + SourceIDs: []string{"pp-2"}, + Bytes: 10, + SHA256: sha, + Status: StatusContentAlias, + KnowledgeID: "knowledge-1", + CanonicalSourceURLHash: missingCanonical, + }}} + + err := ValidateSnapshot(snapshot) + if err == nil || !strings.Contains(err.Error(), "canonical completed item") { + t.Fatalf("ValidateSnapshot() error = %v, want missing canonical error", err) + } +} + +func TestValidateSnapshotRejectsAliasMetadataMismatch(t *testing.T) { + sourceCanonical := strings.Repeat("a", 64) + sourceAlias := strings.Repeat("b", 64) + sha := strings.Repeat("c", 64) + canonical := Item{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sha, Status: StatusCompleted, KnowledgeID: "knowledge-1"} + alias := Item{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: sha, Status: StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical} + + tests := []struct { + name string + mutate func(*Item) + }{ + {"SHA mismatch", func(item *Item) { item.SHA256 = strings.Repeat("d", 64) }}, + {"knowledge mismatch", func(item *Item) { item.KnowledgeID = "knowledge-2" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + candidate := alias + tt.mutate(&candidate) + err := ValidateSnapshot(Snapshot{KBID: "kb-1", Items: []Item{canonical, candidate}}) + if err == nil || !strings.Contains(err.Error(), "alias metadata") { + t.Fatalf("ValidateSnapshot() error = %v, want alias metadata error", err) + } + }) + } +} + +func TestValidateSnapshotRequiresRecoveredSHA256(t *testing.T) { + source := strings.Repeat("a", 64) + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: source, + SourceIDs: []string{"pp-1"}, + Bytes: 10, + Status: StatusCompleted, + KnowledgeID: "knowledge-1", + }}} + + err := ValidateSnapshot(snapshot) + if err == nil || !strings.Contains(err.Error(), "verified SHA-256") { + t.Fatalf("ValidateSnapshot() error = %v, want missing SHA error", err) + } +} diff --git a/internal/connectors/pp/manifest.go b/internal/connectors/pp/manifest.go new file mode 100644 index 0000000..4c923fc --- /dev/null +++ b/internal/connectors/pp/manifest.go @@ -0,0 +1,91 @@ +package pp + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + + "knowledge-governance/internal/governance" +) + +type RecoveryReport struct { + Recovered int + VerifiedBytes int64 +} + +const LegacyMaxBytes int64 = 50 * 1024 * 1024 + +type manifestDocument struct { + Results []manifestItem `json:"results"` +} + +type manifestItem struct { + URL string `json:"url"` + LocalPath string `json:"local_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + SourceIDs []string `json:"source_ids"` +} + +func RecoverMissingSHA256(snapshot Snapshot, reader io.Reader) (Snapshot, RecoveryReport, error) { + var manifest manifestDocument + if err := json.NewDecoder(reader).Decode(&manifest); err != nil { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("decode download manifest: %w", err) + } + bySourceHash := make(map[string]manifestItem, len(manifest.Results)) + for _, item := range manifest.Results { + digest := sha256.Sum256([]byte(item.URL)) + bySourceHash[hex.EncodeToString(digest[:])] = item + } + + result := snapshot + result.Items = append([]Item(nil), snapshot.Items...) + report := RecoveryReport{} + for index := range result.Items { + item := &result.Items[index] + if item.SHA256 != "" { + continue + } + artifact, ok := bySourceHash[item.SourceURLHash] + if !ok { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("missing manifest artifact for source hash %s", item.SourceURLHash) + } + if !governance.ValidSHA256(artifact.SHA256) { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact has invalid SHA-256 for source hash %s", item.SourceURLHash) + } + if artifact.Bytes != item.Bytes { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest size differs from ledger for source hash %s", item.SourceURLHash) + } + if artifact.Bytes > LegacyMaxBytes { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact exceeds the legacy 50 MiB limit for source hash %s", item.SourceURLHash) + } + file, err := os.Open(artifact.LocalPath) + if err != nil { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("open manifest artifact for source hash %s: %w", item.SourceURLHash, err) + } + digest := sha256.New() + bytesRead, copyErr := io.Copy(digest, file) + closeErr := file.Close() + if copyErr != nil { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("hash manifest artifact for source hash %s: %w", item.SourceURLHash, copyErr) + } + if closeErr != nil { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("close manifest artifact for source hash %s: %w", item.SourceURLHash, closeErr) + } + if bytesRead != artifact.Bytes { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact size mismatch for source hash %s", item.SourceURLHash) + } + actual := hex.EncodeToString(digest.Sum(nil)) + if actual != artifact.SHA256 { + return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact SHA-256 mismatch for source hash %s", item.SourceURLHash) + } + item.SHA256 = actual + report.Recovered++ + report.VerifiedBytes += bytesRead + } + return result, report, nil +} diff --git a/internal/connectors/pp/overlimit.go b/internal/connectors/pp/overlimit.go new file mode 100644 index 0000000..c3a8bcf --- /dev/null +++ b/internal/connectors/pp/overlimit.go @@ -0,0 +1,55 @@ +package pp + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" +) + +type OverLimitReport struct { + Added int +} + +func MergeOverLimitManifest(snapshot Snapshot, reader io.Reader) (Snapshot, OverLimitReport, error) { + var manifest manifestDocument + if err := json.NewDecoder(reader).Decode(&manifest); err != nil { + return Snapshot{}, OverLimitReport{}, fmt.Errorf("decode download manifest: %w", err) + } + + result := snapshot + result.Items = append([]Item(nil), snapshot.Items...) + seen := make(map[string]struct{}, len(result.Items)) + for _, item := range result.Items { + seen[item.SourceURLHash] = struct{}{} + } + report := OverLimitReport{} + for _, artifact := range manifest.Results { + if artifact.Status != string(StatusOverLimit) { + continue + } + if artifact.URL == "" || len(artifact.SourceIDs) == 0 || artifact.Bytes <= LegacyMaxBytes { + return Snapshot{}, OverLimitReport{}, fmt.Errorf("invalid over-limit manifest record") + } + digest := sha256.Sum256([]byte(artifact.URL)) + sourceHash := hex.EncodeToString(digest[:]) + if _, exists := seen[sourceHash]; exists { + continue + } + result.Items = append(result.Items, Item{ + KBID: snapshot.KBID, + SourceURLHash: sourceHash, + SourceIDs: append([]string(nil), artifact.SourceIDs...), + Bytes: artifact.Bytes, + Status: StatusOverLimit, + }) + seen[sourceHash] = struct{}{} + report.Added++ + } + sort.Slice(result.Items, func(i, j int) bool { + return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash + }) + return result, report, nil +} diff --git a/internal/connectors/pp/overlimit_test.go b/internal/connectors/pp/overlimit_test.go new file mode 100644 index 0000000..8bb0ef3 --- /dev/null +++ b/internal/connectors/pp/overlimit_test.go @@ -0,0 +1,63 @@ +package pp + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func TestMergeOverLimitManifestAddsBlockedSourceItem(t *testing.T) { + canonicalSource := strings.Repeat("a", 64) + contentHash := strings.Repeat("c", 64) + url := "https://source.invalid/large-document" + urlDigest := sha256.Sum256([]byte(url)) + overLimitSource := hex.EncodeToString(urlDigest[:]) + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: canonicalSource, + SourceIDs: []string{"pp-1"}, + Bytes: 20, + SHA256: contentHash, + Status: StatusCompleted, + KnowledgeID: "knowledge-1", + }}} + manifest := `{"results":[` + + `{"url":"https://source.invalid/existing","status":"existing","bytes":20,"source_ids":["pp-1"]},` + + `{"url":"` + url + `","local_path":"/not-opened","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}` + + `]}` + + merged, report, err := MergeOverLimitManifest(snapshot, strings.NewReader(manifest)) + if err != nil { + t.Fatalf("MergeOverLimitManifest() error = %v", err) + } + if report.Added != 1 { + t.Fatalf("Added = %d, want 1", report.Added) + } + if len(merged.Items) != 2 { + t.Fatalf("items = %d, want 2", len(merged.Items)) + } + var blocked Item + for _, item := range merged.Items { + if item.SourceURLHash == overLimitSource { + blocked = item + } + } + if blocked.Status != StatusOverLimit || blocked.Bytes != 52428801 || blocked.SHA256 != "" { + t.Fatalf("blocked item = %#v", blocked) + } +} + +func TestValidateSnapshotAcceptsOverLimitItemWithoutContentHash(t *testing.T) { + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: strings.Repeat("a", 64), + SourceIDs: []string{"pp-1"}, + Bytes: LegacyMaxBytes + 1, + Status: StatusOverLimit, + }}} + + if err := ValidateSnapshot(snapshot); err != nil { + t.Fatalf("ValidateSnapshot() error = %v", err) + } +} diff --git a/internal/connectors/pp/plan.go b/internal/connectors/pp/plan.go new file mode 100644 index 0000000..0bb46b4 --- /dev/null +++ b/internal/connectors/pp/plan.go @@ -0,0 +1,150 @@ +package pp + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strconv" + + "knowledge-governance/internal/governance" +) + +type ImportPlan struct { + Connection governance.SourceConnection + SourceItems []governance.SourceItem + ContentObjects []governance.ContentObject + Versions []governance.SourceItemVersion + Publications []governance.Publication +} + +func BuildImportPlan(snapshot Snapshot) (ImportPlan, error) { + if err := ValidateSnapshot(snapshot); err != nil { + return ImportPlan{}, err + } + connectionID := stableID("sc", snapshot.KBID) + plan := ImportPlan{ + Connection: governance.SourceConnection{ + ID: connectionID, + Type: governance.SourceTypePP, + Name: "PP legacy " + connectionID, + SecretRef: "", + Enabled: false, + }, + SourceItems: make([]governance.SourceItem, 0, len(snapshot.Items)), + ContentObjects: make([]governance.ContentObject, 0), + Versions: make([]governance.SourceItemVersion, 0, len(snapshot.Items)), + Publications: make([]governance.Publication, 0), + } + contentBySHA := make(map[string]governance.ContentObject) + publicationByKnowledgeID := make(map[string]struct{}) + for _, item := range snapshot.Items { + itemID := stableID("si", connectionID, item.SourceURLHash) + sizeBytes := item.Bytes + revision := item.SHA256 + if item.Status == StatusOverLimit { + revision = metadataRevision(item) + } + plan.SourceItems = append(plan.SourceItems, governance.SourceItem{ + ID: itemID, + SourceConnectionID: connectionID, + StableExternalID: item.SourceURLHash, + CanonicalURIHash: item.SourceURLHash, + Title: "", + ItemType: governance.ItemTypeDocument, + SizeBytes: &sizeBytes, + Revision: revision, + Visibility: governance.VisibilityExternal, + VisibilityOrigin: governance.VisibilityOriginPPPublicTag, + SourceChange: governance.SourceChangeUnchanged, + }) + + rawSummary := map[string]any{ + "legacy_status": item.Status, + "source_ids": item.SourceIDs, + } + if item.Status == StatusContentAlias { + rawSummary["canonical_source_url_hash"] = item.CanonicalSourceURLHash + } + rawJSON, err := json.Marshal(rawSummary) + if err != nil { + return ImportPlan{}, fmt.Errorf("marshal legacy record summary: %w", err) + } + versionID := stableID("sv", itemID, revision) + if item.Status == StatusOverLimit { + plan.Versions = append(plan.Versions, governance.SourceItemVersion{ + ID: versionID, + SourceItemID: itemID, + Revision: revision, + ContentState: governance.ContentOverLimit, + DownloadError: "legacy_50_mib_limit", + RawRecordSummary: rawJSON, + }) + continue + } + + content, ok := contentBySHA[item.SHA256] + if !ok { + content = governance.ContentObject{ + ID: stableID("co", item.SHA256), + SHA256: item.SHA256, + SizeBytes: item.Bytes, + State: governance.ContentVerified, + } + contentBySHA[item.SHA256] = content + plan.ContentObjects = append(plan.ContentObjects, content) + } else if content.SizeBytes != item.Bytes { + return ImportPlan{}, fmt.Errorf("same SHA-256 has conflicting byte sizes") + } + + contentID := content.ID + plan.Versions = append(plan.Versions, governance.SourceItemVersion{ + ID: versionID, + SourceItemID: itemID, + Revision: item.SHA256, + ContentObjectID: &contentID, + ContentState: governance.ContentVerified, + RawRecordSummary: rawJSON, + }) + if item.Status != StatusCompleted { + continue + } + if _, exists := publicationByKnowledgeID[item.KnowledgeID]; exists { + return ImportPlan{}, fmt.Errorf("completed records share a knowledge ID") + } + publicationByKnowledgeID[item.KnowledgeID] = struct{}{} + plan.Publications = append(plan.Publications, governance.Publication{ + ID: stableID("tp", snapshot.KBID, item.KnowledgeID), + SourceItemVersionID: versionID, + TestKBID: snapshot.KBID, + KnowledgeID: item.KnowledgeID, + IngestState: governance.IngestUploaded, + ProcessingState: governance.ProcessingCompleted, + RetrievalState: governance.RetrievalUnpublished, + }) + } + return plan, nil +} + +func stableID(prefix string, parts ...string) string { + digest := sha256.New() + for _, part := range parts { + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(part)) + } + return prefix + "-" + hex.EncodeToString(digest.Sum(nil))[:32] +} + +func metadataRevision(item Item) string { + sourceIDs := append([]string(nil), item.SourceIDs...) + sort.Strings(sourceIDs) + parts := []string{item.SourceURLHash, strconv.FormatInt(item.Bytes, 10)} + parts = append(parts, sourceIDs...) + digest := sha256.New() + for _, part := range parts { + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(part)) + } + return "metadata-sha256:" + hex.EncodeToString(digest.Sum(nil)) +} diff --git a/internal/connectors/pp/plan_test.go b/internal/connectors/pp/plan_test.go new file mode 100644 index 0000000..0881236 --- /dev/null +++ b/internal/connectors/pp/plan_test.go @@ -0,0 +1,60 @@ +package pp + +import ( + "strings" + "testing" + + "knowledge-governance/internal/governance" +) + +func TestBuildImportPlanSharesAliasContentWithoutDuplicatePublication(t *testing.T) { + sourceCanonical := strings.Repeat("a", 64) + sourceAlias := strings.Repeat("b", 64) + contentHash := strings.Repeat("c", 64) + snapshot := Snapshot{KBID: "kb-1", Items: []Item{ + {KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: StatusCompleted, KnowledgeID: "knowledge-1"}, + {KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: contentHash, Status: StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical}, + }} + + plan, err := BuildImportPlan(snapshot) + if err != nil { + t.Fatalf("BuildImportPlan() error = %v", err) + } + if len(plan.SourceItems) != 2 || len(plan.Versions) != 2 || len(plan.ContentObjects) != 1 || len(plan.Publications) != 1 { + t.Fatalf("BuildImportPlan() counts = items:%d versions:%d content:%d publications:%d", len(plan.SourceItems), len(plan.Versions), len(plan.ContentObjects), len(plan.Publications)) + } + if plan.Connection.Enabled || plan.Connection.SecretRef != "" || plan.Connection.Type != governance.SourceTypePP { + t.Fatalf("connection safety defaults = %#v", plan.Connection) + } + if plan.Publications[0].RetrievalState != governance.RetrievalUnpublished || plan.Publications[0].ProcessingState != governance.ProcessingCompleted { + t.Fatalf("publication states = %#v", plan.Publications[0]) + } + if *plan.Versions[0].ContentObjectID != plan.ContentObjects[0].ID || *plan.Versions[1].ContentObjectID != plan.ContentObjects[0].ID { + t.Fatal("canonical and alias versions must reference the same content object") + } +} + +func TestBuildImportPlanMapsOverLimitWithoutContentOrPublication(t *testing.T) { + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: strings.Repeat("a", 64), + SourceIDs: []string{"pp-1"}, + Bytes: LegacyMaxBytes + 1, + Status: StatusOverLimit, + }}} + + plan, err := BuildImportPlan(snapshot) + if err != nil { + t.Fatalf("BuildImportPlan() error = %v", err) + } + if len(plan.SourceItems) != 1 || len(plan.Versions) != 1 || len(plan.ContentObjects) != 0 || len(plan.Publications) != 0 { + t.Fatalf("plan counts = source:%d version:%d content:%d publication:%d", len(plan.SourceItems), len(plan.Versions), len(plan.ContentObjects), len(plan.Publications)) + } + if !strings.HasPrefix(plan.SourceItems[0].Revision, "metadata-sha256:") { + t.Fatalf("source revision = %q, want metadata revision", plan.SourceItems[0].Revision) + } + version := plan.Versions[0] + if version.ContentObjectID != nil || version.ContentState != governance.ContentOverLimit || version.DownloadError != "legacy_50_mib_limit" { + t.Fatalf("over-limit version = %#v", version) + } +} diff --git a/internal/connectors/pp/reconcile.go b/internal/connectors/pp/reconcile.go new file mode 100644 index 0000000..6c386cc --- /dev/null +++ b/internal/connectors/pp/reconcile.go @@ -0,0 +1,38 @@ +package pp + +import "fmt" + +type CurrentItem struct { + Revision string +} + +type ReconciliationSummary struct { + Unchanged int + Adopted int + OverLimit int + UploadRequired int +} + +func PlanReconciliation(snapshot Snapshot, current map[string]CurrentItem) (ReconciliationSummary, error) { + if err := ValidateSnapshot(snapshot); err != nil { + return ReconciliationSummary{}, fmt.Errorf("validate reconciliation snapshot: %w", err) + } + summary := ReconciliationSummary{} + for _, item := range snapshot.Items { + if item.Status == StatusOverLimit { + summary.OverLimit++ + continue + } + stored, exists := current[item.SourceURLHash] + if !exists { + summary.Adopted++ + continue + } + if stored.Revision == item.SHA256 { + summary.Unchanged++ + continue + } + summary.UploadRequired++ + } + return summary, nil +} diff --git a/internal/connectors/pp/reconcile_test.go b/internal/connectors/pp/reconcile_test.go new file mode 100644 index 0000000..591cd95 --- /dev/null +++ b/internal/connectors/pp/reconcile_test.go @@ -0,0 +1,58 @@ +package pp + +import ( + "strings" + "testing" +) + +func TestPlanReconciliationSchedulesNoUploadForAdoptedSnapshot(t *testing.T) { + completedSource := strings.Repeat("a", 64) + overLimitSource := strings.Repeat("b", 64) + contentHash := strings.Repeat("c", 64) + overLimit := Item{ + KBID: "kb-1", + SourceURLHash: overLimitSource, + SourceIDs: []string{"pp-2"}, + Bytes: LegacyMaxBytes + 1, + Status: StatusOverLimit, + } + snapshot := Snapshot{KBID: "kb-1", Items: []Item{ + {KBID: "kb-1", SourceURLHash: completedSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: StatusCompleted, KnowledgeID: "knowledge-1"}, + overLimit, + }} + current := map[string]CurrentItem{ + completedSource: {Revision: contentHash}, + overLimitSource: {Revision: metadataRevision(overLimit)}, + } + + summary, err := PlanReconciliation(snapshot, current) + if err != nil { + t.Fatalf("PlanReconciliation() error = %v", err) + } + if summary.UploadRequired != 0 || summary.Unchanged != 1 || summary.OverLimit != 1 { + t.Fatalf("summary = %#v", summary) + } +} + +func TestPlanReconciliationSchedulesUploadWhenSameSourceContentChanges(t *testing.T) { + source := strings.Repeat("a", 64) + oldHash := strings.Repeat("b", 64) + newHash := strings.Repeat("c", 64) + snapshot := Snapshot{KBID: "kb-1", Items: []Item{{ + KBID: "kb-1", + SourceURLHash: source, + SourceIDs: []string{"pp-1"}, + Bytes: 10, + SHA256: newHash, + Status: StatusCompleted, + KnowledgeID: "knowledge-1", + }}} + + summary, err := PlanReconciliation(snapshot, map[string]CurrentItem{source: {Revision: oldHash}}) + if err != nil { + t.Fatalf("PlanReconciliation() error = %v", err) + } + if summary.UploadRequired != 1 || summary.Unchanged != 0 { + t.Fatalf("summary = %#v", summary) + } +} diff --git a/internal/governance/change_set_execution.go b/internal/governance/change_set_execution.go new file mode 100644 index 0000000..d41d48d --- /dev/null +++ b/internal/governance/change_set_execution.go @@ -0,0 +1,26 @@ +package governance + +import "time" + +// ConfirmCatalogChangeSetRequest is the P3 human confirmation: explicit items of +// one planned ChangeSet are executed into publications and processing jobs. +type ConfirmCatalogChangeSetRequest struct { + ChangeSetID string + ItemSeqs []int64 + OperatorRef string + Reason string + InternalTargetKBID string + ExternalTargetKBID string + Now time.Time +} + +type ConfirmCatalogChangeSetResult struct { + ChangeSetID string `json:"change_set_id"` + ConfirmedItems int `json:"confirmed_items"` + UploadsCreated int `json:"uploads_created"` + UploadsExisting int `json:"uploads_existing"` + RemovalsRequested int `json:"removals_requested"` + SkippedNoArtifact int `json:"skipped_no_artifact"` + SkippedNoTarget int `json:"skipped_no_target"` + NoChangeIgnored int `json:"no_change_ignored"` +} diff --git a/internal/governance/domain_model_contract_test.go b/internal/governance/domain_model_contract_test.go new file mode 100644 index 0000000..8212a6a --- /dev/null +++ b/internal/governance/domain_model_contract_test.go @@ -0,0 +1,40 @@ +package governance + +import ( + "os" + "strings" + "testing" +) + +func TestDomainModelContractDeclaresCanonicalObjectsAndCurrentMappings(t *testing.T) { + content, err := os.ReadFile("../../docs/DOMAIN_MODEL.md") + if err != nil { + t.Fatalf("read domain model contract: %v", err) + } + + text := string(content) + for _, required := range []string{ + "## Source", + "## SourceFile", + "## Snapshot", + "## ChangeSet", + "## Job", + "## Release", + "SourceConnection → SourceItem → SourceItemVersion → ContentObject", + "SourceConnection → SourceCatalogSnapshot → SourceCatalogNode", + "SourceCatalogChangeSet → SourceCatalogChangeItem", + "ReleaseBatch → ReleaseExecution → ReleaseItemExecution", + "source_items", + "source_item_versions", + "content_objects", + "source_catalog_snapshots", + "source_catalog_change_sets", + "jobs", + "release_executions", + "Current implementation gaps", + } { + if !strings.Contains(text, required) { + t.Fatalf("domain model contract is missing %q", required) + } + } +} diff --git a/internal/governance/job_failures.go b/internal/governance/job_failures.go new file mode 100644 index 0000000..cc9aaae --- /dev/null +++ b/internal/governance/job_failures.go @@ -0,0 +1,64 @@ +package governance + +import "time" + +// JobFailure is one terminal-attempt dead-letter projection (P6). It never +// replaces the original jobs audit row; it points at it. +type JobFailure struct { + ID string `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + PublicationID string `json:"publication_id,omitempty"` + SourceItemID string `json:"source_item_id,omitempty"` + ErrorCode string `json:"error_code"` + Attempt int `json:"attempt"` + MaxAttempts int `json:"max_attempts_at_failure"` + ChangeSetID string `json:"change_set_id,omitempty"` + Action string `json:"action,omitempty"` + TargetKBID string `json:"target_kb_id,omitempty"` + State string `json:"state"` + CreatedAt time.Time `json:"created_at"` +} + +type JobFailureFilter struct { + State string // open | requeued | abandoned | "" (defaults to open) + Limit int + Offset int +} + +type JobFailurePage struct { + Items []JobFailure `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +const ( + RetryStarted = "started" + RetryAlreadyRunning = "already_running" + RetryNotRetryable = "not_retryable" +) + +// RetryPublicationsBatchRequest is the server-side batch retry (P6): one +// atomic operation covering 1..100 publications with per-item outcomes. +type RetryPublicationsBatchRequest struct { + PublicationIDs []string `json:"publication_ids"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + Now time.Time `json:"-"` +} + +type RetryOutcome struct { + PublicationID string `json:"publication_id"` + Result string `json:"result"` + Detail string `json:"detail,omitempty"` +} + +type RetryPublicationsBatchResult struct { + BatchKey string `json:"batch_key"` + Total int `json:"total"` + Started int `json:"started"` + AlreadyRunning int `json:"already_running"` + NotRetryable int `json:"not_retryable"` + Outcomes []RetryOutcome `json:"outcomes"` +} diff --git a/internal/governance/model.go b/internal/governance/model.go new file mode 100644 index 0000000..fd7edf0 --- /dev/null +++ b/internal/governance/model.go @@ -0,0 +1,285 @@ +package governance + +import ( + "encoding/json" + "time" +) + +type SourceType string + +const ( + SourceTypePP SourceType = "pp" + SourceTypeAList SourceType = "alist" + SourceTypeDingTalk SourceType = "dingtalk" + SourceTypeManual SourceType = "manual" +) + +func (sourceType SourceType) Valid() bool { + switch sourceType { + case SourceTypePP, SourceTypeAList, SourceTypeDingTalk, SourceTypeManual: + return true + default: + return false + } +} + +type ItemType string + +const ( + ItemTypeDocument ItemType = "document" + ItemTypeFile ItemType = "file" + ItemTypeLink ItemType = "link" + ItemTypeVideo ItemType = "video" +) + +type Visibility string + +const ( + VisibilityInternal Visibility = "internal" + VisibilityExternal Visibility = "external" + VisibilityBoth Visibility = "both" + VisibilityBlocked Visibility = "blocked" +) + +type VisibilityOrigin string + +const ( + VisibilityOriginPPPublicTag VisibilityOrigin = "pp_public_tag" + VisibilityOriginConfiguredScope VisibilityOrigin = "configured_scope" + VisibilityOriginManual VisibilityOrigin = "manual" +) + +type SourceChange string + +const ( + SourceChangeUnchanged SourceChange = "unchanged" + SourceChangeAdded SourceChange = "added" + SourceChangeUpdated SourceChange = "updated" + SourceChangeDeleted SourceChange = "deleted" + SourceChangeUnavailable SourceChange = "unavailable" +) + +func (state SourceChange) Valid() bool { + switch state { + case SourceChangeUnchanged, SourceChangeAdded, SourceChangeUpdated, SourceChangeDeleted, SourceChangeUnavailable: + return true + default: + return false + } +} + +type ReviewState string + +const ( + ReviewPending ReviewState = "pending" + ReviewApproved ReviewState = "approved" + ReviewRejected ReviewState = "rejected" + ReviewNotRequired ReviewState = "not_required" +) + +func (state ReviewState) Valid() bool { + switch state { + case ReviewPending, ReviewApproved, ReviewRejected, ReviewNotRequired: + return true + default: + return false + } +} + +type ContentState string + +const ( + ContentUnknown ContentState = "unknown" + ContentDownloading ContentState = "downloading" + ContentVerified ContentState = "verified" + ContentInvalid ContentState = "invalid" + ContentOverLimit ContentState = "over_limit" + ContentFailed ContentState = "failed" +) + +func (state ContentState) Valid() bool { + switch state { + case ContentUnknown, ContentDownloading, ContentVerified, ContentInvalid, ContentOverLimit, ContentFailed: + return true + default: + return false + } +} + +type IngestState string + +const ( + IngestPending IngestState = "pending" + IngestUploading IngestState = "uploading" + IngestUploaded IngestState = "uploaded" + IngestFailed IngestState = "failed" +) + +func (state IngestState) Valid() bool { + switch state { + case IngestPending, IngestUploading, IngestUploaded, IngestFailed: + return true + default: + return false + } +} + +type ProcessingState string + +const ( + ProcessingPending ProcessingState = "pending" + ProcessingProcessing ProcessingState = "processing" + ProcessingFinalizing ProcessingState = "finalizing" + ProcessingCompleted ProcessingState = "completed" + ProcessingFailed ProcessingState = "failed" + ProcessingStalled ProcessingState = "stalled" +) + +func (state ProcessingState) Valid() bool { + switch state { + case ProcessingPending, ProcessingProcessing, ProcessingFinalizing, ProcessingCompleted, ProcessingFailed, ProcessingStalled: + return true + default: + return false + } +} + +type RetrievalState string + +const ( + RetrievalUnpublished RetrievalState = "unpublished" + RetrievalEnabled RetrievalState = "enabled" + RetrievalDisabling RetrievalState = "disabling" + RetrievalDisabled RetrievalState = "disabled" + RetrievalEnabling RetrievalState = "enabling" + RetrievalFailed RetrievalState = "failed" +) + +func (state RetrievalState) Valid() bool { + switch state { + case RetrievalUnpublished, RetrievalEnabled, RetrievalDisabling, RetrievalDisabled, RetrievalEnabling, RetrievalFailed: + return true + default: + return false + } +} + +type AcceptanceState string + +const ( + AcceptancePending AcceptanceState = "pending" + AcceptancePassed AcceptanceState = "passed" + AcceptanceFailed AcceptanceState = "failed" + AcceptanceManualRequired AcceptanceState = "manual_required" +) + +func (state AcceptanceState) Valid() bool { + switch state { + case AcceptancePending, AcceptancePassed, AcceptanceFailed, AcceptanceManualRequired: + return true + default: + return false + } +} + +func ValidSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, character := range value { + if !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')) { + return false + } + } + return true +} + +type SourceConnection struct { + ID string + Type SourceType + Name string + BaseURL string + SecretRef string + Enabled bool + Schedule json.RawMessage + LastHealth json.RawMessage + CreatedAt time.Time + UpdatedAt time.Time +} + +type SourceItem struct { + ID string + SourceConnectionID string + StableExternalID string + CanonicalURIHash string + ParentExternalID string + NormalizedPath string + Title string + ItemType ItemType + SizeBytes *int64 + MIME string + Revision string + UpdatedAtSource *time.Time + Visibility Visibility + VisibilityOrigin VisibilityOrigin + LastSeenScanID string + SourceChange SourceChange + AbsenceStreak int + CreatedAt time.Time + UpdatedAt time.Time +} + +type ContentObject struct { + ID string + SHA256 string + SizeBytes int64 + MIME string + ValidatedFormat string + LocalSpoolRef string + State ContentState + FirstDiscoveredAt time.Time + LastDiscoveredAt time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +type SourceItemVersion struct { + ID string + SourceItemID string + Revision string + ContentObjectID *string + ContentState ContentState + DownloadError string + RawRecordSummary json.RawMessage + SourceUpdatedAt *time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +type Publication struct { + ID string + SourceItemVersionID string + TestKBID string + KnowledgeID string + IngestState IngestState + ProcessingState ProcessingState + RetrievalState RetrievalState + ParserGeneration string + IndexGeneration string + LastError string + RetryCount int + CreatedAt time.Time + UpdatedAt time.Time +} + +type AcceptanceResult struct { + ID string + PublicationID string + RunID string + CheckType string + State AcceptanceState + PolicyVersion string + EvidenceSummary json.RawMessage + ObservedAt time.Time + CreatedAt time.Time +} diff --git a/internal/governance/model_test.go b/internal/governance/model_test.go new file mode 100644 index 0000000..657066b --- /dev/null +++ b/internal/governance/model_test.go @@ -0,0 +1,57 @@ +package governance + +import "testing" + +func TestDeclaredWorkflowStatesAreValid(t *testing.T) { + tests := []struct { + name string + valid bool + }{ + {"source type", SourceTypeManual.Valid() && SourceTypePP.Valid() && !SourceType("__invalid__").Valid()}, + {"source change", SourceChangeAdded.Valid() && !SourceChange("__invalid__").Valid()}, + {"review", ReviewPending.Valid() && !ReviewState("__invalid__").Valid()}, + {"content", ContentVerified.Valid() && !ContentState("__invalid__").Valid()}, + {"ingest", IngestUploaded.Valid() && !IngestState("__invalid__").Valid()}, + {"processing", ProcessingCompleted.Valid() && !ProcessingState("__invalid__").Valid()}, + {"retrieval", RetrievalEnabled.Valid() && !RetrievalState("__invalid__").Valid()}, + {"acceptance", AcceptanceManualRequired.Valid() && AcceptancePassed.Valid() && AcceptanceFailed.Valid() && !AcceptanceState("__invalid__").Valid()}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !tt.valid { + t.Fatal("declared state must be valid and unknown state must be invalid") + } + }) + } +} + +func TestValidSHA256RequiresCanonicalDigest(t *testing.T) { + valid := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + for _, tt := range []struct { + name string + value string + want bool + }{ + {"canonical", valid, true}, + {"short", valid[:63], false}, + {"uppercase", "A" + valid[1:], false}, + {"non hex", "g" + valid[1:], false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := ValidSHA256(tt.value); got != tt.want { + t.Fatalf("ValidSHA256() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestModelsKeepSourceVersionContentAndPublicationSeparate(t *testing.T) { + contentID := "content-1" + item := SourceItem{ID: "item-1", StableExternalID: "pp-42", SourceChange: SourceChangeAdded} + version := SourceItemVersion{ID: "version-1", SourceItemID: item.ID, ContentObjectID: &contentID, ContentState: ContentVerified} + publication := Publication{ID: "publication-1", SourceItemVersionID: version.ID, IngestState: IngestUploaded, ProcessingState: ProcessingCompleted, RetrievalState: RetrievalEnabled} + + if version.SourceItemID != item.ID || *version.ContentObjectID != contentID || publication.SourceItemVersionID != version.ID { + t.Fatal("source, version, content, and publication identities must remain independently linked") + } +} diff --git a/internal/governance/monitor.go b/internal/governance/monitor.go new file mode 100644 index 0000000..6dd254f --- /dev/null +++ b/internal/governance/monitor.go @@ -0,0 +1,117 @@ +package governance + +import "time" + +type MonitorHost struct { + CPUPercent float64 `json:"cpu_percent"` + MemoryUsedBytes int64 `json:"memory_used_bytes"` + MemoryTotalBytes int64 `json:"memory_total_bytes"` + DiskUsedBytes int64 `json:"disk_used_bytes"` + DiskTotalBytes int64 `json:"disk_total_bytes"` + GPUState string `json:"gpu_state"` + GPUDetail string `json:"gpu_detail"` + Simulated bool `json:"simulated"` +} + +type MonitorService struct { + Key string `json:"key"` + Name string `json:"name"` + State string `json:"state"` + Detail string `json:"detail"` + LatencyMS int `json:"latency_ms"` + ObservedAt time.Time `json:"observed_at"` + Simulated bool `json:"simulated"` +} + +// ServiceHealth (P10) separates "process reachable" (Liveness) from "can +// accept work" (Readiness). not_configured and not_applicable are explicit and +// never collapse into unknown. +type ServiceHealth struct { + Key string `json:"key"` + Name string `json:"name"` + Liveness string `json:"liveness"` + Readiness string `json:"readiness"` + Reason string `json:"reason"` + LatencyMS int64 `json:"latency_ms"` + ObservedAt time.Time `json:"observed_at"` + LastSuccessAt *time.Time `json:"last_success_at"` + ConsecutiveFailures int `json:"consecutive_failures"` + Configured bool `json:"configured"` + Applicable bool `json:"applicable"` +} + +type KnowledgeBaseProgress struct { + ID string `json:"id"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Imported int `json:"imported"` + Completed int `json:"completed"` + Pending int `json:"pending"` + Processing int `json:"processing"` + Finalizing int `json:"finalizing"` + Failed int `json:"failed"` + Cancelled int `json:"cancelled"` + InternalImported int `json:"internal_imported"` + InternalCompleted int `json:"internal_completed"` + ExternalImported int `json:"external_imported"` + ExternalCompleted int `json:"external_completed"` +} + +// JobWindowMetrics reports a frozen statistical window; insufficient samples +// stay null and are never rendered as 0. +type JobWindowMetrics struct { + WindowSeconds int `json:"window_seconds"` + JobsStarted int `json:"jobs_started"` + JobsCompleted int `json:"jobs_completed"` + JobsFailed int `json:"jobs_failed"` + SuccessRate *float64 `json:"success_rate"` + FailureRate *float64 `json:"failure_rate"` + DurationAvgSeconds *float64 `json:"duration_avg_seconds"` + DurationP50Seconds *float64 `json:"duration_p50_seconds"` + DurationP95Seconds *float64 `json:"duration_p95_seconds"` +} + +type QueueGauges struct { + Pending int `json:"pending"` + Running int `json:"running"` + Retry int `json:"retry"` + Failed int `json:"failed"` + OldestPendingAgeSeconds *float64 `json:"oldest_pending_age_seconds"` +} + +type RunMetrics struct { + ObservedAt time.Time `json:"observed_at"` + ActiveWorkers int `json:"active_workers"` + WorkerSource string `json:"worker_source"` + Queue QueueGauges `json:"queue"` + ShortWindow JobWindowMetrics `json:"short_window"` + LongWindow JobWindowMetrics `json:"long_window"` +} + +type MonitorOverview struct { + Mode string `json:"mode"` + Simulated bool `json:"simulated"` + ObservedAt time.Time `json:"observed_at"` + Host MonitorHost `json:"host"` + Services []MonitorService `json:"services"` + KnowledgeBases []KnowledgeBaseProgress `json:"knowledge_bases"` + Health []ServiceHealth `json:"health"` +} + +type KnowledgeMonitorFile struct { + Name string `json:"name"` + Directory string `json:"directory"` + Visibility string `json:"visibility"` + Status string `json:"status"` + StageDetail string `json:"stage_detail"` + ElapsedSecond int `json:"elapsed_seconds"` + ErrorCode string `json:"error_code"` +} + +type KnowledgeMonitorFilePage struct { + Items []KnowledgeMonitorFile `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + SampleOnly bool `json:"sample_only"` +} diff --git a/internal/governance/release_audit_chain.go b/internal/governance/release_audit_chain.go new file mode 100644 index 0000000..e55fe13 --- /dev/null +++ b/internal/governance/release_audit_chain.go @@ -0,0 +1,63 @@ +package governance + +import "time" + +// ReleaseAuditChain is the R6 read-only evidence projection: one execution can +// be traced to its batch, the confirmed release change set, the bound source +// snapshots and catalog change sets, and every item's review, acceptance run +// and source identity. Only safe identifiers, labels, counts and timestamps +// travel here — never bundle paths, credentials, or signed URLs. +type ReleaseAuditChain struct { + ExecutionID string `json:"execution_id"` + Operation string `json:"operation"` + State string `json:"state"` + FailureLevel string `json:"failure_level"` + BatchID string `json:"release_batch_id"` + BatchName string `json:"release_batch_name"` + CreatedBy string `json:"created_by"` + SealedBy string `json:"sealed_by"` + AuthorizedBy string `json:"authorized_by"` + ReleaseChangeSetID string `json:"release_change_set_id"` + ChangeSetName string `json:"release_change_set_name"` + ChangeSetNumber int `json:"release_change_set_number"` + ChangeSetState string `json:"release_change_set_state"` + ChangeSetCreatedBy string `json:"release_change_set_created_by"` + ChangeSetConfirmedBy string `json:"release_change_set_confirmed_by"` + ChangeSetConfirmedAt *time.Time `json:"release_change_set_confirmed_at"` + ManifestSHA256 string `json:"manifest_sha256"` + Snapshots []AuditSnapshot `json:"snapshots"` + CatalogChangeSets []AuditCatalogSet `json:"catalog_change_sets"` + PriorExecutionID string `json:"prior_execution_id"` + RollbackExecutionID string `json:"rollback_execution_id"` + Items []AuditChainItem `json:"items"` +} + +type AuditSnapshot struct { + ID string `json:"id"` + SourceName string `json:"source_name"` + NodeCount int `json:"node_count"` + FileCount int `json:"file_count"` + ObservedAt time.Time `json:"observed_at"` +} + +type AuditCatalogSet struct { + ID string `json:"id"` + State string `json:"state"` + AddCount int `json:"add_count"` + UpdateCount int `json:"update_count"` + MoveCount int `json:"move_count"` + DeleteCount int `json:"delete_count"` +} + +type AuditChainItem struct { + PublicationID string `json:"publication_id"` + Title string `json:"title"` + Action string `json:"action"` + ItemState string `json:"item_state"` + ReviewID string `json:"review_id"` + AcceptanceRunID string `json:"acceptance_run_id"` + SourceItemID string `json:"source_item_id"` + SourceVersionID string `json:"source_item_version_id"` + ProductionRevision string `json:"production_revision"` + ErrorCode string `json:"error_code"` +} diff --git a/internal/governance/release_batch.go b/internal/governance/release_batch.go new file mode 100644 index 0000000..13e6bec --- /dev/null +++ b/internal/governance/release_batch.go @@ -0,0 +1,75 @@ +package governance + +import "time" + +type ReleaseBatchState string + +const ( + ReleaseBatchDraft ReleaseBatchState = "draft" + ReleaseBatchSealed ReleaseBatchState = "sealed" + ReleaseBatchCancelled ReleaseBatchState = "cancelled" +) + +func (state ReleaseBatchState) Valid() bool { + switch state { + case ReleaseBatchDraft, ReleaseBatchSealed, ReleaseBatchCancelled: + return true + default: + return false + } +} + +type ReleaseBatch struct { + ID string `json:"id"` + Name string `json:"name"` + State ReleaseBatchState `json:"state"` + PolicyVersion string `json:"policy_version"` + CreatedBy string `json:"created_by"` + SealedBy string `json:"sealed_by"` + ItemCount int `json:"item_count"` + CreatedAt time.Time `json:"created_at"` + SealedAt *time.Time `json:"sealed_at"` + ReleaseChangeSetID string `json:"release_change_set_id,omitempty"` +} + +type CreateReleaseBatchRequest struct { + Name string + CreatedBy string + PublicationIDs []string + DeletePublicationIDs []string + // ReleaseChangeSetID (R1): a formal batch must cite a CONFIRMED persistent + // release change set; empty keeps legacy ad-hoc batches draft-only. + ReleaseChangeSetID string + Now time.Time +} + +type SealReleaseBatchRequest struct { + BatchID string + SealedBy string + Now time.Time +} + +type ReleaseBatchFilter struct { + State string + Limit int + Offset int +} + +type ReleaseBatchPage struct { + Items []ReleaseBatch `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type ReleaseBatchItemView struct { + PublicationID string `json:"publication_id"` + Title string `json:"title"` + DecidedVisibility Visibility `json:"decided_visibility"` + Action string `json:"action"` +} + +type ReleaseBatchDetail struct { + ReleaseBatch + Items []ReleaseBatchItemView `json:"items"` +} diff --git a/internal/governance/release_change_set.go b/internal/governance/release_change_set.go new file mode 100644 index 0000000..bd78815 --- /dev/null +++ b/internal/governance/release_change_set.go @@ -0,0 +1,63 @@ +package governance + +import "time" + +// ReleaseChangeSet is the R1 persistent, immutable release plan: generated from +// the live plan exactly once, bound to the source snapshots it was computed +// against, hash-sealed, and confirmed before batches may cite it. +type ReleaseChangeSet struct { + ID string `json:"id"` + Number int `json:"number"` + Name string `json:"name"` + State string `json:"state"` + SourceSnapshotIDs []string `json:"source_snapshot_ids"` + SourceChangeSetIDs []string `json:"source_change_set_ids"` + AcceptanceCutoff time.Time `json:"acceptance_cutoff"` + ProductionBaselineRef string `json:"production_baseline_ref"` + ItemCount int `json:"item_count"` + AddCount int `json:"add_count"` + UpdateCount int `json:"update_count"` + DeleteCount int `json:"delete_count"` + BlockedCount int `json:"blocked_count"` + ManifestSHA256 string `json:"manifest_sha256"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + ConfirmedBy string `json:"confirmed_by"` + ConfirmedAt *time.Time `json:"confirmed_at"` +} + +type GenerateReleaseChangeSetRequest struct { + Name string + CreatedBy string + Now time.Time +} + +type ConfirmReleaseChangeSetRequest struct { + ChangeSetID string + ConfirmedBy string + Now time.Time +} + +type ReleaseChangeSetItem struct { + Seq int `json:"seq"` + PublicationID string `json:"publication_id"` + Title string `json:"title"` + TargetScope string `json:"target_scope"` + Action string `json:"action"` + TestState string `json:"test_state"` + ProductionState string `json:"production_state"` + CanRelease bool `json:"can_release"` + Blocker string `json:"blocker"` +} + +type ReleaseChangeSetDetail struct { + ReleaseChangeSet + Items []ReleaseChangeSetItem `json:"items"` +} + +type ReleaseChangeSetPage struct { + Items []ReleaseChangeSet `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} diff --git a/internal/governance/release_execution.go b/internal/governance/release_execution.go new file mode 100644 index 0000000..63fbab0 --- /dev/null +++ b/internal/governance/release_execution.go @@ -0,0 +1,115 @@ +package governance + +import "time" + +type ReleaseOperation string + +const ( + ReleaseOperationPublish ReleaseOperation = "publish" + ReleaseOperationRollback ReleaseOperation = "rollback" +) + +func (operation ReleaseOperation) Valid() bool { + return operation == ReleaseOperationPublish || operation == ReleaseOperationRollback +} + +type ReleaseExecutionState string + +const ( + ReleaseExecutionPending ReleaseExecutionState = "pending" + ReleaseExecutionRunning ReleaseExecutionState = "running" + // R4: a rollback execution compensates in rolling_back and terminates in + // rolled_back (all compensated) or rollback_failed (manual action needed). + ReleaseExecutionRollingBack ReleaseExecutionState = "rolling_back" + ReleaseExecutionRolledBack ReleaseExecutionState = "rolled_back" + ReleaseExecutionRollbackFailed ReleaseExecutionState = "rollback_failed" + ReleaseExecutionCompleted ReleaseExecutionState = "completed" + ReleaseExecutionFailed ReleaseExecutionState = "failed" +) + +func (state ReleaseExecutionState) Valid() bool { + switch state { + case ReleaseExecutionPending, ReleaseExecutionRunning, ReleaseExecutionRollingBack, + ReleaseExecutionRolledBack, ReleaseExecutionRollbackFailed, + ReleaseExecutionCompleted, ReleaseExecutionFailed: + return true + default: + return false + } +} + +type ReleaseExecution struct { + ID string `json:"id"` + ReleaseBatchID string `json:"release_batch_id"` + Operation ReleaseOperation `json:"operation"` + PriorExecutionID string `json:"prior_execution_id"` + State ReleaseExecutionState `json:"state"` + FailureLevel string `json:"failure_level"` + RestoreTrigger string `json:"restore_trigger"` + ReleaseChangeSetID string `json:"release_change_set_id"` + TargetEnvironment string `json:"target_environment"` + AuthorizedBy string `json:"authorized_by"` + AuthorizationCode string `json:"authorization_code"` + JobID string `json:"job_id"` + ManifestSHA256 string `json:"manifest_sha256"` + ItemCount int `json:"item_count"` + BatchSize int `json:"batch_size"` + CurrentBatch int `json:"current_batch"` + TotalBatches int `json:"total_batches"` + CompletedItemCount int `json:"completed_item_count"` + FailedItemCount int `json:"failed_item_count"` + ErrorCode string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` +} + +type AuthorizeReleaseBatchRequest struct { + BatchID string + AuthorizedBy string + AuthorizationCode string + Now time.Time +} + +type AuthorizeReleaseRollbackRequest struct { + PriorExecutionID string + AuthorizedBy string + AuthorizationCode string + Now time.Time +} + +type RetryReleaseExecutionRequest struct { + ExecutionID string + Now time.Time +} + +type ReleaseExecutionFilter struct { + Operation string + State string + Limit int + Offset int +} + +type ReleaseExecutionPage struct { + Items []ReleaseExecution + Total int + Limit int + Offset int +} + +type ReleaseItemExecutionView struct { + PublicationID string `json:"publication_id"` + Title string `json:"title"` + Action string `json:"action"` + State string `json:"state"` + PreviousProductionRevision string `json:"previous_production_revision"` + ProductionRevision string `json:"production_revision"` + ErrorCode string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at"` +} + +type ReleaseExecutionDetail struct { + ReleaseExecution + Items []ReleaseItemExecutionView `json:"items"` +} diff --git a/internal/governance/release_plan.go b/internal/governance/release_plan.go new file mode 100644 index 0000000..faf4b3b --- /dev/null +++ b/internal/governance/release_plan.go @@ -0,0 +1,25 @@ +package governance + +type ReleasePlanItem struct { + PublicationID string `json:"publication_id"` + Title string `json:"title"` + TargetScope string `json:"target_scope"` + Action string `json:"action"` + TestState string `json:"test_state"` + ProductionState string `json:"production_state"` + CanRelease bool `json:"can_release"` + Blocker string `json:"blocker"` +} + +type ReleasePlanSummary struct { + Add int `json:"add"` + Update int `json:"update"` + Delete int `json:"delete"` + Blocked int `json:"blocked"` + Unchanged int `json:"unchanged"` +} + +type ReleasePlan struct { + Items []ReleasePlanItem `json:"items"` + Summary ReleasePlanSummary `json:"summary"` +} diff --git a/internal/governance/review.go b/internal/governance/review.go new file mode 100644 index 0000000..734a08d --- /dev/null +++ b/internal/governance/review.go @@ -0,0 +1,67 @@ +package governance + +import "time" + +type ReviewGateStatus string + +const ( + ReviewGateReady ReviewGateStatus = "ready" + ReviewGateBlocked ReviewGateStatus = "blocked" +) + +func (status ReviewGateStatus) Valid() bool { + return status == ReviewGateReady || status == ReviewGateBlocked +} + +type ReviewCandidate struct { + PublicationID string `json:"publication_id"` + Title string `json:"title"` + SourceType SourceType `json:"source_type"` + CurrentVisibility Visibility `json:"current_visibility"` + AcceptanceRunID string `json:"-"` + KnowledgeExists AcceptanceState `json:"knowledge_exists"` + ChunksPresent AcceptanceState `json:"chunks_present"` + RetrievalHit AcceptanceState `json:"retrieval_hit"` + GateStatus ReviewGateStatus `json:"gate_status"` + GateCode string `json:"gate_code"` + ReviewState ReviewState `json:"review_state"` + DecidedVisibility string `json:"decided_visibility"` + DecisionCode string `json:"decision_code"` + ReviewerRef string `json:"reviewer_ref"` + ReviewedAt *time.Time `json:"reviewed_at"` + PublicationUpdated time.Time `json:"updated_at"` +} + +type ReviewCandidateFilter struct { + ReviewState string + Limit int + Offset int +} + +type ReviewCandidatePage struct { + Items []ReviewCandidate `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type PublicationReview struct { + ID string `json:"id"` + PublicationID string `json:"publication_id"` + Decision ReviewState `json:"decision"` + DecidedVisibility Visibility `json:"decided_visibility"` + ReviewerRef string `json:"reviewer_ref"` + DecisionCode string `json:"decision_code"` + AcceptanceRunID string `json:"-"` + PublicationUpdated time.Time `json:"-"` + CreatedAt time.Time `json:"created_at"` +} + +type ReviewPublicationRequest struct { + PublicationID string + Decision ReviewState + DecidedVisibility Visibility + ReviewerRef string + DecisionCode string + Now time.Time +} diff --git a/internal/governance/scope_rule.go b/internal/governance/scope_rule.go new file mode 100644 index 0000000..8602818 --- /dev/null +++ b/internal/governance/scope_rule.go @@ -0,0 +1,80 @@ +package governance + +import ( + "errors" + "strings" + "time" + "unicode" +) + +type ScopeTargetType string + +const ( + ScopeTargetDirectory ScopeTargetType = "directory" + ScopeTargetItem ScopeTargetType = "item" +) + +type ScopeRuleMutation struct { + TargetType ScopeTargetType + ConnectionID string + DirectoryPath string + SourceItemID string + Visibility Visibility + OperatorRef string + Reason string + Clear bool + Now time.Time +} + +type ScopeRule struct { + TargetType ScopeTargetType `json:"target_type"` + ConnectionID string `json:"connection_id,omitempty"` + DirectoryPath string `json:"directory_path,omitempty"` + SourceItemID string `json:"source_item_id,omitempty"` + Visibility Visibility `json:"visibility"` + UpdatedBy string `json:"updated_by"` + Reason string `json:"reason"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (request ScopeRuleMutation) Validate() error { + if request.TargetType != ScopeTargetDirectory && request.TargetType != ScopeTargetItem { + return errors.New("scope target must be directory or item") + } + if request.TargetType == ScopeTargetDirectory { + if strings.TrimSpace(request.ConnectionID) == "" || request.SourceItemID != "" || !validScopePath(request.DirectoryPath) { + return errors.New("directory scope target is invalid") + } + } else if strings.TrimSpace(request.SourceItemID) == "" || request.ConnectionID != "" || request.DirectoryPath != "" { + return errors.New("item scope target is invalid") + } + if !request.Clear && request.Visibility != VisibilityInternal && request.Visibility != VisibilityExternal { + return errors.New("visibility must be internal or external") + } + if request.Clear && request.Visibility != "" { + return errors.New("clear scope must not include visibility") + } + operator := strings.TrimSpace(request.OperatorRef) + reason := strings.TrimSpace(request.Reason) + if operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() { + return errors.New("operator, reason, and time are required") + } + return nil +} + +func validScopePath(value string) bool { + if len(value) > 2048 || strings.HasPrefix(value, "/") || strings.Contains(value, "\\") { + return false + } + for _, part := range strings.Split(value, "/") { + if part == "." || part == ".." { + return false + } + } + for _, value := range value { + if unicode.IsControl(value) { + return false + } + } + return true +} diff --git a/internal/governance/scope_rule_test.go b/internal/governance/scope_rule_test.go new file mode 100644 index 0000000..a6f4446 --- /dev/null +++ b/internal/governance/scope_rule_test.go @@ -0,0 +1,42 @@ +package governance + +import ( + "testing" + "time" +) + +func TestValidateScopeRuleMutationAcceptsDirectoryRootAndItemOverride(t *testing.T) { + now := time.Now() + for _, request := range []ScopeRuleMutation{ + {TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "仅供内部使用", Now: now}, + {TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "产品/说明书", Visibility: VisibilityExternal, OperatorRef: "engineer-1", Reason: "官网公开资料", Now: now}, + {TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "文件单独包含内部参数", Now: now}, + } { + if err := request.Validate(); err != nil { + t.Errorf("Validate(%#v) error = %v", request, err) + } + } +} + +func TestValidateScopeRuleMutationRejectsAmbiguousOrUnsafeTargets(t *testing.T) { + valid := ScopeRuleMutation{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "范围调整", Now: time.Now()} + cases := []ScopeRuleMutation{ + {}, + {TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()}, + {TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "/private", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()}, + {TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "a/../private", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()}, + {TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: Visibility("public"), OperatorRef: "engineer", Reason: "x", Now: time.Now()}, + {TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "", Reason: "x", Now: time.Now()}, + {TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "", Now: time.Now()}, + } + for _, request := range cases { + if err := request.Validate(); err == nil { + t.Errorf("Validate(%#v) error=nil", request) + } + } + valid.Clear = true + valid.Visibility = "" + if err := valid.Validate(); err != nil { + t.Fatalf("clear validation error = %v", err) + } +} diff --git a/internal/governance/source_asset.go b/internal/governance/source_asset.go new file mode 100644 index 0000000..55bf38c --- /dev/null +++ b/internal/governance/source_asset.go @@ -0,0 +1,45 @@ +package governance + +import "time" + +type SourceAsset struct { + CanRetry bool `json:"can_retry"` + Title string `json:"title"` + DirectoryPath string `json:"directory_path"` + ConnectionID string `json:"connection_id"` + VisibilityOrigin string `json:"visibility_origin"` + ScopeRulePath string `json:"scope_rule_path"` + IngestState string `json:"ingest_state"` + LastError string `json:"last_error"` + PublicationID string `json:"publication_id"` + ID string `json:"id"` + SourceType SourceType `json:"source_type"` + ConnectionName string `json:"connection_name"` + SizeBytes int64 `json:"size_bytes"` + Visibility Visibility `json:"visibility"` + SourceChange SourceChange `json:"source_change"` + ContentState ContentState `json:"content_state"` + LegacyStatus string `json:"legacy_status"` + HasPublication bool `json:"has_publication"` + ProcessingState string `json:"processing_state"` + RetrievalState string `json:"retrieval_state"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SourceAssetFilter struct { + State string + Limit int + Offset int +} + +type SourceAssetSummary struct { + Total int `json:"total"` + Completed int `json:"completed"` + ContentAlias int `json:"content_alias"` + OverLimit int `json:"over_limit"` +} + +type SourceAssetPage struct { + Items []SourceAsset `json:"items"` + Summary SourceAssetSummary `json:"summary"` +} diff --git a/internal/governance/source_asset_test.go b/internal/governance/source_asset_test.go new file mode 100644 index 0000000..08ff4e2 --- /dev/null +++ b/internal/governance/source_asset_test.go @@ -0,0 +1,19 @@ +package governance + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestSourceAssetIncludesDocumentNavigationMetadata(t *testing.T) { + data, err := json.Marshal(SourceAsset{}) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"title", "directory_path", "connection_id", "visibility_origin", "ingest_state", "last_error", "publication_id", "can_retry"} { + if !strings.Contains(string(data), `"`+field+`"`) { + t.Errorf("missing document field %s", field) + } + } +} diff --git a/internal/governance/source_catalog.go b/internal/governance/source_catalog.go new file mode 100644 index 0000000..a371ce6 --- /dev/null +++ b/internal/governance/source_catalog.go @@ -0,0 +1,154 @@ +package governance + +import "time" + +type CatalogSource struct { + ConnectionID string `json:"connection_id"` + SourceType SourceType `json:"source_type"` + Name string `json:"name"` + RootNodeID string `json:"root_node_id"` + FolderCount int `json:"folder_count"` + FileCount int `json:"file_count"` + ObservedAt time.Time `json:"observed_at"` + SnapshotOnly bool `json:"snapshot_only"` + SnapshotStatus string `json:"snapshot_status"` + LastRefreshDurationMS int64 `json:"last_refresh_duration_ms"` +} + +type CatalogBreadcrumb struct { + NodeID string `json:"node_id"` + Name string `json:"name"` +} + +type CatalogNode struct { + NodeID string `json:"node_id"` + ParentNodeID string `json:"parent_node_id,omitempty"` + NodeType string `json:"node_type"` + Name string `json:"name"` + DirectChildCount int `json:"direct_child_count"` + SubtreeFileCount int `json:"subtree_file_count"` + SubtreeBytes int64 `json:"subtree_bytes"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + ModifiedAt *time.Time `json:"modified_at,omitempty"` + ContentSHA256 *string `json:"content_sha256,omitempty"` + Visibility string `json:"visibility"` + ImportedCount int `json:"imported_count"` + ParsedCount int `json:"parsed_count"` + ActiveCount int `json:"active_count"` + FailedCount int `json:"failed_count"` + RoutableCount int `json:"routable_count"` +} + +type CatalogNodeFilter struct { + ConnectionID string + ParentNodeID string + Limit int + Offset int +} + +type CatalogNodePage struct { + Current CatalogNode `json:"current"` + Breadcrumbs []CatalogBreadcrumb `json:"breadcrumbs"` + Items []CatalogNode `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type CatalogSnapshot struct { + SnapshotID string `json:"snapshot_id"` + ConnectionID string `json:"connection_id"` + RootNodeID string `json:"root_node_id"` + SourceName string `json:"source_name"` + NodeCount int `json:"node_count"` + FolderCount int `json:"folder_count"` + FileCount int `json:"file_count"` + ObservedAt time.Time `json:"observed_at"` + CreatedAt time.Time `json:"created_at"` + IsHead bool `json:"is_head"` +} + +type CatalogSnapshotPage struct { + Items []CatalogSnapshot `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type CatalogSnapshotNodeFilter struct { + ConnectionID string + SnapshotID string + ParentNodeID string + Limit int + Offset int +} + +type CatalogRouteRequest struct { + ConnectionID string + NodeIDs []string + Visibility Visibility + TargetKnowledgeBaseID string + TargetKnowledgeBaseIDs []string + OperatorRef string + Reason string + Now time.Time +} + +// CatalogRouteResult describes the desired-state write only. S7 guarantees the +// annotation never creates publications or jobs; counts explain what the +// subsequent ChangeSet/P3 pipeline will be able to act on. +type CatalogRouteResult struct { + SelectedFiles int `json:"selected_files"` + RoutableFiles int `json:"routable_files"` + UnboundFiles int `json:"unbound_files"` + HashUnknownFiles int `json:"hash_unknown_files"` + UnavailableFiles int `json:"unavailable_files"` +} + +type CatalogChangeSetFilter struct { + ConnectionID string + Action string + Limit int + Offset int +} + +type CatalogChangeSet struct { + ID string `json:"id"` + ConnectionID string `json:"connection_id"` + SourceName string `json:"source_name"` + State string `json:"state"` + AddCount int `json:"add_count"` + UpdateCount int `json:"update_count"` + MoveCount int `json:"move_count"` + MoveScopeCount int `json:"move_scope_count"` + DeleteCount int `json:"delete_count"` + NoChangeCount int `json:"no_change_count"` + CreatedAt time.Time `json:"created_at"` +} + +type CatalogChangeItem struct { + Action string `json:"action"` + Title string `json:"title"` + OldPath string `json:"old_path,omitempty"` + NewPath string `json:"new_path,omitempty"` + OldSizeBytes *int64 `json:"old_size_bytes,omitempty"` + NewSizeBytes *int64 `json:"new_size_bytes,omitempty"` + OldModified *time.Time `json:"old_modified_at,omitempty"` + NewModified *time.Time `json:"new_modified_at,omitempty"` + OldContentSHA256 string `json:"old_content_sha256,omitempty"` + NewContentSHA256 string `json:"new_content_sha256,omitempty"` + ReasonCode string `json:"reason_code"` + Seq int64 `json:"seq"` + OldScope string `json:"old_scope,omitempty"` + NewScope string `json:"new_scope,omitempty"` + OldVersionID string `json:"old_version_id,omitempty"` + NewVersionID string `json:"new_version_id,omitempty"` +} + +type CatalogChangeSetDetail struct { + CatalogChangeSet + Items []CatalogChangeItem `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} diff --git a/internal/governance/source_connection.go b/internal/governance/source_connection.go new file mode 100644 index 0000000..647b444 --- /dev/null +++ b/internal/governance/source_connection.go @@ -0,0 +1,84 @@ +package governance + +import "time" + +type SourceHealthStatus string + +const ( + SourceHealthUnknown SourceHealthStatus = "unknown" + SourceHealthHealthy SourceHealthStatus = "healthy" + SourceHealthDegraded SourceHealthStatus = "degraded" + SourceHealthUnhealthy SourceHealthStatus = "unhealthy" +) + +func (status SourceHealthStatus) Valid() bool { + switch status { + case SourceHealthUnknown, SourceHealthHealthy, SourceHealthDegraded, SourceHealthUnhealthy: + return true + default: + return false + } +} + +type SourceConnectionView struct { + ID string `json:"id"` + SourceType SourceType `json:"source_type"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + ScheduleConfigured bool `json:"schedule_configured"` + HealthStatus SourceHealthStatus `json:"health_status"` + ItemCount int `json:"item_count"` + UpdatedAt time.Time `json:"updated_at"` + NextRefreshAt *time.Time `json:"next_refresh_at"` + LastScan *SourceScanSummary `json:"last_scan"` +} + +// SourceConnectionRevision is one validated/active/retired settings revision of +// a source connection (V1). The JSON surface uses endpoint/credential_ref: the +// credential field is a reference NAME only — real secret values never live in +// this table and can never be read back through any endpoint. +type SourceConnectionRevision struct { + ID string `json:"id"` + RevisionNumber int `json:"revision_number"` + State string `json:"state"` + Name string `json:"name"` + BaseURL string `json:"endpoint"` + SecretRef string `json:"credential_ref"` + Enabled bool `json:"enabled"` + RefreshIntervalSeconds string `json:"refresh_interval_seconds"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + ValidatedAt *time.Time `json:"validated_at"` + ActivatedAt *time.Time `json:"activated_at"` + RetiredAt *time.Time `json:"retired_at"` +} + +type SourceScanSummary struct { + ID string `json:"id"` + State SourceScanState `json:"state"` + ObservedItems int `json:"observed_items"` + UnchangedItems int `json:"unchanged_items"` + AddedItems int `json:"added_items"` + UpdatedItems int `json:"updated_items"` + OverLimitItems int `json:"over_limit_items"` + MissingItems int `json:"missing_items"` + PlannedUploads int `json:"planned_uploads"` + ErrorCode string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at"` +} + +type SourceConnectionSummary struct { + Total int `json:"total"` + Enabled int `json:"enabled"` + Disabled int `json:"disabled"` + Healthy int `json:"healthy"` + Degraded int `json:"degraded"` + Unhealthy int `json:"unhealthy"` + Unknown int `json:"unknown"` +} + +type SourceConnectionPage struct { + Items []SourceConnectionView `json:"items"` + Summary SourceConnectionSummary `json:"summary"` +} diff --git a/internal/governance/source_connection_projection_test.go b/internal/governance/source_connection_projection_test.go new file mode 100644 index 0000000..aa9b315 --- /dev/null +++ b/internal/governance/source_connection_projection_test.go @@ -0,0 +1,34 @@ +package governance + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// The frontend source-settings parser requires the next_refresh_at KEY to +// exist whenever a connection is listed — null is a first-class value ("manual +// only"), not something to omit. A `omitempty` here once blanked the entire +// settings page in the deployed stack because every source defaulted to +// manual-only and the strict parser rejected the key-less payload. +func TestSourceConnectionViewAlwaysProjectsNextRefreshAt(t *testing.T) { + view := SourceConnectionView{ID: "conn-1", SourceType: SourceTypePP, Name: "PP", Enabled: true, ScheduleConfigured: false, HealthStatus: SourceHealthHealthy, ItemCount: 621, UpdatedAt: time.Now().UTC()} + payload, err := json.Marshal([]SourceConnectionView{view}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(payload), `"next_refresh_at":null`) { + t.Fatalf("manual-only connection must project next_refresh_at:null, got %s", payload) + } + + at := time.Now().UTC().Truncate(time.Second) + view.NextRefreshAt = &at + payload, err = json.Marshal([]SourceConnectionView{view}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(payload), `"next_refresh_at":"`) { + t.Fatalf("scheduled connection must project a timestamp, got %s", payload) + } +} diff --git a/internal/governance/source_scan.go b/internal/governance/source_scan.go new file mode 100644 index 0000000..968cc86 --- /dev/null +++ b/internal/governance/source_scan.go @@ -0,0 +1,92 @@ +package governance + +import "time" + +type SourceScanState string + +const ( + SourceScanPending SourceScanState = "pending" + SourceScanRunning SourceScanState = "running" + SourceScanCompleted SourceScanState = "completed" + SourceScanFailed SourceScanState = "failed" +) + +func (s SourceScanState) Valid() bool { + switch s { + case SourceScanPending, SourceScanRunning, SourceScanCompleted, SourceScanFailed: + return true + default: + return false + } +} + +type SourceScan struct { + ID string `json:"id"` + SourceConnectionID string `json:"source_connection_id"` + SourceType SourceType `json:"source_type"` + State SourceScanState `json:"state"` + CreatedAt time.Time `json:"created_at"` +} + +type SourceScanSummaryView struct { + ID string `json:"id"` + State SourceScanState `json:"state"` + ObservedItems int `json:"observed_items"` + UnchangedItems int `json:"unchanged_items"` + AddedItems int `json:"added_items"` + UpdatedItems int `json:"updated_items"` + OverLimitItems int `json:"over_limit_items"` + MissingItems int `json:"missing_items"` + PlannedUploads int `json:"planned_uploads"` + ErrorCode string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at"` +} + +type SourceScanPage struct { + Items []SourceScanSummaryView + Total int + Limit int + Offset int +} + +type SourceScanChangeView struct { + ID string `json:"id"` + SourceItemID *string `json:"source_item_id"` + ChangeType SourceScanChangeType `json:"change_type"` + ObservedSizeBytes *int64 `json:"observed_size_bytes"` + CreatedAt time.Time `json:"created_at"` +} + +type SourceScanDetailView struct { + SourceScanSummaryView + Changes []SourceScanChangeView `json:"changes"` +} + +type SourceScanResult struct { + SnapshotSHA256 string `json:"-"` + ObservedItems int `json:"observed_items"` + UnchangedItems int `json:"unchanged_items"` + AddedItems int `json:"added_items"` + UpdatedItems int `json:"updated_items"` + OverLimitItems int `json:"over_limit_items"` + MissingItems int `json:"missing_items"` + PlannedUploads int `json:"planned_uploads"` + Changes []SourceScanChange `json:"-"` +} + +type SourceScanChangeType string + +const ( + SourceScanAdded SourceScanChangeType = "added" + SourceScanUpdated SourceScanChangeType = "updated" + SourceScanUnavailable SourceScanChangeType = "unavailable" +) + +type SourceScanChange struct { + StableExternalID string + ChangeType SourceScanChangeType + PreviousRevision string + ObservedRevision string + ObservedSizeBytes *int64 +} diff --git a/internal/governance/source_schedule.go b/internal/governance/source_schedule.go new file mode 100644 index 0000000..905ac10 --- /dev/null +++ b/internal/governance/source_schedule.go @@ -0,0 +1,42 @@ +package governance + +import ( + "encoding/json" + "errors" + "time" +) + +// V1 per-source refresh schedule. The schedule JSONB lives on the ACTIVE +// revision; next_refresh_at on source_connections is ALWAYS derived from it by +// the server (clients can never push an arbitrary due time). + +const MinRefreshInterval = 5 * time.Minute + +type SourceSchedule struct { + RefreshIntervalSeconds *int `json:"refresh_interval_seconds,omitempty"` +} + +// ParseSourceSchedule accepts the stored JSON and normalises it. Missing, +// zero, or sub-5-minute values are rejected as "not configured" (interval 0, +// no error) rather than silently accepted: a source that could stampede the +// portal every few seconds is worse than a manual-only source. +func ParseSourceSchedule(raw []byte) (time.Duration, error) { + if len(raw) == 0 || string(raw) == "{}" { + return 0, nil + } + var schedule SourceSchedule + if err := json.Unmarshal(raw, &schedule); err != nil { + return 0, errors.New("invalid source schedule") + } + if schedule.RefreshIntervalSeconds == nil || *schedule.RefreshIntervalSeconds == 0 { + return 0, nil + } + if *schedule.RefreshIntervalSeconds < 0 { + return 0, errors.New("invalid source schedule") + } + interval := time.Duration(*schedule.RefreshIntervalSeconds) * time.Second + if interval < MinRefreshInterval { + return 0, errors.New("source refresh interval must be at least five minutes") + } + return interval, nil +} diff --git a/internal/governance/source_schedule_test.go b/internal/governance/source_schedule_test.go new file mode 100644 index 0000000..ec90853 --- /dev/null +++ b/internal/governance/source_schedule_test.go @@ -0,0 +1,49 @@ +package governance + +import ( + "encoding/json" + "testing" + "time" +) + +func TestParseSourceSchedule(t *testing.T) { + cases := []struct { + name string + raw string + want time.Duration + wantErr bool + }{ + {"empty object = manual only", "{}", 0, false}, + {"missing key", `{"other":1}`, 0, false}, + {"explicit zero disables", `{"refresh_interval_seconds":0}`, 0, false}, + {"valid hourly", `{"refresh_interval_seconds":3600}`, time.Hour, false}, + {"valid daily", `{"refresh_interval_seconds":86400}`, 24 * time.Hour, false}, + {"below floor rejected", `{"refresh_interval_seconds":60}`, 0, true}, + {"negative rejected", `{"refresh_interval_seconds":-5}`, 0, true}, + {"malformed rejected", `not json`, 0, true}, + {"string value rejected", `{"refresh_interval_seconds":"3600"}`, 0, true}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + got, err := ParseSourceSchedule([]byte(test.raw)) + if (err != nil) != test.wantErr { + t.Fatalf("error=%v wantErr=%v", err, test.wantErr) + } + if got != test.want { + t.Fatalf("interval=%v want %v", got, test.want) + } + }) + } +} + +func TestSourceScheduleRoundTripsThroughJSON(t *testing.T) { + seconds := 7200 + encoded, err := json.Marshal(SourceSchedule{RefreshIntervalSeconds: &seconds}) + if err != nil { + t.Fatal(err) + } + interval, err := ParseSourceSchedule(encoded) + if err != nil || interval != 2*time.Hour { + t.Fatalf("interval=%v err=%v", interval, err) + } +} diff --git a/internal/governance/state_machine_contract_test.go b/internal/governance/state_machine_contract_test.go new file mode 100644 index 0000000..a9fc108 --- /dev/null +++ b/internal/governance/state_machine_contract_test.go @@ -0,0 +1,38 @@ +package governance + +import ( + "os" + "strings" + "testing" +) + +func TestStateMachineContractSeparatesDimensionsAndReservedStates(t *testing.T) { + content, err := os.ReadFile("../../docs/STATE_MACHINE.md") + if err != nil { + t.Fatalf("read state machine contract: %v", err) + } + + text := string(content) + for _, required := range []string{ + "## State dimensions", + "### Job", + "pending | running | retry | completed | failed | cancelled", + "### Scope policy", + "unset | internal | external | blocked", + "historical compatibility value: both", + "### Source change", + "### Review", + "### Content", + "### Ingest", + "### Processing", + "### Retrieval", + "### Release", + "## Reserved future values", + "rolling_back", + "must not be persisted before R4", + } { + if !strings.Contains(text, required) { + t.Fatalf("state machine contract is missing %q", required) + } + } +} diff --git a/internal/governance/test_publication.go b/internal/governance/test_publication.go new file mode 100644 index 0000000..b07c045 --- /dev/null +++ b/internal/governance/test_publication.go @@ -0,0 +1,63 @@ +package governance + +import ( + "encoding/json" + "time" +) + +// TestPublication is the safe projection of a test publication for the API and web console. +// It deliberately omits knowledge_id, test_kb_id, source URL, SHA-256, spool path, +// raw summaries and any credential material. +type TestPublication struct { + ID string `json:"id"` + Title string `json:"title"` + IngestState string `json:"ingest_state"` + ProcessingState string `json:"processing_state"` + RetrievalState string `json:"retrieval_state"` + TargetScope string `json:"target_scope"` + ChangeSetID string `json:"change_set_id"` + ChangeType string `json:"change_type"` + CanStart bool `json:"can_start"` + RetryCount int `json:"retry_count"` + LastError string `json:"last_error"` + CanRetry bool `json:"can_retry"` + UpdatedAt time.Time `json:"updated_at"` +} + +type TestPublicationFilter struct { + ProcessingState string + // TargetScope filters the test queue by KB routing target (P4). + TargetScope string + Limit int + Offset int +} + +type TestPublicationPage struct { + Items []TestPublication `json:"items"` + Total int `json:"total"` +} + +type StartTestPublicationsRequest struct { + PublicationIDs []string + OperatorRef string + Reason string + Now time.Time +} + +type StartTestPublicationsResult struct { + Selected int `json:"selected"` + Started int `json:"started"` + AlreadyStarted int `json:"already_started"` + Unavailable int `json:"unavailable"` +} + +// AcceptanceRecord is the safe projection of an acceptance result for the API. +// EvidenceSummary is returned as an object but only carries counts and booleans, +// never remote bodies, paths or credentials. +type AcceptanceRecord struct { + CheckType string `json:"check_type"` + State string `json:"state"` + PolicyVersion string `json:"policy_version"` + EvidenceSummary json.RawMessage `json:"evidence_summary"` + ObservedAt time.Time `json:"observed_at"` +} diff --git a/internal/governance/ui_settings.go b/internal/governance/ui_settings.go new file mode 100644 index 0000000..43d2968 --- /dev/null +++ b/internal/governance/ui_settings.go @@ -0,0 +1,51 @@ +package governance + +import ( + "strings" + "time" +) + +type UISettings struct { + ID string `json:"id"` + State string `json:"state"` + DocumentRefreshSeconds int `json:"document_refresh_seconds"` + ResourceRefreshSeconds int `json:"resource_refresh_seconds"` + StalledAfterSeconds int `json:"stalled_after_seconds"` + FinalizingAfterSeconds int `json:"finalizing_after_seconds"` + UpdatedBy string `json:"updated_by"` + CreatedAt time.Time `json:"created_at"` + ActivatedAt *time.Time `json:"activated_at"` + RetiredAt *time.Time `json:"retired_at"` + ErrorCode string `json:"error_code"` +} + +type CreateUISettingsCandidateRequest struct { + DocumentRefreshSeconds int + ResourceRefreshSeconds int + StalledAfterSeconds int + FinalizingAfterSeconds int + UpdatedBy string + Now time.Time +} + +func (request CreateUISettingsCandidateRequest) Valid() bool { + return oneOfInt(request.DocumentRefreshSeconds, 15, 30, 60) && + oneOfInt(request.ResourceRefreshSeconds, 30, 60) && + oneOfInt(request.StalledAfterSeconds, 1800, 3600) && + oneOfInt(request.FinalizingAfterSeconds, 7200, 14400) && + validSettingsActor(request.UpdatedBy) && !request.Now.IsZero() +} + +func oneOfInt(value int, allowed ...int) bool { + for _, candidate := range allowed { + if value == candidate { + return true + } + } + return false +} + +func validSettingsActor(value string) bool { + trimmed := strings.TrimSpace(value) + return trimmed == value && len(trimmed) >= 2 && len(trimmed) <= 128 && !strings.ContainsAny(trimmed, "\x00\r\n\t") +} diff --git a/internal/httpapi/openapi_contract_test.go b/internal/httpapi/openapi_contract_test.go new file mode 100644 index 0000000..aa923d7 --- /dev/null +++ b/internal/httpapi/openapi_contract_test.go @@ -0,0 +1,110 @@ +package httpapi + +import ( + "encoding/json" + "os" + "regexp" + "strings" + "testing" +) + +func TestOpenAPIContractMatchesRegisteredRoutesAndOmitsSecrets(t *testing.T) { + contract, err := os.ReadFile("../../api/openapi.yaml") + if err != nil { + t.Fatalf("read OpenAPI contract: %v", err) + } + + var document struct { + OpenAPI string `json:"openapi"` + Info map[string]any `json:"info"` + Paths map[string]map[string]any `json:"paths"` + Components map[string]map[string]any `json:"components"` + } + if err := json.Unmarshal(contract, &document); err != nil { + t.Fatalf("OpenAPI contract must use JSON-compatible YAML: %v", err) + } + if document.OpenAPI != "3.1.0" || document.Info["title"] != "Knowledge Governance API" || len(document.Paths) == 0 { + t.Fatalf("invalid OpenAPI header: %#v", document) + } + + routes, err := os.ReadFile("server.go") + if err != nil { + t.Fatalf("read registered routes: %v", err) + } + matcher := regexp.MustCompile(`mux\.HandleFunc\("(GET|POST|PUT|DELETE) ([^"]+)"`) + for _, match := range matcher.FindAllStringSubmatch(string(routes), -1) { + method, path := strings.ToLower(match[1]), match[2] + operation, ok := document.Paths[path][method].(map[string]any) + if !ok { + t.Fatalf("OpenAPI is missing %s %s", strings.ToUpper(method), path) + } + if strings.TrimSpace(stringValue(operation["operationId"])) == "" { + t.Fatalf("OpenAPI operation %s %s has no operationId", strings.ToUpper(method), path) + } + responses, ok := operation["responses"].(map[string]any) + if !ok || responses["400"] == nil || responses["500"] == nil { + t.Fatalf("OpenAPI operation %s %s must document 400 and 500 responses", strings.ToUpper(method), path) + } + } + + lower := strings.ToLower(string(contract)) + for _, forbidden := range []string{"secret_ref", "authorization_code", "manifest_sha256", "knowledge_id", "test_kb_id", "base_url", "download_code", "cookie"} { + if strings.Contains(lower, forbidden) { + t.Fatalf("OpenAPI contract must not expose %q", forbidden) + } + } +} + +func TestOpenAPICatalogMutationsRequireIdempotencyKeyHeader(t *testing.T) { + contract, err := os.ReadFile("../../api/openapi.yaml") + if err != nil { + t.Fatal(err) + } + var document struct { + Paths map[string]map[string]any `json:"paths"` + Components struct { + Parameters map[string]map[string]any `json:"parameters"` + } `json:"components"` + } + if err := json.Unmarshal(contract, &document); err != nil { + t.Fatal(err) + } + parameter, ok := document.Components.Parameters["IdempotencyKey"] + if !ok || parameter["name"] != "Idempotency-Key" || parameter["in"] != "header" || parameter["required"] != true { + t.Fatalf("IdempotencyKey header must be required: %#v", parameter) + } + schema, ok := parameter["schema"].(map[string]any) + if !ok || schema["type"] != "string" || schema["minLength"] != float64(1) || schema["maxLength"] != float64(128) { + t.Fatalf("IdempotencyKey header schema is unsafe: %#v", parameter["schema"]) + } + for _, path := range []string{"/api/v1/source-catalog/route", "/api/v1/source-catalog/sources/{id}/refresh"} { + operation, ok := document.Paths[path]["post"].(map[string]any) + if !ok { + t.Fatalf("missing POST %s", path) + } + parameters, ok := operation["parameters"].([]any) + if !ok { + t.Fatalf("POST %s must declare the idempotency header", path) + } + found := false + for _, value := range parameters { + parameterRef, _ := value.(map[string]any) + if parameterRef["$ref"] == "#/components/parameters/IdempotencyKey" { + found = true + } + } + if !found { + t.Fatalf("POST %s must reference IdempotencyKey", path) + } + responses, _ := operation["responses"].(map[string]any) + accepted, _ := responses["202"].(map[string]any) + if accepted["$ref"] != "#/components/responses/OperationEnvelope" { + t.Fatalf("POST %s must document created/replayed operation response: %#v", path, accepted) + } + } +} + +func stringValue(value any) string { + result, _ := value.(string) + return result +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go new file mode 100644 index 0000000..13a6558 --- /dev/null +++ b/internal/httpapi/server.go @@ -0,0 +1,1965 @@ +package httpapi + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/monitoring" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +type JobStore interface { + Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error) + List(ctx context.Context, limit int) ([]jobs.Job, error) +} + +type MonitorProvider interface { + Overview(ctx context.Context) (governance.MonitorOverview, error) + ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) + Metrics(ctx context.Context) (governance.RunMetrics, error) + Services(ctx context.Context) []governance.ServiceHealth +} + +type Store interface { + JobStore + ListSourceConnections(ctx context.Context) (governance.SourceConnectionPage, error) + CreateSourceSettingsRevisionWithOperation(ctx context.Context, connectionID string, draft postgresstore.SourceSettingsDraft, operationID string, now time.Time) (string, bool, error) + ActivateSourceSettingsRevisionWithOperation(ctx context.Context, connectionID, revisionID, operationID string, now time.Time) (bool, error) + ListSourceConnectionRevisions(ctx context.Context, connectionID string) ([]governance.SourceConnectionRevision, error) + TriggerPPSourceScan(ctx context.Context, connectionID string, now time.Time) (governance.SourceScan, error) + TriggerPPSourceScanWithOperation(ctx context.Context, connectionID string, now time.Time, operationID string) (governance.SourceScan, bool, error) + ListSourceScans(ctx context.Context, connectionID string, limit, offset int) (governance.SourceScanPage, error) + GetSourceScanDetail(ctx context.Context, connectionID, scanID string) (governance.SourceScanDetailView, error) + ListReviewCandidates(ctx context.Context, filter governance.ReviewCandidateFilter) (governance.ReviewCandidatePage, error) + ReviewPublication(ctx context.Context, request governance.ReviewPublicationRequest) (governance.PublicationReview, error) + ReviewPublicationWithOperation(ctx context.Context, request governance.ReviewPublicationRequest, operationID string) (governance.PublicationReview, bool, error) + CreateReleaseBatch(ctx context.Context, request governance.CreateReleaseBatchRequest) (governance.ReleaseBatch, error) + CreateReleaseBatchWithOperation(ctx context.Context, request governance.CreateReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) + GetReleaseAuditChain(ctx context.Context, executionID string) (governance.ReleaseAuditChain, error) + GenerateReleaseChangeSetWithOperation(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) + ConfirmReleaseChangeSetWithOperation(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) + ListReleaseChangeSets(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseChangeSetPage, error) + GetReleaseChangeSet(ctx context.Context, id string) (governance.ReleaseChangeSetDetail, error) + LatestReleaseChangeSet(ctx context.Context) (governance.ReleaseChangeSetDetail, error) + SealReleaseBatch(ctx context.Context, request governance.SealReleaseBatchRequest) (governance.ReleaseBatch, error) + SealReleaseBatchWithOperation(ctx context.Context, request governance.SealReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) + ListReleaseBatches(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseBatchPage, error) + GetReleaseBatch(ctx context.Context, batchID string) (governance.ReleaseBatchDetail, error) + AuthorizeReleaseBatch(ctx context.Context, request governance.AuthorizeReleaseBatchRequest) (governance.ReleaseExecution, error) + AuthorizeReleaseBatchWithOperation(ctx context.Context, request governance.AuthorizeReleaseBatchRequest, operationID string) (governance.ReleaseExecution, bool, error) + AuthorizeReleaseRollback(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest) (governance.ReleaseExecution, error) + AuthorizeReleaseRollbackWithOperation(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest, operationID string) (governance.ReleaseExecution, bool, error) + RetryReleaseExecution(ctx context.Context, request governance.RetryReleaseExecutionRequest) (governance.ReleaseExecution, error) + RetryReleaseExecutionWithOperation(ctx context.Context, request governance.RetryReleaseExecutionRequest, operationID string) (governance.ReleaseExecution, bool, error) + ListReleaseExecutions(ctx context.Context, filter governance.ReleaseExecutionFilter) (governance.ReleaseExecutionPage, error) + GetReleaseExecution(ctx context.Context, executionID string) (governance.ReleaseExecutionDetail, error) + ListReleasePlan(ctx context.Context) (governance.ReleasePlan, error) + GetActiveUISettings(ctx context.Context) (governance.UISettings, error) + CreateUISettingsCandidate(ctx context.Context, request governance.CreateUISettingsCandidateRequest) (governance.UISettings, error) + CreateUISettingsCandidateWithOperation(ctx context.Context, request governance.CreateUISettingsCandidateRequest, operationID string) (governance.UISettings, bool, error) + ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) + ActivateUISettingsWithOperation(ctx context.Context, revisionID string, now time.Time, operationID string) (governance.UISettings, bool, error) + ListSourceAssets(ctx context.Context, filter governance.SourceAssetFilter) (governance.SourceAssetPage, error) + ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) + ListCatalogNodes(ctx context.Context, filter governance.CatalogNodeFilter) (governance.CatalogNodePage, error) + ListCatalogSnapshots(ctx context.Context, connectionID string, limit, offset int) (governance.CatalogSnapshotPage, error) + ListCatalogNodesInSnapshot(ctx context.Context, filter governance.CatalogSnapshotNodeFilter) (governance.CatalogNodePage, error) + GetLatestCatalogChangeSet(ctx context.Context, filter governance.CatalogChangeSetFilter) (governance.CatalogChangeSetDetail, error) + RouteCatalogFiles(ctx context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) + RouteCatalogFilesWithOperation(ctx context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) + EnqueueCatalogRefreshWithOperation(ctx context.Context, source governance.CatalogSource, operationID string, now time.Time) (jobs.Job, bool, error) + ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error) + MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error) + MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error) + ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error) + StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) + StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) + ConfirmCatalogChangeSetWithOperation(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest, operationID string) (governance.ConfirmCatalogChangeSetResult, bool, error) + ListAcceptanceResults(ctx context.Context, publicationID string) ([]governance.AcceptanceRecord, error) + RetryPublication(ctx context.Context, publicationID string, now time.Time) error + RetryPublicationWithOperation(ctx context.Context, publicationID string, now time.Time, operationID string) (bool, error) + ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) + RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) +} + +type Config struct { + Store Store + Monitor MonitorProvider + InternalRoutingKnowledgeBaseID string + ExternalRoutingKnowledgeBaseID string + Now func() time.Time + RequireWeKnoraReady bool + ProductionReleaseRequestsEnabled bool + ProductionBackupRestoreVerified bool + ProductionReleaseAuthorizerRef string + ProductionReleaseAuthorizationToken string + NewID func() (string, error) +} + +func NewHandler(config Config) (http.Handler, error) { + if config.Store == nil { + return nil, errors.New("job store is required") + } + if config.Now == nil { + config.Now = time.Now + } + if config.Monitor == nil { + config.Monitor = monitoring.NewSimulationProvider(config.Now) + } + for _, knowledgeBaseID := range []string{config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID} { + if knowledgeBaseID != "" && !validCatalogID(knowledgeBaseID) { + return nil, errors.New("routing knowledge base ID is invalid") + } + } + if config.ProductionReleaseRequestsEnabled && + (strings.TrimSpace(config.ProductionReleaseAuthorizerRef) == "" || strings.TrimSpace(config.ProductionReleaseAuthorizationToken) == "") { + return nil, errors.New("production release authorization identity and token are required") + } + if config.NewID == nil { + config.NewID = newID + } + mux := http.NewServeMux() + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + }) + // P10: liveness answers "is this process serving HTTP" and NEVER depends on + // third parties; readiness answers "can this instance accept dependent work". + mux.HandleFunc("GET /livez", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"status": "live", "observed_at": config.Now().UTC()}) + }) + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + health := config.Monitor.Services(r.Context()) + // The API's own hard dependencies gate readiness. Worker and MinerU + // outages degrade the matrix (visible in Services) but must not take + // the admin UI out of rotation — the page is how operators fix them. + critical := map[string]bool{"postgres": true, "queue": true} + blocked := make([]string, 0) + degraded := make([]string, 0) + warnings := make([]string, 0) + for _, item := range health { + if !item.Applicable { + continue + } + if !item.Configured { + warnings = append(warnings, item.Key) + continue + } + key := strings.SplitN(item.Key, ":", 2)[0] + if item.Readiness == "ready" { + continue + } + if critical[key] || key == "weknora" && config.RequireWeKnoraReady { + blocked = append(blocked, item.Key) + } else { + degraded = append(degraded, item.Key) + } + } + status := map[string]any{"status": "ready", "blocked": blocked, "degraded": degraded, "not_configured": warnings, "observed_at": config.Now().UTC()} + if len(blocked) > 0 { + status["status"] = "blocked" + writeJSON(w, http.StatusServiceUnavailable, status) + return + } + if len(degraded) > 0 { + status["status"] = "degraded" + } + writeJSON(w, http.StatusOK, status) + }) + mux.HandleFunc("POST /api/v1/jobs", func(w http.ResponseWriter, r *http.Request) { + createJob(w, r, config) + }) + mux.HandleFunc("GET /api/v1/jobs", func(w http.ResponseWriter, r *http.Request) { + listJobs(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-items", func(w http.ResponseWriter, r *http.Request) { + listSourceAssets(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/sources", func(w http.ResponseWriter, r *http.Request) { + listCatalogSources(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/nodes", func(w http.ResponseWriter, r *http.Request) { + listCatalogNodes(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/snapshots", func(w http.ResponseWriter, r *http.Request) { + listCatalogSnapshots(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/snapshots/{id}/nodes", func(w http.ResponseWriter, r *http.Request) { + listCatalogSnapshotNodes(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/change-sets/latest", func(w http.ResponseWriter, r *http.Request) { + getLatestCatalogChangeSet(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-catalog/routing-capabilities", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]bool{"internal_enabled": config.InternalRoutingKnowledgeBaseID != "", "external_enabled": config.ExternalRoutingKnowledgeBaseID != ""}}) + }) + mux.HandleFunc("POST /api/v1/source-catalog/route", func(w http.ResponseWriter, r *http.Request) { + routeCatalogFiles(w, r, config) + }) + mux.HandleFunc("POST /api/v1/source-catalog/sources/{id}/refresh", func(w http.ResponseWriter, r *http.Request) { + triggerCatalogRefresh(w, r, config) + }) + mux.HandleFunc("POST /api/v1/source-catalog/change-sets/{id}/confirm", func(w http.ResponseWriter, r *http.Request) { + confirmCatalogChangeSet(w, r, config) + }) + mux.HandleFunc("GET /api/v1/monitor/overview", func(w http.ResponseWriter, r *http.Request) { + monitorOverview(w, r, config.Monitor) + }) + mux.HandleFunc("GET /api/v1/monitor/knowledge-bases/{id}/files", func(w http.ResponseWriter, r *http.Request) { + monitorFiles(w, r, config.Monitor) + }) + mux.HandleFunc("GET /api/v1/monitor/metrics", func(w http.ResponseWriter, r *http.Request) { + monitorMetrics(w, r, config.Monitor) + }) + mux.HandleFunc("GET /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) { + listScopeRules(w, r, config.Store) + }) + mux.HandleFunc("PUT /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) { + mutateScopeRule(w, r, config, false) + }) + mux.HandleFunc("DELETE /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) { + mutateScopeRule(w, r, config, true) + }) + mux.HandleFunc("GET /api/v1/source-connections", func(w http.ResponseWriter, r *http.Request) { + listSourceConnections(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/source-connections/{id}/scan", func(w http.ResponseWriter, r *http.Request) { + triggerPPSourceScan(w, r, config) + }) + mux.HandleFunc("GET /api/v1/source-connections/{id}/scans", func(w http.ResponseWriter, r *http.Request) { + listSourceScans(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-connections/{id}/scans/{scan_id}", func(w http.ResponseWriter, r *http.Request) { + getSourceScanDetail(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/source-connections/{id}/settings/revisions", func(w http.ResponseWriter, r *http.Request) { + listSourceSettingsRevisions(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/source-connections/{id}/settings/revisions", func(w http.ResponseWriter, r *http.Request) { + createSourceSettingsRevision(w, r, config) + }) + mux.HandleFunc("POST /api/v1/source-connections/{id}/settings/revisions/{revision_id}/activate", func(w http.ResponseWriter, r *http.Request) { + activateSourceSettingsRevision(w, r, config) + }) + mux.HandleFunc("GET /api/v1/review-candidates", func(w http.ResponseWriter, r *http.Request) { + listReviewCandidates(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/review-candidates/{id}/decision", func(w http.ResponseWriter, r *http.Request) { + reviewPublication(w, r, config) + }) + mux.HandleFunc("GET /api/v1/release-batches", func(w http.ResponseWriter, r *http.Request) { + listReleaseBatches(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/release-executions/{id}/audit-chain", func(w http.ResponseWriter, r *http.Request) { + getReleaseAuditChain(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/release-change-sets", func(w http.ResponseWriter, r *http.Request) { + generateReleaseChangeSet(w, r, config) + }) + mux.HandleFunc("GET /api/v1/release-change-sets", func(w http.ResponseWriter, r *http.Request) { + listReleaseChangeSets(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/release-change-sets/latest", func(w http.ResponseWriter, r *http.Request) { + getReleaseChangeSet(w, r, config.Store, "latest") + }) + mux.HandleFunc("GET /api/v1/release-change-sets/{id}", func(w http.ResponseWriter, r *http.Request) { + getReleaseChangeSet(w, r, config.Store, r.PathValue("id")) + }) + mux.HandleFunc("POST /api/v1/release-change-sets/{id}/confirm", func(w http.ResponseWriter, r *http.Request) { + confirmReleaseChangeSet(w, r, config) + }) + mux.HandleFunc("GET /api/v1/release-plan", func(w http.ResponseWriter, r *http.Request) { + plan, err := config.Store.ListReleasePlan(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not build release plan"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": plan}) + }) + mux.HandleFunc("GET /api/v1/ui-settings", func(w http.ResponseWriter, r *http.Request) { + getActiveUISettings(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/ui-settings/candidates", func(w http.ResponseWriter, r *http.Request) { + createUISettingsCandidate(w, r, config) + }) + mux.HandleFunc("POST /api/v1/ui-settings/{id}/activate", func(w http.ResponseWriter, r *http.Request) { + activateUISettings(w, r, config) + }) + mux.HandleFunc("POST /api/v1/release-batches", func(w http.ResponseWriter, r *http.Request) { + createReleaseBatch(w, r, config) + }) + mux.HandleFunc("GET /api/v1/release-batches/{id}", func(w http.ResponseWriter, r *http.Request) { + getReleaseBatch(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/release-batches/{id}/seal", func(w http.ResponseWriter, r *http.Request) { + sealReleaseBatch(w, r, config) + }) + mux.HandleFunc("POST /api/v1/release-batches/{id}/authorize-production", func(w http.ResponseWriter, r *http.Request) { + authorizeProductionRelease(w, r, config) + }) + mux.HandleFunc("POST /api/v1/release-executions/{id}/authorize-rollback", func(w http.ResponseWriter, r *http.Request) { + authorizeProductionRollback(w, r, config) + }) + mux.HandleFunc("POST /api/v1/release-executions/{id}/retry", func(w http.ResponseWriter, r *http.Request) { + retryProductionRelease(w, r, config) + }) + mux.HandleFunc("GET /api/v1/release-executions", func(w http.ResponseWriter, r *http.Request) { + listReleaseExecutions(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/release-executions/{id}", func(w http.ResponseWriter, r *http.Request) { + getReleaseExecution(w, r, config.Store) + }) + mux.HandleFunc("GET /api/v1/release-capabilities", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]bool{ + "production_release_requests_enabled": config.ProductionReleaseRequestsEnabled, + "production_backup_restore_verified": config.ProductionBackupRestoreVerified, + }}) + }) + mux.HandleFunc("GET /api/v1/test-publications", func(w http.ResponseWriter, r *http.Request) { + listTestPublications(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/test-publications/start", func(w http.ResponseWriter, r *http.Request) { + startTestPublications(w, r, config) + }) + mux.HandleFunc("GET /api/v1/test-publications/{id}/acceptance", func(w http.ResponseWriter, r *http.Request) { + listAcceptance(w, r, config.Store) + }) + mux.HandleFunc("POST /api/v1/test-publications/{id}/retry", func(w http.ResponseWriter, r *http.Request) { + retryPublication(w, r, config) + }) + mux.HandleFunc("POST /api/v1/test-publications/retry-batch", func(w http.ResponseWriter, r *http.Request) { + retryPublicationsBatch(w, r, config) + }) + mux.HandleFunc("GET /api/v1/job-failures", func(w http.ResponseWriter, r *http.Request) { + listJobFailures(w, r, config.Store) + }) + return mux, nil +} + +type createJobRequest struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + Priority int `json:"priority"` + MaxAttempts int `json:"max_attempts"` + IdempotencyKey string `json:"idempotency_key"` +} + +func createJob(w http.ResponseWriter, r *http.Request, config Config) { + var input createJobRequest + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<20)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON request"}) + return + } + if strings.TrimSpace(input.Type) == "" || len(input.Type) > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type is required and must not exceed 100 characters"}) + return + } + if strings.TrimSpace(input.IdempotencyKey) == "" || len(input.IdempotencyKey) > 255 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "idempotency_key is required and must not exceed 255 characters"}) + return + } + if input.MaxAttempts == 0 { + input.MaxAttempts = 5 + } + if input.MaxAttempts < 1 || input.MaxAttempts > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "max_attempts must be between 1 and 100"}) + return + } + if len(input.Payload) == 0 { + input.Payload = json.RawMessage(`{}`) + } + if !json.Valid(input.Payload) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "payload must be valid JSON"}) + return + } + id, err := config.NewID() + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not create job ID"}) + return + } + now := config.Now().UTC() + job, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ + ID: id, + Type: input.Type, + Payload: input.Payload, + Priority: input.Priority, + AvailableAt: now, + MaxAttempts: input.MaxAttempts, + IdempotencyKey: input.IdempotencyKey, + }, now) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not enqueue job"}) + return + } + writeJSON(w, http.StatusCreated, map[string]any{"data": job}) +} + +func listJobs(w http.ResponseWriter, r *http.Request, store JobStore) { + limit := 50 + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + limit = value + } + result, err := store.List(r.Context(), limit) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list jobs"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": result}) +} + +func listSourceAssets(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.SourceAssetFilter{State: r.URL.Query().Get("state"), Limit: 50} + if filter.State != "" && filter.State != "completed" && filter.State != "content_alias" && filter.State != "over_limit" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "state must be completed, content_alias, or over_limit"}) + return + } + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + filter.Limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + filter.Offset = value + } + page, err := store.ListSourceAssets(r.Context(), filter) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source items"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "data": page.Items, "summary": page.Summary, "limit": filter.Limit, "offset": filter.Offset, + }) +} + +func listCatalogSources(w http.ResponseWriter, r *http.Request, store Store) { + sources, err := store.ListCatalogSources(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source catalogs"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": sources, "summary": map[string]int{"total": len(sources)}}) +} + +func listCatalogNodes(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.CatalogNodeFilter{ConnectionID: r.URL.Query().Get("connection_id"), ParentNodeID: r.URL.Query().Get("parent_id"), Limit: 50} + if !validCatalogID(filter.ConnectionID) || !validCatalogID(filter.ParentNodeID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source catalog location"}) + return + } + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + filter.Limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + filter.Offset = value + } + page, err := store.ListCatalogNodes(r.Context(), filter) + if errors.Is(err, postgresstore.ErrCatalogNodeNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source catalog location not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source catalog"}) + return + } + writeJSON(w, http.StatusOK, page) +} + +func listCatalogSnapshots(w http.ResponseWriter, r *http.Request, store Store) { + connectionID := r.URL.Query().Get("connection_id") + if !validCatalogID(connectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source catalog"}) + return + } + limit, offset, ok := parseLimitOffset(w, r, 20) + if !ok { + return + } + page, err := store.ListCatalogSnapshots(r.Context(), connectionID, limit, offset) + if errors.Is(err, postgresstore.ErrCatalogSnapshotNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source catalog not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list snapshot history"}) + return + } + writeJSON(w, http.StatusOK, page) +} + +func listCatalogSnapshotNodes(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.CatalogSnapshotNodeFilter{ConnectionID: r.URL.Query().Get("connection_id"), SnapshotID: r.PathValue("id"), ParentNodeID: r.URL.Query().Get("parent_id"), Limit: 50} + if !validCatalogID(filter.ConnectionID) || !validCatalogID(filter.SnapshotID) || !validCatalogID(filter.ParentNodeID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid snapshot location"}) + return + } + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + filter.Limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + filter.Offset = value + } + page, err := store.ListCatalogNodesInSnapshot(r.Context(), filter) + if errors.Is(err, postgresstore.ErrCatalogSnapshotNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "snapshot not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list snapshot nodes"}) + return + } + writeJSON(w, http.StatusOK, page) +} + +func getLatestCatalogChangeSet(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.CatalogChangeSetFilter{ConnectionID: strings.TrimSpace(r.URL.Query().Get("connection_id")), Action: strings.TrimSpace(r.URL.Query().Get("action")), Limit: 20} + if !validCatalogID(filter.ConnectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source catalog"}) + return + } + allowedActions := map[string]bool{"": true, "add": true, "update": true, "move": true, "move_scope": true, "delete": true, "no_change": true} + if !allowedActions[filter.Action] { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid change action"}) + return + } + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + filter.Limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + filter.Offset = value + } + detail, err := store.GetLatestCatalogChangeSet(r.Context(), filter) + if errors.Is(err, postgresstore.ErrCatalogChangeSetNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source catalog change set not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load source catalog change set"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": detail}) +} + +func validCatalogID(value string) bool { + if value == "" || len(value) > 36 { + return false + } + for _, character := range value { + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '-' || character == '_' { + continue + } + return false + } + return true +} + +func requireOperationID(w http.ResponseWriter, r *http.Request) (string, bool) { + operationID := strings.TrimSpace(r.Header.Get("Idempotency-Key")) + if operationID == "" || len(operationID) > 128 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Idempotency-Key is required and must not exceed 128 characters"}) + return "", false + } + return operationID, true +} + +type catalogRouteInput struct { + ConnectionID string `json:"connection_id"` + NodeIDs []string `json:"node_ids"` + Policy string `json:"policy"` + Visibility string `json:"visibility"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` +} + +func routeCatalogFiles(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input catalogRouteInput + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil || len(input.NodeIDs) == 0 || len(input.NodeIDs) > 100 || !validCatalogID(input.ConnectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid catalog route request"}) + return + } + policy := strings.TrimSpace(input.Policy) + if policy == "" { + policy = strings.TrimSpace(input.Visibility) + } + visibility := governance.Visibility(policy) + targetKnowledgeBaseIDs := make([]string, 0, 2) + switch visibility { + case governance.VisibilityInternal: + if config.InternalRoutingKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = append(targetKnowledgeBaseIDs, config.InternalRoutingKnowledgeBaseID) + } + case governance.VisibilityExternal: + if config.ExternalRoutingKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = append(targetKnowledgeBaseIDs, config.ExternalRoutingKnowledgeBaseID) + } + case governance.VisibilityBoth: + if config.InternalRoutingKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = append(targetKnowledgeBaseIDs, config.InternalRoutingKnowledgeBaseID) + } + if config.ExternalRoutingKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = append(targetKnowledgeBaseIDs, config.ExternalRoutingKnowledgeBaseID) + } + case governance.VisibilityBlocked: + default: + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "policy must be internal, external, both, or blocked"}) + return + } + if visibility != governance.VisibilityBlocked && + ((visibility == governance.VisibilityBoth && len(targetKnowledgeBaseIDs) != 2) || + (visibility != governance.VisibilityBoth && len(targetKnowledgeBaseIDs) != 1)) { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "routing target is not configured"}) + return + } + targetKnowledgeBaseID := "" + if len(targetKnowledgeBaseIDs) > 0 { + targetKnowledgeBaseID = targetKnowledgeBaseIDs[0] + } + result, replayed, err := config.Store.RouteCatalogFilesWithOperation(r.Context(), governance.CatalogRouteRequest{ + ConnectionID: input.ConnectionID, NodeIDs: input.NodeIDs, Visibility: visibility, + TargetKnowledgeBaseID: targetKnowledgeBaseID, TargetKnowledgeBaseIDs: targetKnowledgeBaseIDs, + OperatorRef: strings.TrimSpace(input.OperatorRef), + Reason: strings.TrimSpace(input.Reason), Now: config.Now().UTC(), + }, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "could not route selected files"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusAccepted, map[string]any{"data": result, "operation": operation}) +} + +func triggerCatalogRefresh(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + connectionID := r.PathValue("id") + if !validCatalogID(connectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source catalog"}) + return + } + sources, err := config.Store.ListCatalogSources(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not inspect source catalog"}) + return + } + var source *governance.CatalogSource + for index := range sources { + if sources[index].ConnectionID == connectionID { + source = &sources[index] + break + } + } + if source == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source catalog not found"}) + return + } + job, replayed, err := config.Store.EnqueueCatalogRefreshWithOperation(r.Context(), *source, operationID, config.Now().UTC()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not enqueue catalog refresh"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusAccepted, map[string]any{"data": job, "operation": operation}) +} + +func monitorOverview(w http.ResponseWriter, r *http.Request, provider MonitorProvider) { + overview, err := provider.Overview(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load monitor overview"}) + return + } + writeJSON(w, http.StatusOK, overview) +} + +func monitorMetrics(w http.ResponseWriter, r *http.Request, provider MonitorProvider) { + metrics, err := provider.Metrics(r.Context()) + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "run metrics unavailable"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": metrics}) +} + +func monitorFiles(w http.ResponseWriter, r *http.Request, provider MonitorProvider) { + knowledgeBaseID := r.PathValue("id") + status := r.URL.Query().Get("status") + allowedStatus := map[string]bool{"": true, "completed": true, "pending": true, "processing": true, "finalizing": true, "failed": true, "cancelled": true} + if !validCatalogID(knowledgeBaseID) || !allowedStatus[status] { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid monitor file query"}) + return + } + limit, offset := 20, 0 + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + offset = value + } + page, err := provider.ListFiles(r.Context(), knowledgeBaseID, status, limit, offset) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load monitored files"}) + return + } + writeJSON(w, http.StatusOK, page) +} + +type scopeRuleInput struct { + TargetType governance.ScopeTargetType `json:"target_type"` + ConnectionID string `json:"connection_id"` + DirectoryPath string `json:"directory_path"` + SourceItemID string `json:"source_item_id"` + Visibility governance.Visibility `json:"visibility"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` +} + +func listScopeRules(w http.ResponseWriter, r *http.Request, store Store) { + connectionID := strings.TrimSpace(r.URL.Query().Get("connection_id")) + if connectionID == "" || len(connectionID) > 36 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "connection_id is required"}) + return + } + rules, err := store.ListScopeRules(r.Context(), connectionID) + if errors.Is(err, postgresstore.ErrScopeTargetNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "scope target not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list scope rules"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": rules, "summary": map[string]int{"total": len(rules)}}) +} + +func mutateScopeRule(w http.ResponseWriter, r *http.Request, config Config, clear bool) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input scopeRuleInput + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid scope rule request"}) + return + } + request := governance.ScopeRuleMutation{ + TargetType: input.TargetType, ConnectionID: strings.TrimSpace(input.ConnectionID), + DirectoryPath: input.DirectoryPath, SourceItemID: strings.TrimSpace(input.SourceItemID), + Visibility: input.Visibility, OperatorRef: strings.TrimSpace(input.OperatorRef), Reason: strings.TrimSpace(input.Reason), + Clear: clear, Now: config.Now().UTC(), + } + if clear { + request.Visibility = "" + } + if err := request.Validate(); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid scope rule request"}) + return + } + rule, replayed, err := config.Store.MutateScopeRuleWithOperation(r.Context(), request, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrScopeTargetNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "scope target not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not update scope rule"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": rule, "operation": operation}) +} + +type sourceSettingsRevisionRequest struct { + SourceType string `json:"source_type"` + Name string `json:"name"` + BaseURL string `json:"endpoint"` + SecretRef string `json:"credential_ref"` + Enabled bool `json:"enabled"` + Schedule struct { + RefreshIntervalSeconds *int `json:"refresh_interval_seconds"` + } `json:"schedule"` + Operator string `json:"operator"` +} + +func (req sourceSettingsRevisionRequest) toDraft() postgresstore.SourceSettingsDraft { + schedule := "{}" + if req.Schedule.RefreshIntervalSeconds != nil { + schedule = fmt.Sprintf(`{"refresh_interval_seconds":%d}`, *req.Schedule.RefreshIntervalSeconds) + } + return postgresstore.SourceSettingsDraft{ + SourceType: req.SourceType, Name: req.Name, BaseURL: req.BaseURL, + SecretRef: req.SecretRef, Enabled: req.Enabled, + Schedule: json.RawMessage(schedule), Operator: req.Operator, + } +} + +func listSourceSettingsRevisions(w http.ResponseWriter, r *http.Request, store Store) { + connectionID := r.PathValue("id") + if !validCatalogID(connectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source connection"}) + return + } + revisions, err := store.ListSourceConnectionRevisions(r.Context(), connectionID) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source settings revisions"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": revisions}) +} + +func createSourceSettingsRevision(w http.ResponseWriter, r *http.Request, config Config) { + if r.Header.Get("X-Governance-Action") != "source-settings" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "source settings write intent is required"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + connectionID := r.PathValue("id") + if !validCatalogID(connectionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source connection"}) + return + } + var request sourceSettingsRevisionRequest + if err := decodeJSONStrict(r, &request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source settings revision"}) + return + } + revisionID, replayed, err := config.Store.CreateSourceSettingsRevisionWithOperation(r.Context(), connectionID, request.toDraft(), operationID, config.Now().UTC()) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrInvalidSourceSettings) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source settings revision"}) + return + } + if errors.Is(err, postgresstore.ErrSourceConnectionNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source connection not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not create source settings revision"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusCreated, map[string]any{"data": map[string]string{"id": revisionID}, "operation": operation}) +} + +func activateSourceSettingsRevision(w http.ResponseWriter, r *http.Request, config Config) { + if r.Header.Get("X-Governance-Action") != "source-settings" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "source settings write intent is required"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + connectionID, revisionID := r.PathValue("id"), r.PathValue("revision_id") + if !validCatalogID(connectionID) || !validCatalogID(revisionID) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source settings revision"}) + return + } + var request struct{} + if err := decodeJSONStrict(r, &request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source settings activation"}) + return + } + replayed, err := config.Store.ActivateSourceSettingsRevisionWithOperation(r.Context(), connectionID, revisionID, operationID, config.Now().UTC()) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrInvalidSourceSettings) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "only validated revisions can be activated"}) + return + } + if errors.Is(err, postgresstore.ErrSourceConnectionNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source settings revision not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not activate source settings revision"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]string{"connection_id": connectionID, "revision_id": revisionID}, "operation": operation}) +} + +func listSourceConnections(w http.ResponseWriter, r *http.Request, store Store) { + page, err := store.ListSourceConnections(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source connections"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page.Items, "summary": page.Summary}) +} + +func triggerPPSourceScan(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + scan, replayed, err := config.Store.TriggerPPSourceScanWithOperation(r.Context(), r.PathValue("id"), config.Now().UTC(), operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrSourceConnectionNotScannable) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source connection is not scannable"}) + return + } + if errors.Is(err, postgresstore.ErrSourceScanAlreadyActive) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "source scan already active"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not trigger source scan"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusAccepted, map[string]any{"data": scan, "operation": operation}) +} + +func listSourceScans(w http.ResponseWriter, r *http.Request, store Store) { + connectionID := r.PathValue("id") + if connectionID == "" || len(connectionID) > 64 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source connection ID"}) + return + } + limit, offset := 20, 0 + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + offset = value + } + page, err := store.ListSourceScans(r.Context(), connectionID, limit, offset) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list source scans"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page.Items, "total": page.Total, "limit": limit, "offset": offset}) +} + +func getSourceScanDetail(w http.ResponseWriter, r *http.Request, store Store) { + connectionID, scanID := r.PathValue("id"), r.PathValue("scan_id") + if connectionID == "" || len(connectionID) > 64 || scanID == "" || len(scanID) > 64 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid source scan ID"}) + return + } + detail, err := store.GetSourceScanDetail(r.Context(), connectionID, scanID) + if errors.Is(err, postgresstore.ErrSourceScanNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "source scan not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load source scan"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": detail}) +} + +func listReviewCandidates(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.ReviewCandidateFilter{ReviewState: r.URL.Query().Get("review_state"), Limit: 20} + if filter.ReviewState != "" && filter.ReviewState != "pending" && filter.ReviewState != "approved" && filter.ReviewState != "rejected" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "review_state must be pending, approved, or rejected"}) + return + } + limit, offset, ok := parseLimitOffset(w, r, 20) + if !ok { + return + } + filter.Limit, filter.Offset = limit, offset + page, err := store.ListReviewCandidates(r.Context(), filter) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list review candidates"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page.Items, "total": page.Total, "limit": limit, "offset": offset}) +} + +type reviewPublicationRequest struct { + Decision string `json:"decision"` + DecidedVisibility string `json:"decided_visibility"` + ReviewerRef string `json:"reviewer_ref"` + DecisionCode string `json:"decision_code"` +} + +func reviewPublication(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input reviewPublicationRequest + if err := decodeStrictJSON(r, &input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid review request"}) + return + } + review, replayed, err := config.Store.ReviewPublicationWithOperation(r.Context(), governance.ReviewPublicationRequest{ + PublicationID: r.PathValue("id"), Decision: governance.ReviewState(input.Decision), + DecidedVisibility: governance.Visibility(input.DecidedVisibility), ReviewerRef: input.ReviewerRef, + DecisionCode: input.DecisionCode, Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidPublicationReview): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid review request"}) + case errors.Is(err, postgresstore.ErrPublicationReviewNotEligible): + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "publication is not eligible for this review decision"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not record review decision"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusCreated, map[string]any{"data": review, "operation": operation}) + } +} + +type createUISettingsCandidateRequest struct { + DocumentRefreshSeconds int `json:"document_refresh_seconds"` + ResourceRefreshSeconds int `json:"resource_refresh_seconds"` + StalledAfterSeconds int `json:"stalled_after_seconds"` + FinalizingAfterSeconds int `json:"finalizing_after_seconds"` + UpdatedBy string `json:"updated_by"` +} + +func getActiveUISettings(w http.ResponseWriter, r *http.Request, store Store) { + settings, err := store.GetActiveUISettings(r.Context()) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load active UI settings"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": settings}) +} + +func createUISettingsCandidate(w http.ResponseWriter, r *http.Request, config Config) { + if r.Header.Get("X-Governance-Action") != "ui-settings" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "UI settings write intent is required"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var request createUISettingsCandidateRequest + if err := decodeJSONStrict(r, &request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid UI settings candidate"}) + return + } + settings, replayed, err := config.Store.CreateUISettingsCandidateWithOperation(r.Context(), governance.CreateUISettingsCandidateRequest{ + DocumentRefreshSeconds: request.DocumentRefreshSeconds, + ResourceRefreshSeconds: request.ResourceRefreshSeconds, + StalledAfterSeconds: request.StalledAfterSeconds, + FinalizingAfterSeconds: request.FinalizingAfterSeconds, + UpdatedBy: request.UpdatedBy, + Now: config.Now(), + }, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrInvalidUISettings) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid UI settings candidate"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not create UI settings candidate"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusCreated, map[string]any{"data": settings, "operation": operation}) +} + +func activateUISettings(w http.ResponseWriter, r *http.Request, config Config) { + if r.Header.Get("X-Governance-Action") != "ui-settings" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "UI settings write intent is required"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var request struct{} + if err := decodeJSONStrict(r, &request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid UI settings activation"}) + return + } + settings, replayed, err := config.Store.ActivateUISettingsWithOperation(r.Context(), r.PathValue("id"), config.Now(), operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrInvalidUISettings) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid UI settings activation"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not activate UI settings"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": settings, "operation": operation}) +} + +type createReleaseBatchRequest struct { + Name string `json:"name"` + CreatedBy string `json:"created_by"` + PublicationIDs []string `json:"publication_ids"` + DeletePublicationIDs []string `json:"delete_publication_ids"` + ReleaseChangeSetID string `json:"release_change_set_id"` +} + +func createReleaseBatch(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input createReleaseBatchRequest + if err := decodeStrictJSON(r, &input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release batch request"}) + return + } + batch, replayed, err := config.Store.CreateReleaseBatchWithOperation(r.Context(), governance.CreateReleaseBatchRequest{ + Name: input.Name, CreatedBy: input.CreatedBy, PublicationIDs: input.PublicationIDs, DeletePublicationIDs: input.DeletePublicationIDs, ReleaseChangeSetID: input.ReleaseChangeSetID, Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseBatch): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release batch request"}) + case errors.Is(err, postgresstore.ErrReleaseBatchNotEligible): + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "release batch contains ineligible publications"}) + case errors.Is(err, postgresstore.ErrReleaseChangeSetUnconfirmed): + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "release change set must be confirmed and cover every requested publication"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not create release batch"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusCreated, map[string]any{"data": batch, "operation": operation}) + } +} + +type generateReleaseChangeSetRequest struct { + Name string `json:"name"` + CreatedBy string `json:"created_by"` +} + +func getReleaseAuditChain(w http.ResponseWriter, r *http.Request, store Store) { + id := r.PathValue("id") + if !validCatalogID(id) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release execution id"}) + return + } + chain, err := store.GetReleaseAuditChain(r.Context(), id) + switch { + case errors.Is(err, postgresstore.ErrReleaseAuditChainNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release audit chain not found"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load release audit chain"}) + default: + writeJSON(w, http.StatusOK, map[string]any{"data": chain}) + } +} + +func generateReleaseChangeSet(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input generateReleaseChangeSetRequest + if err := decodeStrictJSON(r, &input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release change set request"}) + return + } + set, replayed, err := config.Store.GenerateReleaseChangeSetWithOperation(r.Context(), governance.GenerateReleaseChangeSetRequest{ + Name: input.Name, CreatedBy: input.CreatedBy, Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseChangeSet): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release change set request"}) + case errors.Is(err, postgresstore.ErrReleaseChangeSetEmpty): + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "当前发布计划没有可发布项"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not generate release change set"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusCreated, map[string]any{"data": set, "operation": operation}) + } +} + +func listReleaseChangeSets(w http.ResponseWriter, r *http.Request, store Store) { + limit, offset, ok := parseLimitOffset(w, r, 20) + if !ok { + return + } + page, err := store.ListReleaseChangeSets(r.Context(), governance.ReleaseBatchFilter{Limit: limit, Offset: offset}) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list release change sets"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page}) +} + +func getReleaseChangeSet(w http.ResponseWriter, r *http.Request, store Store, id string) { + if id == "" { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release change set not found"}) + return + } + var detail governance.ReleaseChangeSetDetail + var err error + if id == "latest" { + detail, err = store.LatestReleaseChangeSet(r.Context()) + } else { + if !validCatalogID(id) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release change set id"}) + return + } + detail, err = store.GetReleaseChangeSet(r.Context(), id) + } + switch { + case errors.Is(err, postgresstore.ErrReleaseChangeSetNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release change set not found"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load release change set"}) + default: + writeJSON(w, http.StatusOK, map[string]any{"data": detail}) + } +} + +func confirmReleaseChangeSet(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input struct { + ConfirmedBy string `json:"confirmed_by"` + } + if err := decodeStrictJSON(r, &input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid confirmation request"}) + return + } + id := r.PathValue("id") + set, replayed, err := config.Store.ConfirmReleaseChangeSetWithOperation(r.Context(), governance.ConfirmReleaseChangeSetRequest{ + ChangeSetID: id, ConfirmedBy: input.ConfirmedBy, Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseChangeSet): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid confirmation request"}) + case errors.Is(err, postgresstore.ErrReleaseChangeSetNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release change set not found"}) + case errors.Is(err, postgresstore.ErrReleaseChangeSetStateConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "只有 planned 状态的发布计划可以确认"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not confirm release change set"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": set, "operation": operation}) + } +} + +type sealReleaseBatchRequest struct { + SealedBy string `json:"sealed_by"` +} + +func sealReleaseBatch(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input sealReleaseBatchRequest + if err := decodeStrictJSON(r, &input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid seal request"}) + return + } + batch, replayed, err := config.Store.SealReleaseBatchWithOperation(r.Context(), governance.SealReleaseBatchRequest{ + BatchID: r.PathValue("id"), SealedBy: input.SealedBy, Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseBatch): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid seal request"}) + case errors.Is(err, postgresstore.ErrReleaseBatchGateConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "release batch no longer satisfies the seal gate"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not seal release batch"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": batch, "operation": operation}) + } +} + +type authorizeProductionReleaseRequest struct { + Confirmation string `json:"confirmation"` +} + +func authorizeProductionRelease(w http.ResponseWriter, r *http.Request, config Config) { + if !config.ProductionReleaseRequestsEnabled { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "production release requests are disabled"}) + return + } + if !config.ProductionBackupRestoreVerified { + writeJSON(w, http.StatusPreconditionFailed, map[string]string{"error": "production backup restore verification is required"}) + return + } + if !productionReleaseAuthorized(r, config.ProductionReleaseAuthorizationToken) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "production release authorization required"}) + return + } + var input authorizeProductionReleaseRequest + if err := decodeStrictJSON(r, &input); err != nil || input.Confirmation != "publish-to-production" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid production release authorization"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + execution, replayed, err := config.Store.AuthorizeReleaseBatchWithOperation(r.Context(), governance.AuthorizeReleaseBatchRequest{ + BatchID: r.PathValue("id"), AuthorizedBy: config.ProductionReleaseAuthorizerRef, + AuthorizationCode: "production_release_authorized", Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseAuthorization): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid production release authorization"}) + case errors.Is(err, postgresstore.ErrReleaseAuthorizationGateConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "sealed batch no longer satisfies the production release gate"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not authorize production release"}) + default: + writeJSON(w, http.StatusAccepted, map[string]any{"data": map[string]any{ + "id": execution.ID, "release_batch_id": execution.ReleaseBatchID, + "operation": execution.Operation, "state": execution.State, + "target_environment": execution.TargetEnvironment, "job_id": execution.JobID, + "item_count": execution.ItemCount, "created_at": execution.CreatedAt, + }, "operation": map[bool]string{true: "replayed", false: "created"}[replayed]}) + } +} + +type authorizeProductionRollbackRequest struct { + Confirmation string `json:"confirmation"` +} + +func authorizeProductionRollback(w http.ResponseWriter, r *http.Request, config Config) { + if !config.ProductionReleaseRequestsEnabled { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "production release requests are disabled"}) + return + } + if !productionReleaseAuthorized(r, config.ProductionReleaseAuthorizationToken) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "production release authorization required"}) + return + } + var input authorizeProductionRollbackRequest + if err := decodeStrictJSON(r, &input); err != nil || input.Confirmation != "rollback-production-release" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid production rollback authorization"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + execution, replayed, err := config.Store.AuthorizeReleaseRollbackWithOperation(r.Context(), governance.AuthorizeReleaseRollbackRequest{ + PriorExecutionID: r.PathValue("id"), AuthorizedBy: config.ProductionReleaseAuthorizerRef, + AuthorizationCode: "production_rollback_authorized", Now: config.Now().UTC(), + }, operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrInvalidReleaseAuthorization): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid production rollback authorization"}) + case errors.Is(err, postgresstore.ErrReleaseAuthorizationGateConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "release execution is not eligible for rollback"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not authorize production rollback"}) + default: + writeJSON(w, http.StatusAccepted, map[string]any{"data": map[string]any{ + "id": execution.ID, "release_batch_id": execution.ReleaseBatchID, + "operation": execution.Operation, "state": execution.State, + "target_environment": execution.TargetEnvironment, "job_id": execution.JobID, + "item_count": execution.ItemCount, "created_at": execution.CreatedAt, + }, "operation": map[bool]string{true: "replayed", false: "created"}[replayed]}) + } +} + +type retryProductionReleaseRequest struct { + Confirmation string `json:"confirmation"` +} + +func retryProductionRelease(w http.ResponseWriter, r *http.Request, config Config) { + if !config.ProductionReleaseRequestsEnabled { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "production release requests are disabled"}) + return + } + if !productionReleaseAuthorized(r, config.ProductionReleaseAuthorizationToken) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "production release authorization required"}) + return + } + var input retryProductionReleaseRequest + if err := decodeStrictJSON(r, &input); err != nil || input.Confirmation != "retry-production-release" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid production release retry confirmation"}) + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + execution, replayed, err := config.Store.RetryReleaseExecutionWithOperation(r.Context(), governance.RetryReleaseExecutionRequest{ExecutionID: r.PathValue("id"), Now: config.Now().UTC()}, operationID) + if err != nil { + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, postgresstore.ErrReleaseExecutionRetryConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "release execution cannot be retried"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "retry release execution"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusAccepted, map[string]any{"data": safeReleaseExecution(execution), "operation": operation}) +} + +func productionReleaseAuthorized(r *http.Request, token string) bool { + provided := r.Header.Get("Authorization") + expected := "Bearer " + token + return subtle.ConstantTimeCompare([]byte(provided), []byte(expected)) == 1 +} + +func listReleaseExecutions(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.ReleaseExecutionFilter{Operation: r.URL.Query().Get("operation"), State: r.URL.Query().Get("state"), Limit: 20} + if filter.Operation != "" && !governance.ReleaseOperation(filter.Operation).Valid() { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "operation must be publish or rollback"}) + return + } + if filter.State != "" && !governance.ReleaseExecutionState(filter.State).Valid() { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid release execution state"}) + return + } + limit, offset, ok := parseLimitOffset(w, r, 20) + if !ok { + return + } + filter.Limit, filter.Offset = limit, offset + page, err := store.ListReleaseExecutions(r.Context(), filter) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list release executions"}) + return + } + items := make([]map[string]any, 0, len(page.Items)) + for _, execution := range page.Items { + items = append(items, safeReleaseExecution(execution)) + } + writeJSON(w, http.StatusOK, map[string]any{"data": items, "total": page.Total, "limit": limit, "offset": offset}) +} + +func getReleaseExecution(w http.ResponseWriter, r *http.Request, store Store) { + detail, err := store.GetReleaseExecution(r.Context(), r.PathValue("id")) + if errors.Is(err, postgresstore.ErrReleaseExecutionNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release execution not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load release execution"}) + return + } + result := safeReleaseExecution(detail.ReleaseExecution) + items := make([]map[string]any, 0, len(detail.Items)) + for _, item := range detail.Items { + items = append(items, map[string]any{ + "publication_id": item.PublicationID, "title": item.Title, "state": item.State, + "previous_production_revision": item.PreviousProductionRevision, + "production_revision": item.ProductionRevision, "error_code": item.ErrorCode, + "created_at": item.CreatedAt, "completed_at": item.CompletedAt, + }) + } + result["items"] = items + writeJSON(w, http.StatusOK, map[string]any{"data": result}) +} + +func safeReleaseExecution(execution governance.ReleaseExecution) map[string]any { + return map[string]any{ + "id": execution.ID, "release_batch_id": execution.ReleaseBatchID, + "operation": execution.Operation, "prior_execution_id": execution.PriorExecutionID, + "state": execution.State, "target_environment": execution.TargetEnvironment, + "authorized_by": execution.AuthorizedBy, "job_id": execution.JobID, + "item_count": execution.ItemCount, "completed_item_count": execution.CompletedItemCount, + "failed_item_count": execution.FailedItemCount, "error_code": execution.ErrorCode, + "failure_level": execution.FailureLevel, "restore_trigger": execution.RestoreTrigger, + "release_change_set_id": execution.ReleaseChangeSetID, + "created_at": execution.CreatedAt, "started_at": execution.StartedAt, "completed_at": execution.CompletedAt, + } +} + +func listReleaseBatches(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.ReleaseBatchFilter{State: r.URL.Query().Get("state"), Limit: 20} + if filter.State != "" && filter.State != "draft" && filter.State != "sealed" && filter.State != "cancelled" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "state must be draft, sealed, or cancelled"}) + return + } + limit, offset, ok := parseLimitOffset(w, r, 20) + if !ok { + return + } + filter.Limit, filter.Offset = limit, offset + page, err := store.ListReleaseBatches(r.Context(), filter) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list release batches"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page.Items, "total": page.Total, "limit": limit, "offset": offset}) +} + +func getReleaseBatch(w http.ResponseWriter, r *http.Request, store Store) { + detail, err := store.GetReleaseBatch(r.Context(), r.PathValue("id")) + if errors.Is(err, postgresstore.ErrReleaseBatchNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "release batch not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load release batch"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": detail}) +} + +func listTestPublications(w http.ResponseWriter, r *http.Request, store Store) { + filter := governance.TestPublicationFilter{ProcessingState: r.URL.Query().Get("processing_state"), TargetScope: r.URL.Query().Get("target_scope"), Limit: 50} + if filter.ProcessingState != "" && !validProcessingState(filter.ProcessingState) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "processing_state must be pending, processing, finalizing, completed, failed, or stalled"}) + return + } + if filter.TargetScope != "" && filter.TargetScope != "internal" && filter.TargetScope != "external" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "target_scope must be internal or external"}) + return + } + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return + } + filter.Limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return + } + filter.Offset = value + } + page, err := store.ListTestPublications(r.Context(), filter) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list test publications"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "data": page.Items, "total": page.Total, "limit": filter.Limit, "offset": filter.Offset, + }) +} + +type confirmChangeSetInput struct { + ItemSeqs []int64 `json:"item_seqs"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` +} + +func confirmCatalogChangeSet(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input confirmChangeSetInput + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil || len(input.ItemSeqs) == 0 || len(input.ItemSeqs) > 200 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid change set confirmation"}) + return + } + result, replayed, err := config.Store.ConfirmCatalogChangeSetWithOperation(r.Context(), governance.ConfirmCatalogChangeSetRequest{ + ChangeSetID: r.PathValue("id"), ItemSeqs: input.ItemSeqs, + OperatorRef: input.OperatorRef, Reason: input.Reason, + InternalTargetKBID: config.InternalRoutingKnowledgeBaseID, ExternalTargetKBID: config.ExternalRoutingKnowledgeBaseID, + Now: config.Now().UTC(), + }, operationID) + if err != nil { + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrCatalogChangeSetNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "change set not found"}) + default: + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "could not confirm change set"}) + } + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": result, "operation": operation}) +} + +type startTestPublicationsInput struct { + PublicationIDs []string `json:"publication_ids"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` +} + +func startTestPublications(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input startTestPublicationsInput + if err := decodeStrictJSON(r, &input); err != nil || len(input.PublicationIDs) == 0 || len(input.PublicationIDs) > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid processing confirmation"}) + return + } + operator := strings.TrimSpace(input.OperatorRef) + reason := strings.TrimSpace(input.Reason) + if operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "operator_ref and reason are required"}) + return + } + seen := make(map[string]bool, len(input.PublicationIDs)) + for _, publicationID := range input.PublicationIDs { + if !validCatalogID(publicationID) || seen[publicationID] { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid publication_ids"}) + return + } + seen[publicationID] = true + } + result, replayed, err := config.Store.StartTestPublicationsWithOperation(r.Context(), governance.StartTestPublicationsRequest{ + PublicationIDs: input.PublicationIDs, + OperatorRef: operator, + Reason: reason, + Now: config.Now().UTC(), + }, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not start test processing"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusAccepted, map[string]any{"data": result, "operation": operation}) +} + +func listAcceptance(w http.ResponseWriter, r *http.Request, store Store) { + records, err := store.ListAcceptanceResults(r.Context(), r.PathValue("id")) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list acceptance results"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": records}) +} + +func listJobFailures(w http.ResponseWriter, r *http.Request, store Store) { + limit, offset, ok := parseLimitOffset(w, r, 50) + if !ok { + return + } + page, err := store.ListJobFailures(r.Context(), governance.JobFailureFilter{ + State: r.URL.Query().Get("state"), Limit: limit, Offset: offset, + }) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid failure filter"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": page}) +} + +func retryPublicationsBatch(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input startTestPublicationsInput + if err := decodeStrictJSON(r, &input); err != nil || len(input.PublicationIDs) == 0 || len(input.PublicationIDs) > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid retry batch"}) + return + } + operator := strings.TrimSpace(input.OperatorRef) + reason := strings.TrimSpace(input.Reason) + if operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "operator_ref and reason are required"}) + return + } + seen := make(map[string]bool, len(input.PublicationIDs)) + for _, publicationID := range input.PublicationIDs { + if !validCatalogID(publicationID) || seen[publicationID] { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid publication_ids"}) + return + } + seen[publicationID] = true + } + result, replayed, err := config.Store.RetryPublicationsBatchWithOperation(r.Context(), governance.RetryPublicationsBatchRequest{ + PublicationIDs: input.PublicationIDs, + OperatorRef: operator, + Reason: reason, + Now: config.Now().UTC(), + }, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not retry publications"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": result, "operation": operation}) +} + +func retryPublication(w http.ResponseWriter, r *http.Request, config Config) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + publicationID := r.PathValue("id") + replayed, err := config.Store.RetryPublicationWithOperation(r.Context(), publicationID, config.Now().UTC(), operationID) + switch { + case errors.Is(err, postgresstore.ErrOperationRequestConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + case errors.Is(err, postgresstore.ErrPublicationNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "publication not found"}) + case errors.Is(err, postgresstore.ErrPublicationStateConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "publication is not in a retryable state"}) + case errors.Is(err, postgresstore.ErrPublicationNotRetryable): + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "publication has no verified spool artifact"}) + case err != nil: + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not retry publication"}) + default: + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]string{"id": publicationID, "status": "retry_queued"}, "operation": operation}) + } +} + +func parseLimitOffset(w http.ResponseWriter, r *http.Request, defaultLimit int) (int, int, bool) { + limit, offset := defaultLimit, 0 + if raw := r.URL.Query().Get("limit"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "limit must be between 1 and 100"}) + return 0, 0, false + } + limit = value + } + if raw := r.URL.Query().Get("offset"); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 0 || value > 1000000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "offset must be between 0 and 1000000"}) + return 0, 0, false + } + offset = value + } + return limit, offset, true +} + +func decodeStrictJSON(r *http.Request, destination any) error { + decoder := json.NewDecoder(io.LimitReader(r.Body, 64<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return err + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return errors.New("trailing JSON value") + } + return nil +} + +func validProcessingState(state string) bool { + switch state { + case "pending", "processing", "finalizing", "completed", "failed", "stalled": + return true + default: + return false + } +} + +func newID() (string, error) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", fmt.Errorf("read random ID: %w", err) + } + raw[6] = (raw[6] & 0x0f) | 0x40 + raw[8] = (raw[8] & 0x3f) | 0x80 + encoded := hex.EncodeToString(raw[:]) + return encoded[0:8] + "-" + encoded[8:12] + "-" + encoded[12:16] + "-" + encoded[16:20] + "-" + encoded[20:32], nil +} + +func decodeJSONStrict(r *http.Request, target any) error { + decoder := json.NewDecoder(io.LimitReader(r.Body, 16*1024)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("multiple JSON values are not allowed") + } + return err + } + return nil +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go new file mode 100644 index 0000000..11f9f2d --- /dev/null +++ b/internal/httpapi/server_test.go @@ -0,0 +1,1888 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +func TestHandlerHealth(t *testing.T) { + handler, err := NewHandler(Config{Store: &fakeJobStore{}}) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "/health", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", response.Code) + } + if got := response.Header().Get("Content-Type"); got != "application/json" { + t.Fatalf("content type = %q", got) + } + if got := response.Body.String(); got != "{\"status\":\"ok\"}\n" { + t.Fatalf("body = %q", got) + } +} + +func TestHandlerCreatesJob(t *testing.T) { + now := time.Date(2026, time.September, 3, 13, 0, 0, 0, time.UTC) + store := &fakeJobStore{enqueueResult: &jobs.Job{ID: "job-fixed", Type: "demo", State: jobs.StatePending}} + handler, err := NewHandler(Config{ + Store: store, + Now: func() time.Time { return now }, + NewID: func() (string, error) { return "job-fixed", nil }, + }) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + + body := bytes.NewBufferString(`{"type":"demo","payload":{"message":"hello"},"idempotency_key":"demo:fixed"}`) + request := httptest.NewRequest(http.MethodPost, "/api/v1/jobs", body) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201; body=%s", response.Code, response.Body.String()) + } + if store.enqueued.ID != "job-fixed" || store.enqueued.Type != "demo" || store.enqueued.MaxAttempts != 5 { + t.Fatalf("enqueued = %#v", store.enqueued) + } + if store.enqueued.IdempotencyKey != "demo:fixed" || !store.enqueued.AvailableAt.Equal(now) { + t.Fatalf("enqueued idempotency/time = %#v", store.enqueued) + } +} + +func TestHandlerTestPublicationQueueTargetScopeFilter(t *testing.T) { + store := &fakeJobStore{publicationResult: governance.TestPublicationPage{Total: 0}} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/test-publications?target_scope=internal", nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("filter status = %d body %s", recorder.Code, recorder.Body.String()) + } + if store.publicationFilter.TargetScope != "internal" { + t.Fatalf("filter = %+v", store.publicationFilter) + } + invalid := httptest.NewRequest(http.MethodGet, "/api/v1/test-publications?target_scope=prod", nil) + bad := httptest.NewRecorder() + handler.ServeHTTP(bad, invalid) + if bad.Code != http.StatusBadRequest { + t.Fatalf("invalid scope status = %d", bad.Code) + } +} + +func TestHandlerListsJobFailuresAndRunsServerSideBatchRetry(t *testing.T) { + store := &fakeJobStore{ + jobFailurePage: governance.JobFailurePage{ + Items: []governance.JobFailure{{ID: "jf-1", JobID: "job-9", JobType: "test.upload", PublicationID: "tp-1", ErrorCode: "remote_permanent", Attempt: 5, MaxAttempts: 5, State: "open"}}, + Total: 1, Limit: 50, + }, + retryBatchResult: governance.RetryPublicationsBatchResult{ + BatchKey: "op-batch-1", Total: 2, Started: 1, AlreadyRunning: 1, + Outcomes: []governance.RetryOutcome{{PublicationID: "tp-1", Result: "started"}, {PublicationID: "tp-2", Result: "already_running"}}, + }, + } + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/job-failures?state=open&limit=50", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("list failures status = %d body %s", recorder.Code, recorder.Body.String()) + } + if store.jobFailureFilter.State != "open" || store.jobFailureFilter.Limit != 50 { + t.Fatalf("filter = %+v", store.jobFailureFilter) + } + if !strings.Contains(recorder.Body.String(), "\"data\"") { + t.Fatalf("job failures must be an envelope: %s", recorder.Body.String()) + } + + recorder = httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/retry-batch", + strings.NewReader(`{"publication_ids":["tp-1","tp-2"],"operator_ref":"姚工","reason":"解析服务恢复"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "op-batch-1") + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("retry-batch status = %d body %s", recorder.Code, recorder.Body.String()) + } + var envelope struct { + Data governance.RetryPublicationsBatchResult `json:"data"` + Operation string `json:"operation"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Operation != "created" || envelope.Data.Started != 1 || envelope.Data.AlreadyRunning != 1 { + t.Fatalf("envelope = %+v", envelope) + } + if store.retryBatchOperation != "op-batch-1" || store.retryBatchRequest == nil || len(store.retryBatchRequest.PublicationIDs) != 2 { + t.Fatalf("batch request = %+v op=%q", store.retryBatchRequest, store.retryBatchOperation) + } + if store.retryBatchRequest.Now.IsZero() { + t.Fatal("retry batch request now not set") + } + + // Missing Idempotency-Key is rejected before touching the store. + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/retry-batch", + strings.NewReader(`{"publication_ids":["tp-1"],"operator_ref":"姚工","reason":"r"}`))) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("missing key status = %d", recorder.Code) + } + // Duplicate ids rejected. + recorder = httptest.NewRecorder() + dup := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/retry-batch", + strings.NewReader(`{"publication_ids":["tp-1","tp-1"],"operator_ref":"姚工","reason":"r"}`)) + dup.Header.Set("Idempotency-Key", "op-dup") + handler.ServeHTTP(recorder, dup) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("duplicate ids status = %d", recorder.Code) + } +} + +func TestHandlerListsJobsWithJSONPayload(t *testing.T) { + store := &fakeJobStore{listResult: []jobs.Job{{ + ID: "job-6", + Type: "demo", + State: jobs.StateCompleted, + Payload: []byte(`{"message":"done"}`), + }}} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "/api/v1/jobs?limit=10", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", response.Code, response.Body.String()) + } + want := `"id":"job-6"` + if !bytes.Contains(response.Body.Bytes(), []byte(want)) { + t.Fatalf("body missing %s: %s", want, response.Body.String()) + } + if !bytes.Contains(response.Body.Bytes(), []byte(`"payload":{"message":"done"}`)) { + t.Fatalf("payload is not embedded JSON: %s", response.Body.String()) + } + if store.listLimit != 10 { + t.Fatalf("list limit = %d, want 10", store.listLimit) + } +} + +func TestHandlerListsSourceAssetsWithFilterAndSummary(t *testing.T) { + store := &fakeJobStore{assetResult: governance.SourceAssetPage{ + Items: []governance.SourceAsset{{ID: "si-safe", SourceType: governance.SourceTypePP, SizeBytes: 52428801, ContentState: governance.ContentOverLimit, LegacyStatus: "over_limit"}}, + Summary: governance.SourceAssetSummary{Total: 613, Completed: 530, ContentAlias: 29, OverLimit: 54}, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + + request := httptest.NewRequest(http.MethodGet, "/api/v1/source-items?state=over_limit&limit=25&offset=50", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", response.Code, response.Body.String()) + } + if !bytes.Contains(response.Body.Bytes(), []byte(`"total":613`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"content_state":"over_limit"`)) { + t.Fatalf("body = %s", response.Body.String()) + } + if store.assetFilter != (governance.SourceAssetFilter{State: "over_limit", Limit: 25, Offset: 50}) { + t.Fatalf("filter = %#v", store.assetFilter) + } +} + +func TestHandlerSetsListsAndClearsAuditedScopeRules(t *testing.T) { + now := time.Date(2026, 9, 8, 2, 0, 0, 0, time.UTC) + store := &fakeJobStore{scopeRuleResult: governance.ScopeRule{TargetType: governance.ScopeTargetDirectory, ConnectionID: "conn-safe", DirectoryPath: "产品", Visibility: governance.VisibilityInternal, UpdatedBy: "engineer-1", Reason: "内部资料", UpdatedAt: now}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + set := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPut, "/api/v1/scope-rules", bytes.NewBufferString(`{"target_type":"directory","connection_id":"conn-safe","directory_path":"产品","visibility":"internal","operator_ref":"engineer-1","reason":"内部资料"}`)) + request.Header.Set("Idempotency-Key", "op-scope-set") + handler.ServeHTTP(set, request) + if set.Code != http.StatusOK || store.scopeRuleRequest.ConnectionID != "conn-safe" || store.scopeRuleOperationID != "op-scope-set" || store.scopeRuleRequest.Clear || store.scopeRuleRequest.Now != now || !strings.Contains(set.Body.String(), `"operation":"created"`) { + t.Fatalf("set=%d request=%#v body=%s", set.Code, store.scopeRuleRequest, set.Body.String()) + } + store.scopeRules = []governance.ScopeRule{store.scopeRuleResult} + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/api/v1/scope-rules?connection_id=conn-safe", nil)) + if list.Code != http.StatusOK || store.scopeRuleConnectionID != "conn-safe" || !strings.Contains(list.Body.String(), `"directory_path":"产品"`) { + t.Fatalf("list=%d id=%q body=%s", list.Code, store.scopeRuleConnectionID, list.Body.String()) + } + clear := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodDelete, "/api/v1/scope-rules", bytes.NewBufferString(`{"target_type":"directory","connection_id":"conn-safe","directory_path":"产品","operator_ref":"engineer-1","reason":"恢复来源设置"}`)) + request.Header.Set("Idempotency-Key", "op-scope-clear") + handler.ServeHTTP(clear, request) + if clear.Code != http.StatusOK || !store.scopeRuleRequest.Clear || store.scopeRuleRequest.Visibility != "" || !strings.Contains(clear.Body.String(), `"operation":"created"`) { + t.Fatalf("clear=%d request=%#v body=%s", clear.Code, store.scopeRuleRequest, clear.Body.String()) + } +} + +func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) { + store := &fakeJobStore{} + handler, _ := NewHandler(Config{Store: store, Now: time.Now}) + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPut, "/api/v1/scope-rules", bytes.NewBufferString(`{"target_type":"directory","connection_id":"conn-safe","directory_path":"../private","visibility":"external","operator_ref":"engineer","reason":"bad"}`)) + handler.ServeHTTP(response, request) + if response.Code != http.StatusBadRequest || store.scopeRuleRequest.TargetType != "" { + t.Fatalf("status=%d request=%#v", response.Code, store.scopeRuleRequest) + } +} + +func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) { + now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC) + hash := strings.Repeat("c", 64) + store := &fakeJobStore{ + snapshotPageResult: governance.CatalogSnapshotPage{Items: []governance.CatalogSnapshot{{SnapshotID: "ss-a", ConnectionID: "conn-cat", RootNodeID: "root", SourceName: "PP", NodeCount: 2, FolderCount: 1, FileCount: 1, ObservedAt: now, CreatedAt: now, IsHead: true}}, Total: 1, Limit: 20}, + snapshotNodePageResult: governance.CatalogNodePage{Current: governance.CatalogNode{NodeID: "root", NodeType: "folder", Name: "PP", Visibility: "unknown"}, Items: []governance.CatalogNode{{NodeID: "file-1", ParentNodeID: "root", NodeType: "file", Name: "手册.pdf", Visibility: "unknown", ContentSHA256: &hash}}, Total: 1, Limit: 50}, + } + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/snapshots?connection_id=conn-cat", nil)) + if list.Code != http.StatusOK || store.snapshotConnectionID != "conn-cat" || !strings.Contains(list.Body.String(), `"is_head":true`) { + t.Fatalf("list=%d id=%q body=%s", list.Code, store.snapshotConnectionID, list.Body.String()) + } + if strings.Contains(list.Body.String(), "snapshot_sha256") { + t.Fatalf("snapshot history leaked internal hash: %s", list.Body.String()) + } + nodes := httptest.NewRecorder() + handler.ServeHTTP(nodes, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/snapshots/ss-old/nodes?connection_id=conn-cat&parent_id=root", nil)) + if nodes.Code != http.StatusOK || store.snapshotNodeFilter.SnapshotID != "ss-old" || store.snapshotNodeFilter.ParentNodeID != "root" || !strings.Contains(nodes.Body.String(), `"content_sha256":"`) { + t.Fatalf("nodes=%d filter=%#v body=%s", nodes.Code, store.snapshotNodeFilter, nodes.Body.String()) + } + bad := httptest.NewRecorder() + handler.ServeHTTP(bad, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/snapshots/..%2Fprivate/nodes?connection_id=conn-cat&parent_id=root", nil)) + if bad.Code != http.StatusBadRequest { + t.Fatalf("bad snapshot id status=%d", bad.Code) + } +} + +func TestHandlerBrowsesCatalogSourcesAndOnlyDirectChildren(t *testing.T) { + now := time.Date(2026, 9, 8, 10, 0, 0, 0, time.UTC) + store := &fakeJobStore{ + catalogSources: []governance.CatalogSource{{ConnectionID: "conn-cat", SourceType: governance.SourceTypeDingTalk, Name: "公司级技术支持知识库", RootNodeID: "root-cat", FolderCount: 8, FileCount: 3468, ObservedAt: now, SnapshotOnly: true}}, + catalogPage: governance.CatalogNodePage{Current: governance.CatalogNode{NodeID: "root-cat", NodeType: "folder", Name: "公司级技术支持知识库"}, Breadcrumbs: []governance.CatalogBreadcrumb{{NodeID: "root-cat", Name: "公司级技术支持知识库"}}, Items: []governance.CatalogNode{{NodeID: "folder-cat", ParentNodeID: "root-cat", NodeType: "folder", Name: "01 产品总览与定位", SubtreeFileCount: 10, Visibility: "internal", ImportedCount: 8, ParsedCount: 7}}, Total: 1, Limit: 50}, + } + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + sources := httptest.NewRecorder() + handler.ServeHTTP(sources, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/sources", nil)) + if sources.Code != http.StatusOK || !strings.Contains(sources.Body.String(), `"name":"公司级技术支持知识库"`) { + t.Fatalf("sources=%d body=%s", sources.Code, sources.Body.String()) + } + nodes := httptest.NewRecorder() + handler.ServeHTTP(nodes, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/nodes?connection_id=conn-cat&parent_id=root-cat&limit=50&offset=0", nil)) + if nodes.Code != http.StatusOK || store.catalogFilter != (governance.CatalogNodeFilter{ConnectionID: "conn-cat", ParentNodeID: "root-cat", Limit: 50, Offset: 0}) || !strings.Contains(nodes.Body.String(), `"name":"01 产品总览与定位"`) { + t.Fatalf("nodes=%d filter=%#v body=%s", nodes.Code, store.catalogFilter, nodes.Body.String()) + } + for _, forbidden := range []string{"relative_path", "source_locator_hash", "snapshot_sha256", "stable_external_id"} { + if strings.Contains(nodes.Body.String()+sources.Body.String(), forbidden) { + t.Fatalf("catalog API leaked %s", forbidden) + } + } +} + +func TestHandlerPreviewsLatestPersistedCatalogChangeSet(t *testing.T) { + now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC) + store := &fakeJobStore{catalogChangeSetResult: governance.CatalogChangeSetDetail{ + CatalogChangeSet: governance.CatalogChangeSet{ID: "cs-safe", ConnectionID: "conn-cat", SourceName: "国内工业", State: "planned", AddCount: 2, UpdateCount: 1, MoveScopeCount: 1, DeleteCount: 1, NoChangeCount: 10, CreatedAt: now}, + Items: []governance.CatalogChangeItem{{Action: "delete", Title: "旧手册.pdf", OldPath: "旧目录/旧手册.pdf"}}, Total: 1, Limit: 20, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/change-sets/latest?connection_id=conn-cat&action=delete&limit=20&offset=0", nil)) + if response.Code != http.StatusOK || store.catalogChangeSetFilter.ConnectionID != "conn-cat" || store.catalogChangeSetFilter.Action != "delete" || !strings.Contains(response.Body.String(), `"title":"旧手册.pdf"`) || !strings.Contains(response.Body.String(), `"add_count":2`) { + t.Fatalf("status=%d filter=%#v body=%s", response.Code, store.catalogChangeSetFilter, response.Body.String()) + } + for _, forbidden := range []string{"source_locator_hash", "snapshot_sha256", "secret_ref"} { + if strings.Contains(response.Body.String(), forbidden) { + t.Fatalf("change-set response leaked %s", forbidden) + } + } +} + +func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) { + store := &fakeJobStore{} + handler, _ := NewHandler(Config{Store: store}) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/nodes?connection_id=conn-cat&parent_id=../private", nil)) + if response.Code != http.StatusBadRequest || store.catalogFilter.ParentNodeID != "" { + t.Fatalf("status=%d filter=%#v", response.Code, store.catalogFilter) + } +} + +func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) { + now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC) + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/route", bytes.NewBufferString(`{"connection_id":"conn-cat","node_ids":["file-a","file-b"],"visibility":"internal","operator_ref":"engineer-1","reason":"当前页批量标注"}`)) + request.Header.Set("Idempotency-Key", "op-route-internal") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String()) + } + if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) { + t.Fatalf("unsafe route response=%s", response.Body.String()) + } + capabilities := httptest.NewRecorder() + handler.ServeHTTP(capabilities, httptest.NewRequest(http.MethodGet, "/api/v1/source-catalog/routing-capabilities", nil)) + if capabilities.Code != http.StatusOK || !strings.Contains(capabilities.Body.String(), `"internal_enabled":true`) || strings.Contains(capabilities.Body.String(), "kb-") { + t.Fatalf("capabilities=%s", capabilities.Body.String()) + } +} + +func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) { + now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC) + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/route", bytes.NewBufferString(`{"connection_id":"conn-cat","node_ids":["file-a"],"policy":"both","operator_ref":"engineer-1","reason":"同时供内外知识库使用"}`)) + request.Header.Set("Idempotency-Key", "op-route-both") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBoth { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String()) + } + if got := store.catalogRouteRequest.TargetKnowledgeBaseIDs; len(got) != 2 || got[0] != "kb-internal" || got[1] != "kb-external" { + t.Fatalf("target knowledge bases = %#v", got) + } + if strings.Contains(response.Body.String(), "kb-internal") || strings.Contains(response.Body.String(), "kb-external") { + t.Fatalf("route response leaked target identity: %s", response.Body.String()) + } +} + +func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) { + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/route", bytes.NewBufferString(`{"connection_id":"conn-cat","node_ids":["file-a"],"policy":"blocked","operator_ref":"engineer-1","reason":"资料已禁止导入"}`)) + request.Header.Set("Idempotency-Key", "op-route-blocked") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"routable_files":1`) { + t.Fatalf("blocked route response=%s", response.Body.String()) + } +} + +func TestHandlerEnqueuesCatalogRefreshWithSafeSourceIdentity(t *testing.T) { + now := time.Date(2026, 9, 9, 2, 0, 0, 0, time.UTC) + store := &fakeJobStore{ + catalogSources: []governance.CatalogSource{{ConnectionID: "conn-cat", SourceType: governance.SourceTypePP, Name: "PP资料下载中心", RootNodeID: "root-cat"}}, + catalogRefreshResult: jobs.Job{ID: "job-refresh", Type: "source.catalog.refresh", State: jobs.StatePending, Payload: json.RawMessage(`{"connection_id":"conn-cat","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新"}`)}, + } + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/sources/conn-cat/refresh", nil) + request.Header.Set("Idempotency-Key", "op-refresh-safe") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.catalogRefreshSource.ConnectionID != "conn-cat" || store.catalogRefreshOperationID != "op-refresh-safe" || !strings.Contains(response.Body.String(), `"operation":"created"`) { + t.Fatalf("status=%d source=%#v operation_id=%q body=%s", response.Code, store.catalogRefreshSource, store.catalogRefreshOperationID, response.Body.String()) + } + for _, forbidden := range []string{"base_url", "secret", "capture", "token"} { + if strings.Contains(strings.ToLower(response.Body.String()), forbidden) { + t.Fatalf("refresh response leaked %s", forbidden) + } + } +} + +func TestHandlerSourceSettingsRevisionEndpoints(t *testing.T) { + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + + forbidden := httptest.NewRecorder() + handler.ServeHTTP(forbidden, httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions", strings.NewReader(`{"source_type":"alist","name":"n","operator":"o","schedule":{}}`))) + if forbidden.Code != http.StatusForbidden { + t.Fatalf("intent header gate = %d, want 403", forbidden.Code) + } + + missingKey := httptest.NewRecorder() + request := httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions", strings.NewReader(`{"source_type":"alist","name":"国内工业AList","operator":"zherom","enabled":true,"schedule":{"refresh_interval_seconds":3600}}`)) + request.Header.Set("X-Governance-Action", "source-settings") + handler.ServeHTTP(missingKey, request) + if missingKey.Code != http.StatusBadRequest { + t.Fatalf("missing key = %d, want 400", missingKey.Code) + } + + created := httptest.NewRecorder() + request = httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions", strings.NewReader(`{"source_type":"alist","name":"国内工业AList","operator":"zherom","enabled":true,"schedule":{"refresh_interval_seconds":3600}}`)) + request.Header.Set("X-Governance-Action", "source-settings") + request.Header.Set("Idempotency-Key", "create-v1") + handler.ServeHTTP(created, request) + if created.Code != http.StatusCreated { + t.Fatalf("create = %d body=%s", created.Code, created.Body.String()) + } + if store.createSettingsConnectionID != "conn-1" || store.createSettingsOperationID != "create-v1" || store.createSettingsDraft.Name != "国内工业AList" || store.createSettingsDraft.Operator != "zherom" { + t.Fatalf("create request = %s %+v op=%s", store.createSettingsConnectionID, store.createSettingsDraft, store.createSettingsOperationID) + } + if !strings.Contains(created.Body.String(), "scr-fake") || !strings.Contains(created.Body.String(), "created") { + t.Fatalf("create body=%s", created.Body.String()) + } + + store.createSettingsErr = postgresstore.ErrInvalidSourceSettings + rejected := httptest.NewRecorder() + request = httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions", strings.NewReader(`{"source_type":"ftp","name":"x","operator":"o","schedule":{}}`)) + request.Header.Set("X-Governance-Action", "source-settings") + request.Header.Set("Idempotency-Key", "create-v2") + handler.ServeHTTP(rejected, request) + if rejected.Code != http.StatusBadRequest { + t.Fatalf("validation = %d want 400", rejected.Code) + } + + store.createSettingsErr = postgresstore.ErrOperationRequestConflict + conflicted := httptest.NewRecorder() + request = httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions", strings.NewReader(`{"source_type":"alist","name":"x","operator":"o","schedule":{}}`)) + request.Header.Set("X-Governance-Action", "source-settings") + request.Header.Set("Idempotency-Key", "create-v3") + handler.ServeHTTP(conflicted, request) + if conflicted.Code != http.StatusConflict { + t.Fatalf("conflict = %d want 409", conflicted.Code) + } + + store.createSettingsErr = nil + activated := httptest.NewRecorder() + request = httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions/scr-1/activate", strings.NewReader(`{}`)) + request.Header.Set("X-Governance-Action", "source-settings") + request.Header.Set("Idempotency-Key", "activate-v1") + handler.ServeHTTP(activated, request) + if activated.Code != http.StatusOK || store.activateSettingsConnectionID != "conn-1" || store.activateSettingsRevisionID != "scr-1" || store.activateSettingsOperationID != "activate-v1" { + t.Fatalf("activate = %d conn=%s rev=%s op=%s", activated.Code, store.activateSettingsConnectionID, store.activateSettingsRevisionID, store.activateSettingsOperationID) + } + + store.activateSettingsErr = postgresstore.ErrInvalidSourceSettings + badActivate := httptest.NewRecorder() + request = httptest.NewRequest("POST", "/api/v1/source-connections/conn-1/settings/revisions/scr-9/activate", strings.NewReader(`{}`)) + request.Header.Set("X-Governance-Action", "source-settings") + request.Header.Set("Idempotency-Key", "activate-v2") + handler.ServeHTTP(badActivate, request) + if badActivate.Code != http.StatusBadRequest { + t.Fatalf("bad activate = %d want 400", badActivate.Code) + } + + store.activateSettingsErr = nil + store.settingsRevisions = []governance.SourceConnectionRevision{{ID: "scr-2", RevisionNumber: 2, State: "active", Name: "国内工业AList", SecretRef: "alist-file", Enabled: true, RefreshIntervalSeconds: "3600", CreatedBy: "zherom"}} + history := httptest.NewRecorder() + handler.ServeHTTP(history, httptest.NewRequest("GET", "/api/v1/source-connections/conn-1/settings/revisions", nil)) + if history.Code != http.StatusOK { + t.Fatalf("history = %d", history.Code) + } + body := history.Body.String() + if !strings.Contains(body, "alist-file") || !strings.Contains(body, "3600") { + t.Fatalf("history body=%s", body) + } +} + +func TestHandlerReleaseAuditChain(t *testing.T) { + store := &fakeJobStore{auditChain: governance.ReleaseAuditChain{ + ExecutionID: "rx-1", Operation: "publish", State: "completed", + BatchID: "rb-1", BatchName: "9月批次", ReleaseChangeSetID: "rcs-1", + ChangeSetName: "9月发布", ManifestSHA256: strings.Repeat("a", 64), + Snapshots: []governance.AuditSnapshot{{ID: "snap-1", SourceName: "PP资料下载中心", NodeCount: 66, FileCount: 621}}, + CatalogChangeSets: []governance.AuditCatalogSet{{ID: "cs-1", State: "applied", AddCount: 2, MoveCount: 1}}, + Items: []governance.AuditChainItem{{PublicationID: "tp-1", Title: "手册.pdf", Action: "add", ItemState: "verified", ReviewID: "rev-1", AcceptanceRunID: "run-1", SourceItemID: "si-1", SourceVersionID: "siv-1", ProductionRevision: "rev-tp-1"}}, + }} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return time.Now() }}) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/release-executions/rx-1/audit-chain", nil)) + if rec.Code != http.StatusOK || store.auditChainID != "rx-1" { + t.Fatalf("audit chain = %d %q body=%s", rec.Code, store.auditChainID, rec.Body.String()) + } + body := rec.Body.String() + if !strings.Contains(body, `"release_batch_name":"9月批次"`) || !strings.Contains(body, `"snapshots"`) || !strings.Contains(body, `"items"`) || !strings.Contains(body, `"catalog_change_sets":[{"id":"cs-1"`) { + t.Fatalf("incomplete chain body=%s", body) + } + // R6: the projection must never leak bundle paths, base URLs, credentials. + for _, forbidden := range []string{"spool/", "http://", "https://", "bundle_path", "api_key", "secret_ref", "cookie", "bearer", "password"} { + if strings.Contains(strings.ToLower(body), forbidden) { + t.Fatalf("audit chain leaks %q body=%s", forbidden, body) + } + } + // 404 path. + store.auditChainErr = postgresstore.ErrReleaseAuditChainNotFound + missing := httptest.NewRecorder() + handler.ServeHTTP(missing, httptest.NewRequest(http.MethodGet, "/api/v1/release-executions/rx-missing/audit-chain", nil)) + if missing.Code != http.StatusNotFound { + t.Fatalf("missing = %d %s", missing.Code, missing.Body.String()) + } +} + +func TestHandlerReleaseChangeSetEndpoints(t *testing.T) { + store := &fakeJobStore{generateChangeSetResult: governance.ReleaseChangeSet{ID: "rcs-1", ManifestSHA256: strings.Repeat("a", 64)}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return time.Date(2026, 9, 12, 12, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + body := strings.NewReader(`{"name":"9月发布","created_by":"yaozhanghao"}`) + generated := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-change-sets", body) + request.Header.Set("Idempotency-Key", "11111111-1111-4111-8111-111111111111") + handler.ServeHTTP(generated, request) + if generated.Code != http.StatusCreated || store.generateChangeSetRequest.Name != "9月发布" || store.generateChangeSetOperationID == "" { + t.Fatalf("generate = %d %#v %q body=%s", generated.Code, store.generateChangeSetRequest, store.generateChangeSetOperationID, generated.Body.String()) + } + // Missing header → 400 (C4 contract). + noKey := httptest.NewRecorder() + handler.ServeHTTP(noKey, httptest.NewRequest(http.MethodPost, "/api/v1/release-change-sets", strings.NewReader(`{"name":"x","created_by":"y"}`))) + if noKey.Code != http.StatusBadRequest { + t.Fatalf("generate without key = %d", noKey.Code) + } + // Empty plan → 422 honest reason. + store.generateChangeSetErr = postgresstore.ErrReleaseChangeSetEmpty + empty := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPost, "/api/v1/release-change-sets", strings.NewReader(`{"name":"x","created_by":"y"}`)) + request.Header.Set("Idempotency-Key", "22222222-2222-4222-8222-222222222222") + handler.ServeHTTP(empty, request) + if empty.Code != http.StatusUnprocessableEntity { + t.Fatalf("empty plan = %d %s", empty.Code, empty.Body.String()) + } + store.generateChangeSetErr = nil + // Confirm path with envelope. + store.confirmReleaseSetResult = governance.ReleaseChangeSet{ID: "rcs-1", State: "confirmed"} + confirmed := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPost, "/api/v1/release-change-sets/rcs-1/confirm", strings.NewReader(`{"confirmed_by":"yaozhanghao"}`)) + request.Header.Set("Idempotency-Key", "33333333-3333-4333-8333-333333333333") + handler.ServeHTTP(confirmed, request) + if confirmed.Code != http.StatusOK || store.confirmReleaseSetRequest.ChangeSetID != "rcs-1" || !strings.Contains(confirmed.Body.String(), `"operation":"created"`) { + t.Fatalf("confirm = %d %#v body=%s", confirmed.Code, store.confirmReleaseSetRequest, confirmed.Body.String()) + } + // State conflict → 409. + store.confirmReleaseSetErr = postgresstore.ErrReleaseChangeSetStateConflict + conflict := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPost, "/api/v1/release-change-sets/rcs-1/confirm", strings.NewReader(`{"confirmed_by":"yaozhanghao"}`)) + request.Header.Set("Idempotency-Key", "44444444-4444-4444-8444-444444444444") + handler.ServeHTTP(conflict, request) + if conflict.Code != http.StatusConflict { + t.Fatalf("confirm conflict = %d", conflict.Code) + } + // Detail endpoints. + store.changeSetDetail = governance.ReleaseChangeSetDetail{ReleaseChangeSet: governance.ReleaseChangeSet{ID: "rcs-1"}, Items: []governance.ReleaseChangeSetItem{{Seq: 1, PublicationID: "tp-1", Action: "add", CanRelease: true}}} + detail := httptest.NewRecorder() + handler.ServeHTTP(detail, httptest.NewRequest(http.MethodGet, "/api/v1/release-change-sets/rcs-1", nil)) + if detail.Code != http.StatusOK || !strings.Contains(detail.Body.String(), `"publication_id":"tp-1"`) { + t.Fatalf("detail = %d %s", detail.Code, detail.Body.String()) + } + latest := httptest.NewRecorder() + handler.ServeHTTP(latest, httptest.NewRequest(http.MethodGet, "/api/v1/release-change-sets/latest", nil)) + if latest.Code != http.StatusOK { + t.Fatalf("latest = %d %s", latest.Code, latest.Body.String()) + } + store.changeSetDetailErr = postgresstore.ErrReleaseChangeSetNotFound + missing := httptest.NewRecorder() + handler.ServeHTTP(missing, httptest.NewRequest(http.MethodGet, "/api/v1/release-change-sets/rcs-404", nil)) + if missing.Code != http.StatusNotFound { + t.Fatalf("missing = %d", missing.Code) + } + // Batch citing a change set passes the binding through; unconfirmed → 422. + store.createBatchResult = governance.ReleaseBatch{ID: "rb-1"} + batch := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPost, "/api/v1/release-batches", strings.NewReader(`{"name":"批次","created_by":"yaozhanghao","publication_ids":["tp-1"],"delete_publication_ids":[],"release_change_set_id":"rcs-1"}`)) + request.Header.Set("Idempotency-Key", "55555555-5555-4555-8555-555555555555") + handler.ServeHTTP(batch, request) + if batch.Code != http.StatusCreated || store.createBatchRequest.ReleaseChangeSetID != "rcs-1" { + t.Fatalf("batch = %d %#v", batch.Code, store.createBatchRequest) + } + store.createBatchErr = postgresstore.ErrReleaseChangeSetUnconfirmed + unconf := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodPost, "/api/v1/release-batches", strings.NewReader(`{"name":"批次","created_by":"yaozhanghao","publication_ids":["tp-1"],"delete_publication_ids":[],"release_change_set_id":"rcs-9"}`)) + request.Header.Set("Idempotency-Key", "66666666-6666-4666-8666-666666666666") + handler.ServeHTTP(unconf, request) + if unconf.Code != http.StatusUnprocessableEntity || !strings.Contains(unconf.Body.String(), "confirmed") { + t.Fatalf("unconfirmed batch = %d %s", unconf.Code, unconf.Body.String()) + } +} + +func TestHandlerLivenessReadinessAreIndependent(t *testing.T) { + now := time.Date(2026, 9, 12, 10, 0, 0, 0, time.UTC) + healthy := governance.ServiceHealth{Key: "postgres", Name: "PostgreSQL", Liveness: "up", Readiness: "ready", Configured: true, Applicable: true} + degraded := governance.ServiceHealth{Key: "worker", Name: "解析Worker", Liveness: "down", Readiness: "blocked", Reason: "队列无新鲜心跳", Configured: true, Applicable: true} + unconfigured := governance.ServiceHealth{Key: "mineru", Name: "MinerU", Liveness: "unknown", Readiness: "not_configured", Configured: false, Applicable: true} + notDeployed := governance.ServiceHealth{Key: "redis", Name: "Redis / MQ", Liveness: "unknown", Readiness: "not_applicable", Configured: false, Applicable: false} + + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, Monitor: &fakeMonitor{services: []governance.ServiceHealth{healthy, unconfigured, notDeployed}}}) + if err != nil { + t.Fatal(err) + } + live := httptest.NewRecorder() + handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/livez", nil)) + if live.Code != http.StatusOK || !strings.Contains(live.Body.String(), "live") { + t.Fatalf("livez = %d %s", live.Code, live.Body.String()) + } + ready := httptest.NewRecorder() + handler.ServeHTTP(ready, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if ready.Code != http.StatusOK || !strings.Contains(ready.Body.String(), `"not_configured":["mineru"]`) { + t.Fatalf("readyz = %d %s", ready.Code, ready.Body.String()) + } + if strings.Contains(ready.Body.String(), "redis") { + t.Fatal("not-applicable service must not appear in readiness") + } + // A worker outage degrades the matrix but keeps the API itself in rotation + // (admin UI must stay reachable to fix it) and never touches liveness. + handler, err = NewHandler(Config{Store: store, Now: func() time.Time { return now }, Monitor: &fakeMonitor{services: []governance.ServiceHealth{healthy, degraded}}}) + if err != nil { + t.Fatal(err) + } + live = httptest.NewRecorder() + handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/livez", nil)) + if live.Code != http.StatusOK { + t.Fatalf("livez during dependency outage = %d", live.Code) + } + degradedRec := httptest.NewRecorder() + handler.ServeHTTP(degradedRec, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if degradedRec.Code != http.StatusOK || !strings.Contains(degradedRec.Body.String(), `"degraded":["worker"]`) { + t.Fatalf("readyz worker outage = %d %s", degradedRec.Code, degradedRec.Body.String()) + } + if strings.Contains(degradedRec.Body.String(), `"blocked":["worker"]`) { + t.Fatal("worker outage must not block API readiness") + } + // PostgreSQL failure blocks readiness (hard dependency). + pgDown := governance.ServiceHealth{Key: "postgres", Name: "PostgreSQL", Liveness: "down", Readiness: "blocked", Reason: "网络不可达", Configured: true, Applicable: true} + handler, err = NewHandler(Config{Store: store, Now: func() time.Time { return now }, Monitor: &fakeMonitor{services: []governance.ServiceHealth{pgDown}}}) + if err != nil { + t.Fatal(err) + } + blocked := httptest.NewRecorder() + handler.ServeHTTP(blocked, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if blocked.Code != http.StatusServiceUnavailable || !strings.Contains(blocked.Body.String(), `"blocked":["postgres"]`) { + t.Fatalf("readyz postgres outage = %d %s", blocked.Code, blocked.Body.String()) + } + // weknora gating is opt-in: configured-but-blocked weknora only blocks when + // RequireWeKnoraReady is set. + wkDown := governance.ServiceHealth{Key: "weknora", Name: "WeKnora API", Liveness: "up", Readiness: "blocked", Reason: "服务可达但目标知识库无读取权限", Configured: true, Applicable: true} + handler, err = NewHandler(Config{Store: store, Now: func() time.Time { return now }, RequireWeKnoraReady: true, Monitor: &fakeMonitor{services: []governance.ServiceHealth{healthy, wkDown}}}) + if err != nil { + t.Fatal(err) + } + blocked = httptest.NewRecorder() + handler.ServeHTTP(blocked, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if blocked.Code != http.StatusServiceUnavailable || !strings.Contains(blocked.Body.String(), `"blocked":["weknora"]`) { + t.Fatalf("readyz gated weknora = %d %s", blocked.Code, blocked.Body.String()) + } +} + +func TestHandlerServesExplicitlyLabelledMonitorSimulation(t *testing.T) { + now := time.Date(2026, 9, 8, 12, 0, 0, 0, time.UTC) + monitor := &fakeMonitor{ + overview: governance.MonitorOverview{Mode: "simulation", Simulated: true, ObservedAt: now, Host: governance.MonitorHost{CPUPercent: 42, Simulated: true}, Services: []governance.MonitorService{{Key: "mineru", Name: "MinerU", State: "healthy", Detail: "模拟", ObservedAt: now, Simulated: true}}, KnowledgeBases: []governance.KnowledgeBaseProgress{{ID: "kb-sim", Name: "公司级技术支持知识库", Imported: 5750, Completed: 4530}}}, + files: governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{{Name: "[模拟] 文件.pdf", Status: "failed"}}, Total: 1, Limit: 20, SampleOnly: true}, + } + handler, err := NewHandler(Config{Store: &fakeJobStore{}, Monitor: monitor}) + if err != nil { + t.Fatal(err) + } + overview := httptest.NewRecorder() + handler.ServeHTTP(overview, httptest.NewRequest(http.MethodGet, "/api/v1/monitor/overview", nil)) + if overview.Code != http.StatusOK || !strings.Contains(overview.Body.String(), `"mode":"simulation"`) || !strings.Contains(overview.Body.String(), `"simulated":true`) { + t.Fatalf("overview=%d body=%s", overview.Code, overview.Body.String()) + } + metricsReq := httptest.NewRecorder() + monitor.metrics = governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease", ActiveWorkers: 2, + Queue: governance.QueueGauges{Pending: 3, Running: 2, Failed: 1}} + handler.ServeHTTP(metricsReq, httptest.NewRequest(http.MethodGet, "/api/v1/monitor/metrics", nil)) + if metricsReq.Code != http.StatusOK || !strings.Contains(metricsReq.Body.String(), `"worker_source":"pg_lease"`) || !strings.Contains(metricsReq.Body.String(), `"success_rate":null`) { + t.Fatalf("metrics=%d body=%s", metricsReq.Code, metricsReq.Body.String()) + } + metricsDown := httptest.NewRecorder() + monitor.metricsErr = errors.New("db down") + handler.ServeHTTP(metricsDown, httptest.NewRequest(http.MethodGet, "/api/v1/monitor/metrics", nil)) + if metricsDown.Code != http.StatusServiceUnavailable { + t.Fatalf("metrics down status = %d", metricsDown.Code) + } + monitor.metricsErr = nil + files := httptest.NewRecorder() + handler.ServeHTTP(files, httptest.NewRequest(http.MethodGet, "/api/v1/monitor/knowledge-bases/kb-sim/files?status=failed&limit=20&offset=0", nil)) + if files.Code != http.StatusOK || monitor.knowledgeBaseID != "kb-sim" || monitor.status != "failed" || !strings.Contains(files.Body.String(), `"sample_only":true`) { + t.Fatalf("files=%d monitor=%#v body=%s", files.Code, monitor, files.Body.String()) + } + for _, forbidden := range []string{"api_key", "base_url", "authorization", "secret"} { + if strings.Contains(strings.ToLower(overview.Body.String()+files.Body.String()), forbidden) { + t.Fatalf("monitor response leaked %s", forbidden) + } + } +} + +func TestHandlerListsSourceConnectionsSafeProjection(t *testing.T) { + store := &fakeJobStore{connectionResult: governance.SourceConnectionPage{ + Items: []governance.SourceConnectionView{{ + ID: "conn-safe", SourceType: governance.SourceTypeAList, Name: "AList只读源", + Enabled: true, ScheduleConfigured: true, HealthStatus: governance.SourceHealthHealthy, ItemCount: 42, + }}, + Summary: governance.SourceConnectionSummary{Total: 1, Enabled: 1, Healthy: 1}, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/source-connections", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", response.Code, response.Body.String()) + } + for _, expected := range []string{`"id":"conn-safe"`, `"source_type":"alist"`, `"item_count":42`, `"total":1`} { + if !bytes.Contains(response.Body.Bytes(), []byte(expected)) { + t.Fatalf("body missing %s: %s", expected, response.Body.String()) + } + } + for _, forbidden := range []string{"base_url", "secret_ref", "https://", "api_key"} { + if bytes.Contains(bytes.ToLower(response.Body.Bytes()), []byte(forbidden)) { + t.Fatalf("body leaked %q: %s", forbidden, response.Body.String()) + } + } +} + +func TestHandlerTriggersPPSourceScanWithSafeAcceptedResponse(t *testing.T) { + now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC) + store := &fakeJobStore{triggerScanResult: governance.SourceScan{ + ID: "scan-safe", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, + State: governance.SourceScanPending, CreatedAt: now, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-connections/conn-pp/scan", nil) + request.Header.Set("Idempotency-Key", "op-scan-1") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusAccepted { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if store.triggerScanConnectionID != "conn-pp" { + t.Fatalf("connection ID = %q", store.triggerScanConnectionID) + } + for _, expected := range []string{`"id":"scan-safe"`, `"source_connection_id":"conn-pp"`, `"state":"pending"`} { + if !bytes.Contains(response.Body.Bytes(), []byte(expected)) { + t.Errorf("response missing %s: %s", expected, response.Body.String()) + } + } + for _, forbidden := range []string{"snapshot_sha256", "stable_external_id", "base_url", "secret_ref"} { + if bytes.Contains(response.Body.Bytes(), []byte(forbidden)) { + t.Errorf("response leaked %q: %s", forbidden, response.Body.String()) + } + } +} + +func TestHandlerListsSourceScanHistoryWithSafeProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 30, 0, 0, time.UTC) + store := &fakeJobStore{scanPageResult: governance.SourceScanPage{ + Items: []governance.SourceScanSummaryView{{ + ID: "scan-safe", State: governance.SourceScanCompleted, ObservedItems: 613, + UnchangedItems: 559, OverLimitItems: 54, CreatedAt: now, CompletedAt: &now, + }}, Total: 5, Limit: 2, Offset: 1, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/source-connections/conn-pp/scans?limit=2&offset=1", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + if store.scanPageConnectionID != "conn-pp" || store.scanPageLimit != 2 || store.scanPageOffset != 1 { + t.Fatalf("list args=%q/%d/%d", store.scanPageConnectionID, store.scanPageLimit, store.scanPageOffset) + } + for _, expected := range []string{`"id":"scan-safe"`, `"observed_items":613`, `"over_limit_items":54`, `"total":5`, `"limit":2`, `"offset":1`} { + if !bytes.Contains(response.Body.Bytes(), []byte(expected)) { + t.Errorf("body missing %s: %s", expected, response.Body.String()) + } + } + assertNoSourceScanSecrets(t, response.Body.Bytes()) +} + +func TestHandlerGetsSourceScanDetailWithSafeChanges(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 40, 0, 0, time.UTC) + itemID := "si-safe" + size := int64(42) + store := &fakeJobStore{scanDetailResult: governance.SourceScanDetailView{ + SourceScanSummaryView: governance.SourceScanSummaryView{ID: "scan-safe", State: governance.SourceScanCompleted, ObservedItems: 1, UpdatedItems: 1, PlannedUploads: 1, CreatedAt: now, CompletedAt: &now}, + Changes: []governance.SourceScanChangeView{{ID: "chg-safe", SourceItemID: &itemID, ChangeType: governance.SourceScanUpdated, ObservedSizeBytes: &size, CreatedAt: now}}, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/source-connections/conn-pp/scans/scan-safe", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.scanDetailConnectionID != "conn-pp" || store.scanDetailID != "scan-safe" { + t.Fatalf("status=%d args=%q/%q body=%s", response.Code, store.scanDetailConnectionID, store.scanDetailID, response.Body.String()) + } + for _, expected := range []string{`"id":"chg-safe"`, `"source_item_id":"si-safe"`, `"change_type":"updated"`, `"observed_size_bytes":42`} { + if !bytes.Contains(response.Body.Bytes(), []byte(expected)) { + t.Errorf("body missing %s: %s", expected, response.Body.String()) + } + } + assertNoSourceScanSecrets(t, response.Body.Bytes()) +} + +func TestHandlerReturnsNotFoundForUnknownSourceScanDetail(t *testing.T) { + store := &fakeJobStore{scanDetailErr: postgresstore.ErrSourceScanNotFound} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/source-connections/conn-pp/scans/missing", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNotFound || response.Body.String() != "{\"error\":\"source scan not found\"}\n" { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestHandlerListsSafeReviewCandidates(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &fakeJobStore{reviewPageResult: governance.ReviewCandidatePage{ + Items: []governance.ReviewCandidate{{PublicationID: "tp-safe", Title: "Synthetic", SourceType: governance.SourceTypeManual, CurrentVisibility: governance.VisibilityInternal, AcceptanceRunID: "hidden-run", KnowledgeExists: governance.AcceptancePassed, ChunksPresent: governance.AcceptancePassed, RetrievalHit: governance.AcceptanceManualRequired, GateStatus: governance.ReviewGateReady, GateCode: "ready", ReviewState: governance.ReviewPending, PublicationUpdated: now}}, + Total: 1, Limit: 20, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/review-candidates?review_state=pending&limit=20&offset=0", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.reviewFilter.ReviewState != "pending" { + t.Fatalf("status=%d filter=%#v body=%s", response.Code, store.reviewFilter, response.Body.String()) + } + for _, expected := range []string{`"publication_id":"tp-safe"`, `"gate_status":"ready"`, `"retrieval_hit":"manual_required"`, `"total":1`} { + if !bytes.Contains(response.Body.Bytes(), []byte(expected)) { + t.Errorf("missing %s", expected) + } + } + for _, forbidden := range []string{"hidden-run", "acceptance_run_id", "publication_updated_at", "knowledge_id", "test_kb_id"} { + if bytes.Contains(response.Body.Bytes(), []byte(forbidden)) { + t.Errorf("response leaked %q", forbidden) + } + } +} + +func TestHandlerRecordsPublicationReview(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 10, 0, 0, time.UTC) + store := &fakeJobStore{reviewResult: governance.PublicationReview{ID: "rv-safe", PublicationID: "tp-safe", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", AcceptanceRunID: "hidden-run", CreatedAt: now}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/review-candidates/tp-safe/decision", bytes.NewBufferString(`{"decision":"approved","decided_visibility":"internal","reviewer_ref":"engineer-1","decision_code":"approved_for_release"}`)) + request.Header.Set("Idempotency-Key", "op-review-1") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusCreated || store.reviewRequest.PublicationID != "tp-safe" || store.reviewRequest.Now != now { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.reviewRequest, response.Body.String()) + } + if bytes.Contains(response.Body.Bytes(), []byte("hidden-run")) || bytes.Contains(response.Body.Bytes(), []byte("acceptance_run_id")) { + t.Fatalf("review response leaked internal gate identity: %s", response.Body.String()) + } +} + +func TestHandlerVersionsAndActivatesUISettings(t *testing.T) { + now := time.Date(2026, 9, 10, 8, 0, 0, 0, time.UTC) + activeAt := now.Add(time.Second) + store := &fakeJobStore{ + activeUISettingsResult: governance.UISettings{ID: "settings-default", State: "active", DocumentRefreshSeconds: 15, ResourceRefreshSeconds: 30, StalledAfterSeconds: 1800, FinalizingAfterSeconds: 7200, UpdatedBy: "system:migration", CreatedAt: now, ActivatedAt: &now}, + candidateUISettingsResult: governance.UISettings{ID: "uis-candidate", State: "validated", DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", CreatedAt: now}, + activatedUISettingsResult: governance.UISettings{ID: "uis-candidate", State: "active", DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", CreatedAt: now, ActivatedAt: &activeAt}, + } + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + get := httptest.NewRecorder() + handler.ServeHTTP(get, httptest.NewRequest(http.MethodGet, "/api/v1/ui-settings", nil)) + if get.Code != http.StatusOK || !strings.Contains(get.Body.String(), `"state":"active"`) { + t.Fatalf("get status=%d body=%s", get.Code, get.Body.String()) + } + noIntent := httptest.NewRecorder() + handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"updated_by":"engineer-1"}`))) + if noIntent.Code != http.StatusForbidden { + t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String()) + } + create := httptest.NewRecorder() + createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"updated_by":"engineer-1"}`)) + createRequest.Header.Set("X-Governance-Action", "ui-settings") + createRequest.Header.Set("Idempotency-Key", "op-uis-create") + handler.ServeHTTP(create, createRequest) + if create.Code != http.StatusCreated || store.createUISettingsRequest.Now != now || store.createUISettingsRequest.UpdatedBy != "engineer-1" || !strings.Contains(create.Body.String(), `"operation":"created"`) { + t.Fatalf("create status=%d request=%#v body=%s", create.Code, store.createUISettingsRequest, create.Body.String()) + } + unknown := httptest.NewRecorder() + unknownRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"updated_by":"engineer-1","secret":"must-reject"}`)) + unknownRequest.Header.Set("X-Governance-Action", "ui-settings") + handler.ServeHTTP(unknown, unknownRequest) + if unknown.Code != http.StatusBadRequest { + t.Fatalf("unknown field status=%d body=%s", unknown.Code, unknown.Body.String()) + } + activate := httptest.NewRecorder() + activateRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/uis-candidate/activate", bytes.NewBufferString(`{}`)) + activateRequest.Header.Set("X-Governance-Action", "ui-settings") + activateRequest.Header.Set("Idempotency-Key", "op-uis-activate") + handler.ServeHTTP(activate, activateRequest) + if activate.Code != http.StatusOK || store.activateUISettingsID != "uis-candidate" || !strings.Contains(activate.Body.String(), `"state":"active"`) { + t.Fatalf("activate status=%d id=%q body=%s", activate.Code, store.activateUISettingsID, activate.Body.String()) + } +} + +func TestHandlerCreatesAndSealsReleaseBatchWithoutReleaseJob(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 20, 0, 0, time.UTC) + store := &fakeJobStore{ + createBatchResult: governance.ReleaseBatch{ID: "rb-safe", Name: "Batch", State: governance.ReleaseBatchDraft, PolicyVersion: "release-gate-v1", CreatedBy: "engineer-1", ItemCount: 1, CreatedAt: now}, + sealBatchResult: governance.ReleaseBatch{ID: "rb-safe", Name: "Batch", State: governance.ReleaseBatchSealed, PolicyVersion: "release-gate-v1", CreatedBy: "engineer-1", SealedBy: "engineer-2", ItemCount: 1, CreatedAt: now, SealedAt: &now}, + } + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches", bytes.NewBufferString(`{"name":"Batch","created_by":"engineer-1","publication_ids":["tp-safe"],"delete_publication_ids":["tp-delete"]}`)) + createRequest.Header.Set("Idempotency-Key", "op-batch-create") + createResponse := httptest.NewRecorder() + handler.ServeHTTP(createResponse, createRequest) + if createResponse.Code != http.StatusCreated || len(store.createBatchRequest.PublicationIDs) != 1 || len(store.createBatchRequest.DeletePublicationIDs) != 1 || store.createBatchRequest.DeletePublicationIDs[0] != "tp-delete" { + t.Fatalf("create status=%d request=%#v body=%s", createResponse.Code, store.createBatchRequest, createResponse.Body.String()) + } + sealRequest := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches/rb-safe/seal", bytes.NewBufferString(`{"sealed_by":"engineer-2"}`)) + sealRequest.Header.Set("Idempotency-Key", "op-batch-seal") + sealResponse := httptest.NewRecorder() + handler.ServeHTTP(sealResponse, sealRequest) + if sealResponse.Code != http.StatusOK || store.sealBatchRequest.BatchID != "rb-safe" || store.sealBatchRequest.SealedBy != "engineer-2" { + t.Fatalf("seal status=%d request=%#v body=%s", sealResponse.Code, store.sealBatchRequest, sealResponse.Body.String()) + } + if store.enqueued.Type != "" { + t.Fatalf("P5 API enqueued unexpected job: %#v", store.enqueued) + } +} + +func TestHandlerListsSafeSourceTruthReleasePlan(t *testing.T) { + store := &fakeJobStore{releasePlanResult: governance.ReleasePlan{ + Items: []governance.ReleasePlanItem{{PublicationID: "tp-safe", Title: "B.pdf", TargetScope: "external", Action: "add", TestState: "ready", ProductionState: "absent", CanRelease: true}, {Title: "未就绪.pdf", TargetScope: "internal", Action: "blocked", TestState: "not_ready", ProductionState: "absent", Blocker: "test_not_ready"}}, + Summary: governance.ReleasePlanSummary{Add: 1}, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/release-plan", nil)) + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"action":"add"`) || !strings.Contains(response.Body.String(), `"add":1`) || !strings.Contains(response.Body.String(), `"publication_id":"","title":"未就绪.pdf"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + for _, forbidden := range []string{"production_revision", "test_kb_id", "knowledge_id", "bundle_sha256"} { + if strings.Contains(response.Body.String(), forbidden) { + t.Fatalf("release plan leaked %q: %s", forbidden, response.Body.String()) + } + } +} + +func TestHandlerListsAndGetsReleaseBatches(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 30, 0, 0, time.UTC) + batch := governance.ReleaseBatch{ID: "rb-safe", Name: "Batch", State: governance.ReleaseBatchSealed, PolicyVersion: "release-gate-v1", CreatedBy: "engineer-1", SealedBy: "engineer-2", ItemCount: 1, CreatedAt: now, SealedAt: &now} + store := &fakeJobStore{batchPageResult: governance.ReleaseBatchPage{Items: []governance.ReleaseBatch{batch}, Total: 1, Limit: 20}, batchDetailResult: governance.ReleaseBatchDetail{ReleaseBatch: batch, Items: []governance.ReleaseBatchItemView{{PublicationID: "tp-safe", Title: "Synthetic", DecidedVisibility: governance.VisibilityInternal}}}} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + listResponse := httptest.NewRecorder() + handler.ServeHTTP(listResponse, httptest.NewRequest(http.MethodGet, "/api/v1/release-batches?state=sealed&limit=20", nil)) + detailResponse := httptest.NewRecorder() + handler.ServeHTTP(detailResponse, httptest.NewRequest(http.MethodGet, "/api/v1/release-batches/rb-safe", nil)) + if listResponse.Code != http.StatusOK || detailResponse.Code != http.StatusOK || store.batchID != "rb-safe" { + t.Fatalf("list=%d detail=%d batchID=%q", listResponse.Code, detailResponse.Code, store.batchID) + } + for _, body := range [][]byte{listResponse.Body.Bytes(), detailResponse.Body.Bytes()} { + for _, forbidden := range []string{"knowledge_id", "test_kb_id", "acceptance_run_id", "publication_updated_at"} { + if bytes.Contains(body, []byte(forbidden)) { + t.Errorf("batch response leaked %q", forbidden) + } + } + } +} + +func TestHandlerBlocksProductionReleaseAuthorizationByDefault(t *testing.T) { + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches/rb-safe/authorize-production", bytes.NewBufferString(`{"authorized_by":"release-owner","confirmation":"publish-to-production"}`)) + request.Header.Set("Content-Type", "application/json") + handler.ServeHTTP(response, request) + if response.Code != http.StatusForbidden || store.authorizeReleaseCalled { + t.Fatalf("status=%d called=%v body=%s", response.Code, store.authorizeReleaseCalled, response.Body.String()) + } +} + +func TestHandlerBlocksProductionAuthorizationUntilBackupRestoreIsVerified(t *testing.T) { + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionReleaseAuthorizerRef: "release-owner", ProductionReleaseAuthorizationToken: "safe-token"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches/rb-safe/authorize-production", bytes.NewBufferString(`{"confirmation":"publish-to-production"}`)) + request.Header.Set("Authorization", "Bearer safe-token") + handler.ServeHTTP(response, request) + if response.Code != http.StatusPreconditionFailed || store.authorizeReleaseCalled || !strings.Contains(response.Body.String(), "production backup restore verification is required") { + t.Fatalf("status=%d called=%v body=%s", response.Code, store.authorizeReleaseCalled, response.Body.String()) + } + capabilities := httptest.NewRecorder() + handler.ServeHTTP(capabilities, httptest.NewRequest(http.MethodGet, "/api/v1/release-capabilities", nil)) + if !strings.Contains(capabilities.Body.String(), `"production_backup_restore_verified":false`) { + t.Fatalf("capabilities=%s", capabilities.Body.String()) + } +} + +func TestHandlerAuthorizesProductionReleaseOnlyBehindExplicitGate(t *testing.T) { + now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC) + store := &fakeJobStore{authorizeReleaseResult: governance.ReleaseExecution{ + ID: "rx-safe", ReleaseBatchID: "rb-safe", Operation: governance.ReleaseOperationPublish, + State: governance.ReleaseExecutionPending, TargetEnvironment: "production", AuthorizedBy: "release-owner", + AuthorizationCode: "production_release_authorized", JobID: "job-safe", ItemCount: 1, CreatedAt: now, + }} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true, ProductionReleaseAuthorizerRef: "release-owner", ProductionReleaseAuthorizationToken: "safe-token"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches/rb-safe/authorize-production", bytes.NewBufferString(`{"confirmation":"publish-to-production"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", "Bearer safe-token") + request.Header.Set("Idempotency-Key", "op-authz-publish") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + if store.authorizeReleaseRequest.BatchID != "rb-safe" || store.authorizeReleaseRequest.AuthorizedBy != "release-owner" || store.authorizeReleaseRequest.AuthorizationCode != "production_release_authorized" { + t.Fatalf("request=%#v", store.authorizeReleaseRequest) + } + body := response.Body.String() + if strings.Contains(body, "manifest_sha256") || strings.Contains(body, "production_database") { + t.Fatalf("unsafe authorization response: %s", body) + } +} + +func TestHandlerAuthorizesRollbackOnlyBehindExplicitGate(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 0, 0, 0, time.UTC) + store := &fakeJobStore{authorizeRollbackResult: governance.ReleaseExecution{ + ID: "rx-rollback", ReleaseBatchID: "rb-safe", Operation: governance.ReleaseOperationRollback, + State: governance.ReleaseExecutionPending, TargetEnvironment: "production", AuthorizedBy: "release-owner", + AuthorizationCode: "production_rollback_authorized", JobID: "job-rollback", ItemCount: 1, CreatedAt: now, + }} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, ProductionReleaseRequestsEnabled: true, ProductionReleaseAuthorizerRef: "release-owner", ProductionReleaseAuthorizationToken: "safe-token"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-executions/rx-publish/authorize-rollback", bytes.NewBufferString(`{"confirmation":"rollback-production-release"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", "Bearer safe-token") + request.Header.Set("Idempotency-Key", "op-authz-rollback") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.authorizeRollbackRequest.PriorExecutionID != "rx-publish" || store.authorizeRollbackRequest.AuthorizationCode != "production_rollback_authorized" { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.authorizeRollbackRequest, response.Body.String()) + } +} + +func TestHandlerRetriesFailedReleaseOnlyBehindExplicitGate(t *testing.T) { + now := time.Date(2026, 9, 6, 17, 0, 0, 0, time.UTC) + store := &fakeJobStore{retryReleaseResult: governance.ReleaseExecution{ID: "rx-failed", ReleaseBatchID: "rb-safe", Operation: governance.ReleaseOperationPublish, State: governance.ReleaseExecutionPending, TargetEnvironment: "production", AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", JobID: "job-safe", ItemCount: 1, CreatedAt: now}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, ProductionReleaseRequestsEnabled: true, ProductionReleaseAuthorizerRef: "release-owner", ProductionReleaseAuthorizationToken: "safe-token"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-executions/rx-failed/retry", bytes.NewBufferString(`{"confirmation":"retry-production-release"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", "Bearer safe-token") + request.Header.Set("Idempotency-Key", "op-retry-release") + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.retryReleaseRequest.ExecutionID != "rx-failed" { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.retryReleaseRequest, response.Body.String()) + } + if body := response.Body.String(); strings.Contains(body, "authorization_code") || strings.Contains(body, "manifest_sha256") { + t.Fatalf("unsafe retry response: %s", body) + } +} + +func TestHandlerRejectsProductionAuthorizationWithoutServerIdentityAndToken(t *testing.T) { + for _, config := range []Config{ + {Store: &fakeJobStore{}, ProductionReleaseRequestsEnabled: true}, + {Store: &fakeJobStore{}, ProductionReleaseRequestsEnabled: true, ProductionReleaseAuthorizerRef: "release-owner"}, + } { + if _, err := NewHandler(config); err == nil { + t.Fatal("NewHandler() error=nil") + } + } +} + +func TestHandlerRejectsMissingProductionAuthorizationBearer(t *testing.T) { + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true, ProductionReleaseAuthorizerRef: "release-owner", ProductionReleaseAuthorizationToken: "safe-token"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/release-batches/rb-safe/authorize-production", bytes.NewBufferString(`{"confirmation":"publish-to-production"}`)) + request.Header.Set("Content-Type", "application/json") + handler.ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized || store.authorizeReleaseCalled { + t.Fatalf("status=%d called=%v", response.Code, store.authorizeReleaseCalled) + } +} + +func TestHandlerListsAndGetsSafeReleaseExecutions(t *testing.T) { + now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC) + execution := governance.ReleaseExecution{ + ID: "rx-safe", ReleaseBatchID: "rb-safe", Operation: governance.ReleaseOperationPublish, + State: governance.ReleaseExecutionCompleted, TargetEnvironment: "production", AuthorizedBy: "owner", + AuthorizationCode: "hidden-auth-code", JobID: "job-safe", ManifestSHA256: strings.Repeat("a", 64), + ItemCount: 1, CompletedItemCount: 1, CreatedAt: now, CompletedAt: &now, + } + store := &fakeJobStore{ + releaseExecutionPageResult: governance.ReleaseExecutionPage{Items: []governance.ReleaseExecution{execution}, Total: 1, Limit: 20}, + releaseExecutionDetailResult: governance.ReleaseExecutionDetail{ReleaseExecution: execution, Items: []governance.ReleaseItemExecutionView{{PublicationID: "tp-safe", Title: "Synthetic", State: "verified", ProductionRevision: "prod-v1", CreatedAt: now, CompletedAt: &now}}}, + } + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/api/v1/release-executions?operation=publish&state=completed&limit=20", nil)) + detail := httptest.NewRecorder() + handler.ServeHTTP(detail, httptest.NewRequest(http.MethodGet, "/api/v1/release-executions/rx-safe", nil)) + if list.Code != http.StatusOK || detail.Code != http.StatusOK || store.releaseExecutionID != "rx-safe" { + t.Fatalf("list=%d detail=%d id=%q", list.Code, detail.Code, store.releaseExecutionID) + } + serialized := strings.ToLower(list.Body.String() + detail.Body.String()) + for _, forbidden := range []string{"authorization_code", "manifest_sha256", "knowledge_id", "test_kb_id", "database_url", "secret"} { + if strings.Contains(serialized, forbidden) { + t.Fatalf("release execution response leaked %q", forbidden) + } + } +} + +func TestHandlerReportsProductionReleaseCapabilityDisabledByDefault(t *testing.T) { + handler, err := NewHandler(Config{Store: &fakeJobStore{}}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/release-capabilities", nil)) + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"production_release_requests_enabled":false`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func assertNoSourceScanSecrets(t *testing.T, body []byte) { + t.Helper() + for _, forbidden := range []string{"snapshot_sha256", "stable_external_id", "previous_revision", "observed_revision", "base_url", "secret_ref", "local_path"} { + if bytes.Contains(bytes.ToLower(body), []byte(forbidden)) { + t.Errorf("response leaked %q: %s", forbidden, body) + } + } +} + +func TestHandlerListsTestPublicationsSafeProjection(t *testing.T) { + store := &fakeJobStore{publicationResult: governance.TestPublicationPage{ + Items: []governance.TestPublication{{ID: "pub-safe", Title: "Safe title", ProcessingState: "failed", LastError: "remote_parse_failed", CanRetry: true}}, + Total: 1, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/test-publications?processing_state=failed&limit=25", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", response.Code, response.Body.String()) + } + if !bytes.Contains(response.Body.Bytes(), []byte(`"id":"pub-safe"`)) || !bytes.Contains(response.Body.Bytes(), []byte(`"last_error":"remote_parse_failed"`)) { + t.Fatalf("body = %s", response.Body.String()) + } + if store.publicationFilter != (governance.TestPublicationFilter{ProcessingState: "failed", Limit: 25, Offset: 0}) { + t.Fatalf("filter = %#v", store.publicationFilter) + } +} + +func TestHandlerConfirmsChangeSetItems(t *testing.T) { + now := time.Date(2026, 9, 12, 5, 0, 0, 0, time.UTC) + store := &fakeJobStore{confirmChangeSetResult: governance.ConfirmCatalogChangeSetResult{ChangeSetID: "cs-1", ConfirmedItems: 2, UploadsCreated: 2}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/change-sets/cs-1/confirm", bytes.NewBufferString(`{"item_seqs":[4,9],"operator_ref":"engineer-1","reason":"确认新增与更新"}`)) + request.Header.Set("Idempotency-Key", "op-confirm-1") + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.confirmChangeSetOperationID != "op-confirm-1" || len(store.confirmChangeSetRequest.ItemSeqs) != 2 || store.confirmChangeSetRequest.InternalTargetKBID != "kb-internal" || store.confirmChangeSetRequest.Now != now { + t.Fatalf("status=%d id=%q request=%#v body=%s", response.Code, store.confirmChangeSetOperationID, store.confirmChangeSetRequest, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"operation":"created"`) || strings.Contains(response.Body.String(), "source_locator_hash") { + t.Fatalf("unsafe confirm response=%s", response.Body.String()) + } + missing := httptest.NewRecorder() + store.confirmChangeSetErr = postgresstore.ErrCatalogChangeSetNotFound + missingRequest := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/change-sets/cs-nope/confirm", bytes.NewBufferString(`{"item_seqs":[1],"operator_ref":"engineer-1","reason":"确认"}`)) + missingRequest.Header.Set("Idempotency-Key", "op-confirm-missing") + handler.ServeHTTP(missing, missingRequest) + if missing.Code != http.StatusNotFound { + t.Fatalf("missing change set status=%d", missing.Code) + } + bad := httptest.NewRecorder() + handler.ServeHTTP(bad, httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/change-sets/cs-1/confirm", bytes.NewBufferString(`{"item_seqs":[],"operator_ref":"engineer-1","reason":"确认"}`))) + if bad.Code != http.StatusBadRequest { + t.Fatalf("empty selection status=%d", bad.Code) + } + noKey := httptest.NewRecorder() + handler.ServeHTTP(noKey, httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/change-sets/cs-1/confirm", bytes.NewBufferString(`{"item_seqs":[1],"operator_ref":"engineer-1","reason":"确认"}`))) + if noKey.Code != http.StatusBadRequest { + t.Fatalf("missing idempotency key status=%d", noKey.Code) + } +} + +func TestHandlerStartsConfirmedTestPublications(t *testing.T) { + now := time.Date(2026, 9, 9, 3, 0, 0, 0, time.UTC) + store := &fakeJobStore{startPublicationsResult: governance.StartTestPublicationsResult{Selected: 2, Started: 1, AlreadyStarted: 1}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/start", bytes.NewBufferString(`{"publication_ids":["pub-a","pub-b"],"operator_ref":"engineer-1","reason":"确认差异后开始处理"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "op-processing-start") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted || store.startPublicationsRequest.Now != now || len(store.startPublicationsRequest.PublicationIDs) != 2 || store.startPublicationsRequest.OperatorRef != "engineer-1" || !strings.Contains(response.Body.String(), `"operation":"created"`) { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.startPublicationsRequest, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"started":1`) || strings.Contains(response.Body.String(), "test_kb_id") { + t.Fatalf("unsafe response=%s", response.Body.String()) + } +} + +func TestHandlerListsAcceptanceResults(t *testing.T) { + store := &fakeJobStore{acceptanceResult: []governance.AcceptanceRecord{ + {CheckType: "chunks_present", State: "passed", EvidenceSummary: json.RawMessage(`{"chunk_total":2}`)}, + }} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/api/v1/test-publications/pub-1/acceptance", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.acceptanceID != "pub-1" { + t.Fatalf("status=%d id=%q", response.Code, store.acceptanceID) + } + if !bytes.Contains(response.Body.Bytes(), []byte(`"chunks_present"`)) || !bytes.Contains(response.Body.Bytes(), []byte(`{"chunk_total":2}`)) { + t.Fatalf("body = %s", response.Body.String()) + } +} + +func TestHandlerRetriesPublication(t *testing.T) { + store := &fakeJobStore{} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/pub-1/retry", nil) + request.Header.Set("Idempotency-Key", "op-retry-1") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.retryID != "pub-1" { + t.Fatalf("status=%d id=%q body=%s", response.Code, store.retryID, response.Body.String()) + } +} + +func TestHandlerRetryReturnsConflictWhenNotRetryable(t *testing.T) { + store := &fakeJobStore{retryErr: postgresstore.ErrPublicationStateConflict} + handler, err := NewHandler(Config{Store: store}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/test-publications/pub-1/retry", nil) + request.Header.Set("Idempotency-Key", "op-retry-1") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409; body=%s", response.Code, response.Body.String()) + } +} + +func TestHandlerRejectsCatalogMutationsWithoutOperationID(t *testing.T) { + now := time.Date(2026, 9, 11, 5, 0, 0, 0, time.UTC) + store := &fakeJobStore{catalogSources: []governance.CatalogSource{{ConnectionID: "conn-cat", SourceType: governance.SourceTypePP, Name: "PP资料下载中心", RootNodeID: "root-cat"}}} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal"}) + if err != nil { + t.Fatal(err) + } + + for _, request := range []*http.Request{ + httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/route", bytes.NewBufferString(`{"connection_id":"conn-cat","node_ids":["file-a"],"policy":"internal","operator_ref":"engineer-1","reason":"人工标注"}`)), + httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/sources/conn-cat/refresh", nil), + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "Idempotency-Key") { + t.Fatalf("%s %s status=%d body=%s", request.Method, request.URL.Path, response.Code, response.Body.String()) + } + } +} + +type fakeJobStore struct { + createSettingsConnectionID string + createSettingsDraft postgresstore.SourceSettingsDraft + createSettingsOperationID string + createSettingsErr error + createSettingsReplayed bool + activateSettingsConnectionID string + activateSettingsRevisionID string + activateSettingsOperationID string + activateSettingsReplayed bool + activateSettingsErr error + listSettingsConnectionID string + listSettingsErr error + settingsRevisions []governance.SourceConnectionRevision + jobFailureFilter governance.JobFailureFilter + jobFailurePage governance.JobFailurePage + jobFailureErr error + retryBatchRequest *governance.RetryPublicationsBatchRequest + retryBatchOperation string + retryBatchResult governance.RetryPublicationsBatchResult + retryBatchReplayed bool + retryBatchErr error + enqueued jobs.EnqueueRequest + enqueueResult *jobs.Job + listResult []jobs.Job + listLimit int + assetResult governance.SourceAssetPage + assetFilter governance.SourceAssetFilter + connectionResult governance.SourceConnectionPage + triggerScanResult governance.SourceScan + triggerScanConnectionID string + triggerScanOperationID string + triggerScanErr error + scanPageResult governance.SourceScanPage + scanPageConnectionID string + scanPageLimit int + scanPageOffset int + scanDetailResult governance.SourceScanDetailView + scanDetailErr error + scanDetailConnectionID string + scanDetailID string + reviewPageResult governance.ReviewCandidatePage + reviewFilter governance.ReviewCandidateFilter + reviewResult governance.PublicationReview + reviewRequest governance.ReviewPublicationRequest + createBatchResult governance.ReleaseBatch + createBatchRequest governance.CreateReleaseBatchRequest + auditChainID string + auditChain governance.ReleaseAuditChain + auditChainErr error + generateChangeSetRequest governance.GenerateReleaseChangeSetRequest + generateChangeSetOperationID string + generateChangeSetResult governance.ReleaseChangeSet + generateChangeSetReplayed bool + generateChangeSetErr error + confirmReleaseSetRequest governance.ConfirmReleaseChangeSetRequest + confirmReleaseSetOperationID string + confirmReleaseSetResult governance.ReleaseChangeSet + confirmReleaseSetReplayed bool + confirmReleaseSetErr error + listChangeSetsFilter governance.ReleaseBatchFilter + changeSetPage governance.ReleaseChangeSetPage + getChangeSetID string + changeSetDetail governance.ReleaseChangeSetDetail + changeSetDetailErr error + createBatchOperationID string + createBatchErr error + sealBatchResult governance.ReleaseBatch + sealBatchRequest governance.SealReleaseBatchRequest + sealBatchOperationID string + sealBatchErr error + batchPageResult governance.ReleaseBatchPage + batchFilter governance.ReleaseBatchFilter + batchDetailResult governance.ReleaseBatchDetail + batchID string + authorizeReleaseResult governance.ReleaseExecution + authorizeReleaseRequest governance.AuthorizeReleaseBatchRequest + authorizeReleaseOperationID string + authorizeReleaseErr error + authorizeReleaseCalled bool + authorizeRollbackResult governance.ReleaseExecution + authorizeRollbackRequest governance.AuthorizeReleaseRollbackRequest + authorizeRollbackOperationID string + authorizeRollbackErr error + retryReleaseResult governance.ReleaseExecution + retryReleaseRequest governance.RetryReleaseExecutionRequest + retryReleaseOperationID string + retryReleaseErr error + releaseExecutionPageResult governance.ReleaseExecutionPage + releaseExecutionFilter governance.ReleaseExecutionFilter + releaseExecutionDetailResult governance.ReleaseExecutionDetail + releaseExecutionID string + releasePlanResult governance.ReleasePlan + activeUISettingsResult governance.UISettings + candidateUISettingsResult governance.UISettings + activatedUISettingsResult governance.UISettings + createUISettingsRequest governance.CreateUISettingsCandidateRequest + createUISettingsOperationID string + createUISettingsErr error + activateUISettingsID string + activateUISettingsOperationID string + activateUISettingsErr error + publicationResult governance.TestPublicationPage + publicationFilter governance.TestPublicationFilter + confirmChangeSetResult governance.ConfirmCatalogChangeSetResult + confirmChangeSetRequest governance.ConfirmCatalogChangeSetRequest + confirmChangeSetOperationID string + confirmChangeSetReplayed bool + confirmChangeSetErr error + startPublicationsResult governance.StartTestPublicationsResult + startPublicationsRequest governance.StartTestPublicationsRequest + startPublicationsOperationID string + startPublicationsReplayed bool + acceptanceResult []governance.AcceptanceRecord + acceptanceID string + retryErr error + retryID string + retryOperationID string + retryReplayed bool + scopeRules []governance.ScopeRule + scopeRuleConnectionID string + scopeRuleResult governance.ScopeRule + scopeRuleRequest governance.ScopeRuleMutation + scopeRuleOperationID string + scopeRuleReplayed bool + scopeRuleErr error + catalogSources []governance.CatalogSource + catalogPage governance.CatalogNodePage + catalogFilter governance.CatalogNodeFilter + catalogErr error + catalogRouteRequest governance.CatalogRouteRequest + catalogRouteOperationID string + catalogRouteReplayed bool + catalogRouteResult governance.CatalogRouteResult + catalogRouteErr error + catalogRefreshSource governance.CatalogSource + catalogRefreshOperationID string + catalogRefreshReplayed bool + catalogRefreshResult jobs.Job + catalogRefreshErr error + snapshotPageResult governance.CatalogSnapshotPage + snapshotNodePageResult governance.CatalogNodePage + snapshotConnectionID string + snapshotNodeFilter governance.CatalogSnapshotNodeFilter + snapshotErr error + catalogChangeSetResult governance.CatalogChangeSetDetail + catalogChangeSetFilter governance.CatalogChangeSetFilter +} + +func (s *fakeJobStore) ListSourceAssets(_ context.Context, filter governance.SourceAssetFilter) (governance.SourceAssetPage, error) { + s.assetFilter = filter + return s.assetResult, nil +} + +func (s *fakeJobStore) ListSourceConnections(context.Context) (governance.SourceConnectionPage, error) { + return s.connectionResult, nil +} + +func (s *fakeJobStore) CreateSourceSettingsRevisionWithOperation(_ context.Context, connectionID string, draft postgresstore.SourceSettingsDraft, operationID string, _ time.Time) (string, bool, error) { + s.createSettingsConnectionID = connectionID + s.createSettingsDraft = draft + s.createSettingsOperationID = operationID + if s.createSettingsErr != nil { + return "", false, s.createSettingsErr + } + return "scr-fake", s.createSettingsReplayed, nil +} + +func (s *fakeJobStore) ActivateSourceSettingsRevisionWithOperation(_ context.Context, connectionID, revisionID, operationID string, _ time.Time) (bool, error) { + s.activateSettingsConnectionID = connectionID + s.activateSettingsRevisionID = revisionID + s.activateSettingsOperationID = operationID + return s.activateSettingsReplayed, s.activateSettingsErr +} + +func (s *fakeJobStore) ListSourceConnectionRevisions(_ context.Context, connectionID string) ([]governance.SourceConnectionRevision, error) { + s.listSettingsConnectionID = connectionID + return s.settingsRevisions, s.listSettingsErr +} + +func (s *fakeJobStore) TriggerPPSourceScan(_ context.Context, connectionID string, _ time.Time) (governance.SourceScan, error) { + s.triggerScanConnectionID = connectionID + return s.triggerScanResult, nil +} + +func (s *fakeJobStore) TriggerPPSourceScanWithOperation(_ context.Context, connectionID string, _ time.Time, operationID string) (governance.SourceScan, bool, error) { + s.triggerScanConnectionID = connectionID + s.triggerScanOperationID = operationID + return s.triggerScanResult, false, s.triggerScanErr +} + +func (s *fakeJobStore) ListSourceScans(_ context.Context, connectionID string, limit, offset int) (governance.SourceScanPage, error) { + s.scanPageConnectionID, s.scanPageLimit, s.scanPageOffset = connectionID, limit, offset + return s.scanPageResult, nil +} + +func (s *fakeJobStore) GetSourceScanDetail(_ context.Context, connectionID, scanID string) (governance.SourceScanDetailView, error) { + s.scanDetailConnectionID, s.scanDetailID = connectionID, scanID + return s.scanDetailResult, s.scanDetailErr +} + +func (s *fakeJobStore) ListReviewCandidates(_ context.Context, filter governance.ReviewCandidateFilter) (governance.ReviewCandidatePage, error) { + s.reviewFilter = filter + return s.reviewPageResult, nil +} + +func (s *fakeJobStore) ReviewPublication(_ context.Context, request governance.ReviewPublicationRequest) (governance.PublicationReview, error) { + s.reviewRequest = request + return s.reviewResult, nil +} + +func (s *fakeJobStore) ReviewPublicationWithOperation(_ context.Context, request governance.ReviewPublicationRequest, _ string) (governance.PublicationReview, bool, error) { + s.reviewRequest = request + return s.reviewResult, false, nil +} + +func (s *fakeJobStore) CreateReleaseBatch(_ context.Context, request governance.CreateReleaseBatchRequest) (governance.ReleaseBatch, error) { + s.createBatchRequest = request + return s.createBatchResult, nil +} + +func (s *fakeJobStore) CreateReleaseBatchWithOperation(_ context.Context, request governance.CreateReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) { + s.createBatchRequest = request + s.createBatchOperationID = operationID + return s.createBatchResult, false, s.createBatchErr +} + +func (s *fakeJobStore) GenerateReleaseChangeSetWithOperation(_ context.Context, request governance.GenerateReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) { + s.generateChangeSetRequest = request + s.generateChangeSetOperationID = operationID + return s.generateChangeSetResult, s.generateChangeSetReplayed, s.generateChangeSetErr +} + +func (s *fakeJobStore) ConfirmReleaseChangeSetWithOperation(_ context.Context, request governance.ConfirmReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) { + s.confirmReleaseSetRequest = request + s.confirmReleaseSetOperationID = operationID + return s.confirmReleaseSetResult, s.confirmReleaseSetReplayed, s.confirmReleaseSetErr +} + +func (s *fakeJobStore) ListReleaseChangeSets(_ context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseChangeSetPage, error) { + s.listChangeSetsFilter = filter + return s.changeSetPage, nil +} + +func (s *fakeJobStore) GetReleaseChangeSet(_ context.Context, id string) (governance.ReleaseChangeSetDetail, error) { + s.getChangeSetID = id + return s.changeSetDetail, s.changeSetDetailErr +} + +func (s *fakeJobStore) LatestReleaseChangeSet(_ context.Context) (governance.ReleaseChangeSetDetail, error) { + return s.changeSetDetail, s.changeSetDetailErr +} + +func (s *fakeJobStore) SealReleaseBatch(_ context.Context, request governance.SealReleaseBatchRequest) (governance.ReleaseBatch, error) { + s.sealBatchRequest = request + return s.sealBatchResult, nil +} + +func (s *fakeJobStore) SealReleaseBatchWithOperation(_ context.Context, request governance.SealReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) { + s.sealBatchRequest = request + s.sealBatchOperationID = operationID + return s.sealBatchResult, false, s.sealBatchErr +} + +func (s *fakeJobStore) ListReleaseBatches(_ context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseBatchPage, error) { + s.batchFilter = filter + return s.batchPageResult, nil +} + +func (s *fakeJobStore) GetReleaseBatch(_ context.Context, batchID string) (governance.ReleaseBatchDetail, error) { + s.batchID = batchID + return s.batchDetailResult, nil +} + +func (s *fakeJobStore) AuthorizeReleaseBatch(_ context.Context, request governance.AuthorizeReleaseBatchRequest) (governance.ReleaseExecution, error) { + s.authorizeReleaseCalled = true + s.authorizeReleaseRequest = request + return s.authorizeReleaseResult, nil +} + +func (s *fakeJobStore) AuthorizeReleaseBatchWithOperation(_ context.Context, request governance.AuthorizeReleaseBatchRequest, operationID string) (governance.ReleaseExecution, bool, error) { + s.authorizeReleaseCalled = true + s.authorizeReleaseRequest = request + s.authorizeReleaseOperationID = operationID + return s.authorizeReleaseResult, false, s.authorizeReleaseErr +} + +func (s *fakeJobStore) AuthorizeReleaseRollback(_ context.Context, request governance.AuthorizeReleaseRollbackRequest) (governance.ReleaseExecution, error) { + s.authorizeRollbackRequest = request + return s.authorizeRollbackResult, nil +} + +func (s *fakeJobStore) AuthorizeReleaseRollbackWithOperation(_ context.Context, request governance.AuthorizeReleaseRollbackRequest, operationID string) (governance.ReleaseExecution, bool, error) { + s.authorizeRollbackRequest = request + s.authorizeRollbackOperationID = operationID + return s.authorizeRollbackResult, false, s.authorizeRollbackErr +} + +func (s *fakeJobStore) RetryReleaseExecution(_ context.Context, request governance.RetryReleaseExecutionRequest) (governance.ReleaseExecution, error) { + s.retryReleaseRequest = request + return s.retryReleaseResult, nil +} + +func (s *fakeJobStore) RetryReleaseExecutionWithOperation(_ context.Context, request governance.RetryReleaseExecutionRequest, operationID string) (governance.ReleaseExecution, bool, error) { + s.retryReleaseRequest = request + s.retryReleaseOperationID = operationID + return s.retryReleaseResult, false, s.retryReleaseErr +} + +func (s *fakeJobStore) ListReleaseExecutions(_ context.Context, filter governance.ReleaseExecutionFilter) (governance.ReleaseExecutionPage, error) { + s.releaseExecutionFilter = filter + return s.releaseExecutionPageResult, nil +} + +func (s *fakeJobStore) GetReleaseAuditChain(_ context.Context, executionID string) (governance.ReleaseAuditChain, error) { + s.auditChainID = executionID + return s.auditChain, s.auditChainErr +} + +func (s *fakeJobStore) GetReleaseExecution(_ context.Context, executionID string) (governance.ReleaseExecutionDetail, error) { + s.releaseExecutionID = executionID + return s.releaseExecutionDetailResult, nil +} + +func (s *fakeJobStore) ListReleasePlan(context.Context) (governance.ReleasePlan, error) { + return s.releasePlanResult, nil +} + +func (s *fakeJobStore) GetActiveUISettings(context.Context) (governance.UISettings, error) { + return s.activeUISettingsResult, nil +} + +func (s *fakeJobStore) CreateUISettingsCandidate(_ context.Context, request governance.CreateUISettingsCandidateRequest) (governance.UISettings, error) { + s.createUISettingsRequest = request + return s.candidateUISettingsResult, nil +} + +func (s *fakeJobStore) CreateUISettingsCandidateWithOperation(_ context.Context, request governance.CreateUISettingsCandidateRequest, operationID string) (governance.UISettings, bool, error) { + s.createUISettingsRequest = request + s.createUISettingsOperationID = operationID + return s.candidateUISettingsResult, false, s.createUISettingsErr +} + +func (s *fakeJobStore) ActivateUISettings(_ context.Context, revisionID string, _ time.Time) (governance.UISettings, error) { + s.activateUISettingsID = revisionID + return s.activatedUISettingsResult, nil +} + +func (s *fakeJobStore) ActivateUISettingsWithOperation(_ context.Context, revisionID string, _ time.Time, operationID string) (governance.UISettings, bool, error) { + s.activateUISettingsID = revisionID + s.activateUISettingsOperationID = operationID + return s.activatedUISettingsResult, false, s.activateUISettingsErr +} + +func (s *fakeJobStore) Enqueue(_ context.Context, request jobs.EnqueueRequest, _ time.Time) (*jobs.Job, error) { + s.enqueued = request + return s.enqueueResult, nil +} + +func (s *fakeJobStore) List(_ context.Context, limit int) ([]jobs.Job, error) { + s.listLimit = limit + return s.listResult, nil +} + +func (s *fakeJobStore) ListTestPublications(_ context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error) { + s.publicationFilter = filter + return s.publicationResult, nil +} + +func (s *fakeJobStore) ConfirmCatalogChangeSetWithOperation(_ context.Context, request governance.ConfirmCatalogChangeSetRequest, operationID string) (governance.ConfirmCatalogChangeSetResult, bool, error) { + s.confirmChangeSetRequest = request + s.confirmChangeSetOperationID = operationID + return s.confirmChangeSetResult, s.confirmChangeSetReplayed, s.confirmChangeSetErr +} + +func (s *fakeJobStore) StartTestPublications(_ context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) { + s.startPublicationsRequest = request + return s.startPublicationsResult, nil +} + +func (s *fakeJobStore) StartTestPublicationsWithOperation(_ context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) { + s.startPublicationsRequest = request + s.startPublicationsOperationID = operationID + return s.startPublicationsResult, s.startPublicationsReplayed, nil +} + +func (s *fakeJobStore) ListAcceptanceResults(_ context.Context, id string) ([]governance.AcceptanceRecord, error) { + s.acceptanceID = id + return s.acceptanceResult, nil +} + +func (s *fakeJobStore) RetryPublication(_ context.Context, id string, _ time.Time) error { + s.retryID = id + return s.retryErr +} + +func (s *fakeJobStore) RetryPublicationWithOperation(_ context.Context, id string, _ time.Time, operationID string) (bool, error) { + s.retryID = id + s.retryOperationID = operationID + return s.retryReplayed, s.retryErr +} + +func (s *fakeJobStore) ListJobFailures(_ context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) { + s.jobFailureFilter = filter + return s.jobFailurePage, s.jobFailureErr +} + +func (s *fakeJobStore) RetryPublicationsBatchWithOperation(_ context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) { + s.retryBatchRequest = &request + s.retryBatchOperation = operationID + return s.retryBatchResult, s.retryBatchReplayed, s.retryBatchErr +} + +func (s *fakeJobStore) ListScopeRules(_ context.Context, connectionID string) ([]governance.ScopeRule, error) { + s.scopeRuleConnectionID = connectionID + return s.scopeRules, s.scopeRuleErr +} + +func (s *fakeJobStore) MutateScopeRule(_ context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error) { + s.scopeRuleRequest = request + return s.scopeRuleResult, s.scopeRuleErr +} + +func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error) { + s.scopeRuleRequest = request + s.scopeRuleOperationID = operationID + return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr +} + +func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) { + return s.catalogSources, s.catalogErr +} + +func (s *fakeJobStore) ListCatalogNodes(_ context.Context, filter governance.CatalogNodeFilter) (governance.CatalogNodePage, error) { + s.catalogFilter = filter + return s.catalogPage, s.catalogErr +} + +func (s *fakeJobStore) ListCatalogSnapshots(_ context.Context, connectionID string, limit, offset int) (governance.CatalogSnapshotPage, error) { + s.snapshotConnectionID = connectionID + return s.snapshotPageResult, s.snapshotErr +} + +func (s *fakeJobStore) ListCatalogNodesInSnapshot(_ context.Context, filter governance.CatalogSnapshotNodeFilter) (governance.CatalogNodePage, error) { + s.snapshotNodeFilter = filter + return s.snapshotNodePageResult, s.snapshotErr +} + +func (s *fakeJobStore) RouteCatalogFiles(_ context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) { + s.catalogRouteRequest = request + return s.catalogRouteResult, s.catalogRouteErr +} + +func (s *fakeJobStore) RouteCatalogFilesWithOperation(_ context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) { + s.catalogRouteRequest = request + s.catalogRouteOperationID = operationID + return s.catalogRouteResult, s.catalogRouteReplayed, s.catalogRouteErr +} + +func (s *fakeJobStore) EnqueueCatalogRefreshWithOperation(_ context.Context, source governance.CatalogSource, operationID string, _ time.Time) (jobs.Job, bool, error) { + s.catalogRefreshSource = source + s.catalogRefreshOperationID = operationID + return s.catalogRefreshResult, s.catalogRefreshReplayed, s.catalogRefreshErr +} + +func (s *fakeJobStore) GetLatestCatalogChangeSet(_ context.Context, filter governance.CatalogChangeSetFilter) (governance.CatalogChangeSetDetail, error) { + s.catalogChangeSetFilter = filter + return s.catalogChangeSetResult, nil +} + +type fakeMonitor struct { + overview governance.MonitorOverview + files governance.KnowledgeMonitorFilePage + knowledgeBaseID string + status string + metrics governance.RunMetrics + metricsErr error + services []governance.ServiceHealth +} + +func (monitor *fakeMonitor) Overview(context.Context) (governance.MonitorOverview, error) { + return monitor.overview, nil +} + +func (monitor *fakeMonitor) Services(context.Context) []governance.ServiceHealth { + return monitor.services +} + +func (monitor *fakeMonitor) Metrics(context.Context) (governance.RunMetrics, error) { + return monitor.metrics, monitor.metricsErr +} + +func (monitor *fakeMonitor) ListFiles(_ context.Context, knowledgeBaseID, status string, _, _ int) (governance.KnowledgeMonitorFilePage, error) { + monitor.knowledgeBaseID = knowledgeBaseID + monitor.status = status + return monitor.files, nil +} diff --git a/internal/httpapi/ui_settings_integration_test.go b/internal/httpapi/ui_settings_integration_test.go new file mode 100644 index 0000000..6a8c4ab --- /dev/null +++ b/internal/httpapi/ui_settings_integration_test.go @@ -0,0 +1,78 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +func TestUISettingsWriteReadBackIntegration(t *testing.T) { + databaseURL := os.Getenv("TEST_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 10, 9, 0, 0, 0, time.UTC) + handler, err := NewHandler(Config{Store: postgresstore.NewStore(pool), Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + + create := httptest.NewRecorder() + createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"updated_by":"engineer-e2e"}`)) + createRequest.Header.Set("X-Governance-Action", "ui-settings") + createRequest.Header.Set("Idempotency-Key", "op-e2e-uis-create") + handler.ServeHTTP(create, createRequest) + if create.Code != http.StatusCreated { + t.Fatalf("create status=%d body=%s", create.Code, create.Body.String()) + } + var candidate struct { + Data governance.UISettings `json:"data"` + } + if err := json.NewDecoder(create.Body).Decode(&candidate); err != nil || candidate.Data.State != "validated" { + t.Fatalf("candidate=%#v error=%v", candidate, err) + } + + activate := httptest.NewRecorder() + activateRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/"+candidate.Data.ID+"/activate", bytes.NewBufferString(`{}`)) + activateRequest.Header.Set("X-Governance-Action", "ui-settings") + activateRequest.Header.Set("Idempotency-Key", "op-e2e-uis-activate") + handler.ServeHTTP(activate, activateRequest) + if activate.Code != http.StatusOK { + t.Fatalf("activate status=%d body=%s", activate.Code, activate.Body.String()) + } + + get := httptest.NewRecorder() + handler.ServeHTTP(get, httptest.NewRequest(http.MethodGet, "/api/v1/ui-settings", nil)) + if get.Code != http.StatusOK { + t.Fatalf("get status=%d body=%s", get.Code, get.Body.String()) + } + var active struct { + Data governance.UISettings `json:"data"` + } + if err := json.NewDecoder(get.Body).Decode(&active); err != nil || active.Data.ID != candidate.Data.ID || active.Data.State != "active" || active.Data.DocumentRefreshSeconds != 30 { + t.Fatalf("active=%#v error=%v", active, err) + } + var activeCount, retiredCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FILTER(WHERE state='active'),COUNT(*) FILTER(WHERE state='retired') FROM ui_setting_revisions`).Scan(&activeCount, &retiredCount); err != nil || activeCount != 1 || retiredCount != 1 { + t.Fatalf("active=%d retired=%d error=%v", activeCount, retiredCount, err) + } +} diff --git a/internal/jobs/demo.go b/internal/jobs/demo.go new file mode 100644 index 0000000..90e2b17 --- /dev/null +++ b/internal/jobs/demo.go @@ -0,0 +1,35 @@ +package jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" +) + +const maxDemoSleep = 60 * time.Second + +func DemoSleepHandler(ctx context.Context, job Job) error { + var payload struct { + SleepMilliseconds int `json:"sleep_ms"` + } + if err := json.Unmarshal(job.Payload, &payload); err != nil { + return fmt.Errorf("decode demo sleep payload: %w", err) + } + if payload.SleepMilliseconds < 1 { + return errors.New("sleep_ms must be positive") + } + duration := time.Duration(payload.SleepMilliseconds) * time.Millisecond + if duration > maxDemoSleep { + return errors.New("sleep_ms must not exceed 60000") + } + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/jobs/demo_test.go b/internal/jobs/demo_test.go new file mode 100644 index 0000000..6bae946 --- /dev/null +++ b/internal/jobs/demo_test.go @@ -0,0 +1,18 @@ +package jobs + +import ( + "context" + "errors" + "testing" +) + +func TestDemoSleepHandlerStopsWhenContextIsCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + job := Job{Payload: []byte(`{"sleep_ms":30000}`)} + + err := DemoSleepHandler(ctx, job) + if !errors.Is(err, context.Canceled) { + t.Fatalf("DemoSleepHandler() error = %v, want context.Canceled", err) + } +} diff --git a/internal/jobs/errors.go b/internal/jobs/errors.go new file mode 100644 index 0000000..e2fc13d --- /dev/null +++ b/internal/jobs/errors.go @@ -0,0 +1,51 @@ +package jobs + +import "errors" + +// Error codes persisted with terminal failures (P6 DLQ). The table is closed: +// a handler without an explicit code lands in "handler_error" so the DLQ never +// shows a blank classification. +const ( + ErrCodeNoHandler = "no_handler" + ErrCodeArtifactInvalid = "artifact_invalid" + ErrCodeRemotePermanent = "remote_permanent" + ErrCodeUploadIdentity = "upload_identity_ambiguous" + ErrCodeValidationFailed = "validation_failed" + ErrCodeStateConflict = "state_conflict" + ErrCodeHandlerFallback = "handler_error" +) + +// PermanentError marks a failure that must not consume the retry budget: +// the job goes straight to the terminal (dead-letter) state. +type PermanentError struct { + Code string + Err error +} + +func (e *PermanentError) Error() string { return e.Err.Error() } +func (e *PermanentError) Unwrap() error { return e.Err } + +func Permanent(code string, err error) error { + return &PermanentError{Code: safeCode(code), Err: err} +} + +// Terminal reports whether an error must fail immediately plus its stable code. +func Terminal(err error) (bool, string) { + var permanent *PermanentError + if errors.As(err, &permanent) { + return true, safeCode(permanent.Code) + } + return false, ErrCodeHandlerFallback +} + +func safeCode(code string) string { + if code == "" || len(code) > 64 { + return ErrCodeHandlerFallback + } + for _, r := range code { + if !(r == '_' || r >= 'a' && r <= 'z' || r >= '0' && r <= '9') { + return ErrCodeHandlerFallback + } + } + return code +} diff --git a/internal/jobs/job.go b/internal/jobs/job.go new file mode 100644 index 0000000..309fb89 --- /dev/null +++ b/internal/jobs/job.go @@ -0,0 +1,57 @@ +package jobs + +import ( + "encoding/json" + "time" +) + +type State string + +const ( + StatePending State = "pending" + StateRunning State = "running" + StateRetry State = "retry" + StateCompleted State = "completed" + StateFailed State = "failed" + StateCancelled State = "cancelled" +) + +func (state State) Valid() bool { + switch state { + case StatePending, StateRunning, StateRetry, StateCompleted, StateFailed, StateCancelled: + return true + default: + return false + } +} + +func (state State) Terminal() bool { + return state == StateCompleted || state == StateFailed || state == StateCancelled +} + +type Job struct { + ID string `json:"id"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + State State `json:"state"` + Priority int `json:"priority"` + AvailableAt time.Time `json:"available_at"` + LeaseOwner string `json:"lease_owner,omitempty"` + LeaseUntil *time.Time `json:"lease_until,omitempty"` + Attempt int `json:"attempt"` + MaxAttempts int `json:"max_attempts"` + LastError string `json:"last_error,omitempty"` + IdempotencyKey string `json:"idempotency_key"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type EnqueueRequest struct { + ID string + Type string + Payload json.RawMessage + Priority int + AvailableAt time.Time + MaxAttempts int + IdempotencyKey string +} diff --git a/internal/jobs/job_test.go b/internal/jobs/job_test.go new file mode 100644 index 0000000..b324524 --- /dev/null +++ b/internal/jobs/job_test.go @@ -0,0 +1,21 @@ +package jobs + +import "testing" + +func TestStateCancelledIsValidTerminalAndNotClaimable(t *testing.T) { + if !StateCancelled.Valid() { + t.Fatal("cancelled must be a valid job state") + } + if !StateCancelled.Terminal() { + t.Fatal("cancelled must be terminal") + } + if StatePending.Terminal() || StateRunning.Terminal() || StateRetry.Terminal() { + t.Fatal("active job states must not be terminal") + } + if !StateCompleted.Terminal() || !StateFailed.Terminal() { + t.Fatal("completed and failed job states must be terminal") + } + if State("invalid").Valid() { + t.Fatal("unknown job state must be invalid") + } +} diff --git a/internal/jobs/worker.go b/internal/jobs/worker.go new file mode 100644 index 0000000..77516c1 --- /dev/null +++ b/internal/jobs/worker.go @@ -0,0 +1,179 @@ +package jobs + +import ( + "context" + "errors" + "sort" + "time" +) + +type Queue interface { + Claim(ctx context.Context, workerID string, allowedTypes []string, now time.Time, leaseDuration time.Duration) (*Job, error) + Complete(ctx context.Context, id, owner string, now time.Time) error + Fail(ctx context.Context, id, owner, message, code string, now, retryAt time.Time) error + Heartbeat(ctx context.Context, id, owner string, now time.Time, leaseDuration time.Duration) error +} + +// TerminalFailer is an optional Queue extension (P6): a permanent failure goes +// straight to the dead-letter state with a stable error code instead of +// consuming the retry budget. +type TerminalFailer interface { + FailTerminal(ctx context.Context, id, owner, message, code string, now time.Time) error +} + +type Handler func(context.Context, Job) error + +type WorkerConfig struct { + Queue Queue + WorkerID string + LeaseDuration time.Duration + HeartbeatInterval time.Duration + RetryDelay time.Duration + Now func() time.Time + Handlers map[string]Handler + OnError func(error) +} + +type Worker struct { + queue Queue + workerID string + leaseDuration time.Duration + heartbeatInterval time.Duration + retryDelay time.Duration + now func() time.Time + handlers map[string]Handler + allowedTypes []string + onError func(error) +} + +func NewWorker(config WorkerConfig) (*Worker, error) { + if config.Queue == nil { + return nil, errors.New("queue is required") + } + if config.WorkerID == "" { + return nil, errors.New("worker ID is required") + } + if config.LeaseDuration <= 0 { + return nil, errors.New("lease duration must be positive") + } + if config.Now == nil { + config.Now = time.Now + } + if config.RetryDelay <= 0 { + config.RetryDelay = time.Second + } + if config.HeartbeatInterval <= 0 { + config.HeartbeatInterval = config.LeaseDuration / 3 + } + if config.OnError == nil { + config.OnError = func(error) {} + } + allowedTypes := make([]string, 0, len(config.Handlers)) + for jobType := range config.Handlers { + allowedTypes = append(allowedTypes, jobType) + } + sort.Strings(allowedTypes) + return &Worker{ + queue: config.Queue, + workerID: config.WorkerID, + leaseDuration: config.LeaseDuration, + heartbeatInterval: config.HeartbeatInterval, + retryDelay: config.RetryDelay, + now: config.Now, + handlers: config.Handlers, + allowedTypes: allowedTypes, + onError: config.OnError, + }, nil +} + +func (w *Worker) Run(ctx context.Context, pollInterval time.Duration) error { + if pollInterval <= 0 { + return errors.New("poll interval must be positive") + } + for { + claimed, err := w.RunOnce(ctx) + if err != nil { + w.onError(err) + } + if claimed && err == nil { + continue + } + timer := time.NewTimer(pollInterval) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return nil + case <-timer.C: + } + } +} + +func (w *Worker) failJob(ctx context.Context, job Job, cause error, failedAt time.Time) error { + code := "" + if terminal, c := Terminal(cause); terminal { + code = c + if failer, ok := w.queue.(TerminalFailer); ok { + return failer.FailTerminal(ctx, job.ID, w.workerID, cause.Error(), code, failedAt) + } + } + return w.queue.Fail(ctx, job.ID, w.workerID, cause.Error(), code, failedAt, failedAt.Add(w.retryDelay)) +} + +func (w *Worker) RunOnce(ctx context.Context) (bool, error) { + claimedAt := w.now() + job, err := w.queue.Claim(ctx, w.workerID, w.allowedTypes, claimedAt, w.leaseDuration) + if err != nil { + return false, err + } + if job == nil { + return false, nil + } + + handler := w.handlers[job.Type] + if handler == nil { + err := Permanent(ErrCodeNoHandler, errors.New("no handler registered for job type "+job.Type)) + failedAt := w.now() + if failErr := w.failJob(ctx, *job, err, failedAt); failErr != nil { + return true, failErr + } + return true, err + } + heartbeatCtx, stopHeartbeat := context.WithCancel(ctx) + heartbeatResult := make(chan error, 1) + go func() { + ticker := time.NewTicker(w.heartbeatInterval) + defer ticker.Stop() + for { + select { + case <-heartbeatCtx.Done(): + heartbeatResult <- nil + return + case <-ticker.C: + if err := w.queue.Heartbeat(ctx, job.ID, w.workerID, w.now(), w.leaseDuration); err != nil { + heartbeatResult <- err + return + } + } + } + }() + + handlerErr := handler(ctx, *job) + stopHeartbeat() + heartbeatErr := <-heartbeatResult + if handlerErr == nil { + handlerErr = heartbeatErr + } + if handlerErr != nil { + failedAt := w.now() + if failErr := w.failJob(ctx, *job, handlerErr, failedAt); failErr != nil { + return true, failErr + } + return true, handlerErr + } + if err := w.queue.Complete(ctx, job.ID, w.workerID, w.now()); err != nil { + return true, err + } + return true, nil +} diff --git a/internal/jobs/worker_test.go b/internal/jobs/worker_test.go new file mode 100644 index 0000000..1b95d6f --- /dev/null +++ b/internal/jobs/worker_test.go @@ -0,0 +1,295 @@ +package jobs + +import ( + "context" + "errors" + "slices" + "testing" + "time" +) + +func TestWorkerRunOnceCompletesHandledJob(t *testing.T) { + now := time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) + queue := &recordingQueue{claimJob: &Job{ID: "job-1", Type: "demo", State: StateRunning, LeaseOwner: "worker-1"}} + called := false + + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + Now: func() time.Time { return now }, + Handlers: map[string]Handler{ + "demo": func(context.Context, Job) error { + called = true + return nil + }, + }, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + + claimed, err := worker.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if !claimed { + t.Fatal("RunOnce() claimed = false, want true") + } + if !called { + t.Fatal("handler was not called") + } + if queue.completedID != "job-1" { + t.Fatalf("completed job = %q, want job-1", queue.completedID) + } + if queue.completedOwner != "worker-1" { + t.Fatalf("completion owner = %q, want worker-1", queue.completedOwner) + } +} + +func TestWorkerCompletesAtHandlerFinishRatherThanClaimTime(t *testing.T) { + claimedAt := time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) + completedAt := claimedAt.Add(7*time.Minute + 13*time.Second) + times := []time.Time{claimedAt, completedAt} + queue := &recordingQueue{claimJob: &Job{ID: "job-duration", Type: "demo", State: StateRunning, LeaseOwner: "worker-1"}} + worker, err := NewWorker(WorkerConfig{ + Queue: queue, WorkerID: "worker-1", LeaseDuration: 30 * time.Second, + Now: func() time.Time { value := times[0]; times = times[1:]; return value }, + Handlers: map[string]Handler{"demo": func(context.Context, Job) error { return nil }}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatal(err) + } + if !queue.completedAt.Equal(completedAt) { + t.Fatalf("completed at = %s, want handler finish %s", queue.completedAt, completedAt) + } +} + +func TestWorkerRunOnceSchedulesFailedJobForRetry(t *testing.T) { + now := time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) + wantErr := errors.New("temporary failure") + queue := &recordingQueue{claimJob: &Job{ID: "job-2", Type: "demo", State: StateRunning, LeaseOwner: "worker-1"}} + + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + RetryDelay: 5 * time.Second, + Now: func() time.Time { return now }, + Handlers: map[string]Handler{ + "demo": func(context.Context, Job) error { return wantErr }, + }, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + + claimed, err := worker.RunOnce(context.Background()) + if !errors.Is(err, wantErr) { + t.Fatalf("RunOnce() error = %v, want %v", err, wantErr) + } + if !claimed { + t.Fatal("RunOnce() claimed = false, want true") + } + if queue.failedID != "job-2" { + t.Fatalf("failed job = %q, want job-2", queue.failedID) + } + if queue.failedMessage != wantErr.Error() { + t.Fatalf("failure message = %q, want %q", queue.failedMessage, wantErr.Error()) + } + if want := now.Add(5 * time.Second); !queue.retryAt.Equal(want) { + t.Fatalf("retry at = %s, want %s", queue.retryAt, want) + } +} + +func TestWorkerRunOnceParksPermanentFailureImmediately(t *testing.T) { + now := time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) + queue := &recordingQueue{claimJob: &Job{ID: "job-perm", Type: "demo", State: StateRunning, LeaseOwner: "worker-1"}} + + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + RetryDelay: 5 * time.Second, + Now: func() time.Time { return now }, + Handlers: map[string]Handler{ + "demo": func(context.Context, Job) error { return Permanent("artifact_invalid", errors.New("bad file")) }, + }, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + + claimed, err := worker.RunOnce(context.Background()) + if err == nil || !claimed { + t.Fatalf("RunOnce() = (%v, %v), want claimed error", claimed, err) + } + if queue.terminalID != "job-perm" { + t.Fatalf("terminal job = %q, want job-perm (FailTerminal path)", queue.terminalID) + } + if queue.terminalCode != "artifact_invalid" { + t.Fatalf("terminal code = %q, want artifact_invalid", queue.terminalCode) + } + if !queue.terminalAt.Equal(now) { + t.Fatalf("terminal at = %s, want %s", queue.terminalAt, now) + } + if queue.retryAt.IsZero() == false && queue.failedID != "" { + t.Fatal("permanent failure must not schedule a retry") + } +} + +func TestPermanentFallsBackToCodeOnOrdinaryFail(t *testing.T) { + // A non-permanent error keeps the retry path with an empty code so the + // database preserves the previous classification. + if terminal, code := Terminal(errors.New("boom")); terminal || code != ErrCodeHandlerFallback { + t.Fatalf("Terminal(plain) = (%v, %q)", terminal, code) + } +} + +func TestWorkerRunOnceHeartbeatsWhileHandlerRuns(t *testing.T) { + heartbeat := make(chan struct{}, 1) + queue := &recordingQueue{ + claimJob: &Job{ID: "job-3", Type: "demo", State: StateRunning, LeaseOwner: "worker-1"}, + heartbeat: heartbeat, + } + + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + HeartbeatInterval: 5 * time.Millisecond, + Handlers: map[string]Handler{ + "demo": func(ctx context.Context, _ Job) error { + select { + case <-heartbeat: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + return errors.New("heartbeat was not sent") + } + }, + }, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + + claimed, err := worker.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if !claimed { + t.Fatal("RunOnce() claimed = false, want true") + } + if queue.heartbeatCount == 0 { + t.Fatal("heartbeat count = 0, want at least 1") + } +} + +func TestWorkerRunStopsWhenContextIsCanceled(t *testing.T) { + queue := &recordingQueue{} + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.Run(ctx, 5*time.Millisecond) }() + cancel() + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run() error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Run() did not stop after cancellation") + } +} + +func TestWorkerClaimsOnlyRegisteredHandlerTypes(t *testing.T) { + queue := &recordingQueue{} + worker, err := NewWorker(WorkerConfig{ + Queue: queue, + WorkerID: "worker-1", + LeaseDuration: 30 * time.Second, + Handlers: map[string]Handler{ + "source.scan": func(context.Context, Job) error { return nil }, + "test.ingest": func(context.Context, Job) error { return nil }, + }, + }) + if err != nil { + t.Fatalf("NewWorker() error = %v", err) + } + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + want := []string{"source.scan", "test.ingest"} + if !slices.Equal(queue.claimTypes, want) { + t.Fatalf("claim types = %#v, want %#v", queue.claimTypes, want) + } +} + +type recordingQueue struct { + claimJob *Job + claimTypes []string + completedID string + completedOwner string + completedAt time.Time + failedID string + failedMessage string + failedCode string + retryAt time.Time + terminalID string + terminalCode string + terminalAt time.Time + heartbeat chan struct{} + heartbeatCount int +} + +func (q *recordingQueue) Claim(_ context.Context, _ string, types []string, _ time.Time, _ time.Duration) (*Job, error) { + q.claimTypes = append([]string(nil), types...) + return q.claimJob, nil +} + +func (q *recordingQueue) Complete(_ context.Context, id, owner string, now time.Time) error { + q.completedID = id + q.completedOwner = owner + q.completedAt = now + return nil +} + +func (q *recordingQueue) Fail(_ context.Context, id, _ string, message, code string, _, retryAt time.Time) error { + q.failedID = id + q.failedMessage = message + q.failedCode = code + q.retryAt = retryAt + return nil +} + +func (q *recordingQueue) FailTerminal(_ context.Context, id, _, message, code string, now time.Time) error { + q.terminalID = id + q.failedMessage = message + q.terminalCode = code + q.terminalAt = now + return nil +} + +func (q *recordingQueue) Heartbeat(context.Context, string, string, time.Time, time.Duration) error { + q.heartbeatCount++ + if q.heartbeat != nil { + select { + case q.heartbeat <- struct{}{}: + default: + } + } + return nil +} diff --git a/internal/monitoring/health.go b/internal/monitoring/health.go new file mode 100644 index 0000000..bcdad29 --- /dev/null +++ b/internal/monitoring/health.go @@ -0,0 +1,328 @@ +package monitoring + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "knowledge-governance/internal/governance" +) + +// P10: every configured dependency reports Liveness ("process reachable") and +// Readiness ("can accept work") independently. Missing deployment and missing +// configuration are explicit states and never collapse into "unknown". + +const ( + LiveUp = "up" + LiveDown = "down" + LiveUnknown = "unknown" + ReadReady = "ready" + ReadBlocked = "blocked" + ReadNotConfigured = "not_configured" + ReadNotApplicable = "not_applicable" +) + +// ProbeOutcome is one raw probe attempt. +type ProbeOutcome struct { + Reachability error // liveness result (nil = reachable) + WorkReadiness error // readiness result (nil = can accept work) + Latency time.Duration + ObservedAt time.Time +} + +var ( + errCapacityFull = errors.New("capacity_full") + errNoPermission = errors.New("no_permission") +) + +// classify turns one probe outcome into health states. Explicit not-configured +// and not-applicable results win over generic reachability failures. +func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth { + health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt} + switch { + case !applicable: + health.Liveness, health.Reason = LiveUnknown, "未部署：本平台不使用该依赖" + health.Readiness = ReadNotApplicable + return health + case !configured: + health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数" + health.Readiness = ReadNotConfigured + return health + } + if outcome.Reachability != nil { + health.Readiness = ReadBlocked + health.Reason = sanitizeProbeError(outcome.Reachability) + if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) { + health.Liveness = LiveUp // reachable, but not workable + } else { + health.Liveness = LiveDown + } + return health + } + health.Liveness = LiveUp + if outcome.WorkReadiness != nil { + health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness) + return health + } + health.Readiness = ReadReady + return health +} + +// sanitizeProbeError keeps operators useful text without leaking endpoints, +// credentials, or hosts that live inside URLs. +func sanitizeProbeError(err error) string { + switch { + case errors.Is(err, errCapacityFull): + return "服务可达但容量已满（429），暂时不可接单" + case errors.Is(err, errNoPermission): + return "服务可达但目标知识库无读取权限" + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return "探测超时" + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return "地址解析失败（DNS）" + } + var netErr net.Error + if errors.As(err, &netErr) { + return "网络不可达" + } + message := err.Error() + if len(message) > 160 { + return message[:160] + } + return message +} + +// healthTracker remembers recent success and consecutive failures per service. +type healthTracker struct { + records map[string]*trackedHealth +} + +type trackedHealth struct { + lastSuccess *time.Time + failStreak int +} + +func newHealthTracker() *healthTracker { + return &healthTracker{records: map[string]*trackedHealth{}} +} + +func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth { + record := tracker.records[key] + if record == nil { + record = &trackedHealth{} + tracker.records[key] = record + } + switch { + case health.Liveness == LiveUp && health.Readiness == ReadReady: + now := health.ObservedAt + record.lastSuccess = &now + record.failStreak = 0 + case health.Liveness == LiveUnknown: + // Config-level states say nothing about runtime; keep the history. + default: + record.failStreak++ + } + health.LastSuccessAt = record.lastSuccess + health.ConsecutiveFailures = record.failStreak + return health +} + +// httpProbe performs a liveness-only HTTP GET on /health, mapped through +// classify. configured=false when the base URL is absent. +func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth { + baseURL = strings.TrimSpace(baseURL) + if baseURL == "" { + return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now})) + } + parsed, err := url.Parse(baseURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now})) + } + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" + parsed.RawQuery, parsed.Fragment = "", "" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now})) + } + started := time.Now() + response, err := provider.httpClient.Do(request) + outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now} + if err != nil { + outcome.Reachability = err + return provider.track(classify(key, name, true, true, outcome)) + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + if response.StatusCode < 200 || response.StatusCode >= 300 { + outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode) + return provider.track(classify(key, name, true, true, outcome)) + } + return provider.track(classify(key, name, true, true, outcome)) +} + +// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an +// optional real read against a routing knowledge base. +func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth { + if provider.weknoraBaseURL == "" { + return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now})) + } + outcome := provider.httpProbeOutcome(ctx, provider.weknoraBaseURL, now) + if outcome.Reachability == nil && provider.weknoraReadiness != nil { + outcome.WorkReadiness = provider.weknoraReadiness(ctx) + } + return provider.track(classify("weknora", "WeKnora API", true, true, outcome)) +} + +func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome { + parsed, err := url.Parse(baseURL) + outcome := ProbeOutcome{ObservedAt: now} + if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" { + outcome.Reachability = errors.New("探测配置无效") + return outcome + } + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" + parsed.RawQuery, parsed.Fragment = "", "" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + outcome.Reachability = err + return outcome + } + started := time.Now() + response, err := provider.httpClient.Do(request) + outcome.Latency = time.Since(started) + if err != nil { + outcome.Reachability = err + return outcome + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + switch { + case response.StatusCode == http.StatusTooManyRequests: + // Reachable but at capacity: liveness stays up, readiness blocks. + outcome.Reachability = errCapacityFull + case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized: + outcome.Reachability = errNoPermission + case response.StatusCode < 200 || response.StatusCode >= 300: + outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode) + } + return outcome +} + +// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means +// the worker loop is not touching the queue (liveness down), while the queue +// itself can still be ready for a future worker. +func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth { + rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter)) + if err != nil { + return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))} + } + defer rows.Close() + type entry struct { + id string + seen time.Time + } + live := []entry{} + for rows.Next() { + var id string + var seen time.Time + if err := rows.Scan(&id, &seen); err != nil { + return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))} + } + live = append(live, entry{id, seen.UTC()}) + } + if len(live) == 0 { + return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳（worker未运行或全部停跳）"), ObservedAt: now}))} + } + out := make([]governance.ServiceHealth, 0, len(live)) + for _, item := range live { + outcome := ProbeOutcome{ObservedAt: item.seen} + if now.Sub(item.seen) > provider.workerStaleAfter { + outcome.Reachability = errors.New("心跳过期") + } + out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome))) + } + return out +} + +func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth { + return provider.health.apply(health.Key, health) +} + +// Services reports the independent Liveness/Readiness matrix for every service +// the deployment actually has (P10). Missing services are explicit states. +func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth { + now := provider.now().UTC() + health := make([]governance.ServiceHealth, 0, 8) + // PostgreSQL: liveness = ping, readiness = queryable. + pgOutcome := ProbeOutcome{ObservedAt: now} + started := time.Now() + if err := provider.db.Ping(ctx); err != nil { + pgOutcome.Reachability = err + } else { + pgOutcome.Latency = time.Since(started) + var ok int + if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil { + pgOutcome.WorkReadiness = err + } + } + health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome))) + // Queue readiness rides on the jobs table. + queueOutcome := ProbeOutcome{ObservedAt: now} + var pending int + if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil { + queueOutcome.Reachability = err + } + health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome))) + health = append(health, provider.workerProbe(ctx, now)...) + health = append(health, provider.weKnoraProbe(ctx, now)) + health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now)) + // Not-applicable dependencies stay explicit (embedding/llm managed by + // WeKnora, redis/mq/oss not deployed). + health = append(health, + provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})), + provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})), + provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})), + provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})), + ) + sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key }) + return health +} + +// Readiness blocks only on services that ARE deployed and configured and +// currently fail readiness; not_configured is surfaced as a warning elsewhere. +func overallReadiness(health []governance.ServiceHealth) (bool, []string) { + blocked := []string{} + for _, item := range health { + if !item.Applicable || !item.Configured { + continue + } + if item.Readiness != ReadReady { + blocked = append(blocked, item.Key) + } + } + return len(blocked) == 0, blocked +} + +func (tracker *healthTracker) snapshot() []governance.ServiceHealth { + keys := make([]string, 0, len(tracker.records)) + for key := range tracker.records { + keys = append(keys, key) + } + sort.Strings(keys) + records := make([]governance.ServiceHealth, 0, len(keys)) + for _, key := range keys { + record := tracker.records[key] + item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak} + records = append(records, item) + } + return records +} diff --git a/internal/monitoring/health_test.go b/internal/monitoring/health_test.go new file mode 100644 index 0000000..334e74f --- /dev/null +++ b/internal/monitoring/health_test.go @@ -0,0 +1,84 @@ +package monitoring + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "testing" + "time" +) + +func testNow() time.Time { return time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) } + +func TestClassifySeparatesNotApplicableNotConfiguredAndFailures(t *testing.T) { + now := testNow() + notDeployed := classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now}) + if notDeployed.Liveness != LiveUnknown || notDeployed.Readiness != ReadNotApplicable { + t.Fatalf("not deployed = %+v", notDeployed) + } + notConfigured := classify("mineru", "MinerU", false, true, ProbeOutcome{ObservedAt: now}) + if notConfigured.Liveness != LiveUnknown || notConfigured.Readiness != ReadNotConfigured { + t.Fatalf("not configured = %+v", notConfigured) + } + healthy := classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now}) + if healthy.Liveness != LiveUp || healthy.Readiness != ReadReady { + t.Fatalf("healthy = %+v", healthy) + } + // DNS failure: down + blocked with sanitized reason, never the raw host. + dns := &net.DNSError{Err: "no such host", Name: "weknora.internal.example"} + down := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: dns}) + if down.Liveness != LiveDown || down.Reason == "" || strings.Contains(down.Reason, "weknora.internal") { + t.Fatalf("dns down = %+v", down) + } + // Timeout. + timeout := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: context.DeadlineExceeded}) + if timeout.Liveness != LiveDown || timeout.Readiness != ReadBlocked { + t.Fatalf("timeout = %+v", timeout) + } + // Reachable but capacity full => live up, readiness blocked, distinct reason. + capacity := classify("mineru", "MinerU", true, true, ProbeOutcome{ObservedAt: now, Reachability: errCapacityFull}) + if capacity.Liveness != LiveUp || capacity.Readiness != ReadBlocked || capacity.Reason == "" { + t.Fatalf("capacity = %+v", capacity) + } + // Reachable but permission denied on target KB (readiness error only). + perm := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, WorkReadiness: errNoPermission}) + if perm.Liveness != LiveUp || perm.Readiness != ReadBlocked { + t.Fatalf("permission = %+v", perm) + } +} + +func TestHealthTrackerRemembersSuccessAndFailStreak(t *testing.T) { + tracker := newHealthTracker() + now := testNow() + first := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now})) + if first.LastSuccessAt == nil || !first.LastSuccessAt.Equal(now) || first.ConsecutiveFailures != 0 { + t.Fatalf("first = %+v", first) + } + later := now.Add(time.Minute) + bad := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: later, Reachability: context.DeadlineExceeded})) + if !bad.LastSuccessAt.Equal(now) || bad.ConsecutiveFailures != 1 { + t.Fatalf("after one failure = %+v", bad) + } + bad2 := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad.ObservedAt.Add(time.Minute), Reachability: context.DeadlineExceeded})) + if bad2.ConsecutiveFailures != 2 { + t.Fatalf("streak = %d", bad2.ConsecutiveFailures) + } + good := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad2.ObservedAt.Add(time.Minute)})) + if good.ConsecutiveFailures != 0 || !good.LastSuccessAt.Equal(bad2.ObservedAt.Add(time.Minute)) { + t.Fatalf("recovered = %+v", good) + } + // Unknown probes (not configured) must not reset or extend a real streak. + neutral := tracker.apply("postgres", classify("postgres", "PostgreSQL", false, true, ProbeOutcome{ObservedAt: good.ObservedAt.Add(time.Minute)})) + if neutral.ConsecutiveFailures != 0 { + t.Fatalf("neutral streak = %d", neutral.ConsecutiveFailures) + } +} + +func TestSanitizeProbeErrorTruncatesLongMessages(t *testing.T) { + long := errors.New(fmt.Sprintf("%0200d", 1)) + if got := sanitizeProbeError(long); len(got) > 160 { + t.Fatalf("len=%d", len(got)) + } +} diff --git a/internal/monitoring/metrics.go b/internal/monitoring/metrics.go new file mode 100644 index 0000000..3a297a1 --- /dev/null +++ b/internal/monitoring/metrics.go @@ -0,0 +1,171 @@ +package monitoring + +import ( + "context" + "fmt" + "math" + "sort" + "time" + + "knowledge-governance/internal/governance" +) + +type windowJobOutcome struct { + state string + durationSeconds float64 // <0 when duration is unknowable +} + +const ( + metricsShortWindow = time.Hour + metricsLongWindow = 24 * time.Hour + metricsWorkerLease = 90 * time.Second // workers heartbeat every <= lease; fresher than stale leases counts active + metricsSampleMargin = 2 * time.Second +) + +func computeJobWindowMetrics(outcomes []windowJobOutcome, windowSeconds int) governance.JobWindowMetrics { + metrics := governance.JobWindowMetrics{WindowSeconds: windowSeconds} + var durations []float64 + for _, outcome := range outcomes { + switch outcome.state { + case "completed": + metrics.JobsCompleted++ + case "failed": + metrics.JobsFailed++ + } + if outcome.durationSeconds >= 0 { + durations = append(durations, outcome.durationSeconds) + } + } + if finished := metrics.JobsCompleted + metrics.JobsFailed; finished > 0 { + success := math.Round(float64(metrics.JobsCompleted)/float64(finished)*10000) / 100 + failure := math.Round(float64(metrics.JobsFailed)/float64(finished)*10000) / 100 + metrics.SuccessRate = &success + metrics.FailureRate = &failure + } + if len(durations) > 0 { + sort.Float64s(durations) + var sum float64 + for _, value := range durations { + sum += value + } + average := math.Round(sum/float64(len(durations))*10) / 10 + p50 := math.Round(percentile(durations, 0.50)*10) / 10 + p95 := math.Round(percentile(durations, 0.95)*10) / 10 + metrics.DurationAvgSeconds = &average + metrics.DurationP50Seconds = &p50 + metrics.DurationP95Seconds = &p95 + } + return metrics +} + +// percentile uses nearest-rank so tiny samples never interpolate a fake value. +func percentile(sorted []float64, fraction float64) float64 { + rank := int(math.Ceil(fraction * float64(len(sorted)))) + if rank < 1 { + rank = 1 + } + if rank > len(sorted) { + rank = len(sorted) + } + return sorted[rank-1] +} + +func (provider *RealProvider) jobMetrics(ctx context.Context, window time.Duration) (governance.JobWindowMetrics, error) { + windowSeconds := int(window.Seconds()) + var metrics governance.JobWindowMetrics + // Pull the per-outcome detail (bounded) for exact-rate and percentile math. + // Duration is only measurable when the whole lifecycle lives inside the + // window; older-created jobs report -1 so percentiles never mix partials. + rows, err := provider.db.Query(ctx, ` +SELECT state, + CASE WHEN created_at > $2::timestamptz THEN COALESCE(EXTRACT(EPOCH FROM (updated_at - created_at)), -1) ELSE -1 END +FROM jobs +WHERE updated_at > $2::timestamptz AND updated_at <= $1::timestamptz + AND state IN ('completed','failed') +ORDER BY updated_at DESC +LIMIT 10000`, provider.now(), provider.now().Add(-window)) + if err != nil { + return metrics, fmt.Errorf("query job window outcomes: %w", err) + } + defer rows.Close() + var outcomes []windowJobOutcome + for rows.Next() { + var outcome windowJobOutcome + if err := rows.Scan(&outcome.state, &outcome.durationSeconds); err != nil { + return metrics, fmt.Errorf("scan job window outcome: %w", err) + } + outcomes = append(outcomes, outcome) + } + if err := rows.Err(); err != nil { + return metrics, fmt.Errorf("iterate job window outcomes: %w", err) + } + metrics = computeJobWindowMetrics(outcomes, windowSeconds) + + // Started inside the window uses created_at (any state). + var started int + if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE created_at > $2::timestamptz AND created_at <= $1::timestamptz`, provider.now(), provider.now().Add(-window)).Scan(&started); err != nil { + return metrics, fmt.Errorf("count jobs started: %w", err) + } + metrics.JobsStarted = started + return metrics, nil +} + +func (provider *RealProvider) queueGauges(ctx context.Context) (governance.QueueGauges, error) { + var gauges governance.QueueGauges + var oldestPendingSeconds *float64 + err := provider.db.QueryRow(ctx, ` +SELECT COUNT(*) FILTER (WHERE state='pending')::integer, + COUNT(*) FILTER (WHERE state='running')::integer, + COUNT(*) FILTER (WHERE state='retry')::integer, + COUNT(*) FILTER (WHERE state='failed')::integer, + CASE WHEN COUNT(*) FILTER (WHERE state IN ('pending','retry')) = 0 THEN NULL + ELSE GREATEST(EXTRACT(EPOCH FROM ($1::timestamptz - MIN(created_at) FILTER (WHERE state IN ('pending','retry')))), 0) END +FROM jobs`, provider.now()).Scan(&gauges.Pending, &gauges.Running, &gauges.Retry, &gauges.Failed, &oldestPendingSeconds) + if err != nil { + return gauges, fmt.Errorf("scan queue gauges: %w", err) + } + gauges.OldestPendingAgeSeconds = oldestPendingSeconds + return gauges, nil +} + +func (provider *RealProvider) activeWorkerCount(ctx context.Context) (int, error) { + // Count distinct fresh leases only; never infer from container counts. + var active int + err := provider.db.QueryRow(ctx, ` +SELECT COUNT(DISTINCT lease_owner)::integer +FROM jobs +WHERE state = 'running' AND lease_until > $1::timestamptz AND lease_owner IS NOT NULL`, + provider.now().Add(-metricsWorkerLease)).Scan(&active) + if err != nil { + return 0, fmt.Errorf("count active workers: %w", err) + } + return active, nil +} + +// Metrics implements the P9 run-metrics endpoint: frozen 1h/24h windows, +// lease-based active workers, and null-safe samples. +func (provider *RealProvider) Metrics(ctx context.Context) (governance.RunMetrics, error) { + now := provider.now().UTC() + metrics := governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease"} + short, err := provider.jobMetrics(ctx, metricsShortWindow) + if err != nil { + return metrics, err + } + longWindow, err := provider.jobMetrics(ctx, metricsLongWindow) + if err != nil { + return metrics, err + } + gauges, err := provider.queueGauges(ctx) + if err != nil { + return metrics, err + } + active, err := provider.activeWorkerCount(ctx) + if err != nil { + return metrics, err + } + metrics.ShortWindow = short + metrics.LongWindow = longWindow + metrics.Queue = gauges + metrics.ActiveWorkers = active + return metrics, nil +} diff --git a/internal/monitoring/metrics_test.go b/internal/monitoring/metrics_test.go new file mode 100644 index 0000000..3dedc9b --- /dev/null +++ b/internal/monitoring/metrics_test.go @@ -0,0 +1,71 @@ +package monitoring + +import ( + "encoding/json" + "testing" +) + +func TestComputeJobWindowMetricsRates(t *testing.T) { + // Mixed completed/failed inside the window: rates must be exact. + m := computeJobWindowMetrics([]windowJobOutcome{ + {state: "completed", durationSeconds: 10}, + {state: "completed", durationSeconds: 20}, + {state: "completed", durationSeconds: 30}, + {state: "failed", durationSeconds: 40}, + }, 60) + if m.WindowSeconds != 60 || m.JobsCompleted != 3 || m.JobsFailed != 1 { + t.Fatalf("counts = %+v", m) + } + if m.SuccessRate == nil || *m.SuccessRate != 75 { + t.Fatalf("success rate = %v, want 75", m.SuccessRate) + } + if m.FailureRate == nil || *m.FailureRate != 25 { + t.Fatalf("failure rate = %v, want 25", m.FailureRate) + } + // Durations: avg 25s, p50 20s (nearest-rank on 4 sorted samples 10/20/30/40). + if m.DurationAvgSeconds == nil || *m.DurationAvgSeconds != 25 { + t.Fatalf("avg = %v, want 25", m.DurationAvgSeconds) + } + if m.DurationP50Seconds == nil || *m.DurationP50Seconds != 20 { + t.Fatalf("p50 = %v, want 20", m.DurationP50Seconds) + } + if m.DurationP95Seconds == nil || *m.DurationP95Seconds != 40 { + t.Fatalf("p95 = %v, want 40", m.DurationP95Seconds) + } +} + +func TestComputeJobWindowMetricsNullsWhenNoSamples(t *testing.T) { + // Insufficient data must be null, never a fabricated 0%. + m := computeJobWindowMetrics([]windowJobOutcome{{state: "completed", durationSeconds: -1}}, 3600) + if m.JobsCompleted != 1 { + t.Fatalf("completed = %d", m.JobsCompleted) + } + if m.SuccessRate == nil || *m.SuccessRate != 100 { + t.Fatalf("success rate = %v, want 100 (1/1)", m.SuccessRate) + } + if m.DurationAvgSeconds != nil || m.DurationP50Seconds != nil || m.DurationP95Seconds != nil { + t.Fatalf("durations must be null without terminal-time samples: %+v", m) + } + empty := computeJobWindowMetrics(nil, 3600) + if empty.SuccessRate != nil || empty.FailureRate != nil || empty.JobsStarted != 0 { + t.Fatalf("empty window = %+v, want all-null rates", empty) + } + blob, err := json.Marshal(empty) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{`"success_rate":null`, `"failure_rate":null`, `"duration_avg_seconds":null`} { + if !contains(string(blob), key) { + t.Fatalf("metrics JSON missing %s: %s", key, blob) + } + } +} + +func contains(haystack, needle string) bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false +} diff --git a/internal/monitoring/p10_health_integration_test.go b/internal/monitoring/p10_health_integration_test.go new file mode 100644 index 0000000..a95636a --- /dev/null +++ b/internal/monitoring/p10_health_integration_test.go @@ -0,0 +1,116 @@ +package monitoring + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/platform/postgres" +) + +// TestRealServicesHealthMatrix freezes docs plan P10: liveness and readiness +// are per-service and independent, worker liveness rides on queue heartbeats, +// and missing configuration is an explicit state — never "unknown". +func TestRealServicesHealthMatrix(t *testing.T) { + databaseURL := isolatedMetricsURL(t) + pool, err := pgxpool.New(context.Background(), databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + ctx := context.Background() + if err := postgres.ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, time.September, 12, 12, 0, 0, 0, time.UTC) + provider, err := NewRealProvider(RealProviderConfig{ + DB: metricsPool{pool}, + Now: func() time.Time { return now }, + HTTPClient: &http.Client{Timeout: time.Second}, + WorkerStaleAfter: 2 * time.Minute, + }) + if err != nil { + t.Fatal(err) + } + + // Before any worker claims: queue/postgres healthy, worker aggregated down. + health := provider.Services(ctx) + byKey := map[string]string{} + for _, item := range health { + byKey[item.Key+"/l"] = item.Liveness + byKey[item.Key+"/r"] = item.Readiness + } + if byKey["postgres/l"] != LiveUp || byKey["postgres/r"] != ReadReady { + t.Fatalf("postgres health = %v/%v", byKey["postgres/l"], byKey["postgres/r"]) + } + if byKey["worker/l"] != LiveDown || byKey["worker/r"] != ReadBlocked { + t.Fatalf("worker health with no heartbeat = %v/%v", byKey["worker/l"], byKey["worker/r"]) + } + if byKey["weknora/r"] != ReadNotConfigured || byKey["mineru/r"] != ReadNotConfigured { + t.Fatalf("unconfigured deps must be explicit: %v %v", byKey["weknora/r"], byKey["mineru/r"]) + } + for _, key := range []string{"redis", "oss", "embedding", "llm"} { + if byKey[key+"/r"] != ReadNotApplicable || byKey[key+"/l"] != LiveUnknown { + t.Fatalf("not-deployed %s = %v/%v", key, byKey[key+"/l"], byKey[key+"/r"]) + } + } + // Readiness must be blocked ONLY by configured+deployed failures: worker. + ok, blocked := overallReadiness(health) + if ok || len(blocked) != 1 || blocked[0] != "worker" { + t.Fatalf("overall readiness ok=%v blocked=%v", ok, blocked) + } + + // A worker claim refreshes the heartbeat row and flips liveness independently. + store := postgres.NewStore(pool) + if _, err := store.Claim(ctx, "kg-test-worker", []string{"test.upload"}, now, time.Minute); err != nil { + t.Fatal(err) + } + health = provider.Services(ctx) + found := false + for _, item := range health { + if strings.HasPrefix(item.Key, "worker:") { + found = true + if item.Liveness != LiveUp || item.Readiness != ReadReady { + t.Fatalf("fresh worker health = %v/%v", item.Liveness, item.Readiness) + } + } + if item.Key == "worker" { + t.Fatalf("stale aggregate row must disappear once a worker is live") + } + } + if !found { + t.Fatal("live worker row missing after claim") + } + + // Heartbeat older than the stale window: liveness down, and the API-side + // process itself stays independent (liveness of the API is never blocked). + if _, err := pool.Exec(ctx, `UPDATE worker_heartbeats SET last_seen_at = $1`, now.Add(-10*time.Minute)); err != nil { + t.Fatal(err) + } + health = provider.Services(ctx) + staleDown := false + for _, item := range health { + if (item.Key == "worker" || strings.HasPrefix(item.Key, "worker:")) && item.Liveness == LiveDown { + staleDown = true + } + } + if !staleDown { + t.Fatal("stale heartbeat must mark worker down") + } + if ok, _ := overallReadiness(health); ok { + t.Fatal("stale worker must block readiness") + } + // Tracker history: postgres was seen successful and keeps no failures. + for _, item := range health { + if item.Key == "postgres" { + if item.LastSuccessAt == nil || item.ConsecutiveFailures != 0 { + t.Fatalf("postgres tracker history = %v failures=%d", item.LastSuccessAt, item.ConsecutiveFailures) + } + } + } +} diff --git a/internal/monitoring/p9_metrics_integration_test.go b/internal/monitoring/p9_metrics_integration_test.go new file mode 100644 index 0000000..77bcf54 --- /dev/null +++ b/internal/monitoring/p9_metrics_integration_test.go @@ -0,0 +1,190 @@ +package monitoring + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/platform/postgres" +) + +type metricsPool struct{ pool *pgxpool.Pool } + +func (d metricsPool) Ping(ctx context.Context) error { return d.pool.Ping(ctx) } + +func (d metricsPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + return d.pool.Query(ctx, sql, args...) +} +func (d metricsPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + return d.pool.QueryRow(ctx, sql, args...) +} +func (d metricsPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + return d.pool.Exec(ctx, sql, args...) +} + +func isolatedMetricsURL(t *testing.T) string { + t.Helper() + databaseURL := os.Getenv("TEST_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + parsed, err := url.Parse(databaseURL) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256([]byte(t.Name())) + schema := "kgtest_" + hex.EncodeToString(digest[:8]) + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanup, err := pgxpool.New(context.Background(), databaseURL) + if err == nil { + _, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE") + cleanup.Close() + } + }) + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// TestRealMetricsWindowsAndLeases freezes docs plan P9: rates and percentiles +// come from frozen 1h/24h windows, active workers count fresh distinct leases +// only, jobs created outside the window never fake a duration, and empty +// windows return nulls instead of 0. +func TestRealMetricsWindowsAndLeases(t *testing.T) { + ctx := context.Background() + pool, err := pgxpool.New(ctx, isolatedMetricsURL(t)) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := postgres.ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, time.September, 12, 8, 0, 0, 0, time.UTC) + seed := func(id, state string, createdAt, updatedAt time.Time, owner string, leaseUntil *time.Time, attempt int) { + var lease any + if leaseUntil != nil { + lease = *leaseUntil + } + var ownerArg any + if owner != "" { + ownerArg = owner + } + _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,lease_owner,lease_until,attempt,max_attempts,idempotency_key,created_at,updated_at) +VALUES($1::varchar,'test.upload','{}',$2,$4,$5,$6,$7,5,'k-'||$1::text,$3,$4)`, id, state, createdAt, updatedAt, ownerArg, lease, attempt) + if err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + + // Completed inside both windows with exact durations. + for i, d := range []time.Duration{10 * time.Second, 20 * time.Second, 30 * time.Second, 15 * time.Minute, 30 * time.Minute} { + updated := now.Add(-time.Duration(6-i) * time.Minute) + seed(fmt.Sprintf("job-c%d", i), "completed", updated.Add(-d), updated, "", nil, 1) + } + // Failed: two inside 1h, one inside 24h only, one outside 24h. + seed("job-f1", "failed", now.Add(-20*time.Minute), now.Add(-19*time.Minute), "w1", nil, 5) + seed("job-f2", "failed", now.Add(-10*time.Minute), now.Add(-9*time.Minute), "w1", nil, 5) + seed("job-f3", "failed", now.Add(-20*time.Hour), now.Add(-19*time.Hour), "w1", nil, 5) + seed("job-f4", "failed", now.Add(-40*time.Hour), now.Add(-39*time.Hour), "w1", nil, 5) + // Completed now but created 30h ago: duration unknowable, must not pollute percentiles. + seed("job-old", "completed", now.Add(-30*time.Hour), now.Add(-10*time.Minute), "", nil, 1) + // Fresh leases: two rows share wA, one wB, stale wC must not count. + seed("job-run-a1", "running", now.Add(-5*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1) + seed("job-run-a2", "running", now.Add(-4*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1) + seed("job-run-b", "running", now.Add(-3*time.Minute), now.Add(-time.Minute), "wB", ptrTime(now.Add(time.Minute)), 1) + seed("job-run-stale", "running", now.Add(-9*time.Hour), now.Add(-60*time.Minute), "wC", ptrTime(now.Add(-10*time.Minute)), 1) + // Gauges. + seed("job-pending", "pending", now.Add(-3*time.Hour), now.Add(-3*time.Hour), "", nil, 0) + seed("job-retry", "retry", now.Add(-30*time.Minute), now.Add(-25*time.Minute), "", nil, 1) + + provider, err := NewRealProvider(RealProviderConfig{ + DB: metricsPool{pool: pool}, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + metrics, err := provider.Metrics(ctx) + if err != nil { + t.Fatalf("metrics: %v", err) + } + if !metrics.ObservedAt.Equal(now) || metrics.WorkerSource != "pg_lease" { + t.Fatalf("observed/source = %s/%s", metrics.ObservedAt, metrics.WorkerSource) + } + if metrics.ActiveWorkers != 2 { + t.Fatalf("active workers = %d, want 2 (fresh distinct owners only)", metrics.ActiveWorkers) + } + if metrics.Queue.Pending != 1 || metrics.Queue.Running != 4 || metrics.Queue.Retry != 1 || metrics.Queue.Failed != 4 { + t.Fatalf("queue gauges = %+v", metrics.Queue) + } + if metrics.Queue.OldestPendingAgeSeconds == nil || int(*metrics.Queue.OldestPendingAgeSeconds) != 3*3600 { + t.Fatalf("oldest pending = %v, want 10800s", metrics.Queue.OldestPendingAgeSeconds) + } + + short := metrics.ShortWindow + // completed inside 1h: 5 seeded + job-old + job-c* (all updated within 1h except none) -> 6; failed 2. + if short.WindowSeconds != 3600 || short.JobsCompleted != 6 || short.JobsFailed != 2 { + t.Fatalf("short counts = %+v, want 6/2", short) + } + if short.SuccessRate == nil || *short.SuccessRate != 75 || short.FailureRate == nil || *short.FailureRate != 25 { + t.Fatalf("short rates = %v/%v, want 75/25", short.SuccessRate, short.FailureRate) + } + // Durations measurable inside 1h: 10,20,30,900,1800 (job-old excluded). + if short.DurationAvgSeconds == nil || *short.DurationAvgSeconds != 411.4 { + t.Fatalf("short avg = %v, want 411.4(含2条60s失败样本)", *short.DurationAvgSeconds) + } + if short.DurationP95Seconds == nil || *short.DurationP95Seconds != 1800 { + t.Fatalf("short p95 = %v, want 1800", *short.DurationP95Seconds) + } + + long := metrics.LongWindow + if long.JobsFailed != 3 { + t.Fatalf("long failures = %d, want 3 (job-f4 outside window)", long.JobsFailed) + } + // 8 measurable samples (5 completed + 2 failed@60s + job-f3 3600s): p50=60, p95=3600. + if long.DurationP50Seconds == nil || *long.DurationP50Seconds != 60 { + t.Fatalf("long p50 = %v, want 60", *long.DurationP50Seconds) + } + if long.DurationP95Seconds == nil || *long.DurationP95Seconds != 3600 { + t.Fatalf("long p95 = %v, want 3600", *long.DurationP95Seconds) + } + + // Empty-window honesty: a fresh schema with no rows yields null rates. + empty, err := NewRealProvider(RealProviderConfig{DB: metricsPool{pool: pool}, Now: func() time.Time { return now.Add(-100 * time.Hour) }}) + if err != nil { + t.Fatal(err) + } + idle, err := empty.Metrics(ctx) + if err != nil { + t.Fatalf("idle metrics: %v", err) + } + if idle.ShortWindow.SuccessRate != nil || idle.ShortWindow.DurationAvgSeconds != nil { + t.Fatalf("empty window must be null, got %+v", idle.ShortWindow) + } + if idle.Queue.OldestPendingAgeSeconds == nil { + t.Fatalf("oldest pending must remain observable: %+v", idle.Queue) + } +} + +func ptrTime(value time.Time) *time.Time { return &value } diff --git a/internal/monitoring/real.go b/internal/monitoring/real.go new file mode 100644 index 0000000..89e4387 --- /dev/null +++ b/internal/monitoring/real.go @@ -0,0 +1,320 @@ +package monitoring + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +type Database interface { + Ping(context.Context) error + Query(context.Context, string, ...any) (pgx.Rows, error) + QueryRow(context.Context, string, ...any) pgx.Row +} + +type RealProviderConfig struct { + DB Database + Now func() time.Time + HTTPClient *http.Client + WeKnoraBaseURL string + MinerUBaseURL string + // WeKnoraReadiness optionally exercises a real read against a routing + // knowledge base (nil = API key not configured; liveness-only). + WeKnoraReadiness func(ctx context.Context) error + // WorkerStaleAfter bounds how long a missing worker heartbeat still counts + // as alive (P10). + WorkerStaleAfter time.Duration +} + +type RealProvider struct { + db Database + now func() time.Time + httpClient *http.Client + weknoraBaseURL string + minerUBaseURL string + weknoraReadiness func(ctx context.Context) error + workerStaleAfter time.Duration + health *healthTracker +} + +func NewRealProvider(config RealProviderConfig) (*RealProvider, error) { + if config.DB == nil { + return nil, errors.New("monitor database is required") + } + if config.Now == nil { + config.Now = time.Now + } + if config.HTTPClient == nil { + config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + } + stale := config.WorkerStaleAfter + if stale <= 0 { + stale = time.Minute + } + return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil +} + +func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) { + now := provider.now().UTC() + overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}} + overview.Host = collectHostMetrics() + overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now)) + overview.Health = provider.Services(ctx) + overview.Services = append(overview.Services, + provider.httpHealth(ctx, "weknora", "WeKnora API", provider.weknoraBaseURL, now), + governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理，尚无独立探测端点", ObservedAt: now}, + governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理，尚无独立探测端点", ObservedAt: now}, + provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now), + governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署；任务由PostgreSQL持久队列承载", ObservedAt: now}, + governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now}, + ) + rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id, + CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END, + COUNT(*)::integer, + COUNT(*) FILTER (WHERE p.processing_state='completed')::integer, + COUNT(*) FILTER (WHERE p.processing_state='pending')::integer, + COUNT(*) FILTER (WHERE p.processing_state='processing')::integer, + COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer, + COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer, + COUNT(*) FILTER (WHERE p.target_scope='internal')::integer, + COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer, + COUNT(*) FILTER (WHERE p.target_scope='external')::integer, + COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer +FROM test_publications p +GROUP BY p.test_kb_id +ORDER BY p.test_kb_id`) + if err != nil { + return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.KnowledgeBaseProgress + if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil { + return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err) + } + switch item.Visibility { + case "internal": + item.Name = "对内知识库" + case "external": + item.Name = "对外知识库" + default: + item.Name = "测试知识库" + } + overview.KnowledgeBases = append(overview.KnowledgeBases, item) + } + if err := rows.Err(); err != nil { + return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err) + } + return overview, nil +} + +func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) { + if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) { + return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query") + } + page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false} + if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil { + return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err) + } + rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope, + CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END, + GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer), + publication.ingest_state,publication.last_error +FROM test_publications publication +JOIN source_item_versions version ON version.id=publication.source_item_version_id +JOIN source_items item ON item.id=version.source_item_id +WHERE publication.test_kb_id=$1 + AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2) +ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END, + publication.updated_at DESC,publication.id +LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC()) + if err != nil { + return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.KnowledgeMonitorFile + var ingestState string + if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil { + return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err) + } + item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode) + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err) + } + return page, nil +} + +func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService { + started := time.Now() + state, detail := "healthy", "可连接并可查询" + if err := provider.db.Ping(ctx); err != nil { + state, detail = "unhealthy", "连接或查询失败" + } + return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now} +} + +func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService { + started := time.Now() + var pending, running, failed int + err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed) + state, detail := "healthy", fmt.Sprintf("可接单；等待%d，运行%d，失败%d", pending, running, failed) + if err != nil { + state, detail = "unhealthy", "队列不可查询" + } + return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now} +} + +func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService { + if baseURL == "" { + return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now} + } + parsed, err := url.Parse(baseURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now} + } + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" + parsed.RawQuery, parsed.Fragment = "", "" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now} + } + started := time.Now() + response, err := provider.httpClient.Do(request) + latency := int(time.Since(started).Milliseconds()) + if err != nil { + return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now} + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + if response.StatusCode < 200 || response.StatusCode >= 300 { + return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now} + } + return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now} +} + +func collectHostMetrics() governance.MonitorHost { + host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false} + first, err := os.ReadFile("/proc/stat") + if err == nil { + idle1, total1, parseErr := parseCPUStat(string(first)) + if parseErr == nil { + time.Sleep(100 * time.Millisecond) + if second, readErr := os.ReadFile("/proc/stat"); readErr == nil { + idle2, total2, secondErr := parseCPUStat(string(second)) + if secondErr == nil { + host.CPUPercent = cpuPercent(idle1, total1, idle2, total2) + } + } + } + } + if raw, err := os.ReadFile("/proc/meminfo"); err == nil { + host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw)) + } + var stats syscall.Statfs_t + if syscall.Statfs("/", &stats) == nil { + host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize) + host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize) + } + if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 { + host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU；未读取利用率", len(entries)) + } + return host +} + +func parseCPUStat(raw string) (idle, total uint64, err error) { + line := strings.SplitN(raw, "\n", 2)[0] + fields := strings.Fields(line) + if len(fields) < 5 || fields[0] != "cpu" { + return 0, 0, errors.New("invalid cpu stat") + } + values := make([]uint64, 0, len(fields)-1) + for _, field := range fields[1:] { + value, parseErr := strconv.ParseUint(field, 10, 64) + if parseErr != nil { + return 0, 0, errors.New("invalid cpu counter") + } + values = append(values, value) + total += value + } + idle = values[3] + if len(values) > 4 { + idle += values[4] + } + return idle, total, nil +} + +func parseMemoryInfo(raw string) (used, total int64, err error) { + values := map[string]int64{} + for _, line := range strings.Split(raw, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + key := strings.TrimSuffix(fields[0], ":") + if key != "MemTotal" && key != "MemAvailable" { + continue + } + value, parseErr := strconv.ParseInt(fields[1], 10, 64) + if parseErr != nil || value < 0 { + return 0, 0, errors.New("invalid memory counter") + } + values[key] = value * 1024 + } + total, totalOK := values["MemTotal"] + available, availableOK := values["MemAvailable"] + if !totalOK || !availableOK || available > total { + return 0, 0, errors.New("incomplete memory info") + } + return total - available, total, nil +} + +func cpuPercent(idle1, total1, idle2, total2 uint64) float64 { + if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 { + return 0 + } + return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1) +} + +func validMonitorStatus(status string) bool { + switch status { + case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled": + return true + default: + return false + } +} + +func monitorStageDetail(ingestState, processingState, errorCode string) string { + if errorCode != "" { + return "处理失败，可查看错误码后重试" + } + switch { + case ingestState == "pending": + return "等待确认或上传" + case ingestState == "uploading": + return "正在上传测试知识库" + case processingState == "processing": + return "WeKnora正在解析、切片或向量化" + case processingState == "finalizing": + return "基础解析完成，索引或增强处理中" + case processingState == "completed": + return "解析完成，等待或已完成验收" + default: + return "等待远端状态更新" + } +} diff --git a/internal/monitoring/real_test.go b/internal/monitoring/real_test.go new file mode 100644 index 0000000..c405b76 --- /dev/null +++ b/internal/monitoring/real_test.go @@ -0,0 +1,24 @@ +package monitoring + +import "testing" + +func TestParseCPUStatAndMemoryInfo(t *testing.T) { + idle, total, err := parseCPUStat("cpu 100 20 30 400 50 6 7 8 0 0\n") + if err != nil || idle != 450 || total != 621 { + t.Fatalf("idle=%d total=%d err=%v", idle, total, err) + } + used, memoryTotal, err := parseMemoryInfo("MemTotal: 1000 kB\nMemAvailable: 250 kB\n") + if err != nil || used != 750*1024 || memoryTotal != 1000*1024 { + t.Fatalf("used=%d total=%d err=%v", used, memoryTotal, err) + } +} + +func TestCPUPercentUsesTwoMonotonicSamples(t *testing.T) { + value := cpuPercent(450, 621, 500, 721) + if value != 50 { + t.Fatalf("cpu percent=%v, want 50", value) + } + if cpuPercent(500, 721, 450, 621) != 0 { + t.Fatal("non-monotonic CPU counters must fail closed to zero") + } +} diff --git a/internal/monitoring/simulation.go b/internal/monitoring/simulation.go new file mode 100644 index 0000000..5ff34b7 --- /dev/null +++ b/internal/monitoring/simulation.go @@ -0,0 +1,87 @@ +package monitoring + +import ( + "context" + "errors" + "time" + + "knowledge-governance/internal/governance" +) + +type SimulationProvider struct{ now func() time.Time } + +func NewSimulationProvider(now func() time.Time) *SimulationProvider { + if now == nil { + now = time.Now + } + return &SimulationProvider{now: now} +} + +func (provider *SimulationProvider) Overview(context.Context) (governance.MonitorOverview, error) { + now := provider.now().UTC() + return governance.MonitorOverview{ + Mode: "simulation", Simulated: true, ObservedAt: now, + Host: governance.MonitorHost{CPUPercent: 42, MemoryUsedBytes: 19971597926, MemoryTotalBytes: 34359738368, DiskUsedBytes: 128849018880, DiskTotalBytes: 536870912000, GPUState: "unknown", GPUDetail: "未接入真实GPU监控", Simulated: true}, + Services: []governance.MonitorService{ + {Key: "weknora", Name: "WeKnora API", State: "healthy", Detail: "模拟连接正常", LatencyMS: 86, ObservedAt: now, Simulated: true}, + {Key: "embedding", Name: "Embedding模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 142, ObservedAt: now, Simulated: true}, + {Key: "llm", Name: "LLM模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 318, ObservedAt: now, Simulated: true}, + {Key: "mineru", Name: "MinerU", State: "healthy", Detail: "模拟处理中2，队列5", LatencyMS: 24, ObservedAt: now, Simulated: true}, + {Key: "postgres", Name: "PostgreSQL", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true}, + {Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true}, + {Key: "oss", Name: "OSS", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true}, + }, + KnowledgeBases: []governance.KnowledgeBaseProgress{{ID: "simulation-company-support", Name: "公司级技术支持知识库", Visibility: "mixed", Imported: 5750, Completed: 4530, Pending: 450, Processing: 50, Finalizing: 714, Failed: 6, InternalImported: 4200, InternalCompleted: 3300, ExternalImported: 1550, ExternalCompleted: 1230}}, + }, nil +} + +var simulationFiles = []governance.KnowledgeMonitorFile{ + {Name: "[模拟] CR5用户手册.pdf", Directory: "02｜产品资料/CR系列", Visibility: "internal", Status: "completed", StageDetail: "解析、切片和增强任务均完成", ElapsedSecond: 0}, + {Name: "[模拟] MG400部署说明.pdf", Directory: "03｜机器人部署与使用", Visibility: "internal", Status: "pending", StageDetail: "等待进入解析队列", ElapsedSecond: 420}, + {Name: "[模拟] 产品参数总表.xlsx", Directory: "02｜产品资料", Visibility: "external", Status: "finalizing", StageDetail: "基础检索可用，Wiki与图谱仍在生成", ElapsedSecond: 2100}, + {Name: "[模拟] 二次开发接口说明.docx", Directory: "05｜二次开发", Visibility: "internal", Status: "processing", StageDetail: "正在切片和向量化", ElapsedSecond: 480}, + {Name: "[模拟] CRA-IP68产品介绍.pdf", Directory: "06｜应用案例与解决方案", Visibility: "external", Status: "failed", StageDetail: "MinerU解析失败", ElapsedSecond: 720, ErrorCode: "mineru_parse_failed"}, +} + +func (provider *SimulationProvider) ListFiles(_ context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) { + if knowledgeBaseID != "simulation-company-support" || limit < 1 || limit > 100 || offset < 0 { + return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query") + } + allowed := map[string]bool{"": true, "completed": true, "pending": true, "processing": true, "finalizing": true, "failed": true, "cancelled": true} + if !allowed[status] { + return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor status") + } + filtered := make([]governance.KnowledgeMonitorFile, 0) + for _, item := range simulationFiles { + if status == "" || item.Status == status { + filtered = append(filtered, item) + } + } + page := governance.KnowledgeMonitorFilePage{Total: len(filtered), Limit: limit, Offset: offset, SampleOnly: true, Items: []governance.KnowledgeMonitorFile{}} + if offset < len(filtered) { + end := offset + limit + if end > len(filtered) { + end = len(filtered) + } + page.Items = append(page.Items, filtered[offset:end]...) + } + return page, nil +} + +// Metrics declares that no real run metrics exist in simulation mode: every +// sample-dependent number stays null instead of pretending to be zero (P9). +func (provider *SimulationProvider) Metrics(context.Context) (governance.RunMetrics, error) { + now := provider.now().UTC() + return governance.RunMetrics{ + ObservedAt: now, + WorkerSource: "simulation", + ShortWindow: governance.JobWindowMetrics{WindowSeconds: int((1 * time.Hour).Seconds())}, + LongWindow: governance.JobWindowMetrics{WindowSeconds: int((24 * time.Hour).Seconds())}, + }, nil +} + +// Services declares that a simulation provider owns no real dependencies: every +// readiness judgement belongs to the real provider. +func (provider *SimulationProvider) Services(_ context.Context) []governance.ServiceHealth { + return []governance.ServiceHealth{} +} diff --git a/internal/monitoring/simulation_test.go b/internal/monitoring/simulation_test.go new file mode 100644 index 0000000..153a973 --- /dev/null +++ b/internal/monitoring/simulation_test.go @@ -0,0 +1,52 @@ +package monitoring + +import ( + "context" + "testing" + "time" +) + +func TestSimulationProviderLabelsEveryValueAndReconcilesKnowledgeCounts(t *testing.T) { + now := time.Date(2026, 9, 8, 12, 0, 0, 0, time.UTC) + provider := NewSimulationProvider(func() time.Time { return now }) + overview, err := provider.Overview(context.Background()) + if err != nil { + t.Fatal(err) + } + if overview.Mode != "simulation" || !overview.Simulated || overview.ObservedAt != now || len(overview.Services) < 4 || len(overview.KnowledgeBases) != 1 { + t.Fatalf("overview=%#v", overview) + } + if overview.Host.DiskTotalBytes <= 0 || overview.Host.DiskUsedBytes <= 0 || overview.Host.GPUState != "unknown" || overview.Host.GPUDetail != "未接入真实GPU监控" { + t.Fatalf("host resources=%#v", overview.Host) + } + requiredServices := map[string]bool{"weknora": false, "mineru": false, "postgres": false, "redis": false, "oss": false, "embedding": false, "llm": false} + for _, service := range overview.Services { + if _, ok := requiredServices[service.Key]; ok { + requiredServices[service.Key] = true + } + } + for key, present := range requiredServices { + if !present { + t.Errorf("missing service status %q", key) + } + } + kb := overview.KnowledgeBases[0] + if kb.Imported != 5750 || kb.Completed != 4530 || kb.Pending != 450 || kb.Finalizing != 714 || kb.Processing != 50 || kb.Failed != 6 { + t.Fatalf("kb=%#v", kb) + } + if kb.Completed+kb.Pending+kb.Finalizing+kb.Processing+kb.Failed+kb.Cancelled != kb.Imported { + t.Fatalf("status counts do not reconcile: %#v", kb) + } + for _, service := range overview.Services { + if !service.Simulated { + t.Fatalf("unlabelled service=%#v", service) + } + } + files, err := provider.ListFiles(context.Background(), kb.ID, "failed", 20, 0) + if err != nil { + t.Fatal(err) + } + if files.Total != 1 || len(files.Items) != 1 || files.Items[0].Status != "failed" || files.Items[0].Name[:len("[模拟]")] != "[模拟]" || !files.SampleOnly { + t.Fatalf("files=%#v", files) + } +} diff --git a/internal/platform/postgres/c4_concurrent_operation_integration_test.go b/internal/platform/postgres/c4_concurrent_operation_integration_test.go new file mode 100644 index 0000000..11db74c --- /dev/null +++ b/internal/platform/postgres/c4_concurrent_operation_integration_test.go @@ -0,0 +1,104 @@ +package postgres + +import ( + "context" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestC4ConcurrentOperationCreatesOnceAndSurvivesPoolRestart(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp','pp','C4 scan',true)`); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 11, 13, 0, 0, 0, time.UTC) + store := NewStore(pool) + + // 10 concurrent identical requests with the same Idempotency-Key: exactly one must create. + var wg sync.WaitGroup + type outcome struct { + scanID string + replayed bool + err error + } + results := make([]outcome, 10) + for i := range results { + wg.Add(1) + go func(index int) { + defer wg.Done() + scan, replayed, err := store.TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now, "op-c4-scan-1") + results[index] = outcome{scanID: scan.ID, replayed: replayed, err: err} + }(i) + } + wg.Wait() + + var created int + var firstID string + for _, result := range results { + if result.err != nil { + t.Fatalf("concurrent operation error = %v", result.err) + } + if !result.replayed { + created++ + firstID = result.scanID + } + if result.scanID != firstID && firstID != "" { + t.Fatalf("replay returned different scan id: %s vs %s", result.scanID, firstID) + } + } + if created != 1 { + t.Fatalf("created=%d, want exactly 1", created) + } + var scans, jobs, operations int + if err := pool.QueryRow(ctx, `SELECT (SELECT COUNT(*) FROM source_scans),(SELECT COUNT(*) FROM jobs WHERE type='source.scan.pp'),(SELECT COUNT(*) FROM operation_requests WHERE operation_id='op-c4-scan-1')`).Scan(&scans, &jobs, &operations); err != nil { + t.Fatal(err) + } + if scans != 1 || jobs != 1 || operations != 1 { + t.Fatalf("scans=%d jobs=%d operations=%d", scans, jobs, operations) + } + + // Restart: close the pool and reconnect, then replay with the same key. + pool.Close() + restarted, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + scan, replayed, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp", now.Add(time.Hour), "op-c4-scan-1") + if err != nil || !replayed || scan.ID != firstID { + t.Fatalf("post-restart replay scan=%#v replayed=%v err=%v want id=%s", scan, replayed, err, firstID) + } + + // Same key, different payload (different connection): must conflict. + if _, err := restarted.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-c4-pp2','pp','C4 scan 2',true)`); err != nil { + t.Fatal(err) + } + if _, _, err := NewStore(restarted).TriggerPPSourceScanWithOperation(ctx, "conn-c4-pp2", now, "op-c4-scan-1"); err == nil { + t.Fatal("same key different payload should conflict") + } else if !errors.Is(err, ErrOperationRequestConflict) { + t.Fatalf("conflict error=%v", err) + } + var freshOperations int + if err := restarted.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests`).Scan(&freshOperations); err != nil { + t.Fatal(err) + } + if freshOperations != 1 { + t.Fatalf("operations=%d want 1", freshOperations) + } +} diff --git a/internal/platform/postgres/catalog_change_sets.go b/internal/platform/postgres/catalog_change_sets.go new file mode 100644 index 0000000..a84187c --- /dev/null +++ b/internal/platform/postgres/catalog_change_sets.go @@ -0,0 +1,71 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ErrCatalogChangeSetNotFound = errors.New("catalog change set not found") + +func (s *Store) GetLatestCatalogChangeSet(ctx context.Context, filter governance.CatalogChangeSetFilter) (governance.CatalogChangeSetDetail, error) { + if !validRouteID(filter.ConnectionID) || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 || !validCatalogChangeAction(filter.Action) { + return governance.CatalogChangeSetDetail{}, errors.New("invalid catalog change set filter") + } + detail := governance.CatalogChangeSetDetail{Items: []governance.CatalogChangeItem{}, Limit: filter.Limit, Offset: filter.Offset} + err := s.db.QueryRow(ctx, `SELECT change_set.id,change_set.source_connection_id,connection.name,change_set.state, + change_set.add_count,change_set.update_count,change_set.move_count,change_set.move_scope_count, + change_set.delete_count,change_set.no_change_count,change_set.created_at +FROM source_catalog_change_sets change_set +JOIN source_connections connection ON connection.id=change_set.source_connection_id +WHERE change_set.source_connection_id=$1 +ORDER BY change_set.created_at DESC,change_set.id DESC +LIMIT 1`, filter.ConnectionID).Scan( + &detail.ID, &detail.ConnectionID, &detail.SourceName, &detail.State, + &detail.AddCount, &detail.UpdateCount, &detail.MoveCount, &detail.MoveScopeCount, + &detail.DeleteCount, &detail.NoChangeCount, &detail.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return governance.CatalogChangeSetDetail{}, ErrCatalogChangeSetNotFound + } + if err != nil { + return governance.CatalogChangeSetDetail{}, fmt.Errorf("get latest catalog change set: %w", err) + } + if err := s.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND ($2='' OR action=$2)`, detail.ID, filter.Action).Scan(&detail.Total); err != nil { + return governance.CatalogChangeSetDetail{}, fmt.Errorf("count catalog change items: %w", err) + } + rows, err := s.db.Query(ctx, `SELECT action,title,old_path,new_path,old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,COALESCE(old_content_sha256::text,''),COALESCE(new_content_sha256::text,''),reason_code,item_seq,COALESCE(old_scope,''),COALESCE(new_scope,''),COALESCE(old_version_id,''),COALESCE(new_version_id,'') +FROM source_catalog_change_items +WHERE change_set_id=$1 AND ($2='' OR action=$2) +ORDER BY CASE action WHEN 'delete' THEN 1 WHEN 'update' THEN 2 WHEN 'move_scope' THEN 3 WHEN 'add' THEN 4 ELSE 5 END, + lower(title),source_locator_hash +LIMIT $3 OFFSET $4`, detail.ID, filter.Action, filter.Limit, filter.Offset) + if err != nil { + return governance.CatalogChangeSetDetail{}, fmt.Errorf("list catalog change items: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.CatalogChangeItem + if err := rows.Scan(&item.Action, &item.Title, &item.OldPath, &item.NewPath, &item.OldSizeBytes, &item.NewSizeBytes, &item.OldModified, &item.NewModified, &item.OldContentSHA256, &item.NewContentSHA256, &item.ReasonCode, &item.Seq, &item.OldScope, &item.NewScope, &item.OldVersionID, &item.NewVersionID); err != nil { + return governance.CatalogChangeSetDetail{}, fmt.Errorf("scan catalog change item: %w", err) + } + detail.Items = append(detail.Items, item) + } + if err := rows.Err(); err != nil { + return governance.CatalogChangeSetDetail{}, fmt.Errorf("iterate catalog change items: %w", err) + } + return detail, nil +} + +func validCatalogChangeAction(action string) bool { + switch action { + case "", "add", "update", "move", "move_scope", "delete", "no_change": + return true + default: + return false + } +} diff --git a/internal/platform/postgres/catalog_change_sets_integration_test.go b/internal/platform/postgres/catalog_change_sets_integration_test.go new file mode 100644 index 0000000..056260c --- /dev/null +++ b/internal/platform/postgres/catalog_change_sets_integration_test.go @@ -0,0 +1,47 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestCatalogChangeSetPersistsSummaryAndFilteredPreview(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-change','alist','国内工业',true); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-old','conn-change',repeat('1',64),'国内工业',2,1,1,'2026-09-10T12:00:00Z'),('snapshot-new','conn-change',repeat('2',64),'国内工业',3,1,2,'2026-09-10T12:00:00Z'); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,no_change_count,created_at) +VALUES('cs-change','conn-change','snapshot-old','snapshot-new','planned',1,1,0,1,5,'2026-09-10T12:00:00Z'); +INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,old_size_bytes,new_size_bytes,created_at) +VALUES('cs-change',repeat('a',64),'update','新版手册.pdf','手册.pdf','新版手册.pdf',10,12,'2026-09-10T12:00:00Z'), + ('cs-change',repeat('b',64),'delete','旧手册.pdf','旧手册.pdf','',20,NULL,'2026-09-10T12:00:00Z');`) + if err != nil { + t.Fatal(err) + } + store := NewStore(pool) + detail, err := store.GetLatestCatalogChangeSet(ctx, governance.CatalogChangeSetFilter{ConnectionID: "conn-change", Action: "delete", Limit: 20}) + if err != nil { + t.Fatal(err) + } + if detail.SourceName != "国内工业" || detail.AddCount != 1 || detail.UpdateCount != 1 || detail.DeleteCount != 1 || detail.Total != 1 || len(detail.Items) != 1 || detail.Items[0].Title != "旧手册.pdf" || detail.Items[0].Action != "delete" { + t.Fatalf("detail=%#v", detail) + } +} diff --git a/internal/platform/postgres/catalog_refresh_progress.go b/internal/platform/postgres/catalog_refresh_progress.go new file mode 100644 index 0000000..c1d21cf --- /dev/null +++ b/internal/platform/postgres/catalog_refresh_progress.go @@ -0,0 +1,27 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh" +) + +func (s *Store) UpdateCatalogRefreshProgress(ctx context.Context, jobID, owner string, progress catalogrefresh.Progress, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE jobs SET payload=payload||jsonb_build_object( + 'stage',$3::text,'progress',$4::integer,'message',$5::text,'processed',$6::integer,'total',$7::integer, + 'current_directory',NULLIF($9,''),'scanned_directories',$10::integer,'discovered_directories',$11::integer, + 'scanned_files',$12::integer,'estimated_total_files',$13::integer,'total_is_estimate',$14::boolean, + 'source_name',NULLIF($15,'')),updated_at=$8 WHERE id=$1 AND lease_owner=$2 AND state='running'`, + jobID, owner, progress.Stage, progress.Progress, progress.Message, progress.Processed, progress.Total, now, + progress.CurrentDirectory, progress.ScannedDirectories, progress.DiscoveredDirectories, + progress.ScannedFiles, progress.EstimatedTotalFiles, progress.TotalIsEstimate, progress.SourceName) + if err != nil { + return fmt.Errorf("update catalog refresh progress: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrLeaseLost + } + return nil +} diff --git a/internal/platform/postgres/catalog_routing.go b/internal/platform/postgres/catalog_routing.go new file mode 100644 index 0000000..b96fbb5 --- /dev/null +++ b/internal/platform/postgres/catalog_routing.go @@ -0,0 +1,243 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "strings" + "unicode" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +func (s *Store) RouteCatalogFiles(ctx context.Context, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) { + if err := validateCatalogRouteRequest(request); err != nil { + return governance.CatalogRouteResult{}, err + } + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return governance.CatalogRouteResult{}, errors.New("catalog route transactions are unavailable") + } + tx, err := beginner.Begin(ctx) + if err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("begin catalog route: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + result, err := routeCatalogFilesTx(ctx, tx, request) + if err != nil { + return governance.CatalogRouteResult{}, err + } + if err := tx.Commit(ctx); err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("commit catalog route: %w", err) + } + return result, nil +} + +func validateCatalogRouteRequest(request governance.CatalogRouteRequest) error { + targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs + if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID} + } + if !validRouteID(request.ConnectionID) || len(request.NodeIDs) == 0 || len(request.NodeIDs) > 100 || + !validCatalogPolicy(request.Visibility, targetKnowledgeBaseIDs) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 || + strings.TrimSpace(request.Reason) == "" || len(request.Reason) > 500 || request.Now.IsZero() { + return errors.New("invalid catalog route request") + } + seen := map[string]bool{} + for _, id := range request.NodeIDs { + if !validRouteID(id) || seen[id] { + return errors.New("invalid catalog route nodes") + } + seen[id] = true + } + return nil +} + +func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) { + targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs + if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID} + } + nodeIDs := append([]string(nil), request.NodeIDs...) + selectedRows, err := tx.Query(ctx, `SELECT node.node_id,node.node_type,node.relative_path +FROM source_catalog_heads head +JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id +WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[]) +ORDER BY node.node_id +FOR UPDATE OF node`, request.ConnectionID, nodeIDs) + if err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route nodes: %w", err) + } + selectedCount := 0 + directoryPaths := make([]string, 0) + for selectedRows.Next() { + var nodeID, nodeType, directoryPath string + if err := selectedRows.Scan(&nodeID, &nodeType, &directoryPath); err != nil { + selectedRows.Close() + return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route node: %w", err) + } + selectedCount++ + if nodeType == "folder" { + directoryPaths = append(directoryPaths, directoryPath) + } + } + if err := selectedRows.Err(); err != nil { + selectedRows.Close() + return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route nodes: %w", err) + } + selectedRows.Close() + if selectedCount != len(nodeIDs) { + return governance.CatalogRouteResult{}, errors.New("catalog route contains missing nodes") + } + for _, directoryPath := range directoryPaths { + if err := setCatalogDirectoryPolicy(ctx, tx, request, directoryPath); err != nil { + return governance.CatalogRouteResult{}, err + } + } + + rows, err := tx.Query(ctx, `WITH RECURSIVE selected AS ( + SELECT node.snapshot_id,node.node_id,node.node_type + FROM source_catalog_heads head + JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id + WHERE head.source_connection_id=$1 AND node.node_id=ANY($2::text[]) +), descendants AS ( + SELECT snapshot_id,node_id,node_type FROM selected + UNION + SELECT child.snapshot_id,child.node_id,child.node_type + FROM descendants parent + JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id + WHERE parent.node_type='folder' +) +SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected +FROM descendants +JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id +WHERE node.node_type='file' +ORDER BY node.node_id`, request.ConnectionID, nodeIDs) + if err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err) + } + type selectedFile struct { + nodeID string + sourceItemID *string + direct bool + } + files := make([]selectedFile, 0, len(nodeIDs)) + for rows.Next() { + var file selectedFile + if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil { + rows.Close() + return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err) + } + files = append(files, file) + } + if err := rows.Err(); err != nil { + rows.Close() + return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err) + } + rows.Close() + + result := governance.CatalogRouteResult{SelectedFiles: len(files)} + routedItems := map[string]bool{} + for _, file := range files { + if file.sourceItemID == nil { + result.UnboundFiles++ + continue + } + itemID := *file.sourceItemID + if routedItems[itemID] { + continue + } + routedItems[itemID] = true + if file.direct { + if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) +VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err) + } + eventID, err := newOpaqueID("se-") + if err != nil { + return governance.CatalogRouteResult{}, err + } + if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at) +VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { + return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err) + } + } + // S7: classify only. Annotations never create publications, retire targets, or enqueue jobs. + var currentVersionID *string + var contentState, spoolRef string + err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item +LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision +LEFT JOIN content_objects content ON content.id=version.content_object_id +WHERE item.id=$1`, itemID).Scan(¤tVersionID, &contentState, &spoolRef) + switch { + case errors.Is(err, pgx.ErrNoRows): + result.UnavailableFiles++ + case err != nil: + return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err) + case contentState == string(governance.ContentVerified) && spoolRef != "": + result.RoutableFiles++ + default: + result.HashUnknownFiles++ + } + } + return result, nil +} + +func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error { + if _, err := tx.Exec(ctx, `INSERT INTO directory_scope_rules(source_connection_id,directory_path,visibility,updated_by,reason,created_at,updated_at) +VALUES($1,$2,$3,$4,$5,$6,$6) +ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, + request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { + return fmt.Errorf("set routed directory scope: %w", err) + } + eventID, err := newOpaqueID("se-") + if err != nil { + return err + } + if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_connection_id,directory_path,action,decided_visibility,operator_ref,reason,created_at) +VALUES($1,'directory',$2,$3,'set',$4,$5,$6,$7)`, eventID, request.ConnectionID, directoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { + return fmt.Errorf("audit routed directory scope: %w", err) + } + return nil +} + +func validCatalogPolicy(policy governance.Visibility, targetKnowledgeBaseIDs []string) bool { + expectedTargets := 1 + switch policy { + case governance.VisibilityInternal, governance.VisibilityExternal: + case governance.VisibilityBoth: + expectedTargets = 2 + case governance.VisibilityBlocked: + expectedTargets = 0 + default: + return false + } + if len(targetKnowledgeBaseIDs) != expectedTargets { + return false + } + seen := make(map[string]bool, len(targetKnowledgeBaseIDs)) + for _, target := range targetKnowledgeBaseIDs { + if !validRouteID(target) || seen[target] { + return false + } + seen[target] = true + } + return true +} + +func validRouteID(value string) bool { + if value == "" || len(value) > 128 { + return false + } + for _, character := range value { + if unicode.IsLetter(character) || unicode.IsDigit(character) || character == '-' || character == '_' { + continue + } + return false + } + return true +} diff --git a/internal/platform/postgres/catalog_routing_integration_test.go b/internal/platform/postgres/catalog_routing_integration_test.go new file mode 100644 index 0000000..0267b59 --- /dev/null +++ b/internal/platform/postgres/catalog_routing_integration_test.go @@ -0,0 +1,158 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" +) + +// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit). +// It never creates or mutates test_publications and never enqueues jobs. +func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-route','pp','Route PP',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-route',repeat('c',64),10,'verified/file.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-route','conn-route','external-route',repeat('b',64),'工业产品/CRA系列','手册.pdf','document','v1','external','pp_public_tag'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-route','item-route','v1','content-route','verified'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-route','conn-route',repeat('a',64),'Route PP',4,2,2,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) +VALUES + ('snapshot-route','root-route',NULL,'folder','Route PP','',repeat('1',64),NULL,NULL,2,2,20), + ('snapshot-route','folder-route','root-route','folder','工业产品','工业产品',repeat('4',64),NULL,NULL,1,1,10), + ('snapshot-route','file-route','folder-route','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-route',10,0,1,10), + ('snapshot-route','file-unmapped','root-route','file','未接入.pdf','未接入.pdf',repeat('3',64),NULL,10,0,1,10); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-route','snapshot-route',NOW());`) + if err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC) + result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now}) + if err != nil { + t.Fatal(err) + } + if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 { + t.Fatalf("result=%#v", result) + } + // Desired state saved, recursion intact. + var directoryPolicy string + if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil { + t.Fatal(err) + } + if directoryPolicy != "internal" { + t.Fatalf("directory policy=%q", directoryPolicy) + } + // Zero side effects: no publications, no jobs. + var publicationCount, jobCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil { + t.Fatal(err) + } + if publicationCount != 0 || jobCount != 0 { + t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount) + } + + // Re-annotating the same directory is idempotent for rules and still side-effect free. + again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)}) + if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 { + t.Fatalf("again=%#v err=%v", again, err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 { + t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err) + } + var ruleCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 { + t.Fatalf("rule rows=%d err=%v", ruleCount, err) + } + + // Switching policy updates desired state without retiring or removing anything. + both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)}) + if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 { + t.Fatalf("both=%#v err=%v", both, err) + } + if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" { + t.Fatalf("policy after both=%q err=%v", directoryPolicy, err) + } + + // Blocking saves the desired policy only; existing publications (if any) stay untouched. + if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at) +VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil { + t.Fatal(err) + } + blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if blocked.SelectedFiles != 1 { + t.Fatalf("blocked=%#v", blocked) + } + var retrievalState string + var removalJobs int + if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil { + t.Fatal(err) + } + if retrievalState != "enabled" || removalJobs != 0 { + t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs) + } + if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" { + t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err) + } + + // Unbound file and hash-unknown file are reported, never planned. + unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now}) + if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 { + t.Fatalf("unbound=%#v err=%v", unbound, err) + } + if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) +VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil { + t.Fatal(err) + } + hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now}) + if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 { + t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err) + } + // Overrides are still recorded for unbound/pending items so the desired state survives. + var overrideCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 { + t.Fatalf("pending override rows=%d err=%v", overrideCount, err) + } + // Audit trail exists for every annotation action. + var events int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 { + t.Fatalf("scope events=%d err=%v", events, err) + } +} diff --git a/internal/platform/postgres/catalog_routing_operation_integration_test.go b/internal/platform/postgres/catalog_routing_operation_integration_test.go new file mode 100644 index 0000000..23146d0 --- /dev/null +++ b/internal/platform/postgres/catalog_routing_operation_integration_test.go @@ -0,0 +1,113 @@ +package postgres + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestRouteCatalogFilesWithOperationReplaysOriginalResultAndRejectsConflictingPayload(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-op','pp','Operation PP',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-op',repeat('d',64),10,'verified/operation.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-op','conn-op','external-op',repeat('e',64),'工业产品','手册.pdf','document','v1','external','pp_public_tag'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-op','item-op','v1','content-op','verified'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-op','conn-op',repeat('f',64),'Operation PP',2,1,1,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) +VALUES + ('snapshot-op','root-op',NULL,'folder','Operation PP','',repeat('1',64),NULL,NULL,1,1,10), + ('snapshot-op','file-op','root-op','file','手册.pdf','工业产品/手册.pdf',repeat('2',64),'item-op',10,0,1,10); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-op','snapshot-op',NOW());`); err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC) + request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now} + first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1") + if err != nil || replayed || first.RoutableFiles != 1 { + t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err) + } + second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1") + if err != nil || !replayed || second != first { + t.Fatalf("replay result=%#v replayed=%v error=%v", second, replayed, err) + } + var publications, operations int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publications); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil { + t.Fatal(err) + } + if publications != 0 || operations != 1 { + t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations) + } + conflicting := request + conflicting.Reason = "不同的操作内容" + _, _, err = store.RouteCatalogFilesWithOperation(ctx, conflicting, "op-route-1") + if !errors.Is(err, ErrOperationRequestConflict) { + t.Fatalf("conflicting operation error=%v, want ErrOperationRequestConflict", err) + } +} + +func TestEnqueueCatalogRefreshWithOperationReplaysTheOriginalJob(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-refresh-op','pp','Refresh operation',true)`); err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + now := time.Date(2026, 9, 11, 7, 0, 0, 0, time.UTC) + source := governance.CatalogSource{ConnectionID: "conn-refresh-op", SourceType: governance.SourceTypePP, Name: "Refresh operation"} + first, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now) + if err != nil || replayed || first.Type != "source.catalog.refresh" { + t.Fatalf("first job=%#v replayed=%v error=%v", first, replayed, err) + } + second, replayed, err := store.EnqueueCatalogRefreshWithOperation(ctx, source, "op-refresh-1", now.Add(time.Minute)) + if err != nil || !replayed || second.ID != first.ID || second.Type != first.Type { + t.Fatalf("replay job=%#v replayed=%v error=%v", second, replayed, err) + } + var jobs, operations int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh'`).Scan(&jobs); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.refresh' AND operation_id='op-refresh-1'`).Scan(&operations); err != nil { + t.Fatal(err) + } + if jobs != 1 || operations != 1 { + t.Fatalf("jobs=%d operations=%d", jobs, operations) + } +} diff --git a/internal/platform/postgres/change_set_execution.go b/internal/platform/postgres/change_set_execution.go new file mode 100644 index 0000000..ebc09fd --- /dev/null +++ b/internal/platform/postgres/change_set_execution.go @@ -0,0 +1,358 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// ConfirmCatalogChangeSet executes ONLY the human-selected items of one planned +// ChangeSet: it creates publications + confirmations + test.upload jobs, or +// disables retrieval and enqueues test.remove for deletes. NO_CHANGE never +// enqueues. Re-confirmation replays the first stored outcome. +func (s *Store) ConfirmCatalogChangeSet(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) { + operator := strings.TrimSpace(request.OperatorRef) + reason := strings.TrimSpace(request.Reason) + if !validRouteID(request.ChangeSetID) || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() || len(request.ItemSeqs) == 0 || len(request.ItemSeqs) > 200 { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set confirmation") + } + seen := make(map[int64]bool, len(request.ItemSeqs)) + for _, seq := range request.ItemSeqs { + if seq < 1 || seen[seq] { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("invalid change set item selections") + } + seen[seq] = true + } + var tx pgx.Tx + ownsTransaction := false + if existing, ok := s.db.(pgx.Tx); ok { + tx = existing + } else { + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation transactions are unavailable") + } + var err error + tx, err = beginner.Begin(ctx) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("begin change set confirmation: %w", err) + } + ownsTransaction = true + defer tx.Rollback(ctx) + } + result, err := confirmCatalogChangeSetTx(ctx, tx, request) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + if ownsTransaction { + if err := tx.Commit(ctx); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("commit change set confirmation: %w", err) + } + } + return result, nil +} + +func confirmCatalogChangeSetTx(ctx context.Context, tx pgx.Tx, request governance.ConfirmCatalogChangeSetRequest) (governance.ConfirmCatalogChangeSetResult, error) { + result := governance.ConfirmCatalogChangeSetResult{ChangeSetID: request.ChangeSetID} + selections := append([]int64(nil), request.ItemSeqs...) + var state string + err := tx.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id=$1 FOR UPDATE`, request.ChangeSetID).Scan(&state) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ConfirmCatalogChangeSetResult{}, ErrCatalogChangeSetNotFound + } + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("lock change set for confirmation: %w", err) + } + if state != "planned" { + // Even on replay the requested locators must belong to this change set. + var owned int + if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_change_items WHERE change_set_id=$1 AND item_seq=ANY($2::bigint[])`, request.ChangeSetID, selections).Scan(&owned); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set item check: %w", err) + } + if owned != len(selections) { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items") + } + // Replay the first stored outcome from persisted evidence. + if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM processing_confirmations WHERE change_set_id=$1`, request.ChangeSetID).Scan(&result.ConfirmedItems); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set confirmation: %w", err) + } + if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.UploadsCreated); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set uploads: %w", err) + } + if err := tx.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE type='test.remove' AND payload->>'change_set_id'=$1`, request.ChangeSetID).Scan(&result.RemovalsRequested); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("replay change set removals: %w", err) + } + return result, nil + } + + rows, err := tx.Query(ctx, ` +SELECT item.item_seq,item.action,COALESCE(item.new_scope,''),COALESCE(item.old_scope,''),item.title, + COALESCE(item.new_source_item_id,item.old_source_item_id) AS source_item_id, + version.id AS version_id, + version.content_state='verified' AND content.content_state='verified' AND content.local_spool_ref<>'' AS artifact_ready +FROM source_catalog_change_items item +LEFT JOIN source_items source ON source.id=COALESCE(item.new_source_item_id,item.old_source_item_id) +LEFT JOIN source_item_versions version ON version.source_item_id=source.id AND version.revision=source.revision +LEFT JOIN content_objects content ON content.id=version.content_object_id +WHERE item.change_set_id=$1 AND item.item_seq=ANY($2::bigint[]) +ORDER BY item.item_seq`, request.ChangeSetID, selections) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("select change set items: %w", err) + } + type changeRow struct { + seq int64 + action string + newScope string + oldScope string + sourceItemID *string + versionID *string + artifactReady bool + } + changes := make([]changeRow, 0, len(selections)) + for rows.Next() { + var item changeRow + var title string + var versionID *string + var artifactReady *bool + if err := rows.Scan(&item.seq, &item.action, &item.newScope, &item.oldScope, &title, &item.sourceItemID, &versionID, &artifactReady); err != nil { + rows.Close() + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("scan change set item: %w", err) + } + if versionID != nil { + item.versionID = versionID + } + if artifactReady != nil { + item.artifactReady = *artifactReady + } + changes = append(changes, item) + } + if err := rows.Err(); err != nil { + rows.Close() + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("iterate change set items: %w", err) + } + rows.Close() + if len(changes) != len(selections) { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("change set confirmation references unknown items") + } + + for _, item := range changes { + if item.action == "no_change" { + result.NoChangeIgnored++ + continue + } + if item.action == "delete" || item.newScope == string(governance.VisibilityBlocked) { + if item.sourceItemID == nil { + result.SkippedNoArtifact++ + continue + } + removals, err := retireItemPublicationsForChange(ctx, tx, *item.sourceItemID, request) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + result.RemovalsRequested += removals + result.ConfirmedItems++ + continue + } + targets := make([]string, 0, 2) + switch item.newScope { + case string(governance.VisibilityInternal): + if request.InternalTargetKBID == "" { + result.SkippedNoTarget++ + continue + } + targets = append(targets, request.InternalTargetKBID) + case string(governance.VisibilityExternal): + if request.ExternalTargetKBID == "" { + result.SkippedNoTarget++ + continue + } + targets = append(targets, request.ExternalTargetKBID) + case string(governance.VisibilityBoth): + if request.InternalTargetKBID != "" { + targets = append(targets, request.InternalTargetKBID) + } + if request.ExternalTargetKBID != "" { + targets = append(targets, request.ExternalTargetKBID) + } + if len(targets) < 2 { + result.SkippedNoTarget++ + continue + } + default: + result.SkippedNoTarget++ + continue + } + if item.versionID == nil || !item.artifactReady { + result.SkippedNoArtifact++ + continue + } + result.ConfirmedItems++ + for _, target := range targets { + scope := item.newScope + if scope == string(governance.VisibilityBoth) { + scope = string(governance.VisibilityInternal) + if target == request.ExternalTargetKBID { + scope = string(governance.VisibilityExternal) + } + } + var existingPublication string + err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existingPublication) + if err == nil { + if err := ensureChangeSetConfirmation(ctx, tx, existingPublication, request); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + result.UploadsExisting++ + continue + } + if !errors.Is(err, pgx.ErrNoRows) { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("inspect change set publication: %w", err) + } + publicationID, err := newOpaqueID("tp-") + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + tag, err := tx.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,change_set_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$6) +ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`, + publicationID, *item.versionID, target, scope, request.ChangeSetID, request.Now) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("create change set publication: %w", err) + } + if tag.RowsAffected() == 0 { + // A concurrent confirmation already owns this version+target. + var existing string + if err := tx.QueryRow(ctx, `SELECT id FROM test_publications WHERE source_item_version_id=$1 AND test_kb_id=$2`, *item.versionID, target).Scan(&existing); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("resolve concurrent publication: %w", err) + } + if err := ensureChangeSetConfirmation(ctx, tx, existing, request); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + result.UploadsExisting++ + continue + } + jobID, err := newOpaqueID("job-") + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + payload, err := json.Marshal(map[string]string{"publication_id": publicationID, "change_set_id": request.ChangeSetID, "item_seq": strconv.FormatInt(item.seq, 10), "target_scope": scope, "target_kb_id": target}) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, errors.New("encode change set upload job") + } + if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3,$5,$6,$7,'upload') +ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "test-start:"+publicationID, request.ChangeSetID, *item.versionID, target); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("enqueue change set upload: %w", err) + } + if err := ensureChangeSetConfirmation(ctx, tx, publicationID, request); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, err + } + result.UploadsCreated++ + } + } + if _, err := tx.Exec(ctx, `UPDATE source_catalog_change_sets SET state='confirmed',confirmed_at=$2 WHERE id=$1 AND state='planned'`, request.ChangeSetID, request.Now); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, fmt.Errorf("confirm change set: %w", err) + } + return result, nil +} + +func ensureChangeSetConfirmation(ctx context.Context, tx pgx.Tx, publicationID string, request governance.ConfirmCatalogChangeSetRequest) error { + confirmationID, err := newOpaqueID("pc-") + if err != nil { + return err + } + if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,change_set_id) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT (publication_id) DO NOTHING`, + confirmationID, publicationID, request.OperatorRef, request.Reason, request.Now, request.ChangeSetID); err != nil { + return fmt.Errorf("record change set confirmation: %w", err) + } + return nil +} + +func retireItemPublicationsForChange(ctx context.Context, tx pgx.Tx, itemID string, request governance.ConfirmCatalogChangeSetRequest) (int, error) { + rows, err := tx.Query(ctx, `SELECT publication.id,publication.knowledge_id,publication.retrieval_state +FROM test_publications publication +JOIN source_item_versions version ON version.id=publication.source_item_version_id +WHERE version.source_item_id=$1 AND publication.retrieval_state NOT IN ('disabling','disabled','unpublished') +FOR UPDATE OF publication`, itemID) + if err != nil { + return 0, fmt.Errorf("inspect change set removals: %w", err) + } + type target struct{ id, knowledgeID string } + targets := make([]target, 0) + for rows.Next() { + var item target + if err := rows.Scan(&item.id, &item.knowledgeID, new(string)); err != nil { + rows.Close() + return 0, fmt.Errorf("scan change set removal: %w", err) + } + targets = append(targets, item) + } + if err := rows.Err(); err != nil { + rows.Close() + return 0, fmt.Errorf("iterate change set removals: %w", err) + } + rows.Close() + removals := 0 + for _, item := range targets { + if item.knowledgeID == "" { + if _, err := tx.Exec(ctx, `UPDATE test_publications SET ingest_state='failed',processing_state='failed',retrieval_state='disabled',last_error='blocked_by_changeset',updated_at=$2 WHERE id=$1 AND knowledge_id=''`, item.id, request.Now); err != nil { + return 0, fmt.Errorf("stop pre-upload change set removal: %w", err) + } + continue + } + if _, err := tx.Exec(ctx, `UPDATE test_publications SET retrieval_state='disabling',last_error='',updated_at=$2 WHERE id=$1`, item.id, request.Now); err != nil { + return 0, fmt.Errorf("mark change set removal: %w", err) + } + jobID, err := newOpaqueID("job-") + if err != nil { + return 0, err + } + payload, err := json.Marshal(map[string]string{"publication_id": item.id, "change_set_id": request.ChangeSetID}) + if err != nil { + return 0, errors.New("encode change set remove job") + } + // Removals key on the item (version-independent): a confirmed delete must + // not enqueue twice even if version rows rotate between deliveries. + if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES($1,'test.remove',$2,'pending',20,$3,10,$4,$3,$3,$5,$6,'','remove') +ON CONFLICT DO NOTHING`, jobID, payload, request.Now, "changeset-remove:"+item.id+":"+request.ChangeSetID, request.ChangeSetID, item.id); err != nil { + return 0, fmt.Errorf("enqueue change set removal: %w", err) + } + removals++ + } + return removals, nil +} + +func (s *Store) ConfirmCatalogChangeSetWithOperation(ctx context.Context, request governance.ConfirmCatalogChangeSetRequest, operationID string) (governance.ConfirmCatalogChangeSetResult, bool, error) { + fingerprint, err := operationFingerprint(struct { + ChangeSetID string `json:"change_set_id"` + ItemSeqs []int64 `json:"item_seqs"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + }{request.ChangeSetID, request.ItemSeqs, request.OperatorRef, request.Reason}) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, false, err + } + raw, replayed, err := s.executeOperation(ctx, "changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + result, err := NewStore(tx).ConfirmCatalogChangeSet(ctx, request) + if err != nil { + return nil, err + } + return json.Marshal(result) + }) + if err != nil { + return governance.ConfirmCatalogChangeSetResult{}, false, err + } + var result governance.ConfirmCatalogChangeSetResult + if err := json.Unmarshal(raw, &result); err != nil { + return governance.ConfirmCatalogChangeSetResult{}, false, fmt.Errorf("decode stored change set result: %w", err) + } + return result, replayed, nil +} diff --git a/internal/platform/postgres/integration_isolation_external_test.go b/internal/platform/postgres/integration_isolation_external_test.go new file mode 100644 index 0000000..75aa919 --- /dev/null +++ b/internal/platform/postgres/integration_isolation_external_test.go @@ -0,0 +1,44 @@ +package postgres_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/url" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func isolatedExternalIntegrationDatabaseURL(t *testing.T, databaseURL string) string { + t.Helper() + if databaseURL == "" { + return "" + } + parsed, err := url.Parse(databaseURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err) + } + digest := sha256.Sum256([]byte(t.Name())) + schema := "kgtestx_" + hex.EncodeToString(digest[:8]) + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect integration database: %v", err) + } + defer admin.Close() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatalf("create isolated integration schema: %v", err) + } + t.Cleanup(func() { + cleanup, err := pgxpool.New(context.Background(), databaseURL) + if err == nil { + _, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE") + cleanup.Close() + } + }) + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() +} diff --git a/internal/platform/postgres/integration_isolation_test.go b/internal/platform/postgres/integration_isolation_test.go new file mode 100644 index 0000000..8886991 --- /dev/null +++ b/internal/platform/postgres/integration_isolation_test.go @@ -0,0 +1,44 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/url" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func isolatedIntegrationDatabaseURL(t *testing.T, databaseURL string) string { + t.Helper() + if databaseURL == "" { + return "" + } + parsed, err := url.Parse(databaseURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + t.Fatalf("TEST_DATABASE_URL must be a PostgreSQL URL: %v", err) + } + digest := sha256.Sum256([]byte(t.Name())) + schema := "kgtest_" + hex.EncodeToString(digest[:8]) + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect integration database: %v", err) + } + defer admin.Close() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatalf("create isolated integration schema: %v", err) + } + t.Cleanup(func() { + cleanup, err := pgxpool.New(context.Background(), databaseURL) + if err == nil { + _, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE") + cleanup.Close() + } + }) + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() +} diff --git a/internal/platform/postgres/job_failures.go b/internal/platform/postgres/job_failures.go new file mode 100644 index 0000000..9fa2e20 --- /dev/null +++ b/internal/platform/postgres/job_failures.go @@ -0,0 +1,201 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +const listJobFailuresSQL = ` +SELECT jf.id, jf.job_id, jf.job_type, jf.publication_id, jf.error_code, + jf.attempt, jf.max_attempts_at_failure, jf.business_change_set_id, + jf.business_action, jf.business_target_kb_id, jf.state, jf.created_at, + COALESCE(siv.source_item_id,'') +FROM job_failures jf +LEFT JOIN test_publications p ON p.id = jf.publication_id +LEFT JOIN source_item_versions siv ON siv.id = p.source_item_version_id +WHERE ($1 = '' OR jf.state = $1) +ORDER BY jf.created_at DESC, jf.id DESC +LIMIT $2 OFFSET $3 +` + +const countJobFailuresSQL = `SELECT COUNT(*) FROM job_failures WHERE ($1 = '' OR state = $1)` + +// ListJobFailures is the DLQ view (P6): projections of terminal attempts. The +// original jobs and publication audit rows are never moved or deleted. +func (s *Store) ListJobFailures(ctx context.Context, filter governance.JobFailureFilter) (governance.JobFailurePage, error) { + state := filter.State + if state == "" { + state = "open" + } + if state != "open" && state != "requeued" && state != "abandoned" { + return governance.JobFailurePage{}, errors.New("invalid failure state filter") + } + limit, offset := filter.Limit, filter.Offset + if limit <= 0 { + limit = 50 + } + if limit > 100 { + return governance.JobFailurePage{}, errors.New("failure limit must be between 1 and 100") + } + if offset < 0 { + offset = 0 + } + var total int + if err := s.db.QueryRow(ctx, countJobFailuresSQL, state).Scan(&total); err != nil { + return governance.JobFailurePage{}, fmt.Errorf("count job failures: %w", err) + } + rows, err := s.db.Query(ctx, listJobFailuresSQL, state, limit, offset) + if err != nil { + return governance.JobFailurePage{}, fmt.Errorf("list job failures: %w", err) + } + defer rows.Close() + items := make([]governance.JobFailure, 0, limit) + for rows.Next() { + var item governance.JobFailure + if err := rows.Scan(&item.ID, &item.JobID, &item.JobType, &item.PublicationID, &item.ErrorCode, + &item.Attempt, &item.MaxAttempts, &item.ChangeSetID, &item.Action, &item.TargetKBID, + &item.State, &item.CreatedAt, &item.SourceItemID); err != nil { + return governance.JobFailurePage{}, fmt.Errorf("scan job failure: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return governance.JobFailurePage{}, fmt.Errorf("iterate job failures: %w", err) + } + return governance.JobFailurePage{Items: items, Total: total, Limit: limit, Offset: offset}, nil +} + +const failurePublicationStateSQL = ` +SELECT p.processing_state, p.ingest_state, + EXISTS ( + SELECT 1 FROM source_item_versions siv + JOIN content_objects co ON co.id = siv.content_object_id + WHERE siv.id = p.source_item_version_id + AND co.content_state = 'verified' + AND co.local_spool_ref <> '' + ) +FROM test_publications p +WHERE p.id = $1 +` + +const resolvePublicationFailuresSQL = ` +UPDATE job_failures +SET state = 'requeued', resolved_at = $2, resolution_actor_ref = $3, resolution_reason = $4 +WHERE publication_id = $1 AND state = 'open' +` + +const recordRetryAuditSQL = ` +INSERT INTO job_retry_operations(batch_key, publication_id, actor_ref, reason, outcome, created_at) +VALUES ($1, $2, $3, $4, $5, $6) +ON CONFLICT (batch_key, publication_id) DO NOTHING +` + +// RetryPublicationsBatchWithOperation performs the whole selection server-side +// (P6): one operation envelope, one transaction, per-item outcome, atomic audit. +func (s *Store) RetryPublicationsBatchWithOperation(ctx context.Context, request governance.RetryPublicationsBatchRequest, operationID string) (governance.RetryPublicationsBatchResult, bool, error) { + if operationID == "" || len(operationID) > 128 { + return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid operation id") + } + if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 { + return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids must contain between 1 and 100 items") + } + if request.OperatorRef == "" || len(request.OperatorRef) > 255 || request.Reason == "" || len(request.Reason) > 2000 || request.Now.IsZero() { + return governance.RetryPublicationsBatchResult{}, false, errors.New("actor_ref and a non-empty reason are required") + } + seen := make(map[string]bool, len(request.PublicationIDs)) + ids := make([]string, 0, len(request.PublicationIDs)) + for _, id := range request.PublicationIDs { + if id == "" || len(id) > 64 || seen[id] { + return governance.RetryPublicationsBatchResult{}, false, errors.New("publication_ids contain an invalid or duplicate entry") + } + seen[id] = true + ids = append(ids, id) + } + fingerprint, err := operationFingerprint(struct { + PublicationIDs []string `json:"publication_ids"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + }{ids, request.OperatorRef, request.Reason}) + if err != nil { + return governance.RetryPublicationsBatchResult{}, false, err + } + raw, replayed, err := s.executeOperation(ctx, "test.publication.retry_batch", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + result, err := retryPublicationsBatchTx(ctx, tx, request, ids, operationID) + if err != nil { + return nil, err + } + return json.Marshal(result) + }) + if err != nil { + return governance.RetryPublicationsBatchResult{}, false, err + } + var result governance.RetryPublicationsBatchResult + if err := json.Unmarshal(raw, &result); err != nil { + return governance.RetryPublicationsBatchResult{}, false, errors.New("invalid stored retry batch result") + } + return result, replayed, nil +} + +func retryPublicationsBatchTx(ctx context.Context, tx pgx.Tx, request governance.RetryPublicationsBatchRequest, ids []string, batchKey string) (governance.RetryPublicationsBatchResult, error) { + // Stable lock order prevents cross-batch deadlocks. + sorted := append([]string(nil), ids...) + for i := 1; i < len(sorted); i++ { + for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { + sorted[j], sorted[j-1] = sorted[j-1], sorted[j] + } + } + result := governance.RetryPublicationsBatchResult{BatchKey: batchKey, Outcomes: make([]governance.RetryOutcome, 0, len(sorted))} + store := NewStore(tx) + for _, id := range sorted { + var processingState, ingestState string + var spoolAvailable bool + err := tx.QueryRow(ctx, failurePublicationStateSQL, id).Scan(&processingState, &ingestState, &spoolAvailable) + outcome := governance.RetryOutcome{PublicationID: id} + switch { + case errors.Is(err, pgx.ErrNoRows): + outcome.Result = governance.RetryNotRetryable + outcome.Detail = "publication not found" + case err != nil: + return result, fmt.Errorf("classify publication retry: %w", err) + case processingState == "processing" || processingState == "finalizing": + outcome.Result = governance.RetryAlreadyRunning + case processingState == "pending" && ingestState != "failed": + outcome.Result = governance.RetryNotRetryable + outcome.Detail = "publication is already queued" + case processingState != "failed" && processingState != "stalled" && !(processingState == "pending" && ingestState == "failed"): + outcome.Result = governance.RetryNotRetryable + outcome.Detail = "publication is not in a failed state" + case !spoolAvailable: + outcome.Result = governance.RetryNotRetryable + outcome.Detail = "no verified spool artifact" + default: + if err := store.RetryPublication(ctx, id, request.Now); err != nil { + return result, fmt.Errorf("retry publication %s: %w", id, err) + } + if _, err := tx.Exec(ctx, resolvePublicationFailuresSQL, id, request.Now, request.OperatorRef, request.Reason); err != nil { + return result, fmt.Errorf("resolve job failures: %w", err) + } + outcome.Result = governance.RetryStarted + } + if _, err := tx.Exec(ctx, recordRetryAuditSQL, batchKey, id, request.OperatorRef, request.Reason, outcome.Result, request.Now); err != nil { + return result, fmt.Errorf("record retry audit: %w", err) + } + switch outcome.Result { + case governance.RetryStarted: + result.Started++ + case governance.RetryAlreadyRunning: + result.AlreadyRunning++ + default: + result.NotRetryable++ + } + result.Outcomes = append(result.Outcomes, outcome) + } + result.Total = len(sorted) + return result, nil +} diff --git a/internal/platform/postgres/manual_publication.go b/internal/platform/postgres/manual_publication.go new file mode 100644 index 0000000..3838137 --- /dev/null +++ b/internal/platform/postgres/manual_publication.go @@ -0,0 +1,196 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ErrInvalidManualPublicationSeed = errors.New("invalid manual publication seed") + +type ManualPublicationSeedRequest struct { + TestKBID string + Title string + SHA256 string + SizeBytes int64 + MIME string + ValidatedFormat string + LocalSpoolRef string + Now time.Time +} + +type ManualPublicationSeedResult struct { + PublicationID string + JobID string +} + +const seedManualPublicationSQL = ` +WITH ensured_connection AS ( + INSERT INTO source_connections ( + id, source_type, name, base_url, secret_ref, enabled, + schedule, last_health, created_at, updated_at + ) VALUES ($3, 'manual', '本地手工验收', '', '', FALSE, + '{}'::jsonb, '{}'::jsonb, $2, $2) + ON CONFLICT (id) DO UPDATE + SET updated_at = source_connections.updated_at + RETURNING id +), ensured_content AS ( + INSERT INTO content_objects ( + id, sha256, size_bytes, mime, validated_format, + local_spool_ref, content_state, + first_discovered_at, last_discovered_at, created_at, updated_at + ) VALUES ($10, $11, $8, $9, $12, $13, 'verified', $2, $2, $2, $2) + ON CONFLICT (sha256) DO UPDATE + SET last_discovered_at = EXCLUDED.last_discovered_at, + updated_at = EXCLUDED.updated_at, + local_spool_ref = CASE + WHEN content_objects.local_spool_ref = '' THEN EXCLUDED.local_spool_ref + ELSE content_objects.local_spool_ref + END + WHERE content_objects.size_bytes = EXCLUDED.size_bytes + RETURNING id +), ensured_item AS ( + INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, + normalized_path, title, item_type, size_bytes, mime, revision, + visibility, visibility_origin, source_change, created_at, updated_at, + identity_confidence + ) + SELECT $4, id, $5, $6, '', $7, 'document', $8, $9, $15, + 'internal', 'manual', 'added', $2, $2, 'content_hash' + FROM ensured_connection + ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE + SET updated_at = source_items.updated_at, + identity_confidence = CASE + WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence + ELSE EXCLUDED.identity_confidence + END + RETURNING id +), ensured_version AS ( + INSERT INTO source_item_versions ( + id, source_item_id, revision, content_object_id, content_state, + download_error, raw_record_summary, created_at, updated_at + ) + SELECT $14, ensured_item.id, $15, ensured_content.id, 'verified', '', + '{"origin":"manual_e2e"}'::jsonb, $2, $2 + FROM ensured_item CROSS JOIN ensured_content + ON CONFLICT (source_item_id, revision) DO UPDATE + SET updated_at = source_item_versions.updated_at + RETURNING id +), ensured_publication AS ( + INSERT INTO test_publications ( + id, source_item_version_id, test_kb_id, + ingest_state, processing_state, retrieval_state, + created_at, updated_at + ) + SELECT $16, id, $17, 'pending', 'pending', 'unpublished', $2, $2 + FROM ensured_version + ON CONFLICT (source_item_version_id, test_kb_id) DO UPDATE + SET updated_at = test_publications.updated_at + RETURNING id +), ensured_job AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT $18, 'test.upload', $1::jsonb, 'pending', 10, $2, + 5, $19, $2, $2 + FROM ensured_publication + ON CONFLICT (idempotency_key) DO UPDATE + SET idempotency_key = jobs.idempotency_key + RETURNING id +) +SELECT ensured_publication.id, ensured_job.id +FROM ensured_publication CROSS JOIN ensured_job +` + +func (s *Store) SeedManualPublication(ctx context.Context, request ManualPublicationSeedRequest) (ManualPublicationSeedResult, error) { + if !validManualPublicationSeed(request) { + return ManualPublicationSeedResult{}, ErrInvalidManualPublicationSeed + } + identity := request.SHA256 + ":" + request.TestKBID + connectionID := "conn-manual-local" + sourceItemID := deterministicGovernanceID("si-", request.SHA256) + contentID := deterministicGovernanceID("co-", request.SHA256) + versionID := deterministicGovernanceID("sv-", request.SHA256) + publicationID := deterministicGovernanceID("tp-", identity) + jobID := deterministicGovernanceID("job-", "test.upload:"+publicationID) + stableExternalID := "manual-e2e:" + request.SHA256 + canonicalDigest := sha256.Sum256([]byte(stableExternalID)) + revision := "sha256:" + request.SHA256 + payload, err := json.Marshal(struct { + PublicationID string `json:"publication_id"` + }{PublicationID: publicationID}) + if err != nil { + return ManualPublicationSeedResult{}, errors.New("encode manual publication upload job") + } + idempotencyKey := "test-upload:" + publicationID + + result := ManualPublicationSeedResult{} + err = s.db.QueryRow( + ctx, + seedManualPublicationSQL, + payload, + request.Now, + connectionID, + sourceItemID, + stableExternalID, + hex.EncodeToString(canonicalDigest[:]), + request.Title, + request.SizeBytes, + request.MIME, + contentID, + request.SHA256, + request.ValidatedFormat, + request.LocalSpoolRef, + versionID, + revision, + publicationID, + request.TestKBID, + jobID, + idempotencyKey, + ).Scan(&result.PublicationID, &result.JobID) + if errors.Is(err, pgx.ErrNoRows) { + return ManualPublicationSeedResult{}, ErrPublicationStateConflict + } + if err != nil { + return ManualPublicationSeedResult{}, fmt.Errorf("seed manual test publication: %w", err) + } + return result, nil +} + +func validManualPublicationSeed(request ManualPublicationSeedRequest) bool { + if !governance.ValidSHA256(request.SHA256) || request.SizeBytes <= 0 || request.Now.IsZero() { + return false + } + if len(request.TestKBID) == 0 || len(request.TestKBID) > 255 || strings.ContainsAny(request.TestKBID, "/\\\x00\t\r\n") { + return false + } + if strings.TrimSpace(request.Title) == "" || len(request.Title) > 512 || strings.ContainsRune(request.Title, '\x00') { + return false + } + if request.MIME == "" || len(request.MIME) > 128 || strings.ContainsAny(request.MIME, "\x00\r\n") || !validErrorCode(request.ValidatedFormat) { + return false + } + clean := path.Clean(request.LocalSpoolRef) + return len(request.LocalSpoolRef) <= 512 && clean == request.LocalSpoolRef && clean != "." && !strings.HasPrefix(clean, "/") && !strings.HasPrefix(clean, "../") +} + +func deterministicGovernanceID(prefix string, parts ...string) string { + digest := sha256.New() + for _, part := range parts { + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(part)) + } + return prefix + hex.EncodeToString(digest.Sum(nil))[:32] +} diff --git a/internal/platform/postgres/manual_publication_test.go b/internal/platform/postgres/manual_publication_test.go new file mode 100644 index 0000000..59001a1 --- /dev/null +++ b/internal/platform/postgres/manual_publication_test.go @@ -0,0 +1,60 @@ +package postgres + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestSeedManualPublicationCreatesAtomicUploadChain(t *testing.T) { + db := &recordingDB{row: manualSeedRow{publicationID: "tp-seeded", jobID: "job-seeded"}} + store := NewStore(db) + now := time.Date(2026, 9, 5, 5, 0, 0, 0, time.UTC) + + result, err := store.SeedManualPublication(context.Background(), ManualPublicationSeedRequest{ + TestKBID: "c12ff4d5-dd87-423b-ab0f-af745b9d54a7", + Title: "Governance P3 E2E", + SHA256: strings.Repeat("a", 64), + SizeBytes: 128, + MIME: "text/markdown", + ValidatedFormat: "markdown", + LocalSpoolRef: "manual/e2e-a.md", + Now: now, + }) + if err != nil { + t.Fatalf("SeedManualPublication() error = %v", err) + } + if result.PublicationID != "tp-seeded" || result.JobID != "job-seeded" { + t.Fatalf("result = %#v", result) + } + lowerSQL := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "insert into source_connections", + "insert into source_items", + "insert into content_objects", + "insert into source_item_versions", + "insert into test_publications", + "insert into jobs", + "on conflict", + "returning", + } { + if !strings.Contains(lowerSQL, required) { + t.Errorf("seed SQL missing %q", required) + } + } + if !strings.Contains(string(db.queryRowArgs[0].([]byte)), `"publication_id"`) || strings.Contains(string(db.queryRowArgs[0].([]byte)), "c12ff4d5") { + t.Fatalf("job payload = %s, want publication_id only", db.queryRowArgs[0]) + } +} + +type manualSeedRow struct { + publicationID string + jobID string +} + +func (row manualSeedRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.publicationID + *(dest[1].(*string)) = row.jobID + return nil +} diff --git a/internal/platform/postgres/migrate.go b/internal/platform/postgres/migrate.go new file mode 100644 index 0000000..a41aad0 --- /dev/null +++ b/internal/platform/postgres/migrate.go @@ -0,0 +1,140 @@ +package postgres + +import ( + "context" + "embed" + "fmt" + "io/fs" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +type migration struct { + Version int64 + UpSQL string + DownSQL string +} + +func loadMigrations() ([]migration, error) { + paths, err := fs.Glob(migrationFS, "migrations/*.up.sql") + if err != nil { + return nil, fmt.Errorf("glob migrations: %w", err) + } + result := make([]migration, 0, len(paths)) + for _, path := range paths { + base := strings.TrimPrefix(path, "migrations/") + versionText, _, ok := strings.Cut(base, "_") + if !ok { + return nil, fmt.Errorf("invalid migration filename %q", path) + } + version, err := strconv.ParseInt(versionText, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse migration version %q: %w", path, err) + } + up, err := migrationFS.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read up migration %q: %w", path, err) + } + downPath := strings.TrimSuffix(path, ".up.sql") + ".down.sql" + down, err := migrationFS.ReadFile(downPath) + if err != nil { + return nil, fmt.Errorf("read down migration %q: %w", downPath, err) + } + result = append(result, migration{Version: version, UpSQL: string(up), DownSQL: string(down)}) + } + sort.Slice(result, func(i, j int) bool { return result[i].Version < result[j].Version }) + return result, nil +} + +const migrationLockID int64 = 7346729101 + +func ApplyMigrations(ctx context.Context, pool *pgxpool.Pool) error { + if _, err := pool.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil { + return fmt.Errorf("lock migrations: %w", err) + } + defer func() { _, _ = pool.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockID) }() + + if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )`); err != nil { + return fmt.Errorf("create migration ledger: %w", err) + } + + migrations, err := loadMigrations() + if err != nil { + return err + } + for _, item := range migrations { + var applied bool + if err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = $1)", item.Version).Scan(&applied); err != nil { + return fmt.Errorf("check migration %d: %w", item.Version, err) + } + if applied { + continue + } + tx, err := pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin migration %d: %w", item.Version, err) + } + if _, err := tx.Exec(ctx, item.UpSQL); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("apply migration %d: %w", item.Version, err) + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version) VALUES ($1)", item.Version); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("record migration %d: %w", item.Version, err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit migration %d: %w", item.Version, err) + } + } + return nil +} + +func RollbackLatestMigration(ctx context.Context, pool *pgxpool.Pool) error { + migrations, err := loadMigrations() + if err != nil { + return err + } + var version int64 + if err := pool.QueryRow(ctx, "SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1").Scan(&version); err != nil { + if err == pgx.ErrNoRows { + return nil + } + return fmt.Errorf("read latest migration: %w", err) + } + var selected *migration + for i := range migrations { + if migrations[i].Version == version { + selected = &migrations[i] + break + } + } + if selected == nil { + return fmt.Errorf("missing down migration for version %d", version) + } + tx, err := pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin rollback %d: %w", version, err) + } + if _, err := tx.Exec(ctx, selected.DownSQL); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("rollback migration %d: %w", version, err) + } + if _, err := tx.Exec(ctx, "DELETE FROM schema_migrations WHERE version = $1", version); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("delete migration ledger %d: %w", version, err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit rollback %d: %w", version, err) + } + return nil +} diff --git a/internal/platform/postgres/migrations/000001_jobs.down.sql b/internal/platform/postgres/migrations/000001_jobs.down.sql new file mode 100644 index 0000000..ac5b096 --- /dev/null +++ b/internal/platform/postgres/migrations/000001_jobs.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS jobs; diff --git a/internal/platform/postgres/migrations/000001_jobs.up.sql b/internal/platform/postgres/migrations/000001_jobs.up.sql new file mode 100644 index 0000000..3b54bdf --- /dev/null +++ b/internal/platform/postgres/migrations/000001_jobs.up.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS jobs ( + id VARCHAR(36) PRIMARY KEY, + type VARCHAR(100) NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + state VARCHAR(20) NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed')), + priority INTEGER NOT NULL DEFAULT 0, + available_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + lease_owner VARCHAR(255), + lease_until TIMESTAMPTZ, + attempt INTEGER NOT NULL DEFAULT 0 CHECK (attempt >= 0), + max_attempts INTEGER NOT NULL DEFAULT 5 CHECK (max_attempts > 0), + last_error TEXT NOT NULL DEFAULT '', + idempotency_key VARCHAR(255) NOT NULL, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_jobs_claim + ON jobs (priority DESC, available_at, created_at, id) + WHERE state IN ('pending', 'retry', 'running'); + +CREATE INDEX IF NOT EXISTS idx_jobs_state_updated + ON jobs (state, updated_at DESC); diff --git a/internal/platform/postgres/migrations/000002_source_governance.down.sql b/internal/platform/postgres/migrations/000002_source_governance.down.sql new file mode 100644 index 0000000..bea2be6 --- /dev/null +++ b/internal/platform/postgres/migrations/000002_source_governance.down.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS test_publications; +DROP TABLE IF EXISTS source_item_versions; +DROP TABLE IF EXISTS content_objects; +DROP TABLE IF EXISTS source_items; +DROP TABLE IF EXISTS source_connections; diff --git a/internal/platform/postgres/migrations/000002_source_governance.up.sql b/internal/platform/postgres/migrations/000002_source_governance.up.sql new file mode 100644 index 0000000..e65b65f --- /dev/null +++ b/internal/platform/postgres/migrations/000002_source_governance.up.sql @@ -0,0 +1,114 @@ +CREATE TABLE IF NOT EXISTS source_connections ( + id VARCHAR(36) PRIMARY KEY, + source_type VARCHAR(20) NOT NULL + CHECK (source_type IN ('pp', 'alist', 'dingtalk')), + name VARCHAR(255) NOT NULL, + base_url TEXT NOT NULL DEFAULT '', + secret_ref VARCHAR(255) NOT NULL DEFAULT '', + enabled BOOLEAN NOT NULL DEFAULT FALSE, + schedule JSONB NOT NULL DEFAULT '{}'::jsonb + CHECK (jsonb_typeof(schedule) = 'object'), + last_health JSONB NOT NULL DEFAULT '{}'::jsonb + CHECK (jsonb_typeof(last_health) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (name) +); + +CREATE TABLE IF NOT EXISTS source_items ( + id VARCHAR(36) PRIMARY KEY, + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE RESTRICT, + stable_external_id TEXT NOT NULL, + canonical_uri_hash CHAR(64) NOT NULL + CHECK (canonical_uri_hash ~ '^[0-9a-f]{64}$'), + parent_external_id TEXT NOT NULL DEFAULT '', + normalized_path TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + item_type VARCHAR(20) NOT NULL + CHECK (item_type IN ('document', 'file', 'link', 'video')), + size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0), + mime TEXT NOT NULL DEFAULT '', + revision TEXT NOT NULL DEFAULT '', + updated_at_source TIMESTAMPTZ, + visibility VARCHAR(20) NOT NULL DEFAULT 'internal' + CHECK (visibility IN ('internal', 'external')), + visibility_origin VARCHAR(30) NOT NULL DEFAULT 'manual' + CHECK (visibility_origin IN ('pp_public_tag', 'configured_scope', 'manual')), + last_seen_scan_id VARCHAR(36) NOT NULL DEFAULT '', + source_change VARCHAR(20) NOT NULL DEFAULT 'unchanged' + CHECK (source_change IN ('unchanged', 'added', 'updated', 'deleted', 'unavailable')), + absence_streak INTEGER NOT NULL DEFAULT 0 CHECK (absence_streak >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_connection_id, stable_external_id) +); + +CREATE TABLE IF NOT EXISTS content_objects ( + id VARCHAR(36) PRIMARY KEY, + sha256 CHAR(64) NOT NULL + CHECK (sha256 ~ '^[0-9a-f]{64}$'), + size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0), + mime TEXT NOT NULL DEFAULT '', + validated_format VARCHAR(50) NOT NULL DEFAULT '', + local_spool_ref TEXT NOT NULL DEFAULT '', + content_state VARCHAR(20) NOT NULL DEFAULT 'unknown' + CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')), + first_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (sha256) +); + +CREATE TABLE IF NOT EXISTS source_item_versions ( + id VARCHAR(36) PRIMARY KEY, + source_item_id VARCHAR(36) NOT NULL + REFERENCES source_items(id) ON DELETE RESTRICT, + revision TEXT NOT NULL, + content_object_id VARCHAR(36) + REFERENCES content_objects(id) ON DELETE RESTRICT, + content_state VARCHAR(20) NOT NULL DEFAULT 'unknown' + CHECK (content_state IN ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed')), + download_error TEXT NOT NULL DEFAULT '', + raw_record_summary JSONB NOT NULL DEFAULT '{}'::jsonb + CHECK (jsonb_typeof(raw_record_summary) = 'object'), + source_updated_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_item_id, revision) +); + +CREATE TABLE IF NOT EXISTS test_publications ( + id VARCHAR(36) PRIMARY KEY, + source_item_version_id VARCHAR(36) NOT NULL + REFERENCES source_item_versions(id) ON DELETE RESTRICT, + test_kb_id VARCHAR(255) NOT NULL, + knowledge_id VARCHAR(255) NOT NULL DEFAULT '', + ingest_state VARCHAR(20) NOT NULL DEFAULT 'pending' + CHECK (ingest_state IN ('pending', 'uploading', 'uploaded', 'failed')), + processing_state VARCHAR(20) NOT NULL DEFAULT 'pending' + CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')), + retrieval_state VARCHAR(20) NOT NULL DEFAULT 'unpublished' + CHECK (retrieval_state IN ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed')), + parser_generation VARCHAR(255) NOT NULL DEFAULT '', + index_generation VARCHAR(255) NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_item_version_id, test_kb_id) +); + +CREATE INDEX IF NOT EXISTS idx_source_items_change + ON source_items (source_connection_id, source_change, updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_source_items_visibility + ON source_items (visibility, updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_source_item_versions_content + ON source_item_versions (content_object_id) + WHERE content_object_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_test_publications_state + ON test_publications (processing_state, retrieval_state, updated_at DESC); diff --git a/internal/platform/postgres/migrations/000003_test_orchestration.down.sql b/internal/platform/postgres/migrations/000003_test_orchestration.down.sql new file mode 100644 index 0000000..67c1eaa --- /dev/null +++ b/internal/platform/postgres/migrations/000003_test_orchestration.down.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS acceptance_results; +DROP INDEX IF EXISTS idx_test_publications_poll; + +ALTER TABLE test_publications DROP COLUMN IF EXISTS superseded_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS supersedes_publication_id; +ALTER TABLE test_publications DROP COLUMN IF EXISTS stalled_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS completed_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS processing_started_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS last_observed_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_updated_at; +ALTER TABLE test_publications DROP COLUMN IF EXISTS last_remote_status; +ALTER TABLE test_publications DROP COLUMN IF EXISTS poll_sequence; diff --git a/internal/platform/postgres/migrations/000003_test_orchestration.up.sql b/internal/platform/postgres/migrations/000003_test_orchestration.up.sql new file mode 100644 index 0000000..8381f0a --- /dev/null +++ b/internal/platform/postgres/migrations/000003_test_orchestration.up.sql @@ -0,0 +1,33 @@ +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS poll_sequence INTEGER NOT NULL DEFAULT 0 + CHECK (poll_sequence >= 0); +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_status VARCHAR(32) NOT NULL DEFAULT ''; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_remote_updated_at TIMESTAMPTZ; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS last_observed_at TIMESTAMPTZ; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS stalled_at TIMESTAMPTZ; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS supersedes_publication_id VARCHAR(36) + REFERENCES test_publications(id) ON DELETE RESTRICT; +ALTER TABLE test_publications ADD COLUMN IF NOT EXISTS superseded_at TIMESTAMPTZ; + +CREATE TABLE IF NOT EXISTS acceptance_results ( + id VARCHAR(36) PRIMARY KEY, + publication_id VARCHAR(36) NOT NULL + REFERENCES test_publications(id) ON DELETE RESTRICT, + run_id VARCHAR(36) NOT NULL, + check_type VARCHAR(32) NOT NULL + CHECK (check_type IN ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches')), + state VARCHAR(20) NOT NULL + CHECK (state IN ('pending', 'passed', 'failed', 'manual_required')), + policy_version VARCHAR(64) NOT NULL, + evidence_summary JSONB NOT NULL DEFAULT '{}'::jsonb + CHECK (jsonb_typeof(evidence_summary) = 'object'), + observed_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (publication_id, run_id, check_type) +); + +CREATE INDEX IF NOT EXISTS idx_test_publications_poll + ON test_publications (processing_state, last_observed_at, updated_at); +CREATE INDEX IF NOT EXISTS idx_acceptance_results_publication + ON acceptance_results (publication_id, observed_at DESC); diff --git a/internal/platform/postgres/migrations/000004_manual_source.down.sql b/internal/platform/postgres/migrations/000004_manual_source.down.sql new file mode 100644 index 0000000..6fd61ad --- /dev/null +++ b/internal/platform/postgres/migrations/000004_manual_source.down.sql @@ -0,0 +1,18 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM source_connections + WHERE source_type = 'manual' + ) THEN + RAISE EXCEPTION 'cannot downgrade while manual source connections exist'; + END IF; +END +$$; + +ALTER TABLE source_connections + DROP CONSTRAINT IF EXISTS source_connections_source_type_check; + +ALTER TABLE source_connections + ADD CONSTRAINT source_connections_source_type_check + CHECK (source_type IN ('pp', 'alist', 'dingtalk')); diff --git a/internal/platform/postgres/migrations/000004_manual_source.up.sql b/internal/platform/postgres/migrations/000004_manual_source.up.sql new file mode 100644 index 0000000..36cfaa9 --- /dev/null +++ b/internal/platform/postgres/migrations/000004_manual_source.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE source_connections + DROP CONSTRAINT IF EXISTS source_connections_source_type_check; + +ALTER TABLE source_connections + ADD CONSTRAINT source_connections_source_type_check + CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')); diff --git a/internal/platform/postgres/migrations/000005_source_scans.down.sql b/internal/platform/postgres/migrations/000005_source_scans.down.sql new file mode 100644 index 0000000..af2e320 --- /dev/null +++ b/internal/platform/postgres/migrations/000005_source_scans.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS source_scan_changes; +DROP TABLE IF EXISTS source_scans; diff --git a/internal/platform/postgres/migrations/000005_source_scans.up.sql b/internal/platform/postgres/migrations/000005_source_scans.up.sql new file mode 100644 index 0000000..0295005 --- /dev/null +++ b/internal/platform/postgres/migrations/000005_source_scans.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS source_scans ( + id VARCHAR(64) PRIMARY KEY, + source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT, + source_type VARCHAR(32) NOT NULL CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')), + state VARCHAR(16) NOT NULL CHECK (state IN ('pending', 'running', 'completed', 'failed')), + snapshot_sha256 VARCHAR(64) NOT NULL DEFAULT '' CHECK (snapshot_sha256 = '' OR snapshot_sha256 ~ '^[0-9a-f]{64}$'), + observed_items INTEGER NOT NULL DEFAULT 0 CHECK (observed_items >= 0), + unchanged_items INTEGER NOT NULL DEFAULT 0 CHECK (unchanged_items >= 0), + added_items INTEGER NOT NULL DEFAULT 0 CHECK (added_items >= 0), + updated_items INTEGER NOT NULL DEFAULT 0 CHECK (updated_items >= 0), + over_limit_items INTEGER NOT NULL DEFAULT 0 CHECK (over_limit_items >= 0), + missing_items INTEGER NOT NULL DEFAULT 0 CHECK (missing_items >= 0), + planned_uploads INTEGER NOT NULL DEFAULT 0 CHECK (planned_uploads >= 0), + error_code VARCHAR(64) NOT NULL DEFAULT '', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_source_scans_one_active + ON source_scans (source_connection_id) + WHERE state IN ('pending', 'running'); + +CREATE INDEX IF NOT EXISTS idx_source_scans_connection_created + ON source_scans (source_connection_id, created_at DESC, id DESC); + +CREATE TABLE IF NOT EXISTS source_scan_changes ( + id VARCHAR(64) PRIMARY KEY, + source_scan_id VARCHAR(64) NOT NULL REFERENCES source_scans(id) ON DELETE RESTRICT, + source_connection_id VARCHAR(64) NOT NULL REFERENCES source_connections(id) ON DELETE RESTRICT, + stable_external_id VARCHAR(512) NOT NULL, + source_item_id VARCHAR(64) REFERENCES source_items(id) ON DELETE RESTRICT, + change_type VARCHAR(16) NOT NULL CHECK (change_type IN ('added', 'updated', 'unavailable')), + previous_revision VARCHAR(512) NOT NULL DEFAULT '', + observed_revision VARCHAR(512) NOT NULL DEFAULT '', + observed_size_bytes BIGINT CHECK (observed_size_bytes IS NULL OR observed_size_bytes >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_scan_id, stable_external_id) +); + +CREATE INDEX IF NOT EXISTS idx_source_scan_changes_pending + ON source_scan_changes (source_connection_id, change_type, created_at DESC, id DESC); diff --git a/internal/platform/postgres/migrations/000006_review_and_seal.down.sql b/internal/platform/postgres/migrations/000006_review_and_seal.down.sql new file mode 100644 index 0000000..f3450c6 --- /dev/null +++ b/internal/platform/postgres/migrations/000006_review_and_seal.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS release_batch_items; +DROP TABLE IF EXISTS release_batches; +DROP TABLE IF EXISTS publication_reviews; diff --git a/internal/platform/postgres/migrations/000006_review_and_seal.up.sql b/internal/platform/postgres/migrations/000006_review_and_seal.up.sql new file mode 100644 index 0000000..7cde772 --- /dev/null +++ b/internal/platform/postgres/migrations/000006_review_and_seal.up.sql @@ -0,0 +1,58 @@ +CREATE TABLE IF NOT EXISTS publication_reviews ( + id VARCHAR(36) PRIMARY KEY, + publication_id VARCHAR(36) NOT NULL + REFERENCES test_publications(id) ON DELETE RESTRICT, + decision VARCHAR(16) NOT NULL + CHECK (decision IN ('approved', 'rejected')), + decided_visibility VARCHAR(20) NOT NULL + CHECK (decided_visibility IN ('internal', 'external')), + reviewer_ref VARCHAR(128) NOT NULL, + decision_code VARCHAR(64) NOT NULL + CHECK (decision_code ~ '^[a-z0-9_]+$'), + acceptance_run_id VARCHAR(36) NOT NULL, + publication_updated_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (publication_id, acceptance_run_id, publication_updated_at, + reviewer_ref, decision, decided_visibility, decision_code) +); + +CREATE INDEX IF NOT EXISTS idx_publication_reviews_latest + ON publication_reviews (publication_id, created_at DESC, id DESC); + +CREATE TABLE IF NOT EXISTS release_batches ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + state VARCHAR(16) NOT NULL DEFAULT 'draft' + CHECK (state IN ('draft', 'sealed', 'cancelled')), + policy_version VARCHAR(64) NOT NULL, + created_by VARCHAR(128) NOT NULL, + sealed_by VARCHAR(128) NOT NULL DEFAULT '', + item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sealed_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_release_batches_state_created + ON release_batches (state, created_at DESC, id DESC); + +CREATE TABLE IF NOT EXISTS release_batch_items ( + release_batch_id VARCHAR(36) NOT NULL + REFERENCES release_batches(id) ON DELETE RESTRICT, + publication_id VARCHAR(36) NOT NULL + REFERENCES test_publications(id) ON DELETE RESTRICT, + review_id VARCHAR(36) NOT NULL + REFERENCES publication_reviews(id) ON DELETE RESTRICT, + source_item_version_id VARCHAR(36) NOT NULL + REFERENCES source_item_versions(id) ON DELETE RESTRICT, + acceptance_run_id VARCHAR(36) NOT NULL, + publication_updated_at TIMESTAMPTZ NOT NULL, + decided_visibility VARCHAR(20) NOT NULL + CHECK (decided_visibility IN ('internal', 'external')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (release_batch_id, publication_id), + UNIQUE (publication_id), + UNIQUE (review_id) +); + +CREATE INDEX IF NOT EXISTS idx_release_batch_items_batch + ON release_batch_items (release_batch_id, publication_id); diff --git a/internal/platform/postgres/migrations/000007_release_executions.down.sql b/internal/platform/postgres/migrations/000007_release_executions.down.sql new file mode 100644 index 0000000..6f226a0 --- /dev/null +++ b/internal/platform/postgres/migrations/000007_release_executions.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS release_item_executions; +DROP TABLE IF EXISTS release_executions; diff --git a/internal/platform/postgres/migrations/000007_release_executions.up.sql b/internal/platform/postgres/migrations/000007_release_executions.up.sql new file mode 100644 index 0000000..0b50e9d --- /dev/null +++ b/internal/platform/postgres/migrations/000007_release_executions.up.sql @@ -0,0 +1,65 @@ +CREATE TABLE IF NOT EXISTS release_executions ( + id VARCHAR(36) PRIMARY KEY, + release_batch_id VARCHAR(36) NOT NULL + REFERENCES release_batches(id) ON DELETE RESTRICT, + operation VARCHAR(16) NOT NULL + CHECK (operation IN ('publish', 'rollback')), + prior_execution_id VARCHAR(36) + REFERENCES release_executions(id) ON DELETE RESTRICT, + state VARCHAR(16) NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'running', 'completed', 'failed')), + target_environment VARCHAR(16) NOT NULL DEFAULT 'production' + CHECK (target_environment = 'production'), + authorized_by VARCHAR(128) NOT NULL, + authorization_code VARCHAR(64) NOT NULL, + job_id VARCHAR(36) NOT NULL UNIQUE + REFERENCES jobs(id) ON DELETE RESTRICT, + manifest_sha256 CHAR(64) NOT NULL DEFAULT '' + CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'), + item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0), + completed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (completed_item_count >= 0), + failed_item_count INTEGER NOT NULL DEFAULT 0 CHECK (failed_item_count >= 0), + error_code VARCHAR(64) NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + CHECK (operation = 'publish' OR prior_execution_id IS NOT NULL), + CHECK (completed_item_count + failed_item_count <= item_count) +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_release_publish_batch + ON release_executions (release_batch_id) + WHERE operation = 'publish'; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_release_rollback_prior + ON release_executions (prior_execution_id) + WHERE operation = 'rollback'; + +CREATE INDEX IF NOT EXISTS idx_release_executions_batch_created + ON release_executions (release_batch_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_release_executions_state_created + ON release_executions (state, created_at); + +CREATE TABLE IF NOT EXISTS release_item_executions ( + id VARCHAR(36) PRIMARY KEY, + release_execution_id VARCHAR(36) NOT NULL + REFERENCES release_executions(id) ON DELETE RESTRICT, + publication_id VARCHAR(36) NOT NULL + REFERENCES test_publications(id) ON DELETE RESTRICT, + state VARCHAR(24) NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'staged', 'activated', 'verified', 'failed', 'rolled_back')), + bundle_sha256 CHAR(64) NOT NULL DEFAULT '' + CHECK (bundle_sha256 = '' OR bundle_sha256 ~ '^[0-9a-f]{64}$'), + previous_production_revision VARCHAR(128) NOT NULL DEFAULT '', + production_revision VARCHAR(128) NOT NULL DEFAULT '', + error_code VARCHAR(64) NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + UNIQUE (release_execution_id, publication_id) +); + +CREATE INDEX IF NOT EXISTS idx_release_item_executions_execution + ON release_item_executions (release_execution_id, created_at); +CREATE INDEX IF NOT EXISTS idx_release_item_executions_publication + ON release_item_executions (publication_id, created_at DESC); diff --git a/internal/platform/postgres/migrations/000008_scope_rules.down.sql b/internal/platform/postgres/migrations/000008_scope_rules.down.sql new file mode 100644 index 0000000..31a67cf --- /dev/null +++ b/internal/platform/postgres/migrations/000008_scope_rules.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS scope_change_events; +DROP TABLE IF EXISTS source_item_scope_overrides; +DROP TABLE IF EXISTS directory_scope_rules; diff --git a/internal/platform/postgres/migrations/000008_scope_rules.up.sql b/internal/platform/postgres/migrations/000008_scope_rules.up.sql new file mode 100644 index 0000000..73c001d --- /dev/null +++ b/internal/platform/postgres/migrations/000008_scope_rules.up.sql @@ -0,0 +1,57 @@ +CREATE TABLE IF NOT EXISTS directory_scope_rules ( + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE RESTRICT, + directory_path TEXT NOT NULL DEFAULT '', + visibility VARCHAR(20) NOT NULL + CHECK (visibility IN ('internal', 'external')), + updated_by VARCHAR(128) NOT NULL, + reason VARCHAR(500) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source_connection_id, directory_path) +); + +CREATE TABLE IF NOT EXISTS source_item_scope_overrides ( + source_item_id VARCHAR(36) NOT NULL + REFERENCES source_items(id) ON DELETE RESTRICT, + visibility VARCHAR(20) NOT NULL + CHECK (visibility IN ('internal', 'external')), + updated_by VARCHAR(128) NOT NULL, + reason VARCHAR(500) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source_item_id) +); + +CREATE TABLE IF NOT EXISTS scope_change_events ( + id VARCHAR(36) PRIMARY KEY, + target_type VARCHAR(16) NOT NULL + CHECK (target_type IN ('directory', 'item')), + source_connection_id VARCHAR(36) + REFERENCES source_connections(id) ON DELETE RESTRICT, + directory_path TEXT NOT NULL DEFAULT '', + source_item_id VARCHAR(36) + REFERENCES source_items(id) ON DELETE RESTRICT, + action VARCHAR(16) NOT NULL + CHECK (action IN ('set', 'clear')), + previous_visibility VARCHAR(20) + CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external')), + decided_visibility VARCHAR(20) + CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external')), + operator_ref VARCHAR(128) NOT NULL, + reason VARCHAR(500) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK ( + (target_type = 'directory' AND source_connection_id IS NOT NULL AND source_item_id IS NULL) + OR + (target_type = 'item' AND source_connection_id IS NULL AND source_item_id IS NOT NULL) + ), + CHECK ( + (action = 'set' AND decided_visibility IS NOT NULL) + OR + (action = 'clear' AND decided_visibility IS NULL) + ) +); + +CREATE INDEX IF NOT EXISTS idx_scope_change_events_target + ON scope_change_events (target_type, source_connection_id, source_item_id, created_at DESC); diff --git a/internal/platform/postgres/migrations/000009_source_catalog.down.sql b/internal/platform/postgres/migrations/000009_source_catalog.down.sql new file mode 100644 index 0000000..9d47216 --- /dev/null +++ b/internal/platform/postgres/migrations/000009_source_catalog.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS source_catalog_heads; +DROP TABLE IF EXISTS source_catalog_nodes; +DROP TABLE IF EXISTS source_catalog_snapshots; diff --git a/internal/platform/postgres/migrations/000009_source_catalog.up.sql b/internal/platform/postgres/migrations/000009_source_catalog.up.sql new file mode 100644 index 0000000..8c556cf --- /dev/null +++ b/internal/platform/postgres/migrations/000009_source_catalog.up.sql @@ -0,0 +1,56 @@ +CREATE TABLE IF NOT EXISTS source_catalog_snapshots ( + id VARCHAR(36) PRIMARY KEY, + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE RESTRICT, + snapshot_sha256 CHAR(64) NOT NULL + CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'), + source_label VARCHAR(255) NOT NULL, + node_count INTEGER NOT NULL CHECK (node_count >= 1), + folder_count INTEGER NOT NULL CHECK (folder_count >= 1), + file_count INTEGER NOT NULL CHECK (file_count >= 0), + observed_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (source_connection_id, snapshot_sha256), + UNIQUE (source_connection_id, id) +); + +CREATE TABLE IF NOT EXISTS source_catalog_nodes ( + snapshot_id VARCHAR(36) NOT NULL + REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT, + node_id VARCHAR(36) NOT NULL, + parent_node_id VARCHAR(36), + node_type VARCHAR(16) NOT NULL + CHECK (node_type IN ('folder', 'file')), + name TEXT NOT NULL CHECK (name <> '' AND length(name) <= 1024), + relative_path TEXT NOT NULL CHECK (length(relative_path) <= 4096), + source_locator_hash CHAR(64) NOT NULL + CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'), + source_item_id VARCHAR(36) + REFERENCES source_items(id) ON DELETE RESTRICT, + size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes >= 0), + modified_at TIMESTAMPTZ, + sort_order INTEGER NOT NULL DEFAULT 0, + direct_child_count INTEGER NOT NULL DEFAULT 0 CHECK (direct_child_count >= 0), + subtree_file_count INTEGER NOT NULL DEFAULT 0 CHECK (subtree_file_count >= 0), + subtree_bytes BIGINT NOT NULL DEFAULT 0 CHECK (subtree_bytes >= 0), + PRIMARY KEY (snapshot_id, node_id), + FOREIGN KEY (snapshot_id, parent_node_id) + REFERENCES source_catalog_nodes(snapshot_id, node_id) ON DELETE RESTRICT + DEFERRABLE INITIALLY DEFERRED +); + +CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_parent + ON source_catalog_nodes (snapshot_id, parent_node_id, node_type, sort_order, name, node_id); + +CREATE INDEX IF NOT EXISTS idx_source_catalog_nodes_source_item + ON source_catalog_nodes (source_item_id) + WHERE source_item_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS source_catalog_heads ( + source_connection_id VARCHAR(36) PRIMARY KEY + REFERENCES source_connections(id) ON DELETE RESTRICT, + snapshot_id VARCHAR(36) NOT NULL, + activated_at TIMESTAMPTZ NOT NULL, + FOREIGN KEY (source_connection_id, snapshot_id) + REFERENCES source_catalog_snapshots(source_connection_id, id) ON DELETE RESTRICT +); diff --git a/internal/platform/postgres/migrations/000010_catalog_policies.down.sql b/internal/platform/postgres/migrations/000010_catalog_policies.down.sql new file mode 100644 index 0000000..ddd71fd --- /dev/null +++ b/internal/platform/postgres/migrations/000010_catalog_policies.down.sql @@ -0,0 +1,36 @@ +UPDATE scope_change_events +SET previous_visibility = 'internal' +WHERE previous_visibility IN ('both', 'blocked'); +UPDATE scope_change_events +SET decided_visibility = 'internal' +WHERE decided_visibility IN ('both', 'blocked'); +UPDATE source_item_scope_overrides +SET visibility = 'internal' +WHERE visibility IN ('both', 'blocked'); +UPDATE directory_scope_rules +SET visibility = 'internal' +WHERE visibility IN ('both', 'blocked'); + +ALTER TABLE scope_change_events + DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check; +ALTER TABLE scope_change_events + ADD CONSTRAINT scope_change_events_decided_visibility_check + CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external')); + +ALTER TABLE scope_change_events + DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check; +ALTER TABLE scope_change_events + ADD CONSTRAINT scope_change_events_previous_visibility_check + CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external')); + +ALTER TABLE source_item_scope_overrides + DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check; +ALTER TABLE source_item_scope_overrides + ADD CONSTRAINT source_item_scope_overrides_visibility_check + CHECK (visibility IN ('internal', 'external')); + +ALTER TABLE directory_scope_rules + DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check; +ALTER TABLE directory_scope_rules + ADD CONSTRAINT directory_scope_rules_visibility_check + CHECK (visibility IN ('internal', 'external')); diff --git a/internal/platform/postgres/migrations/000010_catalog_policies.up.sql b/internal/platform/postgres/migrations/000010_catalog_policies.up.sql new file mode 100644 index 0000000..6559523 --- /dev/null +++ b/internal/platform/postgres/migrations/000010_catalog_policies.up.sql @@ -0,0 +1,25 @@ +-- Visibility policies used by the source catalog. Publication reviews remain +-- scoped to one concrete internal or external test publication. +ALTER TABLE directory_scope_rules + DROP CONSTRAINT IF EXISTS directory_scope_rules_visibility_check; +ALTER TABLE directory_scope_rules + ADD CONSTRAINT directory_scope_rules_visibility_check + CHECK (visibility IN ('internal', 'external', 'both', 'blocked')); + +ALTER TABLE source_item_scope_overrides + DROP CONSTRAINT IF EXISTS source_item_scope_overrides_visibility_check; +ALTER TABLE source_item_scope_overrides + ADD CONSTRAINT source_item_scope_overrides_visibility_check + CHECK (visibility IN ('internal', 'external', 'both', 'blocked')); + +ALTER TABLE scope_change_events + DROP CONSTRAINT IF EXISTS scope_change_events_previous_visibility_check; +ALTER TABLE scope_change_events + ADD CONSTRAINT scope_change_events_previous_visibility_check + CHECK (previous_visibility IS NULL OR previous_visibility IN ('internal', 'external', 'both', 'blocked')); + +ALTER TABLE scope_change_events + DROP CONSTRAINT IF EXISTS scope_change_events_decided_visibility_check; +ALTER TABLE scope_change_events + ADD CONSTRAINT scope_change_events_decided_visibility_check + CHECK (decided_visibility IS NULL OR decided_visibility IN ('internal', 'external', 'both', 'blocked')); diff --git a/internal/platform/postgres/migrations/000011_processing_confirmations.down.sql b/internal/platform/postgres/migrations/000011_processing_confirmations.down.sql new file mode 100644 index 0000000..a3b4c2f --- /dev/null +++ b/internal/platform/postgres/migrations/000011_processing_confirmations.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS processing_confirmations; +ALTER TABLE test_publications DROP COLUMN IF EXISTS target_scope; diff --git a/internal/platform/postgres/migrations/000011_processing_confirmations.up.sql b/internal/platform/postgres/migrations/000011_processing_confirmations.up.sql new file mode 100644 index 0000000..8ac8d39 --- /dev/null +++ b/internal/platform/postgres/migrations/000011_processing_confirmations.up.sql @@ -0,0 +1,36 @@ +ALTER TABLE test_publications + ADD COLUMN IF NOT EXISTS target_scope VARCHAR(16) NOT NULL DEFAULT 'internal' + CHECK (target_scope IN ('internal', 'external')); + +UPDATE test_publications AS publication +SET target_scope = COALESCE( + ( + SELECT review.decided_visibility + FROM publication_reviews AS review + WHERE review.publication_id = publication.id + AND review.decision = 'approved' + AND review.decided_visibility IN ('internal', 'external') + ORDER BY review.created_at DESC, review.id DESC + LIMIT 1 + ), + CASE WHEN item_rule.visibility IN ('internal', 'external') THEN item_rule.visibility END, + source.visibility +) +FROM source_item_versions AS version +JOIN source_items AS source ON source.id = version.source_item_id +LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id +WHERE publication.source_item_version_id = version.id; + +CREATE TABLE IF NOT EXISTS processing_confirmations ( + id VARCHAR(36) PRIMARY KEY, + publication_id VARCHAR(36) NOT NULL + REFERENCES test_publications(id) ON DELETE RESTRICT, + operator_ref VARCHAR(128) NOT NULL, + reason VARCHAR(500) NOT NULL, + confirmed_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (publication_id) +); + +CREATE INDEX IF NOT EXISTS idx_processing_confirmations_confirmed + ON processing_confirmations (confirmed_at DESC, publication_id); diff --git a/internal/platform/postgres/migrations/000012_release_actions.down.sql b/internal/platform/postgres/migrations/000012_release_actions.down.sql new file mode 100644 index 0000000..cb44e9e --- /dev/null +++ b/internal/platform/postgres/migrations/000012_release_actions.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE release_batch_items + ADD CONSTRAINT release_batch_items_publication_id_key UNIQUE (publication_id); +ALTER TABLE release_batch_items + ADD CONSTRAINT release_batch_items_review_id_key UNIQUE (review_id); + +ALTER TABLE release_item_executions DROP COLUMN IF EXISTS action; +ALTER TABLE release_batch_items DROP COLUMN IF EXISTS action; diff --git a/internal/platform/postgres/migrations/000012_release_actions.up.sql b/internal/platform/postgres/migrations/000012_release_actions.up.sql new file mode 100644 index 0000000..ce9b719 --- /dev/null +++ b/internal/platform/postgres/migrations/000012_release_actions.up.sql @@ -0,0 +1,12 @@ +ALTER TABLE release_batch_items + ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert' + CHECK (action IN ('upsert', 'delete')); + +ALTER TABLE release_item_executions + ADD COLUMN IF NOT EXISTS action VARCHAR(16) NOT NULL DEFAULT 'upsert' + CHECK (action IN ('upsert', 'delete')); + +ALTER TABLE release_batch_items + DROP CONSTRAINT IF EXISTS release_batch_items_publication_id_key; +ALTER TABLE release_batch_items + DROP CONSTRAINT IF EXISTS release_batch_items_review_id_key; diff --git a/internal/platform/postgres/migrations/000013_ui_system_settings.down.sql b/internal/platform/postgres/migrations/000013_ui_system_settings.down.sql new file mode 100644 index 0000000..0f15b77 --- /dev/null +++ b/internal/platform/postgres/migrations/000013_ui_system_settings.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ui_setting_revisions; diff --git a/internal/platform/postgres/migrations/000013_ui_system_settings.up.sql b/internal/platform/postgres/migrations/000013_ui_system_settings.up.sql new file mode 100644 index 0000000..3be4f45 --- /dev/null +++ b/internal/platform/postgres/migrations/000013_ui_system_settings.up.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS ui_setting_revisions ( + id VARCHAR(36) PRIMARY KEY, + state VARCHAR(16) NOT NULL CHECK (state IN ('validated', 'active', 'retired', 'failed')), + document_refresh_seconds INTEGER NOT NULL CHECK (document_refresh_seconds IN (15, 30, 60)), + resource_refresh_seconds INTEGER NOT NULL CHECK (resource_refresh_seconds IN (30, 60)), + stalled_after_seconds INTEGER NOT NULL CHECK (stalled_after_seconds IN (1800, 3600)), + finalizing_after_seconds INTEGER NOT NULL CHECK (finalizing_after_seconds IN (7200, 14400)), + updated_by VARCHAR(128) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + activated_at TIMESTAMPTZ, + retired_at TIMESTAMPTZ, + error_code VARCHAR(64) NOT NULL DEFAULT '', + CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$') +); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_ui_setting_active + ON ui_setting_revisions ((state)) WHERE state = 'active'; + +INSERT INTO ui_setting_revisions ( + id, state, document_refresh_seconds, resource_refresh_seconds, + stalled_after_seconds, finalizing_after_seconds, updated_by, activated_at +) VALUES ( + 'settings-default', 'active', 15, 30, 1800, 7200, 'system:migration', NOW() +) ON CONFLICT (id) DO NOTHING; diff --git a/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.down.sql b/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.down.sql new file mode 100644 index 0000000..e83fecf --- /dev/null +++ b/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE jobs + DROP COLUMN IF EXISTS started_at; diff --git a/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.up.sql b/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.up.sql new file mode 100644 index 0000000..a3d575b --- /dev/null +++ b/internal/platform/postgres/migrations/000014_catalog_refresh_started_at.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE jobs + ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ; diff --git a/internal/platform/postgres/migrations/000015_catalog_change_sets.down.sql b/internal/platform/postgres/migrations/000015_catalog_change_sets.down.sql new file mode 100644 index 0000000..149046c --- /dev/null +++ b/internal/platform/postgres/migrations/000015_catalog_change_sets.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS source_catalog_change_items; +DROP TABLE IF EXISTS source_catalog_change_sets; diff --git a/internal/platform/postgres/migrations/000015_catalog_change_sets.up.sql b/internal/platform/postgres/migrations/000015_catalog_change_sets.up.sql new file mode 100644 index 0000000..aaf0c96 --- /dev/null +++ b/internal/platform/postgres/migrations/000015_catalog_change_sets.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE IF NOT EXISTS source_catalog_change_sets ( + id VARCHAR(36) PRIMARY KEY, + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE RESTRICT, + from_snapshot_id VARCHAR(36) + REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT, + to_snapshot_id VARCHAR(36) NOT NULL UNIQUE + REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT, + state VARCHAR(16) NOT NULL DEFAULT 'planned' + CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled')), + add_count INTEGER NOT NULL DEFAULT 0 CHECK (add_count >= 0), + update_count INTEGER NOT NULL DEFAULT 0 CHECK (update_count >= 0), + move_scope_count INTEGER NOT NULL DEFAULT 0 CHECK (move_scope_count >= 0), + delete_count INTEGER NOT NULL DEFAULT 0 CHECK (delete_count >= 0), + no_change_count INTEGER NOT NULL DEFAULT 0 CHECK (no_change_count >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + confirmed_at TIMESTAMPTZ, + applied_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS source_catalog_change_items ( + change_set_id VARCHAR(36) NOT NULL + REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT, + source_locator_hash CHAR(64) NOT NULL + CHECK (source_locator_hash ~ '^[0-9a-f]{64}$'), + action VARCHAR(16) NOT NULL + CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change')), + title TEXT NOT NULL, + old_path TEXT NOT NULL DEFAULT '', + new_path TEXT NOT NULL DEFAULT '', + old_size_bytes BIGINT CHECK (old_size_bytes IS NULL OR old_size_bytes >= 0), + new_size_bytes BIGINT CHECK (new_size_bytes IS NULL OR new_size_bytes >= 0), + old_modified_at TIMESTAMPTZ, + new_modified_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (change_set_id, source_locator_hash) +); + +CREATE INDEX IF NOT EXISTS idx_source_catalog_change_sets_source + ON source_catalog_change_sets (source_connection_id, created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_source_catalog_change_items_action + ON source_catalog_change_items (change_set_id, action, title); diff --git a/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.down.sql b/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.down.sql new file mode 100644 index 0000000..91ec559 --- /dev/null +++ b/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.down.sql @@ -0,0 +1,4 @@ +DELETE FROM source_catalog_change_items +WHERE change_set_id IN (SELECT id FROM source_catalog_change_sets WHERE origin='backfill'); +DELETE FROM source_catalog_change_sets WHERE origin='backfill'; +ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS origin; diff --git a/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.up.sql b/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.up.sql new file mode 100644 index 0000000..610cf20 --- /dev/null +++ b/internal/platform/postgres/migrations/000016_catalog_change_set_backfill.up.sql @@ -0,0 +1,87 @@ +ALTER TABLE source_catalog_change_sets + ADD COLUMN IF NOT EXISTS origin VARCHAR(16) NOT NULL DEFAULT 'refresh' + CHECK (origin IN ('refresh', 'backfill')); + +WITH current_targets AS ( + SELECT head.source_connection_id,head.snapshot_id AS to_snapshot_id, + ( + SELECT prior.id + FROM source_catalog_snapshots prior + JOIN source_catalog_snapshots current ON current.id=head.snapshot_id + WHERE prior.source_connection_id=head.source_connection_id + AND prior.id<>head.snapshot_id + AND prior.observed_at<=current.observed_at + ORDER BY prior.observed_at DESC,prior.id DESC + LIMIT 1 + ) AS from_snapshot_id + FROM source_catalog_heads head +) +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,origin,created_at) +SELECT 'cs-'||md5(target.source_connection_id||'|'||target.to_snapshot_id), + target.source_connection_id,target.from_snapshot_id,target.to_snapshot_id,'planned','backfill',NOW() +FROM current_targets target +ON CONFLICT (to_snapshot_id) DO NOTHING; + +WITH change_sets AS ( + SELECT id,from_snapshot_id,to_snapshot_id + FROM source_catalog_change_sets + WHERE origin='backfill' +), old_files AS ( + SELECT DISTINCT ON (change_set.id,node.source_locator_hash) + change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at + FROM change_sets change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file' + ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id +), new_files AS ( + SELECT DISTINCT ON (change_set.id,node.source_locator_hash) + change_set.id AS change_set_id,node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at + FROM change_sets change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file' + ORDER BY change_set.id,node.source_locator_hash,node.relative_path,node.node_id +), compared AS ( + SELECT COALESCE(new_files.change_set_id,old_files.change_set_id) AS change_set_id, + COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash, + COALESCE(new_files.name,old_files.name) AS title, + COALESCE(old_files.relative_path,'') AS old_path, + COALESCE(new_files.relative_path,'') AS new_path, + old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes, + old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at, + CASE + WHEN old_files.source_locator_hash IS NULL THEN 'add' + WHEN new_files.source_locator_hash IS NULL THEN 'delete' + WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path + OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope' + WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes + OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update' + ELSE 'no_change' + END AS action + FROM old_files FULL OUTER JOIN new_files + ON new_files.change_set_id=old_files.change_set_id + AND new_files.source_locator_hash=old_files.source_locator_hash +) +INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path, + old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,created_at) +SELECT change_set_id,source_locator_hash,action,title,old_path,new_path, + old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,NOW() +FROM compared +ON CONFLICT (change_set_id,source_locator_hash) DO NOTHING; + +UPDATE source_catalog_change_sets change_set SET + add_count=summary.add_count, + update_count=summary.update_count, + move_scope_count=summary.move_scope_count, + delete_count=summary.delete_count, + no_change_count=summary.no_change_count +FROM ( + SELECT change_set.id AS change_set_id, + COUNT(item.*) FILTER (WHERE item.action='add')::integer AS add_count, + COUNT(item.*) FILTER (WHERE item.action='update')::integer AS update_count, + COUNT(item.*) FILTER (WHERE item.action='move_scope')::integer AS move_scope_count, + COUNT(item.*) FILTER (WHERE item.action='delete')::integer AS delete_count, + COUNT(item.*) FILTER (WHERE item.action='no_change')::integer AS no_change_count + FROM source_catalog_change_sets change_set + LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id + WHERE change_set.origin='backfill' + GROUP BY change_set.id +) summary +WHERE change_set.id=summary.change_set_id; diff --git a/internal/platform/postgres/migrations/000017_release_batch_progress.down.sql b/internal/platform/postgres/migrations/000017_release_batch_progress.down.sql new file mode 100644 index 0000000..ddd83b3 --- /dev/null +++ b/internal/platform/postgres/migrations/000017_release_batch_progress.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE release_executions + DROP CONSTRAINT IF EXISTS release_execution_batch_progress_check, + DROP COLUMN IF EXISTS total_batches, + DROP COLUMN IF EXISTS current_batch, + DROP COLUMN IF EXISTS batch_size; diff --git a/internal/platform/postgres/migrations/000017_release_batch_progress.up.sql b/internal/platform/postgres/migrations/000017_release_batch_progress.up.sql new file mode 100644 index 0000000..8f93a10 --- /dev/null +++ b/internal/platform/postgres/migrations/000017_release_batch_progress.up.sql @@ -0,0 +1,16 @@ +ALTER TABLE release_executions + ADD COLUMN IF NOT EXISTS batch_size INTEGER NOT NULL DEFAULT 50 + CHECK (batch_size BETWEEN 1 AND 100), + ADD COLUMN IF NOT EXISTS current_batch INTEGER NOT NULL DEFAULT 0 + CHECK (current_batch >= 0), + ADD COLUMN IF NOT EXISTS total_batches INTEGER NOT NULL DEFAULT 0 + CHECK (total_batches >= 0), + ADD CONSTRAINT release_execution_batch_progress_check + CHECK (current_batch <= total_batches OR total_batches = 0); + +UPDATE release_executions +SET total_batches = CASE WHEN item_count = 0 THEN 0 ELSE CEIL(item_count::numeric / batch_size)::integer END, + current_batch = CASE + WHEN state IN ('completed', 'failed') AND item_count > 0 THEN CEIL(item_count::numeric / batch_size)::integer + ELSE current_batch + END; diff --git a/internal/platform/postgres/migrations/000018_job_cancelled_state.down.sql b/internal/platform/postgres/migrations/000018_job_cancelled_state.down.sql new file mode 100644 index 0000000..bebe7d9 --- /dev/null +++ b/internal/platform/postgres/migrations/000018_job_cancelled_state.down.sql @@ -0,0 +1,11 @@ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM jobs WHERE state = 'cancelled') THEN + RAISE EXCEPTION 'cannot roll back job cancelled state while cancelled jobs exist'; + END IF; +END +$$; + +ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check; +ALTER TABLE jobs ADD CONSTRAINT jobs_state_check + CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed')); diff --git a/internal/platform/postgres/migrations/000018_job_cancelled_state.up.sql b/internal/platform/postgres/migrations/000018_job_cancelled_state.up.sql new file mode 100644 index 0000000..41fa646 --- /dev/null +++ b/internal/platform/postgres/migrations/000018_job_cancelled_state.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE jobs DROP CONSTRAINT IF EXISTS jobs_state_check; +ALTER TABLE jobs ADD CONSTRAINT jobs_state_check + CHECK (state IN ('pending', 'running', 'retry', 'completed', 'failed', 'cancelled')); diff --git a/internal/platform/postgres/migrations/000019_operation_requests.down.sql b/internal/platform/postgres/migrations/000019_operation_requests.down.sql new file mode 100644 index 0000000..508fc68 --- /dev/null +++ b/internal/platform/postgres/migrations/000019_operation_requests.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS operation_requests; diff --git a/internal/platform/postgres/migrations/000019_operation_requests.up.sql b/internal/platform/postgres/migrations/000019_operation_requests.up.sql new file mode 100644 index 0000000..b3bbd78 --- /dev/null +++ b/internal/platform/postgres/migrations/000019_operation_requests.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS operation_requests ( + operation_type TEXT NOT NULL CHECK (operation_type <> '' AND char_length(operation_type) <= 100), + operation_id TEXT NOT NULL CHECK (operation_id <> '' AND char_length(operation_id) <= 128), + request_fingerprint_sha256 CHAR(64) NOT NULL CHECK (request_fingerprint_sha256 ~ '^[0-9a-f]{64}$'), + state TEXT NOT NULL DEFAULT 'completed' CHECK (state IN ('completed')), + result JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ NOT NULL, + UNIQUE (operation_type, operation_id) +); diff --git a/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.down.sql b/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.down.sql new file mode 100644 index 0000000..7eb61bc --- /dev/null +++ b/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.down.sql @@ -0,0 +1 @@ +ALTER TABLE source_catalog_nodes DROP COLUMN IF EXISTS content_sha256; diff --git a/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.up.sql b/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.up.sql new file mode 100644 index 0000000..cc04076 --- /dev/null +++ b/internal/platform/postgres/migrations/000020_catalog_snapshot_content_hash.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE source_catalog_nodes + ADD COLUMN IF NOT EXISTS content_sha256 CHAR(64) + CHECK (content_sha256 IS NULL OR content_sha256 ~ '^[0-9a-f]{64}$'); diff --git a/internal/platform/postgres/migrations/000021_source_identity_evidence.down.sql b/internal/platform/postgres/migrations/000021_source_identity_evidence.down.sql new file mode 100644 index 0000000..620a471 --- /dev/null +++ b/internal/platform/postgres/migrations/000021_source_identity_evidence.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS reason_code, DROP COLUMN IF EXISTS new_content_sha256, DROP COLUMN IF EXISTS old_content_sha256; +ALTER TABLE source_items DROP COLUMN IF EXISTS identity_confidence; diff --git a/internal/platform/postgres/migrations/000021_source_identity_evidence.up.sql b/internal/platform/postgres/migrations/000021_source_identity_evidence.up.sql new file mode 100644 index 0000000..5388e95 --- /dev/null +++ b/internal/platform/postgres/migrations/000021_source_identity_evidence.up.sql @@ -0,0 +1,12 @@ +-- P2 stable identity: source items record how strongly their identity is anchored. +ALTER TABLE source_items + ADD COLUMN IF NOT EXISTS identity_confidence VARCHAR(16) NOT NULL DEFAULT 'path' + CHECK (identity_confidence IN ('stable_id','object_id','content_hash','path')); + +-- Change items gain content-hash evidence and a reason code so a rename can be +-- classified as a proven move instead of delete+add when the bytes match 1:1. +ALTER TABLE source_catalog_change_items + ADD COLUMN IF NOT EXISTS old_content_sha256 CHAR(64) CHECK (old_content_sha256 IS NULL OR old_content_sha256 ~ '^[0-9a-f]{64}$'), + ADD COLUMN IF NOT EXISTS new_content_sha256 CHAR(64) CHECK (new_content_sha256 IS NULL OR new_content_sha256 ~ '^[0-9a-f]{64}$'), + ADD COLUMN IF NOT EXISTS reason_code VARCHAR(32) NOT NULL DEFAULT 'locator' + CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing')); diff --git a/internal/platform/postgres/migrations/000022_catalog_change_comparison.down.sql b/internal/platform/postgres/migrations/000022_catalog_change_comparison.down.sql new file mode 100644 index 0000000..baa52cf --- /dev/null +++ b/internal/platform/postgres/migrations/000022_catalog_change_comparison.down.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR); diff --git a/internal/platform/postgres/migrations/000022_catalog_change_comparison.up.sql b/internal/platform/postgres/migrations/000022_catalog_change_comparison.up.sql new file mode 100644 index 0000000..e19c8d1 --- /dev/null +++ b/internal/platform/postgres/migrations/000022_catalog_change_comparison.up.sql @@ -0,0 +1,99 @@ +-- P2/S8: canonical snapshot-diff classifier lives in the database so the +-- importer, backfills and future ChangeSet execution share one source of truth. +-- Two sequential statements (not a data-modifying CTE) so the recount sees the rows just written. +CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$ +BEGIN +WITH change_set AS ( + SELECT id,from_snapshot_id,to_snapshot_id FROM source_catalog_change_sets + WHERE id=p_change_set_id +), old_files AS ( + SELECT DISTINCT ON (node.source_locator_hash) + node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256 + FROM change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file' + ORDER BY node.source_locator_hash,node.relative_path,node.node_id +), new_files AS ( + SELECT DISTINCT ON (node.source_locator_hash) + node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256 + FROM change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file' + ORDER BY node.source_locator_hash,node.relative_path,node.node_id +), proven_moves AS ( + -- A content hash proves a move only when exactly one old path carries it and + -- exactly one new path does; copies stay ambiguous and are never auto-merged. + SELECT content_sha256 FROM ( + SELECT old_files.content_sha256 FROM old_files WHERE old_files.content_sha256 IS NOT NULL + GROUP BY old_files.content_sha256 HAVING COUNT(*)=1 + ) one_old + JOIN ( + SELECT new_files.content_sha256 FROM new_files WHERE new_files.content_sha256 IS NOT NULL + GROUP BY new_files.content_sha256 HAVING COUNT(*)=1 + ) one_new USING (content_sha256) +), compared AS ( + SELECT COALESCE(new_files.source_locator_hash,old_files.source_locator_hash) AS source_locator_hash, + COALESCE(new_files.name,old_files.name) AS title, + COALESCE(old_files.relative_path,'') AS old_path, + COALESCE(new_files.relative_path,'') AS new_path, + old_files.size_bytes AS old_size_bytes,new_files.size_bytes AS new_size_bytes, + old_files.modified_at AS old_modified_at,new_files.modified_at AS new_modified_at, + old_files.content_sha256 AS old_content_sha256,new_files.content_sha256 AS new_content_sha256, + CASE + WHEN old_files.source_locator_hash IS NULL THEN 'add' + WHEN new_files.source_locator_hash IS NULL THEN 'delete' + WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path + OR old_files.name IS DISTINCT FROM new_files.name THEN 'move_scope' + WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes + OR old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'update' + ELSE 'no_change' + END AS action, + CASE + WHEN old_files.source_locator_hash IS NULL AND new_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves) + THEN 'content_hash_move' + WHEN new_files.source_locator_hash IS NULL AND old_files.content_sha256 IN (SELECT content_sha256 FROM proven_moves) + THEN 'content_hash_move' + WHEN old_files.source_locator_hash IS NULL THEN 'locator' + WHEN new_files.source_locator_hash IS NULL THEN 'source_missing' + WHEN old_files.content_sha256 IS DISTINCT FROM new_files.content_sha256 + AND (old_files.relative_path<>new_files.relative_path OR old_files.name<>new_files.name) + THEN 'content_hash_conflict' + WHEN old_files.relative_path IS DISTINCT FROM new_files.relative_path OR old_files.name IS DISTINCT FROM new_files.name + THEN 'path_change' + WHEN old_files.size_bytes IS DISTINCT FROM new_files.size_bytes THEN 'size_change' + WHEN old_files.modified_at IS DISTINCT FROM new_files.modified_at THEN 'content_change' + ELSE 'locator' + END AS reason_code + FROM old_files FULL OUTER JOIN new_files USING(source_locator_hash) +) +INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path, + old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code,created_at) +SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title, + compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes, + compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code,NOW() +FROM change_set CROSS JOIN compared +ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET + action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path, + old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes, + old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at, + old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256, + reason_code=EXCLUDED.reason_code; + +UPDATE source_catalog_change_sets change_set SET + add_count=summary.add_count, + update_count=summary.update_count, + move_scope_count=summary.move_scope_count, + delete_count=summary.delete_count, + no_change_count=summary.no_change_count +FROM ( + SELECT items.change_set_id, + COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count, + COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count, + COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count, + COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count, + COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count + FROM source_catalog_change_items items + WHERE items.change_set_id=p_change_set_id + GROUP BY items.change_set_id +) summary +WHERE change_set.id=summary.change_set_id; +END +$$ LANGUAGE plpgsql; diff --git a/internal/platform/postgres/migrations/000023_complete_change_set.down.sql b/internal/platform/postgres/migrations/000023_complete_change_set.down.sql new file mode 100644 index 0000000..6d8c0f5 --- /dev/null +++ b/internal/platform/postgres/migrations/000023_complete_change_set.down.sql @@ -0,0 +1,26 @@ +-- Revert S8 contract. Change rows written with new vocabulary remain but become +-- unclassifiable only if constraints reject them; we keep the checks permissive +-- first, then drop added columns. +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_version_id; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_version_id; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_source_item_id; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_source_item_id; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS new_scope; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS old_scope; +UPDATE source_catalog_change_items SET action='move_scope' WHERE action='move'; +UPDATE source_catalog_change_items SET reason_code='path_change' WHERE reason_code='scope_change'; +ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check; +ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check + CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing')); +ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check; +ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check + CHECK (action IN ('add', 'update', 'move_scope', 'delete', 'no_change')); +ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS failure_reason; +ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS cancelled_at; +ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS applying_at; +UPDATE source_catalog_change_sets SET move_scope_count=move_scope_count+move_count, move_count=0; +ALTER TABLE source_catalog_change_sets DROP COLUMN IF EXISTS move_count; +UPDATE source_catalog_change_sets SET state='confirmed' WHERE state IN ('applying','failed'); +ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check; +ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check + CHECK (state IN ('planned', 'confirmed', 'applied', 'cancelled')); diff --git a/internal/platform/postgres/migrations/000023_complete_change_set.up.sql b/internal/platform/postgres/migrations/000023_complete_change_set.up.sql new file mode 100644 index 0000000..d7835f7 --- /dev/null +++ b/internal/platform/postgres/migrations/000023_complete_change_set.up.sql @@ -0,0 +1,167 @@ +-- S8 complete ChangeSet contract: full lifecycle, distinct move vs move_scope, +-- old/new effective scope, and version identity evidence. +ALTER TABLE source_catalog_change_sets DROP CONSTRAINT IF EXISTS source_catalog_change_sets_state_check; +ALTER TABLE source_catalog_change_sets ADD CONSTRAINT source_catalog_change_sets_state_check + CHECK (state IN ('planned', 'confirmed', 'applying', 'applied', 'cancelled', 'failed')); +ALTER TABLE source_catalog_change_sets + ADD COLUMN IF NOT EXISTS move_count INTEGER NOT NULL DEFAULT 0 CHECK (move_count >= 0), + ADD COLUMN IF NOT EXISTS applying_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS cancelled_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS failure_reason TEXT NOT NULL DEFAULT ''; + +ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_action_check; +ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_action_check + CHECK (action IN ('add', 'update', 'move', 'move_scope', 'delete', 'no_change')); +ALTER TABLE source_catalog_change_items DROP CONSTRAINT IF EXISTS source_catalog_change_items_reason_code_check; +ALTER TABLE source_catalog_change_items ADD CONSTRAINT source_catalog_change_items_reason_code_check + CHECK (reason_code IN ('locator','content_hash_move','content_hash_conflict','content_change','path_change','size_change','source_missing','scope_change')); +ALTER TABLE source_catalog_change_items + ADD COLUMN IF NOT EXISTS old_scope VARCHAR(16), + ADD COLUMN IF NOT EXISTS new_scope VARCHAR(16), + ADD COLUMN IF NOT EXISTS old_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT, + ADD COLUMN IF NOT EXISTS new_source_item_id VARCHAR(36) REFERENCES source_items(id) ON DELETE RESTRICT, + ADD COLUMN IF NOT EXISTS old_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT, + ADD COLUMN IF NOT EXISTS new_version_id VARCHAR(36) REFERENCES source_item_versions(id) ON DELETE RESTRICT; + +-- Canonical classifier v2 (single source of truth; importer and backfills call it). +CREATE OR REPLACE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$ +BEGIN +WITH change_set AS ( + SELECT id,from_snapshot_id,to_snapshot_id,source_connection_id FROM source_catalog_change_sets + WHERE id=p_change_set_id +), old_files AS ( + SELECT DISTINCT ON (node.source_locator_hash) + node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256, + node.source_item_id + FROM change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.from_snapshot_id AND node.node_type='file' + ORDER BY node.source_locator_hash,node.relative_path,node.node_id +), new_files AS ( + SELECT DISTINCT ON (node.source_locator_hash) + node.source_locator_hash,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256, + node.source_item_id + FROM change_set + JOIN source_catalog_nodes node ON node.snapshot_id=change_set.to_snapshot_id AND node.node_type='file' + ORDER BY node.source_locator_hash,node.relative_path,node.node_id +) +-- Old side reports the source-recorded visibility (what the last sync stored); +-- new side reports the currently effective policy (override > item). A policy +-- change between snapshots therefore surfaces as move_scope, not a fake move. +, scoped_old AS ( + SELECT old_files.*, + (SELECT item.visibility FROM source_items item WHERE item.id=old_files.source_item_id) AS item_scope + FROM old_files +), scoped_new AS ( + SELECT new_files.*, + (SELECT COALESCE(override.visibility,item.visibility) + FROM source_items item + LEFT JOIN source_item_scope_overrides override ON override.source_item_id=item.id + WHERE item.id=new_files.source_item_id) AS item_scope + FROM new_files +), with_rule_scope AS ( + SELECT f.*,COALESCE(f.item_scope,( + SELECT rule.visibility FROM directory_scope_rules rule + WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set) + AND (rule.directory_path='' OR f.relative_path=rule.directory_path + OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/') + ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope + FROM (SELECT * FROM scoped_old) f +), with_rule_scope_new AS ( + SELECT f.*,COALESCE(f.item_scope,( + SELECT rule.visibility FROM directory_scope_rules rule + WHERE rule.source_connection_id=(SELECT source_connection_id FROM change_set) + AND (rule.directory_path='' OR f.relative_path=rule.directory_path + OR LEFT(f.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/') + ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1)) AS effective_scope + FROM (SELECT * FROM scoped_new) f +), proven_moves AS ( + -- A content hash proves a move only when exactly one old path carries it and + -- exactly one new path does; copies stay ambiguous and are never auto-merged. + SELECT content_sha256 FROM ( + SELECT content_sha256 FROM scoped_old WHERE content_sha256 IS NOT NULL + GROUP BY content_sha256 HAVING COUNT(*)=1 + ) one_old + JOIN ( + SELECT content_sha256 FROM scoped_new WHERE content_sha256 IS NOT NULL + GROUP BY content_sha256 HAVING COUNT(*)=1 + ) one_new USING (content_sha256) +), compared AS ( + SELECT COALESCE(n.source_locator_hash,o.source_locator_hash) AS source_locator_hash, + COALESCE(n.name,o.name) AS title, + COALESCE(o.relative_path,'') AS old_path, + COALESCE(n.relative_path,'') AS new_path, + o.size_bytes AS old_size_bytes,n.size_bytes AS new_size_bytes, + o.modified_at AS old_modified_at,n.modified_at AS new_modified_at, + o.content_sha256 AS old_content_sha256,n.content_sha256 AS new_content_sha256, + o.effective_scope AS old_scope,n.effective_scope AS new_scope, + o.source_item_id AS old_source_item_id,n.source_item_id AS new_source_item_id, + (SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=o.source_item_id AND i.revision=v.revision WHERE v.source_item_id=o.source_item_id LIMIT 1) AS old_version_id, + (SELECT v.id FROM source_item_versions v JOIN source_items i ON i.id=n.source_item_id AND i.revision=v.revision WHERE v.source_item_id=n.source_item_id LIMIT 1) AS new_version_id, + CASE + WHEN o.source_locator_hash IS NULL THEN 'add' + WHEN n.source_locator_hash IS NULL THEN 'delete' + WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'move_scope' + WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256 + AND (o.relative_path<>n.relative_path OR o.name<>n.name) THEN 'move' + WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256<>n.content_sha256 THEN 'update' + WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'move_scope' + WHEN o.size_bytes IS DISTINCT FROM n.size_bytes OR o.modified_at IS DISTINCT FROM n.modified_at THEN 'update' + ELSE 'no_change' + END AS action, + CASE + WHEN o.source_locator_hash IS NULL AND n.content_sha256 IN (SELECT content_sha256 FROM proven_moves) + THEN 'content_hash_move' + WHEN n.source_locator_hash IS NULL AND o.content_sha256 IN (SELECT content_sha256 FROM proven_moves) + THEN 'content_hash_move' + WHEN o.source_locator_hash IS NULL THEN 'locator' + WHEN n.source_locator_hash IS NULL THEN 'source_missing' + WHEN o.effective_scope IS DISTINCT FROM n.effective_scope THEN 'scope_change' + WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL AND o.content_sha256=n.content_sha256 THEN 'content_hash_move' + WHEN o.content_sha256 IS NOT NULL AND n.content_sha256 IS NOT NULL THEN 'content_hash_conflict' + WHEN o.relative_path IS DISTINCT FROM n.relative_path OR o.name IS DISTINCT FROM n.name THEN 'path_change' + WHEN o.size_bytes IS DISTINCT FROM n.size_bytes THEN 'size_change' + WHEN o.modified_at IS DISTINCT FROM n.modified_at THEN 'content_change' + ELSE 'locator' + END AS reason_code + FROM with_rule_scope o FULL OUTER JOIN with_rule_scope_new n USING(source_locator_hash) +) +INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path, + old_size_bytes,new_size_bytes,old_modified_at,new_modified_at,old_content_sha256,new_content_sha256,reason_code, + old_scope,new_scope,old_source_item_id,new_source_item_id,old_version_id,new_version_id,created_at) +SELECT change_set.id,compared.source_locator_hash,compared.action,compared.title, + compared.old_path,compared.new_path,compared.old_size_bytes,compared.new_size_bytes, + compared.old_modified_at,compared.new_modified_at,compared.old_content_sha256,compared.new_content_sha256,compared.reason_code, + compared.old_scope,compared.new_scope,compared.old_source_item_id,compared.new_source_item_id,compared.old_version_id,compared.new_version_id,NOW() +FROM change_set CROSS JOIN compared +ON CONFLICT (change_set_id,source_locator_hash) DO UPDATE SET + action=EXCLUDED.action,title=EXCLUDED.title,old_path=EXCLUDED.old_path,new_path=EXCLUDED.new_path, + old_size_bytes=EXCLUDED.old_size_bytes,new_size_bytes=EXCLUDED.new_size_bytes, + old_modified_at=EXCLUDED.old_modified_at,new_modified_at=EXCLUDED.new_modified_at, + old_content_sha256=EXCLUDED.old_content_sha256,new_content_sha256=EXCLUDED.new_content_sha256, + reason_code=EXCLUDED.reason_code, + old_scope=EXCLUDED.old_scope,new_scope=EXCLUDED.new_scope, + old_source_item_id=EXCLUDED.old_source_item_id,new_source_item_id=EXCLUDED.new_source_item_id, + old_version_id=EXCLUDED.old_version_id,new_version_id=EXCLUDED.new_version_id; + +UPDATE source_catalog_change_sets change_set SET + add_count=summary.add_count, + update_count=summary.update_count, + move_count=summary.move_count, + move_scope_count=summary.move_scope_count, + delete_count=summary.delete_count, + no_change_count=summary.no_change_count +FROM ( + SELECT items.change_set_id, + COUNT(*) FILTER (WHERE items.action='add')::integer AS add_count, + COUNT(*) FILTER (WHERE items.action='update')::integer AS update_count, + COUNT(*) FILTER (WHERE items.action='move')::integer AS move_count, + COUNT(*) FILTER (WHERE items.action='move_scope')::integer AS move_scope_count, + COUNT(*) FILTER (WHERE items.action='delete')::integer AS delete_count, + COUNT(*) FILTER (WHERE items.action='no_change')::integer AS no_change_count + FROM source_catalog_change_items items + WHERE items.change_set_id=p_change_set_id + GROUP BY items.change_set_id +) summary +WHERE change_set.id=summary.change_set_id; +END +$$ LANGUAGE plpgsql; diff --git a/internal/platform/postgres/migrations/000024_change_set_confirmations.down.sql b/internal/platform/postgres/migrations/000024_change_set_confirmations.down.sql new file mode 100644 index 0000000..82725ff --- /dev/null +++ b/internal/platform/postgres/migrations/000024_change_set_confirmations.down.sql @@ -0,0 +1,5 @@ +DROP INDEX IF EXISTS idx_source_catalog_change_items_seq; +ALTER TABLE source_catalog_change_items DROP COLUMN IF EXISTS item_seq; +ALTER TABLE test_publications DROP COLUMN IF EXISTS change_locator_hash, DROP COLUMN IF EXISTS change_set_id; +DROP INDEX IF EXISTS idx_processing_confirmations_change_set; +ALTER TABLE processing_confirmations DROP COLUMN IF EXISTS change_set_id; diff --git a/internal/platform/postgres/migrations/000024_change_set_confirmations.up.sql b/internal/platform/postgres/migrations/000024_change_set_confirmations.up.sql new file mode 100644 index 0000000..86c40e0 --- /dev/null +++ b/internal/platform/postgres/migrations/000024_change_set_confirmations.up.sql @@ -0,0 +1,14 @@ +-- P3: confirmations bind to the ChangeSet that produced them; publications record +-- provenance so every upload/removal job can be traced back to change evidence. +ALTER TABLE processing_confirmations + ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT; +CREATE INDEX IF NOT EXISTS idx_processing_confirmations_change_set + ON processing_confirmations (change_set_id) WHERE change_set_id IS NOT NULL; + +-- Surrogate public handle for selecting change items without leaking locator hashes. +ALTER TABLE source_catalog_change_items ADD COLUMN IF NOT EXISTS item_seq BIGSERIAL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_source_catalog_change_items_seq ON source_catalog_change_items (item_seq); + +ALTER TABLE test_publications + ADD COLUMN IF NOT EXISTS change_set_id VARCHAR(36) REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT, + ADD COLUMN IF NOT EXISTS change_locator_hash CHAR(64) CHECK (change_locator_hash IS NULL OR change_locator_hash ~ '^[0-9a-f]{64}$'); diff --git a/internal/platform/postgres/migrations/000025_processing_job_identity.down.sql b/internal/platform/postgres/migrations/000025_processing_job_identity.down.sql new file mode 100644 index 0000000..63d5120 --- /dev/null +++ b/internal/platform/postgres/migrations/000025_processing_job_identity.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS jobs_processing_identity; +ALTER TABLE jobs DROP COLUMN IF EXISTS business_action, DROP COLUMN IF EXISTS business_target_kb_id, DROP COLUMN IF EXISTS business_version_id, DROP COLUMN IF EXISTS business_change_set_id; diff --git a/internal/platform/postgres/migrations/000025_processing_job_identity.up.sql b/internal/platform/postgres/migrations/000025_processing_job_identity.up.sql new file mode 100644 index 0000000..c3f54aa --- /dev/null +++ b/internal/platform/postgres/migrations/000025_processing_job_identity.up.sql @@ -0,0 +1,13 @@ +-- P5 formal processing identity: every confirmed change-set job carries an +-- explicit business identity (publication uniqueness already exists since +-- migration 000002) so a duplicate delivery or worker restart cannot enqueue +-- the same action twice. +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_change_set_id VARCHAR(36); +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_version_id VARCHAR(36); +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_target_kb_id VARCHAR(255); +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS business_action VARCHAR(20) + CHECK (business_action IS NULL OR business_action IN ('upload', 'remove')); + +CREATE UNIQUE INDEX IF NOT EXISTS jobs_processing_identity + ON jobs (business_change_set_id, business_version_id, business_target_kb_id, business_action) + WHERE business_change_set_id IS NOT NULL; diff --git a/internal/platform/postgres/migrations/000026_job_failure_queue.down.sql b/internal/platform/postgres/migrations/000026_job_failure_queue.down.sql new file mode 100644 index 0000000..7c121e7 --- /dev/null +++ b/internal/platform/postgres/migrations/000026_job_failure_queue.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS job_retry_operations; +DROP TABLE IF EXISTS job_failures; +ALTER TABLE jobs DROP COLUMN IF EXISTS last_error_code; diff --git a/internal/platform/postgres/migrations/000026_job_failure_queue.up.sql b/internal/platform/postgres/migrations/000026_job_failure_queue.up.sql new file mode 100644 index 0000000..a67fbf7 --- /dev/null +++ b/internal/platform/postgres/migrations/000026_job_failure_queue.up.sql @@ -0,0 +1,48 @@ +-- P6 DLQ: every terminal job failure keeps its immutable jobs audit row AND +-- gains a dead-letter projection. Requeued failures flip to 'requeued'; the +-- history is append-only. Retrying resets the attempt budget on the SAME job +-- row so the P5 business identity can never be duplicated. +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS last_error_code VARCHAR(64) NOT NULL DEFAULT ''; + +CREATE TABLE IF NOT EXISTS job_failures ( + id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text, + job_id VARCHAR(36) NOT NULL REFERENCES jobs(id), + job_type VARCHAR(100) NOT NULL, + publication_id VARCHAR(64) NOT NULL DEFAULT '', + error_code VARCHAR(64) NOT NULL, + last_error TEXT NOT NULL, + attempt INTEGER NOT NULL CHECK (attempt > 0), + max_attempts_at_failure INTEGER NOT NULL CHECK (max_attempts_at_failure > 0), + business_change_set_id VARCHAR(64) NOT NULL DEFAULT '', + business_version_id VARCHAR(64) NOT NULL DEFAULT '', + business_action VARCHAR(16) NOT NULL DEFAULT '', + business_target_kb_id VARCHAR(64) NOT NULL DEFAULT '', + state VARCHAR(16) NOT NULL DEFAULT 'open' + CHECK (state IN ('open', 'requeued', 'abandoned')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + resolved_at TIMESTAMPTZ, + resolution_actor_ref VARCHAR(255) NOT NULL DEFAULT '', + resolution_reason TEXT NOT NULL DEFAULT '', + UNIQUE (job_id, attempt) +); + +CREATE INDEX IF NOT EXISTS idx_job_failures_open + ON job_failures (created_at DESC) WHERE state = 'open'; + +-- Retry audit: who re-queued which publication and with what reason +-- (batch-shared key). job_id is not stored: it is derivable from the +-- publication and would break the audit row on cross-batch duplicates. +CREATE TABLE IF NOT EXISTS job_retry_operations ( + id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text, + batch_key VARCHAR(255) NOT NULL, + publication_id VARCHAR(64) NOT NULL, + actor_ref VARCHAR(255) NOT NULL, + reason TEXT NOT NULL, + outcome VARCHAR(24) NOT NULL + CHECK (outcome IN ('started', 'already_running', 'not_retryable')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (batch_key, publication_id) +); + +CREATE INDEX IF NOT EXISTS idx_job_retry_operations_publication + ON job_retry_operations (publication_id, created_at DESC); diff --git a/internal/platform/postgres/migrations/000027_worker_heartbeats.down.sql b/internal/platform/postgres/migrations/000027_worker_heartbeats.down.sql new file mode 100644 index 0000000..a6a2eac --- /dev/null +++ b/internal/platform/postgres/migrations/000027_worker_heartbeats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS worker_heartbeats; diff --git a/internal/platform/postgres/migrations/000027_worker_heartbeats.up.sql b/internal/platform/postgres/migrations/000027_worker_heartbeats.up.sql new file mode 100644 index 0000000..767ae00 --- /dev/null +++ b/internal/platform/postgres/migrations/000027_worker_heartbeats.up.sql @@ -0,0 +1,9 @@ +-- P10 worker liveness: refreshed whenever a worker touches the queue (claim +-- attempts), read by the API to decide whether the worker process is alive. +CREATE TABLE IF NOT EXISTS worker_heartbeats ( + worker_id VARCHAR(128) PRIMARY KEY, + job_types TEXT NOT NULL DEFAULT '', + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_worker_heartbeats_seen ON worker_heartbeats(last_seen_at DESC); diff --git a/internal/platform/postgres/migrations/000028_release_change_sets.down.sql b/internal/platform/postgres/migrations/000028_release_change_sets.down.sql new file mode 100644 index 0000000..eb6681c --- /dev/null +++ b/internal/platform/postgres/migrations/000028_release_change_sets.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE release_batches DROP COLUMN IF EXISTS release_change_set_id; +DROP TABLE IF EXISTS release_change_set_items; +DROP TABLE IF EXISTS release_change_sets; diff --git a/internal/platform/postgres/migrations/000028_release_change_sets.up.sql b/internal/platform/postgres/migrations/000028_release_change_sets.up.sql new file mode 100644 index 0000000..e65cdbc --- /dev/null +++ b/internal/platform/postgres/migrations/000028_release_change_sets.up.sql @@ -0,0 +1,50 @@ +-- R1 persistent release ChangeSet: the real-time release plan becomes an +-- immutable, snapshot-bound object. Batches reference CONFIRMED sets; later +-- source refreshes can never mutate an existing set or its manifest. +CREATE TABLE IF NOT EXISTS release_change_sets ( + id VARCHAR(40) PRIMARY KEY, + name VARCHAR(200) NOT NULL, + number INTEGER NOT NULL, + state VARCHAR(16) NOT NULL DEFAULT 'planned' + CHECK (state IN ('planned','confirmed','consumed','cancelled')), + source_snapshot_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + source_change_set_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + acceptance_cutoff TIMESTAMPTZ NOT NULL, + production_baseline_ref VARCHAR(64) NOT NULL DEFAULT '', + item_count INTEGER NOT NULL DEFAULT 0, + add_count INTEGER NOT NULL DEFAULT 0, + update_count INTEGER NOT NULL DEFAULT 0, + delete_count INTEGER NOT NULL DEFAULT 0, + blocked_count INTEGER NOT NULL DEFAULT 0, + manifest_sha256 CHAR(64) NOT NULL + CHECK (manifest_sha256 = '' OR manifest_sha256 ~ '^[0-9a-f]{64}$'), + created_by VARCHAR(120) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + confirmed_by VARCHAR(120) NOT NULL DEFAULT '', + confirmed_at TIMESTAMPTZ +); +CREATE UNIQUE INDEX IF NOT EXISTS release_change_sets_number_key + ON release_change_sets(number); + +-- Frozen per-item plan: every field is written once at generation and never +-- updated (delete protection + immutability are enforced by having no UPDATE +-- path and a state machine on the parent). +CREATE TABLE IF NOT EXISTS release_change_set_items ( + change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + publication_id VARCHAR(64) NOT NULL, + title VARCHAR(500) NOT NULL, + target_scope VARCHAR(16) NOT NULL CHECK (target_scope IN ('internal','external')), + action VARCHAR(16) NOT NULL CHECK (action IN ('add','update','delete','blocked','unchanged')), + test_state VARCHAR(16) NOT NULL, + production_state VARCHAR(16) NOT NULL, + can_release BOOLEAN NOT NULL DEFAULT false, + blocker VARCHAR(32) NOT NULL DEFAULT '', + PRIMARY KEY (change_set_id, seq) +); + +-- Batches optionally bind to a confirmed change set (formal releases). +ALTER TABLE release_batches + ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40) + REFERENCES release_change_sets(id); +CREATE INDEX IF NOT EXISTS release_batches_change_set ON release_batches(release_change_set_id) WHERE release_change_set_id IS NOT NULL; diff --git a/internal/platform/postgres/migrations/000029_release_rollback_policy.down.sql b/internal/platform/postgres/migrations/000029_release_rollback_policy.down.sql new file mode 100644 index 0000000..c22f12d --- /dev/null +++ b/internal/platform/postgres/migrations/000029_release_rollback_policy.down.sql @@ -0,0 +1,12 @@ +-- Revert R4 policy. Rollback-stage rows are folded back to running/completed so +-- the old CHECK can be restored without data loss surprises in test schemas. +UPDATE release_executions SET state='running' WHERE state='rolling_back'; +UPDATE release_executions SET state='completed' WHERE state='rolled_back'; +UPDATE release_executions SET state='failed' WHERE state='rollback_failed'; +ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check; +ALTER TABLE release_executions + ADD CONSTRAINT release_executions_state_check CHECK (state IN ('pending','running','completed','failed')); +ALTER TABLE release_executions DROP COLUMN IF EXISTS snapshot_verification_id; +ALTER TABLE release_executions DROP COLUMN IF EXISTS restore_trigger; +ALTER TABLE release_executions DROP COLUMN IF EXISTS failure_level; +DROP INDEX IF EXISTS release_executions_rollback_state; diff --git a/internal/platform/postgres/migrations/000029_release_rollback_policy.up.sql b/internal/platform/postgres/migrations/000029_release_rollback_policy.up.sql new file mode 100644 index 0000000..334cf6c --- /dev/null +++ b/internal/platform/postgres/migrations/000029_release_rollback_policy.up.sql @@ -0,0 +1,22 @@ +-- R4 three-level rollback policy: +-- single_item: one publication permanently failed; later items were skipped +-- and (for publish) compensated by an automatic rollback. +-- batch: remaining batches were never started after a permanent failure. +-- overall: restore the entire release from the pre-release Production +-- Snapshot; only a production authorizer may trigger it, and it +-- requires an R2 snapshot verification id as evidence. +-- The rollback execution walks pending -> rolling_back -> rolled_back / +-- rollback_failed so the UI can show "compensating now" distinctly. +ALTER TABLE release_executions DROP CONSTRAINT IF EXISTS release_executions_state_check; +ALTER TABLE release_executions + ADD CONSTRAINT release_executions_state_check CHECK (state IN ( + 'pending','running','rolling_back','completed','failed','rolled_back','rollback_failed')); + +ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS failure_level VARCHAR(16) NOT NULL DEFAULT '' + CHECK (failure_level IN ('','single_item','batch','overall')); +ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS restore_trigger VARCHAR(32) NOT NULL DEFAULT '' + CHECK (restore_trigger IN ('','auto_rollback_after_failure','snapshot_restore')); +ALTER TABLE release_executions ADD COLUMN IF NOT EXISTS snapshot_verification_id VARCHAR(64) NOT NULL DEFAULT ''; + +CREATE INDEX IF NOT EXISTS release_executions_rollback_state + ON release_executions(prior_execution_id) WHERE operation = 'rollback'; diff --git a/internal/platform/postgres/migrations/000030_release_audit_links.down.sql b/internal/platform/postgres/migrations/000030_release_audit_links.down.sql new file mode 100644 index 0000000..530b3ba --- /dev/null +++ b/internal/platform/postgres/migrations/000030_release_audit_links.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS release_change_set_catalog_links; +DROP TABLE IF EXISTS release_change_set_snapshot_links; +DROP INDEX IF EXISTS release_executions_change_set; +ALTER TABLE release_executions DROP COLUMN IF EXISTS release_change_set_id; diff --git a/internal/platform/postgres/migrations/000030_release_audit_links.up.sql b/internal/platform/postgres/migrations/000030_release_audit_links.up.sql new file mode 100644 index 0000000..b7f2b09 --- /dev/null +++ b/internal/platform/postgres/migrations/000030_release_audit_links.up.sql @@ -0,0 +1,46 @@ +-- R6 release audit chain: promote the evidence links that used to live only in +-- JSONB into real, immutable foreign keys so any execution can be traced +-- end-to-end (execution -> batch -> release change set -> source snapshots -> +-- source change sets -> review/acceptance -> source item/version). +ALTER TABLE release_executions + ADD COLUMN IF NOT EXISTS release_change_set_id VARCHAR(40) + REFERENCES release_change_sets(id); +CREATE INDEX IF NOT EXISTS release_executions_change_set + ON release_executions(release_change_set_id) WHERE release_change_set_id IS NOT NULL; + +-- Backfill the denormalized execution binding from its batch (R1 binding). +UPDATE release_executions AS execution +SET release_change_set_id = batch.release_change_set_id +FROM release_batches AS batch +WHERE execution.release_batch_id = batch.id + AND batch.release_change_set_id IS NOT NULL + AND execution.release_change_set_id IS NULL; + +-- Normalized snapshot bindings (display JSONB stays, but FK rows are truth). +CREATE TABLE IF NOT EXISTS release_change_set_snapshot_links ( + change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE, + snapshot_id VARCHAR(36) NOT NULL REFERENCES source_catalog_snapshots(id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (change_set_id, snapshot_id) +); +CREATE TABLE IF NOT EXISTS release_change_set_catalog_links ( + change_set_id VARCHAR(40) NOT NULL REFERENCES release_change_sets(id) ON DELETE CASCADE, + catalog_change_set_id VARCHAR(36) NOT NULL REFERENCES source_catalog_change_sets(id) ON DELETE RESTRICT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (change_set_id, catalog_change_set_id) +); + +-- Backfill from the JSONB captured at generation time (skip ids that no longer +-- exist: RESTRICT keeps live rows, deleted ones simply do not link). +INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id) +SELECT rcs.id, elem.value +FROM release_change_sets rcs +CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_snapshot_ids) AS elem(value) +JOIN source_catalog_snapshots s ON s.id = elem.value +ON CONFLICT DO NOTHING; +INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id) +SELECT rcs.id, elem.value +FROM release_change_sets rcs +CROSS JOIN LATERAL jsonb_array_elements_text(rcs.source_change_set_ids) AS elem(value) +JOIN source_catalog_change_sets c ON c.id = elem.value +ON CONFLICT DO NOTHING; diff --git a/internal/platform/postgres/migrations/000031_source_connection_revisions.down.sql b/internal/platform/postgres/migrations/000031_source_connection_revisions.down.sql new file mode 100644 index 0000000..bcda918 --- /dev/null +++ b/internal/platform/postgres/migrations/000031_source_connection_revisions.down.sql @@ -0,0 +1,6 @@ +-- Revert V1 revisions. +ALTER TABLE source_connections DROP COLUMN IF EXISTS current_revision_id, + DROP COLUMN IF EXISTS updated_by, + DROP COLUMN IF EXISTS last_scheduled_at, + DROP COLUMN IF EXISTS next_refresh_at; +DROP TABLE IF EXISTS source_connection_revisions CASCADE; diff --git a/internal/platform/postgres/migrations/000031_source_connection_revisions.up.sql b/internal/platform/postgres/migrations/000031_source_connection_revisions.up.sql new file mode 100644 index 0000000..0a20e17 --- /dev/null +++ b/internal/platform/postgres/migrations/000031_source_connection_revisions.up.sql @@ -0,0 +1,64 @@ +-- V1 source connection revisions: settings edits become validated -> active -> retired +-- versions; source_connections keeps ONE authoritative active snapshot that runtime +-- schedulers and existing FKs read. Secrets stay references only: base_url is visible, +-- secret_ref names where a credential lives and is never resolved through this table. + +CREATE TABLE IF NOT EXISTS source_connection_revisions ( + id VARCHAR(36) PRIMARY KEY, + connection_id VARCHAR(36) NOT NULL REFERENCES source_connections(id) ON DELETE CASCADE, + revision_number INTEGER NOT NULL CHECK (revision_number > 0), + state VARCHAR(16) NOT NULL + CHECK (state IN ('validated', 'active', 'retired', 'failed')), + source_type VARCHAR(20) NOT NULL + CHECK (source_type IN ('pp', 'alist', 'dingtalk', 'manual')), + name VARCHAR(255) NOT NULL, + base_url TEXT NOT NULL DEFAULT '', + secret_ref VARCHAR(255) NOT NULL DEFAULT '', + enabled BOOLEAN NOT NULL DEFAULT FALSE, + schedule JSONB NOT NULL DEFAULT '{}'::jsonb + CHECK (jsonb_typeof(schedule) = 'object'), + created_by VARCHAR(128) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + validated_at TIMESTAMPTZ, + activated_at TIMESTAMPTZ, + retired_at TIMESTAMPTZ, + error_code VARCHAR(64) NOT NULL DEFAULT '', + CHECK (error_code = '' OR error_code ~ '^[a-z0-9_]{3,64}$'), + UNIQUE (connection_id, revision_number) +); + +-- Exactly one active revision per connection. +CREATE UNIQUE INDEX IF NOT EXISTS uq_source_connection_active_revision + ON source_connection_revisions (connection_id) WHERE state = 'active'; + +-- Scheduler state lives next to the active snapshot so the existing FKs and UI +-- keep reading source_connections. next_refresh_at is NULL until the operator +-- sets a refresh interval, and it is ALWAYS derived from the schedule on the +-- active revision (never trusted from clients). +ALTER TABLE source_connections + ADD COLUMN IF NOT EXISTS next_refresh_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS last_scheduled_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS current_revision_id VARCHAR(36) REFERENCES source_connection_revisions(id), + ADD COLUMN IF NOT EXISTS updated_by VARCHAR(128) NOT NULL DEFAULT 'system'; + +CREATE INDEX IF NOT EXISTS idx_source_connections_due + ON source_connections (next_refresh_at) + WHERE enabled AND next_refresh_at IS NOT NULL; + +-- Seed: every existing connection gets revision 1 reflecting its current row, +-- marked active, created by the migration itself. +INSERT INTO source_connection_revisions ( + id, connection_id, revision_number, state, source_type, name, base_url, + secret_ref, enabled, schedule, created_by, validated_at, activated_at +) +SELECT + 'scr-' || md5(c.id), c.id, 1, 'active', c.source_type, c.name, c.base_url, + c.secret_ref, c.enabled, c.schedule, 'migration-31', NOW(), NOW() +FROM source_connections AS c +ON CONFLICT (connection_id, revision_number) DO NOTHING; + +UPDATE source_connections AS c +SET current_revision_id = COALESCE(c.current_revision_id, 'scr-' || md5(c.id)) +FROM source_connection_revisions AS r +WHERE r.connection_id = c.id AND r.revision_number = 1 + AND c.current_revision_id IS NULL; diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go new file mode 100644 index 0000000..4f6f0c7 --- /dev/null +++ b/internal/platform/postgres/migrations_test.go @@ -0,0 +1,633 @@ +package postgres + +import ( + "os" + "strings" + "testing" +) + +func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { + migrations, err := loadMigrations() + if err != nil { + t.Fatalf("loadMigrations() error = %v", err) + } + if len(migrations) != 31 { + t.Fatalf("migration count = %d, want 24", len(migrations)) + } + if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") { + t.Fatalf("migration 1 = %#v", migrations[0]) + } + if migrations[1].Version != 2 || !strings.Contains(migrations[1].UpSQL, "CREATE TABLE IF NOT EXISTS source_items") { + t.Fatalf("migration 2 = %#v", migrations[1]) + } + if migrations[2].Version != 3 || !strings.Contains(migrations[2].UpSQL, "CREATE TABLE IF NOT EXISTS acceptance_results") { + t.Fatalf("migration 3 = %#v", migrations[2]) + } + if migrations[3].Version != 4 || !strings.Contains(migrations[3].UpSQL, "source_connections_source_type_check") { + t.Fatalf("migration 4 = %#v", migrations[3]) + } + if migrations[4].Version != 5 || !strings.Contains(migrations[4].UpSQL, "CREATE TABLE IF NOT EXISTS source_scans") { + t.Fatalf("migration 5 = %#v", migrations[4]) + } + if migrations[5].Version != 6 || !strings.Contains(migrations[5].UpSQL, "CREATE TABLE IF NOT EXISTS publication_reviews") { + t.Fatalf("migration 6 = %#v", migrations[5]) + } + if migrations[7].Version != 8 || !strings.Contains(migrations[7].UpSQL, "CREATE TABLE IF NOT EXISTS directory_scope_rules") { + t.Fatalf("migration 8 = %#v", migrations[7]) + } + if migrations[8].Version != 9 || !strings.Contains(migrations[8].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_snapshots") { + t.Fatalf("migration 9 = %#v", migrations[8]) + } + if migrations[9].Version != 10 || !strings.Contains(migrations[9].UpSQL, "Visibility policies used by the source catalog") { + t.Fatalf("migration 10 = %#v", migrations[9]) + } + if migrations[10].Version != 11 || !strings.Contains(migrations[10].UpSQL, "CREATE TABLE IF NOT EXISTS processing_confirmations") { + t.Fatalf("migration 11 = %#v", migrations[10]) + } + if migrations[13].Version != 14 || !strings.Contains(migrations[13].UpSQL, "ADD COLUMN IF NOT EXISTS started_at") { + t.Fatalf("migration 14 = %#v", migrations[13]) + } + if migrations[14].Version != 15 || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_sets") || !strings.Contains(migrations[14].UpSQL, "CREATE TABLE IF NOT EXISTS source_catalog_change_items") { + t.Fatalf("migration 15 = %#v", migrations[14]) + } + if migrations[15].Version != 16 || !strings.Contains(migrations[15].UpSQL, "origin") || !strings.Contains(migrations[15].UpSQL, "source_catalog_change_items") { + t.Fatalf("migration 16 = %#v", migrations[15]) + } + if migrations[16].Version != 17 || !strings.Contains(migrations[16].UpSQL, "current_batch") || !strings.Contains(migrations[16].UpSQL, "total_batches") { + t.Fatalf("migration 17 = %#v", migrations[16]) + } + if migrations[17].Version != 18 || !strings.Contains(migrations[17].UpSQL, "jobs_state_check") || !strings.Contains(migrations[17].UpSQL, "cancelled") { + t.Fatalf("migration 18 = %#v", migrations[17]) + } + if migrations[18].Version != 19 || !strings.Contains(migrations[18].UpSQL, "CREATE TABLE IF NOT EXISTS operation_requests") || !strings.Contains(migrations[18].UpSQL, "UNIQUE (operation_type, operation_id)") { + t.Fatalf("migration 19 = %#v", migrations[18]) + } + if migrations[19].Version != 20 || !strings.Contains(migrations[19].UpSQL, "content_sha256") || !strings.Contains(migrations[19].UpSQL, "IF NOT EXISTS") { + t.Fatalf("migration 20 = %#v", migrations[19]) + } + if migrations[20].Version != 21 || !strings.Contains(migrations[20].UpSQL, "identity_confidence") || !strings.Contains(migrations[20].UpSQL, "reason_code") { + t.Fatalf("migration 21 = %#v", migrations[20]) + } + if migrations[21].Version != 22 || !strings.Contains(migrations[21].UpSQL, "compare_catalog_change_set") { + t.Fatalf("migration 22 = %#v", migrations[21]) + } + if migrations[22].Version != 23 || !strings.Contains(migrations[22].UpSQL, "move_count") || !strings.Contains(migrations[22].UpSQL, "'applying'") || !strings.Contains(migrations[22].UpSQL, "old_scope") { + t.Fatalf("migration 23 = %#v", migrations[22]) + } + if migrations[23].Version != 24 || !strings.Contains(migrations[23].UpSQL, "item_seq") || !strings.Contains(migrations[23].UpSQL, "change_set_id") { + t.Fatalf("migration 24 = %#v", migrations[23]) + } + if migrations[24].Version != 25 || !strings.Contains(migrations[24].UpSQL, "business_change_set_id") || !strings.Contains(migrations[24].UpSQL, "jobs_processing_identity") { + t.Fatalf("migration 25 = %#v", migrations[24]) + } + if migrations[25].Version != 26 || !strings.Contains(migrations[25].UpSQL, "job_failures") || !strings.Contains(migrations[25].UpSQL, "last_error_code") { + t.Fatalf("migration 26 = %#v", migrations[25]) + } + if migrations[26].Version != 27 || !strings.Contains(migrations[26].UpSQL, "worker_heartbeats") { + t.Fatalf("migration 27 = %#v", migrations[26]) + } + if migrations[27].Version != 28 || !strings.Contains(migrations[27].UpSQL, "release_change_sets") || !strings.Contains(migrations[27].UpSQL, "manifest_sha256") { + t.Fatalf("migration 28 = %#v", migrations[27]) + } + if migrations[28].Version != 29 || !strings.Contains(migrations[28].UpSQL, "rolling_back") || !strings.Contains(migrations[28].UpSQL, "failure_level") { + t.Fatalf("migration 29 = %#v", migrations[28]) + } + if migrations[29].Version != 30 || !strings.Contains(migrations[29].UpSQL, "release_change_set_snapshot_links") || !strings.Contains(migrations[29].UpSQL, "release_change_set_catalog_links") { + t.Fatalf("migration 30 = %#v", migrations[29]) + } + if migrations[30].Version != 31 || !strings.Contains(migrations[30].UpSQL, "source_connection_revisions") || !strings.Contains(migrations[30].UpSQL, "uq_source_connection_active_revision") || !strings.Contains(migrations[30].UpSQL, "next_refresh_at") { + t.Fatalf("migration 31 = %#v", migrations[30]) + } +} + +func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) { + up, err := os.ReadFile("migrations/000011_processing_confirmations.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000011_processing_confirmations.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "add column if not exists target_scope", + "update test_publications as publication", + "review.decided_visibility", + "source.visibility", + "check (target_scope in ('internal', 'external'))", + "create table if not exists processing_confirmations", + "references test_publications(id) on delete restrict", + "operator_ref", + "reason", + "unique (publication_id)", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("processing confirmation migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + if !strings.Contains(downSQL, "drop table if exists processing_confirmations") || !strings.Contains(downSQL, "drop column if exists target_scope") { + t.Fatal("processing confirmation down migration must remove only migration 11 objects") + } +} + +func TestVersionedUISystemSettingsMigration(t *testing.T) { + up, err := os.ReadFile("migrations/000013_ui_system_settings.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000013_ui_system_settings.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{"create table if not exists ui_setting_revisions", "'validated', 'active', 'retired', 'failed'", "create unique index", "where state = 'active'", "insert into ui_setting_revisions"} { + if !strings.Contains(upSQL, required) { + t.Fatalf("UI settings migration missing %q", required) + } + } + if !strings.Contains(strings.ToLower(string(down)), "drop table if exists ui_setting_revisions") { + t.Fatal("UI settings rollback must remove only its revision table") + } +} + +func TestReleaseActionMigrationSeparatesUpsertAndDelete(t *testing.T) { + up, err := os.ReadFile("migrations/000012_release_actions.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000012_release_actions.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "alter table release_batch_items", "alter table release_item_executions", + "add column if not exists action", "check (action in ('upsert', 'delete'))", + "drop constraint if exists release_batch_items_publication_id_key", + "drop constraint if exists release_batch_items_review_id_key", + } { + if !strings.Contains(upSQL, required) { + t.Fatalf("release action migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + if !strings.Contains(downSQL, "drop column if exists action") { + t.Fatal("release action rollback must remove action columns") + } +} + +func TestCatalogPolicyMigrationAllowsBothAndBlockedWithoutChangingPublicationReviewScope(t *testing.T) { + up, err := os.ReadFile("migrations/000010_catalog_policies.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000010_catalog_policies.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "visibility policies used by the source catalog", + "directory_scope_rules_visibility_check", + "source_item_scope_overrides_visibility_check", + "scope_change_events_previous_visibility_check", + "scope_change_events_decided_visibility_check", + "'internal', 'external', 'both', 'blocked'", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("catalog policy migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + if !strings.Contains(downSQL, "update source_item_scope_overrides") || !strings.Contains(downSQL, "update directory_scope_rules") { + t.Fatalf("catalog policy down migration must normalize extended policy values before restoring constraints") + } +} + +func TestSourceCatalogMigrationDefinesImmutableHierarchicalSnapshots(t *testing.T) { + up, err := os.ReadFile("migrations/000009_source_catalog.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000009_source_catalog.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "create table if not exists source_catalog_snapshots", + "create table if not exists source_catalog_nodes", + "create table if not exists source_catalog_heads", + "check (node_type in ('folder', 'file'))", + "primary key (snapshot_id, node_id)", + "foreign key (snapshot_id, parent_node_id)", + "references source_items(id) on delete restrict", + "create index if not exists idx_source_catalog_nodes_parent", + "unique (source_connection_id, snapshot_sha256)", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("source catalog migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + ordered := []string{"drop table if exists source_catalog_heads", "drop table if exists source_catalog_nodes", "drop table if exists source_catalog_snapshots"} + previous := -1 + for _, statement := range ordered { + position := strings.Index(downSQL, statement) + if position < 0 || position <= previous { + t.Fatalf("source catalog down order invalid for %q", statement) + } + previous = position + } +} + +func TestScopeRulesMigrationDefinesAuditedDirectoryAndFileOverrides(t *testing.T) { + up, err := os.ReadFile("migrations/000008_scope_rules.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000008_scope_rules.down.sql") + if err != nil { + t.Fatal(err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "create table if not exists directory_scope_rules", + "create table if not exists source_item_scope_overrides", + "create table if not exists scope_change_events", + "primary key (source_connection_id, directory_path)", + "primary key (source_item_id)", + "check (visibility in ('internal', 'external'))", + "check (target_type in ('directory', 'item'))", + "references source_connections(id) on delete restrict", + "references source_items(id) on delete restrict", + "create index if not exists idx_scope_change_events_target", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("scope migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + for _, required := range []string{ + "drop table if exists scope_change_events", + "drop table if exists source_item_scope_overrides", + "drop table if exists directory_scope_rules", + } { + if !strings.Contains(downSQL, required) { + t.Errorf("scope down migration missing %q", required) + } + } +} + +func TestJobsMigrationDefinesDurableLeaseQueue(t *testing.T) { + body, err := os.ReadFile("migrations/000001_jobs.up.sql") + if err != nil { + t.Fatalf("read jobs migration: %v", err) + } + sql := strings.ToLower(string(body)) + for _, required := range []string{ + "create table if not exists schema_migrations", + "create table if not exists jobs", + "idempotency_key", + "lease_owner", + "lease_until", + "available_at", + "max_attempts", + "check (state in", + "unique (idempotency_key)", + } { + if !strings.Contains(sql, required) { + t.Errorf("migration missing %q", required) + } + } +} + +func TestSourceGovernanceMigrationDefinesIdentityAndStateContracts(t *testing.T) { + body, err := os.ReadFile("migrations/000002_source_governance.up.sql") + if err != nil { + t.Fatalf("read source governance migration: %v", err) + } + sql := strings.ToLower(string(body)) + for _, required := range []string{ + "create table if not exists source_connections", + "create table if not exists source_items", + "create table if not exists content_objects", + "create table if not exists source_item_versions", + "create table if not exists test_publications", + "unique (source_connection_id, stable_external_id)", + "unique (sha256)", + "unique (source_item_id, revision)", + "unique (source_item_version_id, test_kb_id)", + "check (source_type in ('pp', 'alist', 'dingtalk'))", + "check (sha256 ~ '^[0-9a-f]{64}$')", + "check (source_change in ('unchanged', 'added', 'updated', 'deleted', 'unavailable'))", + "check (content_state in ('unknown', 'downloading', 'verified', 'invalid', 'over_limit', 'failed'))", + "check (ingest_state in ('pending', 'uploading', 'uploaded', 'failed'))", + "check (processing_state in ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'))", + "check (retrieval_state in ('unpublished', 'enabled', 'disabling', 'disabled', 'enabling', 'failed'))", + "references source_connections(id)", + "references source_items(id)", + "references content_objects(id)", + "references source_item_versions(id)", + "raw_record_summary jsonb", + } { + if !strings.Contains(sql, required) { + t.Errorf("source governance migration missing %q", required) + } + } +} + +func TestSourceGovernanceDownMigrationDropsTablesInDependencyOrder(t *testing.T) { + body, err := os.ReadFile("migrations/000002_source_governance.down.sql") + if err != nil { + t.Fatalf("read source governance down migration: %v", err) + } + sql := strings.ToLower(string(body)) + ordered := []string{ + "drop table if exists test_publications", + "drop table if exists source_item_versions", + "drop table if exists content_objects", + "drop table if exists source_items", + "drop table if exists source_connections", + } + previous := -1 + for _, statement := range ordered { + position := strings.Index(sql, statement) + if position < 0 { + t.Fatalf("down migration missing %q", statement) + } + if position <= previous { + t.Fatalf("%q is not in reverse dependency order", statement) + } + previous = position + } +} + +func TestTestOrchestrationMigrationDefinesPollingAndAcceptanceContracts(t *testing.T) { + body, err := os.ReadFile("migrations/000003_test_orchestration.up.sql") + if err != nil { + t.Fatalf("read test orchestration migration: %v", err) + } + sql := strings.ToLower(string(body)) + for _, required := range []string{ + "alter table test_publications add column if not exists poll_sequence", + "alter table test_publications add column if not exists last_remote_status", + "alter table test_publications add column if not exists last_observed_at", + "alter table test_publications add column if not exists supersedes_publication_id", + "references test_publications(id) on delete restrict", + "create table if not exists acceptance_results", + "references test_publications(id) on delete restrict", + "check (check_type in ('knowledge_exists', 'parse_completed', 'chunks_present', 'retrieval_hit', 'citation_matches'))", + "check (state in ('pending', 'passed', 'failed', 'manual_required'))", + "check (jsonb_typeof(evidence_summary) = 'object')", + "unique (publication_id, run_id, check_type)", + "create index if not exists idx_test_publications_poll", + "create index if not exists idx_acceptance_results_publication", + } { + if !strings.Contains(sql, required) { + t.Errorf("test orchestration migration missing %q", required) + } + } +} + +func TestTestOrchestrationDownMigrationRemovesDependentObjectsFirst(t *testing.T) { + body, err := os.ReadFile("migrations/000003_test_orchestration.down.sql") + if err != nil { + t.Fatalf("read test orchestration down migration: %v", err) + } + sql := strings.ToLower(string(body)) + acceptance := strings.Index(sql, "drop table if exists acceptance_results") + publicationColumns := strings.Index(sql, "alter table test_publications") + if acceptance < 0 || publicationColumns < 0 { + t.Fatalf("down migration missing dependent object removal: %s", sql) + } + if acceptance > publicationColumns { + t.Fatal("acceptance_results must be dropped before test_publications columns") + } +} + +func TestManualSourceMigrationExpandsWithoutRewritingAndDowngradesFailClosed(t *testing.T) { + up, err := os.ReadFile("migrations/000004_manual_source.up.sql") + if err != nil { + t.Fatalf("read manual source up migration: %v", err) + } + down, err := os.ReadFile("migrations/000004_manual_source.down.sql") + if err != nil { + t.Fatalf("read manual source down migration: %v", err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "drop constraint if exists source_connections_source_type_check", + "check (source_type in ('pp', 'alist', 'dingtalk', 'manual'))", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("manual source up migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + for _, required := range []string{ + "if exists", + "source_type = 'manual'", + "raise exception", + "check (source_type in ('pp', 'alist', 'dingtalk'))", + } { + if !strings.Contains(downSQL, required) { + t.Errorf("manual source down migration missing %q", required) + } + } +} + +func TestSourceScanMigrationDefinesAuditableSingleActiveScan(t *testing.T) { + up, err := os.ReadFile("migrations/000005_source_scans.up.sql") + if err != nil { + t.Fatalf("read source scans up migration: %v", err) + } + down, err := os.ReadFile("migrations/000005_source_scans.down.sql") + if err != nil { + t.Fatalf("read source scans down migration: %v", err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "create table if not exists source_scans", + "create table if not exists source_scan_changes", + "references source_connections(id) on delete restrict", + "references source_scans(id) on delete restrict", + "check (state in ('pending', 'running', 'completed', 'failed'))", + "check (change_type in ('added', 'updated', 'unavailable'))", + "check (snapshot_sha256 = '' or snapshot_sha256 ~ '^[0-9a-f]{64}$')", + "check (observed_items >= 0)", + "check (unchanged_items >= 0)", + "check (added_items >= 0)", + "check (updated_items >= 0)", + "check (over_limit_items >= 0)", + "check (missing_items >= 0)", + "check (planned_uploads >= 0)", + "create unique index if not exists idx_source_scans_one_active", + "where state in ('pending', 'running')", + "unique (source_scan_id, stable_external_id)", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("source scans up migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + changes := strings.Index(downSQL, "drop table if exists source_scan_changes") + scans := strings.Index(downSQL, "drop table if exists source_scans") + if changes < 0 || scans < 0 || changes > scans { + t.Fatal("source scans down migration must drop changes before scans") + } +} + +func TestReviewAndSealMigrationDefinesImmutableHumanGate(t *testing.T) { + up, err := os.ReadFile("migrations/000006_review_and_seal.up.sql") + if err != nil { + t.Fatalf("read review and seal up migration: %v", err) + } + down, err := os.ReadFile("migrations/000006_review_and_seal.down.sql") + if err != nil { + t.Fatalf("read review and seal down migration: %v", err) + } + upSQL := strings.ToLower(string(up)) + for _, required := range []string{ + "create table if not exists publication_reviews", + "references test_publications(id) on delete restrict", + "check (decision in ('approved', 'rejected'))", + "check (decided_visibility in ('internal', 'external'))", + "acceptance_run_id", + "publication_updated_at", + "create table if not exists release_batches", + "check (state in ('draft', 'sealed', 'cancelled'))", + "policy_version", + "create table if not exists release_batch_items", + "references publication_reviews(id) on delete restrict", + "unique (publication_id)", + "decided_visibility", + } { + if !strings.Contains(upSQL, required) { + t.Errorf("review and seal migration missing %q", required) + } + } + downSQL := strings.ToLower(string(down)) + ordered := []string{ + "drop table if exists release_batch_items", + "drop table if exists release_batches", + "drop table if exists publication_reviews", + } + previous := -1 + for _, statement := range ordered { + position := strings.Index(downSQL, statement) + if position < 0 || position <= previous { + t.Fatalf("review and seal down migration has invalid dependency order for %q", statement) + } + previous = position + } +} + +func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) { + up, err := os.ReadFile("migrations/000007_release_executions.up.sql") + if err != nil { + t.Fatal(err) + } + down, err := os.ReadFile("migrations/000007_release_executions.down.sql") + if err != nil { + t.Fatal(err) + } + text := strings.ToLower(string(up)) + for _, required := range []string{ + "create table if not exists release_executions", + "operation in ('publish', 'rollback')", + "state in ('pending', 'running', 'completed', 'failed')", + "target_environment = 'production'", + "references release_batches(id) on delete restrict", + "references jobs(id) on delete restrict", + "create unique index if not exists uq_release_publish_batch", + "create table if not exists release_item_executions", + "bundle_sha256", "previous_production_revision", "production_revision", "error_code", + } { + if !strings.Contains(text, required) { + t.Fatalf("release execution migration missing %q", required) + } + } + if strings.Contains(text, "api_key") || strings.Contains(text, "password") || strings.Contains(text, "secret") { + t.Fatal("release execution migration must not persist production credentials") + } + downText := strings.ToLower(string(down)) + items := strings.Index(downText, "drop table if exists release_item_executions") + executions := strings.Index(downText, "drop table if exists release_executions") + if items < 0 || executions < 0 || items > executions { + t.Fatal("release execution down migration must drop item executions before executions") + } +} + +func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) { + sql := strings.ToLower(claimJobSQL) + for _, required := range []string{ + "for update skip locked", + "state in ('pending', 'retry')", + "state = 'running'", + "lease_until <", + "attempt = j.attempt + 1", + } { + if !strings.Contains(sql, required) { + t.Errorf("claim SQL missing %q", required) + } + } +} + +func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) { + tests := []struct { + name string + sql string + required []string + }{ + { + name: "enqueue", + sql: enqueueJobSQL, + required: []string{ + "on conflict (idempotency_key)", + "do update set idempotency_key = excluded.idempotency_key", + "returning id", + }, + }, + { + name: "complete", + sql: completeJobSQL, + required: []string{ + "state = 'running'", + "lease_owner = $2", + "state = 'completed'", + }, + }, + { + name: "heartbeat", + sql: heartbeatJobSQL, + required: []string{ + "state = 'running'", + "lease_owner = $2", + "lease_until = $4", + }, + }, + { + name: "fail", + sql: failJobSQL, + required: []string{ + "case when attempt >= max_attempts then 'failed'", + "else 'retry'", + "lease_owner = null", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sql := strings.ToLower(tt.sql) + for _, required := range tt.required { + if !strings.Contains(sql, required) { + t.Errorf("SQL missing %q", required) + } + } + }) + } +} diff --git a/internal/platform/postgres/operation_requests.go b/internal/platform/postgres/operation_requests.go new file mode 100644 index 0000000..abff998 --- /dev/null +++ b/internal/platform/postgres/operation_requests.go @@ -0,0 +1,180 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +var ErrOperationRequestConflict = errors.New("operation request conflicts with an existing request") + +type operationExecutor func(pgx.Tx) (json.RawMessage, error) + +func (s *Store) RouteCatalogFilesWithOperation(ctx context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) { + if err := validateCatalogRouteRequest(request); err != nil { + return governance.CatalogRouteResult{}, false, err + } + fingerprint, err := catalogRouteFingerprint(request) + if err != nil { + return governance.CatalogRouteResult{}, false, err + } + resultJSON, replayed, err := s.executeOperation(ctx, "catalog.route", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + result, err := routeCatalogFilesTx(ctx, tx, request) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("encode catalog route result: %w", err) + } + return encoded, nil + }) + if err != nil { + return governance.CatalogRouteResult{}, false, err + } + var result governance.CatalogRouteResult + if err := json.Unmarshal(resultJSON, &result); err != nil { + return governance.CatalogRouteResult{}, false, fmt.Errorf("decode stored catalog route result: %w", err) + } + return result, replayed, nil +} + +func catalogRouteFingerprint(request governance.CatalogRouteRequest) (string, error) { + targets := request.TargetKnowledgeBaseIDs + if len(targets) == 0 && request.TargetKnowledgeBaseID != "" { + targets = []string{request.TargetKnowledgeBaseID} + } + payload, err := json.Marshal(struct { + ConnectionID string `json:"connection_id"` + NodeIDs []string `json:"node_ids"` + Visibility governance.Visibility `json:"visibility"` + Targets []string `json:"target_knowledge_base_ids"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + }{ + ConnectionID: request.ConnectionID, + NodeIDs: request.NodeIDs, + Visibility: request.Visibility, + Targets: targets, + OperatorRef: strings.TrimSpace(request.OperatorRef), + Reason: strings.TrimSpace(request.Reason), + }) + if err != nil { + return "", fmt.Errorf("encode catalog route fingerprint: %w", err) + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} + +func (s *Store) EnqueueCatalogRefreshWithOperation(ctx context.Context, source governance.CatalogSource, operationID string, now time.Time) (jobs.Job, bool, error) { + if !validRouteID(source.ConnectionID) || strings.TrimSpace(source.Name) == "" || now.IsZero() { + return jobs.Job{}, false, errors.New("invalid catalog refresh request") + } + fingerprint, err := operationFingerprint(struct { + ConnectionID string `json:"connection_id"` + }{ConnectionID: source.ConnectionID}) + if err != nil { + return jobs.Job{}, false, err + } + resultJSON, replayed, err := s.executeOperation(ctx, "catalog.refresh", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) { + jobID, err := newOpaqueID("job-") + if err != nil { + return nil, err + } + payload, err := json.Marshal(map[string]any{"connection_id": source.ConnectionID, "source_type": source.SourceType, "source_name": source.Name, "stage": "queued", "progress": 0, "message": "等待后台刷新", "processed": 0, "total": 0}) + if err != nil { + return nil, fmt.Errorf("encode catalog refresh job: %w", err) + } + job, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, jobID, "source.catalog.refresh", payload, 20, now, 3, "catalog-refresh:"+jobID, now)) + if err != nil { + return nil, fmt.Errorf("enqueue catalog refresh job: %w", err) + } + encoded, err := json.Marshal(job) + if err != nil { + return nil, fmt.Errorf("encode catalog refresh result: %w", err) + } + return encoded, nil + }) + if err != nil { + return jobs.Job{}, false, err + } + var result jobs.Job + if err := json.Unmarshal(resultJSON, &result); err != nil { + return jobs.Job{}, false, fmt.Errorf("decode stored catalog refresh result: %w", err) + } + return result, replayed, nil +} + +func operationFingerprint(value any) (string, error) { + payload, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode operation fingerprint: %w", err) + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} + +func (s *Store) executeOperation(ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute operationExecutor) (json.RawMessage, bool, error) { + if strings.TrimSpace(operationType) == "" || len(operationType) > 100 || strings.TrimSpace(operationID) == "" || len(operationID) > 128 || len(fingerprint) != 64 || now.IsZero() || execute == nil { + return nil, false, errors.New("invalid operation request") + } + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return nil, false, errors.New("operation request transactions are unavailable") + } + tx, err := beginner.Begin(ctx) + if err != nil { + return nil, false, fmt.Errorf("begin operation request: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + tag, err := tx.Exec(ctx, `INSERT INTO operation_requests(operation_type,operation_id,request_fingerprint_sha256,state,result,created_at,completed_at) +VALUES($1,$2,$3,'completed','{}'::jsonb,$4,$4) +ON CONFLICT(operation_type,operation_id) DO NOTHING`, operationType, operationID, fingerprint, now) + if err != nil { + return nil, false, fmt.Errorf("create operation request: %w", err) + } + if tag.RowsAffected() == 0 { + var storedFingerprint string + var storedResult json.RawMessage + if err := tx.QueryRow(ctx, `SELECT request_fingerprint_sha256,result +FROM operation_requests +WHERE operation_type=$1 AND operation_id=$2 +FOR UPDATE`, operationType, operationID).Scan(&storedFingerprint, &storedResult); err != nil { + return nil, false, fmt.Errorf("load operation request: %w", err) + } + if storedFingerprint != fingerprint { + return nil, false, ErrOperationRequestConflict + } + if err := tx.Commit(ctx); err != nil { + return nil, false, fmt.Errorf("commit replay operation request: %w", err) + } + return storedResult, true, nil + } + result, err := execute(tx) + if err != nil { + return nil, false, err + } + if !json.Valid(result) { + return nil, false, errors.New("operation result must be valid JSON") + } + if _, err := tx.Exec(ctx, `UPDATE operation_requests SET result=$3::jsonb,completed_at=$4 +WHERE operation_type=$1 AND operation_id=$2`, operationType, operationID, result, now); err != nil { + return nil, false, fmt.Errorf("complete operation request: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return nil, false, fmt.Errorf("commit operation request: %w", err) + } + return result, false, nil +} diff --git a/internal/platform/postgres/operation_wrappers.go b/internal/platform/postgres/operation_wrappers.go new file mode 100644 index 0000000..ec566ac --- /dev/null +++ b/internal/platform/postgres/operation_wrappers.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "encoding/json" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +func runOperation[T any](s *Store, ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute func(pgx.Tx) (T, error)) (T, bool, error) { + var zero T + raw, replayed, err := s.executeOperation(ctx, operationType, operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) { + result, err := execute(tx) + if err != nil { + return nil, err + } + return json.Marshal(result) + }) + if err != nil { + return zero, false, err + } + var result T + if err := json.Unmarshal(raw, &result); err != nil { + return zero, false, err + } + return result, replayed, nil +} + +func (s *Store) TriggerPPSourceScanWithOperation(ctx context.Context, connectionID string, now time.Time, operationID string) (governance.SourceScan, bool, error) { + fingerprint, err := operationFingerprint(struct { + ConnectionID string `json:"connection_id"` + }{connectionID}) + if err != nil { + return governance.SourceScan{}, false, err + } + return runOperation(s, ctx, "source.scan.trigger", operationID, fingerprint, now, func(tx pgx.Tx) (governance.SourceScan, error) { + return NewStore(tx).TriggerPPSourceScan(ctx, connectionID, now) + }) +} + +func (s *Store) CreateUISettingsCandidateWithOperation(ctx context.Context, request governance.CreateUISettingsCandidateRequest, operationID string) (governance.UISettings, bool, error) { + fingerprint, err := operationFingerprint(struct { + Document int `json:"document_refresh_seconds"` + Resource int `json:"resource_refresh_seconds"` + Stalled int `json:"stalled_after_seconds"` + Finalize int `json:"finalizing_after_seconds"` + By string `json:"updated_by"` + }{request.DocumentRefreshSeconds, request.ResourceRefreshSeconds, request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy}) + if err != nil { + return governance.UISettings{}, false, err + } + return runOperation(s, ctx, "ui_settings.candidate.create", operationID, fingerprint, request.Now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) { + return NewStore(tx).CreateUISettingsCandidate(ctx, request) + }) +} + +func (s *Store) ActivateUISettingsWithOperation(ctx context.Context, revisionID string, now time.Time, operationID string) (governance.UISettings, bool, error) { + fingerprint, err := operationFingerprint(struct { + RevisionID string `json:"revision_id"` + }{revisionID}) + if err != nil { + return governance.UISettings{}, false, err + } + return runOperation(s, ctx, "ui_settings.activate", operationID, fingerprint, now.UTC(), func(tx pgx.Tx) (governance.UISettings, error) { + return NewStore(tx).ActivateUISettings(ctx, revisionID, now) + }) +} + +func (s *Store) CreateReleaseBatchWithOperation(ctx context.Context, request governance.CreateReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) { + fingerprint, err := operationFingerprint(struct { + Name string `json:"name"` + By string `json:"created_by"` + Upsert []string `json:"publication_ids"` + Delete []string `json:"delete_publication_ids"` + Set string `json:"release_change_set_id"` + }{request.Name, request.CreatedBy, request.PublicationIDs, request.DeletePublicationIDs, request.ReleaseChangeSetID}) + if err != nil { + return governance.ReleaseBatch{}, false, err + } + return runOperation(s, ctx, "release.batch.create", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) { + return NewStore(tx).CreateReleaseBatch(ctx, request) + }) +} + +func (s *Store) GenerateReleaseChangeSetWithOperation(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) { + fingerprint, err := operationFingerprint(struct { + Name string `json:"name"` + By string `json:"created_by"` + }{request.Name, request.CreatedBy}) + if err != nil { + return governance.ReleaseChangeSet{}, false, err + } + return runOperation(s, ctx, "release.changeset.generate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) { + return NewStore(tx).GenerateReleaseChangeSet(ctx, request) + }) +} + +func (s *Store) ConfirmReleaseChangeSetWithOperation(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest, operationID string) (governance.ReleaseChangeSet, bool, error) { + fingerprint, err := operationFingerprint(struct { + ID string `json:"change_set_id"` + By string `json:"confirmed_by"` + }{request.ChangeSetID, request.ConfirmedBy}) + if err != nil { + return governance.ReleaseChangeSet{}, false, err + } + return runOperation(s, ctx, "release.changeset.confirm", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseChangeSet, error) { + return NewStore(tx).ConfirmReleaseChangeSet(ctx, request) + }) +} + +func (s *Store) SealReleaseBatchWithOperation(ctx context.Context, request governance.SealReleaseBatchRequest, operationID string) (governance.ReleaseBatch, bool, error) { + fingerprint, err := operationFingerprint(struct { + BatchID string `json:"batch_id"` + SealedBy string `json:"sealed_by"` + }{request.BatchID, request.SealedBy}) + if err != nil { + return governance.ReleaseBatch{}, false, err + } + return runOperation(s, ctx, "release.batch.seal", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseBatch, error) { + return NewStore(tx).SealReleaseBatch(ctx, request) + }) +} + +func (s *Store) AuthorizeReleaseBatchWithOperation(ctx context.Context, request governance.AuthorizeReleaseBatchRequest, operationID string) (governance.ReleaseExecution, bool, error) { + fingerprint, err := operationFingerprint(struct { + BatchID string `json:"batch_id"` + AuthorizedBy string `json:"authorized_by"` + }{request.BatchID, request.AuthorizedBy}) + if err != nil { + return governance.ReleaseExecution{}, false, err + } + return runOperation(s, ctx, "release.authorize.publish", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) { + return NewStore(tx).AuthorizeReleaseBatch(ctx, request) + }) +} + +func (s *Store) AuthorizeReleaseRollbackWithOperation(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest, operationID string) (governance.ReleaseExecution, bool, error) { + fingerprint, err := operationFingerprint(struct { + PriorID string `json:"prior_execution_id"` + AuthorizedBy string `json:"authorized_by"` + }{request.PriorExecutionID, request.AuthorizedBy}) + if err != nil { + return governance.ReleaseExecution{}, false, err + } + return runOperation(s, ctx, "release.authorize.rollback", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) { + return NewStore(tx).AuthorizeReleaseRollback(ctx, request) + }) +} + +func (s *Store) RetryReleaseExecutionWithOperation(ctx context.Context, request governance.RetryReleaseExecutionRequest, operationID string) (governance.ReleaseExecution, bool, error) { + fingerprint, err := operationFingerprint(struct { + ExecutionID string `json:"execution_id"` + }{request.ExecutionID}) + if err != nil { + return governance.ReleaseExecution{}, false, err + } + return runOperation(s, ctx, "release.execution.retry", operationID, fingerprint, request.Now, func(tx pgx.Tx) (governance.ReleaseExecution, error) { + return NewStore(tx).RetryReleaseExecution(ctx, request) + }) +} diff --git a/internal/platform/postgres/p2_identity_integration_test.go b/internal/platform/postgres/p2_identity_integration_test.go new file mode 100644 index 0000000..c117f10 --- /dev/null +++ b/internal/platform/postgres/p2_identity_integration_test.go @@ -0,0 +1,100 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestP2StableIdentityAndProvenMove freezes docs plan P2: +// identity_confidence is never downgraded from stable_id; the canonical +// compare_catalog_change_set function classifies a 1:1 content-hash relocation +// as proven move evidence, and never merges ambiguous copies. +func TestP2StableIdentityAndProvenMove(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + hash := strings.Repeat("c", 64) + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id','pp','ID PP',true); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snap-old','conn-id',repeat('1',64),'ID PP',2,1,1,'2026-09-11T00:00:00Z'), + ('snap-new','conn-id',repeat('2',64),'ID PP',2,1,1,'2026-09-12T00:00:00Z'), + ('snap-copy','conn-id',repeat('3',64),'ID PP',3,1,2,'2026-09-13T00:00:00Z'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes,content_sha256) +VALUES + ('snap-old','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL), + ('snap-old','f-old','root','file','手册.pdf','旧目录/手册.pdf',repeat('5',64),10,0,1,10,repeat('c',64)), + ('snap-new','root',NULL,'folder','ID PP','',repeat('4',64),NULL,1,1,10,NULL), + ('snap-new','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)), + ('snap-copy','root',NULL,'folder','ID PP','',repeat('4',64),NULL,2,2,20,NULL), + ('snap-copy','f-new','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('6',64),10,0,1,10,repeat('c',64)), + ('snap-copy','f-copy','root','file','副本.pdf','新目录/副本.pdf',repeat('7',64),10,0,1,10,repeat('c',64)); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at) +VALUES('cs-move','conn-id','snap-old','snap-new',NOW()), + ('cs-copy','conn-id','snap-new','snap-copy',NOW()); +SELECT compare_catalog_change_set('cs-move'); +SELECT compare_catalog_change_set('cs-copy');`); err != nil { + t.Fatal(err) + } + + // Proven move: the disappeared old locator is a delete carrying move evidence, + // and the reappearing new locator is an add carrying the same content hash. + var deleteReason, deleteOldHash string + if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(old_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='delete'`).Scan(&deleteReason, &deleteOldHash); err != nil || deleteReason != "content_hash_move" || deleteOldHash != hash { + t.Fatalf("cs-move delete=%q hash=%q err=%v", deleteReason, deleteOldHash, err) + } + var addReason, addNewHash string + if err := pool.QueryRow(ctx, `SELECT reason_code,COALESCE(new_content_sha256::text,'') FROM source_catalog_change_items WHERE change_set_id='cs-move' AND action='add'`).Scan(&addReason, &addNewHash); err != nil || addReason != "content_hash_move" || addNewHash != hash { + t.Fatalf("cs-move add=%q hash=%q err=%v", addReason, addNewHash, err) + } + // Counts were recalculated by the function. + var addCount, deleteCount int + if err := pool.QueryRow(ctx, `SELECT add_count,delete_count FROM source_catalog_change_sets WHERE id='cs-move'`).Scan(&addCount, &deleteCount); err != nil || addCount != 1 || deleteCount != 1 { + t.Fatalf("cs-move counts add=%d delete=%d err=%v", addCount, deleteCount, err) + } + + // Ambiguous copy: hash appears twice on the new side → NOT a proven move; + // the copy is a plain add and the original stays untouched. + var copyReason string + if err := pool.QueryRow(ctx, `SELECT reason_code FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='add' AND new_path LIKE '%副本%'`).Scan(©Reason); err != nil || copyReason != "locator" { + t.Fatalf("ambiguous copy reason=%q err=%v", copyReason, err) + } + var copyDeletes int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_catalog_change_items WHERE change_set_id='cs-copy' AND action='delete'`).Scan(©Deletes); err != nil || copyDeletes != 0 { + t.Fatalf("ambiguous copy must not delete the original: %d err=%v", copyDeletes, err) + } + + // identity_confidence upsert: stable_id never downgrades; path can rise. + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-id2','pp','ID PP2',true) ON CONFLICT DO NOTHING; +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,identity_confidence) +VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'稳.pdf','稳.pdf','document','v1','internal','pp_public_tag','stable_id') ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin,source_change,identity_confidence) +VALUES('item-stable','conn-id','ext-stable',repeat('f',64),'','稳.pdf','document','v1','internal','manual','added','content_hash') +ON CONFLICT (source_connection_id, stable_external_id) DO UPDATE +SET updated_at = source_items.updated_at, + identity_confidence = CASE WHEN source_items.identity_confidence='stable_id' THEN source_items.identity_confidence ELSE EXCLUDED.identity_confidence END;`); err != nil { + t.Fatal(err) + } + var confidence string + if err := pool.QueryRow(ctx, `SELECT identity_confidence FROM source_items WHERE id='item-stable'`).Scan(&confidence); err != nil || confidence != "stable_id" { + t.Fatalf("stable_id downgraded to %q err=%v", confidence, err) + } +} diff --git a/internal/platform/postgres/p3_confirm_integration_test.go b/internal/platform/postgres/p3_confirm_integration_test.go new file mode 100644 index 0000000..a3dc396 --- /dev/null +++ b/internal/platform/postgres/p3_confirm_integration_test.go @@ -0,0 +1,228 @@ +package postgres + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" +) + +func seedP3Fixture(t *testing.T, pool *pgxpool.Pool) string { + t.Helper() + ctx := context.Background() + _, seedErr := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p3','alist','P3 来源',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES + ('co-p3-a',repeat('a',64),10,'verified/a.pdf','verified'), + ('co-p3-b',repeat('b',64),12,'verified/b.pdf','verified'), + ('co-p3-c',repeat('c',64),20,'verified/c.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin) +VALUES('item-a','conn-p3','ext-a',repeat('d',64),'新增.pdf','document','r1','internal','manual'), + ('item-b','conn-p3','ext-b',repeat('e',64),'更新.pdf','document','r2','internal','manual'), + ('item-c','conn-p3','ext-c',repeat('f',64),'删除.pdf','document','r3','internal','manual'), + ('item-d','conn-p3','ext-d',repeat('9',64),'未变.pdf','document','r1','internal','manual'), + ('item-e','conn-p3','ext-e',repeat('8',64),'内部专属.pdf','document','r1','internal','manual'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES + ('version-a','item-a','r1','co-p3-a','verified'), + ('version-b','item-b','r2','co-p3-b','verified'), + ('version-c','item-c','r3','co-p3-c','verified'), + ('version-d','item-d','r1','co-p3-a','verified'), + ('version-e','item-e','r1','co-p3-a','verified'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('p3-old','conn-p3',repeat('1',64),'P3 来源',1,1,0,'2026-09-11T00:00:00Z'), + ('p3-new','conn-p3',repeat('2',64),'P3 来源',6,1,5,'2026-09-12T00:00:00Z'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,content_sha256) +VALUES + ('p3-new','root',NULL,'folder','P3 来源','',repeat('3',64),NULL,NULL,NULL), + ('p3-new','n-a','root','file','新增.pdf','新增.pdf',repeat('a',64),'item-a',10,repeat('a',64)), + ('p3-new','n-b','root','file','更新.pdf','更新.pdf',repeat('b',64),'item-b',12,repeat('b',64)), + ('p3-new','n-d','root','file','未变.pdf','未变.pdf',repeat('c',64),'item-d',10,repeat('a',64)), + ('p3-new','n-e','root','file','内部专属.pdf','内部专属.pdf',repeat('e',64),'item-e',10,repeat('a',64)); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at) +VALUES('tp-p3-c','version-c','kb-internal','internal','uploaded','completed','enabled','kn-c',NOW(),NOW()); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at) +VALUES('cs-p3','conn-p3','p3-old','p3-new',NOW()); +INSERT INTO source_catalog_change_items(change_set_id,source_locator_hash,action,title,old_path,new_path,new_scope,new_source_item_id,old_source_item_id,created_at) +VALUES + ('cs-p3',repeat('a',64),'add','新增.pdf','','新增.pdf','internal','item-a',NULL,NOW()), + ('cs-p3',repeat('b',64),'update','更新.pdf','更新.pdf','更新.pdf','internal','item-b',NULL,NOW()), + ('cs-p3',repeat('c',64),'delete','删除.pdf','删除.pdf','','',NULL,'item-c',NOW()), + ('cs-p3',repeat('d',64),'no_change','未变.pdf','未变.pdf','未变.pdf','internal','item-d',NULL,NOW()), + ('cs-p3',repeat('e',64),'add','内部专属.pdf','','内部专属.pdf','blocked','item-e',NULL,NOW());`) + if seedErr != nil { + t.Fatal(seedErr) + } + return "cs-p3" +} + +func p3Seqs(t *testing.T, pool *pgxpool.Pool, changeSetID string) []int64 { + t.Helper() + rows, err := pool.Query(context.Background(), `SELECT item_seq FROM source_catalog_change_items WHERE change_set_id=$1 ORDER BY item_seq`, changeSetID) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var seqs []int64 + for rows.Next() { + var seq int64 + if err := rows.Scan(&seq); err != nil { + t.Fatal(err) + } + seqs = append(seqs, seq) + } + if len(seqs) != 5 { + t.Fatalf("expected 5 change items, got %d", len(seqs)) + } + return seqs +} + +func TestP3ConfirmExecutesSelectedChangeItems(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + id := seedP3Fixture(t, pool) + store := NewStore(pool) + now := time.Date(2026, 9, 12, 3, 0, 0, 0, time.UTC) + + // Jobs are zero before confirmation. + var jobsBefore int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobsBefore); err != nil || jobsBefore != 0 { + t.Fatalf("jobs before=%d err=%v", jobsBefore, err) + } + + // Partial selection: add, update, delete, no_change, blocked-delete. + result, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{ + ChangeSetID: id, ItemSeqs: p3Seqs(t, pool, "cs-p3"), + OperatorRef: "engineer-1", Reason: "确认P3差异执行", Now: now, + InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external", + }) + if err != nil { + t.Fatal(err) + } + if result.ConfirmedItems != 4 || result.UploadsCreated != 2 || result.RemovalsRequested != 1 || result.NoChangeIgnored != 1 || result.SkippedNoArtifact != 0 || result.SkippedNoTarget != 0 { + t.Fatalf("result=%#v", result) + } + // blocked add became a removal request too (desired state blocked → disable path) + var uploads, removals int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil { + t.Fatal(err) + } + if uploads != 2 || removals != 1 { + t.Fatalf("uploads=%d removals=%d", uploads, removals) + } + // Upload payloads carry the ChangeSet and item identity (P5 will make it a constraint). + var payloadOK bool + if err := pool.QueryRow(ctx, `SELECT COUNT(*)=2 FROM jobs WHERE type='test.upload' AND payload->>'change_set_id'='cs-p3'`).Scan(&payloadOK); err != nil || !payloadOK { + t.Fatalf("upload provenance missing ok=%v err=%v", payloadOK, err) + } + // Publications record their change provenance. + var provenance int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE change_set_id='cs-p3'`).Scan(&provenance); err != nil || provenance != 2 { + t.Fatalf("publication provenance=%d err=%v", provenance, err) + } + // Confirmations bind to the change set. + var confirmations int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM processing_confirmations WHERE change_set_id='cs-p3'`).Scan(&confirmations); err != nil || confirmations != 2 { + t.Fatalf("confirmations=%d err=%v", confirmations, err) + } + // Lifecycle moved to confirmed. + var state string + if err := pool.QueryRow(ctx, `SELECT state FROM source_catalog_change_sets WHERE id='cs-p3'`).Scan(&state); err != nil || state != "confirmed" { + t.Fatalf("state=%q err=%v", state, err) + } + + // Re-confirming the same items replays stored counts; no new jobs or rows. + replaySeqs := p3Seqs(t, pool, "cs-p3")[:1] + replay, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{ + ChangeSetID: id, ItemSeqs: replaySeqs, + OperatorRef: "engineer-2", Reason: "重复确认", Now: now.Add(time.Minute), + InternalTargetKBID: "kb-internal", + }) + if err != nil { + t.Fatal(err) + } + if replay.UploadsCreated != 2 || replay.RemovalsRequested != 1 { + t.Fatalf("replay=%#v", replay) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil || uploads != 2 { + t.Fatalf("replay created uploads=%d err=%v", uploads, err) + } + + // Unknown locator belongs to nobody: rejected before any enqueue. + if _, err := store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{ + ChangeSetID: id, ItemSeqs: []int64{999999}, + OperatorRef: "engineer-2", Reason: "越界确认", Now: now, InternalTargetKBID: "kb-internal", + }); err == nil { + t.Fatal("unknown item selection accepted") + } +} + +func TestP3ConcurrentConfirmationsEnqueueOnce(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + id := seedP3Fixture(t, pool) + store := NewStore(pool) + now := time.Date(2026, 9, 12, 4, 0, 0, 0, time.UTC) + + const workers = 8 + var wg sync.WaitGroup + errs := make([]error, workers) + for index := 0; index < workers; index++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + _, errs[index] = store.ConfirmCatalogChangeSet(ctx, governance.ConfirmCatalogChangeSetRequest{ + ChangeSetID: id, + ItemSeqs: p3Seqs(t, pool, "cs-p3"), + OperatorRef: "engineer-1", Reason: "并发确认", Now: now.Add(time.Duration(index) * time.Second), + InternalTargetKBID: "kb-internal", ExternalTargetKBID: "kb-external", + }) + }(index) + } + wg.Wait() + for index, err := range errs { + if err != nil { + t.Fatalf("worker %d err=%v", index, err) + } + } + // Exactly one enqueue per publication regardless of concurrent confirmations + // (FOR UPDATE on the set serializes; later workers replay from stored counts). + var uploads, removals int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploads); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removals); err != nil { + t.Fatal(err) + } + if uploads != 2 || removals != 1 { + t.Fatalf("concurrent enqueue: uploads=%d removals=%d", uploads, removals) + } +} diff --git a/internal/platform/postgres/p5_job_identity_integration_test.go b/internal/platform/postgres/p5_job_identity_integration_test.go new file mode 100644 index 0000000..2c87595 --- /dev/null +++ b/internal/platform/postgres/p5_job_identity_integration_test.go @@ -0,0 +1,71 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestP5ProcessingJobIdentity freezes docs plan P5: the database rejects a +// duplicate delivery of the same (change_set, version, target, action), while +// a different ChangeSet or action for the same version+target remains legal and +// identity-less jobs never collide. +func TestP5ProcessingJobIdentity(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p5','alist','P5 来源',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p5',repeat('a',64),10,'verified/a.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin) +VALUES('item-p5','conn-p5','ext-p5',repeat('b',64),'P5.pdf','document','r1','internal','manual'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-p5','item-p5','r1','co-p5','verified'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope) VALUES('tp-p5','version-p5','kb-internal','internal'); +INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES('job-p5-1','test.upload','{}','p5:first','cs-p5-a','version-p5','kb-internal','upload')`); err != nil { + t.Fatal(err) + } + + // Duplicate delivery of the same processing identity must be rejected. + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES('job-p5-2','test.upload','{}','p5:replay','cs-p5-a','version-p5','kb-internal','upload')`); err == nil || !strings.Contains(err.Error(), "duplicate key value") { + t.Fatalf("duplicate identity err = %v, want unique violation", err) + } + + // A later ChangeSet touching the same version+target is a different identity. + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES('job-p5-3','test.upload','{}','p5:next','cs-p5-b','version-p5','kb-internal','upload')`); err != nil { + t.Fatalf("different change set rejected: %v", err) + } + + // A different action on the same identity tuple is legal. + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key,business_change_set_id,business_version_id,business_target_kb_id,business_action) +VALUES('job-p5-4','test.remove','{}','p5:remove','cs-p5-a','version-p5','kb-internal','remove')`); err != nil { + t.Fatalf("different action rejected: %v", err) + } + + // Jobs without business identity never collide with the partial index. + for _, id := range []string{"job-p5-5", "job-p5-6"} { + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,idempotency_key) VALUES($1,'test.poll','{}',$2)`, id, "plain-"+id); err != nil { + t.Fatalf("unidentified job %s: %v", id, err) + } + } + + var count int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM jobs WHERE business_version_id='version-p5'`).Scan(&count); err != nil || count != 3 { + t.Fatalf("identified jobs = %d err = %v, want 3", count, err) + } +} diff --git a/internal/platform/postgres/p6_dlq_integration_test.go b/internal/platform/postgres/p6_dlq_integration_test.go new file mode 100644 index 0000000..85f1b97 --- /dev/null +++ b/internal/platform/postgres/p6_dlq_integration_test.go @@ -0,0 +1,179 @@ +package postgres + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// TestP6DeadLetterAndServerSideBatchRetry freezes docs plan P6: an ordinary +// failure that exhausts its budget lands in the DLQ projection without +// touching the original audit row, a permanent failure lands immediately, and +// the server-side batch retry returns mixed outcomes atomically with replay. +func TestP6DeadLetterAndServerSideBatchRetry(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + _, seedErr := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-p6','alist','P6 来源',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-p6',repeat('a',64),10,'verified/p6.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin) +VALUES('item-p6','conn-p6','ext-p6',repeat('d',64),'P6.pdf','document','r1','internal','manual'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES + ('siv-1','item-p6','r1','co-p6','verified'), + ('siv-2','item-p6','r2','co-p6','verified'), + ('siv-3','item-p6','r3','co-p6','verified'), + ('siv-4','item-p6','r4','co-p6','verified'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,updated_at) VALUES + ('tp-1','siv-1','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), + ('tp-2','siv-2','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), + ('tp-3','siv-3','kb-1','uploaded','failed','unpublished','internal','2026-09-12T01:00:00Z'), + ('tp-5','siv-4','kb-1','uploaded','processing','unpublished','internal','2026-09-12T01:00:00Z');`) + if seedErr != nil { + t.Fatalf("seed: %v", seedErr) + } + store := NewStore(pool) + base := time.Date(2026, time.September, 12, 1, 0, 0, 0, time.UTC) + + // 1) Ordinary failure exhausting the budget projects into the DLQ. + uploadPayload, _ := json.Marshal(map[string]string{"publication_id": "tp-1"}) + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-1','test.upload',$1,'running',5,5,'k-dlq-1','w1','cs-p6','siv-1','kb-1','upload',$2)`, uploadPayload, base); err != nil { + t.Fatalf("seed job: %v", err) + } + if err := store.Fail(ctx, "job-dlq-1", "w1", "remote parse failed", "validation_failed", base, base.Add(time.Minute)); err != nil { + t.Fatalf("exhausting fail: %v", err) + } + var jobState, errorCode, failureState string + var attempt int + if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState, &errorCode); err != nil { + t.Fatal(err) + } + if jobState != "failed" || errorCode != "validation_failed" { + t.Fatalf("exhausted job = (%s,%s)", jobState, errorCode) + } + var failedPublicationID string + if err := pool.QueryRow(ctx, `SELECT state,attempt,error_code,publication_id FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &attempt, &errorCode, &failedPublicationID); err != nil { + t.Fatalf("dlq projection missing: %v", err) + } + if failureState != "open" || attempt != 5 || errorCode != "validation_failed" || failedPublicationID != "tp-1" { + t.Fatalf("dlq row = (%s,%d,%s,%s)", failureState, attempt, errorCode, failedPublicationID) + } + + // 2) Permanent failure parks immediately and projects. + removePayload, _ := json.Marshal(map[string]string{"publication_id": "tp-3"}) + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,attempt,max_attempts,idempotency_key,lease_owner,business_change_set_id,business_version_id,business_target_kb_id,business_action,updated_at) VALUES('job-dlq-3','test.remove',$1,'running',1,10,'k-dlq-3','w1','cs-p6','siv-3','','remove',$2)`, removePayload, base); err != nil { + t.Fatalf("seed remove job: %v", err) + } + var _ jobs.TerminalFailer = store + if err := store.FailTerminal(ctx, "job-dlq-3", "w1", "remote rejected", "remote_permanent", base); err != nil { + t.Fatalf("terminal fail: %v", err) + } + if err := pool.QueryRow(ctx, `SELECT state,last_error_code FROM jobs WHERE id='job-dlq-3'`).Scan(&jobState, &errorCode); err != nil { + t.Fatal(err) + } + if jobState != "failed" || errorCode != "remote_permanent" { + t.Fatalf("terminal job = (%s,%s)", jobState, errorCode) + } + + // 3) DLQ query lists both open projections. + page, err := store.ListJobFailures(ctx, governance.JobFailureFilter{Limit: 50}) + if err != nil { + t.Fatalf("list failures: %v", err) + } + if page.Total != 2 || len(page.Items) != 2 { + t.Fatalf("open failures = %d/%d, want 2", len(page.Items), page.Total) + } + + // 4) Server-side batch retry: tp-1 + tp-2 started, unknown id not_retryable. + result, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ + PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"}, + OperatorRef: "姚工", + Reason: "解析服务恢复", + Now: base.Add(2 * time.Minute), + }, "op-retry-batch-1") + if err != nil || replayed { + t.Fatalf("batch retry: %v replayed=%v", err, replayed) + } + if result.Total != 4 || result.Started != 2 || result.NotRetryable != 1 || result.AlreadyRunning != 1 { + t.Fatalf("batch result = %+v", result) + } + if result.Outcomes[0].PublicationID != "tp-1" || result.Outcomes[0].Result != governance.RetryStarted { + t.Fatalf("tp-1 outcome = %+v", result.Outcomes[0]) + } + if result.Outcomes[1].PublicationID != "tp-2" || result.Outcomes[1].Result != governance.RetryStarted { + t.Fatalf("tp-2 outcome = %+v", result.Outcomes[1]) + } + if result.Outcomes[2].PublicationID != "tp-4-missing" || result.Outcomes[2].Result != governance.RetryNotRetryable || result.Outcomes[2].Detail != "publication not found" { + t.Fatalf("missing outcome = %+v", result.Outcomes[2]) + } + if result.Outcomes[3].PublicationID != "tp-5" || result.Outcomes[3].Result != governance.RetryAlreadyRunning { + t.Fatalf("tp-5 outcome = %+v", result.Outcomes[3]) + } + + // tp-1's open DLQ row flipped to requeued with operator and reason. + var actor, reason string + if err := pool.QueryRow(ctx, `SELECT state,resolution_actor_ref,resolution_reason FROM job_failures WHERE job_id='job-dlq-1'`).Scan(&failureState, &actor, &reason); err != nil { + t.Fatal(err) + } + if failureState != "requeued" || actor != "姚工" || reason != "解析服务恢复" { + t.Fatalf("tp-1 failure after retry = (%s,%s,%s)", failureState, actor, reason) + } + // tp-3's failure stays open (not part of this batch). + if err := pool.QueryRow(ctx, `SELECT state FROM job_failures WHERE job_id='job-dlq-3'`).Scan(&failureState); err != nil || failureState != "open" { + t.Fatalf("tp-3 failure state = %s err=%v", failureState, err) + } + + // Audit rows recorded atomically for every selection. + var auditCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM job_retry_operations WHERE batch_key='op-retry-batch-1'`).Scan(&auditCount); err != nil || auditCount != 4 { + t.Fatalf("retry audit rows = %d err=%v", auditCount, err) + } + // Fresh retry jobs exist and the original exhausted job stays untouched. + var newJobs int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND state='pending' AND idempotency_key LIKE 'test-retry:tp-1:%'`).Scan(&newJobs); err != nil || newJobs != 1 { + t.Fatalf("tp-1 retry jobs = %d err=%v", newJobs, err) + } + if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id='job-dlq-1'`).Scan(&jobState); err != nil || jobState != "failed" { + t.Fatalf("original job state = %q err=%v, want failed (untouched)", jobState, err) + } + + // 5) Replaying the same operation id returns the identical stored result. + replay, replayed, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ + PublicationIDs: []string{"tp-1", "tp-2", "tp-5", "tp-4-missing"}, + OperatorRef: "姚工", + Reason: "解析服务恢复", + Now: base.Add(9 * time.Minute), + }, "op-retry-batch-1") + if err != nil || !replayed { + t.Fatalf("replay: %v replayed=%v", err, replayed) + } + if replay.Started != result.Started || replay.Total != result.Total { + t.Fatalf("replay result = %+v, want %+v", replay, result) + } + + // 6) Same key with a different body conflicts. + if _, _, err := store.RetryPublicationsBatchWithOperation(ctx, governance.RetryPublicationsBatchRequest{ + PublicationIDs: []string{"tp-2"}, + OperatorRef: "姚工", + Reason: "解析服务恢复", + Now: base.Add(10 * time.Minute), + }, "op-retry-batch-1"); err == nil { + t.Fatal("expected operation request conflict for same key different body") + } +} diff --git a/internal/platform/postgres/pp_import.go b/internal/platform/postgres/pp_import.go new file mode 100644 index 0000000..4d69963 --- /dev/null +++ b/internal/platform/postgres/pp_import.go @@ -0,0 +1,173 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + ppconnector "knowledge-governance/internal/connectors/pp" +) + +type LegacyImportReport struct { + Connections int + SourceItems int + ContentObjects int + Versions int + Publications int +} + +func LoadPPCurrentItems(ctx context.Context, pool *pgxpool.Pool, connectionID string) (map[string]ppconnector.CurrentItem, error) { + rows, err := pool.Query(ctx, ` + SELECT stable_external_id, revision + FROM source_items + WHERE source_connection_id = $1 + ORDER BY stable_external_id`, connectionID) + if err != nil { + return nil, fmt.Errorf("query current PP items: %w", err) + } + defer rows.Close() + + items := make(map[string]ppconnector.CurrentItem) + for rows.Next() { + var sourceHash string + var item ppconnector.CurrentItem + if err := rows.Scan(&sourceHash, &item.Revision); err != nil { + return nil, fmt.Errorf("scan current PP item: %w", err) + } + items[sourceHash] = item + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate current PP items: %w", err) + } + return items, nil +} + +func ImportPPLegacy(ctx context.Context, pool *pgxpool.Pool, plan ppconnector.ImportPlan) (LegacyImportReport, error) { + tx, err := pool.Begin(ctx) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("begin PP legacy import: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + report := LegacyImportReport{} + tag, err := tx.Exec(ctx, `INSERT INTO source_connections ( + id, source_type, name, base_url, secret_ref, enabled, schedule, last_health + ) VALUES ($1, $2, $3, $4, $5, $6, '{}'::jsonb, '{}'::jsonb) + ON CONFLICT DO NOTHING`, + plan.Connection.ID, + plan.Connection.Type, + plan.Connection.Name, + plan.Connection.BaseURL, + plan.Connection.SecretRef, + plan.Connection.Enabled, + ) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("insert PP source connection: %w", err) + } + report.Connections = int(tag.RowsAffected()) + + for _, content := range plan.ContentObjects { + tag, err = tx.Exec(ctx, `INSERT INTO content_objects ( + id, sha256, size_bytes, mime, validated_format, local_spool_ref, content_state + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT DO NOTHING`, + content.ID, + content.SHA256, + content.SizeBytes, + content.MIME, + content.ValidatedFormat, + content.LocalSpoolRef, + content.State, + ) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("insert PP content object: %w", err) + } + report.ContentObjects += int(tag.RowsAffected()) + } + + for _, item := range plan.SourceItems { + tag, err = tx.Exec(ctx, `INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, + parent_external_id, normalized_path, title, item_type, size_bytes, + mime, revision, visibility, visibility_origin, last_seen_scan_id, + source_change, absence_streak, identity_confidence + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, 'stable_id' + ) + ON CONFLICT DO NOTHING`, + item.ID, + item.SourceConnectionID, + item.StableExternalID, + item.CanonicalURIHash, + item.ParentExternalID, + item.NormalizedPath, + item.Title, + item.ItemType, + item.SizeBytes, + item.MIME, + item.Revision, + item.Visibility, + item.VisibilityOrigin, + item.LastSeenScanID, + item.SourceChange, + item.AbsenceStreak, + ) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("insert PP source item: %w", err) + } + report.SourceItems += int(tag.RowsAffected()) + } + + for _, version := range plan.Versions { + tag, err = tx.Exec(ctx, `INSERT INTO source_item_versions ( + id, source_item_id, revision, content_object_id, content_state, + download_error, raw_record_summary, source_updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, CAST($7 AS JSONB), $8) + ON CONFLICT DO NOTHING`, + version.ID, + version.SourceItemID, + version.Revision, + version.ContentObjectID, + version.ContentState, + version.DownloadError, + string(version.RawRecordSummary), + version.SourceUpdatedAt, + ) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("insert PP source item version: %w", err) + } + report.Versions += int(tag.RowsAffected()) + } + + for _, publication := range plan.Publications { + tag, err = tx.Exec(ctx, `INSERT INTO test_publications ( + id, source_item_version_id, test_kb_id, knowledge_id, + ingest_state, processing_state, retrieval_state, + parser_generation, index_generation, last_error, retry_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT DO NOTHING`, + publication.ID, + publication.SourceItemVersionID, + publication.TestKBID, + publication.KnowledgeID, + publication.IngestState, + publication.ProcessingState, + publication.RetrievalState, + publication.ParserGeneration, + publication.IndexGeneration, + publication.LastError, + publication.RetryCount, + ) + if err != nil { + return LegacyImportReport{}, fmt.Errorf("insert PP test publication: %w", err) + } + report.Publications += int(tag.RowsAffected()) + } + + if err := tx.Commit(ctx); err != nil { + return LegacyImportReport{}, fmt.Errorf("commit PP legacy import: %w", err) + } + return report, nil +} diff --git a/internal/platform/postgres/pp_import_integration_test.go b/internal/platform/postgres/pp_import_integration_test.go new file mode 100644 index 0000000..41337b3 --- /dev/null +++ b/internal/platform/postgres/pp_import_integration_test.go @@ -0,0 +1,155 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + ppconnector "knowledge-governance/internal/connectors/pp" + "knowledge-governance/internal/governance" +) + +func TestImportPPLegacyIsIdempotent(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("pgxpool.New() error = %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("ApplyMigrations() error = %v", err) + } + + sourceCanonical := strings.Repeat("a", 64) + sourceAlias := strings.Repeat("b", 64) + contentHash := strings.Repeat("c", 64) + plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{ + {KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"}, + {KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: contentHash, Status: ppconnector.StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical}, + }}) + if err != nil { + t.Fatalf("BuildImportPlan() error = %v", err) + } + + first, err := ImportPPLegacy(ctx, pool, plan) + if err != nil { + t.Fatalf("first ImportPPLegacy() error = %v", err) + } + if first != (LegacyImportReport{Connections: 1, SourceItems: 2, ContentObjects: 1, Versions: 2, Publications: 1}) { + t.Fatalf("first ImportPPLegacy() report = %#v", first) + } + second, err := ImportPPLegacy(ctx, pool, plan) + if err != nil { + t.Fatalf("second ImportPPLegacy() error = %v", err) + } + if second != (LegacyImportReport{}) { + t.Fatalf("second ImportPPLegacy() report = %#v, want zero inserts", second) + } + + var counts [5]int + if err := pool.QueryRow(ctx, `SELECT + (SELECT COUNT(*) FROM source_connections), + (SELECT COUNT(*) FROM source_items), + (SELECT COUNT(*) FROM content_objects), + (SELECT COUNT(*) FROM source_item_versions), + (SELECT COUNT(*) FROM test_publications)`).Scan(&counts[0], &counts[1], &counts[2], &counts[3], &counts[4]); err != nil { + t.Fatalf("query imported counts: %v", err) + } + if counts != [5]int{1, 2, 1, 2, 1} { + t.Fatalf("imported counts = %#v", counts) + } +} + +func TestLoadPPCurrentItemsSupportsZeroUploadReconciliation(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + + source := strings.Repeat("d", 64) + contentHash := strings.Repeat("e", 64) + snapshot := ppconnector.Snapshot{KBID: "kb-reconcile", Items: []ppconnector.Item{{ + KBID: "kb-reconcile", SourceURLHash: source, SourceIDs: []string{"pp-3"}, Bytes: 12, + SHA256: contentHash, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-3", + }}} + plan, err := ppconnector.BuildImportPlan(snapshot) + if err != nil { + t.Fatal(err) + } + if _, err := ImportPPLegacy(ctx, pool, plan); err != nil { + t.Fatal(err) + } + + current, err := LoadPPCurrentItems(ctx, pool, plan.Connection.ID) + if err != nil { + t.Fatal(err) + } + summary, err := ppconnector.PlanReconciliation(snapshot, current) + if err != nil { + t.Fatal(err) + } + if summary.UploadRequired != 0 || summary.Unchanged != 1 { + t.Fatalf("summary = %#v", summary) + } +} + +func TestStoreListsSourceAssetsWithSummaryAndStateFilter(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + plan, err := ppconnector.BuildImportPlan(ppconnector.Snapshot{KBID: "kb-assets", Items: []ppconnector.Item{ + {KBID: "kb-assets", SourceURLHash: strings.Repeat("1", 64), SourceIDs: []string{"pp-5"}, Bytes: 10, SHA256: strings.Repeat("3", 64), Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-5"}, + {KBID: "kb-assets", SourceURLHash: strings.Repeat("2", 64), SourceIDs: []string{"pp-6"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit}, + }}) + if err != nil { + t.Fatal(err) + } + for index := range plan.SourceItems { + plan.SourceItems[index].Title = "合成手册.pdf" + plan.SourceItems[index].NormalizedPath = "使用指南" + } + if _, err := ImportPPLegacy(ctx, pool, plan); err != nil { + t.Fatal(err) + } + + page, err := NewStore(pool).ListSourceAssets(ctx, governance.SourceAssetFilter{State: "over_limit", Limit: 25}) + if err != nil { + t.Fatal(err) + } + if page.Summary != (governance.SourceAssetSummary{Total: 2, Completed: 1, OverLimit: 1}) { + t.Fatalf("summary = %#v", page.Summary) + } + if len(page.Items) != 1 || page.Items[0].ContentState != governance.ContentOverLimit || page.Items[0].HasPublication { + t.Fatalf("items = %#v", page.Items) + } + item := page.Items[0] + if item.Title != "合成手册.pdf" || item.DirectoryPath != "使用指南" || item.ConnectionID != plan.Connection.ID || item.VisibilityOrigin != "pp_public_tag" || item.CanRetry { + t.Fatalf("document navigation metadata mismatch: %#v", item) + } +} diff --git a/internal/platform/postgres/processing_confirmations.go b/internal/platform/postgres/processing_confirmations.go new file mode 100644 index 0000000..a6bf63b --- /dev/null +++ b/internal/platform/postgres/processing_confirmations.go @@ -0,0 +1,137 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +func (s *Store) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) { + result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)} + operator := strings.TrimSpace(request.OperatorRef) + reason := strings.TrimSpace(request.Reason) + if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() { + return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation") + } + seen := make(map[string]bool, len(request.PublicationIDs)) + for _, publicationID := range request.PublicationIDs { + if !validRouteID(publicationID) || seen[publicationID] { + return governance.StartTestPublicationsResult{}, errors.New("invalid processing publication IDs") + } + seen[publicationID] = true + } + + var tx pgx.Tx + var err error + ownsTransaction := false + if existing, ok := s.db.(pgx.Tx); ok { + tx = existing + } else { + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return governance.StartTestPublicationsResult{}, errors.New("processing confirmation transactions are unavailable") + } + var err error + tx, err = beginner.Begin(ctx) + if err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("begin processing confirmation: %w", err) + } + ownsTransaction = true + defer tx.Rollback(ctx) + } + + for _, publicationID := range request.PublicationIDs { + var ready, confirmed, uploadJobExists bool + err = tx.QueryRow(ctx, ` +SELECT p.ingest_state='pending' + AND p.processing_state='pending' + AND p.retrieval_state='unpublished' + AND p.knowledge_id='' + AND version.content_state='verified' + AND content.content_state='verified' + AND content.local_spool_ref<>'' + AND content.size_bytes>0 AS ready, + EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id), + EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id) +FROM test_publications p +JOIN source_item_versions version ON version.id=p.source_item_version_id +JOIN content_objects content ON content.id=version.content_object_id +WHERE p.id=$1 +FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists) + if errors.Is(err, pgx.ErrNoRows) { + result.Unavailable++ + continue + } + if err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("inspect processing plan: %w", err) + } + if confirmed || uploadJobExists { + result.AlreadyStarted++ + continue + } + if !ready { + result.Unavailable++ + continue + } + confirmationID, err := newOpaqueID("pc-") + if err != nil { + return governance.StartTestPublicationsResult{}, err + } + jobID, err := newOpaqueID("job-") + if err != nil { + return governance.StartTestPublicationsResult{}, err + } + payload, err := json.Marshal(map[string]string{"publication_id": publicationID}) + if err != nil { + return governance.StartTestPublicationsResult{}, errors.New("encode processing upload job") + } + if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,created_at) VALUES($1,$2,$3,$4,$5,$5)`, confirmationID, publicationID, operator, reason, request.Now); err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("record processing confirmation: %w", err) + } + if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at) +VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3)`, jobID, payload, request.Now, "test-start:"+publicationID); err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("enqueue confirmed processing: %w", err) + } + result.Started++ + } + if ownsTransaction { + if err := tx.Commit(ctx); err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("commit processing confirmation: %w", err) + } + } + return result, nil +} + +func (s *Store) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) { + fingerprint, err := operationFingerprint(struct { + PublicationIDs []string `json:"publication_ids"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + }{request.PublicationIDs, request.OperatorRef, request.Reason}) + if err != nil { + return governance.StartTestPublicationsResult{}, false, err + } + raw, replayed, err := s.executeOperation(ctx, "test.publications.start", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + result, err := NewStore(tx).StartTestPublications(ctx, request) + if err != nil { + return nil, err + } + return json.Marshal(result) + }) + if err != nil { + return governance.StartTestPublicationsResult{}, false, err + } + var result governance.StartTestPublicationsResult + if err := json.Unmarshal(raw, &result); err != nil { + return governance.StartTestPublicationsResult{}, false, fmt.Errorf("decode stored processing result: %w", err) + } + return result, replayed, nil +} diff --git a/internal/platform/postgres/production_delete_integration_test.go b/internal/platform/postgres/production_delete_integration_test.go new file mode 100644 index 0000000..4a2f874 --- /dev/null +++ b/internal/platform/postgres/production_delete_integration_test.go @@ -0,0 +1,126 @@ +package postgres + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestProductionDeleteBatchIsReversibleAndReconcilesPlan(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 10, 5, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil { + t.Fatal(err) + } + publicationID := "tp-delete-e2e" + insertReviewFixture(t, ctx, pool, 7, publicationID, now) + review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}) + if err != nil { + t.Fatal(err) + } + initialBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Initial publish", CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: initialBatch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + initialExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: initialBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + initialPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: initialExecution.ID, JobID: initialExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)}) + if err != nil || len(initialPlan) != 1 || initialPlan[0].Action != "upsert" { + t.Fatalf("initial plan=%#v error=%v", initialPlan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: initialExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "absent", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, initialExecution.ID, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-7','blocked','engineer-1','确认删除生产知识',$1,$1)`, now.Add(8*time.Minute)); err != nil { + t.Fatal(err) + } + deletePlan, err := store.ListReleasePlan(ctx) + if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != publicationID { + t.Fatalf("delete plan=%#v error=%v", deletePlan, err) + } + deleteBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Delete production knowledge", CreatedBy: "engineer-1", DeletePublicationIDs: []string{publicationID}, Now: now.Add(9 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + detail, err := store.GetReleaseBatch(ctx, deleteBatch.ID) + if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" { + t.Fatalf("delete batch=%#v error=%v", detail, err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: deleteBatch.ID, SealedBy: "engineer-2", Now: now.Add(10 * time.Minute)}); err != nil { + t.Fatal(err) + } + deleteExecution, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: deleteBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(11 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-7'`); err != nil { + t.Fatal(err) + } + if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12 * time.Minute)}); !errors.Is(err, ErrReleaseExecutionNotClaimable) { + t.Fatalf("unavailable source start error=%v", err) + } + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-7'`); err != nil { + t.Fatal(err) + } + deleteItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: deleteExecution.ID, JobID: deleteExecution.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(12*time.Minute + time.Second)}) + if err != nil || len(deleteItems) != 1 || deleteItems[0].Action != "delete" || deleteItems[0].PreviousProductionRevision != review.ID { + t.Fatalf("delete items=%#v error=%v", deleteItems, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: deleteExecution.ID, PublicationID: publicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: review.ID, ProductionRevision: "absent", Now: now.Add(13 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, deleteExecution.ID, now.Add(14*time.Minute)); err != nil { + t.Fatal(err) + } + afterDelete, err := store.ListReleasePlan(ctx) + if err != nil || len(afterDelete.Items) != 0 { + t.Fatalf("plan after delete=%#v error=%v", afterDelete, err) + } + rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: deleteExecution.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(15 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + rollbackItems, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)}) + if err != nil || len(rollbackItems) != 1 || rollbackItems[0].PreviousProductionRevision != review.ID { + t.Fatalf("rollback items=%#v error=%v", rollbackItems, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: publicationID, PreviousProductionRevision: review.ID, ProductionRevision: review.ID, Now: now.Add(17 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil { + t.Fatal(err) + } + afterRollback, err := store.ListReleasePlan(ctx) + if err != nil || len(afterRollback.Items) != 1 || afterRollback.Items[0].Action != "delete" { + t.Fatalf("plan after rollback=%#v error=%v", afterRollback, err) + } +} diff --git a/internal/platform/postgres/publication_removal.go b/internal/platform/postgres/publication_removal.go new file mode 100644 index 0000000..30159ab --- /dev/null +++ b/internal/platform/postgres/publication_removal.go @@ -0,0 +1,65 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "knowledge-governance/internal/governance" +) + +type PublicationForRemoval struct { + ID string + TestKBID string + KnowledgeID string + RetrievalState governance.RetrievalState +} + +func (s *Store) LoadPublicationForRemoval(ctx context.Context, publicationID string) (PublicationForRemoval, error) { + var candidate PublicationForRemoval + var retrievalState string + err := s.db.QueryRow(ctx, `SELECT id,test_kb_id,knowledge_id,retrieval_state +FROM test_publications +WHERE id=$1 AND knowledge_id<>'' AND retrieval_state='disabling'`, publicationID).Scan( + &candidate.ID, &candidate.TestKBID, &candidate.KnowledgeID, &retrievalState, + ) + if errors.Is(err, pgx.ErrNoRows) { + return PublicationForRemoval{}, ErrPublicationStateConflict + } + if err != nil { + return PublicationForRemoval{}, fmt.Errorf("load publication for removal: %w", err) + } + candidate.RetrievalState = governance.RetrievalState(retrievalState) + return candidate, nil +} + +func (s *Store) RecordRemovalCompleted(ctx context.Context, publicationID string, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE test_publications +SET retrieval_state='disabled',last_error='',updated_at=$2 +WHERE id=$1 AND retrieval_state='disabling'`, publicationID, now) + if err != nil { + return fmt.Errorf("record publication removal: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) RecordRemovalFailure(ctx context.Context, publicationID, code string, now time.Time) error { + if code != "removal_identity_invalid" && code != "removal_remote_rejected" { + return ErrInvalidErrorCode + } + tag, err := s.db.Exec(ctx, `UPDATE test_publications +SET retrieval_state='failed',last_error=$2,updated_at=$3 +WHERE id=$1 AND retrieval_state='disabling'`, publicationID, code, now) + if err != nil { + return fmt.Errorf("record publication removal failure: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} diff --git a/internal/platform/postgres/queue_sql.go b/internal/platform/postgres/queue_sql.go new file mode 100644 index 0000000..37da72d --- /dev/null +++ b/internal/platform/postgres/queue_sql.go @@ -0,0 +1,169 @@ +package postgres + +const enqueueJobSQL = ` +INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at +) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8, $8) +ON CONFLICT (idempotency_key) +DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key +RETURNING id, type, payload, state, priority, available_at, + lease_owner, lease_until, attempt, max_attempts, + last_error, idempotency_key, created_at, updated_at +` + +const claimJobSQL = ` +WITH candidate AS ( + SELECT id + FROM jobs + WHERE ( + (state IN ('pending', 'retry') AND available_at <= $2) + OR (state = 'running' AND lease_until < $2) + ) + AND attempt < max_attempts + AND type = ANY($4::text[]) + ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT 1 +) +UPDATE jobs AS j +SET state = 'running', + started_at = $2, + lease_owner = $1, + lease_until = $3, + attempt = j.attempt + 1, + updated_at = $2 +FROM candidate +WHERE j.id = candidate.id +RETURNING j.id, j.type, j.payload, j.state, j.priority, j.available_at, + j.lease_owner, j.lease_until, j.attempt, j.max_attempts, + j.last_error, j.idempotency_key, j.created_at, j.updated_at +` + +const completeJobSQL = ` +UPDATE jobs +SET state = 'completed', + completed_at = $3, + lease_owner = NULL, + lease_until = NULL, + updated_at = $3 +WHERE id = $1 + AND lease_owner = $2 + AND state = 'running' +` + +const heartbeatJobSQL = ` +UPDATE jobs +SET lease_until = $4, + updated_at = $3 +WHERE id = $1 + AND lease_owner = $2 + AND state = 'running' +` + +const failJobSQL = ` +WITH updated AS ( + UPDATE jobs + SET state = CASE WHEN attempt >= max_attempts THEN 'failed' ELSE 'retry' END, + available_at = $6, + last_error = $3, + last_error_code = CASE WHEN $5 <> '' THEN $5 ELSE last_error_code END, + lease_owner = NULL, + lease_until = NULL, + updated_at = $4 + WHERE id = $1 + AND lease_owner = $2 + AND state = 'running' + RETURNING id, type, payload, last_error, + CASE WHEN last_error_code = '' THEN 'handler_error' ELSE last_error_code END AS error_code, + attempt, max_attempts, + COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv, + COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt, + state +) +INSERT INTO job_failures ( + job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure, + business_change_set_id, business_version_id, business_action, business_target_kb_id, + created_at +) +SELECT updated.id, updated.type, COALESCE(updated.payload->>'publication_id',''), updated.error_code, + updated.last_error, updated.attempt, updated.max_attempts, + updated.bc, updated.bv, updated.ba, updated.bt, $4 +FROM updated +WHERE updated.state = 'failed' AND updated.attempt > 0 +ON CONFLICT (job_id, attempt) DO UPDATE +SET publication_id = EXCLUDED.publication_id, + error_code = EXCLUDED.error_code, + last_error = EXCLUDED.last_error, + max_attempts_at_failure = EXCLUDED.max_attempts_at_failure, + business_change_set_id = EXCLUDED.business_change_set_id, + business_version_id = EXCLUDED.business_version_id, + business_action = EXCLUDED.business_action, + business_target_kb_id = EXCLUDED.business_target_kb_id, + state = 'open', + resolved_at = NULL, + resolution_actor_ref = '', + resolution_reason = '', + created_at = EXCLUDED.created_at +` + +const listJobsSQL = ` +SELECT id, type, payload, state, priority, available_at, + lease_owner, lease_until, attempt, max_attempts, + last_error, idempotency_key, created_at, updated_at +FROM jobs +ORDER BY created_at DESC, id DESC +LIMIT $1 +` + +// P6: a permanent failure parks the job in 'failed' immediately and projects +// the terminal attempt into the dead-letter table. +const failTerminalJobSQL = ` +WITH parked AS ( + UPDATE jobs + SET state = 'failed', + available_at = $5, + last_error = $3, + last_error_code = CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END, + lease_owner = NULL, + lease_until = NULL, + updated_at = $5 + WHERE id = $1 + AND lease_owner = $2 + AND lease_owner IS NOT NULL + AND state = 'running' + RETURNING id, type, payload, last_error, + CASE WHEN $4 = '' THEN 'handler_error' ELSE $4 END AS error_code, + attempt, max_attempts, + COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv, + COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt +) +INSERT INTO job_failures ( + job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure, + business_change_set_id, business_version_id, business_action, business_target_kb_id, + created_at +) +SELECT parked.id, parked.type, COALESCE(parked.payload->>'publication_id',''), parked.error_code, + parked.last_error, GREATEST(parked.attempt,1), parked.max_attempts, + parked.bc, parked.bv, parked.ba, parked.bt, $5 +FROM parked +ON CONFLICT (job_id, attempt) DO UPDATE +SET publication_id = EXCLUDED.publication_id, + error_code = EXCLUDED.error_code, + last_error = EXCLUDED.last_error, + max_attempts_at_failure = EXCLUDED.max_attempts_at_failure, + state = 'open', + resolved_at = NULL, + resolution_actor_ref = '', + resolution_reason = '', + created_at = EXCLUDED.created_at +` + +// P10 liveness: every claim attempt refreshes the worker row so the API can +// distinguish "worker loop alive" from "process dead" without an HTTP port. +const workerHeartbeatSQL = ` +INSERT INTO worker_heartbeats(worker_id, job_types, started_at, last_seen_at) +VALUES($1, $2, $3, $3) +ON CONFLICT (worker_id) DO UPDATE +SET job_types = EXCLUDED.job_types, last_seen_at = EXCLUDED.last_seen_at +` diff --git a/internal/platform/postgres/r1_release_change_set_integration_test.go b/internal/platform/postgres/r1_release_change_set_integration_test.go new file mode 100644 index 0000000..06da473 --- /dev/null +++ b/internal/platform/postgres/r1_release_change_set_integration_test.go @@ -0,0 +1,317 @@ +package postgres + +import ( + "context" + "errors" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// seedR1ReleaseFixture gives us: +// - tp-keep: fully review+acceptance approved, present source -> add +// - tp-drop: was released to prod, source now deleted -> delete +// - tp-block: source unavailable -> blocked +// - tp-unch: already released to prod with same revision -> unchanged +func seedR1ReleaseFixture(t *testing.T, pool *pgxpool.Pool, ctx context.Context) { + t.Helper() + _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name) VALUES + ('conn-r1','alist','R1来源'); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES + ('co-r1',repeat('a',64),10,'spool/r1','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,normalized_path,revision,visibility,visibility_origin,source_change,identity_confidence) VALUES + ('item-keep','conn-r1','ext-keep',repeat('1',64),'新增手册','document','新增手册.pdf','v1','internal','manual','unchanged','stable_id'), + ('item-drop','conn-r1','ext-drop',repeat('2',64),'已删手册','document','已删手册.pdf','v1','internal','manual','deleted','stable_id'), + ('item-block','conn-r1','ext-block',repeat('3',64),'不可用手册','document','不可用手册.pdf','v1','internal','manual','unavailable','stable_id'), + ('item-unch','conn-r1','ext-unch',repeat('4',64),'不变手册','document','不变手册.pdf','v1','internal','manual','unchanged','stable_id'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_state) VALUES + ('siv-keep','item-keep','v1','verified'), + ('siv-drop','item-drop','v1','verified'), + ('siv-block','item-block','v1','verified'), + ('siv-unch','item-unch','v1','verified'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,processing_state,ingest_state,retrieval_state,updated_at) VALUES + ('tp-keep','siv-keep','kb-r1','internal','completed','uploaded','unpublished',now() - interval '1 hour'), + ('tp-drop','siv-drop','kb-r1','internal','completed','uploaded','unpublished',now() - interval '2 hours'), + ('tp-block','siv-block','kb-r1','internal','completed','uploaded','unpublished',now() - interval '3 hours'), + ('tp-unch','siv-unch','kb-r1','internal','completed','uploaded','unpublished',now() - interval '4 hours'); +INSERT INTO acceptance_results(id,publication_id,run_id,check_type,state,policy_version,observed_at) VALUES + ('ace-keep-1','tp-keep','run-keep','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'), + ('ace-keep-2','tp-keep','run-keep','chunks_present','passed','structural-v1',now() - interval '50 minutes'), + ('ace-keep-3','tp-keep','run-keep','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'), + ('ace-unch-1','tp-unch','run-unch','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'), + ('ace-unch-2','tp-unch','run-unch','chunks_present','passed','structural-v1',now() - interval '50 minutes'), + ('ace-unch-3','tp-unch','run-unch','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'), + ('ace-drop-1','tp-drop','run-drop','knowledge_exists','passed','structural-v1',now() - interval '50 minutes'), + ('ace-drop-2','tp-drop','run-drop','chunks_present','passed','structural-v1',now() - interval '50 minutes'), + ('ace-drop-3','tp-drop','run-drop','retrieval_hit','passed','structural-v1',now() - interval '50 minutes'); +INSERT INTO publication_reviews(id,publication_id,decision,decided_visibility,reviewer_ref,decision_code,acceptance_run_id,publication_updated_at,created_at) +SELECT 'rev-'||p.id, p.id, 'approved', 'internal', 'reviewer-r1', 'auto_approve', split_part(a.run_id,'',1), p.updated_at, now() +FROM test_publications p +JOIN LATERAL (SELECT CASE p.id WHEN 'tp-keep' THEN 'run-keep' WHEN 'tp-unch' THEN 'run-unch' ELSE 'run-drop' END AS run_id) a ON TRUE +WHERE p.id IN ('tp-keep','tp-unch','tp-drop'); +INSERT INTO release_batches(id,name,state,policy_version,created_by,item_count,created_at,sealed_at,release_change_set_id) VALUES + ('rb-seed','历史批次','sealed','release-gate-v1','seed',2,now() - interval '1 day',now() - interval '1 day',NULL); +INSERT INTO release_batch_items(release_batch_id,publication_id,review_id,source_item_version_id,acceptance_run_id,publication_updated_at,decided_visibility,action,created_at) VALUES + ('rb-seed','tp-unch','rev-tp-unch','siv-unch','run-unch',(SELECT updated_at FROM test_publications WHERE id='tp-unch'),'internal','upsert',now() - interval '1 day'), + ('rb-seed','tp-drop','rev-tp-drop','siv-drop','run-drop',(SELECT updated_at FROM test_publications WHERE id='tp-drop'),'internal','upsert',now() - interval '1 day'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES + ('snap-r1','conn-r1',repeat('7',64),'R1快照',2,1,1,now() - interval '1 day'); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES + ('conn-r1','snap-r1',now() - interval '1 day'); +INSERT INTO jobs(id,type,payload,state,available_at,attempt,max_attempts,idempotency_key) VALUES + ('job-r1-seed','release.publish','{}','completed',now() - interval '23 hours',1,5,'kr1-seed'); +INSERT INTO release_executions(id,release_batch_id,operation,state,authorized_by,authorization_code,job_id,item_count,started_at,completed_at) VALUES + ('re-seed','rb-seed','publish','completed','seed','seed_auth', + 'job-r1-seed',2,now() - interval '1 day',now() - interval '23 hours'); +INSERT INTO release_item_executions(id,release_execution_id,publication_id,production_revision,state,completed_at) +VALUES ('rie-seed','re-seed','tp-unch','rev-tp-unch','verified',now() - interval '23 hours'), + ('rie-seed-2','re-seed','tp-drop','rev-tp-drop','verified',now() - interval '23 hours');`) + if err != nil { + t.Fatalf("seed r1 fixture: %v", err) + } +} + +// TestR1ReleaseChangeSetLifecycle freezes docs plan R1: generation snapshots the +// live plan immutably, confirmation is required before formal batching, and +// later source refreshes never move a frozen manifest. +func TestR1ReleaseChangeSetLifecycle(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, time.September, 12, 7, 0, 0, 0, time.UTC) + + plan, err := store.ListReleasePlan(ctx) + if err == nil && len(plan.Items) > 0 { + t.Skip("fixture isolation failed: live plan not empty") + } + + seedR1ReleaseFixture(t, pool, ctx) + first, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布", CreatedBy: "yaozhanghao", Now: now}) + if err != nil { + t.Fatal(err) + } + if first.State != "planned" || first.Number != 1 || first.ManifestSHA256 == "" || len(first.ManifestSHA256) != 64 { + t.Fatalf("first set = %#v", first) + } + if len(first.SourceSnapshotIDs) == 0 { + t.Fatalf("R1 core: change set must bind the head catalog snapshots it was computed against") + } + bound := false + for _, id := range first.SourceSnapshotIDs { + if id == "snap-r1" { + bound = true + } + } + if !bound { + t.Fatalf("head snapshot not bound: %#v", first.SourceSnapshotIDs) + } + + detail, err := store.GetReleaseChangeSet(ctx, first.ID) + if err != nil { + t.Fatal(err) + } + actions := map[string]string{} + blockedRows := 0 + for _, item := range detail.Items { + actions[item.PublicationID] = item.Action + if item.Action == "blocked" { + blockedRows++ + } + } + if actions["tp-keep"] != "add" || actions["tp-drop"] != "delete" { + t.Fatalf("frozen actions = %#v", actions) + } + // R1 core rule: an unavailable source can ONLY surface as blocked (never a + // deletable/updatable item with a real publication binding). + if blockedRows != 1 || actions["tp-block"] != "" && actions["tp-block"] != "blocked" { + t.Fatalf("blocked handling = %#v rows=%d", actions, blockedRows) + } + if _, ok := actions["tp-unch"]; ok { + t.Fatal("unchanged items must not enter the manifest") + } + // Manifest recompute must match byte-exact (stable ordering + delimiter). + if got := ComputeReleaseChangeSetManifest(detail.Items); got != first.ManifestSHA256 { + t.Fatalf("manifest not reproducible: %s vs %s", got, first.ManifestSHA256) + } + + // Source mutates AFTER freezing: dropping an item that already exists in + // production turns its live-plan row into delete — but the frozen manifest + // must not move (plan changes, set does not). + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='deleted' WHERE id='item-unch'`); err != nil { + t.Fatal(err) + } + after, err := store.GetReleaseChangeSet(ctx, first.ID) + if err != nil { + t.Fatal(err) + } + if after.ManifestSHA256 != first.ManifestSHA256 || len(after.Items) != len(detail.Items) { + t.Fatal("frozen change set mutated after source change") + } + // Fresh live plan reflects the mutation (proof the set is NOT live-derived). + mutated, err := store.ListReleasePlan(ctx) + if err != nil { + t.Fatal(err) + } + mutationSeen := false + for _, item := range mutated.Items { + if item.PublicationID == "tp-unch" && item.Action == "delete" { + mutationSeen = true + } + } + if !mutationSeen { + t.Fatal("live plan failed to reflect post-freeze source change") + } + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='item-unch'`); err != nil { + t.Fatal(err) + } + + // Dedupe on the SAME plan: repeated generation reuses the latest PLANNED + // set (concurrent clicks must not pile up versions). + deduped, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "重复", CreatedBy: "yaozhanghao", Now: now.Add(time.Minute)}) + if err != nil { + t.Fatal(err) + } + if deduped.ID != first.ID || deduped.Number != 1 { + t.Fatalf("dedupe failed: %#v vs %#v", deduped, first) + } + + // Batching BEFORE confirmation is rejected with the honest reason. + _, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "太早了", CreatedBy: "yaozhanghao", + PublicationIDs: []string{"tp-keep"}, + ReleaseChangeSetID: first.ID, Now: now, + }) + if !errors.Is(err, ErrReleaseChangeSetUnconfirmed) { + t.Fatalf("unconfirmed batching err = %v", err) + } + + // Confirm (idempotent replay on re-confirm). + confirmed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "boss", Now: now}) + if err != nil || confirmed.State != "confirmed" || confirmed.ConfirmedBy != "boss" { + t.Fatalf("confirm = %#v err = %v", confirmed, err) + } + replayed, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: first.ID, ConfirmedBy: "someone-else", Now: now.Add(time.Minute)}) + if err != nil || replayed.ConfirmedBy != "boss" { + t.Fatalf("re-confirm must replay stored confirmation: %#v err=%v", replayed, err) + } + + // A new generation after confirmation takes the next version number — the + // confirmed set stays untouched forever (page refresh cannot alter it). + second, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "9月发布v2", CreatedBy: "yaozhanghao", Now: now.Add(2 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if second.Number != 2 || second.ID == first.ID { + t.Fatalf("post-confirm generation = %#v", second) + } + stillFirst, err := store.GetReleaseChangeSet(ctx, first.ID) + if err != nil || stillFirst.ManifestSHA256 != first.ManifestSHA256 || stillFirst.State != "confirmed" { + t.Fatalf("confirmed set mutated: %#v err=%v", stillFirst.ReleaseChangeSet, err) + } + + // The formal batch citing the CONFIRMED set succeeds and keeps its binding. + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "正式批次", CreatedBy: "yaozhanghao", + PublicationIDs: []string{"tp-keep"}, + ReleaseChangeSetID: first.ID, Now: now, + }) + if err != nil { + t.Fatalf("bound batch: %v", err) + } + if batch.ReleaseChangeSetID != first.ID { + t.Fatalf("batch lost its binding: %#v", batch) + } + + // A batch citing items NOT in the confirmed manifest is rejected. + _, err = store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "越界批次", CreatedBy: "yaozhanghao", + PublicationIDs: []string{"tp-block"}, + ReleaseChangeSetID: first.ID, Now: now, + }) + if !errors.Is(err, ErrReleaseBatchNotEligible) && !errors.Is(err, ErrReleaseChangeSetUnconfirmed) { + t.Fatalf("out-of-manifest batch err = %v", err) + } + + // Concurrency: 8 parallel generations on the SAME plan produce exactly one + // new version (dedupe) or all fail on the gate — never corrupt duplicates. + if _, err := pool.Exec(ctx, `DELETE FROM release_batch_items WHERE release_batch_id IN (SELECT id FROM release_batches WHERE release_change_set_id IS NOT NULL); +DELETE FROM release_batches WHERE release_change_set_id IS NOT NULL; +DELETE FROM release_change_sets`); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + ids := make([]string, 8) + errs := make([]error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + set, genErr := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "并发", CreatedBy: "yaozhanghao", Now: now}) + ids[index], errs[index] = set.ID, genErr + }(i) + } + wg.Wait() + unique := map[string]bool{} + for index, genErr := range errs { + if genErr != nil { + t.Fatalf("concurrent generate[%d]: %v", index, genErr) + } + unique[ids[index]] = true + } + var rowCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&rowCount); err != nil { + t.Fatal(err) + } + if rowCount != len(unique) { + t.Fatalf("concurrent generation produced %d rows but %d distinct ids", rowCount, len(unique)) + } + if len(unique) > 2 { + t.Fatalf("dedupe collapsed under concurrency: %d distinct sets", len(unique)) + } + + // The concurrent winner set is a valid, stable, hex manifest. + var winnerID, manifest string + if err := pool.QueryRow(ctx, `SELECT id, manifest_sha256 FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&winnerID, &manifest); err != nil { + t.Fatal(err) + } + if len(manifest) != 64 || strings.TrimLeft(manifest, "0123456789abcdef") != "" { + t.Fatalf("manifest shape broken: %q", manifest) + } + winner, err := store.GetReleaseChangeSet(ctx, winnerID) + if err != nil || len(winner.Items) == 0 { + t.Fatalf("winner detail = %#v err=%v", winner, err) + } + if ComputeReleaseChangeSetManifest(winner.Items) != manifest { + t.Fatal("stored manifest does not recompute from stored items") + } + + // Page listing. + page, err := store.ListReleaseChangeSets(ctx, governance.ReleaseBatchFilter{Limit: 10}) + if err != nil || page.Total < 1 { + t.Fatalf("page = %#v err = %v", page, err) + } + if _, err := store.LatestReleaseChangeSet(ctx); err != nil { + t.Fatalf("latest: %v", err) + } + if _, err := store.GetReleaseChangeSet(ctx, "rcs-missing"); !errors.Is(err, ErrReleaseChangeSetNotFound) { + t.Fatalf("missing detail err = %v", err) + } +} diff --git a/internal/platform/postgres/r4_rollback_levels_integration_test.go b/internal/platform/postgres/r4_rollback_levels_integration_test.go new file mode 100644 index 0000000..be69996 --- /dev/null +++ b/internal/platform/postgres/r4_rollback_levels_integration_test.go @@ -0,0 +1,236 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +// TestR4RollbackLevels freezes docs plan R4: publish failures grade into +// single_item/batch, rollbacks walk rolling_back -> rolled_back / +// rollback_failed, a failed rollback escalates the prior publish to overall, +// and an overall-graded publish can no longer be compensated by plain +// rollback authorization (snapshot restore is R2 territory and must be gated). +func TestR4RollbackLevels(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) + // The shared review fixture binds to conn-manual and packs its URI hash via + // repeat(, 64), so indexes must stay single-hex-digit. + if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil { + t.Fatal(err) + } + refs := []string{"tp-r4-a", "tp-r4-b"} + for index, publicationID := range refs { + insertReviewFixture(t, ctx, pool, index+1, publicationID, now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + } + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次", CreatedBy: "engineer-1", PublicationIDs: refs, Now: now.Add(2 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)}) + if err != nil || len(plan) != 2 { + t.Fatalf("publish plan=%#v err=%v", plan, err) + } + + // ---- single_item grade: first item fails permanently, nothing verified. + if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + // Second item also fails so FinishExecution can settle the execution. + if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + var level string + if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil { + t.Fatal(err) + } + if level != "single_item" { + t.Fatalf("zero-progress failure must grade single_item, got %q", level) + } + // Nothing was ever verified: auto rollback is a no-op (idempotent, no rows). + if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil { + t.Fatal(err) + } + if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil { + t.Fatal(err) + } + var rollbackRows int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_executions WHERE prior_execution_id=$1`, publish.ID).Scan(&rollbackRows); err != nil { + t.Fatal(err) + } + if rollbackRows != 0 { + t.Fatalf("single_item with zero verified must schedule no rollback, got %d", rollbackRows) + } + + // ---- batch grade: reset items, retry, verify one, fail the other. + // (The job is still pending in this store-level fixture; a real worker run + // would have parked it failed. Retry requires a terminal job, so we set it.) + if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, publish.JobID); err != nil { + t.Fatal(err) + } + retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: publish.ID, Now: now.Add(10 * time.Minute)}) + if err != nil || retried.State != "pending" { + t.Fatalf("retry publish = %#v err=%v", retried, err) + } + plan, err = store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(11 * time.Minute)}) + if err != nil || len(plan) != 2 { + t.Fatalf("retry plan=%#v err=%v", plan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("b", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(12 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(12 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, publish.ID, now.Add(13*time.Minute)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish.ID).Scan(&level); err != nil { + t.Fatal(err) + } + if level != "batch" { + t.Fatalf("partial-progress failure must grade batch, got %q", level) + } + + // ---- batch rollback: rolling_back -> rolled_back with restart re-claim. + if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(14*time.Minute)); err != nil { + t.Fatal(err) + } + rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10}) + if err != nil || rollbacks.Total != 1 { + t.Fatalf("rollbacks=%#v err=%v", rollbacks, err) + } + rollback := rollbacks.Items[0] + rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(15 * time.Minute)}) + if err != nil || len(rollbackPlan) != 1 { + t.Fatalf("rollback plan=%#v err=%v", rollbackPlan, err) + } + // R4: a claimed rollback runs in the dedicated rolling_back stage. + var stage string + if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil { + t.Fatal(err) + } + if stage != "rolling_back" { + t.Fatalf("rollback claim must enter rolling_back, got %q", stage) + } + // Worker restart mid-rollback: the same execution is re-claimable. + if _, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(16 * time.Minute)}); err != nil { + t.Fatalf("re-claim after restart: %v", err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: rollback.ID, PublicationID: rollbackPlan[0].PublicationID, BundleSHA256: "", PreviousProductionRevision: "prod-before", ProductionRevision: "restored-v0", Now: now.Add(17 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, rollback.ID, now.Add(18*time.Minute)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, rollback.ID).Scan(&stage); err != nil { + t.Fatal(err) + } + if stage != "rolled_back" { + t.Fatalf("successful compensation must land rolled_back, got %q", stage) + } + + // ---- second batch with another verified item so we can fail the rollback. + refs2 := []string{"tp-r4-b"} + batch2, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R4批次2", CreatedBy: "engineer-1", PublicationIDs: refs2, DeletePublicationIDs: []string{}, Now: now.Add(19 * time.Minute)}) + if err == nil { + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch2.ID, SealedBy: "engineer-2", Now: now.Add(20 * time.Minute)}); err != nil { + t.Fatalf("seal batch2: %v", err) + } + publish2, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(21 * time.Minute)}) + if err != nil { + t.Fatalf("authorize batch2: %v", err) + } + plan2, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish2.ID, JobID: publish2.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(22 * time.Minute)}) + if err != nil || len(plan2) != 1 { + t.Fatalf("publish2 plan=%#v err=%v", plan2, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish2.ID, PublicationID: plan2[0].PublicationID, BundleSHA256: strings.Repeat("c", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan2[0].ReleaseRevision, Now: now.Add(23 * time.Minute)}); err != nil { + t.Fatal(err) + } + // Finish publish2 successfully so its item stays verified. + if err := store.FinishExecution(ctx, publish2.ID, now.Add(24*time.Minute)); err != nil { + t.Fatal(err) + } + // Authorize a rollback for the completed publish2 and make it fail. + manual, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(25 * time.Minute)}) + if err != nil { + t.Fatalf("authorize rollback: %v", err) + } + manualPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: manual.ID, JobID: manual.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(26 * time.Minute)}) + _ = manualPlan + if err != nil { + t.Fatalf("start manual rollback: %v", err) + } + // The compensated publication now has a newer publication row; fail the + // rollback item permanently. + if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: manual.ID, PublicationID: plan2[0].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(27 * time.Minute)}); err != nil { + t.Fatalf("fail rollback item: %v", err) + } + if err := store.FinishExecution(ctx, manual.ID, now.Add(28*time.Minute)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT state FROM release_executions WHERE id=$1`, manual.ID).Scan(&stage); err != nil { + t.Fatal(err) + } + if stage != "rollback_failed" { + t.Fatalf("failed rollback must land rollback_failed, got %q", stage) + } + // Escalation: the prior publish is now 'overall' (needs snapshot restore). + if err := pool.QueryRow(ctx, `SELECT failure_level FROM release_executions WHERE id=$1`, publish2.ID).Scan(&level); err != nil { + t.Fatal(err) + } + if level != "overall" { + t.Fatalf("failed rollback must escalate prior publish to overall, got %q", level) + } + // Overall-graded publish can no longer be plain-rollback authorized. + if _, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish2.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(29 * time.Minute)}); err == nil { + t.Fatal("overall-graded publish must not accept plain rollback authorization") + } + // A permanently failed rollback stays retriable (re-runs compensation). + if _, err := pool.Exec(ctx, `UPDATE jobs SET state='failed', attempt=max_attempts, lease_owner=NULL, lease_until=NULL WHERE id=$1`, manual.JobID); err != nil { + t.Fatal(err) + } + retriedRollback, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: manual.ID, Now: now.Add(30 * time.Minute)}) + if err != nil || retriedRollback.State != "pending" { + t.Fatalf("retry failed rollback = %#v err=%v", retriedRollback, err) + } + } else { + // Batch creation legitimately refused only when tp-r4-b has no + // publish-eligible state; in this fixture it should succeed. + t.Fatalf("batch2 create: %v", err) + } +} diff --git a/internal/platform/postgres/r5_manifest_seal_integration_test.go b/internal/platform/postgres/r5_manifest_seal_integration_test.go new file mode 100644 index 0000000..729a175 --- /dev/null +++ b/internal/platform/postgres/r5_manifest_seal_integration_test.go @@ -0,0 +1,123 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +// TestR5ExecutionManifestSealAndMismatch freezes docs plan R5: finishing an +// execution seals a deterministic manifest of what actually happened, and a +// "success" that does not match the frozen release change set intent is +// downgraded to failed — never reported as completed. +func TestR5ExecutionManifestSealAndMismatch(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, time.September, 12, 13, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual')`); err != nil { + t.Fatal(err) + } + insertReviewFixture(t, ctx, pool, 1, "tp-r5-a", now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + + runExecution := func(publicationID, tag string) (executionID, jobID string) { + t.Helper() + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R5批次-" + tag, CreatedBy: "engineer-1", PublicationIDs: []string{publicationID}, Now: now.Add(2 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)}) + if err != nil || len(plan) != 1 { + t.Fatalf("plan=%#v err=%v", plan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("d", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + return publish.ID, publish.JobID + } + + // Happy path (unbound batch): manifest is sealed, deterministic, non-empty. + executionID, _ := runExecution("tp-r5-a", "happy") + if err := store.FinishExecution(ctx, executionID, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + var sealed, state, publicationID, bundleSHA, revision string + if err := pool.QueryRow(ctx, `SELECT e.manifest_sha256, e.state, i.publication_id, i.bundle_sha256, i.production_revision +FROM release_executions e JOIN release_item_executions i ON i.release_execution_id=e.id WHERE e.id=$1`, executionID).Scan(&sealed, &state, &publicationID, &bundleSHA, &revision); err != nil { + t.Fatal(err) + } + if state != "completed" { + t.Fatalf("unbound success must stay completed, got %q", state) + } + if len(sealed) != 64 || sealed != ComputeReleaseExecutionManifest("publish", "", []string{publicationID + "|" + bundleSHA + "|" + revision}) { + t.Fatalf("manifest not sealed/deterministic: %q", sealed) + } + // Re-seal must be a no-op (immutable once set). + if err := store.FinishExecution(ctx, executionID, now.Add(8*time.Minute)); err == nil { + t.Fatal("settled execution must not finish twice") + } + + // Mismatch path: simulate a tampered/legacy row where the bound frozen + // change set does NOT contain the publication that was "successfully" + // released — R5 forbids reporting success. + insertReviewFixture(t, ctx, pool, 2, "tp-r5-b", now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r5-b", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + secondExecution, jobID := runExecution("tp-r5-b", "mismatch") + executionID = secondExecution + if _, err := pool.Exec(ctx, `INSERT INTO release_change_sets(id,name,number,state,manifest_sha256,created_by,acceptance_cutoff) VALUES ('rcs-r5-x','不相干计划集',99,'confirmed',repeat('a',64),'tamper',$1)`, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES ('rcs-r5-x',1,'tp-other','别人','internal','add','ready','absent',true,'')`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE release_executions SET release_change_set_id='rcs-r5-x' WHERE id=$1`, executionID); err != nil { + t.Fatal(err) + } + // Reset to running so FinishExecution can settle again for this scenario. + if _, err := pool.Exec(ctx, `UPDATE release_executions SET state='running', manifest_sha256='', completed_at=NULL WHERE id=$1`, executionID); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, executionID, now.Add(9*time.Minute)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT state, error_code, manifest_sha256 FROM release_executions WHERE id=$1`, executionID).Scan(&state, &bundleSHA, &sealed); err != nil { + t.Fatal(err) + } + if state != "failed" || bundleSHA != "release_manifest_mismatch" { + t.Fatalf("mismatch must downgrade to failed, got state=%q code=%q", state, bundleSHA) + } + if len(sealed) != 64 { + t.Fatalf("mismatched execution must still seal its actual manifest, got %q", sealed) + } + _ = jobID +} diff --git a/internal/platform/postgres/r6_audit_chain_integration_test.go b/internal/platform/postgres/r6_audit_chain_integration_test.go new file mode 100644 index 0000000..8a327dc --- /dev/null +++ b/internal/platform/postgres/r6_audit_chain_integration_test.go @@ -0,0 +1,144 @@ +package postgres + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +// TestR6ReleaseAuditChain freezes docs plan R6: one execution traces through +// real FK rows to batch -> confirmed release change set -> bound snapshots and +// catalog change sets -> per-item review/acceptance/source identity, and the +// linked evidence cannot be deleted underneath the chain. +func TestR6ReleaseAuditChain(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, time.September, 12, 11, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name) VALUES ('conn-manual','manual','Manual'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) VALUES + ('snap-r6','conn-manual',repeat('e',64),'R6快照',3,2,1,now() - interval '2 hours'); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES + ('conn-manual','snap-r6',now() - interval '2 hours'); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,state,add_count,update_count,move_scope_count,delete_count,confirmed_at) VALUES + ('cs-r6','conn-manual',NULL,'snap-r6','confirmed',1,0,0,0,now() - interval '90 minutes');`); err != nil { + t.Fatal(err) + } + insertReviewFixture(t, ctx, pool, 1, "tp-r6-a", now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: "tp-r6-a", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + + set, err := store.GenerateReleaseChangeSet(ctx, governance.GenerateReleaseChangeSetRequest{Name: "R6发布", CreatedBy: "yaozhanghao", Now: now}) + if err != nil { + t.Fatal(err) + } + if _, err := store.ConfirmReleaseChangeSet(ctx, governance.ConfirmReleaseChangeSetRequest{ChangeSetID: set.ID, ConfirmedBy: "boss", Now: now}); err != nil { + t.Fatal(err) + } + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "R6批次", CreatedBy: "yaozhanghao", PublicationIDs: []string{"tp-r6-a"}, ReleaseChangeSetID: set.ID, Now: now.Add(time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(2 * time.Minute)}); err != nil { + t.Fatal(err) + } + publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(3 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(4 * time.Minute)}) + if err != nil || len(plan) != 1 { + t.Fatalf("plan=%#v err=%v", plan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(5 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, publish.ID, now.Add(6*time.Minute)); err != nil { + t.Fatal(err) + } + + chain, err := store.GetReleaseAuditChain(ctx, publish.ID) + if err != nil { + t.Fatal(err) + } + if chain.ExecutionID != publish.ID || chain.Operation != "publish" || chain.State != "completed" { + t.Fatalf("head = %#v", chain) + } + if chain.BatchID != batch.ID || chain.BatchName != "R6批次" || chain.CreatedBy != "yaozhanghao" || chain.SealedBy != "engineer-2" || chain.AuthorizedBy != "release-owner" { + t.Fatalf("batch links = %#v", chain) + } + if chain.ReleaseChangeSetID != set.ID || chain.ChangeSetName != "R6发布" || chain.ManifestSHA256 != set.ManifestSHA256 || chain.ChangeSetConfirmedBy != "boss" { + t.Fatalf("change set links = %#v", chain) + } + if len(chain.Snapshots) != 1 || chain.Snapshots[0].ID != "snap-r6" || chain.Snapshots[0].SourceName != "Manual" || chain.Snapshots[0].FileCount != 1 { + t.Fatalf("snapshot links = %#v", chain.Snapshots) + } + if len(chain.CatalogChangeSets) != 1 || chain.CatalogChangeSets[0].ID != "cs-r6" || chain.CatalogChangeSets[0].State != "confirmed" { + t.Fatalf("catalog links = %#v", chain.CatalogChangeSets) + } + if len(chain.ManifestSHA256) != 64 { + t.Fatalf("manifest = %q", chain.ManifestSHA256) + } + if len(chain.Items) != 1 { + t.Fatalf("items = %#v", chain.Items) + } + item := chain.Items[0] + if item.PublicationID != "tp-r6-a" || item.Action != "upsert" || item.ItemState != "verified" || item.ProductionRevision != plan[0].ReleaseRevision { + t.Fatalf("item = %#v", item) + } + if item.ReviewID == "" || item.AcceptanceRunID == "" || item.SourceItemID == "" || item.SourceVersionID == "" { + t.Fatalf("item evidence missing: %#v", item) + } + + // R6 integrity: FK rows protect the evidence — the bound snapshot and the + // confirmed release change set cannot be deleted while the chain exists. + if _, err := pool.Exec(ctx, `DELETE FROM source_catalog_snapshots WHERE id='snap-r6'`); err == nil { + t.Fatal("snapshot delete must be RESTRICTed by the audit chain") + } + if _, err := pool.Exec(ctx, `DELETE FROM release_change_sets WHERE id=$1`, set.ID); err == nil { + t.Fatal("release change set delete must be RESTRICTed by batch/execution bindings") + } + // The catalog chain row links are real FK rows, not just JSONB. + var linkRows int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_set_snapshot_links WHERE change_set_id=$1`, set.ID).Scan(&linkRows); err != nil || linkRows != 1 { + t.Fatalf("snapshot link rows = %d err=%v", linkRows, err) + } + + // Rollback chain: authorize a rollback of the completed publish and see it + // appear on the original execution's projection. + rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{PriorExecutionID: publish.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(7 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + publishChain, err := store.GetReleaseAuditChain(ctx, publish.ID) + if err != nil || publishChain.RollbackExecutionID != rollback.ID { + t.Fatalf("publish chain rollback = %#v err=%v", publishChain, err) + } + rollbackChain, err := store.GetReleaseAuditChain(ctx, rollback.ID) + if err != nil || rollbackChain.PriorExecutionID != publish.ID || rollbackChain.ReleaseChangeSetID != set.ID { + t.Fatalf("rollback chain = %#v err=%v", rollbackChain, err) + } + if _, err := store.GetReleaseAuditChain(ctx, "rx-unknown"); err != ErrReleaseAuditChainNotFound { + t.Fatalf("unknown chain err = %v", err) + } +} diff --git a/internal/platform/postgres/release_audit_chain.go b/internal/platform/postgres/release_audit_chain.go new file mode 100644 index 0000000..da807f9 --- /dev/null +++ b/internal/platform/postgres/release_audit_chain.go @@ -0,0 +1,155 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// ErrReleaseAuditChainNotFound is returned for an unknown execution id. +var ErrReleaseAuditChainNotFound = errors.New("release audit chain not found") + +const releaseAuditChainHeadSQL = ` +SELECT e.id, e.operation, e.state, e.failure_level, + COALESCE(e.release_batch_id,''), COALESCE(b.name,''), COALESCE(b.created_by,''), COALESCE(b.sealed_by,''), + COALESCE(e.authorized_by,''), COALESCE(e.release_change_set_id::text,''), + COALESCE(rcs.name,''), COALESCE(rcs.number,0), COALESCE(rcs.state,''), + COALESCE(rcs.created_by,''), COALESCE(rcs.confirmed_by,''), rcs.confirmed_at, + COALESCE(rcs.manifest_sha256,''), COALESCE(e.prior_execution_id,''), + COALESCE((SELECT rb.id FROM release_executions AS rb + WHERE rb.prior_execution_id = e.id AND rb.operation = 'rollback' + ORDER BY rb.created_at DESC LIMIT 1),'') +FROM release_executions AS e +LEFT JOIN release_batches AS b ON b.id = e.release_batch_id +LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id +WHERE e.id = $1 +` + +const releaseAuditSnapshotLinksSQL = ` +SELECT snap.id, COALESCE(conn.name,''), snap.node_count, snap.file_count, snap.observed_at +FROM release_change_set_snapshot_links AS link +JOIN source_catalog_snapshots AS snap ON snap.id = link.snapshot_id +LEFT JOIN source_connections AS conn ON conn.id = snap.source_connection_id +WHERE link.change_set_id = $1 +ORDER BY snap.id +` + +const releaseAuditCatalogLinksSQL = ` +SELECT cset.id, cset.state, cset.add_count, cset.update_count, + cset.move_count, cset.delete_count +FROM release_change_set_catalog_links AS link +JOIN source_catalog_change_sets AS cset ON cset.id = link.catalog_change_set_id +WHERE link.change_set_id = $1 +ORDER BY cset.id +` + +const releaseAuditItemsSQL = ` +SELECT COALESCE(batch_item.publication_id, item.publication_id), + COALESCE(version_title.title, ''), + COALESCE(item.action, batch_item.action), + COALESCE(item.state, 'pending'), + COALESCE(batch_item.review_id, ''), + COALESCE(batch_item.acceptance_run_id, ''), + COALESCE(version.source_item_id, ''), + COALESCE(batch_item.source_item_version_id, ''), + COALESCE(item.production_revision, ''), + COALESCE(item.error_code, '') +FROM release_item_executions AS item +JOIN test_publications AS publication ON publication.id = item.publication_id +JOIN source_item_versions AS version ON version.id = publication.source_item_version_id +JOIN source_items AS version_title ON version_title.id = version.source_item_id +LEFT JOIN release_batch_items AS batch_item + ON batch_item.release_batch_id = $2 AND batch_item.publication_id = item.publication_id +WHERE item.release_execution_id = $1 +ORDER BY item.id +` + +// GetReleaseAuditChain projects the complete R6 evidence chain for one +// execution: execution -> batch -> release change set -> bound source +// snapshots and catalog change sets -> per-item review/acceptance/source +// identity, plus the compensating rollback row when one exists. Every link is +// read through real foreign keys (migration 30), not denormalized JSON. Only +// safe identifiers, labels, counts and timestamps are selected — never bundle +// paths, credentials, base URLs, or signed object URLs. +func (s *Store) GetReleaseAuditChain(ctx context.Context, executionID string) (governance.ReleaseAuditChain, error) { + if !validShortID(executionID) { + return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound + } + var chain governance.ReleaseAuditChain + err := s.db.QueryRow(ctx, releaseAuditChainHeadSQL, executionID).Scan( + &chain.ExecutionID, &chain.Operation, &chain.State, &chain.FailureLevel, + &chain.BatchID, &chain.BatchName, &chain.CreatedBy, &chain.SealedBy, + &chain.AuthorizedBy, &chain.ReleaseChangeSetID, + &chain.ChangeSetName, &chain.ChangeSetNumber, &chain.ChangeSetState, + &chain.ChangeSetCreatedBy, &chain.ChangeSetConfirmedBy, &chain.ChangeSetConfirmedAt, + &chain.ManifestSHA256, &chain.PriorExecutionID, + &chain.RollbackExecutionID, + ) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseAuditChain{}, ErrReleaseAuditChainNotFound + } + if err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("load release audit chain: %w", err) + } + chain.Snapshots = []governance.AuditSnapshot{} + chain.CatalogChangeSets = []governance.AuditCatalogSet{} + chain.Items = []governance.AuditChainItem{} + if chain.ReleaseChangeSetID != "" { + rows, err := s.db.Query(ctx, releaseAuditSnapshotLinksSQL, chain.ReleaseChangeSetID) + if err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("list release snapshot links: %w", err) + } + for rows.Next() { + var snap governance.AuditSnapshot + if err := rows.Scan(&snap.ID, &snap.SourceName, &snap.NodeCount, &snap.FileCount, &snap.ObservedAt); err != nil { + rows.Close() + return governance.ReleaseAuditChain{}, fmt.Errorf("scan release snapshot link: %w", err) + } + chain.Snapshots = append(chain.Snapshots, snap) + } + if err := rows.Err(); err != nil { + rows.Close() + return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release snapshot links: %w", err) + } + rows.Close() + rows, err = s.db.Query(ctx, releaseAuditCatalogLinksSQL, chain.ReleaseChangeSetID) + if err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("list release catalog links: %w", err) + } + for rows.Next() { + var set governance.AuditCatalogSet + if err := rows.Scan(&set.ID, &set.State, &set.AddCount, &set.UpdateCount, &set.MoveCount, &set.DeleteCount); err != nil { + rows.Close() + return governance.ReleaseAuditChain{}, fmt.Errorf("scan release catalog link: %w", err) + } + chain.CatalogChangeSets = append(chain.CatalogChangeSets, set) + } + if err := rows.Err(); err != nil { + rows.Close() + return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release catalog links: %w", err) + } + rows.Close() + } + rows, err := s.db.Query(ctx, releaseAuditItemsSQL, executionID, chain.BatchID) + if err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("list release audit items: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.AuditChainItem + if err := rows.Scan(&item.PublicationID, &item.Title, &item.Action, &item.ItemState, + &item.ReviewID, &item.AcceptanceRunID, &item.SourceItemID, &item.SourceVersionID, + &item.ProductionRevision, &item.ErrorCode); err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("scan release audit item: %w", err) + } + chain.Items = append(chain.Items, item) + } + if err := rows.Err(); err != nil { + return governance.ReleaseAuditChain{}, fmt.Errorf("iterate release audit items: %w", err) + } + return chain, nil +} diff --git a/internal/platform/postgres/release_authorization_integration_test.go b/internal/platform/postgres/release_authorization_integration_test.go new file mode 100644 index 0000000..d875670 --- /dev/null +++ b/internal/platform/postgres/release_authorization_integration_test.go @@ -0,0 +1,258 @@ +package postgres + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestReleaseAuthorizationPersistentIntegration(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil { + t.Fatal(err) + } + insertReviewFixture(t, ctx, pool, 10, "tp-auth", now) + review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{ + PublicationID: "tp-auth", Decision: governance.ReviewApproved, + DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", + DecisionCode: "approved_for_release", Now: now.Add(time.Minute), + }) + if err != nil || review.ID == "" { + t.Fatalf("review=%#v error=%v", review, err) + } + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "Authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auth"}, Now: now.Add(2 * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + request := governance.AuthorizeReleaseBatchRequest{ + BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute), + } + first, err := store.AuthorizeReleaseBatch(ctx, request) + if err != nil { + t.Fatal(err) + } + second, err := store.AuthorizeReleaseBatch(ctx, request) + if err != nil { + t.Fatal(err) + } + if first.ID != second.ID || first.JobID != second.JobID || first.ItemCount != 1 { + t.Fatalf("first=%#v second=%#v", first, second) + } + plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ + ExecutionID: first.ID, JobID: first.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute), + }) + if err != nil || len(plan) != 1 || plan[0].PublicationID != "tp-auth" { + t.Fatalf("publish plan=%#v error=%v", plan, err) + } + if plan[0].ReleaseExecutionID != first.ID || plan[0].SourceRevision != "revision-10" || plan[0].ParseRevision != "parser-v1@index-v1" || plan[0].ReleaseRevision != review.ID { + t.Fatalf("publish revisions=%#v", plan[0]) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ + ExecutionID: first.ID, PublicationID: "tp-auth", BundleSHA256: strings.Repeat("a", 64), + PreviousProductionRevision: "prod-v0", ProductionRevision: review.ID, Now: now.Add(6 * time.Minute), + }); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, first.ID, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + currentPlan, err := store.ListReleasePlan(ctx) + if err != nil || len(currentPlan.Items) != 0 { + t.Fatalf("current release plan=%#v error=%v", currentPlan, err) + } + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unavailable' WHERE id='si-review-10'`); err != nil { + t.Fatal(err) + } + unavailablePlan, err := store.ListReleasePlan(ctx) + if err != nil || len(unavailablePlan.Items) != 1 || unavailablePlan.Items[0].Action != "blocked" || unavailablePlan.Items[0].CanRelease || unavailablePlan.Items[0].Blocker != "source_unavailable" { + t.Fatalf("unavailable release plan=%#v error=%v", unavailablePlan, err) + } + if _, err := pool.Exec(ctx, `UPDATE source_items SET source_change='unchanged' WHERE id='si-review-10'`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) VALUES('si-review-10','blocked','engineer-1','禁止进入生产',$1,$1)`, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + deletePlan, err := store.ListReleasePlan(ctx) + if err != nil || len(deletePlan.Items) != 1 || deletePlan.Items[0].Action != "delete" || !deletePlan.Items[0].CanRelease || deletePlan.Items[0].PublicationID != "tp-auth" || deletePlan.Items[0].Blocker != "" { + t.Fatalf("delete release plan=%#v error=%v", deletePlan, err) + } + rollback, err := store.AuthorizeReleaseRollback(ctx, governance.AuthorizeReleaseRollbackRequest{ + PriorExecutionID: first.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_rollback_authorized", Now: now.Add(8 * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ + ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(9 * time.Minute), + }) + if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PreviousProductionRevision != "prod-v0" { + t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ + ExecutionID: rollback.ID, PublicationID: "tp-auth", ProductionRevision: "prod-v0", Now: now.Add(10 * time.Minute), + }); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, rollback.ID, now.Add(11*time.Minute)); err != nil { + t.Fatal(err) + } + for _, mutation := range []struct { + sql string + args []any + }{ + {`UPDATE release_item_executions SET state='failed', error_code='release_verification_failed', completed_at=$2 WHERE release_execution_id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}}, + {`UPDATE release_executions SET state='failed', completed_item_count=0, failed_item_count=1, error_code='release_item_failed', completed_at=$2 WHERE id=$1`, []any{rollback.ID, now.Add(12 * time.Minute)}}, + {`UPDATE jobs SET state='completed', attempt=3, completed_at=$2 WHERE id=$1`, []any{rollback.JobID, now.Add(12 * time.Minute)}}, + } { + if _, err := pool.Exec(ctx, mutation.sql, mutation.args...); err != nil { + t.Fatal(err) + } + } + retried, err := store.RetryReleaseExecution(ctx, governance.RetryReleaseExecutionRequest{ExecutionID: rollback.ID, Now: now.Add(13 * time.Minute)}) + if err != nil || retried.State != governance.ReleaseExecutionPending || retried.FailedItemCount != 0 { + t.Fatalf("retried=%#v error=%v", retried, err) + } + var executionState, itemState, jobState string + var attempt int + if err := pool.QueryRow(ctx, `SELECT execution.state, item.state, job.state, job.attempt FROM release_executions execution JOIN release_item_executions item ON item.release_execution_id=execution.id JOIN jobs job ON job.id=execution.job_id WHERE execution.id=$1`, rollback.ID).Scan(&executionState, &itemState, &jobState, &attempt); err != nil { + t.Fatal(err) + } + if executionState != "pending" || itemState != "pending" || jobState != "pending" || attempt != 0 { + t.Fatalf("states=%s/%s/%s attempt=%d", executionState, itemState, jobState, attempt) + } + + insertReviewFixture(t, ctx, pool, 11, "tp-stale-auth", now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{ + PublicationID: "tp-stale-auth", Decision: governance.ReviewApproved, + DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", + DecisionCode: "approved_for_release", Now: now.Add(5 * time.Minute), + }); err != nil { + t.Fatal(err) + } + staleBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "Stale authorization batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-stale-auth"}, Now: now.Add(6 * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: staleBatch.ID, SealedBy: "engineer-2", Now: now.Add(7 * time.Minute)}); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE test_publications SET updated_at = updated_at + INTERVAL '1 second' WHERE id = 'tp-stale-auth'`); err != nil { + t.Fatal(err) + } + _, err = store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{ + BatchID: staleBatch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(8 * time.Minute), + }) + if !errors.Is(err, ErrReleaseAuthorizationGateConflict) { + t.Fatalf("stale authorization error=%v", err) + } + + var executions, jobs, itemExecutions, completed, rolledBack, pending int + if err := pool.QueryRow(ctx, `SELECT + (SELECT COUNT(*)::INTEGER FROM release_executions), + (SELECT COUNT(*)::INTEGER FROM jobs WHERE type = 'release.publish'), + (SELECT COUNT(*)::INTEGER FROM release_item_executions), + (SELECT COUNT(*)::INTEGER FROM release_executions WHERE state = 'completed'), + (SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'rolled_back'), + (SELECT COUNT(*)::INTEGER FROM release_item_executions WHERE state = 'pending')`).Scan(&executions, &jobs, &itemExecutions, &completed, &rolledBack, &pending); err != nil { + t.Fatal(err) + } + if executions != 2 || jobs != 1 || itemExecutions != 2 || completed != 1 || rolledBack != 0 || pending != 1 { + t.Fatalf("executions=%d publishJobs=%d itemExecutions=%d completed=%d rolledBack=%d pending=%d", executions, jobs, itemExecutions, completed, rolledBack, pending) + } +} + +func TestFailedPublishSchedulesIdempotentRollbackForVerifiedItemsOnly(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 10, 3, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual')`); err != nil { + t.Fatal(err) + } + for index, publicationID := range []string{"tp-auto-a", "tp-auto-b"} { + insertReviewFixture(t, ctx, pool, 8+index, publicationID, now) + if _, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{PublicationID: publicationID, Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + } + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{Name: "Automatic rollback batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-auto-a", "tp-auto-b"}, Now: now.Add(2 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + publish, err := store.AuthorizeReleaseBatch(ctx, governance.AuthorizeReleaseBatchRequest{BatchID: batch.ID, AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now.Add(4 * time.Minute)}) + if err != nil { + t.Fatal(err) + } + plan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: publish.ID, JobID: publish.JobID, Operation: governance.ReleaseOperationPublish, Now: now.Add(5 * time.Minute)}) + if err != nil || len(plan) != 2 { + t.Fatalf("publish plan=%#v error=%v", plan, err) + } + if err := store.CompleteItem(ctx, releaseworkflow.CompleteItemRequest{ExecutionID: publish.ID, PublicationID: plan[0].PublicationID, BundleSHA256: strings.Repeat("a", 64), PreviousProductionRevision: "prod-before", ProductionRevision: plan[0].ReleaseRevision, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FailItem(ctx, releaseworkflow.FailItemRequest{ExecutionID: publish.ID, PublicationID: plan[1].PublicationID, ErrorCode: releaseworkflow.ErrorCodeContractMismatch, Now: now.Add(6 * time.Minute)}); err != nil { + t.Fatal(err) + } + if err := store.FinishExecution(ctx, publish.ID, now.Add(7*time.Minute)); err != nil { + t.Fatal(err) + } + if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(8*time.Minute)); err != nil { + t.Fatal(err) + } + if err := store.ScheduleAutomaticRollback(ctx, publish.ID, now.Add(9*time.Minute)); err != nil { + t.Fatal(err) + } + rollbacks, err := store.ListReleaseExecutions(ctx, governance.ReleaseExecutionFilter{Operation: "rollback", Limit: 10}) + if err != nil || rollbacks.Total != 1 || len(rollbacks.Items) != 1 || rollbacks.Items[0].ItemCount != 1 || rollbacks.Items[0].AuthorizationCode != "automatic_rollback_after_failure" { + t.Fatalf("rollbacks=%#v error=%v", rollbacks, err) + } + rollback := rollbacks.Items[0] + rollbackPlan, err := store.StartExecution(ctx, releaseworkflow.StartExecutionRequest{ExecutionID: rollback.ID, JobID: rollback.JobID, Operation: governance.ReleaseOperationRollback, Now: now.Add(10 * time.Minute)}) + if err != nil || len(rollbackPlan) != 1 || rollbackPlan[0].PublicationID != plan[0].PublicationID || rollbackPlan[0].PreviousProductionRevision != "prod-before" { + t.Fatalf("rollback plan=%#v error=%v", rollbackPlan, err) + } +} diff --git a/internal/platform/postgres/release_batches.go b/internal/platform/postgres/release_batches.go new file mode 100644 index 0000000..739b51f --- /dev/null +++ b/internal/platform/postgres/release_batches.go @@ -0,0 +1,450 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +const releaseGatePolicyVersion = "release-gate-v1" + +var ( + ErrInvalidReleaseBatch = errors.New("invalid release batch") + ErrInvalidReleaseBatchFilter = errors.New("invalid release batch filter") + ErrReleaseBatchNotEligible = errors.New("release batch contains ineligible publications") + ErrReleaseBatchGateConflict = errors.New("release batch gate conflict") + ErrReleaseBatchNotFound = errors.New("release batch not found") + ErrReleaseChangeSetUnconfirmed = errors.New("release change set must be confirmed before batching") +) + +const releaseGateCTEs = ` +acceptance_run_summary AS ( + SELECT publication_id, run_id, MAX(observed_at) AS observed_at, + COALESCE(MAX(state) FILTER (WHERE check_type = 'knowledge_exists'), '') AS knowledge_exists, + COALESCE(MAX(state) FILTER (WHERE check_type = 'chunks_present'), '') AS chunks_present, + COALESCE(MAX(state) FILTER (WHERE check_type = 'retrieval_hit'), '') AS retrieval_hit + FROM acceptance_results + GROUP BY publication_id, run_id +), latest_acceptance AS ( + SELECT DISTINCT ON (publication_id) + publication_id, run_id, knowledge_exists, chunks_present, retrieval_hit, observed_at + FROM acceptance_run_summary + ORDER BY publication_id, observed_at DESC, run_id DESC +), latest_review AS ( + SELECT DISTINCT ON (publication_id) + id, publication_id, decision, decided_visibility, acceptance_run_id, + publication_updated_at, created_at + FROM publication_reviews + ORDER BY publication_id, created_at DESC, id DESC +), release_ready AS ( + SELECT publication.id AS publication_id, + publication.source_item_version_id, + publication.updated_at AS publication_updated_at, + review.id AS review_id, + review.acceptance_run_id, + review.decided_visibility + FROM test_publications AS publication + JOIN source_item_versions AS version ON version.id = publication.source_item_version_id + JOIN source_items AS source ON source.id = version.source_item_id + JOIN latest_acceptance AS acceptance ON acceptance.publication_id = publication.id + JOIN publication_reviews AS review ON review.id = ( + SELECT latest_review.id FROM latest_review WHERE latest_review.publication_id = publication.id + ) + WHERE review.decision = 'approved' + AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = publication.updated_at + AND publication.ingest_state = 'uploaded' + AND publication.processing_state = 'completed' + AND publication.retrieval_state = 'unpublished' + AND version.revision = source.revision + AND source.source_change NOT IN ('deleted', 'unavailable') + AND acceptance.knowledge_exists = 'passed' + AND acceptance.chunks_present = 'passed' + AND acceptance.retrieval_hit IN ('passed', 'manual_required') +) +` + +const releaseDeleteReadyCTEs = ` +source_truth AS ( + SELECT source.id, source.source_change, + COALESCE(item_rule.visibility, directory_rule.visibility, source.visibility) AS desired_policy + FROM source_items AS source + LEFT JOIN source_item_scope_overrides AS item_rule ON item_rule.source_item_id = source.id + LEFT JOIN LATERAL ( + SELECT rule.visibility + FROM directory_scope_rules AS rule + WHERE rule.source_connection_id = source.source_connection_id + AND (rule.directory_path = '' OR source.normalized_path = rule.directory_path OR source.normalized_path LIKE rule.directory_path || '/%') + ORDER BY LENGTH(rule.directory_path) DESC, rule.directory_path DESC + LIMIT 1 + ) AS directory_rule ON TRUE +), latest_production AS ( + SELECT DISTINCT ON (version.source_item_id, publication.target_scope) + publication.id AS publication_id, version.source_item_id, publication.target_scope, + execution_item.production_revision, + batch_item.review_id, batch_item.source_item_version_id, + batch_item.acceptance_run_id, batch_item.publication_updated_at, + batch_item.decided_visibility + FROM release_item_executions AS execution_item + JOIN release_executions AS execution ON execution.id = execution_item.release_execution_id + JOIN test_publications AS publication ON publication.id = execution_item.publication_id + JOIN source_item_versions AS version ON version.id = publication.source_item_version_id + JOIN release_batch_items AS batch_item + ON batch_item.release_batch_id = execution.release_batch_id + AND batch_item.publication_id = publication.id + WHERE execution.state IN ('completed', 'rolled_back') + AND execution_item.state IN ('verified', 'rolled_back') + ORDER BY version.source_item_id, publication.target_scope, + execution.completed_at DESC, execution_item.completed_at DESC, execution_item.id DESC +), release_delete_ready AS ( + SELECT production.publication_id, production.source_item_version_id, + production.publication_updated_at, production.review_id, + production.acceptance_run_id, production.decided_visibility, + 'delete' AS action + FROM latest_production AS production + JOIN source_truth AS truth ON truth.id = production.source_item_id + WHERE production.production_revision <> 'absent' + AND truth.source_change <> 'unavailable' + AND ( + truth.source_change = 'deleted' + OR truth.desired_policy = 'blocked' + OR (production.target_scope = 'internal' AND truth.desired_policy NOT IN ('internal', 'both')) + OR (production.target_scope = 'external' AND truth.desired_policy NOT IN ('external', 'both')) + ) +), release_upsert_ready AS ( + SELECT ready.* + FROM release_ready AS ready + JOIN test_publications AS publication ON publication.id = ready.publication_id + JOIN source_item_versions AS version ON version.id = publication.source_item_version_id + JOIN source_truth AS truth ON truth.id = version.source_item_id + LEFT JOIN latest_production AS production + ON production.source_item_id = version.source_item_id + AND production.target_scope = publication.target_scope + WHERE truth.source_change NOT IN ('deleted', 'unavailable') + AND truth.desired_policy <> 'blocked' + AND publication.target_scope = ready.decided_visibility + AND ( + (publication.target_scope = 'internal' AND truth.desired_policy IN ('internal', 'both')) + OR (publication.target_scope = 'external' AND truth.desired_policy IN ('external', 'both')) + ) + AND (production.production_revision IS NULL OR production.production_revision = 'absent' OR production.production_revision <> ready.review_id) +) +` + +const createReleaseBatchSQL = ` +WITH requested AS ( + SELECT value AS publication_id, 'upsert' AS action + FROM jsonb_array_elements_text($5::jsonb) + UNION ALL + SELECT value AS publication_id, 'delete' AS action + FROM jsonb_array_elements_text($6::jsonb) +), ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `, +request_stats AS ( + SELECT COUNT(*)::INTEGER AS requested_count, + COUNT(DISTINCT publication_id)::INTEGER AS distinct_count + FROM requested +), binding AS ( + -- R1: when a batch cites a release change set, the set must be CONFIRMED + -- and every requested publication must exist in its frozen manifest as a + -- releasable item. Source refreshes can never move this goalpost. + SELECT 1 + WHERE $7::text = '' OR EXISTS ( + SELECT 1 FROM release_change_sets rcs + WHERE rcs.id = $7 AND rcs.state = 'confirmed' + AND NOT EXISTS ( + SELECT 1 FROM requested q + WHERE NOT EXISTS ( + SELECT 1 FROM release_change_set_items frozen + WHERE frozen.change_set_id = rcs.id + AND frozen.publication_id = q.publication_id + AND frozen.can_release + ) + ) + ) +), eligible AS ( + SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at, + ready.review_id, ready.acceptance_run_id, ready.decided_visibility, + requested.action + FROM release_upsert_ready AS ready + JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'upsert' + UNION ALL + SELECT ready.publication_id, ready.source_item_version_id, ready.publication_updated_at, + ready.review_id, ready.acceptance_run_id, ready.decided_visibility, + ready.action + FROM release_delete_ready AS ready + JOIN requested ON requested.publication_id = ready.publication_id AND requested.action = 'delete' +), created_batch AS ( + INSERT INTO release_batches ( + id, name, state, policy_version, created_by, item_count, created_at, release_change_set_id + ) + SELECT $1, $2, 'draft', 'release-gate-v1', $3, request_stats.requested_count, $4, NULLIF($7::text, '') + FROM request_stats + WHERE EXISTS (SELECT 1 FROM binding) + AND request_stats.requested_count BETWEEN 1 AND 100 + AND request_stats.requested_count = request_stats.distinct_count + AND request_stats.requested_count = (SELECT COUNT(*) FROM eligible) + RETURNING id, name, state, policy_version, created_by, sealed_by, + item_count, created_at, sealed_at, release_change_set_id +), inserted_items AS ( + INSERT INTO release_batch_items ( + release_batch_id, publication_id, review_id, source_item_version_id, + acceptance_run_id, publication_updated_at, decided_visibility, action, created_at + ) + SELECT created_batch.id, eligible.publication_id, eligible.review_id, + eligible.source_item_version_id, eligible.acceptance_run_id, + eligible.publication_updated_at, eligible.decided_visibility, eligible.action, $4 + FROM created_batch CROSS JOIN eligible + RETURNING release_batch_id +) +SELECT id, name, state, policy_version, created_by, sealed_by, + item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'') +FROM created_batch +WHERE item_count = (SELECT COUNT(*) FROM inserted_items) +` + +const sealReleaseBatchSQL = ` +WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `, +item_stats AS ( + SELECT batch.id AS batch_id, + COUNT(item.publication_id)::INTEGER AS total_count, + COUNT(*) FILTER (WHERE + (item.action = 'upsert' AND ready.publication_id IS NOT NULL) + OR (item.action = 'delete' AND delete_ready.publication_id IS NOT NULL) + )::INTEGER AS valid_count + FROM release_batches AS batch + LEFT JOIN release_batch_items AS item ON item.release_batch_id = batch.id + LEFT JOIN release_upsert_ready AS ready + ON ready.publication_id = item.publication_id + AND item.action = 'upsert' + AND ready.review_id = item.review_id + AND ready.acceptance_run_id = item.acceptance_run_id + AND ready.publication_updated_at = item.publication_updated_at + AND ready.decided_visibility = item.decided_visibility + LEFT JOIN release_delete_ready AS delete_ready + ON delete_ready.publication_id = item.publication_id + AND item.action = 'delete' + AND delete_ready.review_id = item.review_id + AND delete_ready.acceptance_run_id = item.acceptance_run_id + AND delete_ready.publication_updated_at = item.publication_updated_at + AND delete_ready.decided_visibility = item.decided_visibility + WHERE batch.id = $1 AND batch.state = 'draft' + GROUP BY batch.id +), sealed AS ( + UPDATE release_batches AS batch + SET state = 'sealed', sealed_by = $2, sealed_at = $3 + FROM item_stats + WHERE batch.id = item_stats.batch_id + AND batch.policy_version = 'release-gate-v1' + AND item_stats.total_count > 0 + AND item_stats.total_count = item_stats.valid_count + AND batch.item_count = item_stats.total_count + RETURNING batch.id, batch.name, batch.state, batch.policy_version, + batch.created_by, batch.sealed_by, batch.item_count, + batch.created_at, batch.sealed_at, COALESCE(batch.release_change_set_id::text,'') +), already_sealed AS ( + SELECT id, name, state, policy_version, created_by, sealed_by, + item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'') + FROM release_batches + WHERE id = $1 AND state = 'sealed' +) +SELECT * FROM sealed +UNION ALL +SELECT * FROM already_sealed +LIMIT 1 +` + +const countReleaseBatchesSQL = ` +SELECT COUNT(*)::INTEGER +FROM release_batches +WHERE $1 = '' OR state = $1 +` + +const listReleaseBatchesSQL = ` +SELECT id, name, state, policy_version, created_by, sealed_by, + item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'') +FROM release_batches +WHERE $1 = '' OR state = $1 +ORDER BY created_at DESC, id DESC +LIMIT $2 OFFSET $3 +` + +const getReleaseBatchSQL = ` +SELECT id, name, state, policy_version, created_by, sealed_by, + item_count, created_at, sealed_at, COALESCE(release_change_set_id::text,'') +FROM release_batches +WHERE id = $1 +` + +const listReleaseBatchItemsSQL = ` +SELECT item.publication_id, source.title, item.decided_visibility, item.action +FROM release_batch_items AS item +JOIN test_publications AS publication ON publication.id = item.publication_id +JOIN source_item_versions AS version ON version.id = publication.source_item_version_id +JOIN source_items AS source ON source.id = version.source_item_id +WHERE item.release_batch_id = $1 +ORDER BY item.publication_id +` + +func (s *Store) CreateReleaseBatch(ctx context.Context, request governance.CreateReleaseBatchRequest) (governance.ReleaseBatch, error) { + if !validCreateReleaseBatch(request) { + return governance.ReleaseBatch{}, ErrInvalidReleaseBatch + } + batchID, err := newOpaqueID("rb-") + if err != nil { + return governance.ReleaseBatch{}, err + } + publicationIDs, err := json.Marshal(append([]string{}, request.PublicationIDs...)) + if err != nil { + return governance.ReleaseBatch{}, ErrInvalidReleaseBatch + } + deletePublicationIDs, err := json.Marshal(append([]string{}, request.DeletePublicationIDs...)) + if err != nil { + return governance.ReleaseBatch{}, ErrInvalidReleaseBatch + } + batch, err := scanReleaseBatch(s.db.QueryRow(ctx, createReleaseBatchSQL, + batchID, request.Name, request.CreatedBy, request.Now, publicationIDs, deletePublicationIDs, + strings.TrimSpace(request.ReleaseChangeSetID), + )) + if errors.Is(err, pgx.ErrNoRows) { + // R1: distinguish an unconfirmed/non-covering change set from plain + // gate ineligibility so operators get the honest reason. + if request.ReleaseChangeSetID != "" { + var state string + if lookup := s.db.QueryRow(ctx, `SELECT state FROM release_change_sets WHERE id=$1`, request.ReleaseChangeSetID).Scan(&state); lookup != nil || state != "confirmed" { + return governance.ReleaseBatch{}, ErrReleaseChangeSetUnconfirmed + } + } + return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return governance.ReleaseBatch{}, ErrReleaseBatchNotEligible + } + if err != nil { + return governance.ReleaseBatch{}, fmt.Errorf("create release batch: %w", err) + } + return batch, nil +} + +func (s *Store) SealReleaseBatch(ctx context.Context, request governance.SealReleaseBatchRequest) (governance.ReleaseBatch, error) { + if !validShortID(request.BatchID) || !validReviewerRef(request.SealedBy) || request.Now.IsZero() { + return governance.ReleaseBatch{}, ErrInvalidReleaseBatch + } + batch, err := scanReleaseBatch(s.db.QueryRow(ctx, sealReleaseBatchSQL, request.BatchID, request.SealedBy, request.Now)) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseBatch{}, ErrReleaseBatchGateConflict + } + if err != nil { + return governance.ReleaseBatch{}, fmt.Errorf("seal release batch: %w", err) + } + return batch, nil +} + +func (s *Store) ListReleaseBatches(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseBatchPage, error) { + if filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 || + filter.State != "" && filter.State != string(governance.ReleaseBatchDraft) && + filter.State != string(governance.ReleaseBatchSealed) && filter.State != string(governance.ReleaseBatchCancelled) { + return governance.ReleaseBatchPage{}, ErrInvalidReleaseBatchFilter + } + page := governance.ReleaseBatchPage{Items: []governance.ReleaseBatch{}, Limit: filter.Limit, Offset: filter.Offset} + if err := s.db.QueryRow(ctx, countReleaseBatchesSQL, filter.State).Scan(&page.Total); err != nil { + return governance.ReleaseBatchPage{}, fmt.Errorf("count release batches: %w", err) + } + rows, err := s.db.Query(ctx, listReleaseBatchesSQL, filter.State, filter.Limit, filter.Offset) + if err != nil { + return governance.ReleaseBatchPage{}, fmt.Errorf("list release batches: %w", err) + } + defer rows.Close() + for rows.Next() { + batch, err := scanReleaseBatch(rows) + if err != nil { + return governance.ReleaseBatchPage{}, fmt.Errorf("scan release batch: %w", err) + } + page.Items = append(page.Items, batch) + } + if err := rows.Err(); err != nil { + return governance.ReleaseBatchPage{}, fmt.Errorf("iterate release batches: %w", err) + } + return page, nil +} + +func (s *Store) GetReleaseBatch(ctx context.Context, batchID string) (governance.ReleaseBatchDetail, error) { + if !validShortID(batchID) { + return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound + } + batch, err := scanReleaseBatch(s.db.QueryRow(ctx, getReleaseBatchSQL, batchID)) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseBatchDetail{}, ErrReleaseBatchNotFound + } + if err != nil { + return governance.ReleaseBatchDetail{}, fmt.Errorf("get release batch: %w", err) + } + rows, err := s.db.Query(ctx, listReleaseBatchItemsSQL, batchID) + if err != nil { + return governance.ReleaseBatchDetail{}, fmt.Errorf("list release batch items: %w", err) + } + defer rows.Close() + detail := governance.ReleaseBatchDetail{ReleaseBatch: batch, Items: []governance.ReleaseBatchItemView{}} + for rows.Next() { + var item governance.ReleaseBatchItemView + var visibility string + if err := rows.Scan(&item.PublicationID, &item.Title, &visibility, &item.Action); err != nil { + return governance.ReleaseBatchDetail{}, fmt.Errorf("scan release batch item: %w", err) + } + item.DecidedVisibility = governance.Visibility(visibility) + if (item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) || (item.Action != "upsert" && item.Action != "delete") { + return governance.ReleaseBatchDetail{}, errors.New("invalid release batch item visibility") + } + detail.Items = append(detail.Items, item) + } + if err := rows.Err(); err != nil { + return governance.ReleaseBatchDetail{}, fmt.Errorf("iterate release batch items: %w", err) + } + return detail, nil +} + +func validCreateReleaseBatch(request governance.CreateReleaseBatchRequest) bool { + if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 255 || + strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) || + request.Now.IsZero() || len(request.PublicationIDs)+len(request.DeletePublicationIDs) < 1 || len(request.PublicationIDs)+len(request.DeletePublicationIDs) > 100 { + return false + } + if request.ReleaseChangeSetID != "" && !validRouteID(request.ReleaseChangeSetID) { + return false + } + seen := make(map[string]struct{}, len(request.PublicationIDs)+len(request.DeletePublicationIDs)) + for _, publicationID := range append(append([]string{}, request.PublicationIDs...), request.DeletePublicationIDs...) { + if !validShortID(publicationID) { + return false + } + if _, exists := seen[publicationID]; exists { + return false + } + seen[publicationID] = struct{}{} + } + return true +} + +func scanReleaseBatch(row rowScanner) (governance.ReleaseBatch, error) { + var batch governance.ReleaseBatch + var state string + if err := row.Scan( + &batch.ID, &batch.Name, &state, &batch.PolicyVersion, + &batch.CreatedBy, &batch.SealedBy, &batch.ItemCount, + &batch.CreatedAt, &batch.SealedAt, &batch.ReleaseChangeSetID, + ); err != nil { + return governance.ReleaseBatch{}, err + } + batch.State = governance.ReleaseBatchState(state) + if !batch.State.Valid() || batch.PolicyVersion != releaseGatePolicyVersion { + return governance.ReleaseBatch{}, errors.New("invalid release batch row") + } + return batch, nil +} diff --git a/internal/platform/postgres/release_batches_test.go b/internal/platform/postgres/release_batches_test.go new file mode 100644 index 0000000..edda9de --- /dev/null +++ b/internal/platform/postgres/release_batches_test.go @@ -0,0 +1,228 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestCreateReleaseBatchAtomicallyLocksOnlyApprovedCurrentItems(t *testing.T) { + now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC) + db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "draft", policy: "release-gate-v1", createdBy: "engineer-1", itemCount: 3, createdAt: now}} + batch, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{ + Name: "Synthetic batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-b"}, DeletePublicationIDs: []string{"tp-c"}, Now: now, + }) + if err != nil { + t.Fatalf("CreateReleaseBatch() error = %v", err) + } + if batch.ID != "rb-safe" || batch.State != governance.ReleaseBatchDraft || batch.ItemCount != 3 { + t.Fatalf("batch = %#v", batch) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "jsonb_array_elements_text", "publication_reviews", "decision = 'approved'", + "acceptance_run_id", "publication_updated_at", "knowledge_exists = 'passed'", + "chunks_present = 'passed'", "retrieval_hit in ('passed', 'manual_required')", + "insert into release_batches", "insert into release_batch_items", "count(distinct publication_id)", "returning", + "release_delete_ready", "'delete' as action", "eligible.action", + } { + if !strings.Contains(sql, required) { + t.Errorf("create batch SQL missing %q", required) + } + } + if strings.Contains(sql, "insert into jobs") || strings.Contains(sql, "release.probe") { + t.Fatal("P5 batch creation must not enqueue release jobs") + } +} + +func TestCreateReleaseBatchRejectsInvalidOrDuplicateInput(t *testing.T) { + now := time.Now() + requests := []governance.CreateReleaseBatchRequest{ + {Name: "", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: now}, + {Name: "Batch", CreatedBy: "", PublicationIDs: []string{"tp-a"}, Now: now}, + {Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: nil, Now: now}, + {Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a", "tp-a"}, Now: now}, + {Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, DeletePublicationIDs: []string{"tp-a"}, Now: now}, + } + for _, request := range requests { + if _, err := NewStore(&recordingDB{}).CreateReleaseBatch(context.Background(), request); !errors.Is(err, ErrInvalidReleaseBatch) { + t.Fatalf("request=%#v error=%v, want ErrInvalidReleaseBatch", request, err) + } + } +} + +func TestCreateReleaseBatchReturnsNotEligibleWhenAnyItemFailsGate(t *testing.T) { + db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}} + _, err := NewStore(db).CreateReleaseBatch(context.Background(), governance.CreateReleaseBatchRequest{ + Name: "Batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-a"}, Now: time.Now(), + }) + if !errors.Is(err, ErrReleaseBatchNotEligible) { + t.Fatalf("error=%v, want ErrReleaseBatchNotEligible", err) + } +} + +func TestSealReleaseBatchRevalidatesEveryLockedItemWithoutEnqueue(t *testing.T) { + now := time.Date(2026, 9, 6, 9, 10, 0, 0, time.UTC) + db := &recordingDB{row: releaseBatchRow{id: "rb-safe", name: "Synthetic batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 2, createdAt: now.Add(-time.Minute), sealedAt: &now}} + batch, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-2", Now: now}) + if err != nil { + t.Fatalf("SealReleaseBatch() error = %v", err) + } + if batch.State != governance.ReleaseBatchSealed || batch.SealedBy != "engineer-2" || batch.SealedAt == nil { + t.Fatalf("batch = %#v", batch) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "join release_batch_items", "join publication_reviews", + "ready.review_id = item.review_id", "ready.acceptance_run_id = item.acceptance_run_id", "ready.publication_updated_at = item.publication_updated_at", + "publication.ingest_state = 'uploaded'", "publication.processing_state = 'completed'", + "publication.retrieval_state = 'unpublished'", "state = 'sealed'", "state = 'draft'", + "release_delete_ready", "item.action = 'upsert'", "item.action = 'delete'", + } { + if !strings.Contains(sql, required) { + t.Errorf("seal SQL missing %q", required) + } + } + if strings.Contains(sql, "insert into jobs") { + t.Fatal("P5 seal must not enqueue a production release") + } +} + +func TestSealReleaseBatchReturnsGateConflict(t *testing.T) { + db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}} + _, err := NewStore(db).SealReleaseBatch(context.Background(), governance.SealReleaseBatchRequest{BatchID: "rb-safe", SealedBy: "engineer-1", Now: time.Now()}) + if !errors.Is(err, ErrReleaseBatchGateConflict) { + t.Fatalf("error=%v, want ErrReleaseBatchGateConflict", err) + } +} + +func TestListReleaseBatchesAndDetailReturnSafeProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 9, 20, 0, 0, time.UTC) + db := &recordingDB{ + row: releaseCountRow{count: 1}, + rows: &releaseBatchRows{values: []releaseBatchRow{{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now}}}, + } + page, err := NewStore(db).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "sealed", Limit: 20}) + if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].State != governance.ReleaseBatchSealed { + t.Fatalf("page=%#v error=%v", page, err) + } + joined := strings.ToLower(db.queryRowSQL + "\n" + db.querySQL) + if !strings.Contains(joined, "from release_batches") || !strings.Contains(joined, "state = $1") { + t.Fatalf("batch list SQL=%s", joined) + } + + detailDB := &releaseBatchReadDB{ + recordingDB: &recordingDB{}, + row: releaseBatchRow{id: "rb-safe", name: "Batch", state: "sealed", policy: "release-gate-v1", createdBy: "engineer-1", sealedBy: "engineer-2", itemCount: 1, createdAt: now, sealedAt: &now}, + rows: &releaseBatchItemRows{values: []releaseBatchItemRow{{publicationID: "tp-safe", title: "Synthetic", visibility: "internal", action: "delete"}}}, + } + detail, err := NewStore(detailDB).GetReleaseBatch(context.Background(), "rb-safe") + if err != nil || len(detail.Items) != 1 || detail.Items[0].Title != "Synthetic" || detail.Items[0].Action != "delete" { + t.Fatalf("detail=%#v error=%v", detail, err) + } + joined = strings.ToLower(detailDB.queryRowSQL + "\n" + detailDB.querySQL) + for _, forbidden := range []string{"knowledge_id", "test_kb_id", "acceptance_run_id", "publication_updated_at", "stable_external_id", "local_spool_ref"} { + if strings.Contains(joined, forbidden) { + t.Errorf("batch read SQL includes forbidden projection %q", forbidden) + } + } +} + +func TestListReleaseBatchesRejectsInvalidFilter(t *testing.T) { + _, err := NewStore(&recordingDB{}).ListReleaseBatches(context.Background(), governance.ReleaseBatchFilter{State: "released", Limit: 20}) + if !errors.Is(err, ErrInvalidReleaseBatchFilter) { + t.Fatalf("error=%v, want ErrInvalidReleaseBatchFilter", err) + } +} + +type releaseBatchRow struct { + id, name, state, policy, createdBy, sealedBy string + itemCount int + createdAt time.Time + sealedAt *time.Time +} + +func (r releaseBatchRow) Scan(dest ...any) error { + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = r.id, r.name, r.state, r.policy + *dest[4].(*string), *dest[5].(*string), *dest[6].(*int) = r.createdBy, r.sealedBy, r.itemCount + *dest[7].(*time.Time), *dest[8].(**time.Time) = r.createdAt, r.sealedAt + return nil +} + +type releaseCountRow struct{ count int } + +func (r releaseCountRow) Scan(dest ...any) error { + *dest[0].(*int) = r.count + return nil +} + +type releaseBatchRows struct { + values []releaseBatchRow + index int +} + +func (r *releaseBatchRows) Close() {} +func (r *releaseBatchRows) Err() error { return nil } +func (r *releaseBatchRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *releaseBatchRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *releaseBatchRows) RawValues() [][]byte { return nil } +func (r *releaseBatchRows) Conn() *pgx.Conn { return nil } +func (r *releaseBatchRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *releaseBatchRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) } +func (r *releaseBatchRows) Values() ([]any, error) { return nil, nil } + +type releaseBatchItemRow struct{ publicationID, title, visibility, action string } + +type releaseBatchItemRows struct { + values []releaseBatchItemRow + index int +} + +func (r *releaseBatchItemRows) Close() {} +func (r *releaseBatchItemRows) Err() error { return nil } +func (r *releaseBatchItemRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *releaseBatchItemRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *releaseBatchItemRows) RawValues() [][]byte { return nil } +func (r *releaseBatchItemRows) Conn() *pgx.Conn { return nil } +func (r *releaseBatchItemRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *releaseBatchItemRows) Scan(dest ...any) error { + v := r.values[r.index-1] + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = v.publicationID, v.title, v.visibility, v.action + return nil +} +func (r *releaseBatchItemRows) Values() ([]any, error) { return nil, nil } + +type releaseBatchReadDB struct { + *recordingDB + row pgx.Row + rows pgx.Rows +} + +func (d *releaseBatchReadDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row { + d.queryRowSQL, d.queryRowArgs = sql, args + return d.row +} + +func (d *releaseBatchReadDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) { + d.querySQL, d.queryArgs = sql, args + return d.rows, nil +} diff --git a/internal/platform/postgres/release_change_sets.go b/internal/platform/postgres/release_change_sets.go new file mode 100644 index 0000000..8efb0db --- /dev/null +++ b/internal/platform/postgres/release_change_sets.go @@ -0,0 +1,290 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +// Errors for R1 release ChangeSets. +var ( + ErrInvalidReleaseChangeSet = errors.New("invalid release change set request") + ErrReleaseChangeSetNotFound = errors.New("release change set not found") + ErrReleaseChangeSetStateConflict = errors.New("release change set state conflict") + ErrReleaseChangeSetEmpty = errors.New("release plan has no releasable items") +) + +const releaseChangeSetColumns = `id,number,name,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at,confirmed_by,COALESCE(confirmed_at::text,'')` + +func scanReleaseChangeSet(row pgx.Row) (governance.ReleaseChangeSet, error) { + var item governance.ReleaseChangeSet + var snapshots, changeSets []byte + var confirmedAt string + err := row.Scan(&item.ID, &item.Number, &item.Name, &item.State, &snapshots, &changeSets, &item.AcceptanceCutoff, &item.ProductionBaselineRef, + &item.ItemCount, &item.AddCount, &item.UpdateCount, &item.DeleteCount, &item.BlockedCount, &item.ManifestSHA256, &item.CreatedBy, &item.CreatedAt, &item.ConfirmedBy, &confirmedAt) + if err != nil { + return governance.ReleaseChangeSet{}, err + } + _ = json.Unmarshal(snapshots, &item.SourceSnapshotIDs) + _ = json.Unmarshal(changeSets, &item.SourceChangeSetIDs) + if item.SourceSnapshotIDs == nil { + item.SourceSnapshotIDs = []string{} + } + if item.SourceChangeSetIDs == nil { + item.SourceChangeSetIDs = []string{} + } + if confirmedAt != "" { + if parsed, parseErr := time.Parse(time.RFC3339Nano, confirmedAt); parseErr == nil { + item.ConfirmedAt = &parsed + } + } + return item, nil +} + +// ComputeReleaseChangeSetManifest is exported for tests: the manifest seals +// every frozen decision in seq order with an exact delimiter. +func ComputeReleaseChangeSetManifest(items []governance.ReleaseChangeSetItem) string { + lines := make([]string, 0, len(items)) + for _, item := range items { + lines = append(lines, strings.Join([]string{ + strconv.Itoa(item.Seq), item.PublicationID, item.TargetScope, item.Action, + item.TestState, item.ProductionState, strconv.FormatBool(item.CanRelease), item.Blocker, + }, "|")) + } + digest := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return hex.EncodeToString(digest[:]) +} + +// GenerateReleaseChangeSet freezes the current live plan into an immutable +// versioned set. Repeated generation with an identical plan reuses the latest +// PLANNED set (dedup), so concurrent generation cannot create duplicates. +func (s *Store) GenerateReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) { + if request.Name == "" || request.Name != strings.TrimSpace(request.Name) || len(request.Name) > 200 || + strings.ContainsAny(request.Name, "\x00\r\n") || !validReviewerRef(request.CreatedBy) || request.Now.IsZero() { + return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet + } + plan, err := s.ListReleasePlan(ctx) + if err != nil { + return governance.ReleaseChangeSet{}, err + } + items := make([]governance.ReleaseChangeSetItem, 0, len(plan.Items)) + for index, planItem := range plan.Items { + items = append(items, governance.ReleaseChangeSetItem{Seq: index + 1, PublicationID: planItem.PublicationID, Title: planItem.Title, TargetScope: planItem.TargetScope, Action: planItem.Action, TestState: planItem.TestState, ProductionState: planItem.ProductionState, CanRelease: planItem.CanRelease, Blocker: planItem.Blocker}) + } + if len(items) == 0 { + return governance.ReleaseChangeSet{}, ErrReleaseChangeSetEmpty + } + manifest := ComputeReleaseChangeSetManifest(items) + counts := map[string]int{} + for _, item := range items { + counts[item.Action]++ + } + blocked := 0 + for _, item := range items { + if item.Action == "blocked" { + blocked++ + } + } + + // Same plan twice (sequentially) reuses the latest PLANNED set instead of + // piling up identical versions. Concurrent racers fall through to the + // unique-number conflict path below. + if existing, lookupErr := s.findPlannedByManifest(ctx, manifest); lookupErr == nil && existing.ManifestSHA256 == manifest { + return existing, nil + } else if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { + return governance.ReleaseChangeSet{}, lookupErr + } + + for attempt := 0; attempt < 3; attempt++ { + result, retry, err := s.tryInsertReleaseChangeSet(ctx, request, items, manifest, counts["add"], counts["update"], counts["delete"], blocked) + if err == nil { + return result, nil + } + if !retry { + return governance.ReleaseChangeSet{}, err + } + // A concurrent identical generation won; return its set (dedup) if the + // manifest matches, otherwise loop to take the next version number. + existing, lookupErr := s.findPlannedByManifest(ctx, manifest) + if lookupErr == nil && existing.ManifestSHA256 == manifest { + return existing, nil + } + if lookupErr != nil && !errors.Is(lookupErr, ErrReleaseChangeSetNotFound) { + return governance.ReleaseChangeSet{}, lookupErr + } + } + return governance.ReleaseChangeSet{}, fmt.Errorf("release change set generation kept conflicting") +} + +func (s *Store) tryInsertReleaseChangeSet(ctx context.Context, request governance.GenerateReleaseChangeSetRequest, items []governance.ReleaseChangeSetItem, manifest string, add, update, del, blocked int) (governance.ReleaseChangeSet, bool, error) { + tx, err := s.db.Begin(ctx) + if err != nil { + return governance.ReleaseChangeSet{}, false, err + } + defer func() { _ = tx.Rollback(ctx) }() + + id, err := newOpaqueID("rcs-") + if err != nil { + return governance.ReleaseChangeSet{}, false, err + } + set, err := scanReleaseChangeSet(tx.QueryRow(ctx, ` +INSERT INTO release_change_sets(id,name,number,state,source_snapshot_ids,source_change_set_ids,acceptance_cutoff,production_baseline_ref,item_count,add_count,update_count,delete_count,blocked_count,manifest_sha256,created_by,created_at) +SELECT $1,$2,(SELECT COALESCE(MAX(number),0)+1 FROM release_change_sets),'planned', + COALESCE((SELECT jsonb_agg(snapshot_id ORDER BY snapshot_id) FROM (SELECT DISTINCT snapshot_id FROM source_catalog_heads) s),'[]'::jsonb), + COALESCE((SELECT jsonb_agg(change_set_id ORDER BY change_set_id) FROM (SELECT DISTINCT id AS change_set_id FROM source_catalog_change_sets WHERE state IN ('confirmed','applied')) c),'[]'::jsonb), + COALESCE((SELECT MAX(updated_at) FROM test_publications),(SELECT now())), + (SELECT COALESCE(MAX(completed_at)::text,'') FROM release_executions WHERE state='completed'), + $3,$4,$5,$6,$7,$8,$9,$10 +RETURNING `+releaseChangeSetColumns, + id, request.Name, len(items), add, update, del, blocked, manifest, request.CreatedBy, request.Now)) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return governance.ReleaseChangeSet{}, true, err + } + return governance.ReleaseChangeSet{}, false, fmt.Errorf("create release change set: %w", err) + } + // R6 audit chain: mirror the JSONB binding into real FK rows so the trace + // survives even if someone later edits denormalized display fields. + if _, err := tx.Exec(ctx, ` +INSERT INTO release_change_set_snapshot_links(change_set_id, snapshot_id) +SELECT $1, elem.value +FROM jsonb_array_elements_text($2::jsonb) AS elem(value) +ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceSnapshotIDs)); err != nil { + return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release snapshots: %w", err) + } + if _, err := tx.Exec(ctx, ` +INSERT INTO release_change_set_catalog_links(change_set_id, catalog_change_set_id) +SELECT $1, elem.value +FROM jsonb_array_elements_text($2::jsonb) AS elem(value) +ON CONFLICT DO NOTHING`, set.ID, mustMarshalJSON(set.SourceChangeSetIDs)); err != nil { + return governance.ReleaseChangeSet{}, false, fmt.Errorf("link release catalog sets: %w", err) + } + for _, item := range items { + if _, err := tx.Exec(ctx, `INSERT INTO release_change_set_items(change_set_id,seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, + set.ID, item.Seq, item.PublicationID, item.Title, item.TargetScope, item.Action, item.TestState, item.ProductionState, item.CanRelease, item.Blocker); err != nil { + return governance.ReleaseChangeSet{}, false, fmt.Errorf("insert release change set item: %w", err) + } + } + if err := tx.Commit(ctx); err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return governance.ReleaseChangeSet{}, true, err + } + return governance.ReleaseChangeSet{}, false, err + } + return set, false, nil +} + +func (s *Store) findPlannedByManifest(ctx context.Context, manifest string) (governance.ReleaseChangeSet, error) { + return scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE manifest_sha256=$1 AND state='planned' ORDER BY number DESC LIMIT 1`, manifest)) +} + +// ConfirmReleaseChangeSet freezes a planned set for batching. Re-confirming an +// already-confirmed set replays the stored confirmation (immutability). +func (s *Store) ConfirmReleaseChangeSet(ctx context.Context, request governance.ConfirmReleaseChangeSetRequest) (governance.ReleaseChangeSet, error) { + if !validRouteID(request.ChangeSetID) || !validReviewerRef(request.ConfirmedBy) || request.Now.IsZero() { + return governance.ReleaseChangeSet{}, ErrInvalidReleaseChangeSet + } + set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `UPDATE release_change_sets SET state='confirmed',confirmed_by=$2,confirmed_at=$3 WHERE id=$1 AND state='planned' RETURNING `+releaseChangeSetColumns, request.ChangeSetID, request.ConfirmedBy, request.Now)) + if errors.Is(err, pgx.ErrNoRows) { + current, lookupErr := s.GetReleaseChangeSet(ctx, request.ChangeSetID) + if lookupErr != nil { + return governance.ReleaseChangeSet{}, lookupErr + } + if current.State == "confirmed" || current.State == "consumed" { + return current.ReleaseChangeSet, nil + } + return governance.ReleaseChangeSet{}, ErrReleaseChangeSetStateConflict + } + if err != nil { + return governance.ReleaseChangeSet{}, fmt.Errorf("confirm release change set: %w", err) + } + return set, nil +} + +func (s *Store) GetReleaseChangeSet(ctx context.Context, id string) (governance.ReleaseChangeSetDetail, error) { + set, err := scanReleaseChangeSet(s.db.QueryRow(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets WHERE id=$1`, id)) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound + } + if err != nil { + return governance.ReleaseChangeSetDetail{}, err + } + detail := governance.ReleaseChangeSetDetail{ReleaseChangeSet: set, Items: []governance.ReleaseChangeSetItem{}} + rows, err := s.db.Query(ctx, `SELECT seq,publication_id,title,target_scope,action,test_state,production_state,can_release,blocker FROM release_change_set_items WHERE change_set_id=$1 ORDER BY seq`, id) + if err != nil { + return governance.ReleaseChangeSetDetail{}, fmt.Errorf("list release change set items: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.ReleaseChangeSetItem + if err := rows.Scan(&item.Seq, &item.PublicationID, &item.Title, &item.TargetScope, &item.Action, &item.TestState, &item.ProductionState, &item.CanRelease, &item.Blocker); err != nil { + return governance.ReleaseChangeSetDetail{}, err + } + detail.Items = append(detail.Items, item) + } + return detail, rows.Err() +} + +func (s *Store) ListReleaseChangeSets(ctx context.Context, filter governance.ReleaseBatchFilter) (governance.ReleaseChangeSetPage, error) { + limit, offset := filter.Limit, filter.Offset + if limit <= 0 { + limit = 20 + } + if offset < 0 { + offset = 0 + } + page := governance.ReleaseChangeSetPage{Items: []governance.ReleaseChangeSet{}, Limit: limit, Offset: offset} + if limit > 100 { + return page, errors.New("release change set limit must be between 1 and 100") + } + var total int + if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM release_change_sets`).Scan(&total); err != nil { + return page, err + } + page.Total = total + rows, err := s.db.Query(ctx, `SELECT `+releaseChangeSetColumns+` FROM release_change_sets ORDER BY number DESC LIMIT $1 OFFSET $2`, limit, offset) + if err != nil { + return page, err + } + defer rows.Close() + for rows.Next() { + item, err := scanReleaseChangeSet(rows) + if err != nil { + return page, err + } + page.Items = append(page.Items, item) + } + return page, rows.Err() +} + +func (s *Store) LatestReleaseChangeSet(ctx context.Context) (governance.ReleaseChangeSetDetail, error) { + var id string + if err := s.db.QueryRow(ctx, `SELECT id FROM release_change_sets ORDER BY number DESC LIMIT 1`).Scan(&id); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseChangeSetDetail{}, ErrReleaseChangeSetNotFound + } + return governance.ReleaseChangeSetDetail{}, err + } + return s.GetReleaseChangeSet(ctx, id) +} + +func mustMarshalJSON(values []string) []byte { + raw, err := json.Marshal(values) + if err != nil { + return []byte("[]") + } + return raw +} diff --git a/internal/platform/postgres/release_execution_reads_test.go b/internal/platform/postgres/release_execution_reads_test.go new file mode 100644 index 0000000..d6992f8 --- /dev/null +++ b/internal/platform/postgres/release_execution_reads_test.go @@ -0,0 +1,132 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestListAndGetReleaseExecutionsReturnsAuditableProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 9, 0, 0, 0, time.UTC) + row := releaseExecutionRow{id: "rx-safe", batchID: "rb-safe", operation: "publish", state: "completed", target: "production", authorizedBy: "owner", authorizationCode: "production_release_authorized", jobID: "job-safe", manifestSHA: strings.Repeat("a", 64), itemCount: 1, completedCount: 1, createdAt: now, completedAt: &now} + db := &releaseExecutionReadDB{countRow: releaseExecutionCountRow{count: 1}, row: row, rows: &releaseExecutionRows{values: []releaseExecutionRow{row}}} + store := NewStore(db) + page, err := store.ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "publish", State: "completed", Limit: 20}) + if err != nil || page.Total != 1 || len(page.Items) != 1 { + t.Fatalf("page=%#v error=%v", page, err) + } + db.rows = &releaseItemExecutionRows{values: []releaseItemExecutionRow{{publicationID: "tp-safe", title: "Synthetic", action: "delete", state: "verified", previousRevision: "prod-v0", productionRevision: "absent", completedAt: &now}}} + detail, err := store.GetReleaseExecution(context.Background(), "rx-safe") + if err != nil || len(detail.Items) != 1 || detail.Items[0].Action != "delete" || detail.Items[0].ProductionRevision != "absent" { + t.Fatalf("detail=%#v error=%v", detail, err) + } + joined := strings.ToLower(strings.Join(db.queries, "\n")) + for _, required := range []string{"from release_executions", "operation = $1", "state = $2", "from release_item_executions", "publication_id", "item.action"} { + if !strings.Contains(joined, required) { + t.Fatalf("release read SQL missing %q", required) + } + } + for _, forbidden := range []string{"knowledge_id", "test_kb_id", "physical_path", "api_key", "secret"} { + if strings.Contains(joined, forbidden) { + t.Fatalf("release read SQL contains forbidden %q", forbidden) + } + } +} + +func TestListReleaseExecutionsRejectsInvalidFilter(t *testing.T) { + _, err := NewStore(&recordingDB{}).ListReleaseExecutions(context.Background(), governance.ReleaseExecutionFilter{Operation: "deploy", Limit: 20}) + if !errors.Is(err, ErrInvalidReleaseExecutionFilter) { + t.Fatalf("error=%v", err) + } +} + +type releaseExecutionCountRow struct{ count int } + +func (r releaseExecutionCountRow) Scan(dest ...any) error { *dest[0].(*int) = r.count; return nil } + +type releaseExecutionRows struct { + values []releaseExecutionRow + index int +} + +func (r *releaseExecutionRows) Close() {} +func (r *releaseExecutionRows) Err() error { return nil } +func (r *releaseExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *releaseExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *releaseExecutionRows) RawValues() [][]byte { return nil } +func (r *releaseExecutionRows) Conn() *pgx.Conn { return nil } +func (r *releaseExecutionRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *releaseExecutionRows) Scan(dest ...any) error { return r.values[r.index-1].Scan(dest...) } +func (r *releaseExecutionRows) Values() ([]any, error) { return nil, nil } + +type releaseItemExecutionRow struct { + publicationID, title, action, state, previousRevision, productionRevision, errorCode string + createdAt time.Time + completedAt *time.Time +} +type releaseItemExecutionRows struct { + values []releaseItemExecutionRow + index int +} + +func (r *releaseItemExecutionRows) Close() {} +func (r *releaseItemExecutionRows) Err() error { return nil } +func (r *releaseItemExecutionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *releaseItemExecutionRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *releaseItemExecutionRows) RawValues() [][]byte { return nil } +func (r *releaseItemExecutionRows) Conn() *pgx.Conn { return nil } +func (r *releaseItemExecutionRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *releaseItemExecutionRows) Scan(dest ...any) error { + value := r.values[r.index-1] + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = value.publicationID, value.title, value.action, value.state + *dest[4].(*string), *dest[5].(*string), *dest[6].(*string) = value.previousRevision, value.productionRevision, value.errorCode + *dest[7].(*time.Time), *dest[8].(**time.Time) = value.createdAt, value.completedAt + return nil +} +func (r *releaseItemExecutionRows) Values() ([]any, error) { return nil, nil } + +type releaseExecutionReadDB struct { + countRow, row pgx.Row + rows pgx.Rows + queries []string + queryRows int +} + +func (d *releaseExecutionReadDB) Begin(context.Context) (pgx.Tx, error) { + return nil, errors.New("transactions are not supported by the reading fake") +} + +func (d *releaseExecutionReadDB) QueryRow(_ context.Context, sql string, _ ...any) pgx.Row { + d.queries = append(d.queries, sql) + d.queryRows++ + if d.queryRows == 1 { + return d.countRow + } + return d.row +} +func (d *releaseExecutionReadDB) Query(_ context.Context, sql string, _ ...any) (pgx.Rows, error) { + d.queries = append(d.queries, sql) + return d.rows, nil +} +func (d *releaseExecutionReadDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, nil +} diff --git a/internal/platform/postgres/release_execution_worker_store.go b/internal/platform/postgres/release_execution_worker_store.go new file mode 100644 index 0000000..8a5d696 --- /dev/null +++ b/internal/platform/postgres/release_execution_worker_store.go @@ -0,0 +1,498 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +var ErrReleaseExecutionNotClaimable = errors.New("release execution is not claimable") + +const startReleaseExecutionSQL = ` +WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `, +execution_gate AS ( + SELECT execution.id, execution.operation, + COUNT(item.publication_id)::INTEGER AS item_count, + COUNT(*) FILTER ( + WHERE (item.action = 'upsert' + AND ready.publication_id IS NOT NULL + AND ready.review_id = item.review_id + AND ready.acceptance_run_id = item.acceptance_run_id + AND ready.publication_updated_at = item.publication_updated_at + AND ready.decided_visibility = item.decided_visibility) + OR (item.action = 'delete' + AND delete_ready.publication_id IS NOT NULL + AND delete_ready.review_id = item.review_id + AND delete_ready.acceptance_run_id = item.acceptance_run_id + AND delete_ready.publication_updated_at = item.publication_updated_at + AND delete_ready.decided_visibility = item.decided_visibility) + )::INTEGER AS valid_items + FROM release_executions AS execution + JOIN release_batch_items AS item ON item.release_batch_id = execution.release_batch_id + LEFT JOIN release_upsert_ready AS ready ON ready.publication_id = item.publication_id AND item.action = 'upsert' + LEFT JOIN release_delete_ready AS delete_ready ON delete_ready.publication_id = item.publication_id AND item.action = 'delete' + WHERE execution.id = $1 + GROUP BY execution.id, execution.operation +), started AS ( + UPDATE release_executions AS execution + SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolling_back' ELSE 'running' END, + started_at = COALESCE(started_at, $4::TIMESTAMPTZ) + FROM execution_gate + WHERE execution.id = $1::VARCHAR + AND execution_gate.id = execution.id + AND (execution_gate.operation = 'rollback' OR execution_gate.valid_items = execution_gate.item_count) + AND execution.job_id = $2::VARCHAR + AND execution.operation = $3::VARCHAR + AND execution.state IN ('pending', 'running', 'rolling_back') + AND NOT EXISTS ( + SELECT 1 + FROM release_batch_items AS gate_item + JOIN test_publications AS gate_publication ON gate_publication.id = gate_item.publication_id + JOIN source_item_versions AS gate_version ON gate_version.id = gate_item.source_item_version_id + WHERE gate_item.release_batch_id = execution.release_batch_id + AND (gate_version.revision = '' OR gate_publication.parser_generation = '' OR gate_publication.index_generation = '') + ) + RETURNING execution.id, execution.release_batch_id, execution.operation, execution.prior_execution_id +), seeded AS ( + INSERT INTO release_item_executions ( + id, release_execution_id, publication_id, action, state, created_at, updated_at + ) + SELECT 'ri-' || SUBSTR(MD5(started.id || ':' || item.publication_id), 1, 32), + started.id, item.publication_id, item.action, 'pending', $4::TIMESTAMPTZ, $4::TIMESTAMPTZ + FROM started + JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id + WHERE started.operation = 'publish' OR EXISTS ( + SELECT 1 + FROM release_item_executions AS prior_seed + WHERE prior_seed.release_execution_id = started.prior_execution_id + AND prior_seed.publication_id = item.publication_id + AND prior_seed.state = 'verified' + ) + ON CONFLICT (release_execution_id, publication_id) DO NOTHING + RETURNING publication_id, action +), active_items AS ( + SELECT publication_id, action FROM seeded + UNION + SELECT current_item.publication_id, current_item.action + FROM started + JOIN release_item_executions AS current_item + ON current_item.release_execution_id = started.id + WHERE current_item.state = 'pending' +), plan AS ( + SELECT started.id AS release_execution_id, + item.publication_id, + active_items.action, + item.source_item_version_id, + current_version.revision AS source_revision, + publication.parser_generation || '@' || publication.index_generation AS parse_revision, + item.review_id AS release_revision, + publication.test_kb_id, + publication.knowledge_id, + item.decided_visibility, + COALESCE(previous.production_revision, '') AS previous_production_revision + FROM started + JOIN release_batch_items AS item ON item.release_batch_id = started.release_batch_id + JOIN active_items ON active_items.publication_id = item.publication_id + + JOIN test_publications AS publication ON publication.id = item.publication_id + JOIN source_item_versions AS current_version ON current_version.id = item.source_item_version_id + LEFT JOIN LATERAL ( + SELECT CASE + WHEN started.operation = 'rollback' THEN prior_item.previous_production_revision + ELSE prior_item.production_revision + END AS production_revision + FROM release_item_executions AS prior_item + JOIN release_executions AS prior_execution ON prior_execution.id = prior_item.release_execution_id + JOIN test_publications AS prior_publication ON prior_publication.id = prior_item.publication_id + JOIN source_item_versions AS prior_version ON prior_version.id = prior_publication.source_item_version_id + WHERE prior_item.state = 'verified' + AND prior_execution.operation = 'publish' + AND prior_execution.state IN ('completed', 'failed') + AND ( + (started.operation = 'rollback' + AND prior_execution.id = started.prior_execution_id + AND prior_item.publication_id = item.publication_id) + OR + (started.operation = 'publish' + AND prior_version.source_item_id = current_version.source_item_id + AND ( + (active_items.action = 'delete' AND prior_item.publication_id = item.publication_id) + OR (active_items.action = 'upsert' AND prior_item.publication_id <> item.publication_id) + )) + ) + ORDER BY prior_item.completed_at DESC, prior_item.id DESC + LIMIT 1 + ) AS previous ON TRUE + ORDER BY item.publication_id +) +SELECT COUNT(DISTINCT started.id)::INTEGER, + COALESCE( + JSONB_AGG(JSONB_BUILD_OBJECT( + 'release_execution_id', plan.release_execution_id, + 'publication_id', plan.publication_id, + 'action', plan.action, + 'source_item_version_id', plan.source_item_version_id, + 'source_revision', plan.source_revision, + 'parse_revision', plan.parse_revision, + 'release_revision', plan.release_revision, + 'test_knowledge_base_id', plan.test_kb_id, + 'knowledge_id', plan.knowledge_id, + 'decided_visibility', plan.decided_visibility, + 'previous_production_revision', plan.previous_production_revision + ) ORDER BY plan.publication_id) FILTER (WHERE plan.publication_id IS NOT NULL), + '[]'::JSONB + ) +FROM started +LEFT JOIN plan ON TRUE +` + +const beginReleaseBatchSQL = ` +UPDATE release_executions +SET current_batch = $2, + total_batches = $3 +WHERE id = $1 + AND state IN ('running', 'rolling_back') + AND (total_batches = 0 OR total_batches = $3) + AND ($2 = current_batch OR $2 = current_batch + 1) +` + +const completeReleaseItemSQL = ` +UPDATE release_item_executions AS item +SET state = CASE WHEN execution.operation = 'rollback' THEN 'rolled_back' ELSE 'verified' END, + bundle_sha256 = $3, + previous_production_revision = $4, + production_revision = $5, + error_code = '', + updated_at = $6, + completed_at = $6 +FROM release_executions AS execution +WHERE item.release_execution_id = $1 + AND item.publication_id = $2 + AND execution.id = item.release_execution_id + AND execution.state IN ('running', 'rolling_back') + AND item.state IN ('pending', 'staged', 'activated', 'verified', 'rolled_back') +` + +const failReleaseItemSQL = ` +UPDATE release_item_executions AS item +SET state = 'failed', + error_code = $3, + updated_at = $4, + completed_at = $4 +FROM release_executions AS execution +WHERE item.release_execution_id = $1 + AND item.publication_id = $2 + AND execution.id = item.release_execution_id + AND execution.state IN ('running', 'rolling_back') + AND item.state IN ('pending', 'staged', 'activated', 'failed') +` + +const finishReleaseExecutionSQL = ` +WITH item_counts AS ( + SELECT release_execution_id, + COUNT(*)::INTEGER AS total_items, + COUNT(*) FILTER (WHERE state IN ('verified', 'rolled_back'))::INTEGER AS completed_items, + COUNT(*) FILTER (WHERE state = 'failed')::INTEGER AS failed_items + FROM release_item_executions + WHERE release_execution_id = $1 + GROUP BY release_execution_id +) +UPDATE release_executions AS execution +SET state = CASE + WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'rollback_failed' + WHEN execution.operation = 'rollback' THEN 'rolled_back' + WHEN item_counts.failed_items > 0 THEN 'failed' + ELSE 'completed' + END, + completed_item_count = item_counts.completed_items, + failed_item_count = item_counts.failed_items, + -- R4 failure grading on publish: nothing published yet = single_item + -- (compensate that item only); progress existed = batch (stop later + -- batches, compensate this batch's verified items). A failed ROLLBACK + -- escalates the prior publish to overall (snapshot restore required). + failure_level = CASE + WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 AND item_counts.completed_items > 0 THEN 'batch' + WHEN execution.operation = 'publish' AND item_counts.failed_items > 0 THEN 'single_item' + ELSE execution.failure_level + END, + error_code = CASE + WHEN execution.operation = 'rollback' AND item_counts.failed_items > 0 THEN 'release_rollback_failed' + WHEN item_counts.failed_items > 0 THEN 'release_item_failed' + ELSE '' + END, + completed_at = $2 +FROM item_counts +WHERE execution.id = item_counts.release_execution_id + AND execution.state IN ('running', 'rolling_back') + AND item_counts.total_items = execution.item_count + AND item_counts.completed_items + item_counts.failed_items = execution.item_count +` + +// R4: a permanently failed rollback means compensation alone is no longer +// enough: the prior publish escalates to overall (snapshot restore). +const escalatePriorOverallSQL = ` +UPDATE release_executions AS prior +SET failure_level = 'overall' +FROM release_executions AS rollback +WHERE rollback.id = $1 + AND rollback.operation = 'rollback' + AND rollback.state = 'rollback_failed' + AND prior.id = rollback.prior_execution_id + AND prior.operation = 'publish' +` + +func (s *Store) StartExecution(ctx context.Context, request releaseworkflow.StartExecutionRequest) ([]releaseworkflow.PlanItem, error) { + if !validReleaseExecutionRequest(request) { + return nil, ErrReleaseExecutionNotClaimable + } + var executionCount int + var payload []byte + if err := s.db.QueryRow(ctx, startReleaseExecutionSQL, + request.ExecutionID, request.JobID, request.Operation, request.Now.UTC(), + ).Scan(&executionCount, &payload); err != nil { + return nil, fmt.Errorf("start release execution: %w", err) + } + if executionCount != 1 { + return nil, ErrReleaseExecutionNotClaimable + } + var items []releaseworkflow.PlanItem + if err := json.Unmarshal(payload, &items); err != nil { + return nil, errors.New("decode release execution plan") + } + for _, item := range items { + if !validShortID(item.PublicationID) || !validShortID(item.SourceItemVersionID) || + !validRemoteIdentifier(item.TestKnowledgeBaseID) || !validRemoteIdentifier(item.KnowledgeID) || + (item.DecidedVisibility != governance.VisibilityInternal && item.DecidedVisibility != governance.VisibilityExternal) || + len(item.PreviousProductionRevision) > 128 { + return nil, errors.New("invalid release execution plan") + } + } + return items, nil +} + +func (s *Store) BeginBatch(ctx context.Context, executionID string, batch, total int, now time.Time) error { + if !validShortID(executionID) || batch < 1 || total < batch || total > 100000 || now.IsZero() { + return ErrReleaseExecutionNotClaimable + } + result, err := s.db.Exec(ctx, beginReleaseBatchSQL, executionID, batch, total) + if err != nil { + return fmt.Errorf("begin release batch: %w", err) + } + if result.RowsAffected() != 1 { + return ErrReleaseExecutionNotClaimable + } + return nil +} + +func (s *Store) CompleteItem(ctx context.Context, request releaseworkflow.CompleteItemRequest) error { + if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) || + (request.BundleSHA256 != "" && !validLowerSHA256(request.BundleSHA256)) || + !validRevision(request.ProductionRevision) || !validOptionalRevision(request.PreviousProductionRevision) || request.Now.IsZero() { + return ErrReleaseExecutionNotClaimable + } + result, err := s.db.Exec(ctx, completeReleaseItemSQL, request.ExecutionID, request.PublicationID, + request.BundleSHA256, request.PreviousProductionRevision, request.ProductionRevision, request.Now.UTC()) + if err != nil { + return fmt.Errorf("complete release item: %w", err) + } + if result.RowsAffected() != 1 { + return ErrReleaseExecutionNotClaimable + } + return nil +} + +func (s *Store) FailItem(ctx context.Context, request releaseworkflow.FailItemRequest) error { + if !validShortID(request.ExecutionID) || !validShortID(request.PublicationID) || !releaseworkflow.ValidErrorCode(request.ErrorCode) || request.Now.IsZero() { + return ErrReleaseExecutionNotClaimable + } + result, err := s.db.Exec(ctx, failReleaseItemSQL, request.ExecutionID, request.PublicationID, request.ErrorCode, request.Now.UTC()) + if err != nil { + return fmt.Errorf("fail release item: %w", err) + } + if result.RowsAffected() != 1 { + return ErrReleaseExecutionNotClaimable + } + return nil +} + +// ComputeReleaseExecutionManifest seals what ACTUALLY happened: one line per +// item (sorted by publication id) plus the bound release change set digest. +// Publish lines carry the verified bundle digest; rollback lines carry the +// restored revision. The digest is deterministic and re-computable for audits. +func ComputeReleaseExecutionManifest(operation, changeSetManifest string, lines []string) string { + digest := sha256.New() + bound := changeSetManifest + if bound == "" { + bound = "none" + } + fmt.Fprintf(digest, "op:%s\nchangeset:%s\n", operation, bound) + for _, line := range lines { + fmt.Fprintf(digest, "%s\n", line) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +func (s *Store) FinishExecution(ctx context.Context, executionID string, now time.Time) error { + if !validShortID(executionID) || now.IsZero() { + return ErrReleaseExecutionNotClaimable + } + result, err := s.db.Exec(ctx, finishReleaseExecutionSQL, executionID, now.UTC()) + if err != nil { + return fmt.Errorf("finish release execution: %w", err) + } + if result.RowsAffected() != 1 { + return ErrReleaseExecutionNotClaimable + } + // R4: a permanently failed rollback escalates the prior publish to the + // overall level (snapshot restore becomes the only sanctioned path). + if _, err := s.db.Exec(ctx, escalatePriorOverallSQL, executionID); err != nil { + return fmt.Errorf("escalate rollback failure: %w", err) + } + // R5: seal the execution manifest and reconcile it against the confirmed + // release change set. A success that does not match the frozen manifest is + // downgraded to failed (release_manifest_mismatch) — never completed. + if err := s.sealExecutionManifest(ctx, executionID); err != nil { + return err + } + return nil +} + +func (s *Store) sealExecutionManifest(ctx context.Context, executionID string) error { + var operation, state, changeSetManifest, changeSetID string + err := s.db.QueryRow(ctx, ` +SELECT e.operation, e.state, COALESCE(rcs.manifest_sha256,''), COALESCE(e.release_change_set_id::text,'') +FROM release_executions AS e +LEFT JOIN release_change_sets AS rcs ON rcs.id = e.release_change_set_id +WHERE e.id = $1`, executionID).Scan(&operation, &state, &changeSetManifest, &changeSetID) + if errors.Is(err, pgx.ErrNoRows) { + return ErrReleaseExecutionNotClaimable + } + if err != nil { + return fmt.Errorf("load execution for manifest seal: %w", err) + } + rows, err := s.db.Query(ctx, ` +SELECT item.publication_id, item.action, item.state, COALESCE(item.bundle_sha256,''), COALESCE(item.production_revision,'') +FROM release_item_executions AS item +WHERE item.release_execution_id = $1 +ORDER BY item.publication_id`, executionID) + if err != nil { + return fmt.Errorf("load execution items for manifest seal: %w", err) + } + type itemRow struct { + publicationID, action, itemState, bundleSHA, revision string + } + items := []itemRow{} + for rows.Next() { + var item itemRow + if err := rows.Scan(&item.publicationID, &item.action, &item.itemState, &item.bundleSHA, &item.revision); err != nil { + rows.Close() + return fmt.Errorf("scan execution item for manifest seal: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("iterate execution items: %w", err) + } + rows.Close() + lines := make([]string, 0, len(items)) + for _, item := range items { + if operation == "rollback" { + lines = append(lines, fmt.Sprintf("%s|%s", item.publicationID, item.revision)) + } else { + lines = append(lines, fmt.Sprintf("%s|%s|%s", item.publicationID, item.bundleSHA, item.revision)) + } + } + // Only a terminal execution seals its manifest; mid-batch re-finishes must + // never freeze a partial snapshot (the seal below is first-write-wins). + switch state { + case "completed", "failed", "rolled_back", "rollback_failed": + default: + return nil + } + manifest := ComputeReleaseExecutionManifest(operation, changeSetManifest, lines) + if _, err := s.db.Exec(ctx, `UPDATE release_executions SET manifest_sha256 = $2 WHERE id = $1 AND manifest_sha256 = ''`, executionID, manifest); err != nil { + return fmt.Errorf("seal execution manifest: %w", err) + } + if state != "completed" || changeSetID == "" { + // Failed/rolled-back terminal states are honest audit facts; R4 + // overall escalation is already recorded. Nothing further to verify. + return nil + } + // Success must equal the frozen change-set intent for the publications it + // actually touched. + frozen, err := s.changeSetItemActions(ctx, changeSetID) + if err != nil { + return err + } + for _, item := range items { + frozenAction, exists := frozen[item.publicationID] + // The frozen change set records PLAN actions (add/update/delete); + // executions carry EFFECTIVE actions (upsert/delete). Map equivalently. + consistent := exists && item.itemState == "verified" && + ((item.action == "delete" && frozenAction == "delete") || + (item.action == "upsert" && (frozenAction == "add" || frozenAction == "update"))) + if !consistent { + if _, err := s.db.Exec(ctx, `UPDATE release_executions SET state='failed', error_code='release_manifest_mismatch' WHERE id=$1 AND state='completed'`, executionID); err != nil { + return fmt.Errorf("downgrade manifest mismatch: %w", err) + } + return nil + } + } + return nil +} + +func (s *Store) changeSetItemActions(ctx context.Context, changeSetID string) (map[string]string, error) { + rows, err := s.db.Query(ctx, `SELECT publication_id, action FROM release_change_set_items WHERE change_set_id = $1`, changeSetID) + if err != nil { + return nil, fmt.Errorf("load release change set items: %w", err) + } + defer rows.Close() + actions := map[string]string{} + for rows.Next() { + var publicationID, action string + if err := rows.Scan(&publicationID, &action); err != nil { + return nil, fmt.Errorf("scan release change set item: %w", err) + } + actions[publicationID] = action + } + return actions, rows.Err() +} + +func validReleaseExecutionRequest(request releaseworkflow.StartExecutionRequest) bool { + return validShortID(request.ExecutionID) && validShortID(request.JobID) && request.Operation.Valid() && !request.Now.IsZero() +} + +func validRemoteIdentifier(value string) bool { + return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\") +} + +func validRevision(value string) bool { + return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t") +} + +func validOptionalRevision(value string) bool { + return value == "" || validRevision(value) +} + +func validLowerSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} diff --git a/internal/platform/postgres/release_execution_worker_store_test.go b/internal/platform/postgres/release_execution_worker_store_test.go new file mode 100644 index 0000000..71c3542 --- /dev/null +++ b/internal/platform/postgres/release_execution_worker_store_test.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestStartReleaseExecutionSeedsItemsAndReturnsOnlyPendingPlan(t *testing.T) { + payload := `[{"release_execution_id":"rx-safe","publication_id":"tp-safe","action":"delete","source_item_version_id":"sv-safe","source_revision":"source-v1","parse_revision":"parser-v1@index-v1","release_revision":"review-v1","test_knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","decided_visibility":"internal","previous_production_revision":"prod-old"}]` + db := &recordingDB{row: startExecutionRow{executionCount: 1, payload: []byte(payload)}} + items, err := NewStore(db).StartExecution(context.Background(), releaseworkflow.StartExecutionRequest{ + ExecutionID: "rx-safe", JobID: "job-safe", Operation: "publish", Now: time.Now(), + }) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 || items[0].PublicationID != "tp-safe" || items[0].Action != "delete" || items[0].KnowledgeID != "knowledge-safe" || items[0].PreviousProductionRevision != "prod-old" { + t.Fatalf("items=%#v", items) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "update release_executions", "then 'rolling_back' else 'running' end", "job_id = $2", "operation = $3", + "state in ('pending', 'running', 'rolling_back')", + "insert into release_item_executions", "on conflict (release_execution_id, publication_id) do nothing", + "current_item.state = 'pending'", "test_publications", "release_batch_items", + "item.action", "active_items.action", + "started.operation = 'publish' or exists", "prior_seed.state = 'verified'", + "prior_execution.state in ('completed', 'failed')", + "release_delete_ready", "item.action = 'delete'", "execution_gate.valid_items = execution_gate.item_count", + } { + if !strings.Contains(sql, required) { + t.Fatalf("start execution SQL missing %q", required) + } + } +} + +func TestStartReleaseExecutionRejectsUnknownOrMismatchedExecution(t *testing.T) { + _, err := NewStore(&recordingDB{row: startExecutionRow{executionCount: 0, payload: []byte(`[]`)}}).StartExecution(context.Background(), releaseworkflow.StartExecutionRequest{ + ExecutionID: "rx-safe", JobID: "job-safe", Operation: "publish", Now: time.Now(), + }) + if err != ErrReleaseExecutionNotClaimable { + t.Fatalf("error=%v", err) + } +} + +func TestReleaseExecutionItemMutationsAreScopedAndSanitized(t *testing.T) { + now := time.Now() + // The seal step reads the finished execution row (a failed terminal state + // keeps the fake out of the change-set comparison query) and zero items. + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1"), + row: manifestSealRow{operation: "publish", state: "failed"}, + rows: &manifestItemRows{}, + } + store := NewStore(db) + if err := store.CompleteItem(context.Background(), releaseworkflow.CompleteItemRequest{ + ExecutionID: "rx-safe", PublicationID: "tp-safe", BundleSHA256: strings.Repeat("a", 64), + PreviousProductionRevision: "prod-old", ProductionRevision: "prod-new", Now: now, + }); err != nil { + t.Fatal(err) + } + completeSQL := strings.ToLower(db.execSQL) + for _, required := range []string{"update release_item_executions", "state = case", "bundle_sha256", "production_revision", "release_execution_id = $1", "publication_id = $2"} { + if !strings.Contains(completeSQL, required) { + t.Fatalf("complete SQL missing %q", required) + } + } + if err := store.FailItem(context.Background(), releaseworkflow.FailItemRequest{ExecutionID: "rx-safe", PublicationID: "tp-safe", ErrorCode: releaseworkflow.ErrorCodeVerificationFailed, Now: now}); err != nil { + t.Fatal(err) + } + failSQL := strings.ToLower(db.execSQL) + if !strings.Contains(failSQL, "error_code = $3") || strings.Contains(failSQL, "error_message") { + t.Fatalf("unsafe fail SQL: %s", failSQL) + } + if err := store.FinishExecution(context.Background(), "rx-safe", now); err != nil { + t.Fatal(err) + } + // complete, fail, finish, escalation, seal — a terminal failure still seals + // its manifest; escalation must come after the finish write. + if len(db.execSQLs) != 5 { + t.Fatalf("expected complete/fail/finish/escalate/seal, got %d statements: %v", len(db.execSQLs), db.execSQLs) + } + finishSQL := strings.ToLower(db.execSQLs[2]) + for _, required := range []string{"completed_item_count", "failed_item_count", "state = case", "release_item_failed", "'rolling_back'", "rollback_failed"} { + if !strings.Contains(finishSQL, required) { + t.Fatalf("finish SQL missing %q", required) + } + } + escalateSQL := strings.ToLower(db.execSQLs[3]) + if !strings.Contains(escalateSQL, "set failure_level = 'overall'") || !strings.Contains(escalateSQL, "state = 'rollback_failed'") { + t.Fatalf("unsafe escalation SQL: %s", escalateSQL) + } +} + +func TestBeginReleaseBatchPersistsCurrentAndTotalBatch(t *testing.T) { + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(db).BeginBatch(context.Background(), "rx-safe", 2, 3, time.Now()); err != nil { + t.Fatal(err) + } + sql := strings.ToLower(db.execSQL) + for _, required := range []string{"update release_executions", "current_batch = $2", "total_batches = $3", "state in ('running', 'rolling_back')"} { + if !strings.Contains(sql, required) { + t.Fatalf("begin batch SQL missing %q", required) + } + } +} + +type startExecutionRow struct { + executionCount int + payload []byte +} + +func (r startExecutionRow) Scan(dest ...any) error { + *dest[0].(*int) = r.executionCount + *dest[1].(*[]byte) = r.payload + return nil +} + +type manifestSealRow struct { + operation string + state string +} + +func (r manifestSealRow) Scan(dest ...any) error { + *dest[0].(*string) = r.operation + *dest[1].(*string) = r.state + *dest[2].(*string) = "" + *dest[3].(*string) = "" + return nil +} + +type manifestItemRows struct { + values [][5]string + index int +} + +func (r *manifestItemRows) Close() {} +func (r *manifestItemRows) Err() error { return nil } +func (r *manifestItemRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *manifestItemRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *manifestItemRows) RawValues() [][]byte { return nil } +func (r *manifestItemRows) Values() ([]any, error) { return nil, nil } +func (r *manifestItemRows) Conn() *pgx.Conn { return nil } +func (r *manifestItemRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *manifestItemRows) Scan(dest ...any) error { + row := r.values[r.index-1] + for i, target := range dest { + *target.(*string) = row[i] + } + return nil +} diff --git a/internal/platform/postgres/release_executions.go b/internal/platform/postgres/release_executions.go new file mode 100644 index 0000000..33e362b --- /dev/null +++ b/internal/platform/postgres/release_executions.go @@ -0,0 +1,540 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ( + ErrInvalidReleaseAuthorization = errors.New("invalid release authorization") + ErrReleaseAuthorizationGateConflict = errors.New("release authorization gate conflict") + ErrInvalidReleaseExecutionFilter = errors.New("invalid release execution filter") + ErrReleaseExecutionNotFound = errors.New("release execution not found") + ErrReleaseExecutionRetryConflict = errors.New("release execution retry conflict") +) + +const authorizeReleaseBatchSQL = ` +WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `, +batch_gate AS ( + SELECT batch.id AS release_batch_id, + batch.item_count, + batch.release_change_set_id, + COUNT(item.publication_id)::INTEGER AS total_items, + COUNT(*) FILTER (WHERE + (item.action = 'upsert' AND ready.publication_id IS NOT NULL) + OR (item.action = 'delete' AND delete_ready.publication_id IS NOT NULL) + )::INTEGER AS ready_items + FROM release_batches AS batch + JOIN release_batch_items AS item ON item.release_batch_id = batch.id + LEFT JOIN release_upsert_ready AS ready + ON ready.publication_id = item.publication_id + AND item.action = 'upsert' + AND ready.review_id = item.review_id + AND ready.acceptance_run_id = item.acceptance_run_id + AND ready.publication_updated_at = item.publication_updated_at + AND ready.decided_visibility = item.decided_visibility + LEFT JOIN release_delete_ready AS delete_ready + ON delete_ready.publication_id = item.publication_id + AND item.action = 'delete' + AND delete_ready.review_id = item.review_id + AND delete_ready.acceptance_run_id = item.acceptance_run_id + AND delete_ready.publication_updated_at = item.publication_updated_at + AND delete_ready.decided_visibility = item.decided_visibility + WHERE batch.id = $1::VARCHAR + AND batch.state = 'sealed' + AND batch.policy_version = 'release-gate-v1' + GROUP BY batch.id, batch.item_count, batch.release_change_set_id +), eligible_batch AS ( + SELECT release_batch_id, item_count, release_change_set_id + FROM batch_gate + WHERE total_items > 0 + AND total_items = ready_items + AND item_count = total_items +), inserted_job AS ( + INSERT INTO jobs ( + id, type, payload, state, idempotency_key, + max_attempts, available_at, created_at, updated_at + ) + SELECT $3::VARCHAR, 'release.publish', + jsonb_build_object( + 'execution_id', $2::VARCHAR, + 'release_batch_id', eligible_batch.release_batch_id, + 'operation', 'publish' + ), + 'pending', $7::VARCHAR, 3, $6::TIMESTAMPTZ, $6::TIMESTAMPTZ, $6::TIMESTAMPTZ + FROM eligible_batch + ON CONFLICT (idempotency_key) + DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key + RETURNING id +), inserted_execution AS ( + INSERT INTO release_executions ( + id, release_batch_id, operation, state, target_environment, + authorized_by, authorization_code, job_id, item_count, created_at, release_change_set_id + ) + SELECT $2::VARCHAR, eligible_batch.release_batch_id, 'publish', 'pending', 'production', + $4::VARCHAR, $5::VARCHAR, inserted_job.id, eligible_batch.item_count, $6::TIMESTAMPTZ, + eligible_batch.release_change_set_id + FROM eligible_batch + CROSS JOIN inserted_job + ON CONFLICT (release_batch_id) WHERE operation = 'publish' + DO UPDATE SET release_batch_id = EXCLUDED.release_batch_id + RETURNING id, release_batch_id, operation, COALESCE(prior_execution_id, '') AS prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, COALESCE(release_change_set_id::text,'') AS release_change_set_id +) +SELECT id, release_batch_id, operation, prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, release_change_set_id +FROM inserted_execution +` + +const authorizeReleaseRollbackSQL = ` +WITH eligible_prior AS ( + SELECT execution.id, execution.release_batch_id, execution.release_change_set_id, COUNT(*)::INTEGER AS item_count + FROM release_executions AS execution + JOIN release_item_executions AS item ON item.release_execution_id = execution.id + WHERE execution.id = $1::VARCHAR + AND execution.operation = 'publish' + AND execution.state IN ('completed', 'failed') + AND execution.failure_level <> 'overall' + AND item.state = 'verified' + GROUP BY execution.id, execution.release_batch_id, execution.release_change_set_id + HAVING COUNT(*) > 0 +), inserted_job AS ( + INSERT INTO jobs ( + id, type, payload, state, idempotency_key, + max_attempts, available_at, created_at, updated_at + ) + SELECT $3::VARCHAR, 'release.rollback', + jsonb_build_object( + 'execution_id', $2::VARCHAR, + 'prior_execution_id', eligible_prior.id, + 'release_batch_id', eligible_prior.release_batch_id, + 'operation', 'rollback' + ), + 'pending', $7::VARCHAR, 3, $6::TIMESTAMPTZ, $6::TIMESTAMPTZ, $6::TIMESTAMPTZ + FROM eligible_prior + ON CONFLICT (idempotency_key) + DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key + RETURNING id +), inserted_execution AS ( + INSERT INTO release_executions ( + id, release_batch_id, operation, prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, item_count, created_at, release_change_set_id + ) + SELECT $2::VARCHAR, eligible_prior.release_batch_id, 'rollback', eligible_prior.id, + 'pending', 'production', $4::VARCHAR, $5::VARCHAR, + inserted_job.id, eligible_prior.item_count, $6::TIMESTAMPTZ, + eligible_prior.release_change_set_id + FROM eligible_prior + CROSS JOIN inserted_job + ON CONFLICT (prior_execution_id) WHERE operation = 'rollback' + DO UPDATE SET prior_execution_id = EXCLUDED.prior_execution_id + RETURNING id, release_batch_id, operation, COALESCE(prior_execution_id, '') AS prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, COALESCE(release_change_set_id::text,'') AS release_change_set_id +) +SELECT id, release_batch_id, operation, prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, release_change_set_id +FROM inserted_execution +` + +const scheduleAutomaticRollbackSQL = ` +WITH eligible_prior AS ( + SELECT execution.id, execution.release_batch_id, execution.release_change_set_id, COUNT(*)::INTEGER AS item_count + FROM release_executions AS execution + JOIN release_item_executions AS item ON item.release_execution_id = execution.id + WHERE execution.id = $1::VARCHAR + AND execution.operation = 'publish' + AND execution.state = 'failed' + AND item.state = 'verified' + GROUP BY execution.id, execution.release_batch_id, execution.release_change_set_id + HAVING COUNT(*)>0 +), inserted_job AS ( + INSERT INTO jobs(id,type,payload,state,idempotency_key,max_attempts,available_at,created_at,updated_at) + SELECT $3::VARCHAR,'release.rollback',jsonb_build_object( + 'execution_id',$2::VARCHAR, + 'prior_execution_id',eligible_prior.id, + 'release_batch_id',eligible_prior.release_batch_id, + 'operation','rollback'), + 'pending',$5::VARCHAR,3,$4::TIMESTAMPTZ,$4::TIMESTAMPTZ,$4::TIMESTAMPTZ + FROM eligible_prior + ON CONFLICT(idempotency_key) DO UPDATE SET idempotency_key=EXCLUDED.idempotency_key + RETURNING id +), inserted_execution AS ( + INSERT INTO release_executions( + id,release_batch_id,operation,prior_execution_id,state,target_environment, + authorized_by,authorization_code,job_id,item_count,created_at,release_change_set_id) + SELECT $2::VARCHAR,eligible_prior.release_batch_id,'rollback',eligible_prior.id, + 'pending','production','system:auto-rollback','automatic_rollback_after_failure', + inserted_job.id,eligible_prior.item_count,$4::TIMESTAMPTZ, + eligible_prior.release_change_set_id + FROM eligible_prior CROSS JOIN inserted_job + ON CONFLICT(prior_execution_id) WHERE operation='rollback' + DO UPDATE SET prior_execution_id=EXCLUDED.prior_execution_id + RETURNING id,release_batch_id,operation,COALESCE(prior_execution_id,'') AS prior_execution_id,state,target_environment, + authorized_by,authorization_code,job_id,manifest_sha256,item_count, + batch_size,current_batch,total_batches,completed_item_count,failed_item_count,error_code,created_at,started_at,completed_at,failure_level,restore_trigger,COALESCE(release_change_set_id::text,'') AS release_change_set_id +) +SELECT id,release_batch_id,operation,prior_execution_id,state,target_environment, + authorized_by,authorization_code,job_id,manifest_sha256,item_count, + batch_size,current_batch,total_batches,completed_item_count,failed_item_count,error_code,created_at,started_at,completed_at,failure_level,restore_trigger,release_change_set_id +FROM inserted_execution +` + +const retryReleaseExecutionSQL = ` +WITH ` + releaseGateCTEs + `, ` + releaseDeleteReadyCTEs + `, +candidate AS ( + SELECT execution.id, execution.release_batch_id, execution.operation, + execution.prior_execution_id, execution.job_id + FROM release_executions AS execution + WHERE execution.id = $1 + AND (execution.state = 'failed' OR (execution.operation = 'rollback' AND execution.state IN ('rollback_failed', 'rolling_back'))) + AND ( + execution.operation <> 'publish' + OR NOT EXISTS ( + SELECT 1 FROM release_executions AS rollback + WHERE rollback.prior_execution_id = execution.id + AND rollback.operation = 'rollback' + ) + ) + AND ( + execution.operation = 'publish' + OR EXISTS ( + SELECT 1 FROM release_executions AS prior + WHERE prior.id = execution.prior_execution_id + AND prior.operation = 'publish' + AND prior.state IN ('completed', 'failed') + ) + ) +), +gate_stats AS ( + SELECT candidate.id, candidate.release_batch_id, candidate.operation, + candidate.prior_execution_id, candidate.job_id, + COUNT(item.publication_id)::INTEGER AS total_items, + COUNT(*) FILTER ( + WHERE (item.action = 'upsert' + AND ready.publication_id IS NOT NULL + AND ready.review_id = item.review_id + AND ready.acceptance_run_id = item.acceptance_run_id + AND ready.publication_updated_at = item.publication_updated_at + AND ready.decided_visibility = item.decided_visibility) + OR (item.action = 'delete' + AND delete_ready.publication_id IS NOT NULL + AND delete_ready.review_id = item.review_id + AND delete_ready.acceptance_run_id = item.acceptance_run_id + AND delete_ready.publication_updated_at = item.publication_updated_at + AND delete_ready.decided_visibility = item.decided_visibility) + )::INTEGER AS ready_items + FROM candidate + JOIN release_batch_items AS item ON item.release_batch_id = candidate.release_batch_id + LEFT JOIN release_upsert_ready AS ready ON ready.publication_id = item.publication_id AND item.action = 'upsert' + LEFT JOIN release_delete_ready AS delete_ready ON delete_ready.publication_id = item.publication_id AND item.action = 'delete' + GROUP BY candidate.id, candidate.release_batch_id, candidate.operation, + candidate.prior_execution_id, candidate.job_id +), +eligible AS ( + SELECT * FROM gate_stats + WHERE operation = 'rollback' + OR (total_items > 0 AND ready_items = total_items) +), +reset_items AS ( + UPDATE release_item_executions AS item + SET state = 'pending', bundle_sha256 = '', previous_production_revision = '', + production_revision = '', error_code = '', updated_at = $2, completed_at = NULL + FROM eligible + WHERE item.release_execution_id = eligible.id + AND item.state = 'failed' + RETURNING item.release_execution_id +), +reset_item_count AS ( + SELECT release_execution_id, COUNT(*)::INTEGER AS count + FROM reset_items + GROUP BY release_execution_id +), +completed_counts AS ( + SELECT item.release_execution_id, + COUNT(*) FILTER (WHERE item.state IN ('verified', 'rolled_back'))::INTEGER AS completed + FROM release_item_executions AS item + JOIN eligible ON eligible.id = item.release_execution_id + GROUP BY item.release_execution_id +), +reset_job AS ( + UPDATE jobs AS job + SET state = 'pending', attempt = 0, last_error = '', available_at = $2, + lease_owner = NULL, lease_until = NULL, completed_at = NULL, updated_at = $2 + FROM eligible + JOIN reset_item_count ON reset_item_count.release_execution_id = eligible.id + WHERE job.id = eligible.job_id + AND job.state IN ('completed', 'failed') + RETURNING job.id +), +reset_execution AS ( + UPDATE release_executions AS execution + SET state = 'pending', completed_item_count = completed_counts.completed, + failed_item_count = 0, error_code = '', started_at = NULL, completed_at = NULL, + failure_level = '' + FROM eligible + JOIN completed_counts ON completed_counts.release_execution_id = eligible.id + JOIN reset_job ON reset_job.id = eligible.job_id + WHERE execution.id = eligible.id + RETURNING execution.id, execution.release_batch_id, execution.operation, + COALESCE(execution.prior_execution_id, '') AS prior_execution_id, + execution.state, execution.target_environment, execution.authorized_by, + execution.authorization_code, execution.job_id, execution.manifest_sha256, + execution.item_count, execution.batch_size, execution.current_batch, execution.total_batches, + execution.completed_item_count, execution.failed_item_count, + execution.error_code, execution.created_at, execution.started_at, execution.completed_at, + execution.failure_level, execution.restore_trigger, COALESCE(execution.release_change_set_id::text,'') AS release_change_set_id +) +SELECT id, release_batch_id, operation, prior_execution_id, state, target_environment, + authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, release_change_set_id +FROM reset_execution +` + +const countReleaseExecutionsSQL = ` +SELECT COUNT(*)::INTEGER +FROM release_executions +WHERE ($1 = '' OR operation = $1) + AND ($2 = '' OR state = $2) +` + +const listReleaseExecutionsSQL = ` +SELECT id, release_batch_id, operation, COALESCE(prior_execution_id, ''), state, + target_environment, authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, COALESCE(release_change_set_id::text,'') +FROM release_executions +WHERE ($1 = '' OR operation = $1) + AND ($2 = '' OR state = $2) +ORDER BY created_at DESC, id DESC +LIMIT $3 OFFSET $4 +` + +const getReleaseExecutionSQL = ` +SELECT id, release_batch_id, operation, COALESCE(prior_execution_id, ''), state, + target_environment, authorized_by, authorization_code, job_id, manifest_sha256, + item_count, batch_size, current_batch, total_batches, completed_item_count, failed_item_count, error_code, + created_at, started_at, completed_at, failure_level, restore_trigger, COALESCE(release_change_set_id::text,'') +FROM release_executions +WHERE id = $1 +` + +const listReleaseExecutionItemsSQL = ` +SELECT item.publication_id, source.title, item.action, item.state, + item.previous_production_revision, item.production_revision, + item.error_code, item.created_at, item.completed_at +FROM release_item_executions AS item +JOIN test_publications AS publication ON publication.id = item.publication_id +JOIN source_item_versions AS version ON version.id = publication.source_item_version_id +JOIN source_items AS source ON source.id = version.source_item_id +WHERE item.release_execution_id = $1 +ORDER BY item.created_at, item.publication_id +` + +func (s *Store) AuthorizeReleaseBatch(ctx context.Context, request governance.AuthorizeReleaseBatchRequest) (governance.ReleaseExecution, error) { + if !validShortID(request.BatchID) || !validReviewerRef(request.AuthorizedBy) || request.AuthorizationCode != "production_release_authorized" || request.Now.IsZero() { + return governance.ReleaseExecution{}, ErrInvalidReleaseAuthorization + } + executionID, err := newOpaqueID("rx-") + if err != nil { + return governance.ReleaseExecution{}, err + } + jobID, err := newOpaqueID("job-") + if err != nil { + return governance.ReleaseExecution{}, err + } + idempotencyKey := "release:publish:" + request.BatchID + execution, err := scanReleaseExecution(s.db.QueryRow(ctx, authorizeReleaseBatchSQL, + request.BatchID, executionID, jobID, request.AuthorizedBy, + request.AuthorizationCode, request.Now.UTC(), idempotencyKey, + )) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseExecution{}, ErrReleaseAuthorizationGateConflict + } + if err != nil { + return governance.ReleaseExecution{}, fmt.Errorf("authorize release batch: %w", err) + } + return execution, nil +} + +func (s *Store) AuthorizeReleaseRollback(ctx context.Context, request governance.AuthorizeReleaseRollbackRequest) (governance.ReleaseExecution, error) { + if !validShortID(request.PriorExecutionID) || !validReviewerRef(request.AuthorizedBy) || request.AuthorizationCode != "production_rollback_authorized" || request.Now.IsZero() { + return governance.ReleaseExecution{}, ErrInvalidReleaseAuthorization + } + executionID, err := newOpaqueID("rx-") + if err != nil { + return governance.ReleaseExecution{}, err + } + jobID, err := newOpaqueID("job-") + if err != nil { + return governance.ReleaseExecution{}, err + } + idempotencyKey := "release:rollback:" + request.PriorExecutionID + execution, err := scanReleaseExecution(s.db.QueryRow(ctx, authorizeReleaseRollbackSQL, + request.PriorExecutionID, executionID, jobID, request.AuthorizedBy, + request.AuthorizationCode, request.Now.UTC(), idempotencyKey, + )) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseExecution{}, ErrReleaseAuthorizationGateConflict + } + if err != nil { + return governance.ReleaseExecution{}, fmt.Errorf("authorize release rollback: %w", err) + } + return execution, nil +} + +func (s *Store) ScheduleAutomaticRollback(ctx context.Context, priorExecutionID string, now time.Time) error { + if !validShortID(priorExecutionID) || now.IsZero() { + return ErrInvalidReleaseAuthorization + } + executionID, err := newOpaqueID("rx-") + if err != nil { + return err + } + jobID, err := newOpaqueID("job-") + if err != nil { + return err + } + _, err = scanReleaseExecution(s.db.QueryRow(ctx, scheduleAutomaticRollbackSQL, + priorExecutionID, executionID, jobID, now.UTC(), "release:auto-rollback:"+priorExecutionID, + )) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("schedule automatic release rollback: %w", err) + } + return nil +} + +func (s *Store) RetryReleaseExecution(ctx context.Context, request governance.RetryReleaseExecutionRequest) (governance.ReleaseExecution, error) { + if !validShortID(request.ExecutionID) || request.Now.IsZero() { + return governance.ReleaseExecution{}, ErrReleaseExecutionRetryConflict + } + execution, err := scanReleaseExecution(s.db.QueryRow(ctx, retryReleaseExecutionSQL, request.ExecutionID, request.Now.UTC())) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseExecution{}, ErrReleaseExecutionRetryConflict + } + if err != nil { + return governance.ReleaseExecution{}, fmt.Errorf("retry release execution: %w", err) + } + return execution, nil +} + +func (s *Store) ListReleaseExecutions(ctx context.Context, filter governance.ReleaseExecutionFilter) (governance.ReleaseExecutionPage, error) { + if !validReleaseExecutionFilter(filter) { + return governance.ReleaseExecutionPage{}, ErrInvalidReleaseExecutionFilter + } + var total int + if err := s.db.QueryRow(ctx, countReleaseExecutionsSQL, filter.Operation, filter.State).Scan(&total); err != nil { + return governance.ReleaseExecutionPage{}, fmt.Errorf("count release executions: %w", err) + } + rows, err := s.db.Query(ctx, listReleaseExecutionsSQL, filter.Operation, filter.State, filter.Limit, filter.Offset) + if err != nil { + return governance.ReleaseExecutionPage{}, fmt.Errorf("list release executions: %w", err) + } + defer rows.Close() + items := make([]governance.ReleaseExecution, 0) + for rows.Next() { + item, err := scanReleaseExecution(rows) + if err != nil { + return governance.ReleaseExecutionPage{}, fmt.Errorf("scan release execution: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return governance.ReleaseExecutionPage{}, fmt.Errorf("iterate release executions: %w", err) + } + return governance.ReleaseExecutionPage{Items: items, Total: total, Limit: filter.Limit, Offset: filter.Offset}, nil +} + +func (s *Store) GetReleaseExecution(ctx context.Context, executionID string) (governance.ReleaseExecutionDetail, error) { + if !validShortID(executionID) { + return governance.ReleaseExecutionDetail{}, ErrReleaseExecutionNotFound + } + execution, err := scanReleaseExecution(s.db.QueryRow(ctx, getReleaseExecutionSQL, executionID)) + if errors.Is(err, pgx.ErrNoRows) { + return governance.ReleaseExecutionDetail{}, ErrReleaseExecutionNotFound + } + if err != nil { + return governance.ReleaseExecutionDetail{}, fmt.Errorf("get release execution: %w", err) + } + rows, err := s.db.Query(ctx, listReleaseExecutionItemsSQL, executionID) + if err != nil { + return governance.ReleaseExecutionDetail{}, fmt.Errorf("list release execution items: %w", err) + } + defer rows.Close() + items := make([]governance.ReleaseItemExecutionView, 0) + for rows.Next() { + var item governance.ReleaseItemExecutionView + if err := rows.Scan(&item.PublicationID, &item.Title, &item.Action, &item.State, + &item.PreviousProductionRevision, &item.ProductionRevision, &item.ErrorCode, + &item.CreatedAt, &item.CompletedAt); err != nil { + return governance.ReleaseExecutionDetail{}, fmt.Errorf("scan release execution item: %w", err) + } + if item.Action != "upsert" && item.Action != "delete" { + return governance.ReleaseExecutionDetail{}, errors.New("invalid release execution item action") + } + if !validReleaseItemState(item.State) { + return governance.ReleaseExecutionDetail{}, errors.New("invalid release item execution state") + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return governance.ReleaseExecutionDetail{}, fmt.Errorf("iterate release execution items: %w", err) + } + return governance.ReleaseExecutionDetail{ReleaseExecution: execution, Items: items}, nil +} + +func scanReleaseExecution(row rowScanner) (governance.ReleaseExecution, error) { + var execution governance.ReleaseExecution + var operation, state string + if err := row.Scan( + &execution.ID, &execution.ReleaseBatchID, &operation, &execution.PriorExecutionID, &state, + &execution.TargetEnvironment, &execution.AuthorizedBy, &execution.AuthorizationCode, + &execution.JobID, &execution.ManifestSHA256, &execution.ItemCount, + &execution.BatchSize, &execution.CurrentBatch, &execution.TotalBatches, + &execution.CompletedItemCount, &execution.FailedItemCount, &execution.ErrorCode, + &execution.CreatedAt, &execution.StartedAt, &execution.CompletedAt, + &execution.FailureLevel, &execution.RestoreTrigger, &execution.ReleaseChangeSetID, + ); err != nil { + return governance.ReleaseExecution{}, err + } + execution.Operation = governance.ReleaseOperation(operation) + execution.State = governance.ReleaseExecutionState(state) + if !execution.Operation.Valid() || !execution.State.Valid() || execution.TargetEnvironment != "production" { + return governance.ReleaseExecution{}, errors.New("invalid release execution row") + } + return execution, nil +} + +func validReleaseExecutionFilter(filter governance.ReleaseExecutionFilter) bool { + operation := governance.ReleaseOperation(filter.Operation) + state := governance.ReleaseExecutionState(filter.State) + return (filter.Operation == "" || operation.Valid()) && + (filter.State == "" || state.Valid()) && + filter.Limit > 0 && filter.Limit <= 100 && filter.Offset >= 0 && filter.Offset <= 1000000 +} + +func validReleaseItemState(state string) bool { + switch state { + case "pending", "staged", "activated", "verified", "failed", "rolled_back": + return true + default: + return false + } +} diff --git a/internal/platform/postgres/release_executions_test.go b/internal/platform/postgres/release_executions_test.go new file mode 100644 index 0000000..11f1492 --- /dev/null +++ b/internal/platform/postgres/release_executions_test.go @@ -0,0 +1,158 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +func TestAuthorizeReleaseBatchAtomicallyEnqueuesProductionExecution(t *testing.T) { + now := time.Date(2026, 9, 6, 5, 0, 0, 0, time.UTC) + db := &recordingDB{row: releaseExecutionRow{ + id: "rx-safe", batchID: "rb-safe", operation: "publish", state: "pending", + target: "production", authorizedBy: "release-owner", authorizationCode: "production_release_authorized", + jobID: "job-safe", itemCount: 2, createdAt: now, + }} + execution, err := NewStore(db).AuthorizeReleaseBatch(context.Background(), governance.AuthorizeReleaseBatchRequest{ + BatchID: "rb-safe", AuthorizedBy: "release-owner", AuthorizationCode: "production_release_authorized", Now: now, + }) + if err != nil { + t.Fatal(err) + } + if execution.ID != "rx-safe" || execution.Operation != governance.ReleaseOperationPublish || execution.State != governance.ReleaseExecutionPending || execution.JobID != "job-safe" || execution.ItemCount != 2 { + t.Fatalf("execution = %#v", execution) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "state = 'sealed'", "release_ready", "count(*) filter", + "insert into jobs", "'release.publish'", "insert into release_executions", + "'production'", "on conflict (idempotency_key)", "returning", + "release_delete_ready", "item.action = 'upsert'", "item.action = 'delete'", + } { + if !strings.Contains(sql, required) { + t.Fatalf("authorize release SQL missing %q", required) + } + } + for _, forbidden := range []string{"api_key", "password", "secret", "production_database_url"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("authorize release SQL contains forbidden credential field %q", forbidden) + } + } +} + +func TestAuthorizeReleaseBatchRejectsInvalidAuthorization(t *testing.T) { + tests := []governance.AuthorizeReleaseBatchRequest{ + {BatchID: "rb-safe", AuthorizedBy: "owner", AuthorizationCode: "wrong", Now: time.Now()}, + {BatchID: "rb-safe", AuthorizedBy: "bad\nowner", AuthorizationCode: "production_release_authorized", Now: time.Now()}, + } + for _, request := range tests { + if _, err := NewStore(&recordingDB{}).AuthorizeReleaseBatch(context.Background(), request); !errors.Is(err, ErrInvalidReleaseAuthorization) { + t.Fatalf("error = %v", err) + } + } +} + +func TestAuthorizeReleaseBatchRejectsStaleSealGate(t *testing.T) { + _, err := NewStore(&recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}}).AuthorizeReleaseBatch(context.Background(), governance.AuthorizeReleaseBatchRequest{ + BatchID: "rb-safe", AuthorizedBy: "owner", AuthorizationCode: "production_release_authorized", Now: time.Now(), + }) + if !errors.Is(err, ErrReleaseAuthorizationGateConflict) { + t.Fatalf("error = %v", err) + } +} + +func TestAuthorizeReleaseRollbackRequiresVerifiedPriorItems(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 0, 0, 0, time.UTC) + db := &recordingDB{row: releaseExecutionRow{ + id: "rx-rollback", batchID: "rb-safe", operation: "rollback", state: "pending", + target: "production", authorizedBy: "release-owner", authorizationCode: "production_rollback_authorized", + jobID: "job-rollback", itemCount: 2, createdAt: now, + }} + execution, err := NewStore(db).AuthorizeReleaseRollback(context.Background(), governance.AuthorizeReleaseRollbackRequest{ + PriorExecutionID: "rx-publish", AuthorizedBy: "release-owner", + AuthorizationCode: "production_rollback_authorized", Now: now, + }) + if err != nil { + t.Fatal(err) + } + if execution.Operation != governance.ReleaseOperationRollback || execution.ID != "rx-rollback" { + t.Fatalf("execution=%#v", execution) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "operation = 'publish'", "state in ('completed', 'failed')", "item.state = 'verified'", "count(*)::integer", + "'release.rollback'", "insert into jobs", "insert into release_executions", "prior_execution_id", + } { + if !strings.Contains(sql, required) { + t.Fatalf("rollback authorization SQL missing %q", required) + } + } +} + +func TestScheduleAutomaticRollbackUsesOnlyVerifiedItemsFromFailedPublish(t *testing.T) { + now := time.Date(2026, 9, 10, 2, 0, 0, 0, time.UTC) + db := &recordingDB{row: releaseExecutionRow{ + id: "rx-auto", batchID: "rb-safe", operation: "rollback", state: "pending", priorExecutionID: "rx-failed", + target: "production", authorizedBy: "system:auto-rollback", authorizationCode: "automatic_rollback_after_failure", + jobID: "job-auto", itemCount: 1, createdAt: now, + }} + if err := NewStore(db).ScheduleAutomaticRollback(context.Background(), "rx-failed", now); err != nil { + t.Fatal(err) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{"state = 'failed'", "item.state = 'verified'", "count(*)::integer", "'system:auto-rollback'", "'automatic_rollback_after_failure'", "'release.rollback'"} { + if !strings.Contains(sql, required) { + t.Fatalf("automatic rollback SQL missing %q", required) + } + } +} + +func TestAuthorizeReleaseRollbackRejectsInvalidAuthorization(t *testing.T) { + _, err := NewStore(&recordingDB{}).AuthorizeReleaseRollback(context.Background(), governance.AuthorizeReleaseRollbackRequest{ + PriorExecutionID: "rx-publish", AuthorizedBy: "release-owner", AuthorizationCode: "wrong", Now: time.Now(), + }) + if !errors.Is(err, ErrInvalidReleaseAuthorization) { + t.Fatalf("error=%v", err) + } +} + +func TestRetryReleaseExecutionResetsOnlyFailedItemsAndJob(t *testing.T) { + now := time.Date(2026, 9, 6, 17, 0, 0, 0, time.UTC) + db := &recordingDB{row: releaseExecutionRow{id: "rx-safe", batchID: "rb-safe", operation: "publish", state: "pending", target: "production", authorizedBy: "release-owner", authorizationCode: "production_release_authorized", jobID: "job-safe", itemCount: 2, completedCount: 1, createdAt: now}} + store := NewStore(db) + execution, err := store.RetryReleaseExecution(context.Background(), governance.RetryReleaseExecutionRequest{ExecutionID: "rx-safe", Now: now}) + if err != nil { + t.Fatal(err) + } + if execution.ID != "rx-safe" || execution.State != governance.ReleaseExecutionPending || execution.CompletedItemCount != 1 || execution.FailedItemCount != 0 { + t.Fatalf("execution=%#v", execution) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{"release_ready as", "release_delete_ready", "item.action = 'delete'", "update release_item_executions", "state = 'pending'", "and item.state = 'failed'", "update jobs", "attempt = 0", "lease_until = null", "update release_executions", "rollback.prior_execution_id = execution.id"} { + if !strings.Contains(sql, required) { + t.Fatalf("SQL missing %q: %s", required, sql) + } + } +} + +type releaseExecutionRow struct { + id, batchID, operation, priorExecutionID, state, target, authorizedBy, authorizationCode, jobID, manifestSHA, errorCode string + itemCount, batchSize, currentBatch, totalBatches, completedCount, failedCount int + createdAt time.Time + startedAt, completedAt *time.Time +} + +func (r releaseExecutionRow) Scan(dest ...any) error { + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string), *dest[4].(*string) = r.id, r.batchID, r.operation, r.priorExecutionID, r.state + *dest[5].(*string), *dest[6].(*string), *dest[7].(*string), *dest[8].(*string) = r.target, r.authorizedBy, r.authorizationCode, r.jobID + *dest[9].(*string), *dest[10].(*int), *dest[11].(*int), *dest[12].(*int), *dest[13].(*int) = r.manifestSHA, r.itemCount, r.batchSize, r.currentBatch, r.totalBatches + *dest[14].(*int), *dest[15].(*int), *dest[16].(*string) = r.completedCount, r.failedCount, r.errorCode + *dest[17].(*time.Time), *dest[18].(**time.Time), *dest[19].(**time.Time) = r.createdAt, r.startedAt, r.completedAt + return nil +} diff --git a/internal/platform/postgres/release_plan.go b/internal/platform/postgres/release_plan.go new file mode 100644 index 0000000..aaa09a8 --- /dev/null +++ b/internal/platform/postgres/release_plan.go @@ -0,0 +1,132 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "knowledge-governance/internal/governance" +) + +const listReleasePlanSQL = ` +WITH ` + releaseGateCTEs + `, +source_truth AS ( + SELECT source.id AS source_item_id, source.title, source.revision, source.source_change, + COALESCE(item_rule.visibility, directory_rule.visibility, source.visibility) AS desired_policy + FROM source_items source + LEFT JOIN source_item_scope_overrides item_rule ON item_rule.source_item_id=source.id + LEFT JOIN LATERAL ( + SELECT rule.visibility + FROM directory_scope_rules rule + WHERE rule.source_connection_id=source.source_connection_id + AND (rule.directory_path='' OR source.normalized_path=rule.directory_path OR source.normalized_path LIKE rule.directory_path||'/%') + ORDER BY LENGTH(rule.directory_path) DESC + LIMIT 1 + ) directory_rule ON TRUE +), +desired AS ( + SELECT truth.source_item_id, truth.title, scope.target_scope + FROM source_truth truth + CROSS JOIN LATERAL unnest( + CASE + WHEN truth.source_change='deleted' OR truth.desired_policy='blocked' THEN ARRAY[]::TEXT[] + WHEN truth.desired_policy='both' THEN ARRAY['internal','external']::TEXT[] + ELSE ARRAY[truth.desired_policy]::TEXT[] + END + ) scope(target_scope) +), +test_ready AS ( + SELECT DISTINCT ON (version.source_item_id, publication.target_scope) + version.source_item_id, publication.target_scope, publication.id AS publication_id, + ready.review_id AS release_revision + FROM release_ready ready + JOIN test_publications publication ON publication.id=ready.publication_id + JOIN source_item_versions version ON version.id=publication.source_item_version_id + ORDER BY version.source_item_id, publication.target_scope, publication.updated_at DESC, publication.id DESC +), +latest_production AS ( + SELECT DISTINCT ON (version.source_item_id, publication.target_scope) + version.source_item_id, publication.target_scope, + publication.id AS publication_id, item.production_revision + FROM release_item_executions item + JOIN release_executions execution ON execution.id=item.release_execution_id + JOIN test_publications publication ON publication.id=item.publication_id + JOIN source_item_versions version ON version.id=publication.source_item_version_id + WHERE execution.state IN ('completed','rolled_back') + AND item.state IN ('verified','rolled_back') + ORDER BY version.source_item_id, publication.target_scope, execution.completed_at DESC NULLS LAST, execution.id DESC +), +production AS ( + SELECT * FROM latest_production WHERE production_revision<>'' AND production_revision<>'absent' +), +all_scopes AS ( + SELECT source_item_id,target_scope FROM desired + UNION + SELECT source_item_id,target_scope FROM production +), +plan AS ( + SELECT COALESCE(test.publication_id, production.publication_id, '') AS publication_id, + truth.title, + truth.source_change, + scope.target_scope, + CASE + WHEN truth.source_change='unavailable' THEN 'blocked' + WHEN desired.source_item_id IS NULL AND production.source_item_id IS NOT NULL THEN 'delete' + WHEN desired.source_item_id IS NOT NULL AND test.publication_id IS NULL THEN 'blocked' + WHEN production.source_item_id IS NULL THEN 'add' + WHEN production.production_revision<>test.release_revision THEN 'update' + ELSE 'unchanged' + END AS action, + CASE WHEN desired.source_item_id IS NULL THEN 'not_required' WHEN test.publication_id IS NULL THEN 'not_ready' ELSE 'ready' END AS test_state, + CASE WHEN production.source_item_id IS NULL THEN 'absent' ELSE 'present' END AS production_state + FROM all_scopes scope + JOIN source_truth truth ON truth.source_item_id=scope.source_item_id + LEFT JOIN desired ON desired.source_item_id=scope.source_item_id AND desired.target_scope=scope.target_scope + LEFT JOIN test_ready test ON test.source_item_id=scope.source_item_id AND test.target_scope=scope.target_scope + LEFT JOIN production ON production.source_item_id=scope.source_item_id AND production.target_scope=scope.target_scope +) +SELECT publication_id,title,target_scope,action,test_state,production_state, + action IN ('add', 'update', 'delete') AS can_release, + CASE WHEN action='blocked' AND source_change='unavailable' THEN 'source_unavailable' + WHEN action='blocked' THEN 'test_not_ready' + ELSE '' END AS blocker +FROM plan +ORDER BY CASE action WHEN 'delete' THEN 0 WHEN 'blocked' THEN 1 WHEN 'update' THEN 2 WHEN 'add' THEN 3 ELSE 4 END, title, target_scope +` + +func (s *Store) ListReleasePlan(ctx context.Context) (governance.ReleasePlan, error) { + rows, err := s.db.Query(ctx, listReleasePlanSQL) + if err != nil { + return governance.ReleasePlan{}, fmt.Errorf("list release plan: %w", err) + } + defer rows.Close() + plan := governance.ReleasePlan{Items: []governance.ReleasePlanItem{}} + for rows.Next() { + var item governance.ReleasePlanItem + if err := rows.Scan(&item.PublicationID, &item.Title, &item.TargetScope, &item.Action, &item.TestState, &item.ProductionState, &item.CanRelease, &item.Blocker); err != nil { + return governance.ReleasePlan{}, fmt.Errorf("scan release plan: %w", err) + } + if (item.TargetScope != "internal" && item.TargetScope != "external") || (item.Action != "add" && item.Action != "update" && item.Action != "delete" && item.Action != "blocked" && item.Action != "unchanged") { + return governance.ReleasePlan{}, errors.New("invalid release plan row") + } + switch item.Action { + case "add": + plan.Summary.Add++ + case "update": + plan.Summary.Update++ + case "delete": + plan.Summary.Delete++ + case "unchanged": + plan.Summary.Unchanged++ + continue + } + if !item.CanRelease { + plan.Summary.Blocked++ + } + plan.Items = append(plan.Items, item) + } + if err := rows.Err(); err != nil { + return governance.ReleasePlan{}, fmt.Errorf("iterate release plan: %w", err) + } + return plan, nil +} diff --git a/internal/platform/postgres/release_plan_test.go b/internal/platform/postgres/release_plan_test.go new file mode 100644 index 0000000..88c1522 --- /dev/null +++ b/internal/platform/postgres/release_plan_test.go @@ -0,0 +1,91 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestListReleasePlanBuildsSafeSourceTruthActions(t *testing.T) { + rows := &releasePlanRows{items: []releasePlanRow{ + {publicationID: "tp-add", title: "B.pdf", targetScope: "external", action: "add", testState: "ready", productionState: "absent", canRelease: true}, + {publicationID: "tp-delete", title: "C.pdf", targetScope: "internal", action: "delete", testState: "not_required", productionState: "present", canRelease: true, blocker: ""}, + {publicationID: "", title: "D.pdf", targetScope: "internal", action: "blocked", testState: "not_ready", productionState: "absent", canRelease: false, blocker: "test_not_ready"}, + {publicationID: "tp-same", title: "E.pdf", targetScope: "internal", action: "unchanged", testState: "ready", productionState: "present", canRelease: false}, + }} + db := &recordingDB{rows: rows} + plan, err := NewStore(db).ListReleasePlan(context.Background()) + if err != nil { + t.Fatal(err) + } + if plan.Summary.Add != 1 || plan.Summary.Delete != 1 || plan.Summary.Blocked != 1 || plan.Summary.Unchanged != 1 { + t.Fatalf("plan=%#v", plan) + } + if len(plan.Items) != 3 || !plan.Items[0].CanRelease || !plan.Items[1].CanRelease || plan.Items[1].PublicationID != "tp-delete" { + t.Fatalf("items=%#v", plan.Items) + } + if db.querySQL != listReleasePlanSQL { + t.Fatalf("query=%q", db.querySQL) + } + sql := strings.ToLower(db.querySQL) + for _, required := range []string{"coalesce(test.publication_id, production.publication_id, '')", "action in ('add', 'update', 'delete')"} { + if !strings.Contains(sql, required) { + t.Fatalf("release plan SQL missing %q", required) + } + } +} + +type releasePlanRow struct { + publicationID string + title string + targetScope string + action string + testState string + productionState string + canRelease bool + blocker string +} + +func (row releasePlanRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.publicationID + *(dest[1].(*string)) = row.title + *(dest[2].(*string)) = row.targetScope + *(dest[3].(*string)) = row.action + *(dest[4].(*string)) = row.testState + *(dest[5].(*string)) = row.productionState + *(dest[6].(*bool)) = row.canRelease + *(dest[7].(*string)) = row.blocker + return nil +} + +type releasePlanRows struct { + items []releasePlanRow + index int +} + +func (rows *releasePlanRows) Close() {} +func (rows *releasePlanRows) Err() error { return nil } +func (rows *releasePlanRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (rows *releasePlanRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (rows *releasePlanRows) Next() bool { + if rows.index >= len(rows.items) { + return false + } + rows.index++ + return true +} +func (rows *releasePlanRows) Scan(dest ...any) error { return rows.items[rows.index-1].Scan(dest...) } +func (rows *releasePlanRows) Values() ([]any, error) { return nil, errors.New("not implemented") } +func (rows *releasePlanRows) RawValues() [][]byte { return nil } +func (rows *releasePlanRows) Conn() *pgx.Conn { return nil } + +var _ pgx.Rows = (*releasePlanRows)(nil) +var _ = time.Time{} +var _ governance.ReleasePlan diff --git a/internal/platform/postgres/review_and_seal_integration_test.go b/internal/platform/postgres/review_and_seal_integration_test.go new file mode 100644 index 0000000..06f4419 --- /dev/null +++ b/internal/platform/postgres/review_and_seal_integration_test.go @@ -0,0 +1,143 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestReviewAndSealPersistentIntegration(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('conn-manual', 'manual', 'Manual review fixture')`); err != nil { + t.Fatal(err) + } + for index, publicationID := range []string{"tp-ready", "tp-stale", "tp-unreviewed"} { + insertReviewFixture(t, ctx, pool, index, publicationID, now) + } + store := NewStore(pool) + page, err := store.ListReviewCandidates(ctx, governance.ReviewCandidateFilter{ReviewState: "pending", Limit: 20}) + if err != nil || page.Total != 3 || len(page.Items) != 3 { + t.Fatalf("pending review page=%#v error=%v", page, err) + } + + review, err := store.ReviewPublication(ctx, governance.ReviewPublicationRequest{ + PublicationID: "tp-ready", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, + ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(time.Minute), + }) + if err != nil || review.AcceptanceRunID != "run-tp-ready" { + t.Fatalf("review=%#v error=%v", review, err) + } + batch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "Ready batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-ready"}, Now: now.Add(2 * time.Minute), + }) + if err != nil || batch.State != governance.ReleaseBatchDraft || batch.ItemCount != 1 { + t.Fatalf("batch=%#v error=%v", batch, err) + } + sealed, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: batch.ID, SealedBy: "engineer-2", Now: now.Add(3 * time.Minute)}) + if err != nil || sealed.State != governance.ReleaseBatchSealed { + t.Fatalf("sealed=%#v error=%v", sealed, err) + } + batchPage, err := store.ListReleaseBatches(ctx, governance.ReleaseBatchFilter{State: "sealed", Limit: 20}) + if err != nil || batchPage.Total != 1 || len(batchPage.Items) != 1 { + t.Fatalf("batch page=%#v error=%v", batchPage, err) + } + batchDetail, err := store.GetReleaseBatch(ctx, batch.ID) + if err != nil || len(batchDetail.Items) != 1 || batchDetail.Items[0].PublicationID != "tp-ready" || batchDetail.Items[0].DecidedVisibility != governance.VisibilityInternal { + t.Fatalf("batch detail=%#v error=%v", batchDetail, err) + } + + if _, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "Unreviewed batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-unreviewed"}, Now: now.Add(4 * time.Minute), + }); !errors.Is(err, ErrReleaseBatchNotEligible) { + t.Fatalf("unreviewed create error=%v", err) + } + + _, err = store.ReviewPublication(ctx, governance.ReviewPublicationRequest{ + PublicationID: "tp-stale", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, + ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now.Add(5 * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + staleBatch, err := store.CreateReleaseBatch(ctx, governance.CreateReleaseBatchRequest{ + Name: "Stale batch", CreatedBy: "engineer-1", PublicationIDs: []string{"tp-stale"}, Now: now.Add(6 * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE test_publications SET updated_at = updated_at + INTERVAL '1 second' WHERE id = 'tp-stale'`); err != nil { + t.Fatal(err) + } + if _, err := store.SealReleaseBatch(ctx, governance.SealReleaseBatchRequest{BatchID: staleBatch.ID, SealedBy: "engineer-2", Now: now.Add(7 * time.Minute)}); !errors.Is(err, ErrReleaseBatchGateConflict) { + t.Fatalf("stale seal error=%v", err) + } + + var jobs, reviews, batches, sealedBatches int + if err := pool.QueryRow(ctx, `SELECT + (SELECT COUNT(*)::INTEGER FROM jobs), + (SELECT COUNT(*)::INTEGER FROM publication_reviews), + (SELECT COUNT(*)::INTEGER FROM release_batches), + (SELECT COUNT(*)::INTEGER FROM release_batches WHERE state = 'sealed')`).Scan(&jobs, &reviews, &batches, &sealedBatches); err != nil { + t.Fatal(err) + } + if jobs != 0 || reviews != 2 || batches != 2 || sealedBatches != 1 { + t.Fatalf("jobs=%d reviews=%d batches=%d sealed=%d", jobs, reviews, batches, sealedBatches) + } +} + +func insertReviewFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool, index int, publicationID string, now time.Time) { + t.Helper() + itemID := fmt.Sprintf("si-review-%d", index) + versionID := fmt.Sprintf("sv-review-%d", index) + revision := fmt.Sprintf("revision-%d", index) + if _, err := pool.Exec(ctx, `INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, title, + item_type, revision, visibility, visibility_origin, source_change, updated_at + ) VALUES ($1, 'conn-manual', $2, repeat($3, 64), $4, 'document', $5, + 'internal', 'manual', 'added', $6)`, itemID, "external-"+publicationID, fmt.Sprintf("%x", index+1), "Synthetic "+publicationID, revision, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions ( + id, source_item_id, revision, content_state, raw_record_summary, updated_at + ) VALUES ($1, $2, $3, 'verified', '{}'::jsonb, $4)`, versionID, itemID, revision, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO test_publications ( + id, source_item_version_id, test_kb_id, knowledge_id, + ingest_state, processing_state, retrieval_state, + parser_generation, index_generation, updated_at + ) VALUES ($1, $2, 'test-kb', 'knowledge-fixture', 'uploaded', 'completed', 'unpublished', + 'parser-v1', 'index-v1', $3)`, publicationID, versionID, now); err != nil { + t.Fatal(err) + } + states := []struct{ check, state string }{{"knowledge_exists", "passed"}, {"chunks_present", "passed"}, {"retrieval_hit", "manual_required"}} + for checkIndex, state := range states { + if _, err := pool.Exec(ctx, `INSERT INTO acceptance_results ( + id, publication_id, run_id, check_type, state, policy_version, evidence_summary, observed_at + ) VALUES ($1, $2, $3, $4, $5, 'structural-v1', '{}'::jsonb, $6)`, + fmt.Sprintf("ar-%d-%d", index, checkIndex), publicationID, "run-"+publicationID, state.check, state.state, now); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/platform/postgres/reviews.go b/internal/platform/postgres/reviews.go new file mode 100644 index 0000000..209d847 --- /dev/null +++ b/internal/platform/postgres/reviews.go @@ -0,0 +1,253 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ( + ErrInvalidReviewFilter = errors.New("invalid review candidate filter") + ErrInvalidPublicationReview = errors.New("invalid publication review") + ErrPublicationReviewNotEligible = errors.New("publication review is not eligible") +) + +const reviewCandidatesCTE = ` +WITH acceptance_run_summary AS ( + SELECT publication_id, run_id, MAX(observed_at) AS observed_at, + COALESCE(MAX(state) FILTER (WHERE check_type = 'knowledge_exists'), '') AS knowledge_exists, + COALESCE(MAX(state) FILTER (WHERE check_type = 'chunks_present'), '') AS chunks_present, + COALESCE(MAX(state) FILTER (WHERE check_type = 'retrieval_hit'), '') AS retrieval_hit + FROM acceptance_results + GROUP BY publication_id, run_id +), latest_acceptance AS ( + SELECT DISTINCT ON (publication_id) + publication_id, run_id, knowledge_exists, chunks_present, retrieval_hit, observed_at + FROM acceptance_run_summary + ORDER BY publication_id, observed_at DESC, run_id DESC +), candidates AS ( + SELECT p.id AS publication_id, i.title, connection.source_type, + COALESCE(item_scope.visibility, directory_scope.visibility, i.visibility) AS current_visibility, + acceptance.run_id AS acceptance_run_id, + acceptance.knowledge_exists, acceptance.chunks_present, acceptance.retrieval_hit, + CASE + WHEN p.ingest_state <> 'uploaded' OR p.processing_state <> 'completed' OR p.retrieval_state <> 'unpublished' THEN 'blocked' + WHEN version.revision <> i.revision OR i.source_change IN ('deleted', 'unavailable') THEN 'blocked' + WHEN acceptance.knowledge_exists = 'failed' OR acceptance.chunks_present = 'failed' OR acceptance.retrieval_hit = 'failed' THEN 'blocked' + WHEN acceptance.knowledge_exists <> 'passed' OR acceptance.chunks_present <> 'passed' + OR acceptance.retrieval_hit NOT IN ('passed', 'manual_required') THEN 'blocked' + ELSE 'ready' + END AS gate_status, + CASE + WHEN p.ingest_state <> 'uploaded' OR p.processing_state <> 'completed' OR p.retrieval_state <> 'unpublished' THEN 'publication_not_ready' + WHEN version.revision <> i.revision OR i.source_change IN ('deleted', 'unavailable') THEN 'source_not_current' + WHEN acceptance.knowledge_exists = 'failed' OR acceptance.chunks_present = 'failed' OR acceptance.retrieval_hit = 'failed' THEN 'acceptance_failed' + WHEN acceptance.knowledge_exists <> 'passed' OR acceptance.chunks_present <> 'passed' + OR acceptance.retrieval_hit NOT IN ('passed', 'manual_required') THEN 'acceptance_incomplete' + ELSE 'ready' + END AS gate_code, + CASE + WHEN review.id IS NOT NULL AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = p.updated_at THEN review.decision + ELSE 'pending' + END AS review_state, + CASE WHEN review.id IS NOT NULL AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = p.updated_at THEN review.decided_visibility ELSE '' END AS decided_visibility, + CASE WHEN review.id IS NOT NULL AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = p.updated_at THEN review.decision_code ELSE '' END AS decision_code, + CASE WHEN review.id IS NOT NULL AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = p.updated_at THEN review.reviewer_ref ELSE '' END AS reviewer_ref, + CASE WHEN review.id IS NOT NULL AND review.acceptance_run_id = acceptance.run_id + AND review.publication_updated_at = p.updated_at THEN review.created_at ELSE NULL END AS reviewed_at, + p.updated_at + FROM test_publications AS p + JOIN source_item_versions AS version ON version.id = p.source_item_version_id + JOIN source_items AS i ON i.id = version.source_item_id + JOIN source_connections AS connection ON connection.id = i.source_connection_id + LEFT JOIN source_item_scope_overrides AS item_scope ON item_scope.source_item_id = i.id + LEFT JOIN LATERAL ( + SELECT rule.visibility + FROM directory_scope_rules AS rule + WHERE rule.source_connection_id = i.source_connection_id + AND ( + rule.directory_path = '' + OR i.normalized_path = rule.directory_path + OR LEFT(i.normalized_path, LENGTH(rule.directory_path) + 1) = rule.directory_path || '/' + ) + ORDER BY LENGTH(rule.directory_path) DESC + LIMIT 1 + ) AS directory_scope ON TRUE + JOIN latest_acceptance AS acceptance ON acceptance.publication_id = p.id + LEFT JOIN LATERAL ( + SELECT id, decision, decided_visibility, decision_code, reviewer_ref, + acceptance_run_id, publication_updated_at, created_at + FROM publication_reviews + WHERE publication_id = p.id + ORDER BY created_at DESC, id DESC + LIMIT 1 + ) AS review ON TRUE +) +` + +const countReviewCandidatesSQL = reviewCandidatesCTE + ` +SELECT COUNT(*)::INTEGER +FROM candidates +WHERE $1 = '' OR review_state = $1 +` + +const listReviewCandidatesSQL = reviewCandidatesCTE + ` +SELECT publication_id, title, source_type, current_visibility, + acceptance_run_id, knowledge_exists, chunks_present, retrieval_hit, + gate_status, gate_code, review_state, decided_visibility, + decision_code, reviewer_ref, reviewed_at, updated_at +FROM candidates +WHERE $1 = '' OR review_state = $1 +ORDER BY updated_at DESC, publication_id +LIMIT $2 OFFSET $3 +` + +const reviewPublicationSQL = reviewCandidatesCTE + ` +INSERT INTO publication_reviews ( + id, publication_id, decision, decided_visibility, reviewer_ref, + decision_code, acceptance_run_id, publication_updated_at, created_at +) +SELECT $1::varchar, publication_id, $3::varchar, $4::varchar, $5::varchar, $6::varchar, + acceptance_run_id, updated_at, $7::timestamptz +FROM candidates +WHERE publication_id = $2::varchar + AND ($3::varchar = 'rejected' OR gate_status = 'ready') +ON CONFLICT ( + publication_id, acceptance_run_id, publication_updated_at, + reviewer_ref, decision, decided_visibility, decision_code +) DO UPDATE SET id = publication_reviews.id +RETURNING id, publication_id, decision, decided_visibility, reviewer_ref, + decision_code, acceptance_run_id, publication_updated_at, created_at +` + +func (s *Store) ListReviewCandidates(ctx context.Context, filter governance.ReviewCandidateFilter) (governance.ReviewCandidatePage, error) { + if filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 || + filter.ReviewState != "" && filter.ReviewState != string(governance.ReviewPending) && + filter.ReviewState != string(governance.ReviewApproved) && filter.ReviewState != string(governance.ReviewRejected) { + return governance.ReviewCandidatePage{}, ErrInvalidReviewFilter + } + page := governance.ReviewCandidatePage{Items: []governance.ReviewCandidate{}, Limit: filter.Limit, Offset: filter.Offset} + if err := s.db.QueryRow(ctx, countReviewCandidatesSQL, filter.ReviewState).Scan(&page.Total); err != nil { + return governance.ReviewCandidatePage{}, fmt.Errorf("count review candidates: %w", err) + } + rows, err := s.db.Query(ctx, listReviewCandidatesSQL, filter.ReviewState, filter.Limit, filter.Offset) + if err != nil { + return governance.ReviewCandidatePage{}, fmt.Errorf("list review candidates: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.ReviewCandidate + var sourceType, visibility, knowledgeState, chunksState, retrievalState, gateStatus, reviewState string + if err := rows.Scan( + &item.PublicationID, &item.Title, &sourceType, &visibility, + &item.AcceptanceRunID, &knowledgeState, &chunksState, &retrievalState, + &gateStatus, &item.GateCode, &reviewState, &item.DecidedVisibility, + &item.DecisionCode, &item.ReviewerRef, &item.ReviewedAt, &item.PublicationUpdated, + ); err != nil { + return governance.ReviewCandidatePage{}, fmt.Errorf("scan review candidate: %w", err) + } + item.SourceType = governance.SourceType(sourceType) + item.CurrentVisibility = governance.Visibility(visibility) + item.KnowledgeExists = governance.AcceptanceState(knowledgeState) + item.ChunksPresent = governance.AcceptanceState(chunksState) + item.RetrievalHit = governance.AcceptanceState(retrievalState) + item.GateStatus = governance.ReviewGateStatus(gateStatus) + item.ReviewState = governance.ReviewState(reviewState) + if !item.SourceType.Valid() || !item.GateStatus.Valid() || !item.ReviewState.Valid() || + !item.KnowledgeExists.Valid() || !item.ChunksPresent.Valid() || !item.RetrievalHit.Valid() || + item.CurrentVisibility != governance.VisibilityInternal && item.CurrentVisibility != governance.VisibilityExternal { + return governance.ReviewCandidatePage{}, errors.New("invalid review candidate row") + } + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.ReviewCandidatePage{}, fmt.Errorf("iterate review candidates: %w", err) + } + return page, nil +} + +func (s *Store) ReviewPublicationWithOperation(ctx context.Context, request governance.ReviewPublicationRequest, operationID string) (governance.PublicationReview, bool, error) { + fingerprint, err := operationFingerprint(struct { + PublicationID string `json:"publication_id"` + Decision governance.ReviewState `json:"decision"` + Visibility governance.Visibility `json:"decided_visibility"` + ReviewerRef string `json:"reviewer_ref"` + Code string `json:"decision_code"` + }{request.PublicationID, request.Decision, request.DecidedVisibility, request.ReviewerRef, request.DecisionCode}) + if err != nil { + return governance.PublicationReview{}, false, err + } + raw, replayed, err := s.executeOperation(ctx, "publication.review", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + review, err := NewStore(tx).ReviewPublication(ctx, request) + if err != nil { + return nil, err + } + return json.Marshal(review) + }) + if err != nil { + return governance.PublicationReview{}, false, err + } + var review governance.PublicationReview + if err := json.Unmarshal(raw, &review); err != nil { + return governance.PublicationReview{}, false, fmt.Errorf("decode stored review result: %w", err) + } + return review, replayed, nil +} + +func (s *Store) ReviewPublication(ctx context.Context, request governance.ReviewPublicationRequest) (governance.PublicationReview, error) { + if !validPublicationReview(request) { + return governance.PublicationReview{}, ErrInvalidPublicationReview + } + reviewID, err := newOpaqueID("rv-") + if err != nil { + return governance.PublicationReview{}, err + } + var review governance.PublicationReview + var decision, visibility string + err = s.db.QueryRow(ctx, reviewPublicationSQL, + reviewID, request.PublicationID, string(request.Decision), string(request.DecidedVisibility), + request.ReviewerRef, request.DecisionCode, request.Now, + ).Scan( + &review.ID, &review.PublicationID, &decision, &visibility, &review.ReviewerRef, + &review.DecisionCode, &review.AcceptanceRunID, &review.PublicationUpdated, &review.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return governance.PublicationReview{}, ErrPublicationReviewNotEligible + } + if err != nil { + return governance.PublicationReview{}, fmt.Errorf("review publication: %w", err) + } + review.Decision = governance.ReviewState(decision) + review.DecidedVisibility = governance.Visibility(visibility) + return review, nil +} + +func validPublicationReview(request governance.ReviewPublicationRequest) bool { + if !validShortID(request.PublicationID) || request.Now.IsZero() || + request.DecidedVisibility != governance.VisibilityInternal && request.DecidedVisibility != governance.VisibilityExternal || + !validReviewerRef(request.ReviewerRef) { + return false + } + switch request.Decision { + case governance.ReviewApproved: + return request.DecisionCode == "approved_for_release" + case governance.ReviewRejected: + return request.DecisionCode == "needs_correction" || request.DecisionCode == "classification_changed" || request.DecisionCode == "acceptance_rejected" + default: + return false + } +} + +func validReviewerRef(value string) bool { + return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n ") +} diff --git a/internal/platform/postgres/reviews_test.go b/internal/platform/postgres/reviews_test.go new file mode 100644 index 0000000..efabc29 --- /dev/null +++ b/internal/platform/postgres/reviews_test.go @@ -0,0 +1,169 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestListReviewCandidatesReturnsSafeCurrentGateProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC) + db := &recordingDB{ + row: reviewCountRow{count: 1}, + rows: &reviewCandidateRows{values: []reviewCandidateRow{{ + publicationID: "tp-safe", title: "Synthetic review item", sourceType: "manual", + currentVisibility: "internal", acceptanceRunID: "job-safe", knowledgeState: "passed", + chunksState: "passed", retrievalState: "manual_required", gateStatus: "ready", gateCode: "ready", + reviewState: "pending", updatedAt: now, + }}}, + } + page, err := NewStore(db).ListReviewCandidates(context.Background(), governance.ReviewCandidateFilter{ReviewState: "pending", Limit: 20, Offset: 0}) + if err != nil { + t.Fatalf("ListReviewCandidates() error = %v", err) + } + if page.Total != 1 || len(page.Items) != 1 { + t.Fatalf("page = %#v", page) + } + item := page.Items[0] + if item.PublicationID != "tp-safe" || item.GateStatus != governance.ReviewGateReady || item.ReviewState != governance.ReviewPending || item.RetrievalHit != governance.AcceptanceManualRequired { + t.Fatalf("item = %#v", item) + } + joined := strings.ToLower(db.queryRowSQL + "\n" + db.querySQL) + for _, required := range []string{ + "from acceptance_results", "distinct on (publication_id)", "knowledge_exists", "chunks_present", + "retrieval_hit", "manual_required", "from publication_reviews", "publication_updated_at", "acceptance_run_id", + } { + if !strings.Contains(joined, required) { + t.Errorf("review queue SQL missing %q", required) + } + } + for _, forbidden := range []string{"knowledge_id", "test_kb_id", "local_spool_ref", "stable_external_id", "canonical_uri_hash", "raw_record_summary", "secret_ref", "base_url"} { + if strings.Contains(joined, forbidden) { + t.Errorf("review queue SQL includes forbidden projection %q", forbidden) + } + } +} + +func TestListReviewCandidatesRejectsInvalidFilter(t *testing.T) { + _, err := NewStore(&recordingDB{}).ListReviewCandidates(context.Background(), governance.ReviewCandidateFilter{ReviewState: "invalid", Limit: 20}) + if !errors.Is(err, ErrInvalidReviewFilter) { + t.Fatalf("error = %v, want ErrInvalidReviewFilter", err) + } +} + +func TestReviewPublicationAppendsCurrentAcceptanceBoundDecision(t *testing.T) { + now := time.Date(2026, 9, 6, 8, 10, 0, 0, time.UTC) + db := &recordingDB{row: publicationReviewRow{ + id: "rv-safe", publicationID: "tp-safe", decision: "approved", visibility: "internal", + reviewerRef: "engineer-1", decisionCode: "approved_for_release", acceptanceRunID: "job-safe", + publicationUpdatedAt: now.Add(-time.Minute), createdAt: now, + }} + review, err := NewStore(db).ReviewPublication(context.Background(), governance.ReviewPublicationRequest{ + PublicationID: "tp-safe", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, + ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: now, + }) + if err != nil { + t.Fatalf("ReviewPublication() error = %v", err) + } + if review.ID != "rv-safe" || review.Decision != governance.ReviewApproved || review.AcceptanceRunID != "job-safe" { + t.Fatalf("review = %#v", review) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "insert into publication_reviews", "acceptance_run_id", "publication_updated_at", + "gate_status = 'ready'", "on conflict", "returning", + } { + if !strings.Contains(sql, required) { + t.Errorf("review SQL missing %q", required) + } + } +} + +func TestReviewPublicationRejectsInvalidDecisionContract(t *testing.T) { + requests := []governance.ReviewPublicationRequest{ + {PublicationID: "tp-safe", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "needs_correction", Now: time.Now()}, + {PublicationID: "tp-safe", Decision: governance.ReviewRejected, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: time.Now()}, + {PublicationID: "tp-safe", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, ReviewerRef: "bad\nactor", DecisionCode: "approved_for_release", Now: time.Now()}, + } + for _, request := range requests { + if _, err := NewStore(&recordingDB{}).ReviewPublication(context.Background(), request); !errors.Is(err, ErrInvalidPublicationReview) { + t.Fatalf("request=%#v error=%v, want ErrInvalidPublicationReview", request, err) + } + } +} + +func TestReviewPublicationReturnsNotEligibleWhenGateRejects(t *testing.T) { + db := &recordingDB{row: sourceScanErrorRow{err: pgx.ErrNoRows}} + _, err := NewStore(db).ReviewPublication(context.Background(), governance.ReviewPublicationRequest{ + PublicationID: "tp-safe", Decision: governance.ReviewApproved, DecidedVisibility: governance.VisibilityInternal, + ReviewerRef: "engineer-1", DecisionCode: "approved_for_release", Now: time.Now(), + }) + if !errors.Is(err, ErrPublicationReviewNotEligible) { + t.Fatalf("error=%v, want ErrPublicationReviewNotEligible", err) + } +} + +type reviewCountRow struct{ count int } + +func (r reviewCountRow) Scan(dest ...any) error { + *dest[0].(*int) = r.count + return nil +} + +type publicationReviewRow struct { + id, publicationID, decision, visibility, reviewerRef, decisionCode string + acceptanceRunID string + publicationUpdatedAt, createdAt time.Time +} + +func (r publicationReviewRow) Scan(dest ...any) error { + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = r.id, r.publicationID, r.decision, r.visibility + *dest[4].(*string), *dest[5].(*string), *dest[6].(*string) = r.reviewerRef, r.decisionCode, r.acceptanceRunID + *dest[7].(*time.Time), *dest[8].(*time.Time) = r.publicationUpdatedAt, r.createdAt + return nil +} + +type reviewCandidateRow struct { + publicationID, title, sourceType, currentVisibility string + acceptanceRunID, knowledgeState, chunksState string + retrievalState, gateStatus, gateCode, reviewState string + decidedVisibility, decisionCode, reviewerRef string + reviewedAt *time.Time + updatedAt time.Time +} + +type reviewCandidateRows struct { + values []reviewCandidateRow + index int +} + +func (r *reviewCandidateRows) Close() {} +func (r *reviewCandidateRows) Err() error { return nil } +func (r *reviewCandidateRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *reviewCandidateRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *reviewCandidateRows) RawValues() [][]byte { return nil } +func (r *reviewCandidateRows) Conn() *pgx.Conn { return nil } +func (r *reviewCandidateRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *reviewCandidateRows) Scan(dest ...any) error { + v := r.values[r.index-1] + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string), *dest[3].(*string) = v.publicationID, v.title, v.sourceType, v.currentVisibility + *dest[4].(*string), *dest[5].(*string), *dest[6].(*string), *dest[7].(*string) = v.acceptanceRunID, v.knowledgeState, v.chunksState, v.retrievalState + *dest[8].(*string), *dest[9].(*string), *dest[10].(*string) = v.gateStatus, v.gateCode, v.reviewState + *dest[11].(*string), *dest[12].(*string), *dest[13].(*string) = v.decidedVisibility, v.decisionCode, v.reviewerRef + *dest[14].(**time.Time), *dest[15].(*time.Time) = v.reviewedAt, v.updatedAt + return nil +} +func (r *reviewCandidateRows) Values() ([]any, error) { return nil, nil } diff --git a/internal/platform/postgres/s2_snapshot_history_integration_test.go b/internal/platform/postgres/s2_snapshot_history_integration_test.go new file mode 100644 index 0000000..5293e58 --- /dev/null +++ b/internal/platform/postgres/s2_snapshot_history_integration_test.go @@ -0,0 +1,109 @@ +package postgres + +import ( + "context" + "errors" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// TestS2SnapshotHistoryAndContentHash freezes docs plan S2: +// snapshot nodes carry an optional content_sha256 (never faked), +// two snapshots persist, switching head never rewrites the old one, +// and history listing/read-back works read-only with pagination and cross-source rejection. +func TestS2SnapshotHistoryAndContentHash(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-s2','pp','S2 PP',true); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES + ('snapshot-s2-a','conn-s2',repeat('a',64),'S2 PP',2,1,1,'2026-09-10T00:00:00Z'), + ('snapshot-s2-b','conn-s2',repeat('b',64),'S2 PP',2,1,1,'2026-09-11T00:00:00Z'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes,content_sha256) +VALUES + ('snapshot-s2-a','root',NULL,'folder','S2 PP','',repeat('1',64),NULL,1,1,10,NULL), + ('snapshot-s2-a','file-a','root','file','手册.pdf','手册.pdf',repeat('2',64),10,0,1,10,repeat('c',64)), + ('snapshot-s2-b','root',NULL,'folder','S2 PP','',repeat('1',64),NULL,1,1,10,NULL), + ('snapshot-s2-b','file-b','root','file','手册.pdf','手册.pdf',repeat('2',64),20,0,1,20,NULL); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-s2','snapshot-s2-a',NOW());`); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + + // Snapshot history lists both, head first, newest first, without internal hashes/locators. + history, err := store.ListCatalogSnapshots(ctx, "conn-s2", 10, 0) + if err != nil || history.Total != 2 || len(history.Items) != 2 { + t.Fatalf("history=%#v err=%v", history, err) + } + if history.Items[0].SnapshotID != "snapshot-s2-a" || !history.Items[0].IsHead { + t.Fatalf("head-first order=%#v", history.Items[0]) + } + if history.Items[1].SnapshotID != "snapshot-s2-b" || history.Items[1].IsHead { + t.Fatalf("second=%#v", history.Items[1]) + } + + // Historical node read-back: file-a carries the known content hash, file-b reports null. + pageA, err := store.ListCatalogNodesInSnapshot(ctx, governance.CatalogSnapshotNodeFilter{ConnectionID: "conn-s2", SnapshotID: "snapshot-s2-a", ParentNodeID: "root", Limit: 20}) + if err != nil || pageA.Total != 1 || pageA.Items[0].NodeID != "file-a" || pageA.Items[0].ContentSHA256 == nil || *pageA.Items[0].ContentSHA256 != strings.Repeat("c", 64) { + t.Fatalf("snapshot A page=%#v err=%v", pageA, err) + } + pageB, err := store.ListCatalogNodesInSnapshot(ctx, governance.CatalogSnapshotNodeFilter{ConnectionID: "conn-s2", SnapshotID: "snapshot-s2-b", ParentNodeID: "root", Limit: 20}) + if err != nil || pageB.Total != 1 || pageB.Items[0].NodeID != "file-b" || pageB.Items[0].ContentSHA256 != nil { + t.Fatalf("snapshot B page=%#v err=%v", pageB, err) + } + + // The current head view surfaces the content hash too (rule: unknown stays null, known is shown). + headPage, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-s2", ParentNodeID: "root", Limit: 20}) + if err != nil || len(headPage.Items) != 1 || headPage.Items[0].ContentSHA256 == nil { + t.Fatalf("head page=%#v err=%v", headPage, err) + } + + // Switching head to snapshot B must not rewrite snapshot A. + if _, err := pool.Exec(ctx, `UPDATE source_catalog_heads SET snapshot_id='snapshot-s2-b',activated_at=NOW() WHERE source_connection_id='conn-s2'`); err != nil { + t.Fatal(err) + } + var oldHash *string + if err := pool.QueryRow(ctx, `SELECT content_sha256 FROM source_catalog_nodes WHERE snapshot_id='snapshot-s2-a' AND node_id='file-a'`).Scan(&oldHash); err != nil { + t.Fatal(err) + } + if oldHash == nil || *oldHash != strings.Repeat("c", 64) { + t.Fatalf("old snapshot hash mutated: %v", oldHash) + } + afterSwitch, err := store.ListCatalogSnapshots(ctx, "conn-s2", 10, 0) + if err != nil || afterSwitch.Items[0].SnapshotID != "snapshot-s2-b" || !afterSwitch.Items[0].IsHead { + t.Fatalf("after switch=%#v err=%v", afterSwitch.Items, err) + } + + // Cross-source snapshot lookup and unknown snapshots must fail without leaking rows. + if _, err := store.ListCatalogNodesInSnapshot(ctx, governance.CatalogSnapshotNodeFilter{ConnectionID: "conn-other", SnapshotID: "snapshot-s2-a", ParentNodeID: "root", Limit: 20}); !errors.Is(err, ErrCatalogSnapshotNotFound) { + t.Fatalf("cross-source err=%v", err) + } + if _, err := store.ListCatalogSnapshots(ctx, "conn-other", 10, 0); !errors.Is(err, ErrCatalogSnapshotNotFound) { + t.Fatalf("unknown connection err=%v", err) + } + if _, err := store.ListCatalogNodesInSnapshot(ctx, governance.CatalogSnapshotNodeFilter{ConnectionID: "conn-s2", SnapshotID: "missing", ParentNodeID: "root", Limit: 20}); !errors.Is(err, ErrCatalogSnapshotNotFound) { + t.Fatalf("missing snapshot err=%v", err) + } + // Pagination stays stable. + paged, err := store.ListCatalogNodesInSnapshot(ctx, governance.CatalogSnapshotNodeFilter{ConnectionID: "conn-s2", SnapshotID: "snapshot-s2-a", ParentNodeID: "root", Limit: 1, Offset: 1}) + if err != nil || paged.Total != 1 || len(paged.Items) != 0 { + t.Fatalf("paged=%#v err=%v", paged, err) + } +} diff --git a/internal/platform/postgres/s4_refresh_progress_integration_test.go b/internal/platform/postgres/s4_refresh_progress_integration_test.go new file mode 100644 index 0000000..ec603f9 --- /dev/null +++ b/internal/platform/postgres/s4_refresh_progress_integration_test.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh" +) + +// TestS4RefreshProgressPersistsRichSafeFields freezes the S4 contract: the live +// refresh job payload keeps the directory/count fields, survives a pool restart +// (durable read-back), and rejects control/secret-bearing payloads via validation upstream. +func TestS4RefreshProgressPersistsRichSafeFields(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 11, 16, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,state,payload,idempotency_key,priority,available_at,attempt,max_attempts,lease_owner,lease_until,created_at,updated_at) +VALUES('job-s4','source.catalog.refresh','running','{"connection_id":"conn-s4"}'::jsonb,'s4-key',20,$1,1,3,'worker-s4',$1,$1,$1)`, now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + if err := store.UpdateCatalogRefreshProgress(ctx, "job-s4", "worker-s4", catalogrefresh.Progress{ + Stage: "scanning", Progress: 43, Message: "正在扫描目录：已完成 125 个文件夹，待处理 55 个", + Processed: 125, Total: 180, SourceName: "国内工业", CurrentDirectory: "产品资料/CR系列", + ScannedDirectories: 125, DiscoveredDirectories: 180, ScannedFiles: 2438, + EstimatedTotalFiles: 5662, TotalIsEstimate: true, + }, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + pool.Close() + restarted, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + var directory, stage, sourceName string + var progress, scannedFiles, estimatedTotal int + var isEstimate bool + if err := restarted.QueryRow(ctx, `SELECT payload->>'stage',(payload->>'progress')::integer,payload->>'current_directory',(payload->>'scanned_files')::integer,(payload->>'estimated_total_files')::integer,(payload->>'total_is_estimate')::boolean,payload->>'source_name' FROM jobs WHERE id='job-s4'`).Scan(&stage, &progress, &directory, &scannedFiles, &estimatedTotal, &isEstimate, &sourceName); err != nil { + t.Fatal(err) + } + if stage != "scanning" || progress != 43 || directory != "产品资料/CR系列" || scannedFiles != 2438 || estimatedTotal != 5662 || !isEstimate || sourceName != "国内工业" { + t.Fatalf("payload after restart: stage=%s progress=%d dir=%s files=%d estimated=%d estimate=%v source=%s", stage, progress, directory, scannedFiles, estimatedTotal, isEstimate, sourceName) + } + // A stale lease owner must never overwrite the durable last progress snapshot. + if err := NewStore(restarted).UpdateCatalogRefreshProgress(ctx, "job-s4", "worker-stale", catalogrefresh.Progress{Stage: "scanning", Progress: 1, Message: "stale", Processed: 1, Total: 2}, now.Add(2*time.Minute)); err != ErrLeaseLost { + t.Fatalf("stale update error=%v", err) + } + var preserved int + if err := restarted.QueryRow(ctx, `SELECT (payload->>'progress')::integer FROM jobs WHERE id='job-s4'`).Scan(&preserved); err != nil || preserved != 43 { + t.Fatalf("preserved=%d err=%v", preserved, err) + } +} diff --git a/internal/platform/postgres/s6_scope_policy_integration_test.go b/internal/platform/postgres/s6_scope_policy_integration_test.go new file mode 100644 index 0000000..0fecd64 --- /dev/null +++ b/internal/platform/postgres/s6_scope_policy_integration_test.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// TestS6DirectoryPolicyRecursesToDescendantFolders freezes docs/SOURCE_SCOPE_POLICY.md rule 3: +// annotating a first-level directory must recursively govern files inside nested subdirectories, +// and the effective visibility must survive navigation read-back (enter child, return to parent). +func TestS6DirectoryPolicyRecursesToDescendantFolders(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-s6','pp','S6 PP',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-s6',repeat('9',64),10,'verified/s6.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-s6','conn-s6','external-s6',repeat('8',64),'工业产品/CRA系列/手册.pdf','手册.pdf','document','v1','external','pp_public_tag'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-s6','item-s6','v1','content-s6','verified'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-s6','conn-s6',repeat('7',64),'S6 PP',4,2,1,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) +VALUES + ('snapshot-s6','root-s6',NULL,'folder','S6 PP','',repeat('1',64),NULL,NULL,1,1,10), + ('snapshot-s6','folder-top','root-s6','folder','工业产品','工业产品',repeat('2',64),NULL,NULL,1,1,10), + ('snapshot-s6','folder-nested','folder-top','folder','CRA系列','工业产品/CRA系列',repeat('3',64),NULL,NULL,1,1,10), + ('snapshot-s6','file-deep','folder-nested','file','手册.pdf','工业产品/CRA系列/手册.pdf',repeat('4',64),'item-s6',10,0,1,10); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-s6','snapshot-s6',NOW());`); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 11, 15, 0, 0, 0, time.UTC) + result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ + ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal, + TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now, + }) + if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 { + t.Fatalf("route=%#v err=%v", result, err) + } + // Effective visibility of the deep item must be internal via the directory rule. + assets, err := store.ListSourceAssets(ctx, governance.SourceAssetFilter{Limit: 10}) + if err != nil { + t.Fatal(err) + } + if len(assets.Items) != 1 || assets.Items[0].Visibility != governance.VisibilityInternal || assets.Items[0].VisibilityOrigin != "directory_rule" { + t.Fatalf("asset=%#v err=%v", assets.Items, err) + } + // Enter the nested subdirectory: the file row must show internal (recursion read-back). + nested, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-s6", ParentNodeID: "folder-nested", Limit: 20}) + if err != nil || nested.Total != 1 || nested.Items[0].Visibility != "internal" { + t.Fatalf("nested page=%#v err=%v", nested, err) + } + // Return to the parent: the subdirectory aggregates to internal, not mixed. + parent, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-s6", ParentNodeID: "folder-top", Limit: 20}) + if err != nil || parent.Total != 1 || parent.Items[0].Visibility != "internal" { + t.Fatalf("parent page=%#v err=%v", parent, err) + } +} diff --git a/internal/platform/postgres/s8_change_set_contract_integration_test.go b/internal/platform/postgres/s8_change_set_contract_integration_test.go new file mode 100644 index 0000000..c0450fb --- /dev/null +++ b/internal/platform/postgres/s8_change_set_contract_integration_test.go @@ -0,0 +1,166 @@ +package postgres + +import ( + "context" + "errors" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" +) + +// TestS8ChangeSetContract freezes docs plan S8: the canonical classifier +// distinguishes move (proven rename, same bytes) from move_scope (effective +// visibility changed) from update (content changed), records old/new scope and +// version identity, treats source disappearance as delete+source_missing, and +// the change set state machine accepts planned→confirmed→applying→applied plus +// cancelled/failed while GetLatest surfaces every count. +func TestS8ChangeSetContract(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-s8','alist','S8 来源',true); +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES + ('co-move',repeat('a',64),10,'verified/move.pdf','verified'), + ('co-new',repeat('b',64),12,'verified/new.pdf','verified'), + ('co-gone',repeat('c',64),20,'verified/gone.pdf','verified'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-move','conn-s8','ext-move',repeat('d',64),'旧目录/手册.pdf','手册.pdf','document','v1','internal','manual'), + ('item-upd','conn-s8','ext-upd',repeat('e',64),'更新.pdf','更新.pdf','document','v1','internal','manual'), + ('item-scope','conn-s8','ext-scope',repeat('f',64),'范围.pdf','范围.pdf','document','v1','internal','manual'), + ('item-gone','conn-s8','ext-gone',repeat('0',64),'消失.pdf','消失.pdf','document','v1','internal','manual'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES + ('version-move-old','item-move','v1','co-move','verified'), + ('version-upd-old','item-upd','v1','co-move','verified'), + ('version-scope','item-scope','v1','co-move','verified'), + ('version-gone','item-gone','v1','co-gone','verified'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('s8-old','conn-s8',repeat('1',64),'S8 来源',5,1,4,'2026-09-11T00:00:00Z'), + ('s8-new','conn-s8',repeat('2',64),'S8 来源',5,1,4,'2026-09-12T00:00:00Z'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,content_sha256) +VALUES + ('s8-old','root',NULL,'folder','S8 来源','',repeat('3',64),NULL,NULL,NULL), + ('s8-old','n-move','root','file','手册.pdf','旧目录/手册.pdf',repeat('4',64),'item-move',10,repeat('a',64)), + ('s8-old','n-upd','root','file','更新.pdf','更新.pdf',repeat('5',64),'item-upd',10,repeat('a',64)), + ('s8-old','n-scope','root','file','范围.pdf','范围.pdf',repeat('6',64),'item-scope',10,repeat('a',64)), + ('s8-old','n-gone','root','file','消失.pdf','消失.pdf',repeat('7',64),'item-gone',20,repeat('c',64)), + ('s8-new','root',NULL,'folder','S8 来源','',repeat('3',64),NULL,NULL,NULL), + ('s8-new','n-move','root','file','手册改名.pdf','新目录/手册改名.pdf',repeat('4',64),'item-move',10,repeat('a',64)), + ('s8-new','n-upd','root','file','更新.pdf','更新.pdf',repeat('5',64),'item-upd',12,repeat('b',64)), + ('s8-new','n-scope','root','file','范围.pdf','范围.pdf',repeat('6',64),'item-scope',10,repeat('a',64)), + ('s8-new','n-add','root','file','全新.pdf','全新.pdf',repeat('8',64),NULL,5,NULL); +-- effective scope of 范围.pdf becomes external on the new side via an override. +INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) +VALUES('item-scope','external','engineer-1','S8 scope change demo',NOW(),NOW()); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at) +VALUES('cs-s8','conn-s8','s8-old','s8-new',NOW()); +SELECT compare_catalog_change_set('cs-s8');`) + if err != nil { + t.Fatal(err) + } + + actionOf := func(locator string) governance.CatalogChangeItem { + t.Helper() + detail, err := NewStore(pool).GetLatestCatalogChangeSet(ctx, governance.CatalogChangeSetFilter{ConnectionID: "conn-s8", Limit: 50}) + if err != nil { + t.Fatal(err) + } + for _, item := range detail.Items { + if strings.HasPrefix(item.OldPath, locator) || strings.HasPrefix(item.NewPath, locator) { + return item + } + } + t.Fatalf("locator %q not found in %#v", locator, detail.Items) + return governance.CatalogChangeItem{} + } + moved := actionOf("旧目录") + if moved.Action != "move" || moved.ReasonCode != "content_hash_move" || moved.OldContentSHA256 != strings.Repeat("a", 64) || moved.NewContentSHA256 != moved.OldContentSHA256 { + t.Fatalf("proven rename must be move with hash evidence: %#v", moved) + } + if moved.OldVersionID == "" || moved.NewVersionID != moved.OldVersionID { + t.Fatalf("move keeps one version identity: %#v", moved) + } + updated := actionOf("更新.pdf") + if updated.Action != "update" || updated.ReasonCode != "content_hash_conflict" || updated.NewContentSHA256 != strings.Repeat("b", 64) { + t.Fatalf("changed bytes must be update: %#v", updated) + } + scoped := actionOf("范围.pdf") + if scoped.Action != "move_scope" || scoped.ReasonCode != "scope_change" || scoped.OldScope != "internal" || scoped.NewScope != "external" { + t.Fatalf("visibility change must be move_scope with scopes: %#v", scoped) + } + gone := actionOf("消失.pdf") + if gone.Action != "delete" || gone.ReasonCode != "source_missing" || gone.OldContentSHA256 != strings.Repeat("c", 64) { + t.Fatalf("disappearance must be delete+source_missing with old hash: %#v", gone) + } + added := actionOf("全新.pdf") + if added.Action != "add" || added.ReasonCode != "locator" { + t.Fatalf("new file must be add: %#v", added) + } + + // Counts visible on the header. + detail, err := NewStore(pool).GetLatestCatalogChangeSet(ctx, governance.CatalogChangeSetFilter{ConnectionID: "conn-s8", Limit: 50}) + if err != nil { + t.Fatal(err) + } + if detail.MoveCount != 1 || detail.MoveScopeCount != 1 || detail.UpdateCount != 1 || detail.DeleteCount != 1 || detail.AddCount != 1 { + t.Fatalf("counts=%#v", detail) + } + + // Lifecycle: planned→confirmed→applying→applied and cancel/fail transitions. + transitions := []struct { + from, to string + valid bool + }{ + {"planned", "confirmed", true}, + {"confirmed", "applying", true}, + {"applying", "applied", true}, + {"planned", "cancelled", true}, + {"applying", "failed", true}, + {"applied", "planned", false}, + {"cancelled", "confirmed", false}, + {"planned", "applied", false}, + } + for _, step := range transitions { + if _, err := pool.Exec(ctx, `UPDATE source_catalog_change_sets SET state=$1 WHERE id='cs-s8'`, step.from); err != nil { + t.Fatal(err) + } + if err := validateChangeSetTransition(ctx, pool, "cs-s8", step.from, step.to); (err == nil) != step.valid { + t.Fatalf("transition %s→%s valid=%v err=%v", step.from, step.to, step.valid, err) + } + } +} + +// validateChangeSetTransition mirrors the guard the P3 executor will use: it is a +// pure DB check so S8 can freeze the state machine before P3 ships. +func validateChangeSetTransition(ctx context.Context, pool *pgxpool.Pool, id, from, to string) error { + var allowed bool + if err := pool.QueryRow(ctx, `SELECT count(*)>0 FROM source_catalog_change_sets +WHERE id=$1 AND state=$2 AND CASE + WHEN $3='confirmed' THEN $2='planned' + WHEN $3='applying' THEN $2='confirmed' + WHEN $3='applied' THEN $2='applying' + WHEN $3='cancelled' THEN $2 IN ('planned','confirmed') + WHEN $3='failed' THEN $2 IN ('applying','confirmed') + ELSE FALSE END`, id, from, to).Scan(&allowed); err != nil { + return err + } + if !allowed { + return errInvalidChangeSetTransition + } + return nil +} + +var errInvalidChangeSetTransition = errors.New("invalid catalog change set transition") diff --git a/internal/platform/postgres/scope_rules.go b/internal/platform/postgres/scope_rules.go new file mode 100644 index 0000000..ad33db7 --- /dev/null +++ b/internal/platform/postgres/scope_rules.go @@ -0,0 +1,183 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ErrScopeTargetNotFound = errors.New("scope target not found") + +func (s *Store) ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error) { + if connectionID == "" { + return nil, ErrScopeTargetNotFound + } + var exists bool + if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, connectionID).Scan(&exists); err != nil { + return nil, fmt.Errorf("inspect scope connection: %w", err) + } + if !exists { + return nil, ErrScopeTargetNotFound + } + rows, err := s.db.Query(ctx, ` +SELECT 'directory', source_connection_id, directory_path, '', visibility, updated_by, reason, updated_at +FROM directory_scope_rules +WHERE source_connection_id=$1 +UNION ALL +SELECT 'item', item.source_connection_id, '', override.source_item_id, override.visibility, override.updated_by, override.reason, override.updated_at +FROM source_item_scope_overrides override +JOIN source_items item ON item.id=override.source_item_id +WHERE item.source_connection_id=$1 +ORDER BY 1, 3, 4`, connectionID) + if err != nil { + return nil, fmt.Errorf("list scope rules: %w", err) + } + defer rows.Close() + rules := make([]governance.ScopeRule, 0) + for rows.Next() { + var rule governance.ScopeRule + if err := rows.Scan(&rule.TargetType, &rule.ConnectionID, &rule.DirectoryPath, &rule.SourceItemID, &rule.Visibility, &rule.UpdatedBy, &rule.Reason, &rule.UpdatedAt); err != nil { + return nil, fmt.Errorf("scan scope rule: %w", err) + } + rules = append(rules, rule) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate scope rules: %w", err) + } + return rules, nil +} + +func (s *Store) MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error) { + if err := request.Validate(); err != nil { + return governance.ScopeRule{}, err + } + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return governance.ScopeRule{}, errors.New("scope rule transactions are unavailable") + } + tx, err := beginner.Begin(ctx) + if err != nil { + return governance.ScopeRule{}, fmt.Errorf("begin scope rule mutation: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + rule, err := mutateScopeRuleTx(ctx, tx, request) + if err != nil { + return governance.ScopeRule{}, err + } + if err := tx.Commit(ctx); err != nil { + return governance.ScopeRule{}, fmt.Errorf("commit scope rule: %w", err) + } + return rule, nil +} + +func (s *Store) MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error) { + if err := request.Validate(); err != nil { + return governance.ScopeRule{}, false, err + } + fingerprint, err := operationFingerprint(struct { + TargetType governance.ScopeTargetType `json:"target_type"` + ConnectionID string `json:"connection_id"` + DirectoryPath string `json:"directory_path"` + SourceItemID string `json:"source_item_id"` + Visibility governance.Visibility `json:"visibility"` + OperatorRef string `json:"operator_ref"` + Reason string `json:"reason"` + Clear bool `json:"clear"` + }{request.TargetType, request.ConnectionID, request.DirectoryPath, request.SourceItemID, request.Visibility, request.OperatorRef, request.Reason, request.Clear}) + if err != nil { + return governance.ScopeRule{}, false, err + } + resultJSON, replayed, err := s.executeOperation(ctx, "scope.rule.mutate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + rule, err := mutateScopeRuleTx(ctx, tx, request) + if err != nil { + return nil, err + } + return json.Marshal(rule) + }) + if err != nil { + return governance.ScopeRule{}, false, err + } + var rule governance.ScopeRule + if err := json.Unmarshal(resultJSON, &rule); err != nil { + return governance.ScopeRule{}, false, fmt.Errorf("decode stored scope rule result: %w", err) + } + return rule, replayed, nil +} + +func mutateScopeRuleTx(ctx context.Context, tx pgx.Tx, request governance.ScopeRuleMutation) (governance.ScopeRule, error) { + var targetExists bool + var err error + if request.TargetType == governance.ScopeTargetDirectory { + err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, request.ConnectionID).Scan(&targetExists) + } else { + err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_items WHERE id=$1)`, request.SourceItemID).Scan(&targetExists) + } + if err != nil { + return governance.ScopeRule{}, fmt.Errorf("inspect scope target: %w", err) + } + if !targetExists { + return governance.ScopeRule{}, ErrScopeTargetNotFound + } + + var previous *string + var previousValue string + if request.TargetType == governance.ScopeTargetDirectory { + err = tx.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id=$1 AND directory_path=$2 FOR UPDATE`, request.ConnectionID, request.DirectoryPath).Scan(&previousValue) + } else { + err = tx.QueryRow(ctx, `SELECT visibility FROM source_item_scope_overrides WHERE source_item_id=$1 FOR UPDATE`, request.SourceItemID).Scan(&previousValue) + } + if err == nil { + previous = &previousValue + } else if !errors.Is(err, pgx.ErrNoRows) { + return governance.ScopeRule{}, fmt.Errorf("lock scope rule: %w", err) + } + + if request.TargetType == governance.ScopeTargetDirectory { + if request.Clear { + _, err = tx.Exec(ctx, `DELETE FROM directory_scope_rules WHERE source_connection_id=$1 AND directory_path=$2`, request.ConnectionID, request.DirectoryPath) + } else { + _, err = tx.Exec(ctx, `INSERT INTO directory_scope_rules (source_connection_id,directory_path,visibility,updated_by,reason,created_at,updated_at) +VALUES ($1,$2,$3,$4,$5,$6,$6) +ON CONFLICT (source_connection_id,directory_path) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, request.ConnectionID, request.DirectoryPath, request.Visibility, request.OperatorRef, request.Reason, request.Now) + } + } else if request.Clear { + _, err = tx.Exec(ctx, `DELETE FROM source_item_scope_overrides WHERE source_item_id=$1`, request.SourceItemID) + } else { + _, err = tx.Exec(ctx, `INSERT INTO source_item_scope_overrides (source_item_id,visibility,updated_by,reason,created_at,updated_at) +VALUES ($1,$2,$3,$4,$5,$5) +ON CONFLICT (source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, request.SourceItemID, request.Visibility, request.OperatorRef, request.Reason, request.Now) + } + if err != nil { + return governance.ScopeRule{}, fmt.Errorf("write scope rule: %w", err) + } + + eventID, err := newOpaqueID("se-") + if err != nil { + return governance.ScopeRule{}, err + } + var decided any + if !request.Clear { + decided = string(request.Visibility) + } + _, err = tx.Exec(ctx, `INSERT INTO scope_change_events +(id,target_type,source_connection_id,directory_path,source_item_id,action,previous_visibility,decided_visibility,operator_ref,reason,created_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`, eventID, request.TargetType, nullableString(request.ConnectionID), request.DirectoryPath, nullableString(request.SourceItemID), map[bool]string{true: "clear", false: "set"}[request.Clear], previous, decided, request.OperatorRef, request.Reason, request.Now) + if err != nil { + return governance.ScopeRule{}, fmt.Errorf("audit scope rule: %w", err) + } + return governance.ScopeRule{TargetType: request.TargetType, ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, SourceItemID: request.SourceItemID, Visibility: request.Visibility, UpdatedBy: request.OperatorRef, Reason: request.Reason, UpdatedAt: request.Now}, nil +} + +func nullableString(value string) any { + if value == "" { + return nil + } + return value +} diff --git a/internal/platform/postgres/scope_rules_integration_test.go b/internal/platform/postgres/scope_rules_integration_test.go new file mode 100644 index 0000000..0b32884 --- /dev/null +++ b/internal/platform/postgres/scope_rules_integration_test.go @@ -0,0 +1,137 @@ +package postgres + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestScopeRulesResolveItemOverrideThenNearestDirectoryThenSourceDefault(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections (id, source_type, name, enabled) VALUES ('conn-scope', 'pp', 'Scope PP', true); +INSERT INTO source_items (id, source_connection_id, stable_external_id, canonical_uri_hash, normalized_path, title, item_type, revision, visibility, visibility_origin) +VALUES + ('item-a', 'conn-scope', 'a', repeat('a',64), '产品/说明书', 'A.pdf', 'document', 'v1', 'external', 'pp_public_tag'), + ('item-b', 'conn-scope', 'b', repeat('b',64), '产品/说明书', 'B.pdf', 'document', 'v1', 'external', 'pp_public_tag'); +INSERT INTO source_item_versions (id, source_item_id, revision, content_state) +VALUES ('version-a','item-a','v1','over_limit'), ('version-b','item-b','v1','over_limit'); +INSERT INTO test_publications (id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state,updated_at) +VALUES ('publication-a','version-a','test-kb','knowledge-a','uploaded','completed','unpublished',NOW()); +INSERT INTO acceptance_results (id,publication_id,run_id,check_type,state,policy_version,evidence_summary,observed_at) +VALUES + ('accept-a-1','publication-a','run-a','knowledge_exists','passed','v1','{}',NOW()), + ('accept-a-2','publication-a','run-a','chunks_present','passed','v1','{}',NOW()), + ('accept-a-3','publication-a','run-a','retrieval_hit','manual_required','v1','{}',NOW());`) + if err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 8, 1, 0, 0, 0, time.UTC) + if _, err = store.MutateScopeRule(ctx, governance.ScopeRuleMutation{TargetType: governance.ScopeTargetDirectory, ConnectionID: "conn-scope", DirectoryPath: "产品", Visibility: governance.VisibilityInternal, OperatorRef: "engineer-1", Reason: "内部产品资料", Now: now}); err != nil { + t.Fatal(err) + } + reviews, err := store.ListReviewCandidates(ctx, governance.ReviewCandidateFilter{Limit: 10}) + if err != nil || len(reviews.Items) != 1 || reviews.Items[0].CurrentVisibility != governance.VisibilityInternal { + t.Fatalf("review candidate effective scope=%#v err=%v", reviews, err) + } + if _, err = store.MutateScopeRule(ctx, governance.ScopeRuleMutation{TargetType: governance.ScopeTargetItem, SourceItemID: "item-b", Visibility: governance.VisibilityExternal, OperatorRef: "engineer-1", Reason: "该文件已公开", Now: now.Add(time.Minute)}); err != nil { + t.Fatal(err) + } + page, err := store.ListSourceAssets(ctx, governance.SourceAssetFilter{Limit: 10}) + if err != nil { + t.Fatal(err) + } + got := map[string]governance.SourceAsset{} + for _, item := range page.Items { + got[item.ID] = item + } + if got["item-a"].Visibility != governance.VisibilityInternal || got["item-a"].VisibilityOrigin != "directory_rule" || got["item-a"].ScopeRulePath != "产品" { + t.Fatalf("directory effective scope = %#v", got["item-a"]) + } + if got["item-b"].Visibility != governance.VisibilityExternal || got["item-b"].VisibilityOrigin != "file_override" { + t.Fatalf("item override = %#v", got["item-b"]) + } + rules, err := store.ListScopeRules(ctx, "conn-scope") + if err != nil || len(rules) != 2 { + t.Fatalf("rules=%#v err=%v", rules, err) + } + if _, err = store.MutateScopeRule(ctx, governance.ScopeRuleMutation{TargetType: governance.ScopeTargetItem, SourceItemID: "item-b", Clear: true, OperatorRef: "engineer-1", Reason: "恢复目录设置", Now: now.Add(2 * time.Minute)}); err != nil { + t.Fatal(err) + } + page, _ = store.ListSourceAssets(ctx, governance.SourceAssetFilter{Limit: 10}) + for _, item := range page.Items { + if item.ID == "item-b" && (item.Visibility != governance.VisibilityInternal || item.VisibilityOrigin != "directory_rule") { + t.Fatalf("cleared item = %#v", item) + } + } + if _, err = store.MutateScopeRule(ctx, governance.ScopeRuleMutation{TargetType: governance.ScopeTargetDirectory, ConnectionID: "conn-scope", DirectoryPath: "产品", Clear: true, OperatorRef: "engineer-1", Reason: "恢复来源设置", Now: now.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + page, _ = store.ListSourceAssets(ctx, governance.SourceAssetFilter{Limit: 10}) + for _, item := range page.Items { + if item.Visibility != governance.VisibilityExternal || item.VisibilityOrigin != "pp_public_tag" { + t.Fatalf("source fallback = %#v", item) + } + } + var events int + if err = pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events != 4 { + t.Fatalf("events=%d err=%v", events, err) + } +} + +func TestMutateScopeRuleWithOperationReplaysOnceAndRejectsConflict(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-scope-op','pp','Scope operation',true)`); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + request := governance.ScopeRuleMutation{TargetType: governance.ScopeTargetDirectory, ConnectionID: "conn-scope-op", DirectoryPath: "产品", Visibility: governance.VisibilityInternal, OperatorRef: "engineer-1", Reason: "内部资料", Now: time.Date(2026, 9, 11, 9, 0, 0, 0, time.UTC)} + first, replayed, err := store.MutateScopeRuleWithOperation(ctx, request, "op-scope-1") + if err != nil || replayed || first.Visibility != governance.VisibilityInternal { + t.Fatalf("first=%#v replayed=%v err=%v", first, replayed, err) + } + second, replayed, err := store.MutateScopeRuleWithOperation(ctx, request, "op-scope-1") + if err != nil || !replayed || second != first { + t.Fatalf("second=%#v replayed=%v err=%v", second, replayed, err) + } + conflict := request + conflict.Reason = "不同的决定" + if _, _, err := store.MutateScopeRuleWithOperation(ctx, conflict, "op-scope-1"); !errors.Is(err, ErrOperationRequestConflict) { + t.Fatalf("conflict err=%v", err) + } + var events int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events != 1 { + t.Fatalf("events=%d err=%v", events, err) + } +} diff --git a/internal/platform/postgres/source_assets.go b/internal/platform/postgres/source_assets.go new file mode 100644 index 0000000..04e5436 --- /dev/null +++ b/internal/platform/postgres/source_assets.go @@ -0,0 +1,148 @@ +package postgres + +import ( + "context" + "fmt" + "strings" + "unicode" + + "knowledge-governance/internal/governance" +) + +func (s *Store) ListSourceAssets(ctx context.Context, filter governance.SourceAssetFilter) (governance.SourceAssetPage, error) { + var page governance.SourceAssetPage + if err := s.db.QueryRow(ctx, ` + SELECT + COUNT(*), + COUNT(*) FILTER (WHERE v.raw_record_summary->>'legacy_status' = 'completed'), + COUNT(*) FILTER (WHERE v.raw_record_summary->>'legacy_status' = 'content_alias'), + COUNT(*) FILTER (WHERE v.raw_record_summary->>'legacy_status' = 'over_limit') + FROM source_items i + JOIN source_item_versions v ON v.source_item_id = i.id AND v.revision = i.revision`).Scan( + &page.Summary.Total, + &page.Summary.Completed, + &page.Summary.ContentAlias, + &page.Summary.OverLimit, + ); err != nil { + return governance.SourceAssetPage{}, fmt.Errorf("summarize source assets: %w", err) + } + + rows, err := s.db.Query(ctx, ` + SELECT + i.id, + i.title, + COALESCE(i.normalized_path, ''), + i.source_connection_id, + CASE + WHEN item_scope.source_item_id IS NOT NULL THEN 'file_override' + WHEN directory_scope.source_connection_id IS NOT NULL THEN 'directory_rule' + ELSE i.visibility_origin + END, + COALESCE(directory_scope.directory_path, ''), + COALESCE(p.ingest_state, ''), + COALESCE(NULLIF(p.last_error, ''), v.download_error, ''), + COALESCE(p.id, ''), + COALESCE(((p.processing_state IN ('failed','stalled') OR (p.ingest_state = 'failed' AND p.processing_state = 'pending')) + AND co.content_state = 'verified' AND co.local_spool_ref <> ''), false), + c.source_type, + c.name, + COALESCE(i.size_bytes, 0), + COALESCE(item_scope.visibility, directory_scope.visibility, i.visibility), + i.source_change, + v.content_state, + COALESCE(v.raw_record_summary->>'legacy_status', ''), + p.id IS NOT NULL, + COALESCE(p.processing_state, ''), + COALESCE(p.retrieval_state, ''), + GREATEST(i.updated_at, v.updated_at, COALESCE(p.updated_at, i.updated_at)) + FROM source_items i + JOIN source_connections c ON c.id = i.source_connection_id + JOIN source_item_versions v ON v.source_item_id = i.id AND v.revision = i.revision + LEFT JOIN LATERAL ( + SELECT id, ingest_state, processing_state, retrieval_state, last_error, updated_at + FROM test_publications + WHERE source_item_version_id = v.id + ORDER BY updated_at DESC, id + LIMIT 1 + ) p ON TRUE + LEFT JOIN content_objects co ON co.id = v.content_object_id + LEFT JOIN source_item_scope_overrides item_scope ON item_scope.source_item_id = i.id + LEFT JOIN LATERAL ( + SELECT rule.source_connection_id, rule.directory_path, rule.visibility + FROM directory_scope_rules rule + WHERE rule.source_connection_id = i.source_connection_id + AND ( + rule.directory_path = '' + OR i.normalized_path = rule.directory_path + OR LEFT(i.normalized_path, LENGTH(rule.directory_path) + 1) = rule.directory_path || '/' + ) + ORDER BY LENGTH(rule.directory_path) DESC + LIMIT 1 + ) directory_scope ON TRUE + WHERE ($1 = '' OR v.raw_record_summary->>'legacy_status' = $1) + ORDER BY i.updated_at DESC, i.id + LIMIT $2 OFFSET $3`, filter.State, filter.Limit, filter.Offset) + if err != nil { + return governance.SourceAssetPage{}, fmt.Errorf("list source assets: %w", err) + } + defer rows.Close() + + page.Items = make([]governance.SourceAsset, 0) + for rows.Next() { + var item governance.SourceAsset + var sourceType, visibility, sourceChange, contentState string + if err := rows.Scan( + &item.ID, + &item.Title, + &item.DirectoryPath, + &item.ConnectionID, + &item.VisibilityOrigin, + &item.ScopeRulePath, + &item.IngestState, + &item.LastError, + &item.PublicationID, + &item.CanRetry, + &sourceType, + &item.ConnectionName, + &item.SizeBytes, + &visibility, + &sourceChange, + &contentState, + &item.LegacyStatus, + &item.HasPublication, + &item.ProcessingState, + &item.RetrievalState, + &item.UpdatedAt, + ); err != nil { + return governance.SourceAssetPage{}, fmt.Errorf("scan source asset: %w", err) + } + item.SourceType = governance.SourceType(sourceType) + item.DirectoryPath = safeDocumentDirectory(item.DirectoryPath) + item.Visibility = governance.Visibility(visibility) + item.SourceChange = governance.SourceChange(sourceChange) + item.ContentState = governance.ContentState(contentState) + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.SourceAssetPage{}, fmt.Errorf("iterate source assets: %w", err) + } + return page, nil +} + +// Expose only relative source labels, never host paths or signed URLs. +func safeDocumentDirectory(value string) string { + if strings.HasPrefix(value, "/") || strings.ContainsAny(value, ":\\") || len(value) > 2048 { + return "" + } + for _, part := range strings.Split(value, "/") { + if part == ".." || part == "." { + return "" + } + } + for _, r := range value { + if unicode.IsControl(r) { + return "" + } + } + return value +} diff --git a/internal/platform/postgres/source_assets_test.go b/internal/platform/postgres/source_assets_test.go new file mode 100644 index 0000000..950c2b8 --- /dev/null +++ b/internal/platform/postgres/source_assets_test.go @@ -0,0 +1,14 @@ +package postgres + +import "testing" + +func TestSafeDocumentDirectory(t *testing.T) { + for _, input := range []string{"/home/private/file", "https://example.invalid/?token=secret", "../outside", "a/../outside", "C:\\private", "目录\n越界"} { + if safeDocumentDirectory(input) != "" { + t.Errorf("unsafe directory was exposed") + } + } + if safeDocumentDirectory("用户手册/中文") != "用户手册/中文" { + t.Fatal("relative source directory should remain readable") + } +} diff --git a/internal/platform/postgres/source_catalog.go b/internal/platform/postgres/source_catalog.go new file mode 100644 index 0000000..ad252d2 --- /dev/null +++ b/internal/platform/postgres/source_catalog.go @@ -0,0 +1,307 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ErrCatalogNodeNotFound = errors.New("catalog node not found") + +func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) { + rows, err := s.db.Query(ctx, ` +SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id, + snapshot.folder_count - 1, snapshot.file_count, head.activated_at, + 'ready', COALESCE(refresh.duration_ms,0) +FROM source_catalog_heads head +JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id +JOIN source_connections connection ON connection.id=head.source_connection_id +JOIN source_catalog_nodes root ON root.snapshot_id=snapshot.id AND root.parent_node_id IS NULL +LEFT JOIN LATERAL ( + SELECT (EXTRACT(EPOCH FROM (job.completed_at-COALESCE(job.started_at,job.created_at)))*1000)::BIGINT AS duration_ms + FROM jobs job + WHERE job.type='source.catalog.refresh' AND job.state='completed' + AND job.payload->>'connection_id'=connection.id + ORDER BY job.completed_at DESC + LIMIT 1 +) refresh ON TRUE +ORDER BY CASE connection.source_type WHEN 'dingtalk' THEN 1 WHEN 'pp' THEN 2 WHEN 'alist' THEN 3 ELSE 4 END, + snapshot.source_label, connection.id`) + if err != nil { + return nil, fmt.Errorf("list catalog sources: %w", err) + } + defer rows.Close() + result := make([]governance.CatalogSource, 0) + for rows.Next() { + var item governance.CatalogSource + var sourceType string + if err := rows.Scan(&item.ConnectionID, &sourceType, &item.Name, &item.RootNodeID, &item.FolderCount, &item.FileCount, &item.ObservedAt, &item.SnapshotStatus, &item.LastRefreshDurationMS); err != nil { + return nil, fmt.Errorf("scan catalog source: %w", err) + } + item.SourceType = governance.SourceType(sourceType) + item.SnapshotOnly = true + if !item.SourceType.Valid() || item.FolderCount < 0 || item.FileCount < 0 { + return nil, errors.New("invalid catalog source projection") + } + result = append(result, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate catalog sources: %w", err) + } + return result, nil +} + +func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogNodeFilter) (governance.CatalogNodePage, error) { + if filter.ConnectionID == "" || len(filter.ConnectionID) > 36 || filter.ParentNodeID == "" || len(filter.ParentNodeID) > 36 || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 { + return governance.CatalogNodePage{}, errors.New("invalid catalog node filter") + } + page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset} + var parentID *string + err := s.db.QueryRow(ctx, ` +SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count, + node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at +FROM source_catalog_heads head +JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id +WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan( + &page.Current.NodeID, &parentID, &page.Current.NodeType, &page.Current.Name, &page.Current.DirectChildCount, + &page.Current.SubtreeFileCount, &page.Current.SubtreeBytes, &page.Current.SizeBytes, &page.Current.ModifiedAt) + if errors.Is(err, pgx.ErrNoRows) { + return governance.CatalogNodePage{}, ErrCatalogNodeNotFound + } + if err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("get catalog parent: %w", err) + } + if parentID != nil { + page.Current.ParentNodeID = *parentID + } + page.Current.Visibility = "unknown" + + breadcrumbs, err := s.db.Query(ctx, ` +WITH RECURSIVE trail AS ( + SELECT node.snapshot_id,node.node_id,node.parent_node_id,node.name,0 AS depth + FROM source_catalog_heads head + JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id + WHERE head.source_connection_id=$1 AND node.node_id=$2 + UNION ALL + SELECT parent.snapshot_id,parent.node_id,parent.parent_node_id,parent.name,trail.depth+1 + FROM trail JOIN source_catalog_nodes parent + ON parent.snapshot_id=trail.snapshot_id AND parent.node_id=trail.parent_node_id +) +SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter.ParentNodeID) + if err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("list catalog breadcrumbs: %w", err) + } + for breadcrumbs.Next() { + var item governance.CatalogBreadcrumb + if err := breadcrumbs.Scan(&item.NodeID, &item.Name); err != nil { + breadcrumbs.Close() + return governance.CatalogNodePage{}, fmt.Errorf("scan catalog breadcrumb: %w", err) + } + page.Breadcrumbs = append(page.Breadcrumbs, item) + } + if err := breadcrumbs.Err(); err != nil { + breadcrumbs.Close() + return governance.CatalogNodePage{}, fmt.Errorf("iterate catalog breadcrumbs: %w", err) + } + breadcrumbs.Close() + + if err := s.db.QueryRow(ctx, ` +SELECT COUNT(*)::integer FROM source_catalog_heads head +JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id +WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err) + } + + rows, err := s.db.Query(ctx, ` +WITH RECURSIVE current_head AS ( + SELECT snapshot_id FROM source_catalog_heads WHERE source_connection_id=$1 +), selected AS MATERIALIZED ( + SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id + WHERE node.parent_node_id=$2 + ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id + LIMIT $3 OFFSET $4 +), descendants(ancestor_id,snapshot_id,node_id) AS ( + SELECT node_id,snapshot_id,node_id FROM selected + UNION ALL + SELECT descendants.ancestor_id,child.snapshot_id,child.node_id + FROM descendants JOIN source_catalog_nodes child + ON child.snapshot_id=descendants.snapshot_id AND child.parent_node_id=descendants.node_id +), file_state AS ( + SELECT descendants.ancestor_id,leaf.source_item_id, + COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility, + publication.id AS publication_id,publication.ingest_state,publication.processing_state, + current_version.content_state AS version_content_state,content.content_state AS object_content_state, + content.local_spool_ref + FROM descendants + JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file' + LEFT JOIN source_items item ON item.id=leaf.source_item_id + LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision + LEFT JOIN content_objects content ON content.id=current_version.content_object_id + LEFT JOIN source_item_scope_overrides item_scope ON item_scope.source_item_id=item.id + LEFT JOIN LATERAL ( + SELECT rule.visibility FROM directory_scope_rules rule + WHERE rule.source_connection_id=$1 + AND (rule.directory_path='' OR leaf.relative_path=rule.directory_path + OR LEFT(leaf.relative_path,LENGTH(rule.directory_path)+1)=rule.directory_path||'/') + ORDER BY LENGTH(rule.directory_path) DESC LIMIT 1 + ) directory_scope ON TRUE + LEFT JOIN LATERAL ( + SELECT publication.id,publication.ingest_state,publication.processing_state + FROM test_publications publication + JOIN source_item_versions version ON version.id=publication.source_item_version_id + WHERE version.source_item_id=item.id + ORDER BY publication.updated_at DESC,publication.id LIMIT 1 + ) publication ON TRUE +), stats AS ( + SELECT ancestor_id, + COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count, + COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count, + COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count, + COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='failed' OR processing_state IN ('failed','stalled')))::integer AS failed_count, + COUNT(DISTINCT source_item_id) FILTER (WHERE version_content_state='verified' AND object_content_state='verified' AND local_spool_ref<>'')::integer AS routable_count, + CASE WHEN COUNT(DISTINCT effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL)=0 THEN 'unknown' + WHEN COUNT(DISTINCT effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL)=1 THEN MIN(effective_visibility) FILTER (WHERE effective_visibility IS NOT NULL) + ELSE 'mixed' END AS visibility + FROM file_state GROUP BY ancestor_id +) +SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name, + selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes, + selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'), + COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0), + COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0) +FROM selected LEFT JOIN stats ON stats.ancestor_id=selected.node_id +ORDER BY CASE selected.node_type WHEN 'folder' THEN 0 ELSE 1 END,selected.sort_order,lower(selected.name),selected.node_id`, filter.ConnectionID, filter.ParentNodeID, filter.Limit, filter.Offset) + if err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("list catalog nodes: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.CatalogNode + var parent *string + if err := rows.Scan(&item.NodeID, &parent, &item.NodeType, &item.Name, &item.DirectChildCount, &item.SubtreeFileCount, &item.SubtreeBytes, &item.SizeBytes, &item.ModifiedAt, &item.ContentSHA256, &item.Visibility, &item.ImportedCount, &item.ParsedCount, &item.ActiveCount, &item.FailedCount, &item.RoutableCount); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("scan catalog node: %w", err) + } + if parent != nil { + item.ParentNodeID = *parent + } + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("iterate catalog nodes: %w", err) + } + return page, nil +} + +var ErrCatalogSnapshotNotFound = errors.New("source catalog snapshot not found") + +// ListCatalogSnapshots returns the persisted snapshot history for one configured +// connection. The head snapshot is returned first (current view), followed by +// older snapshots newest-first. It never returns locator hashes or snapshot hashes. +func (s *Store) ListCatalogSnapshots(ctx context.Context, connectionID string, limit, offset int) (governance.CatalogSnapshotPage, error) { + if connectionID == "" || len(connectionID) > 36 || limit < 1 || limit > 100 || offset < 0 || offset > 1000000 { + return governance.CatalogSnapshotPage{}, errors.New("invalid catalog snapshot filter") + } + var exists bool + if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, connectionID).Scan(&exists); err != nil { + return governance.CatalogSnapshotPage{}, fmt.Errorf("inspect snapshot connection: %w", err) + } + if !exists { + return governance.CatalogSnapshotPage{}, ErrCatalogSnapshotNotFound + } + page := governance.CatalogSnapshotPage{Items: []governance.CatalogSnapshot{}, Limit: limit, Offset: offset} + if err := s.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM source_catalog_snapshots WHERE source_connection_id=$1`, connectionID).Scan(&page.Total); err != nil { + return governance.CatalogSnapshotPage{}, fmt.Errorf("count catalog snapshots: %w", err) + } + rows, err := s.db.Query(ctx, ` +SELECT snapshot.id,snapshot.source_connection_id,root.node_id,snapshot.source_label,snapshot.node_count, + snapshot.folder_count,snapshot.file_count,snapshot.observed_at,snapshot.created_at, + (head.snapshot_id IS NOT NULL) AS is_head +FROM source_catalog_snapshots snapshot +JOIN source_catalog_nodes root ON root.snapshot_id=snapshot.id AND root.parent_node_id IS NULL +LEFT JOIN source_catalog_heads head ON head.source_connection_id=snapshot.source_connection_id AND head.snapshot_id=snapshot.id +WHERE snapshot.source_connection_id=$1 +ORDER BY is_head DESC,snapshot.observed_at DESC,snapshot.id DESC +LIMIT $2 OFFSET $3`, connectionID, limit, offset) + if err != nil { + return governance.CatalogSnapshotPage{}, fmt.Errorf("list catalog snapshots: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.CatalogSnapshot + if err := rows.Scan(&item.SnapshotID, &item.ConnectionID, &item.RootNodeID, &item.SourceName, &item.NodeCount, &item.FolderCount, &item.FileCount, &item.ObservedAt, &item.CreatedAt, &item.IsHead); err != nil { + return governance.CatalogSnapshotPage{}, fmt.Errorf("scan catalog snapshot: %w", err) + } + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.CatalogSnapshotPage{}, fmt.Errorf("iterate catalog snapshots: %w", err) + } + return page, nil +} + +// ListCatalogNodesInSnapshot is a read-only view of one persisted snapshot. It never +// mutates the current head. content_sha256 is returned only when actually recorded; +// unknown files return null so the caller can render "内容Hash未获取". +func (s *Store) ListCatalogNodesInSnapshot(ctx context.Context, filter governance.CatalogSnapshotNodeFilter) (governance.CatalogNodePage, error) { + if filter.ConnectionID == "" || len(filter.ConnectionID) > 36 || filter.SnapshotID == "" || len(filter.SnapshotID) > 36 || + filter.ParentNodeID == "" || len(filter.ParentNodeID) > 36 || filter.Limit < 1 || filter.Limit > 100 || filter.Offset < 0 || filter.Offset > 1000000 { + return governance.CatalogNodePage{}, errors.New("invalid catalog snapshot node filter") + } + var parent governance.CatalogNode + var parentID *string + err := s.db.QueryRow(ctx, ` +SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count, + node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at,node.content_sha256 +FROM source_catalog_snapshots snapshot +JOIN source_catalog_nodes node ON node.snapshot_id=snapshot.id +WHERE snapshot.id=$1 AND snapshot.source_connection_id=$2 AND node.node_id=$3 AND node.node_type='folder'`, + filter.SnapshotID, filter.ConnectionID, filter.ParentNodeID).Scan( + &parent.NodeID, &parentID, &parent.NodeType, &parent.Name, &parent.DirectChildCount, + &parent.SubtreeFileCount, &parent.SubtreeBytes, &parent.SizeBytes, &parent.ModifiedAt, &parent.ContentSHA256) + if errors.Is(err, pgx.ErrNoRows) { + return governance.CatalogNodePage{}, ErrCatalogSnapshotNotFound + } + if err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("get snapshot parent: %w", err) + } + page := governance.CatalogNodePage{Current: parent, Limit: filter.Limit, Offset: filter.Offset} + if parentID != nil { + page.Current.ParentNodeID = *parentID + } + page.Current.Visibility = "unknown" + if err := s.db.QueryRow(ctx, ` +SELECT COUNT(*)::integer FROM source_catalog_nodes +WHERE snapshot_id=$1 AND parent_node_id=$2`, filter.SnapshotID, filter.ParentNodeID).Scan(&page.Total); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("count snapshot nodes: %w", err) + } + rows, err := s.db.Query(ctx, ` +SELECT node_id,parent_node_id,node_type,name,direct_child_count,subtree_file_count,subtree_bytes,size_bytes,modified_at,content_sha256 +FROM source_catalog_nodes +WHERE snapshot_id=$1 AND parent_node_id=$2 +ORDER BY CASE node_type WHEN 'folder' THEN 0 ELSE 1 END,sort_order,lower(name),node_id +LIMIT $3 OFFSET $4`, filter.SnapshotID, filter.ParentNodeID, filter.Limit, filter.Offset) + if err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("list snapshot nodes: %w", err) + } + defer rows.Close() + for rows.Next() { + var item governance.CatalogNode + var parentPtr *string + if err := rows.Scan(&item.NodeID, &parentPtr, &item.NodeType, &item.Name, &item.DirectChildCount, &item.SubtreeFileCount, &item.SubtreeBytes, &item.SizeBytes, &item.ModifiedAt, &item.ContentSHA256); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("scan snapshot node: %w", err) + } + if parentPtr != nil { + item.ParentNodeID = *parentPtr + } + item.Visibility = "unknown" + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.CatalogNodePage{}, fmt.Errorf("iterate snapshot nodes: %w", err) + } + return page, nil +} diff --git a/internal/platform/postgres/source_catalog_integration_test.go b/internal/platform/postgres/source_catalog_integration_test.go new file mode 100644 index 0000000..e0b85d4 --- /dev/null +++ b/internal/platform/postgres/source_catalog_integration_test.go @@ -0,0 +1,80 @@ +package postgres + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestSourceCatalogListsOnlyDirectChildrenWithBreadcrumbsAndStatus(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-cat','pp','PP资料下载中心',true); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) +VALUES('item-cat','conn-cat','file-a',repeat('b',64),'工业产品/CRA系列','手册.pdf','document','v1','external','pp_public_tag'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_state) VALUES('version-cat','item-cat','v1','verified'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state) +VALUES('publication-cat','version-cat','test-kb','knowledge-cat','uploaded','completed','unpublished'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-cat','conn-cat',repeat('a',64),'PP资料下载中心',4,2,2,'2026-09-08T00:00:00Z'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes) +VALUES + ('snapshot-cat','root-cat',NULL,'folder','PP资料下载中心','',repeat('1',64),NULL,NULL,-1,2,2,30), + ('snapshot-cat','folder-cat','root-cat','folder','工业产品','工业产品',repeat('2',64),NULL,NULL,0,1,1,10), + ('snapshot-cat','file-cat','folder-cat','file','手册.pdf','工业产品/手册.pdf',repeat('3',64),'item-cat',10,0,0,1,10), + ('snapshot-cat','root-file','root-cat','file','未分类.pdf','未分类.pdf',repeat('4',64),NULL,20,0,0,1,20); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-cat','snapshot-cat',NOW());`) + if err == nil { + _, err = pool.Exec(ctx, `INSERT INTO jobs(id,type,state,payload,idempotency_key,created_at,started_at,completed_at) VALUES('job-cat-refresh','source.catalog.refresh','completed','{"connection_id":"conn-cat"}'::jsonb,'catalog-refresh-test','2026-09-08T00:00:00Z','2026-09-08T00:00:05Z','2026-09-08T00:00:12Z'); +INSERT INTO directory_scope_rules(source_connection_id,directory_path,visibility,updated_by,reason) VALUES('conn-cat','','internal','engineer-1','recursive source policy')`) + } + if err != nil { + t.Fatal(err) + } + store := NewStore(pool) + sources, err := store.ListCatalogSources(ctx) + if err != nil || len(sources) != 1 { + t.Fatalf("sources=%#v err=%v", sources, err) + } + if sources[0].RootNodeID != "root-cat" || sources[0].FileCount != 2 || sources[0].FolderCount != 1 || sources[0].SnapshotStatus != "ready" || sources[0].LastRefreshDurationMS != 7000 { + t.Fatalf("source=%#v", sources[0]) + } + root, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-cat", ParentNodeID: "root-cat", Limit: 20}) + if err != nil { + t.Fatal(err) + } + if root.Total != 2 || len(root.Items) != 2 || root.Items[0].NodeType != "folder" || root.Items[0].Name != "工业产品" || root.Items[1].Name != "未分类.pdf" { + t.Fatalf("root page=%#v", root) + } + if root.Items[0].ImportedCount != 1 || root.Items[0].ParsedCount != 1 || root.Items[0].Visibility != "internal" || root.Items[1].Visibility != "internal" { + t.Fatalf("folder status=%#v", root.Items[0]) + } + child, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-cat", ParentNodeID: "folder-cat", Limit: 20}) + if err != nil { + t.Fatal(err) + } + if child.Total != 1 || child.Items[0].Name != "手册.pdf" || child.Items[0].Visibility != "internal" || len(child.Breadcrumbs) != 2 || child.Breadcrumbs[0].Name != "PP资料下载中心" || child.Breadcrumbs[1].Name != "工业产品" { + t.Fatalf("child=%#v", child) + } + _, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-cat", ParentNodeID: "missing", Limit: 20}) + if !errors.Is(err, ErrCatalogNodeNotFound) { + t.Fatalf("missing error=%v", err) + } +} diff --git a/internal/platform/postgres/source_connections.go b/internal/platform/postgres/source_connections.go new file mode 100644 index 0000000..2fdae16 --- /dev/null +++ b/internal/platform/postgres/source_connections.go @@ -0,0 +1,227 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "knowledge-governance/internal/governance" +) + +const listSourceConnectionsSQL = ` +SELECT + c.id, + c.source_type, + c.name, + c.enabled, + c.schedule <> '{}'::jsonb AS schedule_configured, + CASE + WHEN c.last_health->>'status' IN ('healthy', 'degraded', 'unhealthy') + THEN c.last_health->>'status' + ELSE 'unknown' + END AS health_status, + COUNT(i.id)::int AS item_count, + c.updated_at, + c.next_refresh_at, + scan.id, + scan.state, + scan.observed_items, + scan.unchanged_items, + scan.added_items, + scan.updated_items, + scan.over_limit_items, + scan.missing_items, + scan.planned_uploads, + scan.error_code, + scan.created_at, + scan.completed_at +FROM source_connections c +LEFT JOIN source_items i ON i.source_connection_id = c.id +LEFT JOIN LATERAL ( + SELECT id, state, observed_items, unchanged_items, added_items, + updated_items, over_limit_items, missing_items, planned_uploads, + error_code, created_at, completed_at + FROM source_scans + WHERE source_connection_id = c.id + ORDER BY created_at DESC, id DESC + LIMIT 1 +) scan ON TRUE +GROUP BY c.id, c.source_type, c.name, c.enabled, c.schedule, c.last_health, c.updated_at, c.next_refresh_at, + scan.id, scan.state, scan.observed_items, scan.unchanged_items, + scan.added_items, scan.updated_items, scan.over_limit_items, + scan.missing_items, scan.planned_uploads, scan.error_code, + scan.created_at, scan.completed_at +ORDER BY c.source_type, c.name, c.id +` + +func (s *Store) ListSourceConnections(ctx context.Context) (governance.SourceConnectionPage, error) { + rows, err := s.db.Query(ctx, listSourceConnectionsSQL) + if err != nil { + return governance.SourceConnectionPage{}, fmt.Errorf("list source connections: %w", err) + } + defer rows.Close() + + page := governance.SourceConnectionPage{Items: make([]governance.SourceConnectionView, 0)} + for rows.Next() { + var item governance.SourceConnectionView + var sourceType, healthStatus string + var scanID, scanState, scanError *string + var observed, unchanged, added, updated, overLimit, missing, planned *int + var scanCreatedAt, scanCompletedAt *time.Time + if err := rows.Scan( + &item.ID, + &sourceType, + &item.Name, + &item.Enabled, + &item.ScheduleConfigured, + &healthStatus, + &item.ItemCount, + &item.UpdatedAt, + &item.NextRefreshAt, + &scanID, + &scanState, + &observed, + &unchanged, + &added, + &updated, + &overLimit, + &missing, + &planned, + &scanError, + &scanCreatedAt, + &scanCompletedAt, + ); err != nil { + return governance.SourceConnectionPage{}, fmt.Errorf("scan source connection: %w", err) + } + item.SourceType = governance.SourceType(sourceType) + item.HealthStatus = governance.SourceHealthStatus(healthStatus) + if !item.SourceType.Valid() || !item.HealthStatus.Valid() || item.ItemCount < 0 { + return governance.SourceConnectionPage{}, fmt.Errorf("invalid source connection projection") + } + if scanID != nil { + if scanState == nil || observed == nil || unchanged == nil || added == nil || updated == nil || + overLimit == nil || missing == nil || planned == nil || scanError == nil || scanCreatedAt == nil { + return governance.SourceConnectionPage{}, fmt.Errorf("incomplete source scan projection") + } + state := governance.SourceScanState(*scanState) + if !state.Valid() || *observed < 0 || *unchanged < 0 || *added < 0 || *updated < 0 || *overLimit < 0 || *missing < 0 || *planned < 0 { + return governance.SourceConnectionPage{}, fmt.Errorf("invalid source scan projection") + } + item.LastScan = &governance.SourceScanSummary{ + ID: *scanID, State: state, ObservedItems: *observed, UnchangedItems: *unchanged, + AddedItems: *added, UpdatedItems: *updated, OverLimitItems: *overLimit, + MissingItems: *missing, PlannedUploads: *planned, ErrorCode: *scanError, + CreatedAt: *scanCreatedAt, CompletedAt: scanCompletedAt, + } + } + page.Items = append(page.Items, item) + page.Summary.Total++ + if item.Enabled { + page.Summary.Enabled++ + } else { + page.Summary.Disabled++ + } + switch item.HealthStatus { + case governance.SourceHealthHealthy: + page.Summary.Healthy++ + case governance.SourceHealthDegraded: + page.Summary.Degraded++ + case governance.SourceHealthUnhealthy: + page.Summary.Unhealthy++ + case governance.SourceHealthUnknown: + page.Summary.Unknown++ + } + } + if err := rows.Err(); err != nil { + return governance.SourceConnectionPage{}, fmt.Errorf("iterate source connections: %w", err) + } + return page, nil +} + +// V1 per-source scheduler. One statement claims every enabled, due connection +// (FOR UPDATE OF c SKIP LOCKED so parallel dispatchers never share a row), +// skips sources whose manual or scheduled refresh job is still pending/running, +// enqueues one source.catalog.refresh per remaining source, and pushes each +// claimed source's next_refresh_at out by the interval configured on its +// ACTIVE revision. Enqueue and horizon-advance commit atomically in this single +// statement: a crash before commit rolls both back, and two racing dispatcher +// ticks serialize on the row lock, so the same window cannot double-enqueue. +// The deterministic idempotency key (scheduled-refresh::) is a +// second belt. Disabling a source stops it from ever being claimed again while +// every historical scan, snapshot and job stays intact. Sources without a +// configured interval keep next_refresh_at NULL and are never due (manual-only). +const claimDueSourceRefreshesSQL = ` +WITH due AS ( + SELECT c.id, c.name, c.source_type, c.next_refresh_at, + COALESCE((r.schedule->>'refresh_interval_seconds')::INTEGER, 0) AS interval_seconds + FROM source_connections AS c + LEFT JOIN source_connection_revisions AS r + ON r.id = c.current_revision_id AND r.state = 'active' + WHERE c.enabled AND c.source_type <> 'manual' + AND c.next_refresh_at IS NOT NULL AND c.next_refresh_at <= $1 + ORDER BY c.next_refresh_at, c.id + FOR UPDATE OF c SKIP LOCKED +), +enqueueable AS ( + SELECT d.* FROM due AS d + WHERE NOT EXISTS ( + SELECT 1 FROM jobs + WHERE jobs.type = 'source.catalog.refresh' + AND jobs.state IN ('pending', 'running') + AND jobs.payload->>'connection_id' = d.id + ) +), +inserted AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT 'job-' || replace(gen_random_uuid()::text, '-', ''), 'source.catalog.refresh', + jsonb_build_object( + 'connection_id', e.id, 'source_type', e.source_type, 'source_name', e.name, + 'stage', 'queued', 'progress', 0, 'message', '计划刷新已排队', 'processed', 0, 'total', 0), + 'pending', 20, $1, 3, + 'scheduled-refresh:' || e.id || ':' || FLOOR(EXTRACT(EPOCH FROM e.next_refresh_at))::BIGINT::TEXT, + $1, $1 + FROM enqueueable AS e + ON CONFLICT (idempotency_key) DO NOTHING + RETURNING id +), +advanced AS ( + UPDATE source_connections AS c + SET next_refresh_at = $1 + make_interval(secs => GREATEST(d.interval_seconds, 300)), + last_scheduled_at = $1 + FROM due AS d + WHERE c.id = d.id + RETURNING c.id +) +SELECT (SELECT COUNT(*)::INTEGER FROM advanced), (SELECT COUNT(*)::INTEGER FROM inserted) +` + +func (s *Store) ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (claimed int, enqueued int, err error) { + if now.IsZero() { + return 0, 0, errors.New("claim due source refreshes requires a time") + } + if err := s.db.QueryRow(ctx, claimDueSourceRefreshesSQL, now.UTC()).Scan(&claimed, &enqueued); err != nil { + return 0, 0, fmt.Errorf("claim due source refreshes: %w", err) + } + return claimed, enqueued, nil +} + +// NextRefreshAtFor derives the scheduler horizon from the effective schedule; +// it is the ONLY place next_refresh_at values are computed from user input. +func NextRefreshAtFor(enabled bool, scheduleRaw []byte, from time.Time) (*time.Time, error) { + if !enabled { + return nil, nil + } + interval, err := governance.ParseSourceSchedule(scheduleRaw) + if err != nil { + return nil, err + } + if interval <= 0 { + return nil, nil + } + next := from.UTC().Add(interval) + return &next, nil +} diff --git a/internal/platform/postgres/source_connections_integration_test.go b/internal/platform/postgres/source_connections_integration_test.go new file mode 100644 index 0000000..7bcd89b --- /dev/null +++ b/internal/platform/postgres/source_connections_integration_test.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestListSourceConnectionsIntegrationOmitsSensitiveFields(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("ApplyMigrations() error = %v", err) + } + now := time.Date(2026, 9, 6, 2, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, ` + INSERT INTO source_connections ( + id, source_type, name, base_url, secret_ref, enabled, + schedule, last_health, created_at, updated_at + ) VALUES ( + 'conn-integration', 'alist', 'AList只读集成源', + 'https://sensitive-origin.example', 'sensitive-secret-ref', TRUE, + '{"type":"daily"}'::jsonb, '{"status":"healthy","detail":"must-not-leak"}'::jsonb, $1, $1 + )`, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, + title, item_type, visibility, visibility_origin, source_change, + created_at, updated_at + ) VALUES ( + 'item-integration', 'conn-integration', 'external-1', $2, + 'Safe item', 'file', 'internal', 'configured_scope', 'unchanged', $1, $1 + )`, now, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + + page, err := NewStore(pool).ListSourceConnections(ctx) + if err != nil { + t.Fatalf("ListSourceConnections() error = %v", err) + } + if len(page.Items) != 1 || page.Items[0].ItemCount != 1 || !page.Items[0].ScheduleConfigured || page.Items[0].HealthStatus != "healthy" { + t.Fatalf("page = %#v", page) + } + encoded, err := json.Marshal(page) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"sensitive-origin", "sensitive-secret-ref", "must-not-leak", "base_url", "secret_ref"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("projection leaked %q: %s", forbidden, encoded) + } + } +} diff --git a/internal/platform/postgres/source_connections_test.go b/internal/platform/postgres/source_connections_test.go new file mode 100644 index 0000000..3d92b6e --- /dev/null +++ b/internal/platform/postgres/source_connections_test.go @@ -0,0 +1,132 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestListSourceConnectionsReturnsSafeOperationalProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 1, 0, 0, 0, time.UTC) + horizon := now.Add(time.Hour) + rows := &sourceConnectionRows{items: []sourceConnectionRow{ + {id: "conn-pp", sourceType: "pp", name: "PP历史公开知识", enabled: false, scheduleConfigured: false, health: "healthy", itemCount: 613, updatedAt: now, lastScan: true}, + {id: "conn-manual", sourceType: "manual", name: "本地手工验收", enabled: false, scheduleConfigured: false, health: "unknown", itemCount: 5, updatedAt: now}, + {id: "conn-alist", sourceType: "alist", name: "AList只读源", enabled: true, scheduleConfigured: true, health: "healthy", itemCount: 0, updatedAt: now, nextRefreshAt: &horizon}, + }} + db := &recordingDB{rows: rows} + + page, err := NewStore(db).ListSourceConnections(context.Background()) + if err != nil { + t.Fatalf("ListSourceConnections() error = %v", err) + } + if len(page.Items) != 3 || page.Items[0].ID != "conn-pp" || page.Items[0].ItemCount != 613 || page.Items[2].HealthStatus != governance.SourceHealthHealthy { + t.Fatalf("items = %#v", page.Items) + } + if page.Items[0].LastScan == nil || page.Items[0].LastScan.State != governance.SourceScanCompleted || page.Items[0].LastScan.ObservedItems != 613 || page.Items[0].LastScan.OverLimitItems != 54 { + t.Fatalf("last scan = %#v", page.Items[0].LastScan) + } + // V1: only the scheduled source exposes a next-refresh horizon. + if page.Items[0].NextRefreshAt != nil || page.Items[1].NextRefreshAt != nil { + t.Fatalf("unscheduled sources leak horizons: %v %v", page.Items[0].NextRefreshAt, page.Items[1].NextRefreshAt) + } + if page.Items[2].NextRefreshAt == nil || !page.Items[2].NextRefreshAt.Equal(horizon) { + t.Fatalf("alist horizon = %v want %v", page.Items[2].NextRefreshAt, horizon) + } + wantSummary := governance.SourceConnectionSummary{Total: 3, Enabled: 1, Disabled: 2, Healthy: 2, Unknown: 1} + if page.Summary != wantSummary { + t.Fatalf("summary = %#v, want %#v", page.Summary, wantSummary) + } + query := strings.ToLower(db.querySQL) + for _, required := range []string{"count(i.id)", "schedule <> '{}'::jsonb", "last_health", "group by", "left join lateral", "from source_scans", "limit 1", "c.next_refresh_at"} { + if !strings.Contains(query, required) { + t.Errorf("source connection query missing %q", required) + } + } + for _, forbidden := range []string{"base_url", "secret_ref"} { + if strings.Contains(query, forbidden) { + t.Errorf("source connection query selected sensitive field %q", forbidden) + } + } +} + +type sourceConnectionRow struct { + id string + sourceType string + name string + enabled bool + scheduleConfigured bool + health string + itemCount int + updatedAt time.Time + nextRefreshAt *time.Time + lastScan bool +} + +func (row sourceConnectionRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.id + *(dest[1].(*string)) = row.sourceType + *(dest[2].(*string)) = row.name + *(dest[3].(*bool)) = row.enabled + *(dest[4].(*bool)) = row.scheduleConfigured + *(dest[5].(*string)) = row.health + *(dest[6].(*int)) = row.itemCount + *(dest[7].(*time.Time)) = row.updatedAt + // V1: next_refresh_at pointer, present only for scheduled sources. + *(dest[8].(**time.Time)) = row.nextRefreshAt + if !row.lastScan { + for index := 9; index <= 20; index++ { + switch pointer := dest[index].(type) { + case **string: + *pointer = nil + case **int: + *pointer = nil + case **time.Time: + *pointer = nil + } + } + return nil + } + id, state, errorCode := "scan-1", "completed", "" + observed, unchanged, added, updated, overLimit, missing, planned := 613, 559, 0, 0, 54, 0, 0 + createdAt, completedAt := row.updatedAt.Add(-time.Second), row.updatedAt + *(dest[9].(**string)) = &id + *(dest[10].(**string)) = &state + for index, value := range []*int{&observed, &unchanged, &added, &updated, &overLimit, &missing, &planned} { + *(dest[11+index].(**int)) = value + } + *(dest[18].(**string)) = &errorCode + *(dest[19].(**time.Time)) = &createdAt + *(dest[20].(**time.Time)) = &completedAt + return nil +} + +type sourceConnectionRows struct { + items []sourceConnectionRow + index int +} + +func (rows *sourceConnectionRows) Close() {} +func (rows *sourceConnectionRows) Err() error { return nil } +func (rows *sourceConnectionRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (rows *sourceConnectionRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (rows *sourceConnectionRows) Next() bool { + if rows.index >= len(rows.items) { + return false + } + rows.index++ + return true +} +func (rows *sourceConnectionRows) Scan(dest ...any) error { + return rows.items[rows.index-1].Scan(dest...) +} +func (rows *sourceConnectionRows) Values() ([]any, error) { return nil, errors.New("not implemented") } +func (rows *sourceConnectionRows) RawValues() [][]byte { return nil } +func (rows *sourceConnectionRows) Conn() *pgx.Conn { return nil } diff --git a/internal/platform/postgres/source_scan_integration_test.go b/internal/platform/postgres/source_scan_integration_test.go new file mode 100644 index 0000000..562057d --- /dev/null +++ b/internal/platform/postgres/source_scan_integration_test.go @@ -0,0 +1,116 @@ +package postgres_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/workflows/sourcescan" +) + +func TestPPSourceScanPersistentIntegration(t *testing.T) { + databaseURL := isolatedExternalIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("ApplyMigrations() error = %v", err) + } + + sourceID := strings.Repeat("a", 64) + addedSourceID := strings.Repeat("c", 64) + revision := strings.Repeat("b", 64) + addedRevision := strings.Repeat("d", 64) + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections ( + id, source_type, name, base_url, secret_ref, enabled, schedule, last_health +) VALUES ('conn-pp', 'pp', 'PP fixture', '', '', FALSE, '{}'::jsonb, '{}'::jsonb)`, + ); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, + title, item_type, revision, visibility, visibility_origin, source_change +) VALUES ('item-pp', 'conn-pp', $1::varchar, $1::char(64), '', 'document', $2, + 'external', 'pp_public_tag', 'unchanged')`, sourceID, revision); err != nil { + t.Fatal(err) + } + + root := t.TempDir() + ledgerPath := filepath.Join(root, "state.jsonl") + manifestPath := filepath.Join(root, "manifest.json") + ledger := `{"kb_id":"kb-1","source_url_hash":"` + sourceID + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + revision + `","status":"completed","knowledge_id":"knowledge-1"}` + "\n" + + `{"kb_id":"kb-1","source_url_hash":"` + addedSourceID + `","source_ids":["pp-2"],"bytes":11,"sha256":"` + addedRevision + `","status":"completed","knowledge_id":"knowledge-2"}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil { + t.Fatal(err) + } + scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{ + AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath, + }) + if err != nil { + t.Fatal(err) + } + store := postgresstore.NewStore(pool) + now := time.Date(2026, 9, 6, 6, 0, 0, 0, time.UTC) + scan, err := store.TriggerPPSourceScan(ctx, "conn-pp", now) + if err != nil { + t.Fatalf("TriggerPPSourceScan() error = %v", err) + } + job, err := store.Claim(ctx, "integration-worker", []string{"source.scan.pp"}, now, time.Minute) + if err != nil || job == nil { + t.Fatalf("Claim() job=%#v error=%v", job, err) + } + handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now.Add(time.Second) }}) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(ctx, *job); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if err := store.Complete(ctx, job.ID, "integration-worker", now.Add(2*time.Second)); err != nil { + t.Fatalf("Complete() error = %v", err) + } + + var state, health, jobState string + var observed, unchanged, overLimit, changes int + if err := pool.QueryRow(ctx, ` +SELECT scan.state, scan.observed_items, scan.unchanged_items, + scan.over_limit_items, + (SELECT COUNT(*)::INTEGER FROM source_scan_changes WHERE source_scan_id = scan.id), + connection.last_health->>'status' +FROM source_scans AS scan +JOIN source_connections AS connection ON connection.id = scan.source_connection_id +WHERE scan.id = $1`, scan.ID).Scan(&state, &observed, &unchanged, &overLimit, &changes, &health); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT state FROM jobs WHERE id = $1`, job.ID).Scan(&jobState); err != nil { + t.Fatal(err) + } + if state != "completed" || observed != 2 || unchanged != 1 || overLimit != 0 || changes != 1 || health != "healthy" || jobState != "completed" { + t.Fatalf("state=%s observed=%d unchanged=%d overLimit=%d changes=%d health=%s job=%s", state, observed, unchanged, overLimit, changes, health, jobState) + } + page, err := store.ListSourceScans(ctx, "conn-pp", 20, 0) + if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].AddedItems != 1 { + t.Fatalf("ListSourceScans() page=%#v error=%v", page, err) + } + detail, err := store.GetSourceScanDetail(ctx, "conn-pp", scan.ID) + if err != nil || len(detail.Changes) != 1 || detail.Changes[0].ChangeType != "added" || detail.Changes[0].SourceItemID != nil || detail.Changes[0].ObservedSizeBytes == nil || *detail.Changes[0].ObservedSizeBytes != 11 { + t.Fatalf("GetSourceScanDetail() detail=%#v error=%v", detail, err) + } +} diff --git a/internal/platform/postgres/source_scan_migration_integration_test.go b/internal/platform/postgres/source_scan_migration_integration_test.go new file mode 100644 index 0000000..04b56e1 --- /dev/null +++ b/internal/platform/postgres/source_scan_migration_integration_test.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestLatestGovernanceMigrationRoundTripIntegration(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("initial ApplyMigrations() error = %v", err) + } + assertMigrationVersionAndTables(t, ctx, pool, 31, true) + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("idempotent ApplyMigrations() error = %v", err) + } + if err := RollbackLatestMigration(ctx, pool); err != nil { + t.Fatalf("RollbackLatestMigration() error = %v", err) + } + assertMigrationVersionAndTables(t, ctx, pool, 30, true) + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("reapply migrations error = %v", err) + } + assertMigrationVersionAndTables(t, ctx, pool, 31, true) +} + +func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + for version := 31; version > 10; version-- { + if err := RollbackLatestMigration(ctx, pool); err != nil { + t.Fatal(err) + } + } + assertMigrationVersionAndTables(t, ctx, pool, 10, true) + if _, err := pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name) VALUES('conn-m11','manual','Migration 11'); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin) +VALUES('item-m11','conn-m11','external-m11',repeat('a',64),'外部手册.pdf','document','v1','external','manual'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_state) VALUES('version-m11','item-m11','v1','verified'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id) VALUES('publication-m11','version-m11','external-test-kb');`); err != nil { + t.Fatal(err) + } + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var targetScope string + if err := pool.QueryRow(ctx, `SELECT target_scope FROM test_publications WHERE id='publication-m11'`).Scan(&targetScope); err != nil { + t.Fatal(err) + } + if targetScope != "external" { + t.Fatalf("target_scope=%q want external", targetScope) + } +} + +func assertMigrationVersionAndTables(t *testing.T, ctx context.Context, pool *pgxpool.Pool, wantVersion int, wantReleaseTables bool) { + t.Helper() + var version int + var scans, changes, reviews, batches, batchItems, executions, itemExecutions *string + if err := pool.QueryRow(ctx, `SELECT MAX(version)::INTEGER FROM schema_migrations`).Scan(&version); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT to_regclass('source_scans')::text, + to_regclass('source_scan_changes')::text, + to_regclass('publication_reviews')::text, + to_regclass('release_batches')::text, + to_regclass('release_batch_items')::text, + to_regclass('release_executions')::text, + to_regclass('release_item_executions')::text`).Scan(&scans, &changes, &reviews, &batches, &batchItems, &executions, &itemExecutions); err != nil { + t.Fatal(err) + } + if version != wantVersion || scans == nil || changes == nil || reviews == nil || batches == nil || batchItems == nil || + (executions != nil) != wantReleaseTables || (itemExecutions != nil) != wantReleaseTables { + t.Fatalf("version=%d scans=%v changes=%v reviews=%v batches=%v items=%v executions=%v itemExecutions=%v wantVersion=%d wantReleaseTables=%v", version, scans, changes, reviews, batches, batchItems, executions, itemExecutions, wantVersion, wantReleaseTables) + } +} diff --git a/internal/platform/postgres/source_scans.go b/internal/platform/postgres/source_scans.go new file mode 100644 index 0000000..8f006ea --- /dev/null +++ b/internal/platform/postgres/source_scans.go @@ -0,0 +1,438 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + ppconnector "knowledge-governance/internal/connectors/pp" + "knowledge-governance/internal/governance" +) + +var ( + ErrSourceConnectionNotScannable = errors.New("source connection is not scannable") + ErrSourceScanAlreadyActive = errors.New("source scan already active") + ErrSourceScanStateConflict = errors.New("source scan state conflict") + ErrSourceScanNotFound = errors.New("source scan not found") +) + +const triggerPPSourceScanSQL = ` +WITH target AS ( + SELECT id, source_type + FROM source_connections + WHERE id = $1 AND source_type = 'pp' +), active AS ( + SELECT 1 + FROM source_scans + WHERE source_connection_id = $1 + AND state IN ('pending', 'running') +), created_scan AS ( + INSERT INTO source_scans ( + id, source_connection_id, source_type, state, created_at, updated_at + ) + SELECT $2, target.id, target.source_type, 'pending', $5, $5 + FROM target + WHERE NOT EXISTS (SELECT 1 FROM active) + RETURNING id, source_connection_id, source_type, state, created_at +), created_job AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT $4, 'source.scan.pp', $3::jsonb, 'pending', 20, $5, + 3, 'source-scan:' || created_scan.id, $5, $5 + FROM created_scan + RETURNING id +) +SELECT created_scan.id, created_scan.source_connection_id, + created_scan.source_type, created_scan.state, created_scan.created_at +FROM created_scan CROSS JOIN created_job +` + +const inspectPPSourceScanBlockSQL = ` +SELECT connection.source_type, + EXISTS ( + SELECT 1 FROM source_scans AS scan + WHERE scan.source_connection_id = connection.id + AND scan.state IN ('pending', 'running') + ) +FROM source_connections AS connection +WHERE connection.id = $1 +` + +const startSourceScanSQL = ` +WITH transitioned AS ( + UPDATE source_scans + SET state = 'running', + started_at = COALESCE(started_at, $2), + updated_at = $2 + WHERE id = $1 AND state IN ('pending', 'running') + RETURNING id, source_connection_id, source_type, state, created_at +), already_completed AS ( + SELECT id, source_connection_id, source_type, state, created_at + FROM source_scans + WHERE id = $1 AND state = 'completed' +) +SELECT id, source_connection_id, source_type, state, created_at +FROM transitioned +UNION ALL +SELECT id, source_connection_id, source_type, state, created_at +FROM already_completed +LIMIT 1 +` + +const completeSourceScanSQL = ` +WITH completed AS ( + UPDATE source_scans + SET state = 'completed', snapshot_sha256 = $2, + observed_items = $3, unchanged_items = $4, + added_items = $5, updated_items = $6, + over_limit_items = $7, missing_items = $8, + planned_uploads = $9, error_code = '', + completed_at = $10, updated_at = $10 + WHERE id = $1 AND state = 'running' + RETURNING source_connection_id +), inserted_changes AS ( + INSERT INTO source_scan_changes ( + id, source_scan_id, source_connection_id, stable_external_id, + source_item_id, change_type, previous_revision, + observed_revision, observed_size_bytes, created_at + ) + SELECT change.id, $1, completed.source_connection_id, + change.stable_external_id, item.id, change.change_type, + change.previous_revision, change.observed_revision, + change.observed_size_bytes, $10 + FROM completed + CROSS JOIN jsonb_to_recordset($11::jsonb) AS change( + id TEXT, stable_external_id TEXT, change_type TEXT, + previous_revision TEXT, observed_revision TEXT, + observed_size_bytes BIGINT + ) + LEFT JOIN source_items AS item + ON item.source_connection_id = completed.source_connection_id + AND item.stable_external_id = change.stable_external_id + ON CONFLICT (source_scan_id, stable_external_id) DO NOTHING + RETURNING id +), healthy AS ( + UPDATE source_connections AS connection + SET last_health = jsonb_build_object( + 'status', 'healthy', 'checked_at', $10, 'scan_id', $1 + ), + updated_at = $10 + FROM completed + WHERE connection.id = completed.source_connection_id + RETURNING connection.id +) +SELECT (SELECT COUNT(*)::INTEGER FROM completed), + (SELECT COUNT(*)::INTEGER FROM inserted_changes), + (SELECT COUNT(*)::INTEGER FROM healthy) +` + +const failSourceScanSQL = ` +WITH failed AS ( + UPDATE source_scans + SET state = 'failed', error_code = $2, + completed_at = $3, updated_at = $3 + WHERE id = $1 AND state = 'running' + RETURNING source_connection_id +), unhealthy AS ( + UPDATE source_connections AS connection + SET last_health = jsonb_build_object( + 'status', 'unhealthy', 'checked_at', $3, + 'scan_id', $1, 'error_code', $2 + ), + updated_at = $3 + FROM failed + WHERE connection.id = failed.source_connection_id + RETURNING connection.id +) +SELECT (SELECT COUNT(*)::INTEGER FROM failed), + (SELECT COUNT(*)::INTEGER FROM unhealthy) +` + +func (s *Store) TriggerPPSourceScan(ctx context.Context, connectionID string, now time.Time) (governance.SourceScan, error) { + if strings.TrimSpace(connectionID) == "" || len(connectionID) > 64 || now.IsZero() { + return governance.SourceScan{}, ErrSourceConnectionNotScannable + } + scanID, err := newOpaqueID("scan-") + if err != nil { + return governance.SourceScan{}, err + } + jobID, err := newOpaqueID("job-") + if err != nil { + return governance.SourceScan{}, err + } + payload, err := json.Marshal(struct { + ScanID string `json:"scan_id"` + }{ScanID: scanID}) + if err != nil { + return governance.SourceScan{}, errors.New("encode source scan job") + } + scan, err := scanSourceScan(s.db.QueryRow(ctx, triggerPPSourceScanSQL, connectionID, scanID, payload, jobID, now)) + if errors.Is(err, pgx.ErrNoRows) { + var sourceType string + var active bool + inspectErr := s.db.QueryRow(ctx, inspectPPSourceScanBlockSQL, connectionID).Scan(&sourceType, &active) + if inspectErr != nil && !errors.Is(inspectErr, pgx.ErrNoRows) { + return governance.SourceScan{}, fmt.Errorf("inspect blocked PP source scan: %w", inspectErr) + } + if inspectErr == nil && sourceType == string(governance.SourceTypePP) && active { + return governance.SourceScan{}, ErrSourceScanAlreadyActive + } + return governance.SourceScan{}, ErrSourceConnectionNotScannable + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return governance.SourceScan{}, ErrSourceScanAlreadyActive + } + if err != nil { + return governance.SourceScan{}, fmt.Errorf("trigger PP source scan: %w", err) + } + return scan, nil +} + +func (s *Store) ListSourceScans(ctx context.Context, connectionID string, limit, offset int) (governance.SourceScanPage, error) { + rows, err := s.db.Query(ctx, ` +SELECT id, state, observed_items, unchanged_items, added_items, + updated_items, over_limit_items, missing_items, planned_uploads, + error_code, created_at, completed_at, COUNT(*) OVER() +FROM source_scans +WHERE source_connection_id = $1 +ORDER BY created_at DESC, id DESC +LIMIT $2 OFFSET $3`, connectionID, limit, offset) + if err != nil { + return governance.SourceScanPage{}, fmt.Errorf("query source scans: %w", err) + } + defer rows.Close() + page := governance.SourceScanPage{Items: []governance.SourceScanSummaryView{}, Limit: limit, Offset: offset} + for rows.Next() { + var item governance.SourceScanSummaryView + var state string + if err := rows.Scan(&item.ID, &state, &item.ObservedItems, &item.UnchangedItems, &item.AddedItems, + &item.UpdatedItems, &item.OverLimitItems, &item.MissingItems, &item.PlannedUploads, + &item.ErrorCode, &item.CreatedAt, &item.CompletedAt, &page.Total); err != nil { + return governance.SourceScanPage{}, fmt.Errorf("scan source scan summary: %w", err) + } + item.State = governance.SourceScanState(state) + if !item.State.Valid() { + return governance.SourceScanPage{}, errors.New("invalid source scan summary state") + } + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.SourceScanPage{}, fmt.Errorf("iterate source scans: %w", err) + } + return page, nil +} + +func (s *Store) GetSourceScanDetail(ctx context.Context, connectionID, scanID string) (governance.SourceScanDetailView, error) { + var detail governance.SourceScanDetailView + var state string + err := s.db.QueryRow(ctx, ` +SELECT id, state, observed_items, unchanged_items, added_items, + updated_items, over_limit_items, missing_items, planned_uploads, + error_code, created_at, completed_at +FROM source_scans +WHERE source_connection_id = $1 AND id = $2`, connectionID, scanID).Scan( + &detail.ID, &state, &detail.ObservedItems, &detail.UnchangedItems, &detail.AddedItems, + &detail.UpdatedItems, &detail.OverLimitItems, &detail.MissingItems, &detail.PlannedUploads, + &detail.ErrorCode, &detail.CreatedAt, &detail.CompletedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return governance.SourceScanDetailView{}, ErrSourceScanNotFound + } + if err != nil { + return governance.SourceScanDetailView{}, fmt.Errorf("query source scan detail: %w", err) + } + detail.State = governance.SourceScanState(state) + if !detail.State.Valid() { + return governance.SourceScanDetailView{}, errors.New("invalid source scan detail state") + } + rows, err := s.db.Query(ctx, ` +SELECT id, source_item_id, change_type, observed_size_bytes, created_at +FROM source_scan_changes +WHERE source_connection_id = $1 AND source_scan_id = $2 +ORDER BY created_at, id`, connectionID, scanID) + if err != nil { + return governance.SourceScanDetailView{}, fmt.Errorf("query source scan changes: %w", err) + } + defer rows.Close() + detail.Changes = []governance.SourceScanChangeView{} + for rows.Next() { + var change governance.SourceScanChangeView + var changeType string + if err := rows.Scan(&change.ID, &change.SourceItemID, &changeType, &change.ObservedSizeBytes, &change.CreatedAt); err != nil { + return governance.SourceScanDetailView{}, fmt.Errorf("scan source scan change: %w", err) + } + change.ChangeType = governance.SourceScanChangeType(changeType) + switch change.ChangeType { + case governance.SourceScanAdded, governance.SourceScanUpdated, governance.SourceScanUnavailable: + default: + return governance.SourceScanDetailView{}, errors.New("invalid source scan change type") + } + detail.Changes = append(detail.Changes, change) + } + if err := rows.Err(); err != nil { + return governance.SourceScanDetailView{}, fmt.Errorf("iterate source scan changes: %w", err) + } + return detail, nil +} + +func (s *Store) StartSourceScan(ctx context.Context, scanID string, now time.Time) (governance.SourceScan, error) { + scan, err := scanSourceScan(s.db.QueryRow(ctx, startSourceScanSQL, scanID, now)) + if errors.Is(err, pgx.ErrNoRows) { + return governance.SourceScan{}, ErrSourceScanStateConflict + } + if err != nil { + return governance.SourceScan{}, fmt.Errorf("start source scan: %w", err) + } + return scan, nil +} + +func (s *Store) LoadPPCurrentItems(ctx context.Context, connectionID string) (map[string]ppconnector.CurrentItem, error) { + rows, err := s.db.Query(ctx, ` +SELECT stable_external_id, revision +FROM source_items +WHERE source_connection_id = $1 +ORDER BY stable_external_id`, connectionID) + if err != nil { + return nil, fmt.Errorf("query current PP items: %w", err) + } + defer rows.Close() + items := make(map[string]ppconnector.CurrentItem) + for rows.Next() { + var sourceID string + var item ppconnector.CurrentItem + if err := rows.Scan(&sourceID, &item.Revision); err != nil { + return nil, fmt.Errorf("scan current PP item: %w", err) + } + items[sourceID] = item + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate current PP items: %w", err) + } + return items, nil +} + +func (s *Store) CompleteSourceScan(ctx context.Context, scanID string, result governance.SourceScanResult, now time.Time) error { + if !validSourceScanResult(result) || strings.TrimSpace(scanID) == "" || now.IsZero() { + return ErrSourceScanStateConflict + } + type encodedChange struct { + ID string `json:"id"` + StableExternalID string `json:"stable_external_id"` + ChangeType governance.SourceScanChangeType `json:"change_type"` + PreviousRevision string `json:"previous_revision"` + ObservedRevision string `json:"observed_revision"` + ObservedSizeBytes *int64 `json:"observed_size_bytes"` + } + changes := make([]encodedChange, 0, len(result.Changes)) + for _, change := range result.Changes { + changes = append(changes, encodedChange{ + ID: deterministicGovernanceID("chg-", scanID, change.StableExternalID), + StableExternalID: change.StableExternalID, ChangeType: change.ChangeType, + PreviousRevision: change.PreviousRevision, ObservedRevision: change.ObservedRevision, + ObservedSizeBytes: change.ObservedSizeBytes, + }) + } + payload, err := json.Marshal(changes) + if err != nil { + return errors.New("encode source scan changes") + } + var completed, inserted, health int + err = s.db.QueryRow(ctx, completeSourceScanSQL, + scanID, result.SnapshotSHA256, result.ObservedItems, result.UnchangedItems, + result.AddedItems, result.UpdatedItems, result.OverLimitItems, + result.MissingItems, result.PlannedUploads, now, payload, + ).Scan(&completed, &inserted, &health) + if err != nil { + return fmt.Errorf("complete source scan: %w", err) + } + if completed != 1 || inserted != len(changes) || health != 1 { + return ErrSourceScanStateConflict + } + return nil +} + +func (s *Store) FailSourceScan(ctx context.Context, scanID, code string, now time.Time) error { + if strings.TrimSpace(scanID) == "" || now.IsZero() || code != "pp_snapshot_invalid" && code != "source_type_unsupported" { + return ErrSourceScanStateConflict + } + var failed, health int + if err := s.db.QueryRow(ctx, failSourceScanSQL, scanID, code, now).Scan(&failed, &health); err != nil { + return fmt.Errorf("fail source scan: %w", err) + } + if failed != 1 || health != 1 { + return ErrSourceScanStateConflict + } + return nil +} + +func validSourceScanResult(result governance.SourceScanResult) bool { + if !governance.ValidSHA256(result.SnapshotSHA256) || result.ObservedItems < 0 || result.UnchangedItems < 0 || + result.AddedItems < 0 || result.UpdatedItems < 0 || result.OverLimitItems < 0 || result.MissingItems < 0 || + result.PlannedUploads < 0 || result.ObservedItems != result.UnchangedItems+result.AddedItems+result.UpdatedItems+result.OverLimitItems || + result.PlannedUploads != result.UpdatedItems || len(result.Changes) != result.AddedItems+result.UpdatedItems+result.MissingItems { + return false + } + seen := make(map[string]struct{}, len(result.Changes)) + for _, change := range result.Changes { + if change.StableExternalID == "" || len(change.StableExternalID) > 512 || len(change.PreviousRevision) > 512 || len(change.ObservedRevision) > 512 { + return false + } + if _, exists := seen[change.StableExternalID]; exists { + return false + } + seen[change.StableExternalID] = struct{}{} + switch change.ChangeType { + case governance.SourceScanAdded: + if change.PreviousRevision != "" || change.ObservedRevision == "" || change.ObservedSizeBytes == nil { + return false + } + case governance.SourceScanUpdated: + if change.PreviousRevision == "" || change.ObservedRevision == "" || change.ObservedSizeBytes == nil { + return false + } + case governance.SourceScanUnavailable: + if change.PreviousRevision == "" || change.ObservedRevision != "" || change.ObservedSizeBytes != nil { + return false + } + default: + return false + } + if change.ObservedSizeBytes != nil && *change.ObservedSizeBytes < 0 { + return false + } + } + return true +} + +func scanSourceScan(row rowScanner) (governance.SourceScan, error) { + var scan governance.SourceScan + var sourceType, state string + if err := row.Scan(&scan.ID, &scan.SourceConnectionID, &sourceType, &state, &scan.CreatedAt); err != nil { + return governance.SourceScan{}, err + } + scan.SourceType = governance.SourceType(sourceType) + scan.State = governance.SourceScanState(state) + if !scan.SourceType.Valid() || !scan.State.Valid() { + return governance.SourceScan{}, errors.New("invalid source scan row") + } + return scan, nil +} + +func newOpaqueID(prefix string) (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", fmt.Errorf("generate opaque ID: %w", err) + } + return prefix + hex.EncodeToString(value[:]), nil +} diff --git a/internal/platform/postgres/source_scans_test.go b/internal/platform/postgres/source_scans_test.go new file mode 100644 index 0000000..ce29f01 --- /dev/null +++ b/internal/platform/postgres/source_scans_test.go @@ -0,0 +1,395 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestTriggerPPSourceScanAtomicallyEnqueuesScanJob(t *testing.T) { + now := time.Date(2026, 9, 6, 4, 0, 0, 0, time.UTC) + db := &recordingDB{row: sourceScanRow{id: "scan-fixed", connectionID: "conn-pp", sourceType: "pp", state: "pending", createdAt: now}} + store := NewStore(db) + + scan, err := store.TriggerPPSourceScan(context.Background(), "conn-pp", now) + if err != nil { + t.Fatalf("TriggerPPSourceScan() error = %v", err) + } + if scan.ID != "scan-fixed" || scan.SourceConnectionID != "conn-pp" || scan.SourceType != governance.SourceTypePP || scan.State != governance.SourceScanPending { + t.Fatalf("scan = %#v", scan) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "insert into source_scans", + "insert into jobs", + "'source.scan.pp'", + "'pending'", + "not exists", + "returning", + } { + if !strings.Contains(sql, required) { + t.Errorf("trigger SQL missing %q", required) + } + } + if strings.Index(sql, "insert into source_scans") > strings.Index(sql, "insert into jobs") { + t.Fatal("scan must be created before its job") + } + if len(db.queryRowArgs) < 4 { + t.Fatalf("query args = %#v", db.queryRowArgs) + } + payload, ok := db.queryRowArgs[2].([]byte) + if !ok { + t.Fatalf("payload arg type = %T", db.queryRowArgs[2]) + } + var decoded map[string]string + if err := json.Unmarshal(payload, &decoded); err != nil || decoded["scan_id"] == "" { + t.Fatalf("payload = %s err=%v", payload, err) + } +} + +func TestTriggerPPSourceScanDistinguishesAlreadyActive(t *testing.T) { + db := &sourceScanSequenceDB{ + recordingDB: &recordingDB{}, + rows: []pgx.Row{sourceScanErrorRow{err: pgx.ErrNoRows}, sourceScanActiveRow{sourceType: "pp", active: true}}, + } + _, err := NewStore(db).TriggerPPSourceScan(context.Background(), "conn-pp", time.Now()) + if !errors.Is(err, ErrSourceScanAlreadyActive) { + t.Fatalf("TriggerPPSourceScan() error = %v, want ErrSourceScanAlreadyActive", err) + } +} + +func TestStartSourceScanTransitionsOrReturnsCompletedForRecovery(t *testing.T) { + now := time.Date(2026, 9, 6, 4, 10, 0, 0, time.UTC) + db := &recordingDB{row: sourceScanRow{id: "scan-1", connectionID: "conn-pp", sourceType: "pp", state: "running", createdAt: now.Add(-time.Minute)}} + scan, err := NewStore(db).StartSourceScan(context.Background(), "scan-1", now) + if err != nil { + t.Fatalf("StartSourceScan() error = %v", err) + } + if scan.State != governance.SourceScanRunning { + t.Fatalf("state = %q", scan.State) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{"update source_scans", "state in ('pending', 'running')", "coalesce(started_at", "state = 'completed'", "union all"} { + if !strings.Contains(sql, required) { + t.Errorf("start SQL missing %q", required) + } + } +} + +func TestStoreLoadsOnlyPPStableIdentityAndRevision(t *testing.T) { + db := &recordingDB{rows: &ppCurrentRows{values: [][2]string{{"source-a", "revision-a"}, {"source-b", "revision-b"}}}} + current, err := NewStore(db).LoadPPCurrentItems(context.Background(), "conn-pp") + if err != nil { + t.Fatalf("LoadPPCurrentItems() error = %v", err) + } + if len(current) != 2 || current["source-a"].Revision != "revision-a" || current["source-b"].Revision != "revision-b" { + t.Fatalf("current = %#v", current) + } + sql := strings.ToLower(db.querySQL) + if !strings.Contains(sql, "select stable_external_id, revision") || strings.Contains(sql, "normalized_path") || strings.Contains(sql, "raw_record_summary") { + t.Fatalf("unexpected PP current query: %s", sql) + } +} + +func TestCompleteSourceScanAtomicallyPersistsChangesAndHealth(t *testing.T) { + now := time.Date(2026, 9, 6, 4, 20, 0, 0, time.UTC) + db := &recordingDB{row: sourceScanCompletionRow{completed: 1, changes: 2, health: 1}} + result := governance.SourceScanResult{ + SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 613, UnchangedItems: 558, + UpdatedItems: 1, OverLimitItems: 54, MissingItems: 1, PlannedUploads: 1, + Changes: []governance.SourceScanChange{ + {StableExternalID: strings.Repeat("a", 64), ChangeType: governance.SourceScanUpdated, PreviousRevision: "old", ObservedRevision: "new", ObservedSizeBytes: scanInt64Pointer(10)}, + {StableExternalID: strings.Repeat("b", 64), ChangeType: governance.SourceScanUnavailable, PreviousRevision: "old"}, + }, + } + if err := NewStore(db).CompleteSourceScan(context.Background(), "scan-1", result, now); err != nil { + t.Fatalf("CompleteSourceScan() error = %v", err) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{ + "update source_scans", "state = 'running'", "state = 'completed'", + "insert into source_scan_changes", "jsonb_to_recordset", "left join source_items", + "update source_connections", "jsonb_build_object", "'healthy'", + } { + if !strings.Contains(sql, required) { + t.Errorf("complete SQL missing %q", required) + } + } + for _, forbidden := range []string{"normalized_path", "raw_record_summary", "base_url", "secret_ref"} { + if strings.Contains(sql, forbidden) { + t.Errorf("complete SQL includes forbidden field %q", forbidden) + } + } +} + +func TestFailSourceScanStoresOnlyFixedCodeAndUnhealthyState(t *testing.T) { + now := time.Date(2026, 9, 6, 4, 30, 0, 0, time.UTC) + db := &recordingDB{row: sourceScanFailureRow{failed: 1, health: 1}} + if err := NewStore(db).FailSourceScan(context.Background(), "scan-1", "pp_snapshot_invalid", now); err != nil { + t.Fatalf("FailSourceScan() error = %v", err) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{"update source_scans", "state = 'running'", "state = 'failed'", "error_code = $2", "update source_connections", "'unhealthy'"} { + if !strings.Contains(sql, required) { + t.Errorf("failure SQL missing %q", required) + } + } + if len(db.queryRowArgs) != 3 || db.queryRowArgs[1] != "pp_snapshot_invalid" { + t.Fatalf("failure args = %#v", db.queryRowArgs) + } +} + +func TestListSourceScansReturnsOnlySafeSummaryColumns(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 0, 0, 0, time.UTC) + db := &recordingDB{rows: &sourceScanSummaryRows{values: []sourceScanSummaryRow{{ + id: "scan-safe", state: "completed", observed: 613, unchanged: 559, + overLimit: 54, createdAt: now.Add(-time.Minute), completedAt: &now, + }}}} + page, err := NewStore(db).ListSourceScans(context.Background(), "conn-pp", 20, 0) + if err != nil { + t.Fatalf("ListSourceScans() error = %v", err) + } + if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "scan-safe" || page.Items[0].OverLimitItems != 54 { + t.Fatalf("page = %#v", page) + } + sql := strings.ToLower(db.querySQL) + for _, required := range []string{"from source_scans", "source_connection_id = $1", "count(*) over()", "limit $2 offset $3"} { + if !strings.Contains(sql, required) { + t.Errorf("list SQL missing %q", required) + } + } + for _, forbidden := range []string{"snapshot_sha256", "stable_external_id", "previous_revision", "observed_revision"} { + if strings.Contains(sql, forbidden) { + t.Errorf("list SQL includes forbidden field %q", forbidden) + } + } +} + +func TestGetSourceScanDetailReturnsSafeChangeProjection(t *testing.T) { + now := time.Date(2026, 9, 6, 6, 10, 0, 0, time.UTC) + db := &sourceScanReadDB{ + recordingDB: &recordingDB{}, + row: sourceScanDetailRow{id: "scan-safe", state: "completed", observed: 2, added: 1, missing: 1, plannedUploads: 1, createdAt: now, completedAt: &now}, + rows: &sourceScanChangeRows{values: []sourceScanChangeRow{ + {id: "chg-1", sourceItemID: scanStringPointer("si-safe"), changeType: "updated", observedSize: scanInt64Pointer(42), createdAt: now}, + {id: "chg-2", changeType: "unavailable", createdAt: now}, + }}, + } + detail, err := NewStore(db).GetSourceScanDetail(context.Background(), "conn-pp", "scan-safe") + if err != nil { + t.Fatalf("GetSourceScanDetail() error = %v", err) + } + if detail.ID != "scan-safe" || len(detail.Changes) != 2 || detail.Changes[0].SourceItemID == nil || *detail.Changes[0].SourceItemID != "si-safe" { + t.Fatalf("detail = %#v", detail) + } + joined := strings.ToLower(db.queryRowSQL + "\n" + db.querySQL) + for _, required := range []string{"source_connection_id = $1", "id = $2", "from source_scan_changes", "source_scan_id = $2"} { + if !strings.Contains(joined, required) { + t.Errorf("detail SQL missing %q", required) + } + } + for _, forbidden := range []string{"snapshot_sha256", "stable_external_id", "previous_revision", "observed_revision"} { + if strings.Contains(joined, forbidden) { + t.Errorf("detail SQL includes forbidden field %q", forbidden) + } + } +} + +func scanInt64Pointer(value int64) *int64 { return &value } +func scanStringPointer(value string) *string { return &value } + +type sourceScanSequenceDB struct { + *recordingDB + rows []pgx.Row + index int +} + +func (d *sourceScanSequenceDB) QueryRow(_ context.Context, _ string, _ ...any) pgx.Row { + row := d.rows[d.index] + d.index++ + return row +} + +type sourceScanErrorRow struct{ err error } + +func (r sourceScanErrorRow) Scan(...any) error { return r.err } + +type sourceScanActiveRow struct { + sourceType string + active bool +} + +func (r sourceScanActiveRow) Scan(dest ...any) error { + *dest[0].(*string) = r.sourceType + *dest[1].(*bool) = r.active + return nil +} + +type sourceScanRow struct { + id string + connectionID string + sourceType string + state string + createdAt time.Time +} + +func (r sourceScanRow) Scan(dest ...any) error { + *dest[0].(*string) = r.id + *dest[1].(*string) = r.connectionID + *dest[2].(*string) = r.sourceType + *dest[3].(*string) = r.state + *dest[4].(*time.Time) = r.createdAt + return nil +} + +type sourceScanCompletionRow struct{ completed, changes, health int } + +func (r sourceScanCompletionRow) Scan(dest ...any) error { + *dest[0].(*int) = r.completed + *dest[1].(*int) = r.changes + *dest[2].(*int) = r.health + return nil +} + +type sourceScanFailureRow struct{ failed, health int } + +func (r sourceScanFailureRow) Scan(dest ...any) error { + *dest[0].(*int) = r.failed + *dest[1].(*int) = r.health + return nil +} + +type ppCurrentRows struct { + values [][2]string + index int +} + +func (r *ppCurrentRows) Close() {} +func (r *ppCurrentRows) Err() error { return nil } +func (r *ppCurrentRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *ppCurrentRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *ppCurrentRows) RawValues() [][]byte { return nil } +func (r *ppCurrentRows) Conn() *pgx.Conn { return nil } +func (r *ppCurrentRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *ppCurrentRows) Scan(dest ...any) error { + if r.index == 0 || r.index > len(r.values) { + return errors.New("scan outside row") + } + *dest[0].(*string) = r.values[r.index-1][0] + *dest[1].(*string) = r.values[r.index-1][1] + return nil +} +func (r *ppCurrentRows) Values() ([]any, error) { return nil, nil } + +type sourceScanSummaryRow struct { + id, state string + observed, unchanged, added, updated, overLimit, missing int + plannedUploads int + errorCode string + createdAt time.Time + completedAt *time.Time +} + +type sourceScanSummaryRows struct { + values []sourceScanSummaryRow + index int +} + +func (r *sourceScanSummaryRows) Close() {} +func (r *sourceScanSummaryRows) Err() error { return nil } +func (r *sourceScanSummaryRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *sourceScanSummaryRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *sourceScanSummaryRows) RawValues() [][]byte { return nil } +func (r *sourceScanSummaryRows) Conn() *pgx.Conn { return nil } +func (r *sourceScanSummaryRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *sourceScanSummaryRows) Scan(dest ...any) error { + v := r.values[r.index-1] + *dest[0].(*string), *dest[1].(*string) = v.id, v.state + *dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = v.observed, v.unchanged, v.added, v.updated + *dest[6].(*int), *dest[7].(*int), *dest[8].(*int) = v.overLimit, v.missing, v.plannedUploads + *dest[9].(*string), *dest[10].(*time.Time), *dest[11].(**time.Time) = v.errorCode, v.createdAt, v.completedAt + *dest[12].(*int) = len(r.values) + return nil +} +func (r *sourceScanSummaryRows) Values() ([]any, error) { return nil, nil } + +type sourceScanDetailRow sourceScanSummaryRow + +func (r sourceScanDetailRow) Scan(dest ...any) error { + v := sourceScanSummaryRow(r) + *dest[0].(*string), *dest[1].(*string) = v.id, v.state + *dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = v.observed, v.unchanged, v.added, v.updated + *dest[6].(*int), *dest[7].(*int), *dest[8].(*int) = v.overLimit, v.missing, v.plannedUploads + *dest[9].(*string), *dest[10].(*time.Time), *dest[11].(**time.Time) = v.errorCode, v.createdAt, v.completedAt + return nil +} + +type sourceScanChangeRow struct { + id string + sourceItemID *string + changeType string + observedSize *int64 + createdAt time.Time +} + +type sourceScanChangeRows struct { + values []sourceScanChangeRow + index int +} + +func (r *sourceScanChangeRows) Close() {} +func (r *sourceScanChangeRows) Err() error { return nil } +func (r *sourceScanChangeRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *sourceScanChangeRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *sourceScanChangeRows) RawValues() [][]byte { return nil } +func (r *sourceScanChangeRows) Conn() *pgx.Conn { return nil } +func (r *sourceScanChangeRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *sourceScanChangeRows) Scan(dest ...any) error { + v := r.values[r.index-1] + *dest[0].(*string), *dest[1].(**string), *dest[2].(*string) = v.id, v.sourceItemID, v.changeType + *dest[3].(**int64), *dest[4].(*time.Time) = v.observedSize, v.createdAt + return nil +} +func (r *sourceScanChangeRows) Values() ([]any, error) { return nil, nil } + +type sourceScanReadDB struct { + *recordingDB + row pgx.Row + rows pgx.Rows +} + +func (d *sourceScanReadDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row { + d.queryRowSQL, d.queryRowArgs = sql, args + return d.row +} + +func (d *sourceScanReadDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) { + d.querySQL, d.queryArgs = sql, args + return d.rows, nil +} diff --git a/internal/platform/postgres/source_settings.go b/internal/platform/postgres/source_settings.go new file mode 100644 index 0000000..325835f --- /dev/null +++ b/internal/platform/postgres/source_settings.go @@ -0,0 +1,310 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// V1 source settings revisions: every edit becomes a validated revision; +// activation is the ONLY way it takes effect and swaps atomically with the +// previous active revision, syncing the runtime snapshot row that the +// scheduler, FKs and existing readers use. next_refresh_at is always derived +// server-side from the active schedule — clients never write it directly. + +var ErrInvalidSourceSettings = errors.New("invalid source settings") + +// ErrSourceConnectionNotFound marks an unknown connection or revision. +var ErrSourceConnectionNotFound = errors.New("source connection not found") + +const sourceSettingsScheduleFloorSeconds = 300 + +type SourceSettingsDraft struct { + SourceType string `json:"source_type"` + Name string `json:"name"` + BaseURL string `json:"base_url"` + SecretRef string `json:"secret_ref"` + Enabled bool `json:"enabled"` + Schedule json.RawMessage `json:"schedule"` + Operator string `json:"operator"` +} + +func (d SourceSettingsDraft) validate() error { + switch d.SourceType { + case "pp", "alist", "dingtalk", "manual": + case "": + return fmt.Errorf("%w: source type is required", ErrInvalidSourceSettings) + default: + return fmt.Errorf("%w: unknown source type", ErrInvalidSourceSettings) + } + if d.Name != strings.TrimSpace(d.Name) || d.Name == "" || len(d.Name) > 255 { + return fmt.Errorf("%w: name must be trimmed and at most 255 characters", ErrInvalidSourceSettings) + } + if len(d.BaseURL) > 2048 || strings.ContainsAny(d.BaseURL, "\x00\r\n\t") { + return fmt.Errorf("%w: invalid base URL", ErrInvalidSourceSettings) + } + if len(d.SecretRef) > 255 || strings.ContainsAny(d.SecretRef, "\x00\r\n\t") { + return fmt.Errorf("%w: invalid secret reference", ErrInvalidSourceSettings) + } + if d.Operator == "" || d.Operator != strings.TrimSpace(d.Operator) || len(d.Operator) > 128 { + return fmt.Errorf("%w: operator is required", ErrInvalidSourceSettings) + } + if _, err := parseScheduleForValidation(d.Schedule); err != nil { + return err + } + return nil +} + +func parseScheduleForValidation(raw json.RawMessage) (governance.SourceSchedule, error) { + if len(raw) == 0 || string(raw) == "" { + return governance.SourceSchedule{}, fmt.Errorf("%w: schedule is required", ErrInvalidSourceSettings) + } + interval, err := governance.ParseSourceSchedule(raw) + if err != nil { + return governance.SourceSchedule{}, fmt.Errorf("%w: %v", ErrInvalidSourceSettings, err) + } + var decoded governance.SourceSchedule + if err := json.Unmarshal(raw, &decoded); err != nil { + return governance.SourceSchedule{}, fmt.Errorf("%w: schedule must be an object", ErrInvalidSourceSettings) + } + _ = interval + return decoded, nil +} + +// CreateSourceSettingsRevision stores a validated draft for an existing +// connection without touching the active snapshot. +func (s *Store) CreateSourceSettingsRevision(ctx context.Context, connectionID string, draft SourceSettingsDraft, now time.Time) (string, error) { + if !validShortID(connectionID) || now.IsZero() { + return "", ErrInvalidSourceSettings + } + if err := draft.validate(); err != nil { + return "", err + } + return insertSourceSettingsRevisionTx(ctx, s.db, connectionID, draft, now) +} + +// ActivateSourceSettingsRevision atomically promotes a validated revision: +// retire the previous active, sync the runtime snapshot (name/base_url/ +// secret_ref/enabled/schedule), and DERIVE next_refresh_at from the schedule. +// Disabling clears the horizon immediately: no new scheduled scans or jobs. +func (s *Store) ActivateSourceSettingsRevision(ctx context.Context, connectionID, revisionID string, now time.Time) error { + if !validShortID(connectionID) || !validShortID(revisionID) || now.IsZero() { + return ErrInvalidSourceSettings + } + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return errors.New("source settings transactions are unavailable") + } + tx, err := beginner.Begin(ctx) + if err != nil { + return fmt.Errorf("begin source settings activation: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + if err := activateSourceSettingsRevisionTx(ctx, tx, connectionID, revisionID, now); err != nil { + return err + } + return tx.Commit(ctx) +} + +func activateSourceSettingsRevisionTx(ctx context.Context, tx pgx.Tx, connectionID, revisionID string, now time.Time) error { + var ( + state, sourceType, name, baseURL, secretRef, revisionAuthor string + enabled bool + schedule []byte + ) + err := tx.QueryRow(ctx, ` +SELECT state, source_type, name, base_url, secret_ref, enabled, schedule::text, created_by +FROM source_connection_revisions +WHERE id = $1 AND connection_id = $2 FOR UPDATE`, revisionID, connectionID). + Scan(&state, &sourceType, &name, &baseURL, &secretRef, &enabled, &schedule, &revisionAuthor) + if errors.Is(err, pgx.ErrNoRows) { + return ErrSourceConnectionNotFound + } + if err != nil { + return fmt.Errorf("load source settings revision: %w", err) + } + if state == "active" { + return nil // replay: already effective + } + if state != "validated" { + return fmt.Errorf("%w: only validated revisions can be activated", ErrInvalidSourceSettings) + } + interval, err := governance.ParseSourceSchedule(schedule) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidSourceSettings, err) + } + // manual sources are import-only: no live refresh exists, so they can never + // carry a schedule horizon (the dispatcher would enqueue jobs that fail). + if sourceType == "manual" && interval > 0 { + return fmt.Errorf("%w: manual sources cannot be scheduled", ErrInvalidSourceSettings) + } + var nextRefresh any + if enabled && interval > 0 { + nextRefresh = now.UTC().Add(interval) + } + // Guard against a second activation racing in while we hold this row: + // retiring the current active happens under the connection row lock. + var lockID string + if err := tx.QueryRow(ctx, `SELECT id FROM source_connections WHERE id = $1 FOR UPDATE`, connectionID).Scan(&lockID); err != nil { + return fmt.Errorf("lock source connection: %w", err) + } + if _, err := tx.Exec(ctx, ` +UPDATE source_connection_revisions +SET state = 'retired', retired_at = $3 +WHERE connection_id = $1 AND state = 'active' AND id <> $2`, connectionID, revisionID, now.UTC()); err != nil { + return fmt.Errorf("retire previous revision: %w", err) + } + if _, err := tx.Exec(ctx, ` +UPDATE source_connection_revisions +SET state = 'active', activated_at = $2 +WHERE id = $1 AND state = 'validated'`, revisionID, now.UTC()); err != nil { + return fmt.Errorf("activate revision: %w", err) + } + tag, err := tx.Exec(ctx, ` +UPDATE source_connections +SET name = $2, base_url = $3, secret_ref = $4, enabled = $5, schedule = $6::jsonb, + next_refresh_at = $7, current_revision_id = $8, updated_by = $9, updated_at = $10 +WHERE id = $1`, connectionID, name, baseURL, secretRef, enabled, string(schedule), nextRefresh, revisionID, + revisionAuthor, now.UTC()) + if err != nil { + return fmt.Errorf("sync source connection snapshot: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrSourceConnectionNotFound + } + return nil +} + +// ListSourceConnectionRevisions returns safe history (never raw secrets — +// secret_ref is a reference, already safe to display to operators). +func (s *Store) ListSourceConnectionRevisions(ctx context.Context, connectionID string) ([]governance.SourceConnectionRevision, error) { + rows, err := s.db.Query(ctx, ` +SELECT id, revision_number, state, name, base_url, secret_ref, enabled, + COALESCE(schedule->>'refresh_interval_seconds','') AS interval, + created_by, created_at, validated_at, activated_at, retired_at +FROM source_connection_revisions +WHERE connection_id = $1 +ORDER BY revision_number DESC`, connectionID) + if err != nil { + return nil, fmt.Errorf("list source connection revisions: %w", err) + } + defer rows.Close() + items := []governance.SourceConnectionRevision{} + for rows.Next() { + var item governance.SourceConnectionRevision + var interval string + var validatedAt, activatedAt, retiredAt *time.Time + if err := rows.Scan(&item.ID, &item.RevisionNumber, &item.State, &item.Name, &item.BaseURL, &item.SecretRef, + &item.Enabled, &interval, &item.CreatedBy, &item.CreatedAt, &validatedAt, &activatedAt, &retiredAt); err != nil { + return nil, fmt.Errorf("scan source connection revision: %w", err) + } + item.RefreshIntervalSeconds = interval + item.ValidatedAt = validatedAt + item.ActivatedAt = activatedAt + item.RetiredAt = retiredAt + items = append(items, item) + } + return items, rows.Err() +} + +// C4 idempotent write paths for source settings (V1). + +// CreateSourceSettingsRevisionWithOperation stores the validated draft once per +// operation; a replay returns the originally created revision id. +func (s *Store) CreateSourceSettingsRevisionWithOperation(ctx context.Context, connectionID string, draft SourceSettingsDraft, operationID string, now time.Time) (string, bool, error) { + fingerprint, err := operationFingerprint(struct { + ConnectionID string `json:"connection_id"` + Draft SourceSettingsDraft `json:"draft"` + }{ConnectionID: connectionID, Draft: draft}) + if err != nil { + return "", false, err + } + resultJSON, replayed, err := s.executeOperation(ctx, "source.settings.create", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) { + revisionID, err := insertSourceSettingsRevisionTx(ctx, tx, connectionID, draft, now) + if err != nil { + return nil, err + } + return json.Marshal(revisionID) + }) + if err != nil { + return "", false, err + } + var revisionID string + if err := json.Unmarshal(resultJSON, &revisionID); err != nil { + return "", false, fmt.Errorf("decode stored source settings revision: %w", err) + } + return revisionID, replayed, nil +} + +// ActivateSourceSettingsRevisionWithOperation activates the validated revision +// exactly once per operation; replays report "replayed" without re-syncing. +func (s *Store) ActivateSourceSettingsRevisionWithOperation(ctx context.Context, connectionID, revisionID, operationID string, now time.Time) (bool, error) { + fingerprint, err := operationFingerprint(struct { + ConnectionID string `json:"connection_id"` + RevisionID string `json:"revision_id"` + }{ConnectionID: connectionID, RevisionID: revisionID}) + if err != nil { + return false, err + } + _, replayed, err := s.executeOperation(ctx, "source.settings.activate", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) { + if err := activateSourceSettingsRevisionTx(ctx, tx, connectionID, revisionID, now); err != nil { + return nil, err + } + return json.RawMessage(`{}`), nil + }) + return replayed, err +} + +type revisionRowQueryer interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +func insertSourceSettingsRevisionTx(ctx context.Context, db revisionRowQueryer, connectionID string, draft SourceSettingsDraft, now time.Time) (string, error) { + if err := draft.validate(); err != nil { + return "", err + } + revisionID, err := newOpaqueID("scr-") + if err != nil { + return "", err + } + err = db.QueryRow(ctx, ` +WITH base AS ( + SELECT source_type, name, base_url, secret_ref FROM source_connections WHERE id = $1 +), +inherited AS ( + SELECT base.source_type, + base.name AS current_name, base.base_url, base.secret_ref + FROM base +), +next_number AS ( + SELECT COALESCE(MAX(revision_number), 0) + 1 AS number + FROM source_connection_revisions WHERE connection_id = $1 +) +INSERT INTO source_connection_revisions ( + id, connection_id, revision_number, state, source_type, name, base_url, + secret_ref, enabled, schedule, created_by, validated_at +) +SELECT $2, $1, next_number.number, 'validated', inherited.source_type, + $3, + CASE WHEN $4 = '' THEN inherited.base_url ELSE $4 END, + CASE WHEN $5 = '' THEN inherited.secret_ref ELSE $5 END, + $6, $7::jsonb, $8, $9 +FROM inherited, next_number +RETURNING id`, connectionID, revisionID, draft.Name, draft.BaseURL, draft.SecretRef, draft.Enabled, string(draft.Schedule), draft.Operator, now.UTC()).Scan(&revisionID) + if errors.Is(err, pgx.ErrNoRows) { + return "", ErrSourceConnectionNotFound + } + if err != nil { + return "", fmt.Errorf("create source settings revision: %w", err) + } + return revisionID, nil +} diff --git a/internal/platform/postgres/store.go b/internal/platform/postgres/store.go new file mode 100644 index 0000000..eff3694 --- /dev/null +++ b/internal/platform/postgres/store.go @@ -0,0 +1,170 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/jobs" +) + +var ErrLeaseLost = errors.New("job lease lost") + +type database interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + Begin(ctx context.Context) (pgx.Tx, error) +} + +type Store struct { + db database +} + +func NewStore(db database) *Store { + return &Store{db: db} +} + +func (s *Store) Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error) { + job, err := scanJob(s.db.QueryRow( + ctx, + enqueueJobSQL, + request.ID, + request.Type, + request.Payload, + request.Priority, + request.AvailableAt, + request.MaxAttempts, + request.IdempotencyKey, + now, + )) + if err != nil { + return nil, fmt.Errorf("enqueue job: %w", err) + } + return job, nil +} + +func (s *Store) Claim(ctx context.Context, workerID string, allowedTypes []string, now time.Time, leaseDuration time.Duration) (*jobs.Job, error) { + // Liveness heartbeat: refresh even on an empty claim so a healthy idle + // worker still looks alive to /readyz and the monitor matrix. + if _, err := s.db.Exec(ctx, workerHeartbeatSQL, workerID, strings.Join(allowedTypes, ","), now); err != nil { + return nil, fmt.Errorf("record worker heartbeat: %w", err) + } + job, err := scanJob(s.db.QueryRow(ctx, claimJobSQL, workerID, now, now.Add(leaseDuration), allowedTypes)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("claim job: %w", err) + } + return job, nil +} + +func (s *Store) List(ctx context.Context, limit int) ([]jobs.Job, error) { + rows, err := s.db.Query(ctx, listJobsSQL, limit) + if err != nil { + return nil, fmt.Errorf("list jobs: %w", err) + } + defer rows.Close() + + result := make([]jobs.Job, 0) + for rows.Next() { + job, err := scanJob(rows) + if err != nil { + return nil, fmt.Errorf("scan listed job: %w", err) + } + result = append(result, *job) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate jobs: %w", err) + } + return result, nil +} + +func (s *Store) Complete(ctx context.Context, id, owner string, now time.Time) error { + tag, err := s.db.Exec(ctx, completeJobSQL, id, owner, now) + if err != nil { + return fmt.Errorf("complete job: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrLeaseLost + } + return nil +} + +func (s *Store) Heartbeat(ctx context.Context, id, owner string, now time.Time, leaseDuration time.Duration) error { + tag, err := s.db.Exec(ctx, heartbeatJobSQL, id, owner, now, now.Add(leaseDuration)) + if err != nil { + return fmt.Errorf("heartbeat job: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrLeaseLost + } + return nil +} + +func (s *Store) Fail(ctx context.Context, id, owner, message, code string, now, retryAt time.Time) error { + tag, err := s.db.Exec(ctx, failJobSQL, id, owner, message, now, code, retryAt) + if err != nil { + return fmt.Errorf("fail job: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrLeaseLost + } + return nil +} + +// FailTerminal implements jobs.TerminalFailer (P6): park the job in 'failed' +// immediately with a stable error code and project the dead-letter row. +func (s *Store) FailTerminal(ctx context.Context, id, owner, message, code string, now time.Time) error { + tag, err := s.db.Exec(ctx, failTerminalJobSQL, id, owner, message, code, now) + if err != nil { + return fmt.Errorf("fail terminal job: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrLeaseLost + } + return nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanJob(row rowScanner) (*jobs.Job, error) { + job := &jobs.Job{} + var state string + var leaseOwner *string + var lastError *string + if err := row.Scan( + &job.ID, + &job.Type, + &job.Payload, + &state, + &job.Priority, + &job.AvailableAt, + &leaseOwner, + &job.LeaseUntil, + &job.Attempt, + &job.MaxAttempts, + &lastError, + &job.IdempotencyKey, + &job.CreatedAt, + &job.UpdatedAt, + ); err != nil { + return nil, err + } + job.State = jobs.State(state) + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + if lastError != nil { + job.LastError = *lastError + } + return job, nil +} diff --git a/internal/platform/postgres/store_test.go b/internal/platform/postgres/store_test.go new file mode 100644 index 0000000..8f935f4 --- /dev/null +++ b/internal/platform/postgres/store_test.go @@ -0,0 +1,295 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/jobs" +) + +func TestStoreCompleteRejectsLostLease(t *testing.T) { + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 0")} + store := NewStore(db) + now := time.Date(2026, time.September, 3, 10, 0, 0, 0, time.UTC) + + err := store.Complete(context.Background(), "job-1", "worker-1", now) + if !errors.Is(err, ErrLeaseLost) { + t.Fatalf("Complete() error = %v, want ErrLeaseLost", err) + } + if db.execSQL != completeJobSQL { + t.Fatal("Complete() did not use completeJobSQL") + } + if len(db.execArgs) != 3 || db.execArgs[0] != "job-1" || db.execArgs[1] != "worker-1" || db.execArgs[2] != now { + t.Fatalf("Complete() args = %#v", db.execArgs) + } +} + +func TestStoreHeartbeatAndFailRejectLostLease(t *testing.T) { + now := time.Date(2026, time.September, 3, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + call func(*Store) error + }{ + { + name: "heartbeat", + call: func(store *Store) error { + return store.Heartbeat(context.Background(), "job-1", "worker-1", now, 30*time.Second) + }, + }, + { + name: "fail", + call: func(store *Store) error { + return store.Fail(context.Background(), "job-1", "worker-1", "temporary", "", now, now.Add(time.Second)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 0")} + err := tt.call(NewStore(db)) + if !errors.Is(err, ErrLeaseLost) { + t.Fatalf("error = %v, want ErrLeaseLost", err) + } + }) + } +} + +func TestStoreClaimMapsLeasedJob(t *testing.T) { + now := time.Date(2026, time.September, 3, 10, 0, 0, 0, time.UTC) + leaseUntil := now.Add(30 * time.Second) + db := &recordingDB{row: fakeJobRow{ + id: "job-1", + jobType: "demo", + payload: []byte(`{"message":"hello"}`), + state: string(jobs.StateRunning), + priority: 7, + availableAt: now, + leaseOwner: "worker-1", + leaseUntil: &leaseUntil, + attempt: 2, + maxAttempts: 5, + idempotencyKey: "demo:1", + createdAt: now.Add(-time.Minute), + updatedAt: now, + }} + store := NewStore(db) + + job, err := store.Claim(context.Background(), "worker-1", []string{"demo"}, now, 30*time.Second) + if err != nil { + t.Fatalf("Claim() error = %v", err) + } + if job.ID != "job-1" || job.State != jobs.StateRunning || job.Attempt != 2 || job.LeaseOwner != "worker-1" { + t.Fatalf("Claim() job = %#v", job) + } + if db.queryRowSQL != claimJobSQL { + t.Fatal("Claim() did not use claimJobSQL") + } + if len(db.queryRowArgs) != 4 || db.queryRowArgs[0] != "worker-1" || db.queryRowArgs[1] != now || db.queryRowArgs[2] != leaseUntil { + t.Fatalf("Claim() args = %#v", db.queryRowArgs) + } +} + +func TestStoreEnqueueUsesIdempotencyKey(t *testing.T) { + now := time.Date(2026, time.September, 3, 11, 0, 0, 0, time.UTC) + db := &recordingDB{row: fakeJobRow{ + id: "job-4", + jobType: "demo", + payload: []byte(`{"message":"hello"}`), + state: string(jobs.StatePending), + availableAt: now, + maxAttempts: 5, + idempotencyKey: "demo:4", + createdAt: now, + updatedAt: now, + }} + store := NewStore(db) + + job, err := store.Enqueue(context.Background(), jobs.EnqueueRequest{ + ID: "job-4", + Type: "demo", + Payload: []byte(`{"message":"hello"}`), + Priority: 3, + AvailableAt: now, + MaxAttempts: 5, + IdempotencyKey: "demo:4", + }, now) + if err != nil { + t.Fatalf("Enqueue() error = %v", err) + } + if job.ID != "job-4" || job.IdempotencyKey != "demo:4" { + t.Fatalf("Enqueue() job = %#v", job) + } + if db.queryRowSQL != enqueueJobSQL { + t.Fatal("Enqueue() did not use enqueueJobSQL") + } + if len(db.queryRowArgs) != 8 || db.queryRowArgs[0] != "job-4" || db.queryRowArgs[6] != "demo:4" || db.queryRowArgs[7] != now { + t.Fatalf("Enqueue() args = %#v", db.queryRowArgs) + } +} + +func TestStoreListReturnsNewestJobs(t *testing.T) { + now := time.Date(2026, time.September, 3, 12, 0, 0, 0, time.UTC) + rows := &fakeJobRows{items: []fakeJobRow{{ + id: "job-5", + jobType: "demo", + payload: []byte(`{}`), + state: string(jobs.StateCompleted), + availableAt: now, + maxAttempts: 5, + idempotencyKey: "demo:5", + createdAt: now, + updatedAt: now, + }}} + db := &recordingDB{rows: rows} + + got, err := NewStore(db).List(context.Background(), 25) + if err != nil { + t.Fatalf("List() error = %v", err) + } + if len(got) != 1 || got[0].ID != "job-5" || got[0].State != jobs.StateCompleted { + t.Fatalf("List() = %#v", got) + } + if db.querySQL != listJobsSQL || len(db.queryArgs) != 1 || db.queryArgs[0] != 25 { + t.Fatalf("List() SQL/args = %q %#v", db.querySQL, db.queryArgs) + } +} + +func TestScanJobAcceptsNullLastError(t *testing.T) { + job, err := scanJob(nullableLastErrorRow{}) + if err != nil { + t.Fatalf("scanJob() error = %v", err) + } + if job.LastError != "" { + t.Fatalf("last error = %q, want empty", job.LastError) + } +} + +type recordingDB struct { + commandTag pgconn.CommandTag + execSQL string + execSQLs []string + execArgs []any + row pgx.Row + rows pgx.Rows + queryRowSQL string + queryRowArgs []any + querySQL string + queryArgs []any +} + +func (d *recordingDB) Begin(context.Context) (pgx.Tx, error) { + return nil, errors.New("transactions are not supported by the recording fake") +} + +func (d *recordingDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row { + d.queryRowSQL = sql + d.queryRowArgs = args + return d.row +} + +func (d *recordingDB) Exec(_ context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + d.execSQL = sql + d.execSQLs = append(d.execSQLs, sql) + d.execArgs = args + return d.commandTag, nil +} + +func (d *recordingDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) { + d.querySQL = sql + d.queryArgs = args + return d.rows, nil +} + +type fakeJobRow struct { + id string + jobType string + payload []byte + state string + priority int + availableAt time.Time + leaseOwner string + leaseUntil *time.Time + attempt int + maxAttempts int + lastError string + idempotencyKey string + createdAt time.Time + updatedAt time.Time +} + +func (r fakeJobRow) Scan(dest ...any) error { + *dest[0].(*string) = r.id + *dest[1].(*string) = r.jobType + *dest[2].(*json.RawMessage) = r.payload + *dest[3].(*string) = r.state + *dest[4].(*int) = r.priority + *dest[5].(*time.Time) = r.availableAt + *dest[6].(**string) = &r.leaseOwner + *dest[7].(**time.Time) = r.leaseUntil + *dest[8].(*int) = r.attempt + *dest[9].(*int) = r.maxAttempts + *dest[10].(**string) = &r.lastError + *dest[11].(*string) = r.idempotencyKey + *dest[12].(*time.Time) = r.createdAt + *dest[13].(*time.Time) = r.updatedAt + return nil +} + +type nullableLastErrorRow struct{} + +func (nullableLastErrorRow) Scan(dest ...any) error { + lastError, ok := dest[10].(**string) + if !ok { + return errors.New("last_error destination is not nullable") + } + copyDest := append([]any(nil), dest...) + placeholder := "" + placeholderPtr := &placeholder + copyDest[10] = &placeholderPtr + if err := (fakeJobRow{ + id: "job-null-error", + jobType: "demo", + payload: []byte(`{}`), + state: "pending", + availableAt: time.Date(2026, time.September, 3, 13, 0, 0, 0, time.UTC), + maxAttempts: 5, + idempotencyKey: "null-error", + createdAt: time.Date(2026, time.September, 3, 13, 0, 0, 0, time.UTC), + updatedAt: time.Date(2026, time.September, 3, 13, 0, 0, 0, time.UTC), + }).Scan(copyDest...); err != nil { + return err + } + *lastError = nil + return nil +} + +type fakeJobRows struct { + items []fakeJobRow + index int + closed bool + err error +} + +func (r *fakeJobRows) Close() { r.closed = true } +func (r *fakeJobRows) Err() error { return r.err } +func (r *fakeJobRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *fakeJobRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *fakeJobRows) Next() bool { + if r.index >= len(r.items) { + r.closed = true + return false + } + r.index++ + return true +} +func (r *fakeJobRows) Scan(dest ...any) error { return r.items[r.index-1].Scan(dest...) } +func (r *fakeJobRows) Values() ([]any, error) { return nil, errors.New("not implemented") } +func (r *fakeJobRows) RawValues() [][]byte { return nil } +func (r *fakeJobRows) Conn() *pgx.Conn { return nil } diff --git a/internal/platform/postgres/test_publications.go b/internal/platform/postgres/test_publications.go new file mode 100644 index 0000000..d91a3b3 --- /dev/null +++ b/internal/platform/postgres/test_publications.go @@ -0,0 +1,806 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ( + ErrInvalidAcceptanceResult = errors.New("invalid acceptance result") + ErrInvalidErrorCode = errors.New("invalid test publication error code") + ErrPublicationNotUploadable = errors.New("test publication is not uploadable") + ErrPublicationNotRetryable = errors.New("test publication is not retryable") + ErrPublicationNotFound = errors.New("test publication not found") + ErrPublicationStateConflict = errors.New("test publication state conflict") +) + +type PublicationForUpload struct { + ID string + SourceItemVersionID string + TestKBID string + KnowledgeID string + IngestState governance.IngestState + ProcessingState governance.ProcessingState + TargetScope governance.Visibility + ContentObjectID string + SHA256 string + SizeBytes int64 + MIME string + ValidatedFormat string + LocalSpoolRef string + Title string +} + +type PublicationForPoll struct { + ID string + TestKBID string + KnowledgeID string + ProcessingState governance.ProcessingState + PollSequence int + LastRemoteStatus string + LastRemoteUpdatedAt *time.Time + ProcessingStartedAt *time.Time + LastObservedAt *time.Time + LastProgressAt *time.Time +} + +type PublicationForValidation struct { + ID string + TestKBID string + KnowledgeID string +} + +type LegacyVerificationCandidate struct { + ID string + TestKBID string + KnowledgeID string +} + +const loadPublicationForUploadSQL = ` +SELECT p.id, p.source_item_version_id, p.test_kb_id, p.knowledge_id, + p.ingest_state, p.processing_state, p.target_scope, + co.id, co.sha256, co.size_bytes, co.mime, co.validated_format, + co.local_spool_ref, si.title +FROM test_publications AS p +JOIN source_item_versions AS siv ON siv.id = p.source_item_version_id +JOIN content_objects AS co ON co.id = siv.content_object_id +JOIN source_items AS si ON si.id = siv.source_item_id +WHERE p.id = $1 + AND p.ingest_state IN ('pending', 'uploading') + AND p.processing_state = 'pending' + AND p.knowledge_id = '' + AND siv.content_state = 'verified' + AND co.content_state = 'verified' + AND co.local_spool_ref <> '' + AND co.size_bytes > 0 +` + +const loadPublicationForPollSQL = ` +SELECT id, test_kb_id, knowledge_id, processing_state, poll_sequence, + last_remote_status, last_remote_updated_at, processing_started_at, + last_observed_at, updated_at +FROM test_publications +WHERE id = $1 +` + +const loadPublicationForValidationSQL = ` +SELECT id, test_kb_id, knowledge_id +FROM test_publications +WHERE id = $1 + AND ingest_state = 'uploaded' + AND processing_state = 'completed' + AND knowledge_id <> '' +` + +const listLegacyVerificationCandidatesSQL = ` +SELECT p.id, p.test_kb_id, p.knowledge_id +FROM test_publications p +JOIN source_item_versions siv ON siv.id = p.source_item_version_id +JOIN source_items si ON si.id = siv.source_item_id +JOIN source_connections sc ON sc.id = si.source_connection_id +WHERE p.knowledge_id <> '' + AND sc.source_type = 'pp' +ORDER BY p.id +` + +const countTestPublicationsSQL = ` +SELECT COUNT(*) +FROM test_publications +WHERE ($1 = '' OR processing_state = $1) AND ($2 = '' OR target_scope = $2) +` + +const listTestPublicationsSQL = ` +SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state, + p.target_scope, COALESCE(p.change_set_id, ''), + CASE WHEN EXISTS ( + SELECT 1 + FROM test_publications other + JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id + WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id + ) THEN 'updated' ELSE 'added' END AS change_type, + (p.ingest_state='pending' AND p.processing_state='pending' + AND p.retrieval_state='unpublished' AND p.knowledge_id='' + AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0 + AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id) + AND NOT EXISTS (SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)) AS can_start, + p.retry_count, p.last_error, + ((p.processing_state IN ('failed', 'stalled') + OR (p.ingest_state = 'failed' AND p.processing_state = 'pending')) + AND co.content_state = 'verified' + AND co.local_spool_ref <> '') AS can_retry, + p.updated_at +FROM test_publications p +JOIN source_item_versions siv ON siv.id = p.source_item_version_id +JOIN source_items si ON si.id = siv.source_item_id +JOIN content_objects co ON co.id = siv.content_object_id +WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) +ORDER BY p.updated_at DESC, p.id +LIMIT $3 OFFSET $4 +` + +const listAcceptanceResultsSQL = ` +SELECT check_type, state, policy_version, evidence_summary, observed_at +FROM acceptance_results +WHERE publication_id = $1 +ORDER BY observed_at DESC +` + +const loadRetryStateSQL = ` +SELECT p.processing_state, p.ingest_state, + EXISTS ( + SELECT 1 FROM source_item_versions siv + JOIN content_objects co ON co.id = siv.content_object_id + WHERE siv.id = p.source_item_version_id + AND co.content_state = 'verified' + AND co.local_spool_ref <> '' + ) +FROM test_publications p +WHERE p.id = $1 +` + +const retryPublicationSQL = ` +WITH updated AS ( + UPDATE test_publications + SET ingest_state = 'pending', + processing_state = 'pending', + knowledge_id = '', + poll_sequence = 0, + last_remote_status = '', + last_remote_updated_at = NULL, + last_observed_at = NULL, + processing_started_at = NULL, + completed_at = NULL, + stalled_at = NULL, + last_error = '', + retry_count = retry_count + 1, + updated_at = $2 + WHERE id = $1 + AND ( + processing_state IN ('failed', 'stalled') + OR (ingest_state = 'failed' AND processing_state = 'pending') + ) + RETURNING id, retry_count +) +INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at +) +SELECT $3, 'test.upload', $4::jsonb, 'pending', 0, $2, 5, + 'test-retry:' || updated.id || ':' || updated.retry_count, $2, $2 +FROM updated +ON CONFLICT (idempotency_key) DO NOTHING +` + +const markUploadingSQL = ` +UPDATE test_publications +SET ingest_state = 'uploading', + last_error = '', + updated_at = $2 +WHERE id = $1 + AND ingest_state IN ('pending', 'uploading') + AND processing_state = 'pending' + AND knowledge_id = '' +` + +const recordUploadAndSchedulePollSQL = ` +WITH updated AS ( + UPDATE test_publications + SET knowledge_id = $2, + ingest_state = 'uploaded', + processing_state = 'pending', + poll_sequence = 1, + last_remote_status = 'pending', + processing_started_at = COALESCE(processing_started_at, $3), + last_observed_at = $3, + last_error = '', + updated_at = $3 + WHERE id = $1 + AND processing_state = 'pending' + AND ( + (ingest_state = 'uploading' AND knowledge_id = '' AND poll_sequence = 0) + OR (ingest_state = 'uploaded' AND knowledge_id = $2 AND poll_sequence = 1) + ) + RETURNING poll_sequence +), enqueued AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT $4, 'test.poll', $5::jsonb, 'pending', $6, $7, $8, $9, $3, $3 + FROM updated + ON CONFLICT (idempotency_key) DO NOTHING +) +SELECT poll_sequence FROM updated +` + +const recordRemoteStatusAndScheduleNextPollSQL = ` +WITH updated AS ( + UPDATE test_publications + SET processing_state = $3::varchar, + last_remote_status = $3::varchar, + last_remote_updated_at = $4, + last_observed_at = $5, + updated_at = CASE + WHEN last_remote_status IS DISTINCT FROM $3::varchar + OR last_remote_updated_at IS DISTINCT FROM $4 + THEN $5 + ELSE updated_at + END, + processing_started_at = COALESCE(processing_started_at, $5), + poll_sequence = $2 + 1 + WHERE id = $1 + AND poll_sequence = $2 + AND ingest_state = 'uploaded' + AND knowledge_id <> '' + AND processing_state IN ('pending', 'processing', 'finalizing') + RETURNING poll_sequence +), enqueued AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT $6, 'test.poll', $7::jsonb, 'pending', $8, $9, $10, $11, $5, $5 + FROM updated + ON CONFLICT (idempotency_key) DO NOTHING +) +SELECT poll_sequence FROM updated +` + +const recordCompletedAndScheduleValidationSQL = ` +WITH updated AS ( + UPDATE test_publications + SET processing_state = 'completed', + last_remote_status = 'completed', + last_remote_updated_at = $3, + last_observed_at = $4, + completed_at = $4, + parser_generation = $11, + index_generation = $12, + last_error = '', + updated_at = $4 + WHERE id = $1 + AND poll_sequence = $2 + AND ingest_state = 'uploaded' + AND knowledge_id <> '' + AND processing_state IN ('pending', 'processing', 'finalizing') + RETURNING poll_sequence +), enqueued AS ( + INSERT INTO jobs ( + id, type, payload, state, priority, available_at, + max_attempts, idempotency_key, created_at, updated_at + ) + SELECT $5, 'test.validate', $6::jsonb, 'pending', $7, $8, $9, $10, $4, $4 + FROM updated + ON CONFLICT (idempotency_key) DO NOTHING +) +SELECT poll_sequence FROM updated +` + +const recordProcessingFailureSQL = ` +UPDATE test_publications +SET processing_state = 'failed', + last_remote_status = 'failed', + last_observed_at = $4, + last_error = $3, + updated_at = $4 +WHERE id = $1 + AND poll_sequence = $2 + AND ingest_state = 'uploaded' + AND knowledge_id <> '' + AND processing_state IN ('pending', 'processing', 'finalizing') +` + +const recordUploadFailureSQL = ` +UPDATE test_publications +SET ingest_state = 'failed', + processing_state = 'failed', + last_error = $2, + updated_at = $3 +WHERE id = $1 + AND ingest_state IN ('pending', 'uploading') + AND processing_state = 'pending' + AND knowledge_id = '' +` + +const markStalledSQL = ` +UPDATE test_publications +SET processing_state = 'stalled', + last_observed_at = $4, + last_error = 'processing_stalled', + updated_at = $4 +WHERE id = $1 + AND poll_sequence = $2 + AND ingest_state = 'uploaded' + AND knowledge_id <> '' + AND processing_state IN ('pending', 'processing', 'finalizing') + AND updated_at <= $3 +` + +const appendAcceptanceResultSQL = ` +INSERT INTO acceptance_results ( + id, publication_id, run_id, check_type, state, + policy_version, evidence_summary, observed_at +) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8) +ON CONFLICT (id) DO NOTHING +` + +func (s *Store) LoadPublicationForUpload(ctx context.Context, publicationID string) (PublicationForUpload, error) { + var candidate PublicationForUpload + var ingestState string + var processingState string + err := s.db.QueryRow(ctx, loadPublicationForUploadSQL, publicationID).Scan( + &candidate.ID, + &candidate.SourceItemVersionID, + &candidate.TestKBID, + &candidate.KnowledgeID, + &ingestState, + &processingState, + &candidate.TargetScope, + &candidate.ContentObjectID, + &candidate.SHA256, + &candidate.SizeBytes, + &candidate.MIME, + &candidate.ValidatedFormat, + &candidate.LocalSpoolRef, + &candidate.Title, + ) + if errors.Is(err, pgx.ErrNoRows) { + return PublicationForUpload{}, ErrPublicationNotUploadable + } + if err != nil { + return PublicationForUpload{}, fmt.Errorf("load test publication for upload: %w", err) + } + candidate.IngestState = governance.IngestState(ingestState) + candidate.ProcessingState = governance.ProcessingState(processingState) + return candidate, nil +} + +func (s *Store) LoadPublicationForPoll(ctx context.Context, publicationID string) (PublicationForPoll, error) { + var publication PublicationForPoll + var processingState string + err := s.db.QueryRow(ctx, loadPublicationForPollSQL, publicationID).Scan( + &publication.ID, + &publication.TestKBID, + &publication.KnowledgeID, + &processingState, + &publication.PollSequence, + &publication.LastRemoteStatus, + &publication.LastRemoteUpdatedAt, + &publication.ProcessingStartedAt, + &publication.LastObservedAt, + &publication.LastProgressAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return PublicationForPoll{}, ErrPublicationNotFound + } + if err != nil { + return PublicationForPoll{}, fmt.Errorf("load test publication for poll: %w", err) + } + publication.ProcessingState = governance.ProcessingState(processingState) + return publication, nil +} + +func (s *Store) LoadPublicationForValidation(ctx context.Context, publicationID string) (PublicationForValidation, error) { + var publication PublicationForValidation + err := s.db.QueryRow(ctx, loadPublicationForValidationSQL, publicationID).Scan( + &publication.ID, + &publication.TestKBID, + &publication.KnowledgeID, + ) + if errors.Is(err, pgx.ErrNoRows) { + return PublicationForValidation{}, ErrPublicationStateConflict + } + if err != nil { + return PublicationForValidation{}, fmt.Errorf("load test publication for validation: %w", err) + } + return publication, nil +} + +func (s *Store) ListLegacyVerificationCandidates(ctx context.Context) ([]LegacyVerificationCandidate, error) { + rows, err := s.db.Query(ctx, listLegacyVerificationCandidatesSQL) + if err != nil { + return nil, fmt.Errorf("list legacy verification candidates: %w", err) + } + defer rows.Close() + var candidates []LegacyVerificationCandidate + for rows.Next() { + var candidate LegacyVerificationCandidate + if err := rows.Scan(&candidate.ID, &candidate.TestKBID, &candidate.KnowledgeID); err != nil { + return nil, fmt.Errorf("scan legacy verification candidate: %w", err) + } + candidates = append(candidates, candidate) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate legacy verification candidates: %w", err) + } + return candidates, nil +} + +func (s *Store) ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error) { + var page governance.TestPublicationPage + if filter.TargetScope != "" && filter.TargetScope != string(governance.VisibilityInternal) && filter.TargetScope != string(governance.VisibilityExternal) { + return governance.TestPublicationPage{}, errors.New("invalid test publication target scope") + } + if err := s.db.QueryRow(ctx, countTestPublicationsSQL, filter.ProcessingState, filter.TargetScope).Scan(&page.Total); err != nil { + return governance.TestPublicationPage{}, fmt.Errorf("count test publications: %w", err) + } + rows, err := s.db.Query(ctx, listTestPublicationsSQL, filter.ProcessingState, filter.TargetScope, filter.Limit, filter.Offset) + if err != nil { + return governance.TestPublicationPage{}, fmt.Errorf("list test publications: %w", err) + } + defer rows.Close() + page.Items = make([]governance.TestPublication, 0) + for rows.Next() { + var item governance.TestPublication + if err := rows.Scan( + &item.ID, + &item.Title, + &item.IngestState, + &item.ProcessingState, + &item.RetrievalState, + &item.TargetScope, + &item.ChangeSetID, + &item.ChangeType, + &item.CanStart, + &item.RetryCount, + &item.LastError, + &item.CanRetry, + &item.UpdatedAt, + ); err != nil { + return governance.TestPublicationPage{}, fmt.Errorf("scan test publication: %w", err) + } + page.Items = append(page.Items, item) + } + if err := rows.Err(); err != nil { + return governance.TestPublicationPage{}, fmt.Errorf("iterate test publications: %w", err) + } + return page, nil +} + +func (s *Store) ListAcceptanceResults(ctx context.Context, publicationID string) ([]governance.AcceptanceRecord, error) { + rows, err := s.db.Query(ctx, listAcceptanceResultsSQL, publicationID) + if err != nil { + return nil, fmt.Errorf("list acceptance results: %w", err) + } + defer rows.Close() + results := make([]governance.AcceptanceRecord, 0) + for rows.Next() { + var record governance.AcceptanceRecord + if err := rows.Scan(&record.CheckType, &record.State, &record.PolicyVersion, &record.EvidenceSummary, &record.ObservedAt); err != nil { + return nil, fmt.Errorf("scan acceptance result: %w", err) + } + results = append(results, record) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate acceptance results: %w", err) + } + return results, nil +} + +func (s *Store) RetryPublicationWithOperation(ctx context.Context, publicationID string, now time.Time, operationID string) (bool, error) { + fingerprint, err := operationFingerprint(struct { + PublicationID string `json:"publication_id"` + }{publicationID}) + if err != nil { + return false, err + } + raw, replayed, err := s.executeOperation(ctx, "test.publication.retry", operationID, fingerprint, now, func(tx pgx.Tx) (json.RawMessage, error) { + if err := NewStore(tx).RetryPublication(ctx, publicationID, now); err != nil { + return nil, err + } + return json.Marshal(map[string]string{"id": publicationID, "status": "retry_queued"}) + }) + if err != nil { + return false, err + } + if !json.Valid(raw) { + return false, errors.New("invalid stored retry result") + } + return replayed, nil +} + +func (s *Store) RetryPublication(ctx context.Context, publicationID string, now time.Time) error { + var processingState, ingestState string + var spoolAvailable bool + err := s.db.QueryRow(ctx, loadRetryStateSQL, publicationID).Scan(&processingState, &ingestState, &spoolAvailable) + if errors.Is(err, pgx.ErrNoRows) { + return ErrPublicationNotFound + } + if err != nil { + return fmt.Errorf("load test publication retry state: %w", err) + } + legacyUploadFailed := ingestState == string(governance.IngestFailed) && processingState == string(governance.ProcessingPending) + if processingState != string(governance.ProcessingFailed) && processingState != string(governance.ProcessingStalled) && !legacyUploadFailed { + return ErrPublicationStateConflict + } + if !spoolAvailable { + return ErrPublicationNotRetryable + } + payload, err := json.Marshal(struct { + PublicationID string `json:"publication_id"` + }{PublicationID: publicationID}) + if err != nil { + return errors.New("encode retry upload job") + } + jobID := orchestrationJobID("test.upload", publicationID, fmt.Sprint(now.UnixNano())) + tag, err := s.db.Exec(ctx, retryPublicationSQL, publicationID, now, jobID, payload) + if err != nil { + return fmt.Errorf("retry test publication: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) MarkUploading(ctx context.Context, publicationID string, now time.Time) error { + tag, err := s.db.Exec(ctx, markUploadingSQL, publicationID, now) + if err != nil { + return fmt.Errorf("mark test publication uploading: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) RecordUploadAndSchedulePoll(ctx context.Context, publicationID, knowledgeID string, now, nextAt time.Time) (int, error) { + payload, err := json.Marshal(struct { + PublicationID string `json:"publication_id"` + Sequence int `json:"sequence"` + }{PublicationID: publicationID, Sequence: 1}) + if err != nil { + return 0, errors.New("encode first test poll job") + } + idempotencyKey := "test-poll:" + publicationID + ":1" + jobID := orchestrationJobID("test.poll", publicationID, "1") + var sequence int + err = s.db.QueryRow( + ctx, + recordUploadAndSchedulePollSQL, + publicationID, + knowledgeID, + now, + jobID, + payload, + 0, + nextAt, + 5, + idempotencyKey, + ).Scan(&sequence) + if errors.Is(err, pgx.ErrNoRows) { + return 0, ErrPublicationStateConflict + } + if err != nil { + return 0, fmt.Errorf("record test upload and schedule poll: %w", err) + } + return sequence, nil +} + +func (s *Store) RecordRemoteStatusAndScheduleNextPoll( + ctx context.Context, + publicationID string, + sequence int, + status governance.ProcessingState, + remoteUpdatedAt *time.Time, + observedAt time.Time, + nextAt time.Time, +) (int, error) { + if sequence < 1 || (status != governance.ProcessingPending && status != governance.ProcessingProcessing && status != governance.ProcessingFinalizing) { + return 0, ErrPublicationStateConflict + } + nextSequence := sequence + 1 + payload, err := json.Marshal(struct { + PublicationID string `json:"publication_id"` + Sequence int `json:"sequence"` + }{PublicationID: publicationID, Sequence: nextSequence}) + if err != nil { + return 0, errors.New("encode next test poll job") + } + idempotencyKey := fmt.Sprintf("test-poll:%s:%d", publicationID, nextSequence) + jobID := orchestrationJobID("test.poll", publicationID, fmt.Sprint(nextSequence)) + var storedSequence int + err = s.db.QueryRow( + ctx, + recordRemoteStatusAndScheduleNextPollSQL, + publicationID, + sequence, + string(status), + remoteUpdatedAt, + observedAt, + jobID, + payload, + 0, + nextAt, + 5, + idempotencyKey, + ).Scan(&storedSequence) + if errors.Is(err, pgx.ErrNoRows) { + return 0, ErrPublicationStateConflict + } + if err != nil { + return 0, fmt.Errorf("record remote status and schedule next test poll: %w", err) + } + return storedSequence, nil +} + +func (s *Store) RecordCompletedAndScheduleValidation( + ctx context.Context, + publicationID string, + sequence int, + remoteUpdatedAt *time.Time, + observedAt time.Time, +) error { + if sequence < 1 { + return ErrPublicationStateConflict + } + // The release gate requires a parse revision for every completed + // publication. WeKnora reports a single terminal update timestamp for the + // parse+index pipeline, so both generations are stamped from it (falling + // back to the observation time) together with the pinned WeKnora contract + // identity. This keeps manual/seeded publications releasable without + // pretending the document was parsed by a different pipeline. + generationAt := observedAt.UTC() + if remoteUpdatedAt != nil && !remoteUpdatedAt.IsZero() { + generationAt = remoteUpdatedAt.UTC() + } + generation := "weknora-v0.7.1-m74@" + generationAt.Format(time.RFC3339) + payload, err := json.Marshal(struct { + PublicationID string `json:"publication_id"` + }{PublicationID: publicationID}) + if err != nil { + return errors.New("encode test validation job") + } + idempotencyKey := fmt.Sprintf("test-validate:%s:%d", publicationID, sequence) + jobID := orchestrationJobID("test.validate", publicationID, fmt.Sprint(sequence)) + var storedSequence int + err = s.db.QueryRow( + ctx, + recordCompletedAndScheduleValidationSQL, + publicationID, + sequence, + remoteUpdatedAt, + observedAt, + jobID, + payload, + 0, + observedAt, + 5, + idempotencyKey, + generation, + generation, + ).Scan(&storedSequence) + if errors.Is(err, pgx.ErrNoRows) { + return ErrPublicationStateConflict + } + if err != nil { + return fmt.Errorf("record test completion and schedule validation: %w", err) + } + return nil +} + +func (s *Store) RecordProcessingFailure(ctx context.Context, publicationID string, sequence int, code string, observedAt time.Time) error { + if sequence < 1 || !validErrorCode(code) { + return ErrInvalidErrorCode + } + tag, err := s.db.Exec(ctx, recordProcessingFailureSQL, publicationID, sequence, code, observedAt) + if err != nil { + return fmt.Errorf("record test processing failure: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) RecordUploadFailure(ctx context.Context, publicationID, code string, observedAt time.Time) error { + if !validErrorCode(code) { + return ErrInvalidErrorCode + } + tag, err := s.db.Exec(ctx, recordUploadFailureSQL, publicationID, code, observedAt) + if err != nil { + return fmt.Errorf("record test upload failure: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) MarkStalled(ctx context.Context, publicationID string, sequence int, cutoff, observedAt time.Time) error { + if sequence < 1 || cutoff.After(observedAt) { + return ErrPublicationStateConflict + } + tag, err := s.db.Exec(ctx, markStalledSQL, publicationID, sequence, cutoff, observedAt) + if err != nil { + return fmt.Errorf("mark test publication stalled: %w", err) + } + if tag.RowsAffected() != 1 { + return ErrPublicationStateConflict + } + return nil +} + +func (s *Store) AppendAcceptanceResult(ctx context.Context, result governance.AcceptanceResult) error { + if !validShortID(result.ID) || !validShortID(result.PublicationID) || !validShortID(result.RunID) || + !validErrorCode(result.CheckType) || !validErrorCode(result.PolicyVersion) || !result.State.Valid() || + result.ObservedAt.IsZero() || len(result.EvidenceSummary) == 0 || len(result.EvidenceSummary) > 4096 || !json.Valid(result.EvidenceSummary) { + return ErrInvalidAcceptanceResult + } + var evidence map[string]any + if err := json.Unmarshal(result.EvidenceSummary, &evidence); err != nil || evidence == nil { + return ErrInvalidAcceptanceResult + } + _, err := s.db.Exec( + ctx, + appendAcceptanceResultSQL, + result.ID, + result.PublicationID, + result.RunID, + result.CheckType, + string(result.State), + result.PolicyVersion, + []byte(result.EvidenceSummary), + result.ObservedAt, + ) + if err != nil { + return fmt.Errorf("append acceptance result: %w", err) + } + return nil +} + +func validShortID(value string) bool { + return len(value) > 0 && len(value) <= 36 && !strings.ContainsAny(value, " /\\ \r\n") +} + +func validErrorCode(code string) bool { + if len(code) == 0 || len(code) > 64 { + return false + } + for _, character := range code { + if (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || strings.ContainsRune("_.:-", character) { + continue + } + return false + } + return true +} + +func orchestrationJobID(parts ...string) string { + digest := sha256.New() + for _, part := range parts { + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(part)) + } + return "job-" + hex.EncodeToString(digest.Sum(nil))[:32] +} diff --git a/internal/platform/postgres/test_publications_integration_test.go b/internal/platform/postgres/test_publications_integration_test.go new file mode 100644 index 0000000..211bc09 --- /dev/null +++ b/internal/platform/postgres/test_publications_integration_test.go @@ -0,0 +1,253 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestPublicationOrchestrationIsAtomicAndRejectsStaleTransitions(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + + sha := strings.Repeat("a", 64) + if _, err = pool.Exec(ctx, `INSERT INTO source_connections (id, source_type, name) VALUES ('connection-1', 'pp', 'test')`); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO source_items ( + id, source_connection_id, stable_external_id, canonical_uri_hash, + title, item_type, revision, visibility, visibility_origin, source_change + ) VALUES ( + 'item-1', 'connection-1', 'external-1', $1, + 'Safe title', 'document', $2, 'external', 'pp_public_tag', 'added' + )`, sha, sha); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO content_objects ( + id, sha256, size_bytes, mime, validated_format, local_spool_ref, content_state + ) VALUES ('content-1', $1, 24, 'text/markdown', 'md', 'objects/artifact.md', 'verified')`, sha); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO source_item_versions ( + id, source_item_id, revision, content_object_id, content_state + ) VALUES ('version-1', 'item-1', $1, 'content-1', 'verified')`, sha); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, `INSERT INTO test_publications ( + id, source_item_version_id, test_kb_id, ingest_state, processing_state, retrieval_state + ) VALUES ('publication-1', 'version-1', 'kb-1', 'pending', 'pending', 'unpublished')`) + if err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + now := time.Date(2026, time.September, 4, 2, 20, 0, 0, time.UTC) + if _, err := store.LoadPublicationForUpload(ctx, "publication-1"); err != nil { + t.Fatal(err) + } + if err := store.MarkUploading(ctx, "publication-1", now); err != nil { + t.Fatal(err) + } + if sequence, err := store.RecordUploadAndSchedulePoll(ctx, "publication-1", "knowledge-1", now, now.Add(10*time.Second)); err != nil || sequence != 1 { + t.Fatalf("record upload = %d, %v", sequence, err) + } + if sequence, err := store.RecordUploadAndSchedulePoll(ctx, "publication-1", "knowledge-1", now, now.Add(10*time.Second)); err != nil || sequence != 1 { + t.Fatalf("replay upload = %d, %v", sequence, err) + } + if _, err := store.RecordUploadAndSchedulePoll(ctx, "publication-1", "knowledge-conflict", now, now.Add(10*time.Second)); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("conflicting knowledge error = %v", err) + } + + remoteUpdatedAt := now.Add(5 * time.Second) + if sequence, err := store.RecordRemoteStatusAndScheduleNextPoll(ctx, "publication-1", 1, governance.ProcessingProcessing, &remoteUpdatedAt, now.Add(10*time.Second), now.Add(40*time.Second)); err != nil || sequence != 2 { + t.Fatalf("record processing = %d, %v", sequence, err) + } + if _, err := store.RecordRemoteStatusAndScheduleNextPoll(ctx, "publication-1", 1, governance.ProcessingProcessing, &remoteUpdatedAt, now.Add(11*time.Second), now.Add(41*time.Second)); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("stale poll error = %v", err) + } + if err := store.RecordCompletedAndScheduleValidation(ctx, "publication-1", 2, &remoteUpdatedAt, now.Add(20*time.Second)); err != nil { + t.Fatal(err) + } + if _, err := store.RecordRemoteStatusAndScheduleNextPoll(ctx, "publication-1", 2, governance.ProcessingProcessing, &remoteUpdatedAt, now.Add(21*time.Second), now.Add(51*time.Second)); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("completed regression error = %v", err) + } + + var ingestState, processingState, knowledgeID string + var sequence, jobCount int + if err := pool.QueryRow(ctx, `SELECT ingest_state, processing_state, knowledge_id, poll_sequence FROM test_publications WHERE id = 'publication-1'`).Scan(&ingestState, &processingState, &knowledgeID, &sequence); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil { + t.Fatal(err) + } + if ingestState != "uploaded" || processingState != "completed" || knowledgeID != "knowledge-1" || sequence != 2 || jobCount != 3 { + t.Fatalf("final state = %s/%s/%s sequence=%d jobs=%d", ingestState, processingState, knowledgeID, sequence, jobCount) + } + if err := store.AppendAcceptanceResult(ctx, governance.AcceptanceResult{ + ID: "acceptance-1", PublicationID: "publication-1", RunID: "run-1", + CheckType: "chunks_present", State: governance.AcceptancePassed, + PolicyVersion: "structural-v1", EvidenceSummary: json.RawMessage(`{"chunk_total":2}`), ObservedAt: now, + }); err != nil { + t.Fatalf("append acceptance result: %v", err) + } + var acceptanceCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM acceptance_results WHERE publication_id = 'publication-1'`).Scan(&acceptanceCount); err != nil { + t.Fatal(err) + } + if acceptanceCount != 1 { + t.Fatalf("acceptance count = %d", acceptanceCount) + } + + validation, err := store.LoadPublicationForValidation(ctx, "publication-1") + if err != nil { + t.Fatal(err) + } + if validation.ID != "publication-1" || validation.TestKBID != "kb-1" || validation.KnowledgeID != "knowledge-1" { + t.Fatalf("validation context = %#v", validation) + } + if err := store.AppendAcceptanceResult(ctx, governance.AcceptanceResult{ + ID: "acceptance-1", PublicationID: "publication-1", RunID: "run-1", + CheckType: "chunks_present", State: governance.AcceptanceFailed, + PolicyVersion: "structural-v1", EvidenceSummary: json.RawMessage(`{"chunk_total":0}`), ObservedAt: now, + }); err != nil { + t.Fatalf("idempotent append: %v", err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM acceptance_results WHERE publication_id = 'publication-1'`).Scan(&acceptanceCount); err != nil { + t.Fatal(err) + } + if acceptanceCount != 1 { + t.Fatalf("acceptance count after idempotent append = %d", acceptanceCount) + } + if _, err := store.LoadPublicationForValidation(ctx, "missing"); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("non-completed validation error = %v", err) + } + legacy, listErr := store.ListLegacyVerificationCandidates(ctx) + if listErr != nil { + t.Fatal(listErr) + } + if len(legacy) != 1 || legacy[0].ID != "publication-1" || legacy[0].KnowledgeID != "knowledge-1" || legacy[0].TestKBID != "kb-1" { + t.Fatalf("legacy candidates = %#v", legacy) + } + + page, listErr := store.ListTestPublications(ctx, governance.TestPublicationFilter{Limit: 10}) + if listErr != nil { + t.Fatal(listErr) + } + if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "publication-1" || page.Items[0].Title == "" || page.Items[0].CanRetry { + t.Fatalf("test publication page = %#v", page) + } + + records, accErr := store.ListAcceptanceResults(ctx, "publication-1") + if accErr != nil { + t.Fatal(accErr) + } + if len(records) != 1 || records[0].CheckType != "chunks_present" { + t.Fatalf("acceptance records = %#v", records) + } + var evidence map[string]any + if err := json.Unmarshal(records[0].EvidenceSummary, &evidence); err != nil || evidence["chunk_total"] != float64(2) { + t.Fatalf("evidence = %s, %v", records[0].EvidenceSummary, err) + } + + if err := store.RetryPublication(ctx, "publication-1", now); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("completed retry error = %v", err) + } + if _, err := pool.Exec(ctx, `UPDATE test_publications SET processing_state = 'failed', last_error = 'remote_parse_failed' WHERE id = 'publication-1'`); err != nil { + t.Fatal(err) + } + if err := store.RetryPublication(ctx, "publication-1", now); err != nil { + t.Fatalf("retry failed publication: %v", err) + } + var resetProcessing, resetIngest string + if err := pool.QueryRow(ctx, `SELECT processing_state, ingest_state FROM test_publications WHERE id = 'publication-1'`).Scan(&resetProcessing, &resetIngest); err != nil { + t.Fatal(err) + } + if resetProcessing != "pending" || resetIngest != "pending" { + t.Fatalf("retry reset = %s/%s", resetProcessing, resetIngest) + } + var uploadJobs int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type = 'test.upload'`).Scan(&uploadJobs); err != nil { + t.Fatal(err) + } + if uploadJobs != 1 { + t.Fatalf("upload jobs after retry = %d", uploadJobs) + } +} + +func TestSeedManualPublicationIsAtomicAndIdempotent(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("ApplyMigrations() error = %v", err) + } + store := NewStore(pool) + request := ManualPublicationSeedRequest{ + TestKBID: "kb-manual-e2e", + Title: "Manual E2E fixture", + SHA256: strings.Repeat("b", 64), + SizeBytes: 128, + MIME: "text/markdown", + ValidatedFormat: "markdown", + LocalSpoolRef: "manual/fixture.md", + Now: time.Date(2026, 9, 5, 5, 30, 0, 0, time.UTC), + } + first, err := store.SeedManualPublication(ctx, request) + if err != nil { + t.Fatalf("first SeedManualPublication() error = %v", err) + } + second, err := store.SeedManualPublication(ctx, request) + if err != nil { + t.Fatalf("second SeedManualPublication() error = %v", err) + } + if first != second { + t.Fatalf("seed results differ: first=%#v second=%#v", first, second) + } + var sourceType string + var connectionCount, itemCount, contentCount, versionCount, publicationCount, jobCount int + if err := pool.QueryRow(ctx, ` + SELECT + (SELECT source_type FROM source_connections WHERE id = 'conn-manual-local'), + (SELECT COUNT(*) FROM source_connections WHERE id = 'conn-manual-local'), + (SELECT COUNT(*) FROM source_items WHERE source_connection_id = 'conn-manual-local'), + (SELECT COUNT(*) FROM content_objects WHERE sha256 = $1), + (SELECT COUNT(*) FROM source_item_versions WHERE revision = $2), + (SELECT COUNT(*) FROM test_publications WHERE id = $3), + (SELECT COUNT(*) FROM jobs WHERE idempotency_key = $4)`, + request.SHA256, + "sha256:"+request.SHA256, + first.PublicationID, + "test-upload:"+first.PublicationID, + ).Scan(&sourceType, &connectionCount, &itemCount, &contentCount, &versionCount, &publicationCount, &jobCount); err != nil { + t.Fatal(err) + } + if sourceType != "manual" || connectionCount != 1 || itemCount != 1 || contentCount != 1 || versionCount != 1 || publicationCount != 1 || jobCount != 1 { + t.Fatalf("seed counts/type = %q %d/%d/%d/%d/%d/%d", sourceType, connectionCount, itemCount, contentCount, versionCount, publicationCount, jobCount) + } +} diff --git a/internal/platform/postgres/test_publications_test.go b/internal/platform/postgres/test_publications_test.go new file mode 100644 index 0000000..16782ac --- /dev/null +++ b/internal/platform/postgres/test_publications_test.go @@ -0,0 +1,601 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "knowledge-governance/internal/governance" +) + +func TestLoadPublicationForUploadMapsVerifiedCandidate(t *testing.T) { + db := &recordingDB{row: publicationUploadRow{}} + candidate, err := NewStore(db).LoadPublicationForUpload(context.Background(), "publication-1") + if err != nil { + t.Fatalf("LoadPublicationForUpload() error = %v", err) + } + if candidate.ID != "publication-1" || candidate.SourceItemVersionID != "version-1" || candidate.ContentObjectID != "content-1" { + t.Fatalf("candidate identities = %#v", candidate) + } + if candidate.TestKBID != "kb-1" || candidate.IngestState != governance.IngestPending || candidate.ProcessingState != governance.ProcessingPending { + t.Fatalf("candidate states = %#v", candidate) + } + if candidate.TargetScope != "internal" { + t.Fatalf("candidate target scope = %q", candidate.TargetScope) + } + if candidate.SHA256 != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || candidate.SizeBytes != 24 || candidate.ValidatedFormat != "md" || candidate.LocalSpoolRef != "objects/artifact.md" { + t.Fatalf("candidate content = %#v", candidate) + } + if db.queryRowSQL != loadPublicationForUploadSQL || len(db.queryRowArgs) != 1 || db.queryRowArgs[0] != "publication-1" { + t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) + } +} + +func TestMarkUploadingRejectsStalePublicationState(t *testing.T) { + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 0")} + err := NewStore(db).MarkUploading(context.Background(), "publication-1", time.Date(2026, time.September, 4, 2, 0, 0, 0, time.UTC)) + if !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("MarkUploading() error = %v, want ErrPublicationStateConflict", err) + } + if db.execSQL != markUploadingSQL { + t.Fatalf("MarkUploading() SQL = %q", db.execSQL) + } +} + +func TestRecordUploadAndSchedulePollIsAtomicAndIdempotent(t *testing.T) { + db := &recordingDB{row: sequenceRow{sequence: 1}} + now := time.Date(2026, time.September, 4, 2, 5, 0, 0, time.UTC) + nextAt := now.Add(10 * time.Second) + sequence, err := NewStore(db).RecordUploadAndSchedulePoll(context.Background(), "publication-1", "knowledge-1", now, nextAt) + if err != nil { + t.Fatalf("RecordUploadAndSchedulePoll() error = %v", err) + } + if sequence != 1 { + t.Fatalf("sequence = %d, want 1", sequence) + } + updateAt := strings.Index(recordUploadAndSchedulePollSQL, "UPDATE test_publications") + insertAt := strings.Index(recordUploadAndSchedulePollSQL, "INSERT INTO jobs") + if updateAt < 0 || insertAt < 0 || updateAt >= insertAt { + t.Fatalf("atomic SQL missing ordered update/insert: %q", recordUploadAndSchedulePollSQL) + } + if db.queryRowSQL != recordUploadAndSchedulePollSQL || len(db.queryRowArgs) != 9 { + t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) + } + if db.queryRowArgs[8] != "test-poll:publication-1:1" { + t.Fatalf("idempotency key = %#v", db.queryRowArgs[8]) + } + var payload map[string]any + if err := json.Unmarshal(db.queryRowArgs[4].([]byte), &payload); err != nil { + t.Fatal(err) + } + if len(payload) != 2 || payload["publication_id"] != "publication-1" || payload["sequence"] != float64(1) { + t.Fatalf("payload = %#v", payload) + } +} + +func TestRecordRemoteStatusSchedulesNextSequenceAtomically(t *testing.T) { + db := &recordingDB{row: sequenceRow{sequence: 2}} + observedAt := time.Date(2026, time.September, 4, 2, 6, 0, 0, time.UTC) + remoteUpdatedAt := observedAt.Add(-time.Second) + nextAt := observedAt.Add(30 * time.Second) + sequence, err := NewStore(db).RecordRemoteStatusAndScheduleNextPoll( + context.Background(), + "publication-1", + 1, + governance.ProcessingProcessing, + &remoteUpdatedAt, + observedAt, + nextAt, + ) + if err != nil { + t.Fatalf("RecordRemoteStatusAndScheduleNextPoll() error = %v", err) + } + if sequence != 2 { + t.Fatalf("sequence = %d, want 2", sequence) + } + if !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "poll_sequence = $2") || !strings.Contains(recordRemoteStatusAndScheduleNextPollSQL, "INSERT INTO jobs") { + t.Fatalf("SQL lacks sequence guard or atomic enqueue: %q", recordRemoteStatusAndScheduleNextPollSQL) + } + if db.queryRowSQL != recordRemoteStatusAndScheduleNextPollSQL || len(db.queryRowArgs) != 11 { + t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) + } + if db.queryRowArgs[10] != "test-poll:publication-1:2" { + t.Fatalf("idempotency key = %#v", db.queryRowArgs[10]) + } +} + +func TestRecordCompletedSchedulesValidationAtomically(t *testing.T) { + db := &recordingDB{row: sequenceRow{sequence: 3}} + observedAt := time.Date(2026, time.September, 4, 2, 7, 0, 0, time.UTC) + remoteUpdatedAt := observedAt.Add(-time.Second) + err := NewStore(db).RecordCompletedAndScheduleValidation( + context.Background(), "publication-1", 3, &remoteUpdatedAt, observedAt, + ) + if err != nil { + t.Fatalf("RecordCompletedAndScheduleValidation() error = %v", err) + } + if !strings.Contains(recordCompletedAndScheduleValidationSQL, "processing_state = 'completed'") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "'test.validate'") { + t.Fatalf("SQL lacks completed update or validation enqueue: %q", recordCompletedAndScheduleValidationSQL) + } + if !strings.Contains(recordCompletedAndScheduleValidationSQL, "parser_generation = $11") || !strings.Contains(recordCompletedAndScheduleValidationSQL, "index_generation = $12") { + t.Fatalf("SQL lacks release generation stamping: %q", recordCompletedAndScheduleValidationSQL) + } + if db.queryRowSQL != recordCompletedAndScheduleValidationSQL || len(db.queryRowArgs) != 12 { + t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) + } + if db.queryRowArgs[9] != "test-validate:publication-1:3" { + t.Fatalf("idempotency key = %#v", db.queryRowArgs[9]) + } + if db.queryRowArgs[10] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" || db.queryRowArgs[11] != "weknora-v0.7.1-m74@2026-09-04T02:06:59Z" { + t.Fatalf("generation args = %#v", db.queryRowArgs[10:]) + } +} + +func TestLoadPublicationForPollMapsOnlyProcessingProjection(t *testing.T) { + now := time.Date(2026, time.September, 4, 2, 8, 0, 0, time.UTC) + db := &recordingDB{row: publicationPollRow{now: now}} + publication, err := NewStore(db).LoadPublicationForPoll(context.Background(), "publication-1") + if err != nil { + t.Fatalf("LoadPublicationForPoll() error = %v", err) + } + if publication.ID != "publication-1" || publication.TestKBID != "kb-1" || publication.KnowledgeID != "knowledge-1" { + t.Fatalf("publication identities = %#v", publication) + } + if publication.ProcessingState != governance.ProcessingProcessing || publication.PollSequence != 2 || publication.LastRemoteStatus != "processing" { + t.Fatalf("publication state = %#v", publication) + } + if publication.LastProgressAt == nil || !publication.LastProgressAt.Equal(now) { + t.Fatalf("last progress = %#v", publication.LastProgressAt) + } + if db.queryRowSQL != loadPublicationForPollSQL { + t.Fatalf("query = %q", db.queryRowSQL) + } +} + +func TestRecordProcessingFailureStoresOnlyFixedErrorCode(t *testing.T) { + now := time.Date(2026, time.September, 4, 2, 9, 0, 0, time.UTC) + for _, code := range []string{"remote detail", "https://example.com", strings.Repeat("x", 65)} { + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(db).RecordProcessingFailure(context.Background(), "publication-1", 2, code, now); !errors.Is(err, ErrInvalidErrorCode) { + t.Fatalf("RecordProcessingFailure(%q) error = %v", code, err) + } + if db.execSQL != "" { + t.Fatalf("invalid code %q reached database", code) + } + } + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(db).RecordProcessingFailure(context.Background(), "publication-1", 2, "remote_parse_failed", now); err != nil { + t.Fatalf("RecordProcessingFailure() error = %v", err) + } + if db.execSQL != recordProcessingFailureSQL || db.execArgs[2] != "remote_parse_failed" { + t.Fatalf("failure update = %q %#v", db.execSQL, db.execArgs) + } +} + +func TestRecordUploadFailureStoresOnlyFixedErrorCode(t *testing.T) { + now := time.Date(2026, time.September, 4, 2, 9, 30, 0, time.UTC) + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(db).RecordUploadFailure(context.Background(), "publication-1", "upload_identity_ambiguous", now); err != nil { + t.Fatalf("RecordUploadFailure() error = %v", err) + } + if db.execSQL != recordUploadFailureSQL || db.execArgs[1] != "upload_identity_ambiguous" { + t.Fatalf("recorded upload failure args = %#v", db.execArgs) + } + if !strings.Contains(recordUploadFailureSQL, "processing_state = 'failed'") { + t.Fatal("upload terminal failure must enter retryable processing_state=failed") + } + invalidDB := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(invalidDB).RecordUploadFailure(context.Background(), "publication-1", "remote body: token=secret", now); !errors.Is(err, ErrInvalidErrorCode) { + t.Fatalf("invalid error code result = %v", err) + } + if invalidDB.execSQL != "" { + t.Fatal("invalid error code reached database") + } +} + +func TestLoadPublicationForValidationMapsCompletedContext(t *testing.T) { + db := &recordingDB{row: publicationValidationRow{}} + context, err := NewStore(db).LoadPublicationForValidation(context.Background(), "publication-1") + if err != nil { + t.Fatalf("LoadPublicationForValidation() error = %v", err) + } + if context.ID != "publication-1" || context.TestKBID != "kb-1" || context.KnowledgeID != "knowledge-1" { + t.Fatalf("validation context = %#v", context) + } + if db.queryRowSQL != loadPublicationForValidationSQL || len(db.queryRowArgs) != 1 || db.queryRowArgs[0] != "publication-1" { + t.Fatalf("query = %q %#v", db.queryRowSQL, db.queryRowArgs) + } +} + +func TestListLegacyVerificationCandidatesMapsKnowledgeIDs(t *testing.T) { + rows := &legacyVerificationRows{items: []legacyVerificationRow{ + {id: "pub-1", kbID: "kb-1", knowledgeID: "knowledge-1"}, + {id: "pub-2", kbID: "kb-2", knowledgeID: "knowledge-2"}, + }} + db := &recordingDB{rows: rows} + candidates, err := NewStore(db).ListLegacyVerificationCandidates(context.Background()) + if err != nil { + t.Fatalf("ListLegacyVerificationCandidates() error = %v", err) + } + if len(candidates) != 2 || candidates[0].ID != "pub-1" || candidates[0].KnowledgeID != "knowledge-1" || candidates[1].TestKBID != "kb-2" { + t.Fatalf("candidates = %#v", candidates) + } + if db.querySQL != listLegacyVerificationCandidatesSQL { + t.Fatalf("query = %q", db.querySQL) + } + if !strings.Contains(listLegacyVerificationCandidatesSQL, "source_type = 'pp'") { + t.Fatal("legacy verification candidates must exclude manual and future source types") + } +} + +func TestListTestPublicationsReturnsSafeProjection(t *testing.T) { + now := time.Date(2026, time.September, 4, 5, 0, 0, 0, time.UTC) + rows := &testPublicationRows{items: []testPublicationRow{ + {id: "pub-1", title: "Safe title", ingest: "uploaded", processing: string(governance.ProcessingFailed), + retrieval: "unpublished", targetScope: "external", changeSetID: "cs-1", changeType: "updated", canStart: false, retryCount: 2, lastError: "remote_parse_failed", canRetry: true, updated: now}, + }} + db := &recordingDB{rows: rows, row: totalRow{total: 1}} + filter := governance.TestPublicationFilter{ProcessingState: string(governance.ProcessingFailed), Limit: 25, Offset: 0} + page, err := NewStore(db).ListTestPublications(context.Background(), filter) + if err != nil { + t.Fatalf("ListTestPublications() error = %v", err) + } + if page.Total != 1 || len(page.Items) != 1 { + t.Fatalf("page = %#v", page) + } + item := page.Items[0] + if item.ID != "pub-1" || item.Title != "Safe title" || item.TargetScope != "external" || item.ChangeSetID != "cs-1" || item.ChangeType != "updated" || item.CanStart || item.ProcessingState != string(governance.ProcessingFailed) || item.RetryCount != 2 || item.LastError != "remote_parse_failed" || !item.CanRetry { + t.Fatalf("item = %#v", item) + } + if db.queryRowSQL != countTestPublicationsSQL || db.querySQL != listTestPublicationsSQL { + t.Fatalf("count = %q list = %q", db.queryRowSQL, db.querySQL) + } +} + +func TestListAcceptanceResultsMapsSafeProjection(t *testing.T) { + now := time.Date(2026, time.September, 4, 5, 5, 0, 0, time.UTC) + rows := &acceptanceRecordRows{items: []acceptanceRecordRow{ + {checkType: "chunks_present", state: "passed", policy: "structural-v1", evidence: json.RawMessage(`{"chunk_total":2}`), observed: now}, + }} + db := &recordingDB{rows: rows} + results, err := NewStore(db).ListAcceptanceResults(context.Background(), "publication-1") + if err != nil { + t.Fatalf("ListAcceptanceResults() error = %v", err) + } + if len(results) != 1 || results[0].CheckType != "chunks_present" || results[0].State != "passed" || string(results[0].EvidenceSummary) != `{"chunk_total":2}` { + t.Fatalf("results = %#v", results) + } + if db.querySQL != listAcceptanceResultsSQL || len(db.queryArgs) != 1 || db.queryArgs[0] != "publication-1" { + t.Fatalf("query = %q %#v", db.querySQL, db.queryArgs) + } +} + +func TestRetryPublicationResetsAndEnqueuesUploadJob(t *testing.T) { + now := time.Date(2026, time.September, 4, 5, 10, 0, 0, time.UTC) + db := &recordingDB{ + row: retryStateRow{processing: "failed", ingest: "uploaded", spoolAvailable: true}, + commandTag: pgconn.NewCommandTag("INSERT 0 1"), + } + if err := NewStore(db).RetryPublication(context.Background(), "publication-1", now); err != nil { + t.Fatalf("RetryPublication() error = %v", err) + } + if db.queryRowSQL != loadRetryStateSQL || db.execSQL != retryPublicationSQL { + t.Fatalf("query = %q exec = %q", db.queryRowSQL, db.execSQL) + } + if len(db.execArgs) != 4 || db.execArgs[0] != "publication-1" { + t.Fatalf("exec args = %#v", db.execArgs) + } +} + +func TestRetryPublicationRecoversLegacyUploadFailedState(t *testing.T) { + now := time.Date(2026, time.September, 5, 5, 30, 0, 0, time.UTC) + db := &recordingDB{ + row: retryStateRow{processing: "pending", ingest: "failed", spoolAvailable: true}, + commandTag: pgconn.NewCommandTag("INSERT 0 1"), + } + if err := NewStore(db).RetryPublication(context.Background(), "publication-1", now); err != nil { + t.Fatalf("RetryPublication() legacy upload failure error = %v", err) + } + if !strings.Contains(retryPublicationSQL, "ingest_state = 'failed'") { + t.Fatal("retry SQL must recover legacy upload-failed state") + } +} + +func TestRetryPublicationRejectsCompleted(t *testing.T) { + db := &recordingDB{row: retryStateRow{processing: "completed", ingest: "uploaded", spoolAvailable: true}} + err := NewStore(db).RetryPublication(context.Background(), "publication-1", time.Date(2026, time.September, 4, 5, 11, 0, 0, time.UTC)) + if !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("RetryPublication() error = %v, want state conflict", err) + } + if db.execSQL != "" { + t.Fatal("completed publication reached the retry UPDATE") + } +} + +func TestRetryPublicationRejectsMissingSpool(t *testing.T) { + db := &recordingDB{row: retryStateRow{processing: "failed", ingest: "uploaded", spoolAvailable: false}} + err := NewStore(db).RetryPublication(context.Background(), "publication-1", time.Date(2026, time.September, 4, 5, 12, 0, 0, time.UTC)) + if !errors.Is(err, ErrPublicationNotRetryable) { + t.Fatalf("RetryPublication() error = %v, want not retryable", err) + } +} + +func TestMarkStalledRechecksNoProgressCutoffInDatabase(t *testing.T) { + now := time.Date(2026, time.September, 4, 2, 10, 0, 0, time.UTC) + cutoff := now.Add(-30 * time.Minute) + db := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 1")} + if err := NewStore(db).MarkStalled(context.Background(), "publication-1", 3, cutoff, now); err != nil { + t.Fatalf("MarkStalled() error = %v", err) + } + if db.execSQL != markStalledSQL || !strings.Contains(markStalledSQL, "updated_at <= $3") { + t.Fatalf("stalled SQL = %q", db.execSQL) + } + staleDB := &recordingDB{commandTag: pgconn.NewCommandTag("UPDATE 0")} + if err := NewStore(staleDB).MarkStalled(context.Background(), "publication-1", 3, cutoff, now); !errors.Is(err, ErrPublicationStateConflict) { + t.Fatalf("MarkStalled() stale error = %v", err) + } +} + +func TestAppendAcceptanceResultValidatesAndAppendsEvidence(t *testing.T) { + now := time.Date(2026, time.September, 4, 2, 11, 0, 0, time.UTC) + result := governance.AcceptanceResult{ + ID: "acceptance-1", + PublicationID: "publication-1", + RunID: "run-1", + CheckType: "chunk_structure", + State: governance.AcceptancePassed, + PolicyVersion: "structural-v1", + EvidenceSummary: json.RawMessage(`{"chunk_total":2,"parent_match":true}`), + ObservedAt: now, + } + db := &recordingDB{commandTag: pgconn.NewCommandTag("INSERT 0 1")} + if err := NewStore(db).AppendAcceptanceResult(context.Background(), result); err != nil { + t.Fatalf("AppendAcceptanceResult() error = %v", err) + } + if db.execSQL != appendAcceptanceResultSQL || len(db.execArgs) != 8 { + t.Fatalf("acceptance insert = %q %#v", db.execSQL, db.execArgs) + } + + result.EvidenceSummary = json.RawMessage(`"raw text"`) + invalidDB := &recordingDB{commandTag: pgconn.NewCommandTag("INSERT 0 1")} + if err := NewStore(invalidDB).AppendAcceptanceResult(context.Background(), result); !errors.Is(err, ErrInvalidAcceptanceResult) { + t.Fatalf("invalid acceptance error = %v", err) + } + if invalidDB.execSQL != "" { + t.Fatal("invalid acceptance result reached database") + } +} + +type publicationUploadRow struct{} + +type publicationValidationRow struct{} + +func (publicationValidationRow) Scan(dest ...any) error { + *(dest[0].(*string)) = "publication-1" + *(dest[1].(*string)) = "kb-1" + *(dest[2].(*string)) = "knowledge-1" + return nil +} + +type legacyVerificationRow struct { + id string + kbID string + knowledgeID string +} + +func (row legacyVerificationRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.id + *(dest[1].(*string)) = row.kbID + *(dest[2].(*string)) = row.knowledgeID + return nil +} + +type legacyVerificationRows struct { + items []legacyVerificationRow + index int +} + +func (rows *legacyVerificationRows) Close() {} +func (rows *legacyVerificationRows) Err() error { return nil } +func (rows *legacyVerificationRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (rows *legacyVerificationRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (rows *legacyVerificationRows) Next() bool { + if rows.index >= len(rows.items) { + return false + } + rows.index++ + return true +} +func (rows *legacyVerificationRows) Scan(dest ...any) error { + return rows.items[rows.index-1].Scan(dest...) +} +func (rows *legacyVerificationRows) Values() ([]any, error) { + return nil, errors.New("not implemented") +} +func (rows *legacyVerificationRows) RawValues() [][]byte { return nil } +func (rows *legacyVerificationRows) Conn() *pgx.Conn { return nil } + +type testPublicationRow struct { + id string + title string + ingest string + processing string + retrieval string + targetScope string + changeSetID string + changeType string + canStart bool + retryCount int + lastError string + canRetry bool + updated time.Time +} + +func (row testPublicationRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.id + *(dest[1].(*string)) = row.title + *(dest[2].(*string)) = row.ingest + *(dest[3].(*string)) = row.processing + *(dest[4].(*string)) = row.retrieval + *(dest[5].(*string)) = row.targetScope + *(dest[6].(*string)) = row.changeSetID + *(dest[7].(*string)) = row.changeType + *(dest[8].(*bool)) = row.canStart + *(dest[9].(*int)) = row.retryCount + *(dest[10].(*string)) = row.lastError + *(dest[11].(*bool)) = row.canRetry + *(dest[12].(*time.Time)) = row.updated + return nil +} + +type testPublicationRows struct { + items []testPublicationRow + index int +} + +func (rows *testPublicationRows) Close() {} +func (rows *testPublicationRows) Err() error { return nil } +func (rows *testPublicationRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (rows *testPublicationRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (rows *testPublicationRows) Next() bool { + if rows.index >= len(rows.items) { + return false + } + rows.index++ + return true +} +func (rows *testPublicationRows) Scan(dest ...any) error { + return rows.items[rows.index-1].Scan(dest...) +} +func (rows *testPublicationRows) Values() ([]any, error) { return nil, errors.New("not implemented") } +func (rows *testPublicationRows) RawValues() [][]byte { return nil } +func (rows *testPublicationRows) Conn() *pgx.Conn { return nil } + +type totalRow struct { + total int +} + +func (row totalRow) Scan(dest ...any) error { + *(dest[0].(*int)) = row.total + return nil +} + +type acceptanceRecordRow struct { + checkType string + state string + policy string + evidence json.RawMessage + observed time.Time +} + +func (row acceptanceRecordRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.checkType + *(dest[1].(*string)) = row.state + *(dest[2].(*string)) = row.policy + *(dest[3].(*json.RawMessage)) = row.evidence + *(dest[4].(*time.Time)) = row.observed + return nil +} + +type acceptanceRecordRows struct { + items []acceptanceRecordRow + index int +} + +func (rows *acceptanceRecordRows) Close() {} +func (rows *acceptanceRecordRows) Err() error { return nil } +func (rows *acceptanceRecordRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (rows *acceptanceRecordRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (rows *acceptanceRecordRows) Next() bool { + if rows.index >= len(rows.items) { + return false + } + rows.index++ + return true +} +func (rows *acceptanceRecordRows) Scan(dest ...any) error { + return rows.items[rows.index-1].Scan(dest...) +} +func (rows *acceptanceRecordRows) Values() ([]any, error) { return nil, errors.New("not implemented") } +func (rows *acceptanceRecordRows) RawValues() [][]byte { return nil } +func (rows *acceptanceRecordRows) Conn() *pgx.Conn { return nil } + +type retryStateRow struct { + processing string + ingest string + spoolAvailable bool +} + +func (row retryStateRow) Scan(dest ...any) error { + *(dest[0].(*string)) = row.processing + *(dest[1].(*string)) = row.ingest + *(dest[2].(*bool)) = row.spoolAvailable + return nil +} + +type publicationPollRow struct { + now time.Time +} + +func (row publicationPollRow) Scan(dest ...any) error { + *(dest[0].(*string)) = "publication-1" + *(dest[1].(*string)) = "kb-1" + *(dest[2].(*string)) = "knowledge-1" + *(dest[3].(*string)) = string(governance.ProcessingProcessing) + *(dest[4].(*int)) = 2 + *(dest[5].(*string)) = "processing" + *(dest[6].(**time.Time)) = &row.now + *(dest[7].(**time.Time)) = &row.now + *(dest[8].(**time.Time)) = &row.now + *(dest[9].(**time.Time)) = &row.now + return nil +} + +type sequenceRow struct { + sequence int +} + +func (row sequenceRow) Scan(dest ...any) error { + *(dest[0].(*int)) = row.sequence + return nil +} + +func (publicationUploadRow) Scan(dest ...any) error { + values := []any{ + "publication-1", + "version-1", + "kb-1", + "", + string(governance.IngestPending), + string(governance.ProcessingPending), + "internal", + "content-1", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + int64(24), + "text/markdown", + "md", + "objects/artifact.md", + "Safe title", + } + for index, value := range values { + switch target := dest[index].(type) { + case *string: + *target = value.(string) + case *int64: + *target = value.(int64) + case *governance.Visibility: + *target = governance.Visibility(value.(string)) + case nil: + panic("nil destination") + default: + panic("unexpected scan destination") + } + } + return nil +} diff --git a/internal/platform/postgres/ui_settings.go b/internal/platform/postgres/ui_settings.go new file mode 100644 index 0000000..9a6942a --- /dev/null +++ b/internal/platform/postgres/ui_settings.go @@ -0,0 +1,134 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +var ErrInvalidUISettings = errors.New("invalid UI settings") + +const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code` + +const getActiveUISettingsSQL = ` +SELECT ` + uiSettingsColumns + ` +FROM ui_setting_revisions +WHERE state = 'active' +` + +const createUISettingsCandidateSQL = ` +INSERT INTO ui_setting_revisions( + id,state,document_refresh_seconds,resource_refresh_seconds, + stalled_after_seconds,finalizing_after_seconds,updated_by,created_at +) VALUES($1,'validated',$2,$3,$4,$5,$6,$7) +RETURNING ` + uiSettingsColumns + ` +` + +const lockUISettingsRevisionSQL = ` +SELECT ` + uiSettingsColumns + ` +FROM ui_setting_revisions +WHERE id=$1 +FOR UPDATE +` + +const retireActiveUISettingsSQL = ` +UPDATE ui_setting_revisions +SET state='retired',retired_at=$2 +WHERE state='active' AND id<>$1 +` + +const activateUISettingsSQL = ` +UPDATE ui_setting_revisions +SET state='active',activated_at=$2,retired_at=NULL,error_code='' +WHERE id=$1 AND state='validated' +RETURNING ` + uiSettingsColumns + ` +` + +func (s *Store) GetActiveUISettings(ctx context.Context) (governance.UISettings, error) { + return scanUISettings(s.db.QueryRow(ctx, getActiveUISettingsSQL)) +} + +func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governance.CreateUISettingsCandidateRequest) (governance.UISettings, error) { + if !request.Valid() { + return governance.UISettings{}, ErrInvalidUISettings + } + id, err := newOpaqueID("uis-") + if err != nil { + return governance.UISettings{}, fmt.Errorf("create UI settings id: %w", err) + } + return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL, + id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds, + request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC())) +} + +func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) { + if !validShortID(revisionID) || now.IsZero() { + return governance.UISettings{}, ErrInvalidUISettings + } + var tx pgx.Tx + var err error + ownsTransaction := false + if existing, ok := s.db.(pgx.Tx); ok { + tx = existing + } else { + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return governance.UISettings{}, errors.New("store does not support transactions") + } + tx, err = beginner.Begin(ctx) + if err != nil { + return governance.UISettings{}, fmt.Errorf("begin UI settings activation: %w", err) + } + ownsTransaction = true + defer func() { _ = tx.Rollback(ctx) }() + } + candidate, err := scanUISettings(tx.QueryRow(ctx, lockUISettingsRevisionSQL, revisionID)) + if err != nil { + return governance.UISettings{}, err + } + if candidate.State == "active" { + if ownsTransaction { + if err := tx.Commit(ctx); err != nil { + return governance.UISettings{}, fmt.Errorf("commit UI settings activation: %w", err) + } + } + return candidate, nil + } + if candidate.State != "validated" { + return governance.UISettings{}, ErrInvalidUISettings + } + activatedAt := now.UTC() + if _, err := tx.Exec(ctx, retireActiveUISettingsSQL, revisionID, activatedAt); err != nil { + return governance.UISettings{}, fmt.Errorf("retire active UI settings: %w", err) + } + active, err := scanUISettings(tx.QueryRow(ctx, activateUISettingsSQL, revisionID, activatedAt)) + if err != nil { + return governance.UISettings{}, err + } + if ownsTransaction { + if err := tx.Commit(ctx); err != nil { + return governance.UISettings{}, fmt.Errorf("commit UI settings activation: %w", err) + } + } + return active, nil +} + +func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings, error) { + var settings governance.UISettings + if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds, + &settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds, + &settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil { + return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err) + } + if settings.State != "validated" && settings.State != "active" && settings.State != "retired" && settings.State != "failed" { + return governance.UISettings{}, errors.New("invalid UI settings state") + } + return settings, nil +} diff --git a/internal/platform/postgres/ui_settings_integration_test.go b/internal/platform/postgres/ui_settings_integration_test.go new file mode 100644 index 0000000..8a97eaa --- /dev/null +++ b/internal/platform/postgres/ui_settings_integration_test.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + previous, err := store.GetActiveUISettings(ctx) + if err != nil || previous.ID != "settings-default" { + t.Fatalf("previous=%#v error=%v", previous, err) + } + now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC) + candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now}) + if err != nil { + t.Fatal(err) + } + active, err := store.ActivateUISettings(ctx, candidate.ID, now.Add(time.Second)) + if err != nil || active.ID != candidate.ID || active.State != "active" || active.ActivatedAt == nil { + t.Fatalf("active=%#v error=%v", active, err) + } + again, err := store.ActivateUISettings(ctx, candidate.ID, now.Add(2*time.Second)) + if err != nil || again.ID != candidate.ID || again.State != "active" { + t.Fatalf("again=%#v error=%v", again, err) + } + var state string + var retiredAt *time.Time + if err := pool.QueryRow(ctx, `SELECT state,retired_at FROM ui_setting_revisions WHERE id=$1`, previous.ID).Scan(&state, &retiredAt); err != nil || state != "retired" || retiredAt == nil { + t.Fatalf("previous state=%q retiredAt=%v error=%v", state, retiredAt, err) + } +} diff --git a/internal/platform/postgres/ui_settings_test.go b/internal/platform/postgres/ui_settings_test.go new file mode 100644 index 0000000..fa89fba --- /dev/null +++ b/internal/platform/postgres/ui_settings_test.go @@ -0,0 +1,65 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" +) + +func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) { + now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC) + row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now} + db := &recordingDB{row: row} + candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now}) + if err != nil { + t.Fatal(err) + } + if candidate.State != "validated" || candidate.DocumentRefreshSeconds != 30 || candidate.UpdatedBy != "engineer-1" { + t.Fatalf("candidate=%#v", candidate) + } + sql := strings.ToLower(db.queryRowSQL) + for _, required := range []string{"insert into ui_setting_revisions", "'validated'", "returning"} { + if !strings.Contains(sql, required) { + t.Fatalf("candidate SQL missing %q", required) + } + } + for _, forbidden := range []string{"secret", "token", "password", "api_key"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("candidate SQL contains forbidden field %q", forbidden) + } + } + + db.row = uiSettingsRow{id: "settings-active", state: "active", documentRefresh: 15, resourceRefresh: 30, stalledAfter: 1800, finalizingAfter: 7200, updatedBy: "system:migration", createdAt: now, activatedAt: &now} + active, err := NewStore(db).GetActiveUISettings(context.Background()) + if err != nil || active.State != "active" || active.FinalizingAfterSeconds != 7200 { + t.Fatalf("active=%#v error=%v", active, err) + } + if !strings.Contains(strings.ToLower(db.queryRowSQL), "where state = 'active'") { + t.Fatal("active settings query must select only active revision") + } +} + +func TestUISettingsRejectInvalidCandidate(t *testing.T) { + _, err := NewStore(&recordingDB{}).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 31, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: time.Now()}) + if !errors.Is(err, ErrInvalidUISettings) { + t.Fatalf("error=%v", err) + } +} + +type uiSettingsRow struct { + id, state, updatedBy, errorCode string + documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int + createdAt time.Time + activatedAt, retiredAt *time.Time +} + +func (row uiSettingsRow) Scan(dest ...any) error { + *dest[0].(*string), *dest[1].(*string) = row.id, row.state + *dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter + *dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode + return nil +} diff --git a/internal/platform/postgres/v1_migration_seed_test.go b/internal/platform/postgres/v1_migration_seed_test.go new file mode 100644 index 0000000..3980391 --- /dev/null +++ b/internal/platform/postgres/v1_migration_seed_test.go @@ -0,0 +1,53 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestV1Migration31SeedsRealSizedConnectionIDs reproduces the exact live-stack +// failure the first deployment surfaced: source_connections rows carry 36-char +// UUID ids, so a seeded revision id of 'scr-'||c.id overflows VARCHAR(36). +// Migration 31 must seed cleanly for real-sized ids. +func TestV1Migration31SeedsRealSizedConnectionIDs(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + // Roll back to just before migration 31 and re-enter it with live-shaped ids. + if err := RollbackLatestMigration(ctx, pool); err != nil { + t.Fatal(err) + } + const uuidID = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" // exactly 36 chars + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES($1,'alist','真实尺寸UUID源',true)`, uuidID); err != nil { + t.Fatal(err) + } + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migration 31 with 36-char connection id: %v", err) + } + var revisionID, currentRevisionID string + if err := pool.QueryRow(ctx, `SELECT id FROM source_connection_revisions WHERE connection_id=$1 AND revision_number=1 AND state='active'`, uuidID).Scan(&revisionID); err != nil { + t.Fatal(err) + } + if len(revisionID) > 36 { + t.Fatalf("seeded revision id %q exceeds 36 characters", revisionID) + } + if err := pool.QueryRow(ctx, `SELECT current_revision_id FROM source_connections WHERE id=$1`, uuidID).Scan(¤tRevisionID); err != nil { + t.Fatal(err) + } + if currentRevisionID != revisionID { + t.Fatalf("current_revision_id=%q want %q", currentRevisionID, revisionID) + } +} diff --git a/internal/platform/postgres/v1_scheduler_integration_test.go b/internal/platform/postgres/v1_scheduler_integration_test.go new file mode 100644 index 0000000..2d0ead4 --- /dev/null +++ b/internal/platform/postgres/v1_scheduler_integration_test.go @@ -0,0 +1,218 @@ +package postgres + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestV1SourceRefreshScheduler proves the dispatcher ACTUALLY consumes the +// per-source schedule: a due, enabled source with an active revision interval +// gets exactly one catalog refresh job, its next horizon advances by that +// interval, and the disabled / in-flight / not-due / unconfigured states are +// each honoured. +func TestV1SourceRefreshScheduler(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 12, 8, 0, 0, 0, time.UTC) + + seed := func(id, sourceType, name string, enabled bool, schedule string, nextRefresh *time.Time, revisionInterval string) { + var nextArg any + if nextRefresh != nil { + nextArg = nextRefresh.UTC() + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled,schedule,next_refresh_at) VALUES($1,$2,$3,$4,$5::jsonb,$6)`, + id, sourceType, name, enabled, schedule, nextArg); err != nil { + t.Fatal(err) + } + revisionSchedule := "{}" + if revisionInterval != "" { + revisionSchedule = `{"refresh_interval_seconds":` + revisionInterval + `}` + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connection_revisions(id,connection_id,revision_number,state,source_type,name,enabled,schedule,created_by,validated_at,activated_at) +VALUES('active-'||$1,$1,1,'active',$2,$3,$4,$5::jsonb,'tester',$6,$6)`, id, sourceType, name, enabled, revisionSchedule, now.UTC()); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE source_connections SET current_revision_id='active-'||$1 WHERE id=$1`, id); err != nil { + t.Fatal(err) + } + } + past := now.Add(-time.Minute) + + // due hourly source + seed("conn-due", "alist", "到期源", true, "{}", &past, "3600") + // disabled even though due + seed("conn-off", "alist", "停用源", false, "{}", &past, "3600") + // due but the active revision has no interval (defensive floor 300s advances) + seed("conn-nointerval", "alist", "无周期源", true, "{}", &past, "") + // an import-only manual source is never dispatchable even if a horizon is + // somehow present (activation rejects schedules for it) + seed("conn-offline", "manual", "离线手工源", true, "{}", &past, "") + // not configured: next_refresh_at NULL + seed("conn-uncfg", "alist", "未配置源", true, "{}", nil, "3600") + // due and has a pending refresh job already (in-flight window) + seed("conn-busy", "alist", "在途源", true, "{}", &past, "3600") + if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at) +VALUES('job-busy','source.catalog.refresh',jsonb_build_object('connection_id','conn-busy'),'pending',20,$1,3,'manual-busy',$1,$1)`, now.UTC()); err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + claimed, enqueued, err := store.ClaimDueSourceRefreshes(ctx, now) + if err != nil { + t.Fatal(err) + } + if claimed != 3 || enqueued != 2 { + t.Fatalf("claimed=%d enqueued=%d want 3/2", claimed, enqueued) + } + + // exactly one new job for conn-due and conn-manual, none for conn-busy + var dueJobs, busyJobs int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh' AND payload->>'connection_id'='conn-due'`).Scan(&dueJobs); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh' AND payload->>'connection_id'='conn-busy'`).Scan(&busyJobs); err != nil { + t.Fatal(err) + } + if dueJobs != 1 || busyJobs != 1 { + t.Fatalf("dueJobs=%d busyJobs=%d want 1/1", dueJobs, busyJobs) + } + + // horizons advanced by the ACTIVE revision interval (hourly source -> +1h) + var horizon time.Time + if err := pool.QueryRow(ctx, `SELECT next_refresh_at FROM source_connections WHERE id='conn-due'`).Scan(&horizon); err != nil { + t.Fatal(err) + } + if !horizon.Equal(now.Add(time.Hour)) { + t.Fatalf("conn-due horizon=%v want %v", horizon, now.Add(time.Hour)) + } + if err := pool.QueryRow(ctx, `SELECT next_refresh_at FROM source_connections WHERE id='conn-nointerval'`).Scan(&horizon); err != nil { + t.Fatal(err) + } + if !horizon.Equal(now.Add(5 * time.Minute)) { + t.Fatalf("conn-nointerval horizon=%v want five minute floor", horizon) + } + + // manual sources are never dispatchable: horizon frozen at the seeded past + var offlineHorizon time.Time + if err := pool.QueryRow(ctx, `SELECT next_refresh_at FROM source_connections WHERE id='conn-offline'`).Scan(&offlineHorizon); err != nil { + t.Fatal(err) + } + if !offlineHorizon.Equal(past) { + t.Fatalf("manual source was claimed: horizon moved to %v", offlineHorizon) + } + + // disabled and unconfigured sources untouched + // neither was claimed: no last_scheduled_at stamp; conn-uncfg keeps NULL + // horizon, conn-off keeps its seeded past value untouched + var offScheduled, uncfgScheduled bool + if err := pool.QueryRow(ctx, `SELECT last_scheduled_at IS NOT NULL FROM source_connections WHERE id='conn-off'`).Scan(&offScheduled); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT last_scheduled_at IS NOT NULL FROM source_connections WHERE id='conn-uncfg'`).Scan(&uncfgScheduled); err != nil { + t.Fatal(err) + } + if offScheduled || uncfgScheduled { + t.Fatalf("offScheduled=%v uncfgScheduled=%v want both false", offScheduled, uncfgScheduled) + } + + // second tick at the same instant claims nothing (horizons already future) + claimed2, enqueued2, err := store.ClaimDueSourceRefreshes(ctx, now) + if err != nil { + t.Fatal(err) + } + if claimed2 != 0 || enqueued2 != 0 { + t.Fatalf("replay claimed=%d enqueued=%d want 0/0", claimed2, enqueued2) + } + + // simulate the worker finishing the two scheduled jobs (conn-busy's stays + // pending); after one hour all three are due again, but only the two whose + // jobs completed get a NEW job with a NEW idempotency window. + if _, err := pool.Exec(ctx, `UPDATE jobs SET state='completed', updated_at=$1::timestamptz WHERE type='source.catalog.refresh' AND payload->>'connection_id' IN ('conn-due','conn-nointerval') AND state='pending'`, now.Add(30*time.Minute)); err != nil { + t.Fatal(err) + } + claimed3, enqueued3, err := store.ClaimDueSourceRefreshes(ctx, now.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if claimed3 != 3 || enqueued3 != 2 { + t.Fatalf("hour later claimed=%d enqueued=%d want 3/2", claimed3, enqueued3) + } +} + +// TestV1ConcurrentDispatchersClaimOnce proves the SKIP LOCKED lease: two +// dispatcher goroutines ticking at the same instant enqueue exactly ONE refresh +// job for the same due source, never two. +func TestV1ConcurrentDispatchersClaimOnce(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 12, 10, 0, 0, 0, time.UTC) + past := now.Add(-time.Minute) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled,next_refresh_at) +VALUES('conn-race','alist','并发源',true,$1)`, past); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO source_connection_revisions(id,connection_id,revision_number,state,source_type,name,enabled,schedule,created_by,validated_at,activated_at) +VALUES('active-race','conn-race',1,'active','alist','并发源',true,'{"refresh_interval_seconds":3600}','tester',$1,$1)`, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE source_connections SET current_revision_id='active-race' WHERE id='conn-race'`); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + + var wg sync.WaitGroup + totalEnqueued := 0 + var mu sync.Mutex + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, enq, err := store.ClaimDueSourceRefreshes(ctx, now) + if err != nil { + t.Errorf("dispatch: %v", err) + return + } + mu.Lock() + totalEnqueued += enq + mu.Unlock() + }() + } + wg.Wait() + if totalEnqueued != 1 { + t.Fatalf("total enqueued=%d want exactly 1 across 4 concurrent dispatchers", totalEnqueued) + } + var jobs int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='source.catalog.refresh' AND payload->>'connection_id'='conn-race'`).Scan(&jobs); err != nil { + t.Fatal(err) + } + if jobs != 1 { + t.Fatalf("stored jobs=%d want 1", jobs) + } +} diff --git a/internal/platform/postgres/v1_source_settings_integration_test.go b/internal/platform/postgres/v1_source_settings_integration_test.go new file mode 100644 index 0000000..f9d3833 --- /dev/null +++ b/internal/platform/postgres/v1_source_settings_integration_test.go @@ -0,0 +1,225 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// TestV1SourceSettingsLifecycle freezes the V1 promises: an edit only becomes +// effective when its revision is activated; activating derives next_refresh_at +// from the schedule; disabling stops new scheduled jobs immediately while all +// history (revisions, snapshots, scans, jobs) remains readable. +func TestV1SourceSettingsLifecycle(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,base_url,secret_ref,enabled,schedule) +VALUES('conn-v1','alist','原名字','http://alist.internal','alist-secret',false,'{}'::jsonb)`); err != nil { + t.Fatal(err) + } + // Mirror migration 31's seed for a connection created after the migration. + if _, err := pool.Exec(ctx, `INSERT INTO source_connection_revisions(id,connection_id,revision_number,state,source_type,name,base_url,secret_ref,enabled,schedule,created_by,validated_at,activated_at) +VALUES('scr-seed-v1','conn-v1',1,'active','alist','原名字','http://alist.internal','alist-secret',false,'{}'::jsonb,'migration-31',$1,$1)`, now); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE source_connections SET current_revision_id='scr-seed-v1' WHERE id='conn-v1'`); err != nil { + t.Fatal(err) + } + store := NewStore(pool) + + // The migration seeds revision 1 (active) for every connection. + var activeFirst string + var firstNumber int + if err := pool.QueryRow(ctx, `SELECT id, revision_number FROM source_connection_revisions WHERE connection_id='conn-v1' AND state='active'`).Scan(&activeFirst, &firstNumber); err != nil { + t.Fatal(err) + } + if firstNumber != 1 { + t.Fatalf("seed revision=%d want 1", firstNumber) + } + + draft := SourceSettingsDraft{ + SourceType: "alist", Name: "国内工业AList", BaseURL: "http://alist.internal", + SecretRef: "alist-secret", Enabled: true, + Schedule: json.RawMessage(`{"refresh_interval_seconds":3600}`), Operator: "zherom", + } + revisionID, err := store.CreateSourceSettingsRevision(ctx, "conn-v1", draft, now) + if err != nil { + t.Fatal(err) + } + + // Draft alone changes nothing at runtime. + var name string + var enabled bool + var next any + if err := pool.QueryRow(ctx, `SELECT name, enabled, next_refresh_at FROM source_connections WHERE id='conn-v1'`).Scan(&name, &enabled, &next); err != nil { + t.Fatal(err) + } + if name != "原名字" || enabled || next != nil { + t.Fatalf("draft leaked into runtime: name=%v enabled=%v next=%v", name, enabled, next) + } + + // Validation rejects bad schedules and unknown types before any write. + bad := draft + bad.Schedule = json.RawMessage(`{"refresh_interval_seconds":60}`) + if _, err := store.CreateSourceSettingsRevision(ctx, "conn-v1", bad, now); !errors.Is(err, ErrInvalidSourceSettings) { + t.Fatalf("sub-floor schedule err=%v want ErrInvalidSourceSettings", err) + } + wrongType := draft + wrongType.SourceType = "ftp" + if _, err := store.CreateSourceSettingsRevision(ctx, "conn-v1", wrongType, now); !errors.Is(err, ErrInvalidSourceSettings) { + t.Fatalf("unknown type err=%v want ErrInvalidSourceSettings", err) + } + + // Activate: runtime snapshot syncs and horizon derives from schedule. + if err := store.ActivateSourceSettingsRevision(ctx, "conn-v1", revisionID, now); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT name, enabled, next_refresh_at, current_revision_id FROM source_connections WHERE id='conn-v1'`).Scan(&name, &enabled, &next, new(string)); err != nil { + t.Fatal(err) + } + if name != "国内工业AList" || !enabled { + t.Fatalf("activation did not sync snapshot: name=%v enabled=%v", name, enabled) + } + if horizon, ok := next.(time.Time); !ok || !horizon.Equal(now.Add(time.Hour)) { + t.Fatalf("next_refresh_at=%v want %v", next, now.Add(time.Hour)) + } + // Old revision retired, new one active; exactly one active per connection. + var retiredFound, activeCount int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_connection_revisions WHERE id=$1 AND state='retired'`, activeFirst).Scan(&retiredFound); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_connection_revisions WHERE connection_id='conn-v1' AND state='active'`).Scan(&activeCount); err != nil { + t.Fatal(err) + } + if retiredFound != 1 || activeCount != 1 { + t.Fatalf("retired=%d active=%d want 1/1", retiredFound, activeCount) + } + + // Scheduler consumes the new horizon. + claimed, enqueued, err := store.ClaimDueSourceRefreshes(ctx, now.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if claimed != 1 || enqueued != 1 { + t.Fatalf("scheduler claimed=%d enqueued=%d want 1/1", claimed, enqueued) + } + + // Revision history projection: safe fields only. + revisions, err := store.ListSourceConnectionRevisions(ctx, "conn-v1") + if err != nil { + t.Fatal(err) + } + if len(revisions) != 2 || revisions[0].RevisionNumber != 2 || revisions[0].State != "active" || revisions[1].State != "retired" { + t.Fatalf("revisions=%+v", revisions) + } + if revisions[0].RefreshIntervalSeconds != "3600" || revisions[0].CreatedBy != "zherom" { + t.Fatalf("revision projection=%+v", revisions[0]) + } + // The projection must expose the lifecycle timestamps it stores. + if revisions[0].ValidatedAt == nil || revisions[0].ActivatedAt == nil { + t.Fatalf("revision %d lost validated/activated timestamps: %+v", revisions[0].RevisionNumber, revisions[0]) + } + var snapshotUpdatedBy string + if err := pool.QueryRow(ctx, `SELECT updated_by FROM source_connections WHERE id='conn-v1'`).Scan(&snapshotUpdatedBy); err != nil { + t.Fatal(err) + } + if snapshotUpdatedBy != "zherom" { + t.Fatalf("runtime snapshot updated_by=%q want the revision author zherom", snapshotUpdatedBy) + } + + // Disabling (new revision with enabled=false) clears the horizon and stops + // scheduling; history remains queryable. + off := draft + off.Enabled = false + off.Schedule = json.RawMessage(`{}`) + offID, err := store.CreateSourceSettingsRevision(ctx, "conn-v1", off, now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if err := store.ActivateSourceSettingsRevision(ctx, "conn-v1", offID, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT enabled, next_refresh_at FROM source_connections WHERE id='conn-v1'`).Scan(&enabled, &next); err != nil { + t.Fatal(err) + } + if enabled || next != nil { + t.Fatalf("disable left enabled=%v next=%v", enabled, next) + } + if _, err := pool.Exec(ctx, `UPDATE jobs SET state='completed', updated_at=$1::timestamptz WHERE type='source.catalog.refresh' AND payload->>'connection_id'='conn-v1' AND state='pending'`, now.Add(2*time.Minute)); err != nil { + t.Fatal(err) + } + claimed2, enqueued2, err := store.ClaimDueSourceRefreshes(ctx, now.Add(24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if claimed2 != 0 || enqueued2 != 0 { + t.Fatalf("disabled source scheduled claimed=%d enqueued=%d want 0/0", claimed2, enqueued2) + } + var snapshotKept int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_connection_revisions WHERE connection_id='conn-v1'`).Scan(&snapshotKept); err != nil { + t.Fatal(err) + } + if snapshotKept < 3 { + t.Fatalf("history rows=%d want at least 3", snapshotKept) + } + + // Empty endpoint/credential_ref in a draft inherit the connection's stored + // values (the UI must never resend secrets it cannot read back). + inherited := draft + inherited.BaseURL = "" + inherited.SecretRef = "" + inheritedID, err := store.CreateSourceSettingsRevision(ctx, "conn-v1", inherited, now.Add(5*time.Minute)) + if err != nil { + t.Fatal(err) + } + var gotBase, gotSecret string + if err := pool.QueryRow(ctx, `SELECT base_url, secret_ref FROM source_connection_revisions WHERE id=$1`, inheritedID).Scan(&gotBase, &gotSecret); err != nil { + t.Fatal(err) + } + if gotBase != "http://alist.internal" || gotSecret != "alist-secret" { + t.Fatalf("inherited=%q/%q want stored values", gotBase, gotSecret) + } + + // manual sources can never be scheduled (activation rejects the horizon). + if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-manual-v1','manual','手工源',true)`); err != nil { + t.Fatal(err) + } + manualID, err := store.CreateSourceSettingsRevision(ctx, "conn-manual-v1", SourceSettingsDraft{ + SourceType: "manual", Name: "手工源", Schedule: json.RawMessage(`{"refresh_interval_seconds":3600}`), Operator: "zherom", + }, now.Add(6*time.Minute)) + if err != nil { + t.Fatal(err) + } + if err := store.ActivateSourceSettingsRevision(ctx, "conn-manual-v1", manualID, now.Add(6*time.Minute)); !errors.Is(err, ErrInvalidSourceSettings) { + t.Fatalf("manual schedule activation err=%v want ErrInvalidSourceSettings", err) + } + + // Activation is replay-safe and cannot resurrect non-validated states. + if err := store.ActivateSourceSettingsRevision(ctx, "conn-v1", offID, now.Add(3*time.Minute)); err != nil { + t.Fatalf("replay activation: %v", err) + } + if err := store.ActivateSourceSettingsRevision(ctx, "conn-v1", activeFirst, now.Add(3*time.Minute)); errors.Is(err, nil) || !errors.Is(err, ErrInvalidSourceSettings) { + t.Fatalf("retired reactivation err=%v want ErrInvalidSourceSettings", err) + } + + _ = governance.SourceTypeAList +} diff --git a/internal/platform/spool/artifact.go b/internal/platform/spool/artifact.go new file mode 100644 index 0000000..8c430d7 --- /dev/null +++ b/internal/platform/spool/artifact.go @@ -0,0 +1,285 @@ +package spool + +import ( + "archive/zip" + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "hash" + "io" + "os" + "path/filepath" + "strings" + "unicode/utf8" + + "knowledge-governance/internal/governance" +) + +type Expectation struct { + RelativeRef string + SHA256 string + SizeBytes int64 + Format string +} + +type Artifact struct { + file *os.File + expectedSHA string + expectedSize int64 + streamHash hash.Hash + streamed int64 +} + +var supportedFormats = map[string]struct{}{ + "pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, + "md": {}, "txt": {}, "csv": {}, "json": {}, +} + +func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) { + if !filepath.IsAbs(root) || relativeRef == "" || filepath.IsAbs(relativeRef) || maxBytes <= 0 { + return Expectation{}, errors.New("invalid spool reference") + } + cleanRef := filepath.Clean(relativeRef) + if cleanRef == "." || cleanRef != relativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) { + return Expectation{}, errors.New("invalid spool reference") + } + if _, ok := supportedFormats[format]; !ok { + return Expectation{}, errors.New("unsupported spool artifact format") + } + file, err := openRegularNoSymlink(root, cleanRef) + if err != nil { + return Expectation{}, errors.New("open spool artifact") + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxBytes { + return Expectation{}, errors.New("spool artifact size or type mismatch") + } + digest := sha256.New() + bytesRead, err := io.Copy(digest, file) + if err != nil || bytesRead != info.Size() { + return Expectation{}, errors.New("inspect spool artifact content") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return Expectation{}, errors.New("rewind spool artifact") + } + if err := validateFormat(file, format, info.Size()); err != nil { + return Expectation{}, errors.New("spool artifact format mismatch") + } + return Expectation{ + RelativeRef: cleanRef, + SHA256: hex.EncodeToString(digest.Sum(nil)), + SizeBytes: info.Size(), + Format: format, + }, nil +} + +func OpenVerified(root string, expected Expectation, maxBytes int64) (*Artifact, error) { + if !filepath.IsAbs(root) || expected.RelativeRef == "" || filepath.IsAbs(expected.RelativeRef) { + return nil, errors.New("invalid spool reference") + } + cleanRef := filepath.Clean(expected.RelativeRef) + if cleanRef == "." || cleanRef != expected.RelativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) { + return nil, errors.New("invalid spool reference") + } + if !governance.ValidSHA256(expected.SHA256) || expected.SizeBytes <= 0 || maxBytes <= 0 || expected.SizeBytes > maxBytes { + return nil, errors.New("invalid spool expectation") + } + if _, ok := supportedFormats[expected.Format]; !ok { + return nil, errors.New("unsupported spool artifact format") + } + file, err := openRegularNoSymlink(root, cleanRef) + if err != nil { + return nil, errors.New("open spool artifact") + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, errors.New("inspect spool artifact") + } + if !info.Mode().IsRegular() || info.Size() != expected.SizeBytes { + file.Close() + return nil, errors.New("spool artifact size or type mismatch") + } + beforeHash := sha256.New() + bytesRead, err := io.Copy(beforeHash, file) + if err != nil || bytesRead != expected.SizeBytes || hex.EncodeToString(beforeHash.Sum(nil)) != expected.SHA256 { + file.Close() + return nil, errors.New("spool artifact content mismatch") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + file.Close() + return nil, errors.New("rewind spool artifact") + } + if err := validateFormat(file, expected.Format, expected.SizeBytes); err != nil { + file.Close() + return nil, errors.New("spool artifact format mismatch") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + file.Close() + return nil, errors.New("rewind spool artifact after validation") + } + return &Artifact{ + file: file, + expectedSHA: expected.SHA256, + expectedSize: expected.SizeBytes, + streamHash: sha256.New(), + }, nil +} + +func validateFormat(file *os.File, format string, size int64) error { + switch format { + case "pdf": + header := make([]byte, len("%PDF-")) + if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) { + return errors.New("invalid PDF magic") + } + case "docx", "xlsx", "pptx": + archive, err := zip.NewReader(file, size) + if err != nil { + return errors.New("invalid Office ZIP") + } + required := map[string]string{ + "docx": "word/document.xml", + "xlsx": "xl/workbook.xml", + "pptx": "ppt/presentation.xml", + }[format] + foundContentTypes := false + foundRequired := false + for _, entry := range archive.File { + switch entry.Name { + case "[Content_Types].xml": + foundContentTypes = true + case required: + foundRequired = true + } + } + if !foundContentTypes || !foundRequired { + return errors.New("missing Office package entry") + } + case "doc", "xls", "ppt": + header := make([]byte, 8) + oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1} + if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) { + return errors.New("invalid legacy Office magic") + } + case "json": + decoder := json.NewDecoder(file) + var value any + if err := decoder.Decode(&value); err != nil { + return errors.New("invalid JSON") + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return errors.New("multiple JSON values") + } + case "md", "txt", "csv": + prefixSize := size + if prefixSize > 4096 { + prefixSize = 4096 + } + prefix := make([]byte, prefixSize) + if _, err := io.ReadFull(file, prefix); err != nil { + return errors.New("read text prefix") + } + lowerPrefix := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(string(prefix), "\ufeff"))) + if strings.HasPrefix(lowerPrefix, " 0 { + artifact.streamed += int64(count) + _, _ = artifact.streamHash.Write(buffer[:count]) + } + return count, err +} + +func (artifact *Artifact) VerifyComplete() error { + if artifact.streamed != artifact.expectedSize || hex.EncodeToString(artifact.streamHash.Sum(nil)) != artifact.expectedSHA { + return errors.New("spool artifact changed during streaming") + } + return nil +} + +func (artifact *Artifact) Close() error { + return artifact.file.Close() +} diff --git a/internal/platform/spool/artifact_test.go b/internal/platform/spool/artifact_test.go new file mode 100644 index 0000000..ff2904a --- /dev/null +++ b/internal/platform/spool/artifact_test.go @@ -0,0 +1,216 @@ +package spool + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "testing" +) + +func TestOpenVerifiedRechecksSizeAndSHAWhileStreaming(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("safe governance document\n") + ref := filepath.Join("objects", "artifact.md") + if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + artifact, err := OpenVerified(root, Expectation{ + RelativeRef: ref, + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: int64(len(body)), + Format: "md", + }, 50*1024*1024) + if err != nil { + t.Fatalf("OpenVerified() error = %v", err) + } + defer artifact.Close() + streamed, err := io.ReadAll(artifact) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if string(streamed) != string(body) { + t.Fatalf("streamed body = %q", streamed) + } + if err := artifact.VerifyComplete(); err != nil { + t.Fatalf("VerifyComplete() error = %v", err) + } +} + +func TestInspectVerifiedReturnsCanonicalExpectation(t *testing.T) { + root := t.TempDir() + body := []byte("governance p3 unique fact\n") + if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil { + t.Fatal(err) + } + ref := filepath.Join("manual", "fixture.md") + if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + + got, err := InspectVerified(root, ref, "md", 1024) + if err != nil { + t.Fatalf("InspectVerified() error = %v", err) + } + if got.RelativeRef != ref || got.SizeBytes != int64(len(body)) || got.SHA256 != hex.EncodeToString(digest[:]) || got.Format != "md" { + t.Fatalf("expectation = %#v", got) + } +} + +func TestOpenVerifiedRejectsSymlinksAndPathEscapes(t *testing.T) { + root := t.TempDir() + body := []byte("safe governance document\n") + digest := sha256.Sum256(body) + expected := Expectation{ + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: int64(len(body)), + Format: "md", + } + if err := os.Mkdir(filepath.Join(root, "real"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "real", "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("real", "artifact.md"), filepath.Join(root, "file-link.md")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real", filepath.Join(root, "dir-link")); err != nil { + t.Fatal(err) + } + parent := filepath.Dir(root) + rootLink := filepath.Join(parent, filepath.Base(root)+"-link") + if err := os.Symlink(root, rootLink); err != nil { + t.Fatal(err) + } + defer os.Remove(rootLink) + + tests := []struct { + name string + root string + ref string + }{ + {name: "absolute reference", root: root, ref: filepath.Join(root, "real", "artifact.md")}, + {name: "parent escape", root: root, ref: "../artifact.md"}, + {name: "file symlink", root: root, ref: "file-link.md"}, + {name: "directory symlink", root: root, ref: filepath.Join("dir-link", "artifact.md")}, + {name: "root symlink", root: rootLink, ref: filepath.Join("real", "artifact.md")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectation := expected + expectation.RelativeRef = tt.ref + artifact, err := OpenVerified(tt.root, expectation, 50*1024*1024) + if artifact != nil { + artifact.Close() + } + if err == nil { + t.Fatalf("OpenVerified(%q) error = nil", tt.ref) + } + }) + } +} + +func TestOpenVerifiedRejectsUnsupportedFormat(t *testing.T) { + root := t.TempDir() + body := []byte("safe governance document\n") + path := filepath.Join(root, "artifact.bin") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + artifact, err := OpenVerified(root, Expectation{ + RelativeRef: "artifact.bin", + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: int64(len(body)), + Format: "exe", + }, 50*1024*1024) + if artifact != nil { + artifact.Close() + } + if err == nil { + t.Fatal("OpenVerified() error = nil for unsupported format") + } +} + +func TestOpenVerifiedRejectsContentThatDoesNotMatchFormat(t *testing.T) { + tests := []struct { + name string + format string + body []byte + }{ + {name: "PDF magic", format: "pdf", body: []byte("not a PDF")}, + {name: "DOCX container", format: "docx", body: []byte("not a ZIP")}, + {name: "legacy Office magic", format: "doc", body: []byte("not OLE")}, + {name: "JSON syntax", format: "json", body: []byte("{")}, + {name: "text binary", format: "md", body: []byte{'a', 0, 'b'}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "artifact"), tt.body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(tt.body) + artifact, err := OpenVerified(root, Expectation{ + RelativeRef: "artifact", + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: int64(len(tt.body)), + Format: tt.format, + }, 50*1024*1024) + if artifact != nil { + artifact.Close() + } + if err == nil { + t.Fatalf("OpenVerified() error = nil for format %s", tt.format) + } + }) + } +} + +func TestOpenVerifiedRejectsHTMLMasqueradingAsText(t *testing.T) { + root := t.TempDir() + body := []byte(" Sign in") + if err := os.WriteFile(filepath.Join(root, "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + artifact, err := OpenVerified(root, Expectation{ + RelativeRef: "artifact.md", + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: int64(len(body)), + Format: "md", + }, 50*1024*1024) + if artifact != nil { + artifact.Close() + } + if err == nil { + t.Fatal("OpenVerified() error = nil for HTML login page") + } +} + +func TestOpenVerifiedRejectsEmptyArtifact(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "empty.md"), nil, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(nil) + artifact, err := OpenVerified(root, Expectation{ + RelativeRef: "empty.md", + SHA256: hex.EncodeToString(digest[:]), + SizeBytes: 0, + Format: "md", + }, 50*1024*1024) + if artifact != nil { + artifact.Close() + } + if err == nil { + t.Fatal("OpenVerified() error = nil for empty artifact") + } +} diff --git a/internal/platform/weknora/client.go b/internal/platform/weknora/client.go new file mode 100644 index 0000000..2052bd9 --- /dev/null +++ b/internal/platform/weknora/client.go @@ -0,0 +1,368 @@ +package weknora + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +const defaultMaxResponseBytes int64 = 2 * 1024 * 1024 + +var ErrConflict = errors.New("WeKnora resource conflict") + +type HTTPError struct { + StatusCode int +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("WeKnora HTTP status %d", e.StatusCode) +} + +func IsRetryable(err error) bool { + var httpErr *HTTPError + if errors.As(err, &httpErr) { + return httpErr.StatusCode == http.StatusTooManyRequests || httpErr.StatusCode >= 500 + } + if errors.Is(err, context.Canceled) { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var networkErr net.Error + return errors.As(err, &networkErr) +} + +type ClientConfig struct { + BaseURL string + APIKey string + Timeout time.Duration + MaxResponseBytes int64 + AllowDockerHTTP bool +} + +type Client struct { + baseURL *url.URL + apiKey string + httpClient *http.Client + maxResponseBytes int64 +} + +func NewClient(config ClientConfig) (*Client, error) { + baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL)) + if err != nil { + return nil, errors.New("invalid WeKnora base URL") + } + if baseURL.Scheme == "" || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" { + return nil, errors.New("invalid WeKnora base URL") + } + if baseURL.Scheme != "https" && !(baseURL.Scheme == "http" && (isLoopbackHost(baseURL.Hostname()) || (config.AllowDockerHTTP && isSingleLabelDockerHost(baseURL.Hostname())))) { + return nil, errors.New("WeKnora base URL must use HTTPS") + } + baseURL.Path = strings.TrimRight(baseURL.Path, "/") + if strings.TrimSpace(config.APIKey) == "" { + return nil, errors.New("WeKnora API key is required") + } + if config.Timeout <= 0 { + return nil, errors.New("WeKnora HTTP timeout must be positive") + } + if config.MaxResponseBytes <= 0 { + config.MaxResponseBytes = defaultMaxResponseBytes + } + return &Client{ + baseURL: baseURL, + apiKey: config.APIKey, + httpClient: &http.Client{ + Timeout: config.Timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + maxResponseBytes: config.MaxResponseBytes, + }, nil +} + +func isLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func isSingleLabelDockerHost(host string) bool { + if host == "" || strings.Contains(host, ".") || net.ParseIP(host) != nil || len(host) > 63 { + return false + } + for index, char := range host { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') { + continue + } + if char != '-' || index == 0 || index == len(host)-1 { + return false + } + } + return true +} + +func (c *Client) GetKnowledge(ctx context.Context, knowledgeID string) (Knowledge, error) { + if !validPathSegment(knowledgeID) { + return Knowledge{}, errors.New("invalid knowledge ID") + } + var response knowledgeResponse + if err := c.doJSON(ctx, http.MethodGet, "/api/v1/knowledge/"+knowledgeID, nil, &response); err != nil { + return Knowledge{}, err + } + if !response.Success || response.Data.ID == "" { + return Knowledge{}, errors.New("WeKnora returned an unsuccessful knowledge response") + } + return response.Data, nil +} + +func (c *Client) DeleteKnowledge(ctx context.Context, knowledgeID string) error { + if !validPathSegment(knowledgeID) { + return errors.New("invalid knowledge ID") + } + var response struct { + Success bool `json:"success"` + } + if err := c.doJSON(ctx, http.MethodDelete, "/api/v1/knowledge/"+knowledgeID, nil, &response); err != nil { + return err + } + if !response.Success { + return errors.New("WeKnora returned an unsuccessful delete response") + } + return nil +} + +func (c *Client) UploadFile(ctx context.Context, request UploadRequest) (Knowledge, error) { + if !validPathSegment(request.KnowledgeBaseID) { + return Knowledge{}, errors.New("invalid knowledge base ID") + } + if strings.TrimSpace(request.FileName) == "" || strings.ContainsAny(request.FileName, "/\\\x00") { + return Knowledge{}, errors.New("invalid upload file name") + } + if request.Content == nil { + return Knowledge{}, errors.New("upload content is required") + } + + pipeReader, pipeWriter := io.Pipe() + writer := multipart.NewWriter(pipeWriter) + writeResult := make(chan error, 1) + go func() { + err := writeMultipartUpload(writer, request) + if err == nil { + err = writer.Close() + } + _ = pipeWriter.CloseWithError(err) + writeResult <- err + }() + + endpoint := c.endpoint("/api/v1/knowledge-bases/" + request.KnowledgeBaseID + "/knowledge/file") + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, pipeReader) + if err != nil { + _ = pipeReader.CloseWithError(err) + <-writeResult + return Knowledge{}, errors.New("create WeKnora upload request") + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("X-API-Key", c.apiKey) + resp, err := c.httpClient.Do(req) + if err != nil { + _ = pipeReader.CloseWithError(err) + <-writeResult + return Knowledge{}, fmt.Errorf("send WeKnora upload request: %w", err) + } + if writeErr := <-writeResult; writeErr != nil { + resp.Body.Close() + return Knowledge{}, errors.New("stream WeKnora upload body") + } + + var response knowledgeResponse + if resp.StatusCode == http.StatusConflict { + if err := c.decodeBody(resp, &response); err != nil { + return Knowledge{}, err + } + return response.Data, ErrConflict + } + if err := c.decodeResponse(resp, &response); err != nil { + return Knowledge{}, err + } + if !response.Success || response.Data.ID == "" { + return Knowledge{}, errors.New("WeKnora returned an unsuccessful upload response") + } + return response.Data, nil +} + +func (c *Client) ListKnowledge(ctx context.Context, knowledgeBaseID string, page, pageSize int) ([]Knowledge, int64, error) { + if !validPathSegment(knowledgeBaseID) || !validPage(page, pageSize) { + return nil, 0, errors.New("invalid knowledge list request") + } + query := url.Values{} + query.Set("page", fmt.Sprint(page)) + query.Set("page_size", fmt.Sprint(pageSize)) + var response knowledgeListResponse + if err := c.doJSONQuery(ctx, http.MethodGet, "/api/v1/knowledge-bases/"+knowledgeBaseID+"/knowledge", query, nil, &response); err != nil { + return nil, 0, err + } + if !response.Success || response.Total < 0 { + return nil, 0, errors.New("WeKnora returned an unsuccessful knowledge list") + } + return response.Data, response.Total, nil +} + +func (c *Client) ListChunks(ctx context.Context, knowledgeID string, page, pageSize int, chunkTypes []string) ([]Chunk, int64, error) { + if !validPathSegment(knowledgeID) || !validPage(page, pageSize) { + return nil, 0, errors.New("invalid chunk list request") + } + query := url.Values{} + query.Set("page", fmt.Sprint(page)) + query.Set("page_size", fmt.Sprint(pageSize)) + for _, chunkType := range chunkTypes { + if !validPathSegment(chunkType) { + return nil, 0, errors.New("invalid chunk type") + } + query.Add("chunk_type", chunkType) + } + var response chunkListResponse + if err := c.doJSONQuery(ctx, http.MethodGet, "/api/v1/chunks/"+knowledgeID, query, nil, &response); err != nil { + return nil, 0, err + } + if !response.Success || response.Total < 0 { + return nil, 0, errors.New("WeKnora returned an unsuccessful chunk list") + } + return response.Data, response.Total, nil +} + +func (c *Client) SearchKnowledge(ctx context.Context, request SearchRequest) ([]SearchResult, error) { + if strings.TrimSpace(request.Query) == "" || len(request.KnowledgeIDs) == 0 { + return nil, errors.New("search query and knowledge IDs are required") + } + for _, knowledgeID := range request.KnowledgeIDs { + if !validPathSegment(knowledgeID) { + return nil, errors.New("invalid search knowledge ID") + } + } + body, err := json.Marshal(request) + if err != nil { + return nil, errors.New("encode WeKnora search request") + } + var response searchResponse + if err := c.doJSON(ctx, http.MethodPost, "/api/v1/knowledge-search", bytes.NewReader(body), &response); err != nil { + return nil, err + } + if !response.Success { + return nil, errors.New("WeKnora returned an unsuccessful search response") + } + return response.Data, nil +} + +func writeMultipartUpload(writer *multipart.Writer, request UploadRequest) error { + part, err := writer.CreateFormFile("file", request.FileName) + if err != nil { + return err + } + if _, err := io.Copy(part, request.Content); err != nil { + return err + } + if request.Metadata != nil { + value, err := json.Marshal(request.Metadata) + if err != nil { + return err + } + if err := writer.WriteField("metadata", string(value)); err != nil { + return err + } + } + if err := writer.WriteField("fileName", request.FileName); err != nil { + return err + } + if request.Channel != "" { + if err := writer.WriteField("channel", request.Channel); err != nil { + return err + } + } + if request.ProcessConfig != nil { + value, err := json.Marshal(request.ProcessConfig) + if err != nil { + return err + } + if err := writer.WriteField("process_config", string(value)); err != nil { + return err + } + } + return nil +} + +func validPathSegment(value string) bool { + return strings.TrimSpace(value) != "" && !strings.ContainsAny(value, "/?#\\\x00") +} + +func validPage(page, pageSize int) bool { + return page > 0 && pageSize > 0 && pageSize <= 100 +} + +func (c *Client) endpoint(path string) string { + endpoint := *c.baseURL + endpoint.Path = strings.TrimRight(c.baseURL.Path, "/") + path + return endpoint.String() +} + +func (c *Client) doJSON(ctx context.Context, method, path string, body io.Reader, target any) error { + return c.doJSONQuery(ctx, method, path, nil, body, target) +} + +func (c *Client) doJSONQuery(ctx context.Context, method, path string, query url.Values, body io.Reader, target any) error { + endpoint := c.endpoint(path) + if len(query) > 0 { + endpoint += "?" + query.Encode() + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, body) + if err != nil { + return errors.New("create WeKnora request") + } + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("X-API-Key", c.apiKey) + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send WeKnora request: %w", err) + } + return c.decodeResponse(resp, target) +} + +func (c *Client) decodeResponse(resp *http.Response, target any) error { + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + return &HTTPError{StatusCode: resp.StatusCode} + } + return c.decodeBody(resp, target) +} + +func (c *Client) decodeBody(resp *http.Response, target any) error { + defer resp.Body.Close() + limited := io.LimitReader(resp.Body, c.maxResponseBytes+1) + payload, err := io.ReadAll(limited) + if err != nil { + return errors.New("read WeKnora response") + } + if int64(len(payload)) > c.maxResponseBytes { + return errors.New("WeKnora response exceeds size limit") + } + if err := json.Unmarshal(payload, target); err != nil { + return errors.New("decode WeKnora response") + } + return nil +} diff --git a/internal/platform/weknora/client_test.go b/internal/platform/weknora/client_test.go new file mode 100644 index 0000000..e4be40d --- /dev/null +++ b/internal/platform/weknora/client_test.go @@ -0,0 +1,379 @@ +package weknora + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestClientGetKnowledgeUsesPinnedOriginAndAPIKey(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if got := r.Header.Get("X-API-Key"); got != "test-api-key" { + t.Fatalf("X-API-Key = %q", got) + } + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization must be absent, got %q", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"processing","file_hash":"abc","updated_at":"2026-09-04T01:00:00Z","metadata":{"source":"legacy","process_overrides":{"parser":"mineru"}}}}`)) + })) + defer server.Close() + + client, err := NewClient(ClientConfig{ + BaseURL: server.URL, + APIKey: "test-api-key", + Timeout: time.Second, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + knowledge, err := client.GetKnowledge(context.Background(), "knowledge-1") + if err != nil { + t.Fatalf("GetKnowledge() error = %v", err) + } + if knowledge.ID != "knowledge-1" || knowledge.KnowledgeBaseID != "kb-1" || knowledge.ParseStatus != "processing" || knowledge.FileHash != "abc" { + t.Fatalf("knowledge = %#v", knowledge) + } + if knowledge.Metadata["source"] != "legacy" { + t.Fatalf("metadata = %#v", knowledge.Metadata) + } +} + +func TestNewClientRequiresHTTPSExceptForLoopbackTests(t *testing.T) { + for _, rawURL := range []string{ + "http://example.com", + "ftp://example.com", + "https://user:password@example.com", + "https://example.com?redirect=other", + } { + if _, err := NewClient(ClientConfig{BaseURL: rawURL, APIKey: "test-api-key", Timeout: time.Second}); err == nil { + t.Fatalf("NewClient(%q) error = nil", rawURL) + } + } + if _, err := NewClient(ClientConfig{BaseURL: "https://example.com", APIKey: "test-api-key", Timeout: time.Second}); err != nil { + t.Fatalf("NewClient(https) error = %v", err) + } + if _, err := NewClient(ClientConfig{BaseURL: "http://127.0.0.1:8080", APIKey: "key", Timeout: time.Second}); err != nil { + t.Fatalf("NewClient(loopback) error = %v", err) + } +} + +func TestNewClientAllowsExplicitSingleLabelDockerHTTPOnly(t *testing.T) { + if _, err := NewClient(ClientConfig{ + BaseURL: "http://app:8080", + APIKey: "key", + Timeout: time.Second, + AllowDockerHTTP: true, + }); err != nil { + t.Fatalf("NewClient(explicit docker HTTP) error = %v", err) + } + + for _, rawURL := range []string{ + "http://app:8080", + "http://host.docker.internal:8080", + "http://10.0.0.5:8080", + "http://example.com:8080", + } { + allow := rawURL != "http://app:8080" + _, err := NewClient(ClientConfig{ + BaseURL: rawURL, + APIKey: "key", + Timeout: time.Second, + AllowDockerHTTP: allow, + }) + if err == nil { + t.Fatalf("NewClient(%q) error = nil, want insecure HTTP rejected", rawURL) + } + } +} + +func TestClientUploadFileStreamsExpectedMultipartFields(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/knowledge-bases/kb-1/knowledge/file" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if r.ContentLength != -1 { + t.Fatalf("ContentLength = %d, want streaming request", r.ContentLength) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("ParseMultipartForm() error = %v", err) + } + file, header, err := r.FormFile("file") + if err != nil { + t.Fatalf("FormFile() error = %v", err) + } + defer file.Close() + body, err := io.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if header.Filename != "safe.md" || string(body) != "document body" { + t.Fatalf("upload = filename %q body %q", header.Filename, body) + } + if r.FormValue("metadata") != `{"content_sha256":"abc","publication_id":"publication-1"}` { + t.Fatalf("metadata = %q", r.FormValue("metadata")) + } + if r.FormValue("fileName") != "safe.md" || r.FormValue("channel") != "governance" { + t.Fatalf("fileName/channel = %q/%q", r.FormValue("fileName"), r.FormValue("channel")) + } + if r.FormValue("process_config") != `{"graph_enabled":false}` { + t.Fatalf("process_config = %q", r.FormValue("process_config")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-new","knowledge_base_id":"kb-1","parse_status":"pending"}}`)) + })) + defer server.Close() + + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + knowledge, err := client.UploadFile(context.Background(), UploadRequest{ + KnowledgeBaseID: "kb-1", + FileName: "safe.md", + Content: bytes.NewBufferString("document body"), + Metadata: map[string]string{ + "publication_id": "publication-1", + "content_sha256": "abc", + }, + Channel: "governance", + ProcessConfig: map[string]any{"graph_enabled": false}, + }) + if err != nil { + t.Fatalf("UploadFile() error = %v", err) + } + if knowledge.ID != "knowledge-new" || knowledge.KnowledgeBaseID != "kb-1" || knowledge.ParseStatus != "pending" { + t.Fatalf("knowledge = %#v", knowledge) + } +} + +func TestClientUploadFileReturnsTypedConflictWithoutClaimingSuccess(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"success":false,"data":{"id":"candidate-1","knowledge_base_id":"kb-1"},"code":"duplicate"}`)) + })) + defer server.Close() + + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + knowledge, err := client.UploadFile(context.Background(), UploadRequest{ + KnowledgeBaseID: "kb-1", + FileName: "safe.md", + Content: bytes.NewBufferString("document body"), + }) + if !errors.Is(err, ErrConflict) { + t.Fatalf("UploadFile() error = %v, want ErrConflict", err) + } + if knowledge.ID != "candidate-1" { + t.Fatalf("candidate knowledge = %#v", knowledge) + } +} + +func TestClientDeleteKnowledgeUsesPinnedOriginAndAPIKey(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Path != "/api/v1/knowledge/knowledge-1" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if got := r.Header.Get("X-API-Key"); got != "test-api-key" { + t.Fatalf("X-API-Key = %q", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true}`)) + })) + defer server.Close() + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if err := client.DeleteKnowledge(context.Background(), "knowledge-1"); err != nil { + t.Fatalf("DeleteKnowledge() error = %v", err) + } + if err := client.DeleteKnowledge(context.Background(), "../knowledge-1"); err == nil { + t.Fatal("DeleteKnowledge() accepted an unsafe knowledge ID") + } +} + +func TestClientProvidesPaginatedInventoryChunkAndScopedSearchReads(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/knowledge-bases/kb-1/knowledge": + if r.Method != http.MethodGet || r.URL.Query().Get("page") != "2" || r.URL.Query().Get("page_size") != "50" { + t.Fatalf("knowledge list request = %s %s", r.Method, r.URL.RawQuery) + } + _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-1","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1"}}],"total":51,"page":2,"page_size":50}`)) + case "/api/v1/chunks/knowledge-1": + if r.Method != http.MethodGet || r.URL.Query().Get("page") != "1" || r.URL.Query().Get("page_size") != "100" { + t.Fatalf("chunk list request = %s %s", r.Method, r.URL.RawQuery) + } + types := r.URL.Query()["chunk_type"] + if len(types) != 2 || types[0] != "text" || types[1] != "image_ocr" { + t.Fatalf("chunk types = %#v", types) + } + _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"chunk-1","knowledge_id":"knowledge-1","chunk_type":"text"}],"total":1,"page":1,"page_size":100}`)) + case "/api/v1/knowledge-search": + if r.Method != http.MethodPost { + t.Fatalf("search method = %s", r.Method) + } + var request SearchRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + if request.Query != "unique question" || len(request.KnowledgeIDs) != 1 || request.KnowledgeIDs[0] != "knowledge-1" { + t.Fatalf("search request = %#v", request) + } + _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"chunk-1","knowledge_id":"knowledge-1","score":0.02,"match_type":2,"chunk_type":"text"}]}`)) + default: + t.Fatalf("unexpected path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + items, total, err := client.ListKnowledge(context.Background(), "kb-1", 2, 50) + if err != nil || total != 51 || len(items) != 1 || items[0].Metadata["publication_id"] != "publication-1" { + t.Fatalf("ListKnowledge() = %#v, %d, %v", items, total, err) + } + chunks, total, err := client.ListChunks(context.Background(), "knowledge-1", 1, 100, []string{"text", "image_ocr"}) + if err != nil || total != 1 || len(chunks) != 1 || chunks[0].KnowledgeID != "knowledge-1" { + t.Fatalf("ListChunks() = %#v, %d, %v", chunks, total, err) + } + results, err := client.SearchKnowledge(context.Background(), SearchRequest{Query: "unique question", KnowledgeIDs: []string{"knowledge-1"}}) + if err != nil || len(results) != 1 || results[0].KnowledgeID != "knowledge-1" || results[0].Score != 0.02 { + t.Fatalf("SearchKnowledge() = %#v, %v", results, err) + } +} + +func TestClientRejectsRedirectWithoutForwardingAPIKey(t *testing.T) { + var targetHits atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetHits.Add(1) + if r.Header.Get("X-API-Key") != "" { + t.Error("redirect target received X-API-Key") + } + w.WriteHeader(http.StatusOK) + })) + defer target.Close() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer origin.Close() + + client, err := NewClient(ClientConfig{BaseURL: origin.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if _, err := client.GetKnowledge(context.Background(), "knowledge-1"); err == nil { + t.Fatal("GetKnowledge() error = nil, want redirect rejected") + } + if targetHits.Load() != 0 { + t.Fatalf("redirect target hits = %d", targetHits.Load()) + } +} + +func TestClientBoundsAndRedactsRemoteResponses(t *testing.T) { + secret := "remote-secret-that-must-not-leak" + tests := []struct { + name string + body string + code int + }{ + {name: "error body", code: http.StatusInternalServerError, body: secret}, + {name: "malformed body", code: http.StatusOK, body: "{" + secret}, + {name: "oversized body", code: http.StatusOK, body: strings.Repeat("x", 65)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.code) + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second, MaxResponseBytes: 64}) + if err != nil { + t.Fatal(err) + } + _, err = client.GetKnowledge(context.Background(), "knowledge-1") + if err == nil { + t.Fatal("GetKnowledge() error = nil") + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("error leaked remote body: %v", err) + } + }) + } +} + +func TestClientClassifiesHTTPFailuresWithoutLeakingBody(t *testing.T) { + tests := []struct { + status int + retryable bool + }{ + {status: http.StatusUnauthorized, retryable: false}, + {status: http.StatusForbidden, retryable: false}, + {status: http.StatusTooManyRequests, retryable: true}, + {status: http.StatusInternalServerError, retryable: true}, + } + for _, tt := range tests { + t.Run(http.StatusText(tt.status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + _, _ = w.Write([]byte("sensitive remote detail")) + })) + defer server.Close() + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + _, err = client.GetKnowledge(context.Background(), "knowledge-1") + var httpErr *HTTPError + if !errors.As(err, &httpErr) || httpErr.StatusCode != tt.status { + t.Fatalf("GetKnowledge() error = %T %v", err, err) + } + if IsRetryable(err) != tt.retryable { + t.Fatalf("IsRetryable() = %v, want %v", IsRetryable(err), tt.retryable) + } + if strings.Contains(err.Error(), "sensitive remote detail") { + t.Fatalf("error leaked remote body: %v", err) + } + }) + } +} + +func TestClientClassifiesTimeoutAsRetryable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(time.Second): + } + })) + defer server.Close() + client, err := NewClient(ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: 10 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + _, err = client.GetKnowledge(context.Background(), "knowledge-1") + if err == nil || !IsRetryable(err) { + t.Fatalf("GetKnowledge() error = %v, want retryable timeout", err) + } +} diff --git a/internal/platform/weknora/secret.go b/internal/platform/weknora/secret.go new file mode 100644 index 0000000..14632bd --- /dev/null +++ b/internal/platform/weknora/secret.go @@ -0,0 +1,49 @@ +package weknora + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + "syscall" +) + +const maxAPIKeyFileBytes = 16 * 1024 + +func ReadAPIKeyFile(path string) (string, error) { + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return "", fmt.Errorf("open API key file: %w", err) + } + file := os.NewFile(uintptr(fd), "test-weknora-api-key") + if file == nil { + _ = syscall.Close(fd) + return "", errors.New("open API key file: invalid file descriptor") + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return "", fmt.Errorf("inspect API key file: %w", err) + } + if !info.Mode().IsRegular() { + return "", errors.New("API key file must be a regular file") + } + if info.Mode().Perm() != 0o600 { + return "", errors.New("API key file permissions must be 0600") + } + + body, err := io.ReadAll(io.LimitReader(file, maxAPIKeyFileBytes+1)) + if err != nil { + return "", fmt.Errorf("read API key file: %w", err) + } + if len(body) > maxAPIKeyFileBytes { + return "", errors.New("API key file is too large") + } + key := strings.TrimSpace(string(body)) + if key == "" { + return "", errors.New("API key file is empty") + } + return key, nil +} diff --git a/internal/platform/weknora/secret_test.go b/internal/platform/weknora/secret_test.go new file mode 100644 index 0000000..4c4a1b3 --- /dev/null +++ b/internal/platform/weknora/secret_test.go @@ -0,0 +1,55 @@ +package weknora + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadAPIKeyFileRequiresPrivateRegularFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(path, []byte("test-secret-value\n"), 0o600); err != nil { + t.Fatal(err) + } + + key, err := ReadAPIKeyFile(path) + if err != nil { + t.Fatalf("ReadAPIKeyFile() error = %v", err) + } + if key != "test-secret-value" { + t.Fatalf("key = %q", key) + } +} + +func TestReadAPIKeyFileRejectsUnsafeInputsWithoutLeakingContent(t *testing.T) { + dir := t.TempDir() + secret := "secret-value-that-must-not-leak" + valid := filepath.Join(dir, "valid") + if err := os.WriteFile(valid, []byte(secret), 0o600); err != nil { + t.Fatal(err) + } + groupReadable := filepath.Join(dir, "group-readable") + if err := os.WriteFile(groupReadable, []byte(secret), 0o640); err != nil { + t.Fatal(err) + } + symlink := filepath.Join(dir, "symlink") + if err := os.Symlink(valid, symlink); err != nil { + t.Fatal(err) + } + empty := filepath.Join(dir, "empty") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatal(err) + } + + for _, path := range []string{groupReadable, symlink, empty, dir} { + _, err := ReadAPIKeyFile(path) + if err == nil { + t.Fatalf("ReadAPIKeyFile(%q) error = nil", filepath.Base(path)) + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("error leaked secret: %v", err) + } + } +} diff --git a/internal/platform/weknora/types.go b/internal/platform/weknora/types.go new file mode 100644 index 0000000..8a0a4a0 --- /dev/null +++ b/internal/platform/weknora/types.go @@ -0,0 +1,76 @@ +package weknora + +import ( + "io" + "time" +) + +type UploadRequest struct { + KnowledgeBaseID string + FileName string + Content io.Reader + Metadata map[string]string + Channel string + ProcessConfig map[string]any +} + +type Knowledge struct { + ID string `json:"id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + Title string `json:"title"` + ParseStatus string `json:"parse_status"` + FileName string `json:"file_name"` + FileSize int64 `json:"file_size"` + FileHash string `json:"file_hash"` + Metadata map[string]any `json:"metadata"` + UpdatedAt time.Time `json:"updated_at"` + ErrorMessage string `json:"error_message"` +} + +type Chunk struct { + ID string `json:"id"` + KnowledgeID string `json:"knowledge_id"` + ChunkType string `json:"chunk_type"` +} + +type SearchRequest struct { + Query string `json:"query"` + KnowledgeIDs []string `json:"knowledge_ids"` +} + +type SearchResult struct { + ID string `json:"id"` + Content string `json:"content"` + KnowledgeID string `json:"knowledge_id"` + Score float64 `json:"score"` + MatchType int `json:"match_type"` + ChunkType string `json:"chunk_type"` +} + +type knowledgeResponse struct { + Success bool `json:"success"` + Data Knowledge `json:"data"` + Code string `json:"code"` + Message string `json:"message"` +} + +type knowledgeListResponse struct { + Success bool `json:"success"` + Data []Knowledge `json:"data"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} + +type chunkListResponse struct { + Success bool `json:"success"` + Data []Chunk `json:"data"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} + +type searchResponse struct { + Success bool `json:"success"` + Data []SearchResult `json:"data"` +} diff --git a/internal/releaseadapter/v071/archive.go b/internal/releaseadapter/v071/archive.go new file mode 100644 index 0000000..dd3a659 --- /dev/null +++ b/internal/releaseadapter/v071/archive.go @@ -0,0 +1,197 @@ +package v071 + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "syscall" + + "knowledge-governance/internal/releasebundle" +) + +type BundleArchive struct { + root string + maxBytes int64 +} + +func NewBundleArchive(root string, maxBytes int64) (*BundleArchive, error) { + if !filepath.IsAbs(root) || maxBytes < 1024 { + return nil, errors.New("invalid bundle archive configuration") + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o700 { + return nil, errors.New("bundle archive root must be a real 0700 directory") + } + resolved, err := filepath.EvalSymlinks(root) + if err != nil || resolved != filepath.Clean(root) { + return nil, errors.New("bundle archive root must not contain symlinks") + } + return &BundleArchive{root: root, maxBytes: maxBytes}, nil +} + +func (archive *BundleArchive) Save(bundle *DocumentBundle) error { + if err := validateDocumentBundle(bundle); err != nil { + return err + } + payload, err := json.Marshal(bundle) + if err != nil || int64(len(payload)) > archive.maxBytes { + return errors.New("release bundle exceeds archive limit") + } + finalPath := archive.bundlePath(bundle.Manifest.ReleaseRevision) + existing, loadErr := archive.Load(bundle.Manifest.ReleaseRevision) + switch { + case loadErr == nil: + if existing.Manifest.BundleSHA256 == bundle.Manifest.BundleSHA256 { + return nil + } + // A retry of the same release revision re-seals the bundle with a new + // CreatedAt, so a differing digest is expected. The revision belongs to + // this publication and only this worker may write the archive (0700 + // root), so replace the stale file instead of dead-locking the retry. + if existing.Manifest.PublicationID != bundle.Manifest.PublicationID || + existing.Manifest.Contract.KnowledgeID != bundle.Manifest.Contract.KnowledgeID || + existing.Manifest.Contract.KnowledgeBaseID != bundle.Manifest.Contract.KnowledgeBaseID { + return errors.New("release revision archive collision") + } + if removeErr := os.Remove(finalPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return errors.New("replace release bundle archive") + } + case !errors.Is(loadErr, os.ErrNotExist): + return loadErr + } + temporary, err := os.CreateTemp(archive.root, ".bundle-*.tmp") + if err != nil { + return errors.New("create release bundle archive") + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + temporary.Close() + return errors.New("secure release bundle archive") + } + if _, err := temporary.Write(payload); err != nil { + temporary.Close() + return errors.New("write release bundle archive") + } + if err := temporary.Sync(); err != nil { + temporary.Close() + return errors.New("sync release bundle archive") + } + if err := temporary.Close(); err != nil { + return errors.New("close release bundle archive") + } + if err := os.Link(temporaryPath, finalPath); err != nil { + if errors.Is(err, os.ErrExist) { + existing, loadErr := archive.Load(bundle.Manifest.ReleaseRevision) + if loadErr == nil && existing.Manifest.BundleSHA256 == bundle.Manifest.BundleSHA256 { + return nil + } + return errors.New("release revision archive collision") + } + return errors.New("publish release bundle archive") + } + directory, err := os.Open(archive.root) + if err != nil { + return errors.New("open release bundle archive root") + } + syncErr := directory.Sync() + closeErr := directory.Close() + if syncErr != nil || closeErr != nil { + return errors.New("sync release bundle archive root") + } + return nil +} + +func (archive *BundleArchive) Load(releaseRevision string) (*DocumentBundle, error) { + if !validSearchIdentifier(releaseRevision) { + return nil, errors.New("invalid release revision") + } + filePath := archive.bundlePath(releaseRevision) + fd, err := syscall.Open(filePath, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + return nil, errors.New("open release bundle archive") + } + file := os.NewFile(uintptr(fd), "release-bundle") + if file == nil { + syscall.Close(fd) + return nil, errors.New("open release bundle archive") + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() > archive.maxBytes { + return nil, errors.New("invalid release bundle archive file") + } + payload, err := io.ReadAll(io.LimitReader(file, archive.maxBytes+1)) + if err != nil || int64(len(payload)) > archive.maxBytes { + return nil, errors.New("read release bundle archive") + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var bundle DocumentBundle + if err := decoder.Decode(&bundle); err != nil { + return nil, errors.New("decode release bundle archive") + } + if decoder.More() { + return nil, errors.New("decode release bundle archive") + } + if bundle.Manifest.ReleaseRevision != releaseRevision { + return nil, errors.New("release bundle revision mismatch") + } + if err := validateDocumentBundle(&bundle); err != nil { + return nil, err + } + return &bundle, nil +} + +func (archive *BundleArchive) bundlePath(releaseRevision string) string { + return filepath.Join(archive.root, releaseRevision+".bundle.json") +} + +func validateDocumentBundle(bundle *DocumentBundle) error { + if bundle == nil || !validSearchIdentifier(bundle.Manifest.ReleaseRevision) { + return errors.New("invalid release bundle") + } + storedSHA := bundle.Manifest.BundleSHA256 + sealed, err := releaseManifestReseal(bundle.Manifest) + if err != nil || sealed.BundleSHA256 != storedSHA { + return errors.New("release bundle manifest digest mismatch") + } + if len(bundle.Postgres.Tables) != len(bundle.Manifest.Tables) { + return errors.New("release bundle PostgreSQL manifest mismatch") + } + for _, expected := range bundle.Manifest.Tables { + actual := bundle.Postgres.Table(expected.Name) + if actual.RowCount != len(actual.Rows) || actual.RowCount != expected.RowCount || actual.SHA256 != expected.SHA256 || newTableSnapshot(actual.Name, actual.Rows).SHA256 != actual.SHA256 { + return errors.New("release bundle PostgreSQL digest mismatch") + } + } + searchSHA, err := searchDocumentsDigest(bundle.Search.Documents) + if err != nil || searchSHA != bundle.Search.DocumentsSHA256 || bundle.Search.DocumentsSHA256 != bundle.Manifest.Search.SHA256 || len(bundle.Search.Documents) != bundle.Manifest.Search.DocumentCount || bundle.Search.MappingSHA256 != bundle.Manifest.Contract.ElasticsearchMappingSHA256 { + return errors.New("release bundle Elasticsearch digest mismatch") + } + if bundle.Objects.SHA256 != objectSnapshotDigest(bundle.Objects.Objects) || bundle.Objects.SHA256 != bundle.Manifest.Assets.SHA256 || len(bundle.Objects.Objects) != bundle.Manifest.Assets.ObjectCount || bundle.Objects.TotalBytes != bundle.Manifest.Assets.TotalBytes { + return errors.New("release bundle object digest mismatch") + } + var total int64 + for _, object := range bundle.Objects.Objects { + if object.Size != int64(len(object.Data)) || object.SHA256 != shaBytes(object.Data) { + return errors.New("release bundle object content mismatch") + } + total += object.Size + } + if total != bundle.Objects.TotalBytes { + return errors.New("release bundle object total mismatch") + } + return nil +} + +func releaseManifestReseal(manifest releasebundle.Manifest) (releasebundle.Manifest, error) { + manifest.BundleSHA256 = "" + return releasebundle.Seal(manifest) +} diff --git a/internal/releaseadapter/v071/archive_test.go b/internal/releaseadapter/v071/archive_test.go new file mode 100644 index 0000000..b73f0f9 --- /dev/null +++ b/internal/releaseadapter/v071/archive_test.go @@ -0,0 +1,155 @@ +package v071 + +import ( + "os" + "path/filepath" + "testing" + "time" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestBundleArchiveRoundTripUsesRestrictedAtomicFiles(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + bundle := testDocumentBundle(t) + archive, err := NewBundleArchive(root, 16*1024*1024) + if err != nil { + t.Fatal(err) + } + if err := archive.Save(bundle); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "review-v1.bundle.json") + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 || !info.Mode().IsRegular() { + t.Fatalf("mode=%v", info.Mode()) + } + loaded, err := archive.Load("review-v1") + if err != nil { + t.Fatal(err) + } + if loaded.Manifest.BundleSHA256 != bundle.Manifest.BundleSHA256 || len(loaded.Search.Documents) != 1 || loaded.Postgres.Table("chunks").RowCount != 1 { + t.Fatalf("loaded=%#v", loaded) + } + if err := archive.Save(bundle); err != nil { + t.Fatalf("idempotent save: %v", err) + } +} + +func TestBundleArchiveRejectsSymlinkRootAndRevisionCollision(t *testing.T) { + parent := t.TempDir() + realRoot := filepath.Join(parent, "real") + if err := os.Mkdir(realRoot, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(parent, "link") + if err := os.Symlink(realRoot, link); err != nil { + t.Fatal(err) + } + if _, err := NewBundleArchive(link, 1024); err == nil { + t.Fatal("symlink root accepted") + } + archive, err := NewBundleArchive(realRoot, 16*1024*1024) + if err != nil { + t.Fatal(err) + } + bundle := testDocumentBundle(t) + if err := archive.Save(bundle); err != nil { + t.Fatal(err) + } + // A retry re-seals the same publication with a new digest: it must replace + // cleanly rather than dead-lock the release revision forever. + retry := testDocumentBundle(t) + retryManifest := retry.Manifest + retryManifest.CreatedAt = retryManifest.CreatedAt.Add(time.Hour) + sealed, err := releasebundle.Seal(retryManifest) + if err != nil { + t.Fatal(err) + } + retry.Manifest = sealed + if retry.Manifest.BundleSHA256 == bundle.Manifest.BundleSHA256 { + t.Fatal("re-seal produced identical digest") + } + if err := archive.Save(retry); err != nil { + t.Fatalf("retry save rejected: %v", err) + } + // The same revision carrying a different publication identity is a real + // collision and must still fail closed. + foreign := testDocumentBundle(t) + foreignManifest := foreign.Manifest + foreignManifest.PublicationID = "tp-other" + foreign.Manifest, err = releasebundle.Seal(foreignManifest) + if err != nil { + t.Fatal(err) + } + if err := archive.Save(foreign); err == nil { + t.Fatal("cross-publication revision collision accepted") + } +} + +func TestBundleArchiveRejectsTrailingJSON(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + archive, err := NewBundleArchive(root, 16*1024*1024) + if err != nil { + t.Fatal(err) + } + if err := archive.Save(testDocumentBundle(t)); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(filepath.Join(root, "review-v1.bundle.json"), os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString(`{}`); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := archive.Load("review-v1"); err == nil { + t.Fatal("trailing JSON accepted") + } +} + +func testDocumentBundle(t *testing.T) *DocumentBundle { + t.Helper() + pg := PostgresSnapshot{Tables: []TableSnapshot{ + newTableSnapshot("knowledges", mustCanonicalRows(t, `{"id":"knowledge-safe","title":"synthetic release marker"}`)), + newTableSnapshot("chunks", mustCanonicalRows(t, `{"id":"chunk-safe"}`)), + }} + search := SearchSnapshot{Documents: []SearchDocument{{ + ID: "es-safe", Source: mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","content":"synthetic release marker"}`)[0], + }}} + search.DocumentsSHA256, _ = searchDocumentsDigest(search.Documents) + search.MappingSHA256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + objects := ObjectSnapshot{SHA256: objectSnapshotDigest(nil)} + exporter, err := NewDocumentExporter(DocumentExporterConfig{ + Contract: &stubSourceContract{contract: sourceContractFixture(search.MappingSHA256)}, + Postgres: &stubPGExporter{snapshot: pg}, Search: &stubSearchExporter{snapshot: search}, + Objects: &stubObjectExporter{snapshot: objects}, TenantID: 10000, EmbeddingModel: "bge-m3", Now: fixedArchiveTime, + }) + if err != nil { + t.Fatal(err) + } + value, err := exporter.Export(t.Context(), releasePlanForArchive()) + if err != nil { + t.Fatal(err) + } + return value.(*DocumentBundle) +} + +func fixedArchiveTime() time.Time { return time.Date(2026, 9, 6, 11, 0, 0, 0, time.UTC) } + +func releasePlanForArchive() releaseworkflow.PlanItem { + return releaseworkflow.PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "review-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal"} +} diff --git a/internal/releaseadapter/v071/contract_inspector.go b/internal/releaseadapter/v071/contract_inspector.go new file mode 100644 index 0000000..ebe5043 --- /dev/null +++ b/internal/releaseadapter/v071/contract_inspector.go @@ -0,0 +1,101 @@ +package v071 + +import ( + "context" + "errors" + "strings" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type targetContractDB interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +type elasticsearchMappingInspector interface { + MappingSHA256(ctx context.Context) (string, error) +} + +type TargetContractInspectorConfig struct { + DB targetContractDB + Mapping elasticsearchMappingInspector + KnowledgeBaseID string + TenantID uint64 + IndexName string +} + +type TargetContractInspector struct { + config TargetContractInspectorConfig +} + +const targetContractSQL = ` +SELECT migration.version, + migration.dirty, + knowledge_base.tenant_id, + model.name, + COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::INTEGER, 0), + COALESCE(vector_store.engine_type, ''), + COALESCE(vector_store.index_config->>'index_name', '') +FROM knowledge_bases AS knowledge_base +JOIN models AS model + ON model.id = knowledge_base.embedding_model_id + AND model.deleted_at IS NULL +JOIN schema_migrations AS migration ON TRUE +LEFT JOIN vector_stores AS vector_store + ON vector_store.id = knowledge_base.vector_store_id + AND vector_store.deleted_at IS NULL +WHERE knowledge_base.id = $1 + AND knowledge_base.deleted_at IS NULL +` + +func NewTargetContractInspector(config TargetContractInspectorConfig) (*TargetContractInspector, error) { + config.KnowledgeBaseID = strings.TrimSpace(config.KnowledgeBaseID) + config.IndexName = strings.TrimSpace(config.IndexName) + if config.DB == nil || config.Mapping == nil || config.TenantID == 0 || !validSearchIdentifier(config.KnowledgeBaseID) || !validSearchIdentifier(config.IndexName) { + return nil, errors.New("invalid target contract inspector configuration") + } + return &TargetContractInspector{config: config}, nil +} + +func (inspector *TargetContractInspector) Contract(ctx context.Context, item releaseworkflow.PlanItem) (releasebundle.TargetContract, error) { + if item.TestKnowledgeBaseID != inspector.config.KnowledgeBaseID { + return releasebundle.TargetContract{}, errors.New("release plan knowledge base does not match target contract") + } + var ( + migrationVersion int + dirty bool + tenantID uint64 + embeddingModel string + dimension int + engineType string + indexName string + ) + if err := inspector.config.DB.QueryRow(ctx, targetContractSQL, inspector.config.KnowledgeBaseID).Scan( + &migrationVersion, &dirty, &tenantID, &embeddingModel, &dimension, &engineType, &indexName, + ); err != nil { + return releasebundle.TargetContract{}, errors.New("read target WeKnora contract") + } + if migrationVersion != releasebundle.WeKnoraMigrationV071 || dirty || tenantID != inspector.config.TenantID || + !validSearchIdentifier(embeddingModel) || dimension < 1 || + (engineType != "" && engineType != "elasticsearch") || + (indexName != "" && indexName != inspector.config.IndexName) { + return releasebundle.TargetContract{}, errors.New("target WeKnora contract mismatch") + } + mappingSHA256, err := inspector.config.Mapping.MappingSHA256(ctx) + if err != nil || !validLowerSHA256(mappingSHA256) { + return releasebundle.TargetContract{}, errors.New("read target Elasticsearch mapping contract") + } + return releasebundle.TargetContract{ + WeKnoraVersion: releasebundle.WeKnoraVersionV071, + MigrationVersion: releasebundle.WeKnoraMigrationV071, + TenantID: tenantID, + KnowledgeBaseID: inspector.config.KnowledgeBaseID, + RetrieverEngine: "elasticsearch_v8", + EmbeddingModel: embeddingModel, + EmbeddingDimension: dimension, + ElasticsearchMappingSHA256: mappingSHA256, + }, nil +} diff --git a/internal/releaseadapter/v071/contract_inspector_test.go b/internal/releaseadapter/v071/contract_inspector_test.go new file mode 100644 index 0000000..29e64fb --- /dev/null +++ b/internal/releaseadapter/v071/contract_inspector_test.go @@ -0,0 +1,87 @@ +package v071 + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5" +) + +func TestTargetContractInspectorReadsOnlyCompatibilityFields(t *testing.T) { + db := &contractDB{row: contractRow{values: []any{74, false, uint64(10000), "bge-m3", 1024, "elasticsearch", "chunks"}}} + mapping := &mappingInspector{sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + inspector, err := NewTargetContractInspector(TargetContractInspectorConfig{DB: db, Mapping: mapping, KnowledgeBaseID: "kb-safe", TenantID: 10000, IndexName: "chunks"}) + if err != nil { + t.Fatal(err) + } + contract, err := inspector.Contract(context.Background(), releasePlanForArchive()) + if err != nil { + t.Fatal(err) + } + if contract.WeKnoraVersion != "v0.7.1" || contract.MigrationVersion != 74 || contract.KnowledgeBaseID != "kb-safe" || contract.TenantID != 10000 || contract.EmbeddingModel != "bge-m3" || contract.EmbeddingDimension != 1024 || contract.RetrieverEngine != "elasticsearch_v8" || contract.ElasticsearchMappingSHA256 != mapping.sha { + t.Fatalf("contract=%#v", contract) + } + query := strings.ToLower(db.query) + for _, forbidden := range []string{"connection_config", "password", "api_key"} { + if strings.Contains(query, forbidden) { + t.Fatalf("query exposes %s: %s", forbidden, query) + } + } + for _, required := range []string{"schema_migrations", "knowledge_bases", "models", "embedding_parameters", "vector_stores", "index_config"} { + if !strings.Contains(query, required) { + t.Fatalf("query missing %s: %s", required, query) + } + } +} + +func TestTargetContractInspectorRejectsMismatchedPlanKB(t *testing.T) { + db := &contractDB{row: contractRow{values: []any{74, false, uint64(10000), "bge-m3", 1024, "elasticsearch", "chunks"}}} + inspector, _ := NewTargetContractInspector(TargetContractInspectorConfig{DB: db, Mapping: &mappingInspector{sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, KnowledgeBaseID: "kb-safe", TenantID: 10000, IndexName: "chunks"}) + plan := releasePlanForArchive() + plan.TestKnowledgeBaseID = "other-kb" + if _, err := inspector.Contract(context.Background(), plan); err == nil { + t.Fatal("mismatched KB accepted") + } +} + +type contractDB struct { + row contractRow + query string +} + +func (d *contractDB) QueryRow(_ context.Context, query string, args ...any) pgx.Row { + d.query = query + return d.row +} + +type contractRow struct { + values []any + err error +} + +func (r contractRow) Scan(dest ...any) error { + if r.err != nil { + return r.err + } + for i, value := range r.values { + switch target := dest[i].(type) { + case *int: + *target = value.(int) + case *bool: + *target = value.(bool) + case *uint64: + *target = value.(uint64) + case *string: + *target = value.(string) + } + } + return nil +} + +type mappingInspector struct { + sha string + err error +} + +func (m *mappingInspector) MappingSHA256(context.Context) (string, error) { return m.sha, m.err } diff --git a/internal/releaseadapter/v071/document_export.go b/internal/releaseadapter/v071/document_export.go new file mode 100644 index 0000000..38e8210 --- /dev/null +++ b/internal/releaseadapter/v071/document_export.go @@ -0,0 +1,149 @@ +package v071 + +import ( + "context" + "errors" + "strings" + "time" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type postgresSnapshotExporter interface { + Export(ctx context.Context, identity PostgresIdentity) (PostgresSnapshot, error) +} + +type searchSnapshotExporter interface { + Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) +} + +type objectSnapshotExporter interface { + Export(ctx context.Context, snapshot PostgresSnapshot) (ObjectSnapshot, error) +} + +type DocumentExporterConfig struct { + Contract targetContractInspector + Postgres postgresSnapshotExporter + Search searchSnapshotExporter + Objects objectSnapshotExporter + TenantID uint64 + EmbeddingModel string + Now func() time.Time +} + +type DocumentExporter struct { + config DocumentExporterConfig +} + +type DocumentBundle struct { + Manifest releasebundle.Manifest + Postgres PostgresSnapshot + Search SearchSnapshot + Objects ObjectSnapshot +} + +func (bundle *DocumentBundle) ReleaseManifest() releasebundle.Manifest { + if bundle == nil { + return releasebundle.Manifest{} + } + return bundle.Manifest +} + +func NewDocumentExporter(config DocumentExporterConfig) (*DocumentExporter, error) { + if config.Contract == nil || config.Postgres == nil || config.Search == nil || config.Objects == nil || config.TenantID == 0 || + !validSearchIdentifier(config.EmbeddingModel) || config.Now == nil { + return nil, errors.New("invalid v0.7.1 document exporter configuration") + } + return &DocumentExporter{config: config}, nil +} + +func (exporter *DocumentExporter) Export(ctx context.Context, item releaseworkflow.PlanItem) (releaseworkflow.ExportedBundle, error) { + if !validDocumentPlan(item) { + return nil, errors.New("invalid document release plan") + } + sourceContract, err := exporter.config.Contract.Contract(ctx, item) + if err != nil || sourceContract.WeKnoraVersion != releasebundle.WeKnoraVersionV071 || + sourceContract.MigrationVersion != releasebundle.WeKnoraMigrationV071 || sourceContract.TenantID != exporter.config.TenantID || + sourceContract.KnowledgeBaseID != item.TestKnowledgeBaseID || sourceContract.RetrieverEngine != "elasticsearch_v8" || + sourceContract.EmbeddingModel != exporter.config.EmbeddingModel || sourceContract.EmbeddingDimension != releasebundle.EmbeddingDimensionV071 { + return nil, errors.New("source WeKnora release contract mismatch") + } + postgresSnapshot, err := exporter.config.Postgres.Export(ctx, PostgresIdentity{ + TenantID: exporter.config.TenantID, KnowledgeBaseID: item.TestKnowledgeBaseID, KnowledgeID: item.KnowledgeID, + }) + if err != nil { + return nil, errors.New("export PostgreSQL document snapshot") + } + searchSnapshot, err := exporter.config.Search.Export(ctx, SearchIdentity{ + TenantID: exporter.config.TenantID, KnowledgeBaseID: item.TestKnowledgeBaseID, KnowledgeID: item.KnowledgeID, + }) + if err != nil { + return nil, errors.New("export Elasticsearch document snapshot") + } + if len(searchSnapshot.Documents) == 0 || !validLowerSHA256(searchSnapshot.MappingSHA256) || !validLowerSHA256(searchSnapshot.DocumentsSHA256) { + return nil, errors.New("Elasticsearch document snapshot is incomplete") + } + if searchSnapshot.MappingSHA256 != sourceContract.ElasticsearchMappingSHA256 { + return nil, errors.New("source Elasticsearch mapping contract mismatch") + } + objectSnapshot, err := exporter.config.Objects.Export(ctx, postgresSnapshot) + if err != nil { + return nil, errors.New("export object document snapshot") + } + if !validLowerSHA256(objectSnapshot.SHA256) { + return nil, errors.New("object document snapshot is incomplete") + } + tables := make([]releasebundle.TableDigest, 0, len(postgresSnapshot.Tables)) + for _, table := range postgresSnapshot.Tables { + if !validReleaseTable(table.Name) || table.RowCount != len(table.Rows) || !validLowerSHA256(table.SHA256) { + return nil, errors.New("PostgreSQL document snapshot is incomplete") + } + tables = append(tables, releasebundle.TableDigest{Name: table.Name, RowCount: table.RowCount, SHA256: table.SHA256}) + } + manifest, err := releasebundle.Seal(releasebundle.Manifest{ + FormatVersion: releasebundle.FormatVersionV1, ExecutionID: item.ReleaseExecutionID, + PublicationID: item.PublicationID, SourceItemVersionID: item.SourceItemVersionID, + SourceRevision: item.SourceRevision, ParseRevision: item.ParseRevision, ReleaseRevision: item.ReleaseRevision, + CreatedAt: exporter.config.Now().UTC(), + Contract: releasebundle.Contract{ + WeKnoraVersion: sourceContract.WeKnoraVersion, MigrationVersion: sourceContract.MigrationVersion, + TenantID: sourceContract.TenantID, KnowledgeBaseID: sourceContract.KnowledgeBaseID, KnowledgeID: item.KnowledgeID, + RetrieverEngine: sourceContract.RetrieverEngine, EmbeddingModel: sourceContract.EmbeddingModel, + EmbeddingDimension: sourceContract.EmbeddingDimension, ElasticsearchMappingSHA256: sourceContract.ElasticsearchMappingSHA256, + }, + Tables: tables, + Search: releasebundle.SearchDigest{DocumentCount: len(searchSnapshot.Documents), SHA256: searchSnapshot.DocumentsSHA256}, + Assets: releasebundle.AssetDigest{ObjectCount: len(objectSnapshot.Objects), TotalBytes: objectSnapshot.TotalBytes, SHA256: objectSnapshot.SHA256}, + }) + if err != nil { + return nil, errors.New("seal v0.7.1 document release bundle") + } + return &DocumentBundle{Manifest: manifest, Postgres: postgresSnapshot, Search: searchSnapshot, Objects: objectSnapshot}, nil +} + +func validDocumentPlan(item releaseworkflow.PlanItem) bool { + for _, value := range []string{item.ReleaseExecutionID, item.PublicationID, item.SourceItemVersionID, + item.TestKnowledgeBaseID, item.KnowledgeID} { + if !validSearchIdentifier(value) { + return false + } + } + for _, value := range []string{item.SourceRevision, item.ParseRevision, item.ReleaseRevision} { + if value == "" || len(value) > 128 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\t/?#\\") { + return false + } + } + return item.DecidedVisibility == "internal" || item.DecidedVisibility == "external" +} + +func validReleaseTable(name string) bool { + switch name { + case "knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations", "resources", "resource_bindings": + return true + default: + return false + } +} + +var _ releaseworkflow.BundleExporter = (*DocumentExporter)(nil) diff --git a/internal/releaseadapter/v071/document_export_test.go b/internal/releaseadapter/v071/document_export_test.go new file mode 100644 index 0000000..d487f6c --- /dev/null +++ b/internal/releaseadapter/v071/document_export_test.go @@ -0,0 +1,101 @@ +package v071 + +import ( + "context" + "strings" + "testing" + "time" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestDocumentExporterBuildsSealedThreeLaneBundle(t *testing.T) { + pg := PostgresSnapshot{Tables: []TableSnapshot{ + newTableSnapshot("knowledges", mustCanonicalRows(t, `{"id":"knowledge-safe"}`)), + newTableSnapshot("chunks", mustCanonicalRows(t, `{"id":"chunk-safe"}`)), + newTableSnapshot("knowledge_tag_relations", nil), + newTableSnapshot("resources", nil), + newTableSnapshot("resource_bindings", nil), + }} + search := SearchSnapshot{Documents: []SearchDocument{{ID: "es-safe", Source: mustCanonicalRows(t, `{"knowledge_id":"knowledge-safe"}`)[0]}}, MappingSHA256: strings.Repeat("a", 64), DocumentsSHA256: strings.Repeat("b", 64)} + objects := ObjectSnapshot{Objects: []AssetObject{}, TotalBytes: 0, SHA256: strings.Repeat("c", 64)} + exporter, err := NewDocumentExporter(DocumentExporterConfig{Contract: &stubSourceContract{contract: sourceContractFixture(strings.Repeat("a", 64))}, Postgres: &stubPGExporter{snapshot: pg}, Search: &stubSearchExporter{snapshot: search}, Objects: &stubObjectExporter{snapshot: objects}, TenantID: 10000, EmbeddingModel: "bge-m3", Now: func() time.Time { return time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + bundleValue, err := exporter.Export(context.Background(), releaseworkflow.PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "review-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal"}) + if err != nil { + t.Fatal(err) + } + bundle, ok := bundleValue.(*DocumentBundle) + if !ok { + t.Fatalf("bundle type=%T", bundleValue) + } + manifest := bundle.ReleaseManifest() + if len(manifest.BundleSHA256) != 64 || manifest.Contract.MigrationVersion != 74 || manifest.Contract.EmbeddingDimension != 1024 || manifest.Search.DocumentCount != 1 || len(manifest.Tables) != 5 { + t.Fatalf("manifest=%#v", manifest) + } + if bundle.Postgres.Table("chunks").RowCount != 1 || len(bundle.Search.Documents) != 1 || len(bundle.Objects.Objects) != 0 { + t.Fatalf("bundle=%#v", bundle) + } +} + +func TestDocumentExporterRejectsMissingSearchDocuments(t *testing.T) { + pg := PostgresSnapshot{Tables: []TableSnapshot{newTableSnapshot("knowledges", mustCanonicalRows(t, `{"id":"knowledge-safe"}`)), newTableSnapshot("chunks", mustCanonicalRows(t, `{"id":"chunk-safe"}`))}} + exporter, _ := NewDocumentExporter(DocumentExporterConfig{Contract: &stubSourceContract{contract: sourceContractFixture(strings.Repeat("a", 64))}, Postgres: &stubPGExporter{snapshot: pg}, Search: &stubSearchExporter{snapshot: SearchSnapshot{MappingSHA256: strings.Repeat("a", 64), DocumentsSHA256: strings.Repeat("b", 64)}}, Objects: &stubObjectExporter{snapshot: ObjectSnapshot{SHA256: strings.Repeat("c", 64)}}, TenantID: 10000, EmbeddingModel: "bge-m3", Now: time.Now}) + _, err := exporter.Export(context.Background(), releaseworkflow.PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "review-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal"}) + if err == nil { + t.Fatal("missing search documents accepted") + } +} + +func TestDocumentExporterRejectsSourceContractMappingDrift(t *testing.T) { + pg := PostgresSnapshot{Tables: []TableSnapshot{newTableSnapshot("knowledges", mustCanonicalRows(t, `{"id":"knowledge-safe"}`)), newTableSnapshot("chunks", mustCanonicalRows(t, `{"id":"chunk-safe"}`))}} + search := SearchSnapshot{Documents: []SearchDocument{{ID: "es-safe", Source: mustCanonicalRows(t, `{"knowledge_id":"knowledge-safe"}`)[0]}}, MappingSHA256: strings.Repeat("a", 64), DocumentsSHA256: strings.Repeat("b", 64)} + exporter, _ := NewDocumentExporter(DocumentExporterConfig{Contract: &stubSourceContract{contract: sourceContractFixture(strings.Repeat("c", 64))}, Postgres: &stubPGExporter{snapshot: pg}, Search: &stubSearchExporter{snapshot: search}, Objects: &stubObjectExporter{snapshot: ObjectSnapshot{SHA256: strings.Repeat("d", 64)}}, TenantID: 10000, EmbeddingModel: "bge-m3", Now: time.Now}) + _, err := exporter.Export(context.Background(), releaseworkflow.PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "review-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal"}) + if err == nil { + t.Fatal("source mapping drift accepted") + } +} + +type stubSourceContract struct { + contract releasebundle.TargetContract + err error +} + +func (s *stubSourceContract) Contract(context.Context, releaseworkflow.PlanItem) (releasebundle.TargetContract, error) { + return s.contract, s.err +} + +func sourceContractFixture(mapping string) releasebundle.TargetContract { + return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: mapping} +} + +type stubPGExporter struct { + snapshot PostgresSnapshot + err error +} + +func (s *stubPGExporter) Export(context.Context, PostgresIdentity) (PostgresSnapshot, error) { + return s.snapshot, s.err +} + +type stubSearchExporter struct { + snapshot SearchSnapshot + err error +} + +func (s *stubSearchExporter) Export(context.Context, SearchIdentity) (SearchSnapshot, error) { + return s.snapshot, s.err +} + +type stubObjectExporter struct { + snapshot ObjectSnapshot + err error +} + +func (s *stubObjectExporter) Export(context.Context, PostgresSnapshot) (ObjectSnapshot, error) { + return s.snapshot, s.err +} diff --git a/internal/releaseadapter/v071/document_target.go b/internal/releaseadapter/v071/document_target.go new file mode 100644 index 0000000..a07bd4a --- /dev/null +++ b/internal/releaseadapter/v071/document_target.go @@ -0,0 +1,209 @@ +package v071 + +import ( + "context" + "errors" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type documentArchive interface { + Save(bundle *DocumentBundle) error + Load(releaseRevision string) (*DocumentBundle, error) +} + +type targetContractInspector interface { + Contract(ctx context.Context, item releaseworkflow.PlanItem) (releasebundle.TargetContract, error) +} + +type postgresDocumentTarget interface { + Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error + Disable(ctx context.Context, identity PostgresIdentity) error +} + +type searchDocumentTarget interface { + Replace(ctx context.Context, identity SearchIdentity, snapshot SearchSnapshot) (SearchSnapshot, error) + Disable(ctx context.Context, identity SearchIdentity) error +} + +type objectDocumentTarget interface { + Stage(ctx context.Context, releaseRevision string, snapshot ObjectSnapshot) (ObjectStage, error) + Promote(ctx context.Context, stage ObjectStage) error +} + +type ProductionVerifier interface { + Verify(ctx context.Context, item releaseworkflow.PlanItem, bundle *DocumentBundle) error + VerifyDisabled(ctx context.Context, item releaseworkflow.PlanItem) error +} + +type DocumentTargetConfig struct { + Archive documentArchive + Contract targetContractInspector + Postgres postgresDocumentTarget + Search searchDocumentTarget + Objects objectDocumentTarget + Verifier ProductionVerifier +} + +type DocumentTarget struct { + config DocumentTargetConfig +} + +func NewDocumentTarget(config DocumentTargetConfig) (*DocumentTarget, error) { + if config.Archive == nil || config.Contract == nil || config.Postgres == nil || config.Search == nil || config.Objects == nil || config.Verifier == nil { + return nil, errors.New("v0.7.1 document target dependencies are required") + } + return &DocumentTarget{config: config}, nil +} + +func (target *DocumentTarget) Contract(ctx context.Context, item releaseworkflow.PlanItem) (releasebundle.TargetContract, error) { + contract, err := target.config.Contract.Contract(ctx, item) + if err != nil { + return releasebundle.TargetContract{}, errors.New("inspect target release contract") + } + return contract, nil +} + +func (target *DocumentTarget) Prepare(ctx context.Context, item releaseworkflow.PlanItem, exported releaseworkflow.ExportedBundle) (releaseworkflow.PreparedChange, error) { + bundle, ok := exported.(*DocumentBundle) + if !ok || validateDocumentBundle(bundle) != nil || bundle.Manifest.ReleaseRevision != item.ReleaseRevision { + return releaseworkflow.PreparedChange{}, errors.New("invalid v0.7.1 target bundle") + } + if err := target.config.Archive.Save(bundle); err != nil { + return releaseworkflow.PreparedChange{}, errors.New("archive v0.7.1 target bundle") + } + previousRevision := item.PreviousProductionRevision + if previousRevision == "" { + previousRevision = "absent" + } else if previousRevision != "absent" { + if _, err := target.config.Archive.Load(previousRevision); err != nil { + return releaseworkflow.PreparedChange{}, errors.New("prior release bundle is unavailable") + } + } + if _, err := target.config.Objects.Stage(ctx, bundle.Manifest.ReleaseRevision, bundle.Objects); err != nil { + return releaseworkflow.PreparedChange{}, errors.New("stage v0.7.1 release assets") + } + return releaseworkflow.PreparedChange{PreviousRevision: previousRevision, ProductionRevision: bundle.Manifest.ReleaseRevision}, nil +} + +func (target *DocumentTarget) Activate(ctx context.Context, item releaseworkflow.PlanItem, change releaseworkflow.PreparedChange) error { + bundle, err := target.config.Archive.Load(change.ProductionRevision) + if err != nil || bundle.Manifest.ReleaseRevision != item.ReleaseRevision { + return errors.New("load staged v0.7.1 release bundle") + } + stage, err := target.config.Objects.Stage(ctx, bundle.Manifest.ReleaseRevision, bundle.Objects) + if err != nil { + return errors.New("restage v0.7.1 release assets") + } + if err := target.config.Objects.Promote(ctx, stage); err != nil { + return errors.New("promote v0.7.1 release assets") + } + postgresIdentity, searchIdentity := targetIdentities(bundle.Manifest) + if err := target.config.Postgres.Replace(ctx, postgresIdentity, bundle.Postgres); err != nil { + return errors.New("activate v0.7.1 PostgreSQL release") + } + if _, err := target.config.Search.Replace(ctx, searchIdentity, bundle.Search); err != nil { + return errors.New("activate v0.7.1 Elasticsearch release") + } + return nil +} + +func (target *DocumentTarget) Verify(ctx context.Context, item releaseworkflow.PlanItem, change releaseworkflow.PreparedChange) error { + bundle, err := target.config.Archive.Load(change.ProductionRevision) + if err != nil { + return errors.New("load v0.7.1 release for verification") + } + if err := target.config.Verifier.Verify(ctx, item, bundle); err != nil { + return errors.New("verify v0.7.1 production release") + } + return nil +} + +func (target *DocumentTarget) Compensate(ctx context.Context, item releaseworkflow.PlanItem, change releaseworkflow.PreparedChange) error { + if change.PreviousRevision == "" || change.PreviousRevision == "absent" { + manifestBundle, err := target.config.Archive.Load(change.ProductionRevision) + if err != nil { + return errors.New("load failed first release bundle") + } + postgresIdentity, searchIdentity := targetIdentities(manifestBundle.Manifest) + if err := target.config.Postgres.Disable(ctx, postgresIdentity); err != nil { + return errors.New("disable failed first PostgreSQL release") + } + if err := target.config.Search.Disable(ctx, searchIdentity); err != nil { + return errors.New("disable failed first Elasticsearch release") + } + if err := target.config.Verifier.VerifyDisabled(ctx, item); err != nil { + return errors.New("verify disabled first release") + } + return nil + } + return target.restoreBundle(ctx, change.PreviousRevision) +} + +func (target *DocumentTarget) Restore(ctx context.Context, item releaseworkflow.PlanItem, revision string) (string, error) { + if revision == "absent" { + current, err := target.config.Archive.Load(item.ReleaseRevision) + if err != nil { + return "", errors.New("load current release for disable") + } + postgresIdentity, searchIdentity := targetIdentities(current.Manifest) + if err := target.config.Postgres.Disable(ctx, postgresIdentity); err != nil { + return "", errors.New("disable current PostgreSQL release") + } + if err := target.config.Search.Disable(ctx, searchIdentity); err != nil { + return "", errors.New("disable current Elasticsearch release") + } + return "absent", nil + } + if err := target.restoreBundle(ctx, revision); err != nil { + return "", err + } + return revision, nil +} + +func (target *DocumentTarget) VerifyRestored(ctx context.Context, item releaseworkflow.PlanItem, revision string) error { + if revision == "absent" { + if err := target.config.Verifier.VerifyDisabled(ctx, item); err != nil { + return errors.New("verify disabled production release") + } + return nil + } + bundle, err := target.config.Archive.Load(revision) + if err != nil { + return errors.New("load restored v0.7.1 release") + } + if err := target.config.Verifier.Verify(ctx, item, bundle); err != nil { + return errors.New("verify restored v0.7.1 release") + } + return nil +} + +func (target *DocumentTarget) restoreBundle(ctx context.Context, revision string) error { + bundle, err := target.config.Archive.Load(revision) + if err != nil { + return errors.New("load prior v0.7.1 release bundle") + } + stage, err := target.config.Objects.Stage(ctx, revision, bundle.Objects) + if err != nil { + return errors.New("stage prior v0.7.1 release assets") + } + if err := target.config.Objects.Promote(ctx, stage); err != nil { + return errors.New("promote prior v0.7.1 release assets") + } + postgresIdentity, searchIdentity := targetIdentities(bundle.Manifest) + if err := target.config.Postgres.Replace(ctx, postgresIdentity, bundle.Postgres); err != nil { + return errors.New("restore prior v0.7.1 PostgreSQL release") + } + if _, err := target.config.Search.Replace(ctx, searchIdentity, bundle.Search); err != nil { + return errors.New("restore prior v0.7.1 Elasticsearch release") + } + return nil +} + +func targetIdentities(manifest releasebundle.Manifest) (PostgresIdentity, SearchIdentity) { + postgresIdentity := PostgresIdentity{TenantID: manifest.Contract.TenantID, KnowledgeBaseID: manifest.Contract.KnowledgeBaseID, KnowledgeID: manifest.Contract.KnowledgeID} + return postgresIdentity, SearchIdentity(postgresIdentity) +} + +var _ releaseworkflow.SidecarTarget = (*DocumentTarget)(nil) diff --git a/internal/releaseadapter/v071/document_target_test.go b/internal/releaseadapter/v071/document_target_test.go new file mode 100644 index 0000000..83c18c1 --- /dev/null +++ b/internal/releaseadapter/v071/document_target_test.go @@ -0,0 +1,173 @@ +package v071 + +import ( + "context" + "os" + "reflect" + "testing" + + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestDocumentTargetCoordinatesPrepareActivateVerifyAndRollback(t *testing.T) { + root := t.TempDir() + _ = os.Chmod(root, 0o700) + archive, _ := NewBundleArchive(root, 16*1024*1024) + current := testDocumentBundle(t) + prior := cloneBundleWithRevision(t, current, "prod-v0") + if err := archive.Save(prior); err != nil { + t.Fatal(err) + } + calls := []string{} + target, err := NewDocumentTarget(DocumentTargetConfig{Archive: archive, Contract: &stubContractInspector{contract: targetContractForBundle(current)}, Postgres: &stubPostgresTarget{calls: &calls}, Search: &stubSearchTarget{calls: &calls}, Objects: &stubObjectTarget{calls: &calls}, Verifier: &stubProductionVerifier{calls: &calls}}) + if err != nil { + t.Fatal(err) + } + item := releasePlanForArchive() + item.PreviousProductionRevision = "prod-v0" + change, err := target.Prepare(context.Background(), item, current) + if err != nil { + t.Fatal(err) + } + if change.PreviousRevision != "prod-v0" || change.ProductionRevision != "review-v1" { + t.Fatalf("change=%#v", change) + } + if err := target.Activate(context.Background(), item, change); err != nil { + t.Fatal(err) + } + if err := target.Verify(context.Background(), item, change); err != nil { + t.Fatal(err) + } + if err := target.Compensate(context.Background(), item, change); err != nil { + t.Fatal(err) + } + if _, err := target.Restore(context.Background(), item, "prod-v0"); err != nil { + t.Fatal(err) + } + if err := target.VerifyRestored(context.Background(), item, "prod-v0"); err != nil { + t.Fatal(err) + } + want := []string{"object-stage:review-v1", "object-stage:review-v1", "object-promote:review-v1", "postgres-replace", "search-replace", "verify:review-v1", "object-stage:prod-v0", "object-promote:prod-v0", "postgres-replace", "search-replace", "object-stage:prod-v0", "object-promote:prod-v0", "postgres-replace", "search-replace", "verify:prod-v0"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("calls=%v\nwant=%v", calls, want) + } +} + +func TestDocumentTargetCompensatesFirstReleaseByDisableWithoutDelete(t *testing.T) { + root := t.TempDir() + _ = os.Chmod(root, 0o700) + archive, _ := NewBundleArchive(root, 16*1024*1024) + bundle := testDocumentBundle(t) + calls := []string{} + target, _ := NewDocumentTarget(DocumentTargetConfig{Archive: archive, Contract: &stubContractInspector{contract: targetContractForBundle(bundle)}, Postgres: &stubPostgresTarget{calls: &calls}, Search: &stubSearchTarget{calls: &calls}, Objects: &stubObjectTarget{calls: &calls}, Verifier: &stubProductionVerifier{calls: &calls}}) + item := releasePlanForArchive() + change, err := target.Prepare(context.Background(), item, bundle) + if err != nil { + t.Fatal(err) + } + if change.PreviousRevision != "absent" { + t.Fatalf("change=%#v", change) + } + if err := target.Compensate(context.Background(), item, change); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(calls, []string{"object-stage:review-v1", "postgres-disable", "search-disable", "verify-disabled"}) { + t.Fatalf("calls=%v", calls) + } +} + +type stubContractInspector struct { + contract releasebundle.TargetContract + err error +} + +func (s *stubContractInspector) Contract(context.Context, releaseworkflow.PlanItem) (releasebundle.TargetContract, error) { + return s.contract, s.err +} + +type stubPostgresTarget struct { + calls *[]string + err error +} + +func (s *stubPostgresTarget) Replace(_ context.Context, _ PostgresIdentity, _ PostgresSnapshot) error { + *s.calls = append(*s.calls, "postgres-replace") + return s.err +} +func (s *stubPostgresTarget) Disable(context.Context, PostgresIdentity) error { + *s.calls = append(*s.calls, "postgres-disable") + return s.err +} + +type stubSearchTarget struct { + calls *[]string + err error +} + +func (s *stubSearchTarget) Replace(_ context.Context, _ SearchIdentity, _ SearchSnapshot) (SearchSnapshot, error) { + *s.calls = append(*s.calls, "search-replace") + return SearchSnapshot{}, s.err +} +func (s *stubSearchTarget) Disable(context.Context, SearchIdentity) error { + *s.calls = append(*s.calls, "search-disable") + return s.err +} + +type stubObjectTarget struct { + calls *[]string + err error +} + +func (s *stubObjectTarget) Stage(_ context.Context, revision string, _ ObjectSnapshot) (ObjectStage, error) { + *s.calls = append(*s.calls, "object-stage:"+revision) + return ObjectStage{ReleaseRevision: revision}, s.err +} +func (s *stubObjectTarget) Promote(_ context.Context, stage ObjectStage) error { + *s.calls = append(*s.calls, "object-promote:"+stage.ReleaseRevision) + return s.err +} + +type stubProductionVerifier struct { + calls *[]string + err error +} + +func (s *stubProductionVerifier) Verify(_ context.Context, _ releaseworkflow.PlanItem, bundle *DocumentBundle) error { + *s.calls = append(*s.calls, "verify:"+bundle.Manifest.ReleaseRevision) + return s.err +} +func (s *stubProductionVerifier) VerifyDisabled(context.Context, releaseworkflow.PlanItem) error { + *s.calls = append(*s.calls, "verify-disabled") + return s.err +} + +func cloneBundleWithRevision(t *testing.T, bundle *DocumentBundle, revision string) *DocumentBundle { + t.Helper() + clone := *bundle + clone.Manifest = bundle.Manifest + clone.Manifest.ReleaseRevision = revision + clone.Manifest.BundleSHA256 = "" + sealed, err := releasebundle.Seal(clone.Manifest) + if err != nil { + t.Fatal(err) + } + clone.Manifest = sealed + return &clone +} +func targetContractForBundle(bundle *DocumentBundle) releasebundle.TargetContract { + c := bundle.Manifest.Contract + return releasebundle.TargetContract{WeKnoraVersion: c.WeKnoraVersion, MigrationVersion: c.MigrationVersion, TenantID: c.TenantID, KnowledgeBaseID: c.KnowledgeBaseID, RetrieverEngine: c.RetrieverEngine, EmbeddingModel: c.EmbeddingModel, EmbeddingDimension: c.EmbeddingDimension, ElasticsearchMappingSHA256: c.ElasticsearchMappingSHA256} +} +func revisionFromPostgres(snapshot PostgresSnapshot) string { + if snapshot.Table("chunks").Rows[0][0] == '{' { + return "review-v1" + } + return "unknown" +} +func revisionFromSearch(snapshot SearchSnapshot) string { + if len(snapshot.Documents) > 0 { + return "review-v1" + } + return "unknown" +} diff --git a/internal/releaseadapter/v071/object_export.go b/internal/releaseadapter/v071/object_export.go new file mode 100644 index 0000000..c425e12 --- /dev/null +++ b/internal/releaseadapter/v071/object_export.go @@ -0,0 +1,161 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "sort" + "strings" +) + +type ObjectReader interface { + Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) +} + +type ObjectExporterLimits struct { + MaxObjectBytes int64 + MaxTotalBytes int64 +} + +type AssetObject struct { + ResourceID string + Handle string + Provider string + PhysicalPath string + Size int64 + SHA256 string + Data []byte +} + +type ObjectSnapshot struct { + Objects []AssetObject + TotalBytes int64 + SHA256 string +} + +type ObjectExporter struct { + reader ObjectReader + limits ObjectExporterLimits +} + +func NewObjectExporter(reader ObjectReader, limits ObjectExporterLimits) (*ObjectExporter, error) { + if reader == nil { + return nil, errors.New("object reader is required") + } + if limits.MaxObjectBytes == 0 { + limits.MaxObjectBytes = 100 * 1024 * 1024 + } + if limits.MaxTotalBytes == 0 { + limits.MaxTotalBytes = 1024 * 1024 * 1024 + } + if limits.MaxObjectBytes < 1 || limits.MaxTotalBytes < limits.MaxObjectBytes { + return nil, errors.New("invalid object export limits") + } + return &ObjectExporter{reader: reader, limits: limits}, nil +} + +func (exporter *ObjectExporter) Export(ctx context.Context, postgresSnapshot PostgresSnapshot) (ObjectSnapshot, error) { + resources := postgresSnapshot.Table("resources") + objects := make([]AssetObject, 0, len(resources.Rows)) + var total int64 + for _, raw := range resources.Rows { + var resource struct { + ID string `json:"id"` + Handle string `json:"handle"` + Provider string `json:"provider"` + PhysicalPath string `json:"physical_path"` + Size int64 `json:"size"` + ContentHash string `json:"content_hash"` + } + if json.Unmarshal(raw, &resource) != nil || !validSearchIdentifier(resource.ID) || len(resource.Handle) != 22 || + !validObjectProvider(resource.Provider) || !validObjectKey(resource.PhysicalPath) || resource.Size < 0 || resource.Size > exporter.limits.MaxObjectBytes || + (resource.ContentHash != "" && !validLowerSHA256(resource.ContentHash)) { + return ObjectSnapshot{}, errors.New("invalid release resource record") + } + reader, err := exporter.reader.Open(ctx, resource.Provider, resource.PhysicalPath) + if err != nil { + return ObjectSnapshot{}, errors.New("read release object") + } + data, readErr := io.ReadAll(io.LimitReader(reader, exporter.limits.MaxObjectBytes+1)) + closeErr := reader.Close() + if readErr != nil || closeErr != nil || int64(len(data)) > exporter.limits.MaxObjectBytes { + return ObjectSnapshot{}, errors.New("read release object") + } + if resource.Size != int64(len(data)) { + return ObjectSnapshot{}, errors.New("release object size mismatch") + } + digest := sha256.Sum256(data) + contentSHA := hex.EncodeToString(digest[:]) + if resource.ContentHash != "" && resource.ContentHash != contentSHA { + return ObjectSnapshot{}, errors.New("release object digest mismatch") + } + total += int64(len(data)) + if total > exporter.limits.MaxTotalBytes { + return ObjectSnapshot{}, errors.New("release object total exceeds limit") + } + objects = append(objects, AssetObject{ + ResourceID: resource.ID, Handle: resource.Handle, Provider: resource.Provider, + PhysicalPath: resource.PhysicalPath, Size: int64(len(data)), SHA256: contentSHA, Data: data, + }) + } + sort.Slice(objects, func(left, right int) bool { return objects[left].Handle < objects[right].Handle }) + return ObjectSnapshot{Objects: objects, TotalBytes: total, SHA256: objectSnapshotDigest(objects)}, nil +} + +func objectSnapshotDigest(objects []AssetObject) string { + inventory := make([]struct { + Handle string `json:"handle"` + Key string `json:"key"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + }, 0, len(objects)) + for _, object := range objects { + inventory = append(inventory, struct { + Handle string `json:"handle"` + Key string `json:"key"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + }{Handle: object.Handle, Key: object.PhysicalPath, Size: object.Size, SHA256: object.SHA256}) + } + encoded, _ := json.Marshal(inventory) + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func shaBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +func validObjectProvider(provider string) bool { + switch provider { + case "minio", "oss", "s3", "cos": + return true + default: + return false + } +} + +func validObjectKey(value string) bool { + if strings.HasPrefix(value, "storage://") { + rest := strings.TrimPrefix(value, "storage://") + parts := strings.SplitN(rest, "/", 2) + return len(parts) == 2 && validSearchIdentifier(parts[0]) && validObjectLocator(parts[1]) + } + return validObjectLocator(value) +} + +func validLowerSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} diff --git a/internal/releaseadapter/v071/object_export_test.go b/internal/releaseadapter/v071/object_export_test.go new file mode 100644 index 0000000..15fa24c --- /dev/null +++ b/internal/releaseadapter/v071/object_export_test.go @@ -0,0 +1,76 @@ +package v071 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "testing" +) + +func TestObjectExporterReadsAndVerifiesReferencedResources(t *testing.T) { + resources := newTableSnapshot("resources", mustCanonicalRows(t, + `{"id":"res-1","handle":"ABCDEFGHIJKLMNOPQRSTUV","tenant_id":10000,"provider":"minio","physical_path":"storage://source-backend/10000/knowledge/source.txt","size":5,"content_hash":"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"}`, + `{"id":"res-2","handle":"abcdefghijklmnopqrstuv","tenant_id":10000,"provider":"oss","physical_path":"10000/knowledge/image.bin","size":3,"content_hash":"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}`, + )) + reader := &fakeObjectReader{objects: map[string][]byte{"minio|storage://source-backend/10000/knowledge/source.txt": []byte("hello"), "oss|10000/knowledge/image.bin": []byte("abc")}} + exporter, err := NewObjectExporter(reader, ObjectExporterLimits{MaxObjectBytes: 10, MaxTotalBytes: 20}) + if err != nil { + t.Fatal(err) + } + snapshot, err := exporter.Export(context.Background(), PostgresSnapshot{Tables: []TableSnapshot{resources}}) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Objects) != 2 || snapshot.TotalBytes != 8 || len(snapshot.SHA256) != 64 { + t.Fatalf("snapshot=%#v", snapshot) + } + if snapshot.Objects[0].PhysicalPath == "" || !bytes.Equal(snapshot.Objects[0].Data, []byte("hello")) { + t.Fatalf("object=%#v", snapshot.Objects[0]) + } + if len(reader.calls) != 2 { + t.Fatalf("calls=%v", reader.calls) + } +} + +func TestObjectExporterRejectsHashMismatchWithoutLeakingPhysicalPath(t *testing.T) { + resources := newTableSnapshot("resources", mustCanonicalRows(t, `{"id":"res-1","handle":"ABCDEFGHIJKLMNOPQRSTUV","tenant_id":10000,"provider":"oss","physical_path":"sensitive/private/key","size":5,"content_hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}`)) + exporter, _ := NewObjectExporter(&fakeObjectReader{objects: map[string][]byte{"oss|sensitive/private/key": []byte("hello")}}, ObjectExporterLimits{MaxObjectBytes: 10, MaxTotalBytes: 20}) + _, err := exporter.Export(context.Background(), PostgresSnapshot{Tables: []TableSnapshot{resources}}) + if err == nil || strings.Contains(err.Error(), "sensitive") || strings.Contains(err.Error(), "private") { + t.Fatalf("error=%v", err) + } +} + +func mustCanonicalRows(t *testing.T, values ...string) []json.RawMessage { + t.Helper() + rows := make([]json.RawMessage, 0, len(values)) + for _, value := range values { + raw, err := canonicalJSON([]byte(value)) + if err != nil { + t.Fatal(err) + } + rows = append(rows, raw) + } + return rows +} + +type fakeObjectReader struct { + objects map[string][]byte + calls []string + err error +} + +func (f *fakeObjectReader) Open(_ context.Context, provider, path string) (io.ReadCloser, error) { + f.calls = append(f.calls, provider+"|"+path) + if f.err != nil { + return nil, f.err + } + data, ok := f.objects[provider+"|"+path] + if !ok { + return nil, errors.New("not found") + } + return io.NopCloser(bytes.NewReader(data)), nil +} diff --git a/internal/releaseadapter/v071/object_import.go b/internal/releaseadapter/v071/object_import.go new file mode 100644 index 0000000..156d1ce --- /dev/null +++ b/internal/releaseadapter/v071/object_import.go @@ -0,0 +1,130 @@ +package v071 + +import ( + "bytes" + "context" + "errors" + "io" + "strings" +) + +type ObjectStore interface { + Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error + Stat(ctx context.Context, provider, path string) (size int64, sha256 string, err error) + Copy(ctx context.Context, provider, sourcePath, targetPath string) error +} + +type TargetObjectConfig struct { + StorageBackendID string + Provider string +} + +type StagedObject struct { + Handle string + StagingPath string + FinalPath string + Size int64 + SHA256 string +} + +type ObjectStage struct { + ReleaseRevision string + Provider string + Objects []StagedObject +} + +type ObjectImporter struct { + store ObjectStore + config TargetObjectConfig +} + +func NewObjectImporter(store ObjectStore, config TargetObjectConfig) (*ObjectImporter, error) { + config.StorageBackendID = strings.TrimSpace(config.StorageBackendID) + config.Provider = strings.TrimSpace(config.Provider) + if store == nil || !validSearchIdentifier(config.StorageBackendID) || !validObjectProvider(config.Provider) { + return nil, errors.New("invalid target object importer configuration") + } + return &ObjectImporter{store: store, config: config}, nil +} + +func (importer *ObjectImporter) Stage(ctx context.Context, releaseRevision string, snapshot ObjectSnapshot) (ObjectStage, error) { + if !validSearchIdentifier(releaseRevision) || !validLowerSHA256(snapshot.SHA256) || snapshot.SHA256 != objectSnapshotDigest(snapshot.Objects) { + return ObjectStage{}, errors.New("invalid object import snapshot") + } + var total int64 + stage := ObjectStage{ReleaseRevision: releaseRevision, Provider: importer.config.Provider, Objects: make([]StagedObject, 0, len(snapshot.Objects))} + for _, object := range snapshot.Objects { + if !validSearchIdentifier(object.ResourceID) || len(object.Handle) != 22 || object.Size != int64(len(object.Data)) || object.SHA256 != shaBytes(object.Data) || !validObjectKey(object.PhysicalPath) { + return ObjectStage{}, errors.New("invalid object import entry") + } + providerPath, ok := providerPathFromLocator(object.PhysicalPath) + if !ok { + return ObjectStage{}, errors.New("invalid object import locator") + } + total += object.Size + stagingPath := ".governance-release-stage/" + releaseRevision + "/" + providerPath + if !validObjectLocator(stagingPath) { + return ObjectStage{}, errors.New("invalid object staging locator") + } + if err := importer.store.Put(ctx, importer.config.Provider, stagingPath, bytes.NewReader(object.Data), object.Size); err != nil { + return ObjectStage{}, errors.New("write staged release object") + } + size, digest, err := importer.store.Stat(ctx, importer.config.Provider, stagingPath) + if err != nil || size != object.Size || digest != object.SHA256 { + return ObjectStage{}, errors.New("verify staged release object") + } + stage.Objects = append(stage.Objects, StagedObject{Handle: object.Handle, StagingPath: stagingPath, FinalPath: providerPath, Size: object.Size, SHA256: object.SHA256}) + } + if total != snapshot.TotalBytes { + return ObjectStage{}, errors.New("object import total mismatch") + } + return stage, nil +} + +func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage) error { + if stage.Provider != importer.config.Provider || !validSearchIdentifier(stage.ReleaseRevision) { + return errors.New("invalid object promotion stage") + } + for _, object := range stage.Objects { + if !validObjectLocator(object.StagingPath) || !validObjectLocator(object.FinalPath) || !validLowerSHA256(object.SHA256) || object.Size < 0 { + return errors.New("invalid object promotion entry") + } + if err := importer.store.Copy(ctx, importer.config.Provider, object.StagingPath, object.FinalPath); err != nil { + return errors.New("promote release object") + } + size, digest, err := importer.store.Stat(ctx, importer.config.Provider, object.FinalPath) + if err != nil || size != object.Size || digest != object.SHA256 { + return errors.New("verify promoted release object") + } + } + return nil +} + +func providerPathFromLocator(locator string) (string, bool) { + rest := locator + if strings.HasPrefix(locator, "storage://") { + rest = strings.TrimPrefix(locator, "storage://") + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) { + return "", false + } + rest = parts[1] + } else if !validObjectLocator(locator) { + return "", false + } + // WeKnora nests the provider locator inside the storage locator: + // "storage:///oss:///". S3 keys address the + // bucket configured for the store, so strip the inner scheme and bucket. + if separator := strings.Index(rest, "://"); separator >= 0 { + if !validSearchIdentifier(rest[:separator]) { + return "", false + } + schemeless := rest[separator+3:] + boundary := strings.Index(schemeless, "/") + if boundary < 0 || boundary == len(schemeless)-1 { + return "", false + } + rest = schemeless[boundary+1:] + } + return rest, validObjectLocator(rest) +} diff --git a/internal/releaseadapter/v071/object_import_test.go b/internal/releaseadapter/v071/object_import_test.go new file mode 100644 index 0000000..8564dd0 --- /dev/null +++ b/internal/releaseadapter/v071/object_import_test.go @@ -0,0 +1,106 @@ +package v071 + +import ( + "bytes" + "context" + "errors" + "io" + "reflect" + "testing" +) + +func TestProviderPathFromLocatorUnwrapsNestedSchemes(t *testing.T) { + cases := map[string]string{ + "storage://source-backend/10000/knowledge/source.txt": "10000/knowledge/source.txt", + "storage://78854bad-c2a8-4766-9748-e2814ecafbe6/oss://yzh-weknora-test/weknora/10000/545d3943/26f459da.md": "weknora/10000/545d3943/26f459da.md", + "oss://bucket/path/to.bin": "path/to.bin", + "plain/path.txt": "plain/path.txt", + } + invalid := []string{"", "storage://backend", "storage://bad id/x", "oss://bucket", "storage://backend/oss://bucket-only", "storage://backend/oss://bucket/"} + for locator, want := range cases { + got, ok := providerPathFromLocator(locator) + if !ok || got != want { + t.Fatalf("locator %q got %q ok=%v want %q", locator, got, ok, want) + } + } + for _, locator := range invalid { + if got, ok := providerPathFromLocator(locator); ok { + t.Fatalf("locator %q unexpectedly accepted as %q", locator, got) + } + } +} + +func TestObjectImporterStagesThenPromotesWithoutDelete(t *testing.T) { + data := []byte("hello") + snapshot := ObjectSnapshot{Objects: []AssetObject{{ResourceID: "res-1", Handle: "ABCDEFGHIJKLMNOPQRSTUV", Provider: "minio", PhysicalPath: "storage://source-backend/10000/knowledge/source.txt", Size: 5, SHA256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", Data: data}}, TotalBytes: 5} + snapshot.SHA256 = objectSnapshotDigest(snapshot.Objects) + store := &fakeObjectStore{objects: map[string][]byte{}} + importer, err := NewObjectImporter(store, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "oss"}) + if err != nil { + t.Fatal(err) + } + stage, err := importer.Stage(context.Background(), "release-v1", snapshot) + if err != nil { + t.Fatal(err) + } + if len(stage.Objects) != 1 || stage.Objects[0].FinalPath != "10000/knowledge/source.txt" { + t.Fatalf("stage=%#v", stage) + } + if err := importer.Promote(context.Background(), stage); err != nil { + t.Fatal(err) + } + wantCalls := []string{"put|oss|.governance-release-stage/release-v1/10000/knowledge/source.txt", "stat|oss|.governance-release-stage/release-v1/10000/knowledge/source.txt", "copy|oss|.governance-release-stage/release-v1/10000/knowledge/source.txt|10000/knowledge/source.txt", "stat|oss|10000/knowledge/source.txt"} + if !reflect.DeepEqual(store.calls, wantCalls) { + t.Fatalf("calls=%v want=%v", store.calls, wantCalls) + } + if !bytes.Equal(store.objects["oss|10000/knowledge/source.txt"], data) { + t.Fatal("final bytes differ") + } +} + +func TestObjectImporterRejectsTamperedSnapshotBeforeWrite(t *testing.T) { + store := &fakeObjectStore{objects: map[string][]byte{}} + importer, _ := NewObjectImporter(store, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "oss"}) + snapshot := ObjectSnapshot{Objects: []AssetObject{{ResourceID: "res-1", Handle: "ABCDEFGHIJKLMNOPQRSTUV", PhysicalPath: "key", Size: 5, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Data: []byte("hello")}}, TotalBytes: 5, SHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} + if _, err := importer.Stage(context.Background(), "release-v1", snapshot); err == nil { + t.Fatal("tampered snapshot accepted") + } + if len(store.calls) != 0 { + t.Fatalf("writes=%v", store.calls) + } +} + +type fakeObjectStore struct { + objects map[string][]byte + calls []string +} + +func (f *fakeObjectStore) Put(_ context.Context, provider, path string, reader io.Reader, size int64) error { + f.calls = append(f.calls, "put|"+provider+"|"+path) + data, err := io.ReadAll(reader) + if err != nil { + return err + } + if int64(len(data)) != size { + return errors.New("size") + } + f.objects[provider+"|"+path] = data + return nil +} +func (f *fakeObjectStore) Stat(_ context.Context, provider, path string) (int64, string, error) { + f.calls = append(f.calls, "stat|"+provider+"|"+path) + data, ok := f.objects[provider+"|"+path] + if !ok { + return 0, "", errors.New("not found") + } + return int64(len(data)), shaBytes(data), nil +} +func (f *fakeObjectStore) Copy(_ context.Context, provider, source, target string) error { + f.calls = append(f.calls, "copy|"+provider+"|"+source+"|"+target) + data, ok := f.objects[provider+"|"+source] + if !ok { + return errors.New("not found") + } + f.objects[provider+"|"+target] = append([]byte(nil), data...) + return nil +} diff --git a/internal/releaseadapter/v071/postgres_export.go b/internal/releaseadapter/v071/postgres_export.go new file mode 100644 index 0000000..2811836 --- /dev/null +++ b/internal/releaseadapter/v071/postgres_export.go @@ -0,0 +1,340 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "regexp" + "sort" + + "github.com/jackc/pgx/v5" +) + +var resourceReferencePattern = regexp.MustCompile(`resource://([A-Za-z0-9_-]{22})`) + +type PostgresIdentity struct { + TenantID uint64 + KnowledgeBaseID string + KnowledgeID string +} + +type TableSnapshot struct { + Name string + Rows []json.RawMessage + RowCount int + SHA256 string +} + +type PostgresSnapshot struct { + Tables []TableSnapshot +} + +func (snapshot PostgresSnapshot) Table(name string) TableSnapshot { + for _, table := range snapshot.Tables { + if table.Name == name { + return table + } + } + return TableSnapshot{Name: name} +} + +type PostgresExporter struct { + db postgresQueryDB +} + +type postgresQueryDB interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) +} + +func NewPostgresExporter(db postgresQueryDB) (*PostgresExporter, error) { + if db == nil { + return nil, errors.New("PostgreSQL export database is required") + } + return &PostgresExporter{db: db}, nil +} + +const exportKnowledgeSQL = ` +SELECT to_jsonb(knowledge) +FROM knowledges AS knowledge +WHERE knowledge.id = $1 + AND knowledge.tenant_id = $2 + AND knowledge.knowledge_base_id = $3 + AND knowledge.deleted_at IS NULL + AND knowledge.parse_status = 'completed' +` + +const exportChunksSQL = ` +SELECT to_jsonb(chunk) +FROM chunks AS chunk +WHERE chunk.knowledge_id = $1 + AND chunk.tenant_id = $2 + AND chunk.deleted_at IS NULL +ORDER BY chunk.id +` + +const exportTagRelationsSQL = ` +SELECT to_jsonb(relation) +FROM knowledge_tag_relations AS relation +WHERE relation.knowledge_id = $1 + AND EXISTS ( + SELECT 1 + FROM knowledges AS knowledge + WHERE knowledge.id = relation.knowledge_id + AND knowledge.tenant_id = $2 + AND knowledge.knowledge_base_id = $3 + AND knowledge.deleted_at IS NULL + ) +ORDER BY relation.tag_id +` + +const exportKnowledgeTagsSQL = ` +SELECT to_jsonb(tag) +FROM knowledge_tags AS tag +WHERE tag.tenant_id = $2 + AND tag.knowledge_base_id = $3 + AND tag.deleted_at IS NULL + AND EXISTS ( + SELECT 1 + FROM knowledge_tag_relations AS relation + WHERE relation.knowledge_id = $1 + AND relation.tag_id = tag.id + ) +ORDER BY tag.id +` + +const exportResourcesSQL = ` +SELECT to_jsonb(resource) +FROM resources AS resource +WHERE resource.tenant_id = $1 + AND resource.deleted_at IS NULL + AND ( + resource.handle = ANY($4::text[]) + OR EXISTS ( + SELECT 1 + FROM resource_bindings AS binding + WHERE binding.resource_id = resource.id + AND binding.tenant_id = $1 + AND ( + (binding.owner_type = 'knowledge' AND binding.owner_id = $2) + OR (binding.owner_type = 'chunk' AND binding.owner_id = ANY($3::text[])) + ) + ) + ) +ORDER BY resource.id +` + +const exportResourceBindingsSQL = ` +SELECT to_jsonb(binding) +FROM resource_bindings AS binding +WHERE binding.tenant_id = $1 + AND binding.resource_id = ANY($2::text[]) + AND ( + (binding.owner_type = 'knowledge' AND binding.owner_id = $3) + OR (binding.owner_type = 'chunk' AND binding.owner_id = ANY($4::text[])) + ) +ORDER BY binding.id +` + +func (exporter *PostgresExporter) Export(ctx context.Context, identity PostgresIdentity) (PostgresSnapshot, error) { + if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { + return PostgresSnapshot{}, errors.New("invalid PostgreSQL export identity") + } + var knowledgeRaw []byte + if err := exporter.db.QueryRow(ctx, exportKnowledgeSQL, identity.KnowledgeID, identity.TenantID, identity.KnowledgeBaseID).Scan(&knowledgeRaw); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return PostgresSnapshot{}, errors.New("eligible PostgreSQL knowledge not found") + } + return PostgresSnapshot{}, errors.New("read PostgreSQL knowledge") + } + knowledge, err := canonicalJSON(knowledgeRaw) + if err != nil || !matchesKnowledgeRow(knowledge, identity) { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope knowledge row") + } + chunks, err := exporter.queryRows(ctx, exportChunksSQL, identity.KnowledgeID, identity.TenantID) + if err != nil { + return PostgresSnapshot{}, err + } + if len(chunks) == 0 { + return PostgresSnapshot{}, errors.New("PostgreSQL knowledge has no chunks") + } + chunkIDs := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + chunkID, ok := matchesChunkRow(chunk, identity) + if !ok { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope chunk row") + } + chunkIDs = append(chunkIDs, chunkID) + } + knowledgeTags, err := exporter.queryRows(ctx, exportKnowledgeTagsSQL, identity.KnowledgeID, identity.TenantID, identity.KnowledgeBaseID) + if err != nil { + return PostgresSnapshot{}, err + } + for _, tag := range knowledgeTags { + if !matchesKnowledgeTagRow(tag, identity) { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope knowledge tag") + } + } + tagRelations, err := exporter.queryRows(ctx, exportTagRelationsSQL, identity.KnowledgeID, identity.TenantID, identity.KnowledgeBaseID) + if err != nil { + return PostgresSnapshot{}, err + } + for _, relation := range tagRelations { + if !matchesTagRelationRow(relation, identity.KnowledgeID) { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope tag relation") + } + } + handles := referencedResourceHandles(append([]json.RawMessage{knowledge}, chunks...)) + resources, err := exporter.queryRows(ctx, exportResourcesSQL, identity.TenantID, identity.KnowledgeID, chunkIDs, handles) + if err != nil { + return PostgresSnapshot{}, err + } + resourceIDs := make([]string, 0, len(resources)) + for _, resource := range resources { + resourceID, ok := matchesResourceRow(resource, identity.TenantID) + if !ok { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope resource row") + } + resourceIDs = append(resourceIDs, resourceID) + } + bindings, err := exporter.queryRows(ctx, exportResourceBindingsSQL, identity.TenantID, resourceIDs, identity.KnowledgeID, chunkIDs) + if err != nil { + return PostgresSnapshot{}, err + } + for _, binding := range bindings { + if !matchesBindingRow(binding, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) { + return PostgresSnapshot{}, errors.New("PostgreSQL returned a cross-scope resource binding") + } + } + tables := []TableSnapshot{ + newTableSnapshot("knowledges", []json.RawMessage{knowledge}), + newTableSnapshot("chunks", chunks), + newTableSnapshot("knowledge_tags", knowledgeTags), + newTableSnapshot("knowledge_tag_relations", tagRelations), + newTableSnapshot("resources", resources), + newTableSnapshot("resource_bindings", bindings), + } + return PostgresSnapshot{Tables: tables}, nil +} + +func (exporter *PostgresExporter) queryRows(ctx context.Context, query string, arguments ...any) ([]json.RawMessage, error) { + rows, err := exporter.db.Query(ctx, query, arguments...) + if err != nil { + return nil, errors.New("read PostgreSQL release rows") + } + defer rows.Close() + result := make([]json.RawMessage, 0) + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, errors.New("scan PostgreSQL release row") + } + canonical, err := canonicalJSON(raw) + if err != nil { + return nil, errors.New("invalid PostgreSQL release row") + } + result = append(result, canonical) + } + if rows.Err() != nil { + return nil, errors.New("iterate PostgreSQL release rows") + } + return result, nil +} + +func newTableSnapshot(name string, rows []json.RawMessage) TableSnapshot { + rows = append([]json.RawMessage(nil), rows...) + sort.Slice(rows, func(left, right int) bool { return string(rows[left]) < string(rows[right]) }) + encoded, _ := json.Marshal(rows) + digest := sha256.Sum256(encoded) + return TableSnapshot{Name: name, Rows: rows, RowCount: len(rows), SHA256: hex.EncodeToString(digest[:])} +} + +func referencedResourceHandles(rows []json.RawMessage) []string { + seen := map[string]bool{} + for _, row := range rows { + for _, match := range resourceReferencePattern.FindAllSubmatch(row, -1) { + seen[string(match[1])] = true + } + } + handles := make([]string, 0, len(seen)) + for handle := range seen { + handles = append(handles, handle) + } + sort.Strings(handles) + return handles +} + +func matchesKnowledgeRow(raw json.RawMessage, identity PostgresIdentity) bool { + var row struct { + ID string `json:"id"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + ParseStatus string `json:"parse_status"` + } + return json.Unmarshal(raw, &row) == nil && row.ID == identity.KnowledgeID && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID && row.ParseStatus == "completed" +} + +func matchesChunkRow(raw json.RawMessage, identity PostgresIdentity) (string, bool) { + var row struct { + ID string `json:"id"` + TenantID uint64 `json:"tenant_id"` + KnowledgeID string `json:"knowledge_id"` + } + if json.Unmarshal(raw, &row) != nil { + return "", false + } + return row.ID, validSearchIdentifier(row.ID) && row.TenantID == identity.TenantID && row.KnowledgeID == identity.KnowledgeID +} + +func matchesKnowledgeTagRow(raw json.RawMessage, identity PostgresIdentity) bool { + var row struct { + ID string `json:"id"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + } + return json.Unmarshal(raw, &row) == nil && validSearchIdentifier(row.ID) && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID +} + +func matchesTagRelationRow(raw json.RawMessage, knowledgeID string) bool { + var row struct { + KnowledgeID string `json:"knowledge_id"` + TagID string `json:"tag_id"` + } + return json.Unmarshal(raw, &row) == nil && row.KnowledgeID == knowledgeID && validSearchIdentifier(row.TagID) +} + +func matchesResourceRow(raw json.RawMessage, tenantID uint64) (string, bool) { + var row struct { + ID string `json:"id"` + TenantID uint64 `json:"tenant_id"` + Handle string `json:"handle"` + } + if json.Unmarshal(raw, &row) != nil { + return "", false + } + return row.ID, validSearchIdentifier(row.ID) && row.TenantID == tenantID && len(row.Handle) == 22 +} + +func matchesBindingRow(raw json.RawMessage, tenantID uint64, knowledgeID string, chunkIDs, resourceIDs []string) bool { + var row struct { + ResourceID string `json:"resource_id"` + TenantID uint64 `json:"tenant_id"` + OwnerType string `json:"owner_type"` + OwnerID string `json:"owner_id"` + } + if json.Unmarshal(raw, &row) != nil || row.TenantID != tenantID || !containsString(resourceIDs, row.ResourceID) { + return false + } + return row.OwnerType == "knowledge" && row.OwnerID == knowledgeID || row.OwnerType == "chunk" && containsString(chunkIDs, row.OwnerID) +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/internal/releaseadapter/v071/postgres_export_integration_test.go b/internal/releaseadapter/v071/postgres_export_integration_test.go new file mode 100644 index 0000000..3f04baf --- /dev/null +++ b/internal/releaseadapter/v071/postgres_export_integration_test.go @@ -0,0 +1,36 @@ +package v071 + +import ( + "context" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestPostgresExporterV071Integration(t *testing.T) { + databaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_WEKNORA_V071_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + exporter, err := NewPostgresExporter(pool) + if err != nil { + t.Fatal(err) + } + snapshot, err := exporter.Export(ctx, PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-release-safe", KnowledgeID: "knowledge-release-safe"}) + if err != nil { + t.Fatal(err) + } + for table, count := range map[string]int{"knowledges": 1, "chunks": 1, "knowledge_tags": 1, "knowledge_tag_relations": 1, "resources": 2, "resource_bindings": 1} { + actual := snapshot.Table(table) + if actual.RowCount != count || len(actual.SHA256) != 64 { + t.Fatalf("%s=%#v", table, actual) + } + } +} diff --git a/internal/releaseadapter/v071/postgres_export_test.go b/internal/releaseadapter/v071/postgres_export_test.go new file mode 100644 index 0000000..ab6a3fc --- /dev/null +++ b/internal/releaseadapter/v071/postgres_export_test.go @@ -0,0 +1,130 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func TestPostgresExporterReadsOnlyDocumentAllowlist(t *testing.T) { + handle1, handle2 := "ABCDEFGHIJKLMNOPQRSTUV", "abcdefghijklmnopqrstuv" + db := &exportDB{ + knowledge: jsonRow{raw: []byte(`{"id":"knowledge-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://` + handle1 + `"}`)}, + rowsByMarker: map[string][]json.RawMessage{ + "select to_jsonb(chunk)": {json.RawMessage(`{"id":"chunk-1","tenant_id":10000,"knowledge_id":"knowledge-safe","image_info":{"url":"resource://` + handle2 + `"}}`)}, + "select to_jsonb(tag)": {json.RawMessage(`{"id":"tag-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","name":"synthetic"}`)}, + "select to_jsonb(relation)": {json.RawMessage(`{"knowledge_id":"knowledge-safe","tag_id":"tag-safe"}`)}, + "select to_jsonb(resource)": {json.RawMessage(`{"id":"res-1","handle":"` + handle1 + `","tenant_id":10000,"physical_path":"tenant/knowledge/source.pdf"}`), json.RawMessage(`{"id":"res-2","handle":"` + handle2 + `","tenant_id":10000,"physical_path":"tenant/knowledge/image.png"}`)}, + "select to_jsonb(binding)": {json.RawMessage(`{"id":"bind-1","resource_id":"res-1","tenant_id":10000,"owner_type":"knowledge","owner_id":"knowledge-safe"}`)}, + }, + } + exporter, err := NewPostgresExporter(db) + if err != nil { + t.Fatal(err) + } + snapshot, err := exporter.Export(context.Background(), PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}) + if err != nil { + t.Fatal(err) + } + if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount != 1 || snapshot.Table("knowledge_tags").RowCount != 1 || snapshot.Table("knowledge_tag_relations").RowCount != 1 || snapshot.Table("resources").RowCount != 2 { + t.Fatalf("snapshot=%#v", snapshot) + } + if got := db.resourceHandles; len(got) != 2 || got[0] != handle1 || got[1] != handle2 { + t.Fatalf("resource handles=%v", got) + } + if len(db.chunkIDs) != 1 || db.chunkIDs[0] != "chunk-1" { + t.Fatalf("chunk IDs=%v", db.chunkIDs) + } + joined := strings.ToLower(strings.Join(db.queries, "\n")) + for _, required := range []string{"from knowledges", "from chunks", "from knowledge_tags", "from knowledge_tag_relations", "from resources", "from resource_bindings", "tenant_id", "knowledge_base_id", "knowledge_id"} { + if !strings.Contains(joined, required) { + t.Fatalf("SQL missing %q", required) + } + } + for _, forbidden := range []string{"users", "messages", "sessions", "api_keys", "model_providers", "jobs", "feedback"} { + if strings.Contains(joined, forbidden) { + t.Fatalf("SQL included forbidden table %q", forbidden) + } + } +} + +func TestPostgresExporterRejectsCrossScopeRows(t *testing.T) { + db := &exportDB{knowledge: jsonRow{raw: []byte(`{"id":"knowledge-safe","tenant_id":10000,"knowledge_base_id":"wrong","parse_status":"completed"}`)}, rowsByMarker: map[string][]json.RawMessage{}} + exporter, _ := NewPostgresExporter(db) + _, err := exporter.Export(context.Background(), PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}) + if err == nil || strings.Contains(err.Error(), "wrong") || strings.Contains(err.Error(), "knowledge-safe") { + t.Fatalf("error=%v", err) + } +} + +type jsonRow struct { + raw []byte + err error +} + +func (r jsonRow) Scan(dest ...any) error { + if r.err != nil { + return r.err + } + *dest[0].(*[]byte) = append([]byte(nil), r.raw...) + return nil +} + +type jsonRows struct { + values []json.RawMessage + index int +} + +func (r *jsonRows) Close() {} +func (r *jsonRows) Err() error { return nil } +func (r *jsonRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *jsonRows) FieldDescriptions() []pgconn.FieldDescription { return nil } +func (r *jsonRows) RawValues() [][]byte { return nil } +func (r *jsonRows) Conn() *pgx.Conn { return nil } +func (r *jsonRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} +func (r *jsonRows) Scan(dest ...any) error { + *dest[0].(*[]byte) = append([]byte(nil), r.values[r.index-1]...) + return nil +} +func (r *jsonRows) Values() ([]any, error) { return nil, nil } + +type exportDB struct { + knowledge pgx.Row + rowsByMarker map[string][]json.RawMessage + queries []string + resourceHandles []string + chunkIDs []string +} + +func (d *exportDB) QueryRow(_ context.Context, sql string, args ...any) pgx.Row { + d.queries = append(d.queries, sql) + return d.knowledge +} +func (d *exportDB) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) { + d.queries = append(d.queries, sql) + lower := strings.ToLower(sql) + for marker, values := range d.rowsByMarker { + if strings.Contains(lower, marker) { + if marker == "select to_jsonb(resource)" && len(args) >= 4 { + d.chunkIDs = append([]string(nil), args[2].([]string)...) + d.resourceHandles = append([]string(nil), args[3].([]string)...) + } + return &jsonRows{values: values}, nil + } + } + return nil, errors.New("unexpected query") +} +func (d *exportDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, nil +} diff --git a/internal/releaseadapter/v071/postgres_import.go b/internal/releaseadapter/v071/postgres_import.go new file mode 100644 index 0000000..f16115b --- /dev/null +++ b/internal/releaseadapter/v071/postgres_import.go @@ -0,0 +1,416 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + + "github.com/jackc/pgx/v5" +) + +type postgresTransactionalDB interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + +type PostgresImporterConfig struct { + TargetStorageBackendID string + TargetProvider string +} + +type PostgresImporter struct { + db postgresTransactionalDB + config PostgresImporterConfig +} + +func NewPostgresImporter(db postgresTransactionalDB, options ...PostgresImporterConfig) (*PostgresImporter, error) { + if db == nil || len(options) > 1 { + return nil, errors.New("invalid PostgreSQL importer configuration") + } + var config PostgresImporterConfig + if len(options) == 1 { + config = options[0] + } + config.TargetStorageBackendID = strings.TrimSpace(config.TargetStorageBackendID) + config.TargetProvider = strings.TrimSpace(config.TargetProvider) + if (config.TargetStorageBackendID == "") != (config.TargetProvider == "") || + (config.TargetStorageBackendID != "" && (!validSearchIdentifier(config.TargetStorageBackendID) || !validObjectProvider(config.TargetProvider))) { + return nil, errors.New("invalid target storage backend configuration") + } + return &PostgresImporter{db: db, config: config}, nil +} + +func (importer *PostgresImporter) Replace(ctx context.Context, identity PostgresIdentity, snapshot PostgresSnapshot) error { + prepared, chunkIDs, resourcePairs, err := importer.prepareSnapshot(identity, snapshot) + if err != nil { + return err + } + tx, err := importer.db.Begin(ctx) + if err != nil { + return errors.New("begin PostgreSQL document import") + } + defer tx.Rollback(ctx) + var version int + var dirty bool + if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { + return errors.New("target WeKnora schema contract mismatch") + } + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil { + return errors.New("lock target knowledge") + } + var crossScope int + if err := tx.QueryRow(ctx, ` +SELECT COUNT(*)::INTEGER +FROM knowledges +WHERE id = $3 AND (tenant_id <> $1 OR knowledge_base_id <> $2) +`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID).Scan(&crossScope); err != nil || crossScope != 0 { + return errors.New("target knowledge ID conflicts with another scope") + } + for _, pair := range resourcePairs { + if err := tx.QueryRow(ctx, ` +SELECT COUNT(*)::INTEGER +FROM resources +WHERE (id = $1 OR handle = $2) + AND NOT (id = $1 AND handle = $2 AND tenant_id = $3) +`, pair.id, pair.handle, identity.TenantID).Scan(&crossScope); err != nil || crossScope != 0 { + return errors.New("target resource identity conflict") + } + } + for _, raw := range prepared.Table("knowledge_tags").Rows { + var tag struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &tag) != nil { + return errors.New("invalid target knowledge tag") + } + var crossScope int + if err := tx.QueryRow(ctx, `SELECT COUNT(*) FROM knowledge_tags WHERE id = $1 AND NOT (tenant_id = $2 AND knowledge_base_id = $3)`, tag.ID, identity.TenantID, identity.KnowledgeBaseID).Scan(&crossScope); err != nil || crossScope != 0 { + return errors.New("target knowledge tag identity conflict") + } + } + for _, statement := range []struct { + sql string + args []any + }{ + {`DELETE FROM resource_bindings WHERE tenant_id = $1 AND ((owner_type = 'knowledge' AND owner_id = $2) OR (owner_type = 'chunk' AND owner_id = ANY($3::text[])))`, []any{identity.TenantID, identity.KnowledgeID, chunkIDs}}, + {`DELETE FROM knowledge_tag_relations WHERE knowledge_id = $1`, []any{identity.KnowledgeID}}, + {`DELETE FROM chunks WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3`, []any{identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID}}, + } { + if _, err := tx.Exec(ctx, statement.sql, statement.args...); err != nil { + return errors.New("clear prior target document rows") + } + } + for _, tableName := range []string{"knowledges", "resources", "chunks", "knowledge_tags", "knowledge_tag_relations", "resource_bindings"} { + table := prepared.Table(tableName) + if err := insertSnapshotRows(ctx, tx, table); err != nil { + return err + } + } + if _, err := tx.Exec(ctx, `SELECT setval('chunks_seq_id_seq', GREATEST(COALESCE((SELECT MAX(seq_id) FROM chunks), 1), 1), true)`); err != nil { + return errors.New("advance target chunk sequence") + } + if err := tx.Commit(ctx); err != nil { + return errors.New("commit PostgreSQL document import") + } + return nil +} + +func (importer *PostgresImporter) Disable(ctx context.Context, identity PostgresIdentity) error { + if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { + return errors.New("invalid PostgreSQL disable identity") + } + tx, err := importer.db.Begin(ctx) + if err != nil { + return errors.New("begin PostgreSQL disable") + } + defer tx.Rollback(ctx) + var version int + var dirty bool + if err := tx.QueryRow(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { + return errors.New("target WeKnora schema contract mismatch") + } + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, identity.KnowledgeID); err != nil { + return errors.New("lock target knowledge") + } + command, err := tx.Exec(ctx, ` +UPDATE knowledges +SET enable_status = 'disabled', updated_at = NOW() +WHERE id = $3 AND tenant_id = $1 AND knowledge_base_id = $2 AND deleted_at IS NULL +`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID) + if err != nil || command.RowsAffected() != 1 { + return errors.New("disable target knowledge") + } + if _, err := tx.Exec(ctx, ` +UPDATE chunks +SET is_enabled = FALSE, updated_at = NOW() +WHERE tenant_id = $1 AND knowledge_base_id = $2 AND knowledge_id = $3 AND deleted_at IS NULL +`, identity.TenantID, identity.KnowledgeBaseID, identity.KnowledgeID); err != nil { + return errors.New("disable target chunks") + } + if err := tx.Commit(ctx); err != nil { + return errors.New("commit PostgreSQL disable") + } + return nil +} + +type resourceIdentityPair struct{ id, handle string } + +func (importer *PostgresImporter) prepareSnapshot(identity PostgresIdentity, snapshot PostgresSnapshot) (PostgresSnapshot, []string, []resourceIdentityPair, error) { + if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { + return PostgresSnapshot{}, nil, nil, errors.New("invalid PostgreSQL import identity") + } + if snapshot.Table("knowledges").RowCount != 1 || snapshot.Table("chunks").RowCount < 1 { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is incomplete") + } + prepared := cloneSnapshot(snapshot) + known := map[string]bool{} + for index, table := range prepared.Tables { + if !validReleaseTable(table.Name) || known[table.Name] || table.RowCount != len(table.Rows) || newTableSnapshot(table.Name, table.Rows).SHA256 != table.SHA256 { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import snapshot is invalid") + } + known[table.Name] = true + prepared.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...) + } + if !matchesKnowledgeRow(prepared.Table("knowledges").Rows[0], identity) { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge scope mismatch") + } + chunkIDs := make([]string, 0, prepared.Table("chunks").RowCount) + chunkRows := prepared.Table("chunks").Rows + for index, row := range chunkRows { + id, ok := matchesChunkRow(row, identity) + if !ok { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch") + } + chunkIDs = append(chunkIDs, id) + // chunks.seq_id is a per-database global sequence value, not document + // content (WeKnora retrieval and Elasticsearch never reference it). + // Carrying a source value across environments collides with rows the + // target has already written, so let the target sequence assign it and + // advance the sequence after the transaction below. + var fields map[string]any + if err := json.Unmarshal(row, &fields); err != nil { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import chunk scope mismatch") + } + if _, present := fields["seq_id"]; present { + delete(fields, "seq_id") + rewritten, err := json.Marshal(fields) + if err != nil { + return PostgresSnapshot{}, nil, nil, errors.New("rewrite target chunk row") + } + chunkRows[index] = rewritten + } + } + chunks := newTableSnapshot("chunks", chunkRows) + for index := range prepared.Tables { + if prepared.Tables[index].Name == "chunks" { + prepared.Tables[index] = chunks + } + } + tagIDs := map[string]bool{} + for _, row := range prepared.Table("knowledge_tags").Rows { + if !matchesKnowledgeTagRow(row, identity) { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch") + } + var tag struct { + ID string `json:"id"` + } + if json.Unmarshal(row, &tag) != nil { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import knowledge tag scope mismatch") + } + tagIDs[tag.ID] = true + } + for _, row := range prepared.Table("knowledge_tag_relations").Rows { + var relation struct { + TagID string `json:"tag_id"` + } + if !matchesTagRelationRow(row, identity.KnowledgeID) || json.Unmarshal(row, &relation) != nil || !tagIDs[relation.TagID] { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import tag scope mismatch") + } + } + resources := prepared.Table("resources") + resourceIDs := make([]string, 0, len(resources.Rows)) + resourcePairs := make([]resourceIdentityPair, 0, len(resources.Rows)) + for rowIndex, row := range resources.Rows { + rewritten, pair, err := importer.prepareResourceRow(row, identity.TenantID) + if err != nil { + return PostgresSnapshot{}, nil, nil, err + } + resources.Rows[rowIndex] = rewritten + resourceIDs = append(resourceIDs, pair.id) + resourcePairs = append(resourcePairs, pair) + } + resources = newTableSnapshot("resources", resources.Rows) + for index := range prepared.Tables { + if prepared.Tables[index].Name == "resources" { + prepared.Tables[index] = resources + } + } + for _, row := range prepared.Table("resource_bindings").Rows { + if !matchesBindingRow(row, identity.TenantID, identity.KnowledgeID, chunkIDs, resourceIDs) { + return PostgresSnapshot{}, nil, nil, errors.New("PostgreSQL import resource binding scope mismatch") + } + } + return prepared, chunkIDs, resourcePairs, nil +} + +func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenantID uint64) (json.RawMessage, resourceIdentityPair, error) { + var row map[string]any + if json.Unmarshal(raw, &row) != nil { + return nil, resourceIdentityPair{}, errors.New("invalid PostgreSQL import resource") + } + id, _ := row["id"].(string) + handle, _ := row["handle"].(string) + if matchedID, ok := matchesResourceRow(raw, tenantID); !ok || matchedID != id { + return nil, resourceIdentityPair{}, errors.New("PostgreSQL import resource scope mismatch") + } + sourceBackendID, _ := row["storage_backend_id"].(string) + physicalPath, _ := row["physical_path"].(string) + if importer.config.TargetStorageBackendID == "" { + if sourceBackendID != "" { + return nil, resourceIdentityPair{}, errors.New("target storage backend rewrite is required") + } + return raw, resourceIdentityPair{id: id, handle: handle}, nil + } + providerPath := physicalPath + if strings.HasPrefix(physicalPath, "storage://") { + rest := strings.TrimPrefix(physicalPath, "storage://") + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) { + return nil, resourceIdentityPair{}, errors.New("invalid source storage locator") + } + providerPath = parts[1] + } else if !validObjectLocator(physicalPath) { + return nil, resourceIdentityPair{}, errors.New("invalid source storage locator") + } + targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath + digest := sha256.Sum256([]byte(targetPath)) + row["storage_backend_id"] = importer.config.TargetStorageBackendID + row["provider"] = importer.config.TargetProvider + row["physical_path"] = targetPath + row["location_hash"] = hex.EncodeToString(digest[:]) + encoded, err := json.Marshal(row) + if err != nil { + return nil, resourceIdentityPair{}, errors.New("rewrite target resource") + } + canonical, err := canonicalJSON(encoded) + if err != nil { + return nil, resourceIdentityPair{}, errors.New("rewrite target resource") + } + return canonical, resourceIdentityPair{id: id, handle: handle}, nil +} + +func insertSnapshotRows(ctx context.Context, tx pgx.Tx, table TableSnapshot) error { + if len(table.Rows) == 0 { + return nil + } + columns, err := tableColumns(ctx, tx, table.Name) + if err != nil { + return err + } + if table.Name == "chunks" { + // seq_id is assigned by the target sequence; it must not appear in the + // explicit column list (an explicit NULL would violate NOT NULL) and it + // is removed from snapshot rows during prepare. + filtered := columns[:0] + for _, column := range columns { + if column != "seq_id" { + filtered = append(filtered, column) + } + } + columns = filtered + } + quotedTable := pgx.Identifier{table.Name}.Sanitize() + quotedColumns := make([]string, len(columns)) + for index, column := range columns { + quotedColumns[index] = pgx.Identifier{column}.Sanitize() + } + columnList := strings.Join(quotedColumns, ", ") + selectList := make([]string, len(quotedColumns)) + for index, column := range quotedColumns { + selectList[index] = "record_value." + column + } + query := fmt.Sprintf("INSERT INTO %s (%s) SELECT %s FROM jsonb_populate_record(NULL::%s, $1::jsonb) AS record_value", quotedTable, columnList, strings.Join(selectList, ", "), quotedTable) + if table.Name == "knowledges" || table.Name == "knowledge_tags" || table.Name == "resources" { + updates := make([]string, 0, len(quotedColumns)-1) + for index, column := range columns { + if column == "id" { + continue + } + updates = append(updates, quotedColumns[index]+" = EXCLUDED."+quotedColumns[index]) + } + query += " ON CONFLICT (id) DO UPDATE SET " + strings.Join(updates, ", ") + } + for _, row := range table.Rows { + if _, err := tx.Exec(ctx, query, string(row)); err != nil { + return fmt.Errorf("insert target PostgreSQL document row (%s): %w", table.Name, err) + } + } + return nil +} + +func tableColumns(ctx context.Context, tx pgx.Tx, tableName string) ([]string, error) { + if !validReleaseTable(tableName) { + return nil, errors.New("invalid PostgreSQL import table") + } + rows, err := tx.Query(ctx, ` +SELECT column_name +FROM information_schema.columns +WHERE table_schema = 'public' AND table_name = $1 AND is_generated = 'NEVER' +ORDER BY ordinal_position +`, tableName) + if err != nil { + return nil, errors.New("inspect target PostgreSQL table") + } + defer rows.Close() + columns := make([]string, 0) + for rows.Next() { + var column string + if rows.Scan(&column) != nil || !validSQLIdentifier(column) { + return nil, errors.New("invalid target PostgreSQL column") + } + columns = append(columns, column) + } + if rows.Err() != nil || len(columns) == 0 { + return nil, errors.New("inspect target PostgreSQL table") + } + return columns, nil +} + +func validSQLIdentifier(value string) bool { + if value == "" || len(value) > 63 { + return false + } + for index, char := range value { + if (char < 'a' || char > 'z') && char != '_' && (index == 0 || char < '0' || char > '9') { + return false + } + } + return true +} + +func validObjectLocator(value string) bool { + if value == "" || len(value) > 2048 || value != strings.TrimSpace(value) || strings.ContainsAny(value, "\x00\r\n\\?#") || strings.HasPrefix(value, "/") { + return false + } + if separator := strings.Index(value, "://"); separator >= 0 { + if !validSearchIdentifier(value[:separator]) { + return false + } + value = value[separator+3:] + } + cleaned := path.Clean(value) + return cleaned == value && cleaned != "." && cleaned != ".." && !strings.HasPrefix(cleaned, "../") +} + +func cloneSnapshot(snapshot PostgresSnapshot) PostgresSnapshot { + result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))} + for index, table := range snapshot.Tables { + result.Tables[index] = table + result.Tables[index].Rows = append([]json.RawMessage(nil), table.Rows...) + } + return result +} diff --git a/internal/releaseadapter/v071/postgres_import_integration_test.go b/internal/releaseadapter/v071/postgres_import_integration_test.go new file mode 100644 index 0000000..899d7ac --- /dev/null +++ b/internal/releaseadapter/v071/postgres_import_integration_test.go @@ -0,0 +1,115 @@ +package v071 + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestPostgresImporterV071ReplacesAndRestoresDocument(t *testing.T) { + sourceURL, targetURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL"), os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL") + if sourceURL == "" || targetURL == "" { + t.Skip("v0.7.1 source and target URLs are not set") + } + ctx := context.Background() + source, err := pgxpool.New(ctx, sourceURL) + if err != nil { + t.Fatal(err) + } + defer source.Close() + target, err := pgxpool.New(ctx, targetURL) + if err != nil { + t.Fatal(err) + } + defer target.Close() + identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-release-safe", KnowledgeID: "knowledge-release-safe"} + sourceExporter, _ := NewPostgresExporter(source) + newSnapshot, err := sourceExporter.Export(ctx, identity) + if err != nil { + t.Fatal(err) + } + oldSnapshot := clonePostgresSnapshot(newSnapshot) + oldChunks := oldSnapshot.Table("chunks") + var chunk map[string]any + if err := json.Unmarshal(oldChunks.Rows[0], &chunk); err != nil { + t.Fatal(err) + } + chunk["content"] = "Prior production content" + oldRaw, err := json.Marshal(chunk) + if err != nil { + t.Fatal(err) + } + for index, table := range oldSnapshot.Tables { + if table.Name == "chunks" { + oldSnapshot.Tables[index] = newTableSnapshot("chunks", []json.RawMessage{oldRaw}) + } + } + + importer, err := NewPostgresImporter(target) + if err != nil { + t.Fatal(err) + } + if err := importer.Replace(ctx, identity, oldSnapshot); err != nil { + t.Fatal(err) + } + targetExporter, _ := NewPostgresExporter(target) + oldActual, err := targetExporter.Export(ctx, identity) + if err != nil { + t.Fatal(err) + } + assertPostgresSnapshotDigests(t, oldActual, oldSnapshot) + if err := importer.Replace(ctx, identity, newSnapshot); err != nil { + t.Fatal(err) + } + newActual, err := targetExporter.Export(ctx, identity) + if err != nil { + t.Fatal(err) + } + assertPostgresSnapshotDigests(t, newActual, newSnapshot) + if err := importer.Replace(ctx, identity, oldSnapshot); err != nil { + t.Fatal(err) + } + restored, err := targetExporter.Export(ctx, identity) + if err != nil { + t.Fatal(err) + } + assertPostgresSnapshotDigests(t, restored, oldSnapshot) + if err := importer.Disable(ctx, identity); err != nil { + t.Fatal(err) + } + var knowledgeEnabled string + var enabledChunks, knowledgeRows, chunkRows int + if err := target.QueryRow(ctx, `SELECT enable_status FROM knowledges WHERE id=$1`, identity.KnowledgeID).Scan(&knowledgeEnabled); err != nil { + t.Fatal(err) + } + if err := target.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE is_enabled), COUNT(*) FROM chunks WHERE knowledge_id=$1`, identity.KnowledgeID).Scan(&enabledChunks, &chunkRows); err != nil { + t.Fatal(err) + } + if err := target.QueryRow(ctx, `SELECT COUNT(*) FROM knowledges WHERE id=$1`, identity.KnowledgeID).Scan(&knowledgeRows); err != nil { + t.Fatal(err) + } + if knowledgeEnabled != "disabled" || enabledChunks != 0 || knowledgeRows != 1 || chunkRows != 1 { + t.Fatalf("status=%s enabled_chunks=%d rows=%d/%d", knowledgeEnabled, enabledChunks, knowledgeRows, chunkRows) + } +} + +func clonePostgresSnapshot(snapshot PostgresSnapshot) PostgresSnapshot { + result := PostgresSnapshot{Tables: make([]TableSnapshot, len(snapshot.Tables))} + for i, table := range snapshot.Tables { + result.Tables[i] = table + result.Tables[i].Rows = append([]json.RawMessage(nil), table.Rows...) + } + return result +} +func assertPostgresSnapshotDigests(t *testing.T, actual, want PostgresSnapshot) { + t.Helper() + for _, table := range want.Tables { + got := actual.Table(table.Name) + if got.RowCount != table.RowCount || got.SHA256 != table.SHA256 { + t.Fatalf("%s got=%#v want=%#v", table.Name, got, table) + } + } +} diff --git a/internal/releaseadapter/v071/s3_object_store.go b/internal/releaseadapter/v071/s3_object_store.go new file mode 100644 index 0000000..cc65750 --- /dev/null +++ b/internal/releaseadapter/v071/s3_object_store.go @@ -0,0 +1,141 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3ObjectStoreConfig struct { + Provider string + Endpoint string + Region string + AccessKeyID string + SecretAccessKey string + Bucket string + Secure bool + ForcePathStyle bool + MaxObjectBytes int64 + HTTPClient *http.Client +} + +type S3ObjectStore struct { + provider string + bucket string + maxObjectBytes int64 + client *s3.Client +} + +func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) { + config.Provider = strings.TrimSpace(config.Provider) + config.Endpoint = strings.TrimSpace(config.Endpoint) + config.Region = strings.TrimSpace(config.Region) + config.Bucket = strings.TrimSpace(config.Bucket) + if config.Region == "" { + config.Region = "us-east-1" + } + if config.MaxObjectBytes == 0 { + config.MaxObjectBytes = 100 * 1024 * 1024 + } + endpoint, err := url.Parse(config.Endpoint) + if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" || + (endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") || + !validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 { + return nil, errors.New("invalid S3 object store configuration") + } + httpClient := &http.Client{Timeout: 2 * time.Minute} + if config.HTTPClient != nil { + clone := *config.HTTPClient + httpClient = &clone + } + httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") } + forcePathStyle := config.ForcePathStyle || config.Provider == "minio" + client := s3.New(s3.Options{ + Region: config.Region, + BaseEndpoint: aws.String(endpoint.String()), + Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")), + UsePathStyle: forcePathStyle, + HTTPClient: httpClient, + }) + return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil +} + +func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error { + if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil { + return nil + } + if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil { + return errors.New("ensure release object bucket") + } + return nil +} + +func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) { + if provider != store.provider || !validObjectKey(physicalPath) { + return nil, errors.New("invalid release object source") + } + objectPath, ok := providerPathFromLocator(physicalPath) + if !ok { + return nil, errors.New("invalid release object source") + } + object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}) + if err != nil { + return nil, errors.New("read release source object") + } + return object.Body, nil +} + +func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error { + if provider != store.provider || !validObjectLocator(objectPath) || size < 0 || size > store.maxObjectBytes { + return errors.New("invalid release object write") + } + if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath), Body: reader, ContentLength: aws.Int64(size)}); err != nil { + return errors.New("write release object") + } + return nil +} + +func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) { + if provider != store.provider || !validObjectLocator(objectPath) { + return 0, "", errors.New("invalid release object read") + } + head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}) + if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes { + return 0, "", errors.New("inspect release object") + } + object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}) + if err != nil { + return 0, "", errors.New("read release object") + } + defer object.Body.Close() + hash := sha256.New() + written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1)) + if err != nil || written != *head.ContentLength || written > store.maxObjectBytes { + return 0, "", errors.New("verify release object") + } + return written, hex.EncodeToString(hash.Sum(nil)), nil +} + +func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error { + if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) { + return errors.New("invalid release object copy") + } + copySource := url.PathEscape(store.bucket + "/" + sourcePath) + if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil { + return errors.New("copy release object") + } + return nil +} + +var _ ObjectStore = (*S3ObjectStore)(nil) +var _ ObjectReader = (*S3ObjectStore)(nil) diff --git a/internal/releaseadapter/v071/s3_object_store_integration_test.go b/internal/releaseadapter/v071/s3_object_store_integration_test.go new file mode 100644 index 0000000..8b8b555 --- /dev/null +++ b/internal/releaseadapter/v071/s3_object_store_integration_test.go @@ -0,0 +1,57 @@ +package v071 + +import ( + "bytes" + "context" + "io" + "os" + "testing" +) + +func TestS3ObjectStoreIntegration(t *testing.T) { + endpoint := os.Getenv("TEST_MINIO_ENDPOINT") + access := os.Getenv("TEST_MINIO_ACCESS_KEY") + secret := os.Getenv("TEST_MINIO_SECRET_KEY") + bucket := os.Getenv("TEST_MINIO_BUCKET") + if endpoint == "" || access == "" || secret == "" || bucket == "" { + t.Skip("MinIO integration environment is not set") + } + ctx := context.Background() + store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint, AccessKeyID: access, SecretAccessKey: secret, Bucket: bucket, Secure: false}) + if err != nil { + t.Fatal(err) + } + if err := store.EnsureBucket(ctx); err != nil { + t.Fatal(err) + } + data := []byte("synthetic object release content") + if err := store.Put(ctx, "minio", "stage/source.txt", bytes.NewReader(data), int64(len(data))); err != nil { + t.Fatal(err) + } + reader, err := store.Open(ctx, "minio", "storage://source-backend/stage/source.txt") + if err != nil { + t.Fatal(err) + } + opened, err := io.ReadAll(reader) + closeErr := reader.Close() + if err != nil || closeErr != nil || !bytes.Equal(opened, data) { + t.Fatalf("opened=%q err=%v close=%v", opened, err, closeErr) + } + size, digest, err := store.Stat(ctx, "minio", "stage/source.txt") + if err != nil { + t.Fatal(err) + } + if size != int64(len(data)) || digest != shaBytes(data) { + t.Fatalf("size=%d digest=%s", size, digest) + } + if err := store.Copy(ctx, "minio", "stage/source.txt", "final/source.txt"); err != nil { + t.Fatal(err) + } + size, digest, err = store.Stat(ctx, "minio", "final/source.txt") + if err != nil { + t.Fatal(err) + } + if size != int64(len(data)) || digest != shaBytes(data) { + t.Fatalf("copy size=%d digest=%s", size, digest) + } +} diff --git a/internal/releaseadapter/v071/search.go b/internal/releaseadapter/v071/search.go new file mode 100644 index 0000000..e111332 --- /dev/null +++ b/internal/releaseadapter/v071/search.go @@ -0,0 +1,322 @@ +package v071 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const maxSearchResponseBytes = 64 * 1024 * 1024 + +type SearchIdentity struct { + TenantID uint64 + KnowledgeBaseID string + KnowledgeID string +} + +type SearchDocument struct { + ID string + Source json.RawMessage +} + +type SearchSnapshot struct { + Documents []SearchDocument + MappingSHA256 string + DocumentsSHA256 string +} + +type SearchExporterConfig struct { + BaseURL string + Index string + AuthorizationToken string + HTTPClient *http.Client + PageSize int +} + +type SearchExporter struct { + baseURL *url.URL + index string + authorizationToken string + client *http.Client + pageSize int +} + +// searchFieldNames reflects the real v0.7.1 index shape: +// - ID fields may be mapped as "text" with a ".keyword" subfield (community +// default) or as plain "keyword"; exact filters must target the right one. +// - Documents may carry no tenant_id field at all; tenancy is then implied +// by the knowledge_base_id/knowledge_id scope. +type searchFieldNames struct { + tenant string // "" when the index has no tenant_id field + knowledge string + base string +} + +// mappingInspection is the canonical digest of the index mappings plus the +// logical scope field names derived from the real mapping. +type mappingInspection struct { + digest string + fields searchFieldNames +} + +// inspectMapping fetches the index mapping once and derives both the contract +// digest and the exact-match field names from a single response. +func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) { + var response map[string]json.RawMessage + if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil { + return mappingInspection{}, err + } + indexRaw, ok := response[exporter.index] + if !ok { + return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index") + } + var index struct { + Mappings json.RawMessage `json:"mappings"` + } + if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 { + return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid") + } + canonical, err := canonicalJSON(index.Mappings) + if err != nil { + return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid") + } + digest := sha256.Sum256(canonical) + var mappings struct { + Properties map[string]struct { + Type string `json:"type"` + Fields map[string]struct { + Type string `json:"type"` + } `json:"fields"` + } `json:"properties"` + } + if err := json.Unmarshal(canonical, &mappings); err != nil { + return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid") + } + resolve := func(name string) string { + property, ok := mappings.Properties[name] + if !ok { + return "" + } + if property.Type == "keyword" { + return name + } + if _, ok := property.Fields["keyword"]; ok { + return name + ".keyword" + } + return name + } + fields := searchFieldNames{ + tenant: resolve("tenant_id"), + knowledge: resolve("knowledge_id"), + base: resolve("knowledge_base_id"), + } + if fields.knowledge == "" || fields.base == "" { + return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields") + } + return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil +} + +// scopedFilter builds the tenant/KB/knowledge term filter adapted to the +// resolved index field names. +func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any { + filter := make([]any, 0, 3) + if fields.tenant != "" { + filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}}) + } + filter = append(filter, + map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}}, + map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}}, + ) + return filter +} + +func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) { + baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL)) + if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" { + return nil, errors.New("invalid Elasticsearch base URL") + } + baseURL.Path = strings.TrimRight(baseURL.Path, "/") + if !validSearchIdentifier(config.Index) { + return nil, errors.New("invalid Elasticsearch index") + } + pageSize := config.PageSize + if pageSize == 0 { + pageSize = 500 + } + if pageSize < 1 || pageSize > 1000 { + return nil, errors.New("invalid Elasticsearch page size") + } + client := &http.Client{Timeout: 30 * time.Second} + if config.HTTPClient != nil { + clone := *config.HTTPClient + client = &clone + } + client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") } + return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil +} + +func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) { + if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) { + return SearchSnapshot{}, errors.New("invalid search export identity") + } + inspection, err := exporter.inspectMapping(ctx) + if err != nil { + return SearchSnapshot{}, err + } + filter := scopedFilterFor(inspection.fields, identity) + documents := make([]SearchDocument, 0) + for from := 0; ; from += exporter.pageSize { + payload := map[string]any{ + "from": from, + "size": exporter.pageSize, + "query": map[string]any{"bool": map[string]any{"filter": filter}}, + "sort": []any{"_doc"}, + } + var response searchResponse + if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil { + return SearchSnapshot{}, err + } + for _, hit := range response.Hits.Hits { + if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) { + return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document") + } + canonical, err := canonicalJSON(hit.Source) + if err != nil { + return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document") + } + documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical}) + } + if len(response.Hits.Hits) < exporter.pageSize { + break + } + if from+exporter.pageSize >= 10000 { + return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window") + } + } + sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID }) + digest, err := searchDocumentsDigest(documents) + if err != nil { + return SearchSnapshot{}, err + } + return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil +} + +func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) { + inspection, err := exporter.inspectMapping(ctx) + if err != nil { + return "", err + } + return inspection.digest, nil +} + +func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error { + var body io.Reader + if payload != nil { + encoded, err := json.Marshal(payload) + if err != nil { + return errors.New("encode Elasticsearch request") + } + body = bytes.NewReader(encoded) + } + requestURL := *exporter.baseURL + relative, err := url.Parse(path) + if err != nil || relative.IsAbs() || relative.Host != "" { + return errors.New("invalid Elasticsearch request path") + } + requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path + requestURL.RawQuery = relative.RawQuery + request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body) + if err != nil { + return errors.New("create Elasticsearch request") + } + request.Header.Set("Accept", "application/json") + if payload != nil { + request.Header.Set("Content-Type", "application/json") + } + if exporter.authorizationToken != "" { + request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken) + } + response, err := exporter.client.Do(request) + if err != nil { + return errors.New("Elasticsearch request failed") + } + defer response.Body.Close() + if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host { + return errors.New("Elasticsearch response origin mismatch") + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode) + } + decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1)) + if err := decoder.Decode(result); err != nil { + return errors.New("decode Elasticsearch response") + } + return nil +} + +type searchResponse struct { + Hits struct { + Hits []struct { + ID string `json:"_id"` + Source json.RawMessage `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} + +// searchHitMatches verifies the returned document belongs to the requested +// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the +// field is absent the KB/knowledge scope is authoritative, matching the +// export filter semantics. A present tenant must still match exactly. +func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool { + var scoped struct { + TenantID *uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + } + if json.Unmarshal(source, &scoped) != nil { + return false + } + if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID { + return false + } + return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID +} + +func searchDocumentsDigest(documents []SearchDocument) (string, error) { + payload, err := json.Marshal(documents) + if err != nil { + return "", errors.New("encode Elasticsearch document digest") + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} + +func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) { + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, err + } + return json.Marshal(value) +} + +func validSearchIdentifier(value string) bool { + if value == "" || len(value) > 128 || value != strings.TrimSpace(value) { + return false + } + for _, char := range value { + if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' { + return false + } + } + return true +} diff --git a/internal/releaseadapter/v071/search_import.go b/internal/releaseadapter/v071/search_import.go new file mode 100644 index 0000000..cc22c3f --- /dev/null +++ b/internal/releaseadapter/v071/search_import.go @@ -0,0 +1,179 @@ +package v071 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" +) + +type SearchImporter struct { + exporter *SearchExporter +} + +func NewSearchImporter(config SearchExporterConfig) (*SearchImporter, error) { + exporter, err := NewSearchExporter(config) + if err != nil { + return nil, err + } + return &SearchImporter{exporter: exporter}, nil +} + +func (importer *SearchImporter) Replace(ctx context.Context, identity SearchIdentity, snapshot SearchSnapshot) (SearchSnapshot, error) { + if err := validateSearchSnapshot(snapshot, identity); err != nil { + return SearchSnapshot{}, err + } + prior, err := importer.exporter.Export(ctx, identity) + if err != nil { + return SearchSnapshot{}, errors.New("read prior Elasticsearch document snapshot") + } + if prior.MappingSHA256 != snapshot.MappingSHA256 { + return SearchSnapshot{}, errors.New("target Elasticsearch mapping contract mismatch") + } + if err := importer.writeSnapshot(ctx, identity, snapshot); err != nil { + if restoreErr := importer.writeSnapshot(ctx, identity, prior); restoreErr != nil { + return SearchSnapshot{}, errors.New("Elasticsearch replacement and compensation failed") + } + return SearchSnapshot{}, err + } + actual, err := importer.exporter.Export(ctx, identity) + if err != nil || actual.DocumentsSHA256 != snapshot.DocumentsSHA256 || len(actual.Documents) != len(snapshot.Documents) { + if restoreErr := importer.writeSnapshot(ctx, identity, prior); restoreErr != nil { + return SearchSnapshot{}, errors.New("Elasticsearch verification and compensation failed") + } + return SearchSnapshot{}, errors.New("Elasticsearch replacement verification failed") + } + return prior, nil +} + +func (importer *SearchImporter) Disable(ctx context.Context, identity SearchIdentity) error { + prior, err := importer.exporter.Export(ctx, identity) + if err != nil || len(prior.Documents) == 0 { + return errors.New("target Elasticsearch document not found") + } + inspection, err := importer.exporter.inspectMapping(ctx) + if err != nil { + return errors.New("resolve target Elasticsearch scope fields") + } + filter := scopedFilterFor(inspection.fields, identity) + var result struct { + Failures []json.RawMessage `json:"failures"` + } + payload := map[string]any{ + "script": map[string]any{"lang": "painless", "source": "ctx._source.is_enabled = false"}, + "query": map[string]any{"bool": map[string]any{"filter": filter}}, + } + if err := importer.exporter.requestJSON(ctx, http.MethodPost, "/"+importer.exporter.index+"/_update_by_query?refresh=true&conflicts=proceed", payload, &result); err != nil || len(result.Failures) > 0 { + return errors.New("disable target Elasticsearch documents") + } + actual, err := importer.exporter.Export(ctx, identity) + if err != nil || len(actual.Documents) != len(prior.Documents) { + return errors.New("verify disabled Elasticsearch documents") + } + for _, document := range actual.Documents { + var state struct { + IsEnabled *bool `json:"is_enabled"` + } + if json.Unmarshal(document.Source, &state) != nil || state.IsEnabled == nil || *state.IsEnabled { + return errors.New("verify disabled Elasticsearch documents") + } + } + return nil +} + +func validateSearchSnapshot(snapshot SearchSnapshot, identity SearchIdentity) error { + if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) || + !validLowerSHA256(snapshot.MappingSHA256) || !validLowerSHA256(snapshot.DocumentsSHA256) { + return errors.New("invalid Elasticsearch import snapshot") + } + digest, err := searchDocumentsDigest(snapshot.Documents) + if err != nil || digest != snapshot.DocumentsSHA256 { + return errors.New("invalid Elasticsearch import snapshot") + } + seen := map[string]bool{} + for _, document := range snapshot.Documents { + if !validSearchIdentifier(document.ID) || seen[document.ID] || !searchHitMatches(document.Source, identity) { + return errors.New("cross-scope Elasticsearch import document") + } + seen[document.ID] = true + } + return nil +} + +func (importer *SearchImporter) writeSnapshot(ctx context.Context, identity SearchIdentity, snapshot SearchSnapshot) error { + if len(snapshot.Documents) > 0 { + var body bytes.Buffer + encoder := json.NewEncoder(&body) + for _, document := range snapshot.Documents { + if err := encoder.Encode(map[string]any{"index": map[string]string{"_index": importer.exporter.index, "_id": document.ID}}); err != nil { + return errors.New("encode Elasticsearch bulk release") + } + body.Write(document.Source) + body.WriteByte('\n') + } + var result struct { + Errors bool `json:"errors"` + } + if err := importer.requestRawJSON(ctx, http.MethodPost, "/_bulk?refresh=true", &body, "application/x-ndjson", &result); err != nil || result.Errors { + return errors.New("write Elasticsearch release documents") + } + } + inspection, err := importer.exporter.inspectMapping(ctx) + if err != nil { + return errors.New("resolve target Elasticsearch scope fields") + } + filter := scopedFilterFor(inspection.fields, identity) + boolQuery := map[string]any{"filter": filter} + if len(snapshot.Documents) > 0 { + ids := make([]string, len(snapshot.Documents)) + for index, document := range snapshot.Documents { + ids[index] = document.ID + } + boolQuery["must_not"] = []any{map[string]any{"ids": map[string]any{"values": ids}}} + } + var deleted struct { + Failures []json.RawMessage `json:"failures"` + } + if err := importer.exporter.requestJSON(ctx, http.MethodPost, "/"+importer.exporter.index+"/_delete_by_query?refresh=true&conflicts=proceed", map[string]any{"query": map[string]any{"bool": boolQuery}}, &deleted); err != nil || len(deleted.Failures) > 0 { + return errors.New("remove obsolete Elasticsearch release documents") + } + return nil +} + +func (importer *SearchImporter) requestRawJSON(ctx context.Context, method, requestPath string, body io.Reader, contentType string, result any) error { + relative, err := url.Parse(requestPath) + if err != nil || relative.IsAbs() || relative.Host != "" { + return errors.New("invalid Elasticsearch import path") + } + requestURL := *importer.exporter.baseURL + requestURL.Path = strings.TrimRight(importer.exporter.baseURL.Path, "/") + relative.Path + requestURL.RawQuery = relative.RawQuery + request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body) + if err != nil { + return errors.New("create Elasticsearch import request") + } + request.Header.Set("Accept", "application/json") + request.Header.Set("Content-Type", contentType) + if importer.exporter.authorizationToken != "" { + request.Header.Set("Authorization", "Bearer "+importer.exporter.authorizationToken) + } + response, err := importer.exporter.client.Do(request) + if err != nil { + return errors.New("Elasticsearch import request failed") + } + defer response.Body.Close() + if response.Request == nil || response.Request.URL.Scheme != importer.exporter.baseURL.Scheme || response.Request.URL.Host != importer.exporter.baseURL.Host { + return errors.New("Elasticsearch import response origin mismatch") + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return errors.New("Elasticsearch import returned non-success status") + } + if err := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1)).Decode(result); err != nil { + return errors.New("decode Elasticsearch import response") + } + return nil +} diff --git a/internal/releaseadapter/v071/search_import_test.go b/internal/releaseadapter/v071/search_import_test.go new file mode 100644 index 0000000..564d649 --- /dev/null +++ b/internal/releaseadapter/v071/search_import_test.go @@ -0,0 +1,220 @@ +package v071 + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +func TestSearchImporterReplacesAndRestoresScopedDocuments(t *testing.T) { + state := newSearchState(map[string]json.RawMessage{"es-old": mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-old","embedding":[0.1]}`)[0], "other": mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"other-kb","knowledge_id":"other","chunk_id":"other","embedding":[0.9]}`)[0]}) + server := httptest.NewServer(state) + defer server.Close() + config := SearchExporterConfig{BaseURL: server.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: server.Client(), PageSize: 100} + importer, err := NewSearchImporter(config) + if err != nil { + t.Fatal(err) + } + identity := SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"} + exporter, _ := NewSearchExporter(config) + mapping, err := exporter.MappingSHA256(context.Background()) + if err != nil { + t.Fatal(err) + } + newDocs := []SearchDocument{{ID: "es-new", Source: mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-new","embedding":[0.2]}`)[0]}} + newDigest, _ := searchDocumentsDigest(newDocs) + prior, err := importer.Replace(context.Background(), identity, SearchSnapshot{Documents: newDocs, MappingSHA256: mapping, DocumentsSHA256: newDigest}) + if err != nil { + t.Fatal(err) + } + if len(prior.Documents) != 1 || prior.Documents[0].ID != "es-old" { + t.Fatalf("prior=%#v", prior) + } + current, err := exporter.Export(context.Background(), identity) + if err != nil { + t.Fatal(err) + } + if len(current.Documents) != 1 || current.Documents[0].ID != "es-new" || state.has("other") == false { + t.Fatalf("current=%#v other=%v", current, state.has("other")) + } + if _, err := importer.Replace(context.Background(), identity, prior); err != nil { + t.Fatal(err) + } + restored, err := exporter.Export(context.Background(), identity) + if err != nil { + t.Fatal(err) + } + if len(restored.Documents) != 1 || restored.Documents[0].ID != "es-old" || state.has("es-new") { + t.Fatalf("restored=%#v", restored) + } + if err := importer.Disable(context.Background(), identity); err != nil { + t.Fatal(err) + } + if !state.disabled("es-old") || !state.has("es-old") || !state.has("other") { + t.Fatalf("disable state old=%v other=%v", state.disabled("es-old"), state.has("other")) + } + for _, requestPath := range state.paths { + if strings.Contains(requestPath, "knowledge-safe") || strings.Contains(requestPath, "kb-safe") { + t.Fatalf("identity leaked in URL: %s", requestPath) + } + } + joinedPaths := strings.Join(state.paths, "\n") + if !strings.Contains(joinedPaths, "/_bulk?refresh=true") || !strings.Contains(joinedPaths, "/chunks/_delete_by_query?refresh=true") { + t.Fatalf("refresh contract missing: %s", joinedPaths) + } +} + +type searchState struct { + mu sync.Mutex + docs map[string]json.RawMessage + paths []string +} + +func newSearchState(docs map[string]json.RawMessage) *searchState { return &searchState{docs: docs} } +func (s *searchState) has(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.docs[id] + return ok +} + +func (s *searchState) disabled(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + var value struct { + IsEnabled bool `json:"is_enabled"` + } + return json.Unmarshal(s.docs[id], &value) == nil && !value.IsEnabled +} +func (s *searchState) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + s.paths = append(s.paths, r.URL.RequestURI()) + if r.Header.Get("Authorization") != "Bearer safe-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + switch r.URL.Path { + case "/chunks/_mapping": + _, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`)) + case "/chunks/_search": + var q map[string]any + _ = json.NewDecoder(r.Body).Decode(&q) + identity := identityFromQuery(q) + hits := []map[string]any{} + for id, source := range s.docs { + if sourceMatches(source, identity) { + hits = append(hits, map[string]any{"_id": id, "_source": json.RawMessage(source)}) + } + } + _, _ = w.Write(mustJSON(map[string]any{"hits": map[string]any{"hits": hits}})) + case "/_bulk": + scanner := bufio.NewScanner(r.Body) + for scanner.Scan() { + var action struct { + Index struct { + ID string `json:"_id"` + } `json:"index"` + } + _ = json.Unmarshal(scanner.Bytes(), &action) + if !scanner.Scan() { + break + } + s.docs[action.Index.ID] = append([]byte(nil), scanner.Bytes()...) + } + _, _ = w.Write([]byte(`{"errors":false,"items":[]}`)) + case "/chunks/_delete_by_query": + var q map[string]any + _ = json.NewDecoder(r.Body).Decode(&q) + identity := identityFromQuery(q) + keep := idsFromQuery(q) + for id, source := range s.docs { + if sourceMatches(source, identity) && !keep[id] { + delete(s.docs, id) + } + } + _, _ = w.Write([]byte(`{"failures":[]}`)) + case "/chunks/_update_by_query": + var q map[string]any + _ = json.NewDecoder(r.Body).Decode(&q) + identity := identityFromQuery(q) + for id, source := range s.docs { + if sourceMatches(source, identity) { + var value map[string]any + _ = json.Unmarshal(source, &value) + value["is_enabled"] = false + s.docs[id] = mustJSON(value) + } + } + _, _ = w.Write([]byte(`{"failures":[]}`)) + default: + http.NotFound(w, r) + } +} +func identityFromQuery(q map[string]any) SearchIdentity { + raw, _ := json.Marshal(q) + var values map[string]any + _ = json.Unmarshal(raw, &values) + text := string(raw) + id := SearchIdentity{} + for _, part := range []struct { + key string + target *string + }{{"knowledge_base_id", &id.KnowledgeBaseID}, {"knowledge_id", &id.KnowledgeID}} { + found := false + for _, marker := range []string{"\"" + part.key + "\":\"", "\"" + part.key + ".keyword\":\""} { + if at := strings.Index(text, marker); at >= 0 { + rest := text[at+len(marker):] + if end := strings.Index(rest, "\""); end >= 0 { + *part.target = rest[:end] + } + found = true + break + } + } + if found { + continue + } + } + for _, marker := range []string{`"tenant_id":10000`, `"tenant_id.keyword":10000`} { + if strings.Contains(text, marker) { + id.TenantID = 10000 + break + } + } + return id +} +func idsFromQuery(q map[string]any) map[string]bool { + result := map[string]bool{} + var walk func(any) + walk = func(value any) { + switch typed := value.(type) { + case map[string]any: + if rawIDs, ok := typed["ids"].(map[string]any); ok { + if values, ok := rawIDs["values"].([]any); ok { + for _, value := range values { + if id, ok := value.(string); ok { + result[id] = true + } + } + } + } + for _, child := range typed { + walk(child) + } + case []any: + for _, child := range typed { + walk(child) + } + } + } + walk(q) + return result +} +func sourceMatches(raw json.RawMessage, id SearchIdentity) bool { return searchHitMatches(raw, id) } +func mustJSON(value any) []byte { result, _ := json.Marshal(value); return result } diff --git a/internal/releaseadapter/v071/search_test.go b/internal/releaseadapter/v071/search_test.go new file mode 100644 index 0000000..fc88dc7 --- /dev/null +++ b/internal/releaseadapter/v071/search_test.go @@ -0,0 +1,159 @@ +package v071 + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestSearchExporterReadsOnlyScopedDocumentsAndMapping(t *testing.T) { + requests := []recordedSearchRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if r.Body != nil { + _ = json.NewDecoder(r.Body).Decode(&body) + } + requests = append(requests, recordedSearchRequest{method: r.Method, path: r.URL.Path, body: body, authorization: r.Header.Get("Authorization")}) + switch r.URL.Path { + case "/chunks/_mapping": + _, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"knowledge_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"tenant_id":{"type":"keyword"}}}}}`)) + case "/chunks/_search": + if len(requests) == 2 { + _, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-1","embedding":[0.1,0.2]} ,"sort":["es-1"]}]}}`)) + } else { + _, _ = w.Write([]byte(`{"hits":{"hits":[]}}`)) + } + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: server.Client(), PageSize: 1}) + if err != nil { + t.Fatal(err) + } + snapshot, err := exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Documents) != 1 || snapshot.Documents[0].ID != "es-1" || len(snapshot.MappingSHA256) != 64 || len(snapshot.DocumentsSHA256) != 64 { + t.Fatalf("snapshot=%#v", snapshot) + } + if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || requests[2].method != http.MethodPost { + t.Fatalf("requests=%#v", requests) + } + for _, request := range requests { + if request.authorization != "Bearer safe-token" { + t.Fatalf("authorization=%q", request.authorization) + } + if strings.Contains(strings.ToLower(request.path), "knowledge-safe") { + t.Fatalf("identity leaked into URL: %s", request.path) + } + } + encoded, _ := json.Marshal(requests[1].body) + for _, required := range []string{`"tenant_id":10000`, `"knowledge_base_id":"kb-safe"`, `"knowledge_id":"knowledge-safe"`} { + if !strings.Contains(string(encoded), required) { + t.Fatalf("query missing %s: %s", required, encoded) + } + } + second, _ := json.Marshal(requests[2].body) + if !strings.Contains(string(second), `"from":1`) { + t.Fatalf("pagination missing: %s", second) + } +} + +func TestSearchExporterReadsMappingContractWithoutDocumentQuery(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/chunks/_mapping" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`)) + })) + defer server.Close() + exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10}) + if err != nil { + t.Fatal(err) + } + digest, err := exporter.MappingSHA256(context.Background()) + if err != nil || len(digest) != 64 || requests != 1 { + t.Fatalf("digest=%q requests=%d err=%v", digest, requests, err) + } +} + +func TestSearchExporterRejectsCrossScopedHit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "_mapping") { + _, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`)) + return + } + _, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"tenant_id":10000,"knowledge_base_id":"wrong","knowledge_id":"knowledge-safe"},"sort":["es-1"]}]}}`)) + })) + defer server.Close() + exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10}) + if err != nil { + t.Fatal(err) + } + _, err = exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}) + if err == nil || strings.Contains(err.Error(), "wrong") || strings.Contains(err.Error(), "knowledge-safe") { + t.Fatalf("error=%v", err) + } +} + +type recordedSearchRequest struct { + method, path, authorization string + body map[string]any +} + +// TestSearchExporterHandlesCommunityIndexShape locks the fix for the real +// WeKnora v0.7.1 default index shape discovered during cross-environment +// release verification: ID fields mapped as "text" with a ".keyword" subfield +// and documents without any tenant_id field. Export must exact-match through +// the subfields, omit the absent tenant filter, and accept tenant-less hits. +func TestSearchExporterHandlesCommunityIndexShape(t *testing.T) { + seenQueries := []map[string]any{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/chunks/_mapping": + _, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":1024},"knowledge_id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"knowledge_base_id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}}`)) + case "/chunks/_search": + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + seenQueries = append(seenQueries, body) + _, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"content":"text","knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-1","embedding":[0.1],"is_enabled":true}}]}}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10}) + if err != nil { + t.Fatal(err) + } + snapshot, err := exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Documents) != 1 || snapshot.Documents[0].ID != "es-1" { + t.Fatalf("snapshot=%#v", snapshot) + } + if len(seenQueries) == 0 { + t.Fatal("no search issued") + } + encoded, _ := json.Marshal(seenQueries[0]) + text := string(encoded) + for _, required := range []string{`"knowledge_base_id.keyword":"kb-safe"`, `"knowledge_id.keyword":"knowledge-safe"`} { + if !strings.Contains(text, required) { + t.Fatalf("community-shape query missing %s: %s", required, text) + } + } + if strings.Contains(text, "tenant_id") { + t.Fatalf("query must omit absent tenant_id field: %s", text) + } +} diff --git a/internal/releaseadapter/v071/three_lane_integration_test.go b/internal/releaseadapter/v071/three_lane_integration_test.go new file mode 100644 index 0000000..f17f608 --- /dev/null +++ b/internal/releaseadapter/v071/three_lane_integration_test.go @@ -0,0 +1,158 @@ +package v071 + +import ( + "bytes" + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/platform/weknora" + "knowledge-governance/internal/releasebundle" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +func TestThreeLaneReleaseAndRollbackV071Integration(t *testing.T) { + sourceDatabaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL") + targetDatabaseURL := os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL") + minioEndpoint := os.Getenv("TEST_MINIO_ENDPOINT") + accessKey := os.Getenv("TEST_MINIO_ACCESS_KEY") + secretKey := os.Getenv("TEST_MINIO_SECRET_KEY") + if sourceDatabaseURL == "" || targetDatabaseURL == "" || minioEndpoint == "" || accessKey == "" || secretKey == "" { + t.Skip("three-lane integration environment is not set") + } + ctx := context.Background() + sourcePool, err := pgxpool.New(ctx, sourceDatabaseURL) + if err != nil { + t.Fatal(err) + } + defer sourcePool.Close() + targetPool, err := pgxpool.New(ctx, targetDatabaseURL) + if err != nil { + t.Fatal(err) + } + defer targetPool.Close() + + sourceSearchState := newSearchState(map[string]json.RawMessage{"es-release": mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-release-safe","knowledge_id":"knowledge-release-safe","chunk_id":"chunk-release-safe","content":"Synthetic release content","is_enabled":true}`)[0]}) + targetSearchState := newSearchState(map[string]json.RawMessage{}) + sourceSearchServer := httptest.NewServer(sourceSearchState) + defer sourceSearchServer.Close() + targetSearchServer := httptest.NewServer(targetSearchState) + defer targetSearchServer.Close() + sourceSearch, err := NewSearchExporter(SearchExporterConfig{BaseURL: sourceSearchServer.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: sourceSearchServer.Client(), PageSize: 100}) + if err != nil { + t.Fatal(err) + } + targetSearchConfig := SearchExporterConfig{BaseURL: targetSearchServer.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: targetSearchServer.Client(), PageSize: 100} + targetSearch, err := NewSearchImporter(targetSearchConfig) + if err != nil { + t.Fatal(err) + } + mappingSHA, err := sourceSearch.MappingSHA256(ctx) + if err != nil { + t.Fatal(err) + } + + sourceObjects, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: minioEndpoint, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: "release-source", ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + targetObjects, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: minioEndpoint, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: "release-target", ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + if err := sourceObjects.EnsureBucket(ctx); err != nil { + t.Fatal(err) + } + if err := targetObjects.EnsureBucket(ctx); err != nil { + t.Fatal(err) + } + if err := sourceObjects.Put(ctx, "minio", "10000/knowledge-release-safe/synthetic.txt", bytes.NewReader([]byte("Synthetic release content")), 25); err != nil { + t.Fatal(err) + } + if err := sourceObjects.Put(ctx, "minio", "10000/knowledge-release-safe/image.png", bytes.NewReader([]byte("synthetic-image!")), 16); err != nil { + t.Fatal(err) + } + objectExporter, _ := NewObjectExporter(sourceObjects, ObjectExporterLimits{}) + objectImporter, _ := NewObjectImporter(targetObjects, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "minio"}) + postgresExporter, _ := NewPostgresExporter(sourcePool) + postgresImporter, _ := NewPostgresImporter(targetPool, PostgresImporterConfig{TargetStorageBackendID: "target-backend", TargetProvider: "minio"}) + contract := releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-release-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: mappingSHA} + exporter, err := NewDocumentExporter(DocumentExporterConfig{Contract: &stubSourceContract{contract: contract}, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: 10000, EmbeddingModel: "bge-m3", Now: func() time.Time { return time.Date(2026, 9, 6, 12, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + archiveRoot := t.TempDir() + if err := os.Chmod(archiveRoot, 0o700); err != nil { + t.Fatal(err) + } + archive, err := NewBundleArchive(archiveRoot, 128*1024*1024) + if err != nil { + t.Fatal(err) + } + verification := &verificationClient{knowledge: weknora.Knowledge{ID: "knowledge-release-safe", KnowledgeBaseID: "kb-release-safe", ParseStatus: "completed"}, chunkTotal: 1, search: []weknora.SearchResult{{ID: "chunk-release-safe", KnowledgeID: "knowledge-release-safe", Score: 0.8}}} + verifier, _ := NewWeKnoraVerifier(verification) + target, err := NewDocumentTarget(DocumentTargetConfig{Archive: archive, Contract: &stubContractInspector{contract: contract}, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier}) + if err != nil { + t.Fatal(err) + } + publisher, err := releaseworkflow.NewSidecarPublisher(exporter, target) + if err != nil { + t.Fatal(err) + } + + first := releaseworkflow.PlanItem{ReleaseExecutionID: "rx-release-1", PublicationID: "tp-release-1", SourceItemVersionID: "version-release-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "review-release-1", TestKnowledgeBaseID: "kb-release-safe", KnowledgeID: "knowledge-release-safe", DecidedVisibility: "internal"} + if _, err := exporter.Export(ctx, first); err != nil { + t.Fatalf("export preflight: %v", err) + } + firstResult, err := publisher.Publish(ctx, first) + if err != nil { + t.Fatal(err) + } + if firstResult.PreviousProductionRevision != "absent" || firstResult.ProductionRevision != "review-release-1" { + t.Fatalf("first=%#v", firstResult) + } + second := first + second.ReleaseExecutionID = "rx-release-2" + second.PublicationID = "tp-release-2" + second.SourceRevision = "source-v2" + second.ReleaseRevision = "review-release-2" + second.PreviousProductionRevision = firstResult.ProductionRevision + secondResult, err := publisher.Publish(ctx, second) + if err != nil { + t.Fatal(err) + } + if secondResult.PreviousProductionRevision != "review-release-1" || secondResult.ProductionRevision != "review-release-2" { + t.Fatalf("second=%#v", secondResult) + } + rollback, err := publisher.Rollback(ctx, second) + if err != nil { + t.Fatal(err) + } + if rollback.ProductionRevision != "review-release-1" { + t.Fatalf("rollback=%#v", rollback) + } + + targetSnapshot, err := NewPostgresExporter(targetPool) + if err != nil { + t.Fatal(err) + } + actual, err := targetSnapshot.Export(ctx, PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-release-safe", KnowledgeID: "knowledge-release-safe"}) + if err != nil { + t.Fatal(err) + } + if actual.Table("chunks").RowCount != 1 || !targetSearchState.has("es-release") { + t.Fatalf("target rows=%d search=%v", actual.Table("chunks").RowCount, targetSearchState.has("es-release")) + } + for _, key := range []string{"10000/knowledge-release-safe/synthetic.txt", "10000/knowledge-release-safe/image.png"} { + size, _, err := targetObjects.Stat(ctx, "minio", key) + if err != nil || size == 0 { + t.Fatalf("asset=%s size=%d err=%v", filepath.Base(key), size, err) + } + } +} diff --git a/internal/releaseadapter/v071/verifier.go b/internal/releaseadapter/v071/verifier.go new file mode 100644 index 0000000..b37bc29 --- /dev/null +++ b/internal/releaseadapter/v071/verifier.go @@ -0,0 +1,141 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "strings" + "unicode/utf8" + + "knowledge-governance/internal/platform/weknora" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type WeKnoraVerificationClient interface { + GetKnowledge(ctx context.Context, knowledgeID string) (weknora.Knowledge, error) + ListChunks(ctx context.Context, knowledgeID string, page, pageSize int, chunkTypes []string) ([]weknora.Chunk, int64, error) + SearchKnowledge(ctx context.Context, request weknora.SearchRequest) ([]weknora.SearchResult, error) +} + +type WeKnoraVerifier struct { + client WeKnoraVerificationClient +} + +func NewWeKnoraVerifier(client WeKnoraVerificationClient) (*WeKnoraVerifier, error) { + if client == nil { + return nil, errors.New("WeKnora verification client is required") + } + return &WeKnoraVerifier{client: client}, nil +} + +func (verifier *WeKnoraVerifier) Verify(ctx context.Context, item releaseworkflow.PlanItem, bundle *DocumentBundle) error { + if bundle == nil || validateDocumentBundle(bundle) != nil || bundle.Manifest.Contract.KnowledgeID != item.KnowledgeID || bundle.Manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID { + return errors.New("invalid WeKnora verification bundle") + } + knowledge, err := verifier.client.GetKnowledge(ctx, item.KnowledgeID) + if err != nil || knowledge.ID != item.KnowledgeID || knowledge.KnowledgeBaseID != item.TestKnowledgeBaseID || knowledge.ParseStatus != "completed" { + return errors.New("production WeKnora knowledge verification failed") + } + _, chunkTotal, err := verifier.client.ListChunks(ctx, item.KnowledgeID, 1, 1, bundleChunkTypes(bundle)) + expectedChunks := bundle.Postgres.Table("chunks").RowCount + if err != nil || expectedChunks < 1 || chunkTotal != int64(expectedChunks) { + return errors.New("production WeKnora chunk verification failed") + } + query := bundleProbeQuery(bundle) + if query == "" { + return errors.New("production WeKnora retrieval probe is unavailable") + } + results, err := verifier.client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{item.KnowledgeID}}) + if err != nil { + return errors.New("production WeKnora retrieval verification failed") + } + for _, result := range results { + if result.KnowledgeID == item.KnowledgeID { + return nil + } + } + return errors.New("production WeKnora retrieval verification failed") +} + +func (verifier *WeKnoraVerifier) VerifyDisabled(ctx context.Context, item releaseworkflow.PlanItem) error { + if !validSearchIdentifier(item.KnowledgeID) { + return errors.New("invalid disabled WeKnora verification plan") + } + results, err := verifier.client.SearchKnowledge(ctx, weknora.SearchRequest{ + Query: "governance retrieval disable verification sentinel", + KnowledgeIDs: []string{item.KnowledgeID}, + }) + if err != nil { + return errors.New("disabled WeKnora retrieval verification failed") + } + for _, result := range results { + if result.KnowledgeID == item.KnowledgeID { + return errors.New("disabled WeKnora knowledge remains retrievable") + } + } + return nil +} + +// bundleChunkTypes lists the distinct chunk_type values sealed in the bundle. +// WeKnora's chunk list endpoint defaults to text-only results, so verification +// must ask for exactly the types the released document contains. +func bundleChunkTypes(bundle *DocumentBundle) []string { + seen := map[string]bool{} + types := make([]string, 0, 3) + for _, raw := range bundle.Postgres.Table("chunks").Rows { + var row struct { + ChunkType string `json:"chunk_type"` + } + if json.Unmarshal(raw, &row) != nil { + continue + } + chunkType := strings.TrimSpace(row.ChunkType) + if chunkType == "" || seen[chunkType] || !validSearchIdentifier(chunkType) { + continue + } + seen[chunkType] = true + types = append(types, chunkType) + } + return types +} + +func bundleProbeQuery(bundle *DocumentBundle) string { + for _, document := range bundle.Search.Documents { + var value map[string]any + if json.Unmarshal(document.Source, &value) != nil { + continue + } + for _, key := range []string{"content", "question", "answer", "title"} { + if text, ok := value[key].(string); ok { + if query := boundedProbeText(text); query != "" { + return query + } + } + } + } + for _, row := range bundle.Postgres.Table("knowledges").Rows { + var value struct { + Title string `json:"title"` + } + if json.Unmarshal(row, &value) == nil { + if query := boundedProbeText(value.Title); query != "" { + return query + } + } + } + return "" +} + +func boundedProbeText(value string) string { + value = strings.Join(strings.Fields(value), " ") + if value == "" { + return "" + } + if utf8.RuneCountInString(value) <= 128 { + return value + } + runes := []rune(value) + return string(runes[:128]) +} + +var _ ProductionVerifier = (*WeKnoraVerifier)(nil) diff --git a/internal/releaseadapter/v071/verifier_test.go b/internal/releaseadapter/v071/verifier_test.go new file mode 100644 index 0000000..5d28d62 --- /dev/null +++ b/internal/releaseadapter/v071/verifier_test.go @@ -0,0 +1,61 @@ +package v071 + +import ( + "context" + "testing" + + "knowledge-governance/internal/platform/weknora" +) + +func TestWeKnoraVerifierChecksKnowledgeChunksAndScopedRetrieval(t *testing.T) { + bundle := testDocumentBundle(t) + client := &verificationClient{knowledge: weknora.Knowledge{ID: "knowledge-safe", KnowledgeBaseID: "kb-safe", ParseStatus: "completed"}, chunkTotal: 1, search: []weknora.SearchResult{{ID: "chunk-safe", KnowledgeID: "knowledge-safe", Score: 0.8}}} + verifier, err := NewWeKnoraVerifier(client) + if err != nil { + t.Fatal(err) + } + if err := verifier.Verify(context.Background(), releasePlanForArchive(), bundle); err != nil { + t.Fatal(err) + } + if client.searchRequest.Query == "" || len(client.searchRequest.KnowledgeIDs) != 1 || client.searchRequest.KnowledgeIDs[0] != "knowledge-safe" { + t.Fatalf("request=%#v", client.searchRequest) + } +} + +func TestWeKnoraVerifierRejectsWrongRetrievalAndConfirmsDisable(t *testing.T) { + bundle := testDocumentBundle(t) + client := &verificationClient{knowledge: weknora.Knowledge{ID: "knowledge-safe", KnowledgeBaseID: "kb-safe", ParseStatus: "completed"}, chunkTotal: 1, search: []weknora.SearchResult{{KnowledgeID: "other"}}} + verifier, _ := NewWeKnoraVerifier(client) + if err := verifier.Verify(context.Background(), releasePlanForArchive(), bundle); err == nil { + t.Fatal("wrong retrieval accepted") + } + client.search = nil + if err := verifier.VerifyDisabled(context.Background(), releasePlanForArchive()); err != nil { + t.Fatal(err) + } + client.search = []weknora.SearchResult{{KnowledgeID: "knowledge-safe"}} + if err := verifier.VerifyDisabled(context.Background(), releasePlanForArchive()); err == nil { + t.Fatal("retrievable disabled knowledge accepted") + } +} + +type verificationClient struct { + knowledge weknora.Knowledge + knowledgeErr error + chunkTotal int64 + chunkErr error + search []weknora.SearchResult + searchErr error + searchRequest weknora.SearchRequest +} + +func (c *verificationClient) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + return c.knowledge, c.knowledgeErr +} +func (c *verificationClient) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) { + return nil, c.chunkTotal, c.chunkErr +} +func (c *verificationClient) SearchKnowledge(_ context.Context, request weknora.SearchRequest) ([]weknora.SearchResult, error) { + c.searchRequest = request + return c.search, c.searchErr +} diff --git a/internal/releasebundle/manifest.go b/internal/releasebundle/manifest.go new file mode 100644 index 0000000..bd43f46 --- /dev/null +++ b/internal/releasebundle/manifest.go @@ -0,0 +1,187 @@ +package releasebundle + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "sort" + "strings" + "time" +) + +const ( + FormatVersionV1 = "weknora-release-bundle/v1" + WeKnoraVersionV071 = "v0.7.1" + WeKnoraMigrationV071 = 74 + EmbeddingDimensionV071 = 1024 +) + +var allowedTables = map[string]bool{ + "knowledges": true, + "chunks": true, + "knowledge_tags": true, + "knowledge_tag_relations": true, + "resources": true, + "resource_bindings": true, + "wiki_pages": true, + "wiki_folders": true, +} + +type Manifest struct { + FormatVersion string `json:"format_version"` + ExecutionID string `json:"execution_id"` + PublicationID string `json:"publication_id"` + SourceItemVersionID string `json:"source_item_version_id"` + SourceRevision string `json:"source_revision"` + ParseRevision string `json:"parse_revision"` + ReleaseRevision string `json:"release_revision"` + CreatedAt time.Time `json:"created_at"` + Contract Contract `json:"contract"` + Tables []TableDigest `json:"tables"` + Search SearchDigest `json:"search"` + Assets AssetDigest `json:"assets"` + BundleSHA256 string `json:"bundle_sha256"` +} + +type Contract struct { + WeKnoraVersion string `json:"weknora_version"` + MigrationVersion int `json:"migration_version"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + RetrieverEngine string `json:"retriever_engine"` + EmbeddingModel string `json:"embedding_model"` + EmbeddingDimension int `json:"embedding_dimension"` + ElasticsearchMappingSHA256 string `json:"elasticsearch_mapping_sha256"` +} + +type TargetContract struct { + WeKnoraVersion string + MigrationVersion int + TenantID uint64 + KnowledgeBaseID string + RetrieverEngine string + EmbeddingModel string + EmbeddingDimension int + ElasticsearchMappingSHA256 string +} + +type TableDigest struct { + Name string `json:"name"` + RowCount int `json:"row_count"` + SHA256 string `json:"sha256"` +} + +type SearchDigest struct { + DocumentCount int `json:"document_count"` + SHA256 string `json:"sha256"` +} + +type AssetDigest struct { + ObjectCount int `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + SHA256 string `json:"sha256"` +} + +func Seal(manifest Manifest) (Manifest, error) { + manifest.BundleSHA256 = "" + manifest.Tables = append([]TableDigest(nil), manifest.Tables...) + sort.Slice(manifest.Tables, func(left, right int) bool { return manifest.Tables[left].Name < manifest.Tables[right].Name }) + if err := validateManifestFields(manifest); err != nil { + return Manifest{}, err + } + digest, err := manifestDigest(manifest) + if err != nil { + return Manifest{}, err + } + manifest.BundleSHA256 = digest + return manifest, nil +} + +func Validate(manifest Manifest, target TargetContract) error { + if err := validateManifestFields(manifest); err != nil { + return err + } + if !validSHA256(manifest.BundleSHA256) { + return errors.New("invalid bundle digest") + } + expected, err := manifestDigest(manifest) + if err != nil { + return err + } + if expected != manifest.BundleSHA256 { + return errors.New("bundle digest mismatch") + } + contract := manifest.Contract + if target.WeKnoraVersion != WeKnoraVersionV071 || target.MigrationVersion != WeKnoraMigrationV071 || + contract.WeKnoraVersion != target.WeKnoraVersion || contract.MigrationVersion != target.MigrationVersion || + contract.TenantID != target.TenantID || contract.KnowledgeBaseID != target.KnowledgeBaseID || + contract.RetrieverEngine != target.RetrieverEngine || contract.EmbeddingModel != target.EmbeddingModel || + contract.EmbeddingDimension != target.EmbeddingDimension || + contract.ElasticsearchMappingSHA256 != target.ElasticsearchMappingSHA256 { + return errors.New("target release contract mismatch") + } + return nil +} + +func validateManifestFields(manifest Manifest) error { + if manifest.FormatVersion != FormatVersionV1 || manifest.Contract.WeKnoraVersion != WeKnoraVersionV071 || + manifest.Contract.MigrationVersion != WeKnoraMigrationV071 || manifest.Contract.EmbeddingDimension != EmbeddingDimensionV071 || + manifest.Contract.RetrieverEngine != "elasticsearch_v8" || manifest.Contract.TenantID == 0 || manifest.CreatedAt.IsZero() { + return errors.New("unsupported release contract") + } + for _, value := range []string{manifest.ExecutionID, manifest.PublicationID, manifest.SourceItemVersionID, + manifest.SourceRevision, manifest.ParseRevision, manifest.ReleaseRevision, manifest.Contract.KnowledgeBaseID, + manifest.Contract.KnowledgeID, manifest.Contract.EmbeddingModel} { + if !validIdentifier(value) { + return errors.New("invalid release identity") + } + } + if !validSHA256(manifest.Contract.ElasticsearchMappingSHA256) || !validSHA256(manifest.Search.SHA256) || !validSHA256(manifest.Assets.SHA256) || + manifest.Search.DocumentCount < 0 || manifest.Assets.ObjectCount < 0 || manifest.Assets.TotalBytes < 0 { + return errors.New("invalid release digest") + } + seen := make(map[string]bool, len(manifest.Tables)) + hasKnowledge, hasChunks := false, false + for _, table := range manifest.Tables { + if !allowedTables[table.Name] || seen[table.Name] || table.RowCount < 0 || !validSHA256(table.SHA256) { + return errors.New("invalid release table") + } + seen[table.Name] = true + hasKnowledge = hasKnowledge || table.Name == "knowledges" && table.RowCount == 1 + hasChunks = hasChunks || table.Name == "chunks" && table.RowCount > 0 + } + if !hasKnowledge || !hasChunks { + return errors.New("release manifest requires one knowledge and at least one chunk") + } + return nil +} + +func manifestDigest(manifest Manifest) (string, error) { + manifest.BundleSHA256 = "" + manifest.Tables = append([]TableDigest(nil), manifest.Tables...) + sort.Slice(manifest.Tables, func(left, right int) bool { return manifest.Tables[left].Name < manifest.Tables[right].Name }) + payload, err := json.Marshal(manifest) + if err != nil { + return "", errors.New("encode release manifest") + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} + +func validSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} + +func validIdentifier(value string) bool { + return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\") +} diff --git a/internal/releasebundle/manifest_test.go b/internal/releasebundle/manifest_test.go new file mode 100644 index 0000000..0c85b64 --- /dev/null +++ b/internal/releasebundle/manifest_test.go @@ -0,0 +1,119 @@ +package releasebundle + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestSealAndValidateV071Manifest(t *testing.T) { + manifest := Manifest{ + FormatVersion: "weknora-release-bundle/v1", + ExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", + SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", + CreatedAt: time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC), + Contract: Contract{ + WeKnoraVersion: "v0.7.1", MigrationVersion: 74, TenantID: 10000, + KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: 1024, + ElasticsearchMappingSHA256: strings.Repeat("a", 64), + }, + Tables: []TableDigest{ + {Name: "chunks", RowCount: 2, SHA256: strings.Repeat("b", 64)}, + {Name: "knowledge_tags", RowCount: 1, SHA256: strings.Repeat("f", 64)}, + {Name: "knowledges", RowCount: 1, SHA256: strings.Repeat("c", 64)}, + }, + Search: SearchDigest{DocumentCount: 2, SHA256: strings.Repeat("d", 64)}, + Assets: AssetDigest{ObjectCount: 1, TotalBytes: 42, SHA256: strings.Repeat("e", 64)}, + } + sealed, err := Seal(manifest) + if err != nil { + t.Fatal(err) + } + if len(sealed.BundleSHA256) != 64 { + t.Fatalf("bundle sha=%q", sealed.BundleSHA256) + } + target := TargetContract{ + WeKnoraVersion: "v0.7.1", MigrationVersion: 74, TenantID: 10000, + KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", + EmbeddingModel: "bge-m3", EmbeddingDimension: 1024, + ElasticsearchMappingSHA256: strings.Repeat("a", 64), + } + if err := Validate(sealed, target); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(sealed) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"endpoint", "password", "secret", "api_key", "physical_path", "database_url"} { + if strings.Contains(strings.ToLower(string(encoded)), forbidden) { + t.Fatalf("manifest leaked forbidden field %q", forbidden) + } + } +} + +func TestSealCanonicalizesTableOrder(t *testing.T) { + base := validManifest() + first, err := Seal(base) + if err != nil { + t.Fatal(err) + } + base.Tables[0], base.Tables[1] = base.Tables[1], base.Tables[0] + second, err := Seal(base) + if err != nil { + t.Fatal(err) + } + if first.BundleSHA256 != second.BundleSHA256 { + t.Fatalf("digest differs: %s vs %s", first.BundleSHA256, second.BundleSHA256) + } +} + +func TestValidateRejectsContractDriftAndUnknownTables(t *testing.T) { + manifest, err := Seal(validManifest()) + if err != nil { + t.Fatal(err) + } + target := validTarget() + tests := []struct { + name string + mutate func(*Manifest, *TargetContract) + }{ + {name: "version", mutate: func(_ *Manifest, target *TargetContract) { target.WeKnoraVersion = "v0.7.2" }}, + {name: "migration", mutate: func(_ *Manifest, target *TargetContract) { target.MigrationVersion = 73 }}, + {name: "kb", mutate: func(_ *Manifest, target *TargetContract) { target.KnowledgeBaseID = "other" }}, + {name: "embedding", mutate: func(_ *Manifest, target *TargetContract) { target.EmbeddingDimension = 768 }}, + {name: "mapping", mutate: func(_ *Manifest, target *TargetContract) { target.ElasticsearchMappingSHA256 = strings.Repeat("f", 64) }}, + {name: "unknown table", mutate: func(manifest *Manifest, _ *TargetContract) { + manifest.Tables = append(manifest.Tables, TableDigest{Name: "users", RowCount: 1, SHA256: strings.Repeat("1", 64)}) + }}, + {name: "tampered bundle", mutate: func(manifest *Manifest, _ *TargetContract) { manifest.Search.DocumentCount++ }}, + {name: "missing parse revision", mutate: func(manifest *Manifest, _ *TargetContract) { manifest.ParseRevision = "" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + candidate, contract := manifest, target + candidate.Tables = append([]TableDigest(nil), manifest.Tables...) + tt.mutate(&candidate, &contract) + if err := Validate(candidate, contract); err == nil { + t.Fatal("Validate() error=nil") + } + }) + } +} + +func validManifest() Manifest { + return Manifest{ + FormatVersion: "weknora-release-bundle/v1", ExecutionID: "rx-safe", PublicationID: "tp-safe", + SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", + CreatedAt: time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC), + Contract: Contract{WeKnoraVersion: "v0.7.1", MigrationVersion: 74, TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: 1024, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}, + Tables: []TableDigest{{Name: "knowledges", RowCount: 1, SHA256: strings.Repeat("b", 64)}, {Name: "chunks", RowCount: 1, SHA256: strings.Repeat("c", 64)}}, + Search: SearchDigest{DocumentCount: 1, SHA256: strings.Repeat("d", 64)}, Assets: AssetDigest{SHA256: strings.Repeat("e", 64)}, + } +} + +func validTarget() TargetContract { + return TargetContract{WeKnoraVersion: "v0.7.1", MigrationVersion: 74, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: 1024, ElasticsearchMappingSHA256: strings.Repeat("a", 64)} +} diff --git a/internal/workflows/catalogrefresh/handler.go b/internal/workflows/catalogrefresh/handler.go new file mode 100644 index 0000000..4040337 --- /dev/null +++ b/internal/workflows/catalogrefresh/handler.go @@ -0,0 +1,191 @@ +package catalogrefresh + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os/exec" + "path/filepath" + "strings" + "time" + "unicode" + + "knowledge-governance/internal/jobs" +) + +type Request struct { + ConnectionID string `json:"connection_id"` + SourceType string `json:"source_type"` + SourceName string `json:"source_name"` + Stage string `json:"stage"` + Progress int `json:"progress"` + Message string `json:"message"` + Processed int `json:"processed"` + Total int `json:"total"` +} +type Progress struct { + Stage string `json:"stage"` + Progress int `json:"progress"` + Message string `json:"message"` + Processed int `json:"processed"` + Total int `json:"total"` + SourceName string `json:"source_name,omitempty"` + CurrentDirectory string `json:"current_directory,omitempty"` + ScannedDirectories int `json:"scanned_directories,omitempty"` + DiscoveredDirectories int `json:"discovered_directories,omitempty"` + ScannedFiles int `json:"scanned_files,omitempty"` + EstimatedTotalFiles int `json:"estimated_total_files,omitempty"` + TotalIsEstimate bool `json:"total_is_estimate,omitempty"` +} +type Store interface { + UpdateCatalogRefreshProgress(context.Context, string, string, Progress, time.Time) error +} + +var progressForbiddenSubstrings = []string{"http://", "https://", "token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "api-key", "apikey"} + +type Runner interface { + Run(context.Context, Request, func(Progress) error) error +} +type Config struct { + Store Store + Runner Runner + Now func() time.Time +} +type Handler struct { + store Store + runner Runner + now func() time.Time +} + +func NewHandler(config Config) (*Handler, error) { + if config.Store == nil || config.Runner == nil { + return nil, errors.New("catalog refresh dependencies are required") + } + if config.Now == nil { + config.Now = time.Now + } + return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil +} +func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error { + var request Request + decoder := json.NewDecoder(bytes.NewReader(job.Payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil || request.ConnectionID == "" || request.SourceName == "" || (request.SourceType != "pp" && request.SourceType != "dingtalk" && request.SourceType != "alist") || job.LeaseOwner == "" { + return errors.New("catalog_refresh_invalid_payload") + } + var last Progress + return handler.runner.Run(ctx, request, func(progress Progress) error { + progress = clampProgress(last, progress) + if !validProgress(progress) { + return errors.New("catalog_refresh_invalid_progress") + } + last = progress + return handler.store.UpdateCatalogRefreshProgress(ctx, job.ID, job.LeaseOwner, progress, handler.now().UTC()) + }) +} +func validProgress(progress Progress) bool { + if progress.Progress < 0 || progress.Progress > 100 || progress.Processed < 0 || progress.Total < 0 || + progress.ScannedDirectories < 0 || progress.DiscoveredDirectories < 0 || progress.ScannedFiles < 0 || progress.EstimatedTotalFiles < 0 || + strings.TrimSpace(progress.Stage) == "" || len(progress.Stage) > 32 || + strings.TrimSpace(progress.Message) == "" || len(progress.Message) > 300 || + len(progress.CurrentDirectory) > 1024 || len(progress.SourceName) > 255 { + return false + } + for _, text := range []string{progress.Stage, progress.Message, progress.CurrentDirectory, progress.SourceName} { + for _, r := range text { + if unicode.IsControl(r) && r != ' ' { + return false + } + } + lowered := strings.ToLower(text) + for _, forbidden := range progressForbiddenSubstrings { + if strings.Contains(lowered, forbidden) { + return false + } + } + } + return true +} + +// clampProgress enforces that display progress never regresses when discovered +// totals shift mid-scan (BFS estimates grow), and never clamps an explicit +// failure snapshot. +func clampProgress(previous, next Progress) Progress { + if next.Stage == "failed" { + return next + } + if next.Total < previous.Total && previous.Total >= next.Processed { + next.Total = previous.Total + } + if next.Total > 0 { + estimated := next.Processed * 100 / next.Total + if next.Progress < previous.Progress && previous.Progress <= estimated { + next.Progress = previous.Progress + } + } + if next.ScannedDirectories < previous.ScannedDirectories { + next.ScannedDirectories = previous.ScannedDirectories + } + if next.ScannedFiles < previous.ScannedFiles { + next.ScannedFiles = previous.ScannedFiles + } + if next.DiscoveredDirectories < previous.DiscoveredDirectories { + next.DiscoveredDirectories = previous.DiscoveredDirectories + } + if next.EstimatedTotalFiles < previous.EstimatedTotalFiles { + next.EstimatedTotalFiles = previous.EstimatedTotalFiles + } + return next +} + +type CommandRunner struct { + PythonPath string + ScriptPath string +} + +func NewCommandRunner(pythonPath, scriptPath string) (*CommandRunner, error) { + if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) { + return nil, errors.New("catalog refresh command paths must be absolute") + } + return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath}, nil +} +func (runner *CommandRunner) Run(ctx context.Context, request Request, report func(Progress) error) error { + command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, "--connection-id", request.ConnectionID, "--source-type", request.SourceType, "--source-name", request.SourceName) + stdout, err := command.StdoutPipe() + if err != nil { + return errors.New("catalog_refresh_start_failed") + } + command.Stderr = io.Discard + if err = command.Start(); err != nil { + return errors.New("catalog_refresh_start_failed") + } + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 4096), 1<<20) + for scanner.Scan() { + decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes())) + decoder.DisallowUnknownFields() + var progress Progress + if decoder.Decode(&progress) != nil || !validProgress(progress) { + _ = command.Process.Kill() + _ = command.Wait() + return errors.New("catalog_refresh_invalid_output") + } + if err = report(progress); err != nil { + _ = command.Process.Kill() + _ = command.Wait() + return err + } + } + if scanner.Err() != nil { + _ = command.Process.Kill() + _ = command.Wait() + return errors.New("catalog_refresh_output_failed") + } + if err = command.Wait(); err != nil { + return errors.New("catalog_refresh_failed") + } + return nil +} diff --git a/internal/workflows/catalogrefresh/handler_test.go b/internal/workflows/catalogrefresh/handler_test.go new file mode 100644 index 0000000..01dd138 --- /dev/null +++ b/internal/workflows/catalogrefresh/handler_test.go @@ -0,0 +1,126 @@ +package catalogrefresh + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "knowledge-governance/internal/jobs" +) + +type fakeStore struct{ updates []Progress } + +func (s *fakeStore) UpdateCatalogRefreshProgress(_ context.Context, _ string, _ string, p Progress, _ time.Time) error { + s.updates = append(s.updates, p) + return nil +} + +func TestHandlerClampsMonotonicProgressAndKeepsFailuresVisible(t *testing.T) { + store := &fakeStore{} + handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }}) + if err != nil { + t.Fatal(err) + } + runner := func(ctx context.Context, _ Request, report func(Progress) error) error { + _ = report(Progress{Stage: "scanning", Progress: 40, Message: "扫描中", Processed: 60, Total: 100}) + // Discovered more directories than estimated: total must not shrink, percent must not regress. + _ = report(Progress{Stage: "scanning", Progress: 30, Message: "发现更多目录", Processed: 70, Total: 50}) + return report(Progress{Stage: "failed", Progress: 0, Message: "来源不可用"}) + } + custom := *handler + custom.runner = runnerFunc(runner) + job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"alist","source_name":"国内工业","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)} + if err = custom.Handle(context.Background(), job); err != nil { + t.Fatal(err) + } + if len(store.updates) != 3 { + t.Fatalf("updates=%#v", store.updates) + } + if store.updates[1].Progress < 40 || store.updates[1].Total < 70 { + t.Fatalf("regression not clamped: %#v", store.updates[1]) + } + if store.updates[2].Stage != "failed" || store.updates[2].Progress != 0 { + t.Fatalf("failure must keep its own state: %#v", store.updates[2]) + } +} + +func TestValidProgressRejectsControlCharsURLsAndCredentials(t *testing.T) { + base := Progress{Stage: "scanning", Progress: 50, Message: "正在扫描", Processed: 10, Total: 20} + if !validProgress(base) { + t.Fatal("baseline rejected") + } + for name, mutate := range map[string]func(*Progress){ + "control": func(p *Progress) { p.CurrentDirectory = "/产品\a资料" }, + "url": func(p *Progress) { p.CurrentDirectory = "https://evil.invalid/x" }, + "url-message": func(p *Progress) { p.Message = "访问 https://evil.invalid 失败" }, + "token": func(p *Progress) { p.Message = "token=abc123 已过期" }, + "too-long": func(p *Progress) { p.CurrentDirectory = strings.Repeat("目", 1100) }, + "negative": func(p *Progress) { p.ScannedFiles = -1 }, + "secret-dir": func(p *Progress) { p.CurrentDirectory = "/run/source-secrets/pp" }, + } { + item := base + mutate(&item) + if validProgress(item) { + t.Errorf("%s accepted", name) + } + } +} + +func TestCommandRunnerRejectsUnknownProgressFields(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 unavailable") + } + runner, err := NewCommandRunner(python, writeFakeProgressScript(t)) + if err != nil { + t.Fatal(err) + } + err = runner.Run(context.Background(), Request{ConnectionID: "c", SourceType: "pp", SourceName: "n"}, func(Progress) error { return nil }) + if err == nil || err.Error() != "catalog_refresh_invalid_output" { + t.Fatalf("unknown field must fail: %v", err) + } +} + +func writeFakeProgressScript(t *testing.T) string { + t.Helper() + directory := t.TempDir() + path := filepath.Join(directory, "progress.py") + body := "import json\nprint(json.dumps({'stage':'scanning','progress':10,'message':'ok','processed':1,'total':2,'unexpected_field':'leak'}),flush=True)\n" + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +type runnerFunc func(context.Context, Request, func(Progress) error) error + +func (f runnerFunc) Run(ctx context.Context, request Request, report func(Progress) error) error { + return f(ctx, request, report) +} + +type fakeRunner struct{} + +func (fakeRunner) Run(_ context.Context, _ Request, report func(Progress) error) error { + if err := report(Progress{Stage: "scanning", Progress: 50, Message: "正在扫描"}); err != nil { + return err + } + return report(Progress{Stage: "completed", Progress: 100, Message: "完成"}) +} +func TestHandlerRunsLiveRefreshAndPersistsProgress(t *testing.T) { + store := &fakeStore{} + handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }}) + if err != nil { + t.Fatal(err) + } + job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)} + if err = handler.Handle(context.Background(), job); err != nil { + t.Fatal(err) + } + if len(store.updates) != 2 || store.updates[0].Stage != "scanning" || store.updates[1].Progress != 100 { + t.Fatalf("updates=%#v", store.updates) + } +} diff --git a/internal/workflows/release/handler.go b/internal/workflows/release/handler.go new file mode 100644 index 0000000..6db264c --- /dev/null +++ b/internal/workflows/release/handler.go @@ -0,0 +1,299 @@ +package release + +import ( + "context" + "encoding/json" + "errors" + "io" + "log" + "os" + "strings" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// debugReleaseItemErrors enables fail-closed-safe diagnostics: only the +// generic step-level cause text is logged, never payload or credentials. +var debugReleaseItemErrors = os.Getenv("RELEASE_DEBUG_ITEM_ERRORS") == "true" + +const ( + ErrorCodeContractMismatch = "release_contract_mismatch" + ErrorCodeBundleInvalid = "release_bundle_invalid" + ErrorCodeTargetUnavailable = "release_target_unavailable" + ErrorCodeTargetWriteFailed = "release_target_write_failed" + ErrorCodeVerificationFailed = "release_verification_failed" + ErrorCodeRollbackArtifactMissing = "release_rollback_artifact_missing" + ErrorCodeRollbackFailed = "release_rollback_failed" + ErrorCodeStoppedAfterFailure = "release_stopped_after_failure" + ErrorCodeInternal = "release_internal_error" +) + +var ( + ErrInvalidReleaseJob = errors.New("invalid_release_job") + ErrReleaseRetryable = errors.New("release_retryable") + ErrReleaseState = errors.New("release_state_failed") +) + +type PlanItem struct { + ReleaseExecutionID string `json:"release_execution_id"` + PublicationID string `json:"publication_id"` + Action string `json:"action"` + SourceItemVersionID string `json:"source_item_version_id"` + SourceRevision string `json:"source_revision"` + ParseRevision string `json:"parse_revision"` + ReleaseRevision string `json:"release_revision"` + TestKnowledgeBaseID string `json:"test_knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + DecidedVisibility governance.Visibility `json:"decided_visibility"` + PreviousProductionRevision string `json:"previous_production_revision"` +} + +type ItemResult struct { + BundleSHA256 string + PreviousProductionRevision string + ProductionRevision string +} + +type StartExecutionRequest struct { + ExecutionID string + JobID string + Operation governance.ReleaseOperation + Now time.Time +} + +type CompleteItemRequest struct { + ExecutionID string + PublicationID string + BundleSHA256 string + PreviousProductionRevision string + ProductionRevision string + Now time.Time +} + +type FailItemRequest struct { + ExecutionID string + PublicationID string + ErrorCode string + Now time.Time +} + +type Store interface { + StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error) + BeginBatch(context.Context, string, int, int, time.Time) error + CompleteItem(context.Context, CompleteItemRequest) error + FailItem(context.Context, FailItemRequest) error + FinishExecution(context.Context, string, time.Time) error + ScheduleAutomaticRollback(context.Context, string, time.Time) error +} + +type Publisher interface { + Publish(context.Context, PlanItem) (ItemResult, error) + Delete(context.Context, PlanItem) (ItemResult, error) + Rollback(context.Context, PlanItem) (ItemResult, error) +} + +type ItemError struct { + Code string + Retryable bool + Cause error +} + +func (err *ItemError) Error() string { + if err == nil || !ValidErrorCode(err.Code) { + return ErrorCodeInternal + } + return err.Code +} + +func (err *ItemError) Unwrap() error { + if err == nil { + return nil + } + return err.Cause +} + +type jobPayload struct { + ExecutionID string `json:"execution_id"` + ReleaseBatchID string `json:"release_batch_id"` + Operation string `json:"operation"` + PriorExecutionID string `json:"prior_execution_id"` +} + +func NewHandler(store Store, publisher Publisher, now func() time.Time) (jobs.Handler, error) { + if store == nil || publisher == nil || now == nil { + return nil, errors.New("release handler dependencies are required") + } + return func(ctx context.Context, job jobs.Job) error { + payload, operation, err := parseJob(job) + if err != nil { + return err + } + items, err := store.StartExecution(ctx, StartExecutionRequest{ + ExecutionID: payload.ExecutionID, JobID: job.ID, Operation: operation, Now: now().UTC(), + }) + if err != nil { + return ErrReleaseState + } + retryableFailure := false + permanentFailure := false + failureIndex := -1 + const batchSize = 50 + totalBatches := (len(items) + batchSize - 1) / batchSize + for start := 0; start < len(items); start += batchSize { + batchNumber := start/batchSize + 1 + if err := store.BeginBatch(ctx, payload.ExecutionID, batchNumber, totalBatches, now().UTC()); err != nil { + return ErrReleaseState + } + end := start + batchSize + if end > len(items) { + end = len(items) + } + for index := start; index < end; index++ { + item := items[index] + result, itemErr := publishItem(ctx, publisher, operation, item) + if itemErr != nil { + code, retryable := classifyItemError(itemErr) + if debugReleaseItemErrors { + var itemErrDetail *ItemError + causeText := "" + if errors.As(itemErr, &itemErrDetail) && itemErrDetail != nil && itemErrDetail.Cause != nil { + causeText = itemErrDetail.Cause.Error() + } + log.Printf("release item %s (%s/%s) failed code=%s cause=%s", item.PublicationID, item.Action, item.KnowledgeID, code, causeText) + } + if retryable && job.Attempt < job.MaxAttempts { + retryableFailure = true + continue + } + if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: code, Now: now().UTC()}); err != nil { + return ErrReleaseState + } + permanentFailure = true + failureIndex = index + break + } + if !validResult(operation, result) { + if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeBundleInvalid, Now: now().UTC()}); err != nil { + return ErrReleaseState + } + permanentFailure = true + failureIndex = index + break + } + if err := store.CompleteItem(ctx, CompleteItemRequest{ + ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, + BundleSHA256: result.BundleSHA256, PreviousProductionRevision: result.PreviousProductionRevision, + ProductionRevision: result.ProductionRevision, Now: now().UTC(), + }); err != nil { + return ErrReleaseState + } + } + if permanentFailure { + break + } + } + if permanentFailure { + for _, item := range items[failureIndex+1:] { + if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeStoppedAfterFailure, Now: now().UTC()}); err != nil { + return ErrReleaseState + } + } + } + if retryableFailure { + return ErrReleaseRetryable + } + if err := store.FinishExecution(ctx, payload.ExecutionID, now().UTC()); err != nil { + return ErrReleaseState + } + if permanentFailure && operation == governance.ReleaseOperationPublish { + if err := store.ScheduleAutomaticRollback(ctx, payload.ExecutionID, now().UTC()); err != nil { + return ErrReleaseState + } + } + return nil + }, nil +} + +func parseJob(job jobs.Job) (jobPayload, governance.ReleaseOperation, error) { + var payload jobPayload + decoder := json.NewDecoder(strings.NewReader(string(job.Payload))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&payload); err != nil { + return jobPayload{}, "", ErrInvalidReleaseJob + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return jobPayload{}, "", ErrInvalidReleaseJob + } + operation := governance.ReleaseOperation(payload.Operation) + expectedType := "release." + payload.Operation + if !validID(payload.ExecutionID, "rx-") || !validID(payload.ReleaseBatchID, "rb-") || !operation.Valid() || job.Type != expectedType || job.Attempt < 1 || job.MaxAttempts < job.Attempt { + return jobPayload{}, "", ErrInvalidReleaseJob + } + return payload, operation, nil +} + +func publishItem(ctx context.Context, publisher Publisher, operation governance.ReleaseOperation, item PlanItem) (ItemResult, error) { + if operation == governance.ReleaseOperationRollback { + return publisher.Rollback(ctx, item) + } + if item.Action == "delete" { + return publisher.Delete(ctx, item) + } + return publisher.Publish(ctx, item) +} + +func classifyItemError(err error) (string, bool) { + var itemErr *ItemError + if errors.As(err, &itemErr) && ValidErrorCode(itemErr.Code) { + return itemErr.Code, itemErr.Retryable + } + return ErrorCodeInternal, true +} + +func validResult(operation governance.ReleaseOperation, result ItemResult) bool { + if strings.TrimSpace(result.ProductionRevision) == "" || len(result.ProductionRevision) > 128 { + return false + } + if operation == governance.ReleaseOperationPublish { + return validSHA256(result.BundleSHA256) + } + return true +} + +func validSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} + +func validID(value, prefix string) bool { + if !strings.HasPrefix(value, prefix) || len(value) > 36 || len(value) <= len(prefix) { + return false + } + for _, char := range value { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { + return false + } + } + return true +} + +func ValidErrorCode(code string) bool { + switch code { + case ErrorCodeContractMismatch, ErrorCodeBundleInvalid, ErrorCodeTargetUnavailable, + ErrorCodeTargetWriteFailed, ErrorCodeVerificationFailed, ErrorCodeRollbackArtifactMissing, + ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeInternal: + return true + default: + return false + } +} diff --git a/internal/workflows/release/handler_test.go b/internal/workflows/release/handler_test.go new file mode 100644 index 0000000..0bc504d --- /dev/null +++ b/internal/workflows/release/handler_test.go @@ -0,0 +1,209 @@ +package release + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +func TestHandlerStopsAfterPermanentItemFailureAndSchedulesAutomaticRollback(t *testing.T) { + store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}, {PublicationID: "tp-2"}, {PublicationID: "tp-3"}}} + publisher := &fakePublisher{publish: map[string]publishResult{ + "tp-1": {result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}}, + "tp-2": {err: &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}}, + "tp-3": {result: ItemResult{BundleSHA256: repeatHex('b'), ProductionRevision: "prod-3"}}, + }} + handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 6, 7, 0, 0, 0, time.UTC) }) + if err != nil { + t.Fatal(err) + } + job := releaseJob("release.publish", 1, 3) + if err := handler(context.Background(), job); err != nil { + t.Fatal(err) + } + if len(store.completed) != 1 || store.completed[0].PublicationID != "tp-1" { + t.Fatalf("completed=%#v", store.completed) + } + if len(store.failed) != 2 || store.failed[0].ErrorCode != ErrorCodeContractMismatch || store.failed[1].PublicationID != "tp-3" || store.failed[1].ErrorCode != ErrorCodeStoppedAfterFailure { + t.Fatalf("failed=%#v", store.failed) + } + if store.finishedExecutionID != "rx-safe" || store.automaticRollbackExecutionID != "rx-safe" || publisher.publishCalls != 2 { + t.Fatalf("finished=%q rollback=%q publishCalls=%d", store.finishedExecutionID, store.automaticRollbackExecutionID, publisher.publishCalls) + } +} + +func TestHandlerPersistsProgressAtFiftyItemBatchBoundaries(t *testing.T) { + items := make([]PlanItem, 0, 105) + results := make(map[string]publishResult, 105) + for index := 0; index < 105; index++ { + id := fmt.Sprintf("tp-%03d", index) + items = append(items, PlanItem{PublicationID: id}) + results[id] = publishResult{result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}} + } + store := &fakeStore{items: items} + handler, err := NewHandler(store, &fakePublisher{publish: results}, time.Now) + if err != nil { + t.Fatal(err) + } + if err := handler(context.Background(), releaseJob("release.publish", 1, 3)); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(store.startedBatches, [][2]int{{1, 3}, {2, 3}, {3, 3}}) { + t.Fatalf("started batches=%v", store.startedBatches) + } +} + +func TestHandlerPublishesDeleteActionAsVerifiedAbsentRevision(t *testing.T) { + store := &fakeStore{items: []PlanItem{{PublicationID: "tp-delete", Action: "delete", PreviousProductionRevision: "prod-current"}}} + publisher := &fakePublisher{delete: map[string]publishResult{ + "tp-delete": {result: ItemResult{BundleSHA256: repeatHex('d'), PreviousProductionRevision: "prod-current", ProductionRevision: "absent"}}, + }} + handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 10, 4, 0, 0, 0, time.UTC) }) + if err != nil { + t.Fatal(err) + } + if err := handler(context.Background(), jobs.Job{ID: "job-delete", Type: "release.publish", Payload: []byte(`{"execution_id":"rx-delete","release_batch_id":"rb-delete","operation":"publish"}`), Attempt: 1, MaxAttempts: 3}); err != nil { + t.Fatal(err) + } + if publisher.deleteCalls != 1 || publisher.publishCalls != 0 || len(store.completed) != 1 || store.completed[0].ProductionRevision != "absent" { + t.Fatalf("deleteCalls=%d publishCalls=%d completed=%#v", publisher.deleteCalls, publisher.publishCalls, store.completed) + } +} + +func TestHandlerReturnsOnlySanitizedRetryErrorBeforeLastAttempt(t *testing.T) { + store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}} + publisher := &fakePublisher{publish: map[string]publishResult{ + "tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true, Cause: errors.New("raw endpoint and credential text")}}, + }} + handler, _ := NewHandler(store, publisher, time.Now) + err := handler(context.Background(), releaseJob("release.publish", 1, 3)) + if !errors.Is(err, ErrReleaseRetryable) || err.Error() != ErrReleaseRetryable.Error() { + t.Fatalf("error=%v", err) + } + if len(store.failed) != 0 || store.finishedExecutionID != "" { + t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID) + } +} + +func TestHandlerFailsRetryableItemsWithFixedCodeOnLastAttempt(t *testing.T) { + store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}} + publisher := &fakePublisher{publish: map[string]publishResult{ + "tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true}}, + }} + handler, _ := NewHandler(store, publisher, time.Now) + if err := handler(context.Background(), releaseJob("release.publish", 3, 3)); err != nil { + t.Fatal(err) + } + if len(store.failed) != 1 || store.failed[0].ErrorCode != ErrorCodeTargetUnavailable || store.finishedExecutionID != "rx-safe" { + t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID) + } +} + +func TestHandlerDispatchesRollbackAndRejectsInvalidPayload(t *testing.T) { + store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}} + publisher := &fakePublisher{rollback: map[string]publishResult{"tp-1": {result: ItemResult{ProductionRevision: "prod-old"}}}} + handler, _ := NewHandler(store, publisher, time.Now) + if err := handler(context.Background(), releaseJob("release.rollback", 1, 3)); err != nil { + t.Fatal(err) + } + if publisher.rollbackCalls != 1 || publisher.publishCalls != 0 { + t.Fatalf("publish=%d rollback=%d", publisher.publishCalls, publisher.rollbackCalls) + } + bad := releaseJob("release.publish", 1, 3) + bad.Payload = []byte(`{"execution_id":"../bad"}`) + if err := handler(context.Background(), bad); !errors.Is(err, ErrInvalidReleaseJob) { + t.Fatalf("error=%v", err) + } + // Production rollback payloads carry prior_execution_id; the strict + // decoder must accept it (regression: release jobs failed permanently as + // invalid_release_job the moment rollback executions included the field). + withPrior := releaseJob("release.rollback", 1, 3) + withPrior.Payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback","prior_execution_id":"rx-prior"}`) + if err := handler(context.Background(), withPrior); err != nil { + t.Fatalf("rollback payload with prior_execution_id rejected: %v", err) + } +} + +func releaseJob(jobType string, attempt, maxAttempts int) jobs.Job { + payload := []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"publish"}`) + if jobType == "release.rollback" { + payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback"}`) + } + return jobs.Job{ID: "job-safe", Type: jobType, Payload: payload, Attempt: attempt, MaxAttempts: maxAttempts} +} + +type fakeStore struct { + items []PlanItem + completed []CompleteItemRequest + failed []FailItemRequest + finishedExecutionID string + automaticRollbackExecutionID string + startedBatches [][2]int +} + +func (s *fakeStore) StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error) { + return s.items, nil +} +func (s *fakeStore) CompleteItem(_ context.Context, request CompleteItemRequest) error { + s.completed = append(s.completed, request) + return nil +} +func (s *fakeStore) FailItem(_ context.Context, request FailItemRequest) error { + s.failed = append(s.failed, request) + return nil +} +func (s *fakeStore) FinishExecution(_ context.Context, executionID string, _ time.Time) error { + s.finishedExecutionID = executionID + return nil +} +func (s *fakeStore) ScheduleAutomaticRollback(_ context.Context, executionID string, _ time.Time) error { + s.automaticRollbackExecutionID = executionID + return nil +} +func (s *fakeStore) BeginBatch(_ context.Context, _ string, batch, total int, _ time.Time) error { + s.startedBatches = append(s.startedBatches, [2]int{batch, total}) + return nil +} + +type publishResult struct { + result ItemResult + err error +} + +type fakePublisher struct { + publish, rollback, delete map[string]publishResult + publishCalls, rollbackCalls, deleteCalls int +} + +func (p *fakePublisher) Publish(_ context.Context, item PlanItem) (ItemResult, error) { + p.publishCalls++ + value := p.publish[item.PublicationID] + return value.result, value.err +} +func (p *fakePublisher) Rollback(_ context.Context, item PlanItem) (ItemResult, error) { + p.rollbackCalls++ + value := p.rollback[item.PublicationID] + return value.result, value.err +} + +func (p *fakePublisher) Delete(_ context.Context, item PlanItem) (ItemResult, error) { + p.deleteCalls++ + value := p.delete[item.PublicationID] + return value.result, value.err +} + +func repeatHex(value byte) string { + result := make([]byte, 64) + for index := range result { + result[index] = value + } + return string(result) +} + +var _ = governance.ReleaseOperationPublish diff --git a/internal/workflows/release/routing_publisher.go b/internal/workflows/release/routing_publisher.go new file mode 100644 index 0000000..6f08683 --- /dev/null +++ b/internal/workflows/release/routing_publisher.go @@ -0,0 +1,65 @@ +package release + +import ( + "context" + "errors" + "strings" +) + +type RoutingPublisher struct { + publishers map[string]Publisher +} + +func NewRoutingPublisher(publishers map[string]Publisher) (*RoutingPublisher, error) { + if len(publishers) == 0 { + return nil, errors.New("release publisher routes are required") + } + copied := make(map[string]Publisher, len(publishers)) + for knowledgeBaseID, publisher := range publishers { + if !validRoutingID(knowledgeBaseID) || publisher == nil { + return nil, errors.New("invalid release publisher route") + } + copied[knowledgeBaseID] = publisher + } + return &RoutingPublisher{publishers: copied}, nil +} + +func (router *RoutingPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) { + publisher, ok := router.publishers[item.TestKnowledgeBaseID] + if !ok { + return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false} + } + return publisher.Publish(ctx, item) +} + +func (router *RoutingPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) { + publisher, ok := router.publishers[item.TestKnowledgeBaseID] + if !ok { + return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false} + } + return publisher.Rollback(ctx, item) +} + +func (router *RoutingPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) { + publisher, ok := router.publishers[item.TestKnowledgeBaseID] + if !ok { + return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false} + } + return publisher.Delete(ctx, item) +} + +func validRoutingID(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > 128 { + return false + } + for _, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' { + continue + } + return false + } + return true +} + +var _ Publisher = (*RoutingPublisher)(nil) diff --git a/internal/workflows/release/routing_publisher_test.go b/internal/workflows/release/routing_publisher_test.go new file mode 100644 index 0000000..a580576 --- /dev/null +++ b/internal/workflows/release/routing_publisher_test.go @@ -0,0 +1,52 @@ +package release + +import ( + "context" + "testing" +) + +func TestRoutingPublisherSelectsExactKnowledgeBase(t *testing.T) { + first := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-a"}} + second := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-b"}} + router, err := NewRoutingPublisher(map[string]Publisher{"kb-a": first, "kb-b": second}) + if err != nil { + t.Fatal(err) + } + item := testPlanItem() + item.TestKnowledgeBaseID = "kb-b" + result, err := router.Publish(context.Background(), item) + if err != nil { + t.Fatal(err) + } + if result.ProductionRevision != "prod-b" || first.calls != 0 || second.calls != 1 { + t.Fatalf("result=%#v calls=%d/%d", result, first.calls, second.calls) + } + if _, err := router.Delete(context.Background(), item); err != nil || first.deleteCalls != 0 || second.deleteCalls != 1 { + t.Fatalf("delete error=%v deleteCalls=%d/%d", err, first.deleteCalls, second.deleteCalls) + } + item.TestKnowledgeBaseID = "unknown" + if _, err := router.Publish(context.Background(), item); err == nil || err.Error() != ErrorCodeContractMismatch { + t.Fatalf("error=%v", err) + } +} + +type routingPublisherStub struct { + result ItemResult + err error + calls int + deleteCalls int +} + +func (s *routingPublisherStub) Publish(context.Context, PlanItem) (ItemResult, error) { + s.calls++ + return s.result, s.err +} +func (s *routingPublisherStub) Rollback(context.Context, PlanItem) (ItemResult, error) { + s.calls++ + return s.result, s.err +} + +func (s *routingPublisherStub) Delete(context.Context, PlanItem) (ItemResult, error) { + s.deleteCalls++ + return s.result, s.err +} diff --git a/internal/workflows/release/sidecar_publisher.go b/internal/workflows/release/sidecar_publisher.go new file mode 100644 index 0000000..d701204 --- /dev/null +++ b/internal/workflows/release/sidecar_publisher.go @@ -0,0 +1,162 @@ +package release + +import ( + "context" + "errors" + "strings" + + "knowledge-governance/internal/releasebundle" +) + +type BundleExporter interface { + Export(ctx context.Context, item PlanItem) (ExportedBundle, error) +} + +type ExportedBundle interface { + ReleaseManifest() releasebundle.Manifest +} + +type PreparedChange struct { + PreviousRevision string + ProductionRevision string +} + +type SidecarTarget interface { + Contract(ctx context.Context, item PlanItem) (releasebundle.TargetContract, error) + Prepare(ctx context.Context, item PlanItem, bundle ExportedBundle) (PreparedChange, error) + Activate(ctx context.Context, item PlanItem, change PreparedChange) error + Verify(ctx context.Context, item PlanItem, change PreparedChange) error + Compensate(ctx context.Context, item PlanItem, change PreparedChange) error + Restore(ctx context.Context, item PlanItem, revision string) (string, error) + VerifyRestored(ctx context.Context, item PlanItem, revision string) error +} + +type SidecarPublisher struct { + exporter BundleExporter + target SidecarTarget +} + +func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget) (*SidecarPublisher, error) { + if exporter == nil || target == nil { + return nil, errors.New("release sidecar dependencies are required") + } + return &SidecarPublisher{exporter: exporter, target: target}, nil +} + +func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) { + bundle, err := publisher.exporter.Export(ctx, item) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err) + } + if bundle == nil { + return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle")) + } + manifest := bundle.ReleaseManifest() + if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID || + manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision || + manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision || + manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID { + return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch")) + } + targetContract, err := publisher.target.Contract(ctx, item) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err) + } + if err := releasebundle.Validate(manifest, targetContract); err != nil { + return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err) + } + change, err := publisher.target.Prepare(ctx, item, bundle) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err) + } + if !validRevision(change.ProductionRevision) || (change.PreviousRevision != "" && !validRevision(change.PreviousRevision)) { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid prepared revision")) + } + if err := publisher.target.Activate(ctx, item, change); err != nil { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err) + } + if err := publisher.target.Verify(ctx, item, change); err != nil { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err) + } + return ItemResult{ + BundleSHA256: manifest.BundleSHA256, + PreviousProductionRevision: change.PreviousRevision, + ProductionRevision: change.ProductionRevision, + }, nil +} + +func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) { + if !validRevision(item.PreviousProductionRevision) { + return ItemResult{}, newSidecarError(ErrorCodeRollbackArtifactMissing, false, errors.New("missing rollback revision")) + } + restoredRevision, err := publisher.target.Restore(ctx, item, item.PreviousProductionRevision) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err) + } + if restoredRevision != item.PreviousProductionRevision { + return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("restored revision mismatch")) + } + if err := publisher.target.VerifyRestored(ctx, item, restoredRevision); err != nil { + return ItemResult{}, newSidecarError(ErrorCodeVerificationFailed, false, err) + } + return ItemResult{ProductionRevision: restoredRevision}, nil +} + +func (publisher *SidecarPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) { + bundle, err := publisher.exporter.Export(ctx, item) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err) + } + if bundle == nil { + return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle")) + } + manifest := bundle.ReleaseManifest() + if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID || + manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision || + manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision || + manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID { + return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch")) + } + targetContract, err := publisher.target.Contract(ctx, item) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err) + } + if err := releasebundle.Validate(manifest, targetContract); err != nil { + return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err) + } + change, err := publisher.target.Prepare(ctx, item, bundle) + if err != nil { + return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err) + } + if !validRevision(change.PreviousRevision) || change.ProductionRevision != item.ReleaseRevision { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid delete snapshot revision")) + } + restoredRevision, err := publisher.target.Restore(ctx, item, "absent") + if err != nil { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err) + } + if restoredRevision != "absent" { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeContractMismatch, errors.New("deleted revision mismatch")) + } + if err := publisher.target.VerifyRestored(ctx, item, "absent"); err != nil { + return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err) + } + return ItemResult{BundleSHA256: manifest.BundleSHA256, PreviousProductionRevision: change.PreviousRevision, ProductionRevision: "absent"}, nil +} + +func (publisher *SidecarPublisher) compensateError(ctx context.Context, item PlanItem, change PreparedChange, code string, cause error) error { + if err := publisher.target.Compensate(ctx, item, change); err != nil { + return newSidecarError(ErrorCodeRollbackFailed, false, err) + } + return newSidecarError(code, false, cause) +} + +func newSidecarError(code string, retryable bool, cause error) error { + return &ItemError{Code: code, Retryable: retryable, Cause: cause} +} + +func validRevision(value string) bool { + return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\") +} + +var _ Publisher = (*SidecarPublisher)(nil) diff --git a/internal/workflows/release/sidecar_publisher_test.go b/internal/workflows/release/sidecar_publisher_test.go new file mode 100644 index 0000000..036a0c7 --- /dev/null +++ b/internal/workflows/release/sidecar_publisher_test.go @@ -0,0 +1,217 @@ +package release + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "knowledge-governance/internal/releasebundle" +) + +func TestSidecarPublisherPublishesWithContractAndVerification(t *testing.T) { + calls := []string{} + manifest, err := releasebundle.Seal(testManifest()) + if err != nil { + t.Fatal(err) + } + exporter := &fakeExporter{manifest: manifest, calls: &calls} + target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}} + publisher, err := NewSidecarPublisher(exporter, target) + if err != nil { + t.Fatal(err) + } + result, err := publisher.Publish(context.Background(), testPlanItem()) + if err != nil { + t.Fatal(err) + } + if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "prod-v1" { + t.Fatalf("result=%#v", result) + } + want := []string{"export", "contract", "prepare", "activate", "verify"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("calls=%v want=%v", calls, want) + } +} + +func TestSidecarPublisherRollsBackFailedVerificationWithoutLeakingCause(t *testing.T) { + calls := []string{} + manifest, err := releasebundle.Seal(testManifest()) + if err != nil { + t.Fatal(err) + } + target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}, verifyErr: errors.New("signed-url-secret-should-not-leak")} + publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target) + if err != nil { + t.Fatal(err) + } + _, err = publisher.Publish(context.Background(), testPlanItem()) + var itemErr *ItemError + if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeVerificationFailed || itemErr.Retryable { + t.Fatalf("error=%#v", err) + } + if strings.Contains(err.Error(), "signed-url") { + t.Fatalf("error leaked cause: %v", err) + } + want := []string{"export", "contract", "prepare", "activate", "verify", "compensate"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("calls=%v want=%v", calls, want) + } +} + +func TestSidecarPublisherRejectsContractDriftBeforeTargetMutation(t *testing.T) { + calls := []string{} + manifest, err := releasebundle.Seal(testManifest()) + if err != nil { + t.Fatal(err) + } + contract := testTargetContract() + contract.EmbeddingDimension = 768 + publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: contract, calls: &calls}) + if err != nil { + t.Fatal(err) + } + _, err = publisher.Publish(context.Background(), testPlanItem()) + var itemErr *ItemError + if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch { + t.Fatalf("error=%#v", err) + } + if !reflect.DeepEqual(calls, []string{"export", "contract"}) { + t.Fatalf("calls=%v", calls) + } +} + +func TestSidecarPublisherRejectsRevisionMismatchBeforeTargetMutation(t *testing.T) { + calls := []string{} + manifest, err := releasebundle.Seal(testManifest()) + if err != nil { + t.Fatal(err) + } + item := testPlanItem() + item.ParseRevision = "other-parser/index" + publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: testTargetContract(), calls: &calls}) + if err != nil { + t.Fatal(err) + } + _, err = publisher.Publish(context.Background(), item) + var itemErr *ItemError + if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch { + t.Fatalf("error=%#v", err) + } + if !reflect.DeepEqual(calls, []string{"export"}) { + t.Fatalf("calls=%v", calls) + } +} + +func TestSidecarPublisherDeletesOnlyAfterArchivingAndVerifiesAbsent(t *testing.T) { + calls := []string{} + manifest, err := releasebundle.Seal(testManifest()) + if err != nil { + t.Fatal(err) + } + target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "release-v1"}, expectedRestoreRevision: "absent", restoredRevision: "absent"} + publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target) + if err != nil { + t.Fatal(err) + } + result, err := publisher.Delete(context.Background(), testPlanItem()) + if err != nil { + t.Fatal(err) + } + if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "absent" { + t.Fatalf("result=%#v", result) + } + if want := []string{"export", "contract", "prepare", "restore", "verify-restored"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("calls=%v want=%v", calls, want) + } +} + +func TestSidecarPublisherRestoresArchivedRevision(t *testing.T) { + calls := []string{} + target := &fakeSidecarTarget{calls: &calls, restoredRevision: "prod-v0"} + publisher, err := NewSidecarPublisher(&fakeExporter{}, target) + if err != nil { + t.Fatal(err) + } + result, err := publisher.Rollback(context.Background(), testPlanItem()) + if err != nil { + t.Fatal(err) + } + if result.ProductionRevision != "prod-v0" || !reflect.DeepEqual(calls, []string{"restore", "verify-restored"}) { + t.Fatalf("result=%#v calls=%v", result, calls) + } +} + +type fakeExporter struct { + manifest releasebundle.Manifest + calls *[]string + err error +} + +func (f *fakeExporter) Export(context.Context, PlanItem) (ExportedBundle, error) { + if f.calls != nil { + *f.calls = append(*f.calls, "export") + } + return fakeExportedBundle{manifest: f.manifest}, f.err +} + +type fakeExportedBundle struct{ manifest releasebundle.Manifest } + +func (bundle fakeExportedBundle) ReleaseManifest() releasebundle.Manifest { return bundle.manifest } + +type fakeSidecarTarget struct { + contract releasebundle.TargetContract + calls *[]string + prepared PreparedChange + prepareErr, activateErr, verifyErr, compensateErr, restoreErr, verifyRestoredErr error + restoredRevision string + expectedRestoreRevision string +} + +func (f *fakeSidecarTarget) Contract(context.Context, PlanItem) (releasebundle.TargetContract, error) { + *f.calls = append(*f.calls, "contract") + return f.contract, nil +} +func (f *fakeSidecarTarget) Prepare(context.Context, PlanItem, ExportedBundle) (PreparedChange, error) { + *f.calls = append(*f.calls, "prepare") + return f.prepared, f.prepareErr +} +func (f *fakeSidecarTarget) Activate(context.Context, PlanItem, PreparedChange) error { + *f.calls = append(*f.calls, "activate") + return f.activateErr +} +func (f *fakeSidecarTarget) Verify(context.Context, PlanItem, PreparedChange) error { + *f.calls = append(*f.calls, "verify") + return f.verifyErr +} +func (f *fakeSidecarTarget) Compensate(context.Context, PlanItem, PreparedChange) error { + *f.calls = append(*f.calls, "compensate") + return f.compensateErr +} +func (f *fakeSidecarTarget) Restore(_ context.Context, _ PlanItem, revision string) (string, error) { + *f.calls = append(*f.calls, "restore") + expected := f.expectedRestoreRevision + if expected == "" { + expected = "prod-v0" + } + if revision != expected { + return "", errors.New("wrong revision") + } + return f.restoredRevision, f.restoreErr +} +func (f *fakeSidecarTarget) VerifyRestored(context.Context, PlanItem, string) error { + *f.calls = append(*f.calls, "verify-restored") + return f.verifyRestoredErr +} + +func testPlanItem() PlanItem { + return PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal", PreviousProductionRevision: "prod-v0"} +} +func testManifest() releasebundle.Manifest { + return releasebundle.Manifest{FormatVersion: releasebundle.FormatVersionV1, ExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", CreatedAt: time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC), Contract: releasebundle.Contract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}, Tables: []releasebundle.TableDigest{{Name: "knowledges", RowCount: 1, SHA256: strings.Repeat("b", 64)}, {Name: "chunks", RowCount: 1, SHA256: strings.Repeat("c", 64)}}, Search: releasebundle.SearchDigest{DocumentCount: 1, SHA256: strings.Repeat("d", 64)}, Assets: releasebundle.AssetDigest{SHA256: strings.Repeat("e", 64)}} +} +func testTargetContract() releasebundle.TargetContract { + return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)} +} diff --git a/internal/workflows/sourcescan/file_scanner.go b/internal/workflows/sourcescan/file_scanner.go new file mode 100644 index 0000000..fe8136b --- /dev/null +++ b/internal/workflows/sourcescan/file_scanner.go @@ -0,0 +1,194 @@ +package sourcescan + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + ppconnector "knowledge-governance/internal/connectors/pp" +) + +const maxPPMetadataBytes int64 = 64 * 1024 * 1024 + +type FilePPScannerConfig struct { + AllowedRoot string + LedgerPath string + ManifestPath string +} + +type FilePPScanner struct { + root string + ledgerPath string + manifestPath string +} + +func NewFilePPScanner(config FilePPScannerConfig) (*FilePPScanner, error) { + root, err := filepath.Abs(config.AllowedRoot) + if err != nil || config.AllowedRoot == "" { + return nil, errors.New("PP scan allowed root is required") + } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil || resolvedRoot != root { + return nil, errors.New("PP scan allowed root must be a canonical directory") + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("PP scan allowed root must be a canonical directory") + } + if config.LedgerPath == "" || config.ManifestPath == "" { + return nil, errors.New("PP ledger and manifest paths are required") + } + return &FilePPScanner{root: root, ledgerPath: config.LedgerPath, manifestPath: config.ManifestPath}, nil +} + +func (s *FilePPScanner) Scan(ctx context.Context) (ppconnector.Snapshot, string, error) { + if err := ctx.Err(); err != nil { + return ppconnector.Snapshot{}, "", err + } + ledger, err := s.readFile(s.ledgerPath, maxPPMetadataBytes) + if err != nil { + return ppconnector.Snapshot{}, "", fmt.Errorf("read PP ledger: %w", err) + } + manifest, err := s.readFile(s.manifestPath, maxPPMetadataBytes) + if err != nil { + return ppconnector.Snapshot{}, "", fmt.Errorf("read PP manifest: %w", err) + } + if err := s.validateManifestPaths(manifest); err != nil { + return ppconnector.Snapshot{}, "", err + } + snapshot, err := ppconnector.FoldLedger(bytes.NewReader(ledger)) + if err != nil { + return ppconnector.Snapshot{}, "", err + } + snapshot, _, err = ppconnector.RecoverMissingSHA256(snapshot, bytes.NewReader(manifest)) + if err != nil { + return ppconnector.Snapshot{}, "", err + } + snapshot, _, err = ppconnector.MergeOverLimitManifest(snapshot, bytes.NewReader(manifest)) + if err != nil { + return ppconnector.Snapshot{}, "", err + } + if err := ppconnector.ValidateSnapshot(snapshot); err != nil { + return ppconnector.Snapshot{}, "", err + } + digest := sha256.New() + _, _ = digest.Write(ledger) + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write(manifest) + return snapshot, hex.EncodeToString(digest.Sum(nil)), nil +} + +func (s *FilePPScanner) validateManifestPaths(body []byte) error { + var manifest struct { + Results []struct { + LocalPath string `json:"local_path"` + Status string `json:"status"` + } `json:"results"` + } + if err := json.Unmarshal(body, &manifest); err != nil { + return fmt.Errorf("decode PP manifest paths: %w", err) + } + for _, item := range manifest.Results { + if item.LocalPath == "" { + return errors.New("PP manifest artifact local_path is required") + } + if item.Status == string(ppconnector.StatusOverLimit) { + if err := s.validateDeclaredPath(item.LocalPath); err != nil { + return fmt.Errorf("validate PP over-limit artifact path: %w", err) + } + file, err := s.openRegular(item.LocalPath) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return fmt.Errorf("validate PP over-limit artifact: %w", err) + } + if err := file.Close(); err != nil { + return errors.New("close PP over-limit artifact") + } + continue + } + file, err := s.openRegular(item.LocalPath) + if err != nil { + return fmt.Errorf("validate PP manifest artifact: %w", err) + } + if err := file.Close(); err != nil { + return errors.New("close PP manifest artifact") + } + } + return nil +} + +func (s *FilePPScanner) readFile(path string, maxBytes int64) ([]byte, error) { + file, err := s.openRegular(path) + if err != nil { + return nil, err + } + body, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1)) + closeErr := file.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + if int64(len(body)) > maxBytes { + return nil, errors.New("PP metadata file exceeds size limit") + } + return body, nil +} + +func (s *FilePPScanner) validateDeclaredPath(path string) error { + absolute, err := filepath.Abs(path) + if err != nil { + return errors.New("invalid PP source path") + } + relative, err := filepath.Rel(s.root, absolute) + if err != nil || relative == ".." || filepath.IsAbs(relative) || len(relative) >= 3 && relative[:3] == ".."+string(filepath.Separator) { + return errors.New("PP source path is outside allowed root") + } + parent := filepath.Dir(absolute) + resolvedParent, err := filepath.EvalSymlinks(parent) + if err != nil || resolvedParent != parent { + return errors.New("PP source path parent must not contain symlinks") + } + return nil +} + +func (s *FilePPScanner) openRegular(path string) (*os.File, error) { + if err := s.validateDeclaredPath(path); err != nil { + return nil, err + } + absolute, _ := filepath.Abs(path) + resolved, err := filepath.EvalSymlinks(absolute) + if err != nil { + return nil, err + } + if resolved != absolute { + return nil, errors.New("PP source path must not contain symlinks") + } + before, err := os.Lstat(absolute) + if err != nil { + return nil, err + } + if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("PP source path must be a regular file") + } + file, err := os.Open(absolute) + if err != nil { + return nil, err + } + after, err := file.Stat() + if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) { + _ = file.Close() + return nil, errors.New("PP source file changed while opening") + } + return file, nil +} diff --git a/internal/workflows/sourcescan/handler.go b/internal/workflows/sourcescan/handler.go new file mode 100644 index 0000000..c54e99e --- /dev/null +++ b/internal/workflows/sourcescan/handler.go @@ -0,0 +1,127 @@ +package sourcescan + +import ( + "context" + "encoding/json" + "errors" + "sort" + "time" + + ppconnector "knowledge-governance/internal/connectors/pp" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +type PPScanner interface { + Scan(context.Context) (ppconnector.Snapshot, string, error) +} + +type PPScanStore interface { + StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) + LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) + CompleteSourceScan(context.Context, string, governance.SourceScanResult, time.Time) error + FailSourceScan(context.Context, string, string, time.Time) error +} + +type PPHandlerConfig struct { + Store PPScanStore + Scanner PPScanner + Now func() time.Time +} + +type PPHandler struct { + store PPScanStore + scanner PPScanner + now func() time.Time +} + +func NewPPHandler(config PPHandlerConfig) (*PPHandler, error) { + if config.Store == nil || config.Scanner == nil { + return nil, errors.New("PP scan store and scanner are required") + } + if config.Now == nil { + config.Now = time.Now + } + return &PPHandler{store: config.Store, scanner: config.Scanner, now: config.Now}, nil +} + +func (h *PPHandler) Handle(ctx context.Context, job jobs.Job) error { + var payload struct { + ScanID string `json:"scan_id"` + } + if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.ScanID == "" { + return errors.New("invalid source scan payload") + } + now := h.now() + scan, err := h.store.StartSourceScan(ctx, payload.ScanID, now) + if err != nil { + return err + } + if scan.State == governance.SourceScanCompleted { + return nil + } + if scan.State != governance.SourceScanRunning { + return errors.New("source scan is not running") + } + if scan.SourceType != governance.SourceTypePP { + return h.store.FailSourceScan(ctx, scan.ID, "source_type_unsupported", now) + } + snapshot, snapshotSHA, err := h.scanner.Scan(ctx) + if err != nil || !governance.ValidSHA256(snapshotSHA) { + return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now) + } + current, err := h.store.LoadPPCurrentItems(ctx, scan.SourceConnectionID) + if err != nil { + return err + } + reconciliation, err := ppconnector.PlanReconciliation(snapshot, current) + if err != nil { + return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now) + } + observed := make(map[string]struct{}, len(snapshot.Items)) + changes := make([]governance.SourceScanChange, 0) + for _, item := range snapshot.Items { + observed[item.SourceURLHash] = struct{}{} + if item.Status == ppconnector.StatusOverLimit { + continue + } + size := item.Bytes + stored, exists := current[item.SourceURLHash] + if !exists { + changes = append(changes, governance.SourceScanChange{ + StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanAdded, + ObservedRevision: item.SHA256, ObservedSizeBytes: &size, + }) + } else if stored.Revision != item.SHA256 { + changes = append(changes, governance.SourceScanChange{ + StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanUpdated, + PreviousRevision: stored.Revision, ObservedRevision: item.SHA256, ObservedSizeBytes: &size, + }) + } + } + missingIDs := make([]string, 0) + for sourceID := range current { + if _, ok := observed[sourceID]; !ok { + missingIDs = append(missingIDs, sourceID) + } + } + sort.Strings(missingIDs) + for _, sourceID := range missingIDs { + changes = append(changes, governance.SourceScanChange{ + StableExternalID: sourceID, ChangeType: governance.SourceScanUnavailable, + PreviousRevision: current[sourceID].Revision, + }) + } + result := governance.SourceScanResult{ + SnapshotSHA256: snapshotSHA, + ObservedItems: len(snapshot.Items), + UnchangedItems: reconciliation.Unchanged, + AddedItems: reconciliation.Adopted, + UpdatedItems: reconciliation.UploadRequired, + OverLimitItems: reconciliation.OverLimit, + MissingItems: len(missingIDs), + PlannedUploads: reconciliation.UploadRequired, + Changes: changes, + } + return h.store.CompleteSourceScan(ctx, scan.ID, result, now) +} diff --git a/internal/workflows/sourcescan/handler_test.go b/internal/workflows/sourcescan/handler_test.go new file mode 100644 index 0000000..882412c --- /dev/null +++ b/internal/workflows/sourcescan/handler_test.go @@ -0,0 +1,208 @@ +package sourcescan + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + ppconnector "knowledge-governance/internal/connectors/pp" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +func TestFilePPScannerReadsValidatedSnapshotUnderAllowedRoot(t *testing.T) { + root := t.TempDir() + source := strings.Repeat("a", 64) + sha := strings.Repeat("b", 64) + ledgerPath := filepath.Join(root, "state.jsonl") + manifestPath := filepath.Join(root, "manifest.json") + ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil { + t.Fatal(err) + } + + scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath}) + if err != nil { + t.Fatalf("NewFilePPScanner() error = %v", err) + } + snapshot, fingerprint, err := scanner.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan() error = %v", err) + } + if len(snapshot.Items) != 1 || snapshot.Items[0].SourceURLHash != source { + t.Fatalf("snapshot = %#v", snapshot) + } + if !governance.ValidSHA256(fingerprint) { + t.Fatalf("fingerprint = %q", fingerprint) + } +} + +func TestFilePPScannerRejectsManifestItemWithoutLocalPath(t *testing.T) { + root := t.TempDir() + source := strings.Repeat("a", 64) + sha := strings.Repeat("b", 64) + ledgerPath := filepath.Join(root, "state.jsonl") + manifestPath := filepath.Join(root, "manifest.json") + ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(`{"results":[{"local_path":""}]}`), 0o600); err != nil { + t.Fatal(err) + } + scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath}) + if err != nil { + t.Fatal(err) + } + if _, _, err := scanner.Scan(context.Background()); err == nil || !strings.Contains(err.Error(), "local_path") { + t.Fatalf("Scan() error = %v, want missing local_path error", err) + } +} + +func TestFilePPScannerAllowsMissingArtifactForOverLimitRecord(t *testing.T) { + root := t.TempDir() + source := strings.Repeat("a", 64) + sha := strings.Repeat("b", 64) + ledgerPath := filepath.Join(root, "state.jsonl") + manifestPath := filepath.Join(root, "manifest.json") + ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}` + manifest := `{"results":[{"url":"https://source.invalid/large","local_path":"` + filepath.Join(root, "not-downloaded.bin") + `","bytes":52428801,"status":"over_limit","source_ids":["pp-large"]}]}` + if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath}) + if err != nil { + t.Fatal(err) + } + snapshot, _, err := scanner.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan() error = %v", err) + } + if len(snapshot.Items) != 2 || snapshot.Items[1].Status != ppconnector.StatusOverLimit { + t.Fatalf("snapshot = %#v", snapshot) + } +} + +func TestPPHandlerCompletesAggregateReadOnlyReconciliation(t *testing.T) { + now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC) + sameSource := strings.Repeat("a", 64) + updatedSource := strings.Repeat("b", 64) + overLimitSource := strings.Repeat("c", 64) + missingSource := strings.Repeat("d", 64) + addedSource := strings.Repeat("e", 64) + sameSHA := strings.Repeat("1", 64) + newSHA := strings.Repeat("2", 64) + oldSHA := strings.Repeat("3", 64) + addedSHA := strings.Repeat("5", 64) + + store := &fakePPScanStore{ + scan: governance.SourceScan{ID: "scan-1", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanPending}, + current: map[string]ppconnector.CurrentItem{ + sameSource: {Revision: sameSHA}, + updatedSource: {Revision: oldSHA}, + missingSource: {Revision: strings.Repeat("4", 64)}, + }, + } + scanner := &fakePPScanner{snapshot: ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{ + {KBID: "kb-1", SourceURLHash: sameSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sameSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"}, + {KBID: "kb-1", SourceURLHash: updatedSource, SourceIDs: []string{"pp-2"}, Bytes: 20, SHA256: newSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-2"}, + {KBID: "kb-1", SourceURLHash: overLimitSource, SourceIDs: []string{"pp-3"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit}, + {KBID: "kb-1", SourceURLHash: addedSource, SourceIDs: []string{"pp-4"}, Bytes: 30, SHA256: addedSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-4"}, + }}, snapshotSHA: strings.Repeat("f", 64)} + + handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now }}) + if err != nil { + t.Fatalf("NewPPHandler() error = %v", err) + } + job := jobs.Job{Type: "source.scan.pp", Payload: []byte(`{"scan_id":"scan-1"}`)} + if err := handler.Handle(context.Background(), job); err != nil { + t.Fatalf("Handle() error = %v", err) + } + + want := governance.SourceScanResult{ + SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 4, UnchangedItems: 1, AddedItems: 1, + UpdatedItems: 1, OverLimitItems: 1, MissingItems: 1, PlannedUploads: 1, + Changes: []governance.SourceScanChange{ + {StableExternalID: updatedSource, ChangeType: governance.SourceScanUpdated, PreviousRevision: oldSHA, ObservedRevision: newSHA, ObservedSizeBytes: int64Pointer(20)}, + {StableExternalID: addedSource, ChangeType: governance.SourceScanAdded, ObservedRevision: addedSHA, ObservedSizeBytes: int64Pointer(30)}, + {StableExternalID: missingSource, ChangeType: governance.SourceScanUnavailable, PreviousRevision: strings.Repeat("4", 64)}, + }, + } + if store.completedScanID != "scan-1" || !reflect.DeepEqual(store.completed, want) { + t.Fatalf("completed scan = %q %#v, want %#v", store.completedScanID, store.completed, want) + } + if store.completedAt != now || store.failedCode != "" { + t.Fatalf("completedAt=%v failedCode=%q", store.completedAt, store.failedCode) + } +} + +func TestPPHandlerNoOpsWhenReclaimedScanAlreadyCompleted(t *testing.T) { + store := &fakePPScanStore{scan: governance.SourceScan{ + ID: "scan-completed", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanCompleted, + }} + scanner := &fakePPScanner{} + handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner}) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"scan_id":"scan-completed"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if scanner.calls != 0 || store.completedScanID != "" || store.failedCode != "" { + t.Fatalf("scanner calls=%d completed=%q failed=%q", scanner.calls, store.completedScanID, store.failedCode) + } +} + +func int64Pointer(value int64) *int64 { return &value } + +type fakePPScanner struct { + snapshot ppconnector.Snapshot + snapshotSHA string + err error + calls int +} + +func (s *fakePPScanner) Scan(context.Context) (ppconnector.Snapshot, string, error) { + s.calls++ + return s.snapshot, s.snapshotSHA, s.err +} + +type fakePPScanStore struct { + scan governance.SourceScan + current map[string]ppconnector.CurrentItem + completedScanID string + completed governance.SourceScanResult + completedAt time.Time + failedCode string +} + +func (s *fakePPScanStore) StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) { + if s.scan.State == governance.SourceScanPending { + s.scan.State = governance.SourceScanRunning + } + return s.scan, nil +} + +func (s *fakePPScanStore) LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) { + return s.current, nil +} + +func (s *fakePPScanStore) CompleteSourceScan(_ context.Context, scanID string, result governance.SourceScanResult, now time.Time) error { + s.completedScanID, s.completed, s.completedAt = scanID, result, now + return nil +} + +func (s *fakePPScanStore) FailSourceScan(_ context.Context, _ string, code string, _ time.Time) error { + s.failedCode = code + return nil +} diff --git a/internal/workflows/testweknora/legacy_verify.go b/internal/workflows/testweknora/legacy_verify.go new file mode 100644 index 0000000..e0f1242 --- /dev/null +++ b/internal/workflows/testweknora/legacy_verify.go @@ -0,0 +1,196 @@ +package testweknora + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + + "knowledge-governance/internal/platform/weknora" +) + +type LegacyCandidate struct { + ID string + KnowledgeID string + TestKBID string +} + +type LegacyVerifyClient interface { + GetKnowledge(context.Context, string) (weknora.Knowledge, error) + ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) +} + +type LegacyVerificationReport struct { + Total int + Existing int + Missing int + WrongKB int + Pending int + Processing int + Finalizing int + Completed int + Failed int + ChunkChecked int + ChunkEmpty int + HTTPErrors int +} + +func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) { + report := LegacyVerificationReport{Total: len(candidates)} + completedSeen := 0 + for _, candidate := range candidates { + knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID) + if err != nil { + var httpErr *weknora.HTTPError + if errors.As(err, &httpErr) && httpErr.StatusCode == 404 { + report.Missing++ + } else { + report.HTTPErrors++ + } + continue + } + if knowledge.KnowledgeBaseID != candidate.TestKBID { + report.WrongKB++ + continue + } + report.Existing++ + switch knowledge.ParseStatus { + case "pending": + report.Pending++ + case "processing": + report.Processing++ + case "finalizing": + report.Finalizing++ + case "completed": + report.Completed++ + completedSeen++ + if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 { + if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil { + report.HTTPErrors++ + } + } + case "failed": + report.Failed++ + } + } + return report, nil +} + +func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) { + if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 { + return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration") + } + jobs := make(chan LegacyCandidate) + results := make(chan LegacyVerificationReport) + var workers sync.WaitGroup + var completedSeen atomic.Int64 + workerCount := min(concurrency, len(candidates)) + workers.Add(workerCount) + for range workerCount { + go func() { + defer workers.Done() + for candidate := range jobs { + partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0) + if partial.Completed == 1 { + completed := completedSeen.Add(1) + if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 { + if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil { + partial.HTTPErrors++ + } + } + } + results <- partial + } + }() + } + go func() { + defer close(jobs) + for _, candidate := range candidates { + select { + case jobs <- candidate: + case <-ctx.Done(): + return + } + } + }() + go func() { + workers.Wait() + close(results) + }() + + report := LegacyVerificationReport{} + for partial := range results { + mergeLegacyVerificationReport(&report, partial) + } + if err := ctx.Err(); err != nil { + return report, err + } + return report, nil +} + +func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) { + report.Total += partial.Total + report.Existing += partial.Existing + report.Missing += partial.Missing + report.WrongKB += partial.WrongKB + report.Pending += partial.Pending + report.Processing += partial.Processing + report.Finalizing += partial.Finalizing + report.Completed += partial.Completed + report.Failed += partial.Failed + report.ChunkChecked += partial.ChunkChecked + report.ChunkEmpty += partial.ChunkEmpty + report.HTTPErrors += partial.HTTPErrors +} + +func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error { + _, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil) + if err != nil { + return err + } + report.ChunkChecked++ + if total == 0 { + report.ChunkEmpty++ + } + return nil +} + +func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string { + kbCounts := make(map[string]int) + for _, candidate := range candidates { + kbCounts[candidate.TestKBID]++ + } + kbs := make([]string, 0, len(kbCounts)) + for kb := range kbCounts { + kbs = append(kbs, kb) + } + sort.Strings(kbs) + var builder strings.Builder + fmt.Fprintf(&builder, "total=%d\n", len(candidates)) + fmt.Fprintf(&builder, "concurrency=%d\n", concurrency) + for _, kb := range kbs { + fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb]) + } + return builder.String() +} + +func FormatLegacyVerificationReport(report LegacyVerificationReport) string { + return fmt.Sprintf( + "total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n", + report.Total, + report.Existing, + report.Missing, + report.WrongKB, + report.Pending, + report.Processing, + report.Finalizing, + report.Completed, + report.Failed, + report.ChunkChecked, + report.ChunkEmpty, + report.HTTPErrors, + ) +} diff --git a/internal/workflows/testweknora/legacy_verify_test.go b/internal/workflows/testweknora/legacy_verify_test.go new file mode 100644 index 0000000..b0317f1 --- /dev/null +++ b/internal/workflows/testweknora/legacy_verify_test.go @@ -0,0 +1,192 @@ +package testweknora + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "knowledge-governance/internal/platform/weknora" +) + +func TestVerifyLegacyClassifiesWithoutLeaking(t *testing.T) { + client := &legacyClientStub{getResults: map[string]legacyGetResult{ + "knowledge-completed-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}}, + "knowledge-completed-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}}, + "knowledge-pending": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "pending"}}, + "knowledge-processing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "processing"}}, + "knowledge-finalizing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "finalizing"}}, + "knowledge-failed": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "failed"}}, + "knowledge-wrong-kb": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-other", ParseStatus: "completed"}}, + "knowledge-missing": {err: &weknora.HTTPError{StatusCode: 404}}, + "knowledge-server-error": {err: &weknora.HTTPError{StatusCode: 500}}, + }, chunkTotal: map[string]int64{ + "knowledge-completed-a": 2, + "knowledge-completed-b": 0, + }} + + candidates := []LegacyCandidate{ + {ID: "pub-1", KnowledgeID: "knowledge-completed-a", TestKBID: "kb-1"}, + {ID: "pub-2", KnowledgeID: "knowledge-completed-b", TestKBID: "kb-1"}, + {ID: "pub-3", KnowledgeID: "knowledge-pending", TestKBID: "kb-1"}, + {ID: "pub-4", KnowledgeID: "knowledge-processing", TestKBID: "kb-1"}, + {ID: "pub-5", KnowledgeID: "knowledge-finalizing", TestKBID: "kb-1"}, + {ID: "pub-6", KnowledgeID: "knowledge-failed", TestKBID: "kb-1"}, + {ID: "pub-7", KnowledgeID: "knowledge-wrong-kb", TestKBID: "kb-1"}, + {ID: "pub-8", KnowledgeID: "knowledge-missing", TestKBID: "kb-1"}, + {ID: "pub-9", KnowledgeID: "knowledge-server-error", TestKBID: "kb-1"}, + } + + report, err := VerifyLegacy(context.Background(), client, candidates, 1) + if err != nil { + t.Fatalf("VerifyLegacy() error = %v", err) + } + want := LegacyVerificationReport{ + Total: 9, Existing: 6, Missing: 1, WrongKB: 1, + Pending: 1, Processing: 1, Finalizing: 1, Completed: 2, Failed: 1, + ChunkChecked: 2, ChunkEmpty: 1, HTTPErrors: 1, + } + if report != want { + t.Fatalf("report = %#v, want %#v", report, want) + } +} + +func TestVerifyLegacyChunkSampleEverySkipsUnsampledCompleted(t *testing.T) { + client := &legacyClientStub{getResults: map[string]legacyGetResult{ + "knowledge-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}}, + "knowledge-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}}, + }, chunkTotal: map[string]int64{ + "knowledge-b": 5, + }} + candidates := []LegacyCandidate{ + {ID: "pub-1", KnowledgeID: "knowledge-a", TestKBID: "kb-1"}, + {ID: "pub-2", KnowledgeID: "knowledge-b", TestKBID: "kb-1"}, + } + report, err := VerifyLegacy(context.Background(), client, candidates, 2) + if err != nil { + t.Fatal(err) + } + if report.Completed != 2 || report.ChunkChecked != 1 { + t.Fatalf("report = %#v", report) + } + if client.chunkQueries != 1 { + t.Fatalf("chunk queries = %d, want 1", client.chunkQueries) + } +} + +func TestVerifyLegacyConcurrentBoundsConcurrencyAndAggregates(t *testing.T) { + client := &concurrentLegacyClient{} + candidates := make([]LegacyCandidate, 12) + for index := range candidates { + candidates[index] = LegacyCandidate{ID: "publication", KnowledgeID: "knowledge", TestKBID: "kb-1"} + } + + report, err := VerifyLegacyConcurrent(context.Background(), client, candidates, 0, 3) + if err != nil { + t.Fatalf("VerifyLegacyConcurrent() error = %v", err) + } + if report.Total != 12 || report.Existing != 12 || report.Completed != 12 { + t.Fatalf("report = %#v", report) + } + if maximum := client.maximum.Load(); maximum < 2 || maximum > 3 { + t.Fatalf("maximum concurrency = %d, want 2..3", maximum) + } +} + +func TestVerifyLegacyConcurrentReturnsImmediatelyForEmptyInput(t *testing.T) { + report, err := VerifyLegacyConcurrent(context.Background(), &concurrentLegacyClient{}, nil, 10, 4) + if err != nil { + t.Fatalf("VerifyLegacyConcurrent() error = %v", err) + } + if report != (LegacyVerificationReport{}) { + t.Fatalf("report = %#v", report) + } +} + +func TestFormatDryRunReportAggregatesWithoutKnowledgeIDs(t *testing.T) { + candidates := []LegacyCandidate{ + {ID: "pub-1", KnowledgeID: "knowledge-1", TestKBID: "kb-1"}, + {ID: "pub-2", KnowledgeID: "knowledge-2", TestKBID: "kb-1"}, + {ID: "pub-3", KnowledgeID: "knowledge-3", TestKBID: "kb-2"}, + } + report := FormatDryRunReport(candidates, 4) + if !strings.Contains(report, "total=3") || !strings.Contains(report, "concurrency=4") || + !strings.Contains(report, "knowledge_base=kb-1 count=2") || !strings.Contains(report, "knowledge_base=kb-2 count=1") { + t.Fatalf("report = %q", report) + } + if strings.Contains(report, "knowledge-1") || strings.Contains(report, "pub-1") { + t.Fatalf("report leaked knowledge/publication ID: %q", report) + } +} + +func TestFormatLegacyVerificationReportContainsOnlyAggregateCounts(t *testing.T) { + report := FormatLegacyVerificationReport(LegacyVerificationReport{ + Total: 530, Existing: 500, Missing: 20, WrongKB: 10, + Completed: 490, Failed: 10, ChunkChecked: 20, ChunkEmpty: 2, HTTPErrors: 3, + }) + for _, expected := range []string{"total=530", "existing=500", "missing=20", "wrong_kb=10", "completed=490", "failed=10", "chunk_checked=20", "chunk_empty=2", "http_errors=3"} { + if !strings.Contains(report, expected) { + t.Fatalf("report missing %q: %q", expected, report) + } + } + for _, forbidden := range []string{"knowledge_id", "knowledge_base", "publication_id"} { + if strings.Contains(report, forbidden) { + t.Fatalf("report leaked identifier field %q: %q", forbidden, report) + } + } +} + +type legacyClientStub struct { + getResults map[string]legacyGetResult + chunkTotal map[string]int64 + chunkQueries int +} + +type legacyGetResult struct { + knowledge weknora.Knowledge + err error +} + +type concurrentLegacyClient struct { + current atomic.Int64 + maximum atomic.Int64 +} + +func (client *concurrentLegacyClient) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + current := client.current.Add(1) + for { + maximum := client.maximum.Load() + if current <= maximum || client.maximum.CompareAndSwap(maximum, current) { + break + } + } + time.Sleep(10 * time.Millisecond) + client.current.Add(-1) + return weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}, nil +} + +func (*concurrentLegacyClient) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) { + return nil, 1, nil +} + +func (client *legacyClientStub) GetKnowledge(_ context.Context, knowledgeID string) (weknora.Knowledge, error) { + result, ok := client.getResults[knowledgeID] + if !ok { + return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404} + } + return result.knowledge, result.err +} + +func (client *legacyClientStub) ListChunks(_ context.Context, knowledgeID string, _ int, _ int, _ []string) ([]weknora.Chunk, int64, error) { + client.chunkQueries++ + total, ok := client.chunkTotal[knowledgeID] + if !ok { + total = 0 + } + chunks := make([]weknora.Chunk, 0, total) + for i := int64(0); i < total; i++ { + chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: knowledgeID}) + } + return chunks, total, nil +} diff --git a/internal/workflows/testweknora/poll.go b/internal/workflows/testweknora/poll.go new file mode 100644 index 0000000..83a2d20 --- /dev/null +++ b/internal/workflows/testweknora/poll.go @@ -0,0 +1,141 @@ +package testweknora + +import ( + "context" + "errors" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +var ( + ErrInvalidPollJobPayload = errors.New("test_poll_invalid_payload") + ErrPollRemoteFailed = errors.New("test_poll_remote_failed") + ErrPollRemoteIdentity = errors.New("test_poll_remote_identity_invalid") + ErrPollStateConflict = errors.New("test_poll_state_conflict") +) + +type PollStore interface { + LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error) + RecordRemoteStatusAndScheduleNextPoll(context.Context, string, int, governance.ProcessingState, *time.Time, time.Time, time.Time) (int, error) + RecordCompletedAndScheduleValidation(context.Context, string, int, *time.Time, time.Time) error + RecordProcessingFailure(context.Context, string, int, string, time.Time) error + MarkStalled(context.Context, string, int, time.Time, time.Time) error +} + +type PollClient interface { + GetKnowledge(context.Context, string) (weknora.Knowledge, error) +} + +type PollHandlerConfig struct { + Store PollStore + Client PollClient + PollDelay time.Duration + StallAfter time.Duration + Now func() time.Time +} + +type PollHandler struct { + store PollStore + client PollClient + pollDelay time.Duration + stallAfter time.Duration + now func() time.Time +} + +type pollPayload struct { + PublicationID string `json:"publication_id"` + Sequence int `json:"sequence"` +} + +func NewPollHandler(config PollHandlerConfig) (*PollHandler, error) { + if config.Store == nil || config.Client == nil || config.PollDelay <= 0 || config.StallAfter <= 0 { + return nil, errors.New("invalid test poll handler configuration") + } + if config.Now == nil { + config.Now = time.Now + } + return &PollHandler{ + store: config.Store, client: config.Client, pollDelay: config.PollDelay, + stallAfter: config.StallAfter, now: config.Now, + }, nil +} + +func (handler *PollHandler) Handle(ctx context.Context, job jobs.Job) error { + var payload pollPayload + if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" || payload.Sequence < 1 { + return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidPollJobPayload) + } + publication, err := handler.store.LoadPublicationForPoll(ctx, payload.PublicationID) + if err != nil { + return ErrPollStateConflict + } + if publication.KnowledgeID == "" || publication.PollSequence != payload.Sequence { + return ErrPollStateConflict + } + now := handler.now() + knowledge, err := handler.client.GetKnowledge(ctx, publication.KnowledgeID) + if err != nil { + return ErrPollRemoteFailed + } + if knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID { + return ErrPollRemoteIdentity + } + state := mapParseStatus(knowledge.ParseStatus) + if state == governance.ProcessingCompleted { + if err := handler.store.RecordCompletedAndScheduleValidation(ctx, publication.ID, publication.PollSequence, &knowledge.UpdatedAt, now); err != nil { + return ErrPollStateConflict + } + return nil + } + if state == governance.ProcessingFailed { + if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_parse_failed", now); err != nil { + return ErrPollStateConflict + } + return nil + } + if !state.Valid() { + if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_unknown_status", now); err != nil { + return ErrPollStateConflict + } + return nil + } + cutoff := now.Add(-handler.stallAfter) + lastProgress := publication.LastProgressAt + if lastProgress == nil { + lastProgress = publication.ProcessingStartedAt + } + if lastProgress == nil { + lastProgress = publication.LastObservedAt + } + if lastProgress != nil && !lastProgress.After(cutoff) { + if err := handler.store.MarkStalled(ctx, publication.ID, publication.PollSequence, cutoff, now); err != nil { + return ErrPollStateConflict + } + return nil + } + if _, err := handler.store.RecordRemoteStatusAndScheduleNextPoll(ctx, publication.ID, publication.PollSequence, state, &knowledge.UpdatedAt, now, now.Add(handler.pollDelay)); err != nil { + return ErrPollStateConflict + } + return nil +} + +func mapParseStatus(status string) governance.ProcessingState { + switch status { + case "pending": + return governance.ProcessingPending + case "processing": + return governance.ProcessingProcessing + case "finalizing": + return governance.ProcessingFinalizing + case "completed": + return governance.ProcessingCompleted + case "failed": + return governance.ProcessingFailed + default: + return governance.ProcessingState(status) + } +} diff --git a/internal/workflows/testweknora/poll_test.go b/internal/workflows/testweknora/poll_test.go new file mode 100644 index 0000000..d511ad2 --- /dev/null +++ b/internal/workflows/testweknora/poll_test.go @@ -0,0 +1,188 @@ +package testweknora + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +func TestPollHandlerSchedulesNextSequenceForRemoteProgress(t *testing.T) { + remoteUpdated := time.Date(2026, time.September, 4, 3, 39, 0, 0, time.UTC) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:39:00Z"}}`)) + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 40, 0, 0, time.UTC) + lastProgress := now.Add(-time.Minute) + store := &pollStoreStub{publication: postgresstore.PublicationForPoll{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + ProcessingState: governance.ProcessingProcessing, PollSequence: 2, + LastRemoteStatus: "processing", LastProgressAt: &lastProgress, + }} + handler, err := NewPollHandler(PollHandlerConfig{ + Store: store, Client: client, PollDelay: 15 * time.Second, + StallAfter: 30 * time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if store.recordedState != governance.ProcessingFinalizing || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(remoteUpdated) || !store.nextPollAt.Equal(now.Add(15*time.Second)) { + t.Fatalf("store = %#v", store) + } +} + +func TestPollHandlerCompletesAndSchedulesValidationWithoutNextPoll(t *testing.T) { + completedAt := time.Date(2026, time.September, 4, 3, 45, 0, 0, time.UTC) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"completed","updated_at":"2026-09-04T03:45:00Z"}}`)) + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 46, 0, 0, time.UTC) + lastProgress := now.Add(-time.Minute) + store := &pollStoreStub{publication: postgresstore.PublicationForPoll{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + ProcessingState: governance.ProcessingProcessing, PollSequence: 3, + LastRemoteStatus: "processing", LastProgressAt: &lastProgress, + }} + handler, err := NewPollHandler(PollHandlerConfig{ + Store: store, Client: client, PollDelay: 15 * time.Second, + StallAfter: 30 * time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":3}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if !store.completedValidated || store.recordedState != "" || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(completedAt) { + t.Fatalf("store = %#v", store) + } +} + +func TestPollHandlerMarksStalledWhenNoProgressExceedsThreshold(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:10:00Z"}}`)) + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 50, 0, 0, time.UTC) + stalledSince := now.Add(-45 * time.Minute) + store := &pollStoreStub{publication: postgresstore.PublicationForPoll{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + ProcessingState: governance.ProcessingProcessing, PollSequence: 4, + LastRemoteStatus: "processing", LastProgressAt: &stalledSince, + }} + handler, err := NewPollHandler(PollHandlerConfig{ + Store: store, Client: client, PollDelay: 15 * time.Second, + StallAfter: 30 * time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":4}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if !store.stalledMarked || store.recordedState != "" || !store.stalledCutoff.Equal(now.Add(-30*time.Minute)) { + t.Fatalf("store = %#v", store) + } +} + +func TestPollHandlerRejectsStaleSequenceWithoutPollingRemote(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("stale sequence must not poll the remote knowledge endpoint") + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 4, 0, 0, 0, time.UTC) + store := &pollStoreStub{publication: postgresstore.PublicationForPoll{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + ProcessingState: governance.ProcessingProcessing, PollSequence: 5, + }} + handler, err := NewPollHandler(PollHandlerConfig{ + Store: store, Client: client, PollDelay: 15 * time.Second, + StallAfter: 30 * time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); !errors.Is(err, ErrPollStateConflict) { + t.Fatalf("Handle() error = %v, want state conflict", err) + } + if store.recordedState != "" || store.stalledMarked || store.completedValidated { + t.Fatalf("store mutated for stale sequence: %#v", store) + } +} + +type pollStoreStub struct { + publication postgresstore.PublicationForPoll + recordedState governance.ProcessingState + recordedRemoteUpdated *time.Time + nextPollAt time.Time + completedValidated bool + stalledMarked bool + stalledCutoff time.Time +} + +func (store *pollStoreStub) LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error) { + return store.publication, nil +} + +func (store *pollStoreStub) RecordRemoteStatusAndScheduleNextPoll(_ context.Context, _ string, _ int, state governance.ProcessingState, remoteUpdated *time.Time, _ time.Time, nextAt time.Time) (int, error) { + store.recordedState = state + store.recordedRemoteUpdated = remoteUpdated + store.nextPollAt = nextAt + return store.publication.PollSequence + 1, nil +} + +func (store *pollStoreStub) RecordCompletedAndScheduleValidation(_ context.Context, _ string, _ int, remoteUpdated *time.Time, _ time.Time) error { + store.completedValidated = true + store.recordedRemoteUpdated = remoteUpdated + return nil +} + +func (store *pollStoreStub) RecordProcessingFailure(context.Context, string, int, string, time.Time) error { + return nil +} + +func (store *pollStoreStub) MarkStalled(_ context.Context, _ string, _ int, cutoff, _ time.Time) error { + store.stalledMarked = true + store.stalledCutoff = cutoff + return nil +} diff --git a/internal/workflows/testweknora/remove.go b/internal/workflows/testweknora/remove.go new file mode 100644 index 0000000..ebacadd --- /dev/null +++ b/internal/workflows/testweknora/remove.go @@ -0,0 +1,117 @@ +package testweknora + +import ( + "context" + "errors" + "net/http" + "time" + + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +var ( + ErrInvalidRemoveJobPayload = errors.New("test_remove_invalid_payload") + ErrRemoveRemoteFailed = errors.New("test_remove_remote_failed") + ErrRemoveRemoteIdentity = errors.New("test_remove_remote_identity_invalid") + ErrRemovePending = errors.New("test_remove_pending") + ErrRemoveStateConflict = errors.New("test_remove_state_conflict") +) + +type RemoveStore interface { + LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error) + RecordRemovalCompleted(context.Context, string, time.Time) error + RecordRemovalFailure(context.Context, string, string, time.Time) error +} + +type RemoveClient interface { + GetKnowledge(context.Context, string) (weknora.Knowledge, error) + DeleteKnowledge(context.Context, string) error +} + +type RemoveHandlerConfig struct { + Store RemoveStore + Client RemoveClient + Now func() time.Time +} + +type RemoveHandler struct { + store RemoveStore + client RemoveClient + now func() time.Time +} + +type removePayload struct { + PublicationID string `json:"publication_id"` +} + +func NewRemoveHandler(config RemoveHandlerConfig) (*RemoveHandler, error) { + if config.Store == nil || config.Client == nil { + return nil, errors.New("invalid test removal handler configuration") + } + if config.Now == nil { + config.Now = time.Now + } + return &RemoveHandler{store: config.Store, client: config.Client, now: config.Now}, nil +} + +func (handler *RemoveHandler) Handle(ctx context.Context, job jobs.Job) error { + var payload removePayload + if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" { + return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidRemoveJobPayload) + } + candidate, err := handler.store.LoadPublicationForRemoval(ctx, payload.PublicationID) + if err != nil { + return ErrRemoveStateConflict + } + knowledge, err := handler.client.GetKnowledge(ctx, candidate.KnowledgeID) + if isNotFound(err) { + return handler.complete(ctx, candidate.ID) + } + if err != nil { + return ErrRemoveRemoteFailed + } + if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID { + return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid") + } + if err := handler.client.DeleteKnowledge(ctx, candidate.KnowledgeID); err != nil { + if isNotFound(err) { + return handler.complete(ctx, candidate.ID) + } + if !weknora.IsRetryable(err) { + return handler.failClosed(ctx, candidate.ID, "removal_remote_rejected") + } + return ErrRemoveRemoteFailed + } + knowledge, err = handler.client.GetKnowledge(ctx, candidate.KnowledgeID) + if isNotFound(err) { + return handler.complete(ctx, candidate.ID) + } + if err != nil { + return ErrRemoveRemoteFailed + } + if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID { + return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid") + } + return ErrRemovePending +} + +func (handler *RemoveHandler) complete(ctx context.Context, publicationID string) error { + if err := handler.store.RecordRemovalCompleted(ctx, publicationID, handler.now()); err != nil { + return ErrRemoveStateConflict + } + return nil +} + +func (handler *RemoveHandler) failClosed(ctx context.Context, publicationID, code string) error { + if err := handler.store.RecordRemovalFailure(ctx, publicationID, code, handler.now()); err != nil { + return ErrRemoveStateConflict + } + return nil +} + +func isNotFound(err error) bool { + var httpErr *weknora.HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound +} diff --git a/internal/workflows/testweknora/remove_test.go b/internal/workflows/testweknora/remove_test.go new file mode 100644 index 0000000..5d823f2 --- /dev/null +++ b/internal/workflows/testweknora/remove_test.go @@ -0,0 +1,100 @@ +package testweknora + +import ( + "context" + "errors" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +type removalStoreStub struct { + candidate postgresstore.PublicationForRemoval + completed string + failed string +} + +func (store *removalStoreStub) LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error) { + return store.candidate, nil +} + +func (store *removalStoreStub) RecordRemovalCompleted(_ context.Context, publicationID string, _ time.Time) error { + store.completed = publicationID + return nil +} + +func (store *removalStoreStub) RecordRemovalFailure(_ context.Context, publicationID, code string, _ time.Time) error { + store.failed = publicationID + ":" + code + return nil +} + +type removalClientStub struct { + knowledge weknora.Knowledge + getCalls int + deleted string + keepAfterDelete bool +} + +func (client *removalClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + client.getCalls++ + if client.getCalls > 1 && !client.keepAfterDelete { + return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404} + } + return client.knowledge, nil +} + +func (client *removalClientStub) DeleteKnowledge(_ context.Context, knowledgeID string) error { + client.deleted = knowledgeID + return nil +} + +func TestRemoveHandlerVerifiesIdentityAndReadBackBeforeCompletion(t *testing.T) { + store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling, + }} + client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}} + handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client, Now: func() time.Time { return time.Date(2026, 9, 9, 3, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if client.deleted != "knowledge-1" || store.completed != "publication-1" || store.failed != "" { + t.Fatalf("deleted=%q completed=%q failed=%q", client.deleted, store.completed, store.failed) + } +} + +func TestRemoveHandlerRejectsMismatchedKnowledgeBaseWithoutDeleting(t *testing.T) { + store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling, + }} + client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "other-kb"}} + handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client}) + if err != nil { + t.Fatal(err) + } + err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}) + if err != nil || client.deleted != "" || store.failed != "publication-1:removal_identity_invalid" { + t.Fatalf("error=%v deleted=%q failed=%q", err, client.deleted, store.failed) + } +} + +func TestRemoveHandlerRetriesWhileAsynchronousDeleteIsStillVisible(t *testing.T) { + store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling, + }} + client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}, keepAfterDelete: true} + handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client}) + if err != nil { + t.Fatal(err) + } + err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}) + if !errors.Is(err, ErrRemovePending) || store.completed != "" { + t.Fatalf("error=%v completed=%q", err, store.completed) + } +} diff --git a/internal/workflows/testweknora/upload.go b/internal/workflows/testweknora/upload.go new file mode 100644 index 0000000..960e2d5 --- /dev/null +++ b/internal/workflows/testweknora/upload.go @@ -0,0 +1,241 @@ +package testweknora + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "path/filepath" + "strings" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/spool" + "knowledge-governance/internal/platform/weknora" +) + +var ( + ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload") + ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid") + ErrUploadRemoteFailed = errors.New("test_upload_remote_failed") + ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid") + ErrUploadStateConflict = errors.New("test_upload_state_conflict") +) + +type UploadStore interface { + LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) + MarkUploading(context.Context, string, time.Time) error + RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error) + RecordUploadFailure(context.Context, string, string, time.Time) error +} + +type UploadClient interface { + UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) + ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) + GetKnowledge(context.Context, string) (weknora.Knowledge, error) +} + +type UploadHandlerConfig struct { + Store UploadStore + Client UploadClient + SpoolRoot string + MaxUploadBytes int64 + FirstPollDelay time.Duration + Now func() time.Time +} + +type UploadHandler struct { + store UploadStore + client UploadClient + spoolRoot string + maxUploadBytes int64 + firstPollDelay time.Duration + now func() time.Time +} + +type uploadPayload struct { + PublicationID string `json:"publication_id"` + TargetScope string `json:"target_scope,omitempty"` + TargetKBID string `json:"target_kb_id,omitempty"` +} + +func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) { + if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) || + config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 { + return nil, errors.New("invalid test upload handler configuration") + } + if config.Now == nil { + config.Now = time.Now + } + return &UploadHandler{ + store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot, + maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now, + }, nil +} + +func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error { + var payload uploadPayload + if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" { + return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload) + } + candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID) + if err != nil { + return ErrUploadStateConflict + } + // P4: the job snapshots its target; a mismatch against the stored publication + // means the plan changed underneath and this delivery must not guess. + if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID { + return ErrUploadStateConflict + } + if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) { + return ErrUploadStateConflict + } + if candidate.IngestState == governance.IngestUploading { + knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate) + if err != nil { + if errors.Is(err, ErrUploadRemoteIdentity) { + return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous") + } + return err + } + now := handler.now() + if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil { + return ErrUploadStateConflict + } + return nil + } + artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{ + RelativeRef: candidate.LocalSpoolRef, + SHA256: candidate.SHA256, + SizeBytes: candidate.SizeBytes, + Format: candidate.ValidatedFormat, + }, handler.maxUploadBytes) + if err != nil { + return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid") + } + defer artifact.Close() + if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil { + return ErrUploadStateConflict + } + knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{ + KnowledgeBaseID: candidate.TestKBID, + FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat), + Content: artifact, + Metadata: map[string]string{ + "content_sha256": candidate.SHA256, + "publication_id": candidate.ID, + }, + Channel: "governance", + }) + if err := artifact.VerifyComplete(); err != nil { + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid) + } + if uploadErr != nil { + if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) { + return ErrUploadRemoteFailed + } + knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate) + if err != nil { + if errors.Is(err, ErrUploadRemoteIdentity) { + return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous") + } + return err + } + } + if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID { + return ErrUploadRemoteIdentity + } + now := handler.now() + if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil { + return ErrUploadStateConflict + } + return nil +} + +func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) { + const pageSize = 100 + matches := make([]weknora.Knowledge, 0, 2) + for page := 1; page <= 100; page++ { + items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize) + if err != nil { + return weknora.Knowledge{}, ErrUploadRemoteFailed + } + for _, item := range items { + publicationID, publicationOK := metadataString(item.Metadata, "publication_id") + contentSHA, contentOK := metadataString(item.Metadata, "content_sha256") + if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 { + matches = append(matches, item) + if len(matches) > 1 { + return weknora.Knowledge{}, ErrUploadRemoteIdentity + } + } + } + if int64(page*pageSize) >= total { + break + } + if len(items) == 0 { + return weknora.Knowledge{}, ErrUploadRemoteFailed + } + } + if len(matches) != 1 { + return weknora.Knowledge{}, ErrUploadRemoteFailed + } + verified, err := handler.client.GetKnowledge(ctx, matches[0].ID) + if err != nil { + return weknora.Knowledge{}, ErrUploadRemoteFailed + } + publicationID, publicationOK := metadataString(verified.Metadata, "publication_id") + contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256") + if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID || + !publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 { + return weknora.Knowledge{}, ErrUploadRemoteIdentity + } + return verified, nil +} + +func metadataString(metadata map[string]any, key string) (string, bool) { + value, ok := metadata[key].(string) + return value, ok +} + +func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error { + if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil { + return ErrUploadStateConflict + } + return nil +} + +func decodeStrictJSON(data []byte, target any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return errors.New("extra JSON value") + } + return nil +} + +func safeUploadFileName(title, fallback, format string) string { + base := strings.TrimSpace(title) + base = strings.Map(func(character rune) rune { + switch character { + case '/', '\\', '\x00', '\r', '\n', '\t': + return '_' + default: + return character + } + }, base) + base = strings.Trim(base, ". ") + if base == "" { + base = fallback + } + if len(base) > 120 { + base = base[:120] + } + return base + "." + format +} diff --git a/internal/workflows/testweknora/upload_test.go b/internal/workflows/testweknora/upload_test.go new file mode 100644 index 0000000..422f1ec --- /dev/null +++ b/internal/workflows/testweknora/upload_test.go @@ -0,0 +1,354 @@ +package testweknora + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +func TestUploadHandlerStreamsVerifiedArtifactAndSchedulesFirstPoll(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("safe governance document\n") + if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + sha := hex.EncodeToString(digest[:]) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/knowledge-bases/kb-1/knowledge/file" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + file, _, err := r.FormFile("file") + if err != nil { + t.Fatal(err) + } + defer file.Close() + uploaded, err := io.ReadAll(file) + if err != nil || string(uploaded) != string(body) { + t.Fatalf("uploaded body = %q, %v", uploaded, err) + } + if r.FormValue("metadata") != `{"content_sha256":"`+sha+`","publication_id":"publication-1"}` { + t.Fatalf("metadata = %q", r.FormValue("metadata")) + } + if r.FormValue("channel") != "governance" { + t.Fatalf("channel = %q", r.FormValue("channel")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"pending"}}`)) + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + + now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC) + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1", + IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending, + ContentObjectID: "content-1", SHA256: sha, SizeBytes: int64(len(body)), + ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title", + }} + handler, err := NewUploadHandler(UploadHandlerConfig{ + Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024, + FirstPollDelay: 10 * time.Second, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1"}`)} + if err := handler.Handle(context.Background(), job); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if !store.markedUploading || store.recordedKnowledgeID != "knowledge-1" || !store.nextPollAt.Equal(now.Add(10*time.Second)) { + t.Fatalf("store = %#v", store) + } +} + +func TestUploadHandlerRejectsJobSnapshotWithMismatchedTarget(t *testing.T) { + root := t.TempDir() + now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC) + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1", TargetScope: governance.VisibilityInternal, + IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending, + }} + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: "http://127.0.0.1:1", APIKey: "***", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + handler, err := NewUploadHandler(UploadHandlerConfig{Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 1 << 20, FirstPollDelay: time.Second, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"external","target_kb_id":"kb-1"}`)} + if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) { + t.Fatalf("scope mismatch error = %v", err) + } + job = jobs.Job{ID: "job-2", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"internal","target_kb_id":"kb-other"}`)} + if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) { + t.Fatalf("kb mismatch error = %v", err) + } + if store.markedUploading { + t.Fatal("mismatched job must not mark uploading") + } +} + +func TestUploadHandlerRecoversUniqueRemoteKnowledgeAfterResponseLoss(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("safe ambiguous upload\n") + if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + sha := hex.EncodeToString(digest[:]) + var uploads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost: + uploads.Add(1) + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + file, _, err := r.FormFile("file") + if err != nil { + t.Fatal(err) + } + _, _ = io.Copy(io.Discard, file) + _ = file.Close() + connection, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + _ = connection.Close() + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-recovered": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 10, 0, 0, time.UTC) + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending, + ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)), + ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title", + }} + handler, err := NewUploadHandler(UploadHandlerConfig{ + Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024, + FirstPollDelay: time.Second, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if uploads.Load() != 1 || store.recordedKnowledgeID != "knowledge-recovered" { + t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID) + } +} + +func TestUploadHandlerRecoversUploadingStateBeforeAnotherPost(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("safe retry recovery\n") + if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + sha := hex.EncodeToString(digest[:]) + var uploads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost: + uploads.Add(1) + http.Error(w, "must not upload", http.StatusInternalServerError) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-existing": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 20, 0, 0, time.UTC) + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading, + ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)), + ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title", + }} + handler, err := NewUploadHandler(UploadHandlerConfig{ + Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024, + FirstPollDelay: time.Second, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-existing" { + t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID) + } +} + +func TestUploadHandlerFailsClosedWhenAmbiguousRecoveryHasMultipleMatches(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil { + t.Fatal(err) + } + body := []byte("safe ambiguous duplicate\n") + if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(body) + sha := hex.EncodeToString(digest[:]) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost: + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"success":false,"data":{}}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"data":[` + + `{"id":"knowledge-a","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}},` + + `{"id":"knowledge-b","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}` + + `],"total":2,"page":1,"page_size":100}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, time.September, 4, 3, 30, 0, 0, time.UTC) + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending, + ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)), + ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title", + }} + handler, err := NewUploadHandler(UploadHandlerConfig{ + Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024, + FirstPollDelay: time.Second, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if store.uploadFailureCode != "upload_identity_ambiguous" || store.recordedKnowledgeID != "" { + t.Fatalf("failure=%q knowledge=%q", store.uploadFailureCode, store.recordedKnowledgeID) + } +} + +func TestUploadHandlerRecordsInvalidArtifactForManualRetry(t *testing.T) { + root := t.TempDir() + store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{ + ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending, + ProcessingState: governance.ProcessingPending, SHA256: strings.Repeat("a", 64), SizeBytes: 128, + ValidatedFormat: "md", LocalSpoolRef: "missing/artifact.md", Title: "Safe title", + }} + handler, err := NewUploadHandler(UploadHandlerConfig{ + Store: store, Client: uploadClientStub{}, SpoolRoot: root, MaxUploadBytes: 1024, + FirstPollDelay: time.Second, + }) + if err != nil { + t.Fatal(err) + } + + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v, want terminal failure recorded", err) + } + if store.uploadFailureCode != "upload_artifact_invalid" || store.markedUploading { + t.Fatalf("failure=%q markedUploading=%v", store.uploadFailureCode, store.markedUploading) + } +} + +type uploadStoreStub struct { + candidate postgresstore.PublicationForUpload + markedUploading bool + recordedKnowledgeID string + nextPollAt time.Time + uploadFailureCode string +} + +func (store *uploadStoreStub) LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) { + return store.candidate, nil +} + +func (store *uploadStoreStub) MarkUploading(context.Context, string, time.Time) error { + store.markedUploading = true + return nil +} + +func (store *uploadStoreStub) RecordUploadAndSchedulePoll(_ context.Context, _ string, knowledgeID string, _ time.Time, nextAt time.Time) (int, error) { + store.recordedKnowledgeID = knowledgeID + store.nextPollAt = nextAt + return 1, nil +} + +func (store *uploadStoreStub) RecordUploadFailure(_ context.Context, _ string, code string, _ time.Time) error { + store.uploadFailureCode = code + return nil +} + +type uploadClientStub struct{} + +func (uploadClientStub) UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) { + return weknora.Knowledge{}, errors.New("unexpected upload") +} + +func (uploadClientStub) ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) { + return nil, 0, errors.New("unexpected list") +} + +func (uploadClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + return weknora.Knowledge{}, errors.New("unexpected get") +} diff --git a/internal/workflows/testweknora/validate.go b/internal/workflows/testweknora/validate.go new file mode 100644 index 0000000..115401a --- /dev/null +++ b/internal/workflows/testweknora/validate.go @@ -0,0 +1,143 @@ +package testweknora + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +var ( + ErrInvalidValidateJobPayload = errors.New("test_validate_invalid_payload") + ErrValidateStateConflict = errors.New("test_validate_state_conflict") + ErrValidateRecordFailed = errors.New("test_validate_record_failed") +) + +type ValidateStore interface { + LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) + AppendAcceptanceResult(context.Context, governance.AcceptanceResult) error +} + +type ValidateClient interface { + GetKnowledge(context.Context, string) (weknora.Knowledge, error) + ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) +} + +type ValidateHandlerConfig struct { + Store ValidateStore + Client ValidateClient + RunID string + PolicyVersion string + Now func() time.Time +} + +type ValidateHandler struct { + store ValidateStore + client ValidateClient + runID string + policyVersion string + now func() time.Time +} + +type validatePayload struct { + PublicationID string `json:"publication_id"` +} + +func NewValidateHandler(config ValidateHandlerConfig) (*ValidateHandler, error) { + if config.Store == nil || config.Client == nil || config.PolicyVersion == "" { + return nil, errors.New("invalid test validate handler configuration") + } + if config.RunID == "" { + config.RunID = "default" + } + if config.Now == nil { + config.Now = time.Now + } + return &ValidateHandler{ + store: config.Store, client: config.Client, runID: config.RunID, + policyVersion: config.PolicyVersion, now: config.Now, + }, nil +} + +func (handler *ValidateHandler) Handle(ctx context.Context, job jobs.Job) error { + var payload validatePayload + if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" { + return ErrInvalidValidateJobPayload + } + publication, err := handler.store.LoadPublicationForValidation(ctx, payload.PublicationID) + if err != nil { + return ErrValidateStateConflict + } + now := handler.now() + runID := handler.runID + if job.ID != "" { + runID = job.ID + } + + knowledge, knowledgeErr := handler.client.GetKnowledge(ctx, publication.KnowledgeID) + knowledgeState := governance.AcceptancePassed + knowledgeEvidence := json.RawMessage(`{"matched":true}`) + if knowledgeErr != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID { + knowledgeState = governance.AcceptanceFailed + knowledgeEvidence = json.RawMessage(`{"matched":false}`) + } + if err := handler.append(ctx, publication.ID, runID, "knowledge_exists", knowledgeState, knowledgeEvidence, now); err != nil { + return err + } + + _, chunkTotal, chunksErr := handler.client.ListChunks(ctx, publication.KnowledgeID, 1, 1, nil) + chunksState := governance.AcceptancePassed + if chunksErr != nil || chunkTotal <= 0 { + chunksState = governance.AcceptanceFailed + } + if err := handler.append(ctx, publication.ID, runID, "chunks_present", chunksState, mustJSON(map[string]any{"chunk_total": chunkTotal}), now); err != nil { + return err + } + + if err := handler.append(ctx, publication.ID, runID, "retrieval_hit", governance.AcceptanceManualRequired, json.RawMessage(`{"reason":"thresholds_unresolved"}`), now); err != nil { + return err + } + return nil +} + +func (handler *ValidateHandler) append(ctx context.Context, publicationID, runID, checkType string, state governance.AcceptanceState, evidence json.RawMessage, observedAt time.Time) error { + result := governance.AcceptanceResult{ + ID: acceptanceResultID(publicationID, runID, checkType), + PublicationID: publicationID, + RunID: runID, + CheckType: checkType, + State: state, + PolicyVersion: handler.policyVersion, + EvidenceSummary: evidence, + ObservedAt: observedAt, + } + if err := handler.store.AppendAcceptanceResult(ctx, result); err != nil { + return ErrValidateRecordFailed + } + return nil +} + +func acceptanceResultID(publicationID, runID, checkType string) string { + digest := sha256.New() + _, _ = digest.Write([]byte(publicationID)) + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(runID)) + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write([]byte(checkType)) + return "acc-" + hex.EncodeToString(digest.Sum(nil))[:32] +} + +func mustJSON(value any) json.RawMessage { + data, err := json.Marshal(value) + if err != nil { + return json.RawMessage(`{}`) + } + return data +} diff --git a/internal/workflows/testweknora/validate_test.go b/internal/workflows/testweknora/validate_test.go new file mode 100644 index 0000000..e7d3e46 --- /dev/null +++ b/internal/workflows/testweknora/validate_test.go @@ -0,0 +1,118 @@ +package testweknora + +import ( + "context" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/weknora" +) + +func TestValidateHandlerAppendsDeterministicChecksAndManualRetrieval(t *testing.T) { + client := &validateClientStub{ + knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}, + chunkTotal: 3, + } + store := &validateStoreStub{publication: postgresstore.PublicationForValidation{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + }} + now := time.Date(2026, time.September, 4, 4, 10, 0, 0, time.UTC) + handler, err := NewValidateHandler(ValidateHandlerConfig{ + Store: store, Client: client, RunID: "run-1", PolicyVersion: "structural-v1", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if len(store.appended) != 3 { + t.Fatalf("appended = %d, want 3: %#v", len(store.appended), store.appended) + } + byType := map[string]governance.AcceptanceResult{} + for _, result := range store.appended { + byType[result.CheckType] = result + } + if byType["knowledge_exists"].State != governance.AcceptancePassed { + t.Fatalf("knowledge_exists = %#v", byType["knowledge_exists"]) + } + if byType["chunks_present"].State != governance.AcceptancePassed { + t.Fatalf("chunks_present = %#v", byType["chunks_present"]) + } + if byType["retrieval_hit"].State != governance.AcceptanceManualRequired { + t.Fatalf("retrieval_hit = %#v", byType["retrieval_hit"]) + } + for _, result := range store.appended { + if result.PublicationID != "publication-1" || result.RunID != "run-1" || result.ObservedAt != now { + t.Fatalf("acceptance metadata = %#v", result) + } + } +} + +func TestValidateHandlerFailsChecksWithoutAborting(t *testing.T) { + client := &validateClientStub{ + knowledge: weknora.Knowledge{ID: "knowledge-mismatch", KnowledgeBaseID: "kb-other"}, + chunkTotal: 0, + } + store := &validateStoreStub{publication: postgresstore.PublicationForValidation{ + ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", + }} + handler, err := NewValidateHandler(ValidateHandlerConfig{ + Store: store, Client: client, RunID: "run-2", PolicyVersion: "structural-v1", + Now: func() time.Time { return time.Date(2026, time.September, 4, 4, 11, 0, 0, time.UTC) }, + }) + if err != nil { + t.Fatal(err) + } + if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil { + t.Fatalf("Handle() error = %v", err) + } + if len(store.appended) != 3 { + t.Fatalf("appended = %d, want 3", len(store.appended)) + } + byType := map[string]governance.AcceptanceResult{} + for _, result := range store.appended { + byType[result.CheckType] = result + } + if byType["knowledge_exists"].State != governance.AcceptanceFailed || byType["chunks_present"].State != governance.AcceptanceFailed { + t.Fatalf("failed checks = %#v", store.appended) + } + if byType["retrieval_hit"].State != governance.AcceptanceManualRequired { + t.Fatalf("retrieval still manual = %#v", byType["retrieval_hit"]) + } +} + +type validateStoreStub struct { + publication postgresstore.PublicationForValidation + appended []governance.AcceptanceResult +} + +func (store *validateStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) { + return store.publication, nil +} + +func (store *validateStoreStub) AppendAcceptanceResult(_ context.Context, result governance.AcceptanceResult) error { + store.appended = append(store.appended, result) + return nil +} + +type validateClientStub struct { + knowledge weknora.Knowledge + chunkTotal int64 +} + +func (client *validateClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) { + return client.knowledge, nil +} + +func (client *validateClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) { + chunks := make([]weknora.Chunk, 0, client.chunkTotal) + for i := int64(0); i < client.chunkTotal; i++ { + chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: "knowledge-1"}) + } + return chunks, client.chunkTotal, nil +} diff --git a/scripts/__pycache__/import_display_metadata.cpython-313.pyc b/scripts/__pycache__/import_display_metadata.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f1ba722ab0727babcba4db777bf688c6b8c95e6 GIT binary patch literal 6678 zcma)ATWlQHc|J3)w;iq|hmsbB{Ob&-Hz>Sp2mS4m+Y;Gvz?&wvtTFrzT~pLBm1mnO67?{HBwz_vud2( z4pr>o9E}KjIVa>k&IP%j^Kxa4=qiQFIX8S)T&m3}bz zhh;3qm)Q9PFJEJ)r&&Q#Qwe^VO^O;X@EXr9U?s_lHzie*W*_)NUyFT1ZvWI4-g`u%wCVQ8t}QD7*lb>yj$XO9@F^X5&k|yhu6~%ofP6 zi&&Ku8Ri-l7x;8SV{uwmS#3#V)wq%p1(*nSBeIgr@~nn=S>@vzj9__LV6l=&sH~*1 z^L+eT(ESy;r^x_i*rs`HDRVfqq$I^qW=VvTNY~OKL77*y&Xj^RKGCTz%kdC7=vU-% z;)NF?GawuM$-D2YK%7a)0Yn54jrE>2){Husi!cO-li!$Qd+XJf>o4#$N zwV&diVU=C`rLBIEngdD7r$~iWE0wKehu^~e);w(}z)WTiT0F<9%gKZ!UsG8gi|m4e z=OsZ94F^sW-4JWjd z5rz|ss*<=a8h&WOVuIJC>tak(4BJ&zkqx`a)J7SRhL|Sa)B=p*vd9cu3fPY1SW^lA zuqC5eCC?gMPQ;G z$i1rXK(>OmJ>_d1w@$5$ZdcZ1D_&l=edPVn3zZE)uXuUYw&f|med^Y!O;3~VY1*!8 z$~F(`Rj0DvQ(xOr<Yp`z-1K(`Hd_XBErY+-{^{+{-u~5h zH(MriEtA{+`fS57-QSgUb-{|ad*A6@6F!jNmG#=gx~DyBYu|z0{2#*fbLatKDhgT@ z%@qabM4?t>E1+7WqUhViM&8AEQDiX;NB7=ANN5JOEQ$)pG@>ZlS5$%)ZE-aR0wo5^J(-lRRL%lG^b@V=%g^3!3 zh#)UgCz%B*K!3%;Bt&B|3KjyiK?jc+R9q#u!Lpdhhf&tQD9Yl^6dp9{6a2iG2=*tG zIG<1lgT>yHUQ6)zz?8RLJS|)0m0RA*+vB&!e>kxc-gdfIPy8UWHljQ0?sn%6^!{U5KhV3mzc;tP_ks8@ z{VSQ!dn#7WVYnF-FR%er$vqXc@PC0U!6k#eB}#N3I!~+gF{ILMFARYrA!KNnF5My=Wzv%eoJ1wup(Fto`Rpt4}VAmRT$D zFkiGRL_N1Uy!FIyq5b(&gNgcooVtjh1}JL{a_=lUhF-Y$GHX{>3Ac2z@CI&JuV@dW zZi|9*oA?P{a<1Gu;asIvE74i>QxxZrJ)9@%F}c7CD#crxApALVas7suKnz@>YWwKiAPh`4<{bQ|z93xAnj zuxj>V&AGc4sGcPH)Sz3S&qYSVXfGKjrWVWPXpI?TTF&qG$;I#}&GMq^E2<;H?+Uu)~eTpcN>2gTNfVtT63YHm2lQOyyYujIlWa;zuxeNikAQw9SQzJ$0r^4 z`yW<)eq>ayy7>6W=*Gc~be{7dy~$XSV9%c4ggX zC-wS~U!J_r->q4H^P}{K>5cE`2ao-+q3g4gx%!cZ_@R)ie_gK}Uv>Pcyk@(u>F(@C z+s`^a?zq?gph|D;*P916n}>7F!+PDw>X|>4*KIYnt~ag`;xD|_UqmnIHJ2YpFJ-4^ z?uhF%A4NZmZk*DayEmJA?&Hm-!CceegN)wv`X6h@bJ0ty?rhEF?O@maBY&?vtki?U zcSi44uV2+00-skOc|@<4{kzA%7SKKW*R_ph*ya7R5BKY@oZG5zTs^y8+x&t2xAMm9 zy+a?*Z5}$AJ9JV%)T7t-Zq|K=CEeD2U| z4@UGur%W{2r&sl7z5QDUT2|ls+_nGfL#VRto5RRkwRZS}&UZWC3+38-bDqAet?${> zmyvITdiK*MedzD71WdBc{HYz%l6PhrbHQ1Qgn~zv}kYcdo8i-wWsb$2a}GIe+hi?uRzr zAO7ChP9t)a|1|urYa{cxqU%Y6*W3LBdF|bK7!Bi#iRf`tP@V#({CPo{Bev2^9Mwa~ zAUKg?IKt`yVW5f7^w2EnFZx-fehUJCM^p?$lt30&1g*Q;92Vp#iDC(Oan4=Ps~BVe zA$83Wt7|3_0E>cCV6AP<2UKndHUN;@n=I&Pinj;E+v`W&4$jB<$;swwE&2#?t@fl= z6hd723{?V67h7JmhIJawH`ioM0V`DW*kvZlM2P3MClz-^AQ>_4X4(zOO7{XYv|1FU z(mFZ#nJ7?2+)Ixj=0gl1cNu4AXPzb9Dp+xdZs4>CG>&{=Eps51o(Ig+DJb!Dl0;2H zmgx@?Et5HzN_8%z6Nyj}2XtCdi97*71uecL3ImyP_J*V_#Vi~)@El|!8oV3+f7X%a z5Fxb=^Qw>7gE<8PD zxUmSQ$wN~&x2yu%4BwOp(A5MuAzXpOTnQUCbFE8oId~j0m1r@79`jn(M&2BMAKE5J zXPF5Yw|zC6zP87{wqMlcd_4dPZSGB5UCvgwUUu(jmefvec`6}fmLpNK0}xvC)Z9sC z+W-`WfA<&J=yiSUhVHou?Y6SjGk3W4>i6DQ8(;s+pR2j!=W{br{rGu3bUqurkiBp@ zDwK|8`c5-sFf+pdjQ9jL--xF)QUY?x%GeFFog5vRfQEO3UjgJX-xP~wus zVwk9UOsoZdn`A`e=6#V_QX}Tgd;>CqI07&NA`r;jK>nRv{UBE9ggA)*8am*WkeY(* z2}4oT*R+kY?YI$D@g?$qi9G*?>c2#l|Kapxy|3ub6DzhaoSxM)KltvZvpMH%)}048 zo$Wbi`#mb>JhEbY>TpsWPpchN<q#oJ4s~MeX_`>G;tJ6;y JWNS2c^gn*T`=6FwQz5x9;3ZOS2LXezWbu(X15&@t>H{gCX&8tDGc=a|BC>1WU5i zUQ#60loBa5rA1mz8Ie&_gJ@7wqi9r9lW0;?vuIXRi)ewA-fP`s6KxKn+w3gVhfJl; zlGL~U1+AF%l$LWKE{477dosj~J(*(Wo-8qIPqvu7Cr8Y25QRiJ!5WJR*3_N5p4M83 zd93+ZKKvK3S*)d)I7*6zthJaBi&z`P#jG9T5;hIuQZ^mpC2R)7OS{Y1%uO8x;Z7rn zZbv!My|kQIpFn9ntPBR8?4v5P-DQw<^1UK2^*aIr?~no^(Zf6ZzT^BFM^DJ-_d47T z$=wSzf_q5n4+PgZ+yWGOg1+PKAn))6tF2aNNRXhp|D?kwNIoy`2=?=ifavuJZtR3mvr z-%wDhG$`5XGP&G{s7vcc$=AwY0s4Y&|3IQxQgXHOp&=KpunukWxB)8A=aP>3m5k%i zlNXjYQR~6s^^n*+;#Mw{oFlpkAZ!AO8VH&anLfIbQPLCpxy>K&4DjAL8-TtA6EG-d zmrHOD@-CNRb-4xu-jE-^?Jn1`klUZA$#A)PeWDcf`vhJHKy{wWB?a9h2qVzyqa5FV`2g9$$Hl-w#+PdaC~FHz%$)5aN&5gla5at=;+wL!3T zB~BngB%v$>kt*eEEL8#RyApjYfJ>GF9F1_CG1(6w+A+W~YN_eA(nnc?T57(n^m*2( zmRfEreVsL_rPgFAa5B~m<5{r0OXq7umr+N?Y1Y3|X*mHctgLM##hO|BM#@3-5tV7U z^9I<$K3*{oiGe;ia*|>igcyq~P~?Gc-m4gb0U$%grqQrUDoU0oB#LlUTwb5Zdx8P+ zq>>|f+=9S+lSMvWQu4h%$%E&7L3x@EtR7D#501NU5Qy9BQU?QCgHzkCWNA5{vQcOeclPd2(ig(NlVh&a_{Gcx=-VdJ{ z@Gm_BQG}Rf^5j(yL<3XV1(*BeZg zceyUwCof$a%m3OmX?!MlI{Z{PS}_)inW`=~#L127!SU$q0+1q*a?}&^8ho51xa3*z z0#qN|kIwnf*a8Ae6%$?9N(~&L-yS)4+$RFuk2%CHSXzPVSmsPptlpMLw%80sUO17> z01FgK+92kDjj;?vtRN~4Eg~|v+i*t=*1`WR0ixvqkNCrAs|g~IwHV&|ZKt0nFPs(} z18+rjY}jNyj~s&~TS<;QPMkA1h~_GSK%Dxh2k2h1l2%Ad2od;?=o!2PY(7aPFL#fh zucQ>Jx>jL4{ccf0CgX5uRAJfD#|!-Mkob_YQgREvpfAj)$hCS4ux7aC9;!})A>~Op z#W^4!In(#^jhE{tcXh;ebxiK+jP2@_5BJF4{@CIEcvki3iZSN(`(MhtxMCu!`sW+} zDj_R)YUz6NLe_e+(iS%L2ZKYKYHGq}{E*-akM;mBC`O5gTQ&%Mz%5B|7kia7cPQ8& z5Pf0b@Nig+J-l1wMaAd|1O|M3IHyJo2{jr+sL~osVUv5vS2e(&RE+QesgGa?0@jH+ z8t@4U4Hvp%fQy?Kv24g4>{m=4Kko*Xp;&qYVvo-Y_q>El3a3e1uk1S)NqhtV@Muaq zA&L;6o9)qUQx@CVf-?oNOy7s=#)BVjjA#1d7GI>`7DeQ=QIUgFhV)mfUaI;=&A%F! zy_Nk(8cTm~&WIYKxg+T^nSbZ8fQvvo7N5h%xp`NBGkIx1MItOl)*#eTx`-zL5hY0u zuB$GS-jb#Dqr;P3xL;~;s-mo((^3!v8P>oWSrc3UBx|WLaRe}RVC%rr4FbhcPUJ)j zff7tL)w7K*&e^0Mv~Vs68}1&r#DnnUKJNDUk+NaqKo|*A zRUPnuJe<+m;%am{Tb(Ypy|r!YS}62}>8ko*``i`41f&jwPZ$aT&kgd!K_%Ve9ttAo z4upbOY67Ou!*{qSar5i)BhZ|q7l!1$l_rG>z`eu^$9*DDBHagcY6rm*n>LDKKvb+! zs3&n-DKyZfVg*_biGnBK<$=^_l%D92=vRzz#fALA+eyp}EnGM$kp+yG!@tBs6d|S! zrqi9LI!_-ybvW8G_O-ZSRV3$gv+dd!tFNY>1aO%K42llc$j#jFt?zg?IvP z6$6?5|B{`mS58I-hXP~7-ydTol4n^8bUu9dN2QzHtQ32iG!#m1jAC)qB8f|_Sgj@%o8tg zH8_z~EC_(V5oQZqQLAv`)xIF;H-!!(7`m3}ARUc5{ws)OL|3|AZX(=SaL>_Q5Nm-o z>+Lz3r&y?~hr6Sf*h-fXz2t(nnRxHrwwl>&PHNrDh?Z@7%gWpq5&2NqmSDnC&Cn3s zI>2B1fY)a9@;W+S;_9W-h#Y^UC-C@Zk$L z-+3RppZ(_dXMY}@`^GP3|LOa)uV0ycfkF0?9%(9ET8&A$5H z&38VSIr~am+sw0X&wLoskeeNUcIL%%idj{bu;pHC=}5Mm{nj%xAN(BK{P4!RAKZBT z5=?C7;~(C5_Dx_zpb<)NO*3t9@q}yUEn5xpxtgI+kKgC1@&-JiL0B}YW)QCDUXVJ~ z8P2ufNDt#kZ@fM8#^sr>KRa{jXBs}UuRV2r{Kc8G@7#F#B20Vc^4D)(x(p*OSe7thX)eP`|mFH~j$KRPLiTcIn(2wE{O29-=*4|Rd8JOpyB$U(KZA0w0?6-(o= zhabY5SXG4-+RqCjP9CMuV87qjBOXRzR0@y=K%-SGs2RJE?TRSkRhs5?iUNgV;8pHv zNl}SepR+=_+9bTA;3CGLUB9LU6Ts z!xd?KB*r_X`2OGo8Je=%BZXg3 z4An*cyJ7isdiL3)&mBFVEtfXOi&`d&I$}i~@uJRndRL_5Du9eB!zkHQP zShC~f;Dx1ftvkM~NA4e#O9uZglY@W%Dif-15hV0EZLo~oA9c!w2mj5``2~$df0J

5~j zlOlBnB9L7mPHDl3JGvkMQzM-ywJZpCFfx!7SPFTk6J&ygL5I#E_S5?riLN0e3Iu9k zkRjkVLv_H?lUGQpZAyyaQld04-j2FDs*P#}#)*QnU=V15=|X)-4V*!r8_R4u0cI(b zQBtfkS<90g$UTU$io=GKI=+}JHvbv}Nr?ujonS2(S3&=Ri8DEq7nF*I8tL;+%1gkE z0WvQIp?~R+aJj0omwM%)`ltpjd4;kW9GyKV2kw~)z?u~s(=yz9+}gEps%G9&(ngS*8wHkC8}1V!#Wt-4r^*pDvPkE zoPDwOY-Y;Ds!`8B0LS1?xv1t135rb<>|LvM#JIFYX}^Rm)x`mQ_CoqSTP?Y>Rp7t4 z45U*b^Pa7j-q~8nVvD#;ecjz=*zGbdy$UA8Wmdt?wvud_+Iw~vCTifUT(*w8V4X*m z4$jIh0GAzpT%VX9j(}CbpSOqgEXNk>J}#o0Skh@hpZ9Lw>M8c zQ)?FhK~{&;vG>BRcHt$Z26m;suAn2`iwBziAs(31r4LDPIePCxE|&|`mM2jOt_M6o z$m8QBSNo;}d-5xnIpAFzMu)^IQ&g3UK^5v@U4oeH{ged2uUGpL3HB+%Z1;=rH{0FnIoWB0Sdove)hN9;$o}Iby%#G;R9e=`Og4LCVyIL?q zTL4CznI^ek|M0b&?|c)-xbW`mxsgRh__@a5F(Beap$gGrGgcegK)@D23)tP(sF)fW_qT9OP2zE^ zvcj1VgVG5f7^c834$nA{4EcRQg+{9}J?I`*m=kWuo4_88d!{{3AakfJ5F^+JqtYF; zhSR0cXbUsIvrUKxySf`RGBAjn5{7fdsM|n1fkBYFy+99|#a&_0hz&8kw4j{~6a=H% ziKH-Ic#9LmFp_4WeiX;d)J^d^t%-P>Dkg0YU?sE<#h{`fp{U|mJZNHmrY`2|$4nf9 zCL7^jIt7v|2>Zsg)BdOYa*i`@WFxr~orfnoz1KRuGVhz@r5G>8`B1#`xZHkls$$K> z1K)XKvSM?rV)I1B)(MOETE*7$m7~kA=KZ?plOlON?wCr;jqW?sd8#Cm^Tvs9Kla*V7k9+V)?Mzp zn)9yfYIkh?zF3)a)Ck6w)6J)vPq&_GjqZ50`K9KUTc@l!a_%EnOXTL^FW^{`4b)cz z(Mav0Ah(-prEX!a4RiabgP0tmj$qPFxiN85J(zf?qnHd(Pg0=vc&JBc$n{Y5m~>Lx zFxgHYz@&rj#>9GvjudEW&{{d~p$YQgX-m$erRbWaXtd((;&;m4D*Mzr`G_<2h*R$9 zj#&;x3Z~2%XVcH5$IRt2SuUH)r;O%EF3kJv;TI1_+NTVb)2>r4IdA2pVHH@Urs^Ai zUHnP$FH0xu*;qY0QP0J#{Iz;+JXb#0Eg$j54tnFZx{ayusd zu->w}cPcmk(#lb4wC=oVEbq1S%g$KtgEF1-q37xfdFO%nLmlI})3j;C8m*IyH;%6u z&zBztrOz!N?2&tVXYGUGqzFPrrOZn{Xr2#4x!7`#}WN zToW~U;1INs+c+r3Mo8F(r}sq3r<>of%PV)q3wO)eyOX-__6f3K%A7XhK4XrQ{q5HGHG zz70C!z4Xp|*|S0SLAM7qERtYYr1lsE+Cgx1`8;EoGZWBU5I{F$bomEu*&9K(1T9mS zw?H!tG_a-x(pDNsTMS5BSxGIDnP)CcYKg2lsY8OCsRh=ej|S3K(oA<3%&a=BU|b3Y zLMJlH1wpm7lg^~vma5jpNz^ZIlVVGB7z%mbY?BE znKdaJfURq}UY;D$UPB0J@So0R2^n20^?KAb*`&(|?2b*xYbWLYCs@<}RwENKLE2Zf zEPMuQb^!dMS{9e7uNsVCbtZKs9Sdlega8eN+)4`B@Sg*+c9yD5LeF3mNa_w?Us#)J ztADZiY=OQu`gs5CE`Zdo&(;mL5|H5wQ4mn)0%8wG!K!C+d3v8heoEW83kF5{xO$rG zA7;R1v*s$GQz4tnhgAc~*_{KWHmANx`a~e*lKKHWDLw^j`wZwh59k0a*@ZW)8t#Hg zQpet!dQ*Zu!ZWGkmYi1smy^IOhs)&(R2*};^u_VQ72yi*xfdn+D0hrspx4YhYn$}h zq*7S>JRQ=BLg<3vL_6z(pf4m_T5Z(Zs`@9K2UY(}TXar9#R-=uSRuBcb!}>P_rx(t zx6VfvAOl$)hdb8b_7TZe$UC>+*`7`I&Mx7yIoKWGs3hpsORJN72p$7~inPGl^%3Wl z+4M4$s)a%}iz`f0C2&i&Y#wZ~y=oCCyv-s|l%hW_gDXl>PsU>HSw|I|D3(;8%)u6v z;wxTIUgq(iWn2bZzE}@vPncJpoCTDxSZA15lf;=T>}pJ|L~`Zs0-0K~bbWp1K_<)C zmF%jGHpe{t-84b$q?&;@cA*BR2InsQE0w04=7i#Z7_y{mkKTh56_o`6&O~unt6s>Z zX}5!pKU=Yp5=yyJ=RO@GmFg!0W>V^8Q|0;>m%Mj?p6-Q1Ne#i4?jzOlmT*h%G2W6z z$4jm-U{vhnbUadNicbMV$-N-vQIu*WnYWio`dF+i^*qR~X4kM)Y&Ba`&2R+Xw8GM^ z13Et3QV#IsO4c)4Ny6qr2}~~m4|HI=)I!R#T1O=w`pjLZsjGoa&YCl;4}Io(^KB1( zYjns(+9tW-Vr?sH6Hk5VNehbVQV4d39Mx;l3GY0u;A6Sn?}LXc#e&`zF4esPG&y(z z(x3hUf&^BWJN-6lG!MmjVAaEZX&4=?(nRoV1P3p0%0f30GrZFCeej+r4#8}~<(kuE zLf6*~EvmzaZXpUUzx#mTB%<{jHiUw`RS&{=W*#5y!JH7bf&XUkq(dDy^azZmjRl^U zj&{VwG5f|3=AM1=#z&VlGsVqQFW>md2RAN#2dp16FO6z1joRCz{@#eL3@&s7AZt|P zjB4)@L(q2!g{t}X9(AxAuV1>J#B? z?WxcPb^vs@!+q77zYVyEKY~cHz#AVp&fx3d>yWRss!ha8SO>nTLtxJU3za(@pkyi``JwW{8B>NBGF7>qWDg0X7C5+BE} zCop;vqIQ@D9;Cb4qG0c>FkQbFElQHug|Md}3gNMtZ!gkT|L*3ZaE=$8PQjVb16EIO z2+cTde>khpALw!W=NDxs;dpNd=o(fpoS6VimQ*q!J0LLg`bYTS`~LT@kH4lC%>MNK znG3JXE6^Zj&z-;d)Q{&uXa$vP#3Rs)wkvMwdH?X-+VvR#yGcY*KKO5%AE=YaWPTS_%yL`Cc$$xxHQ`UG+## zI!THi>=ngh?Ma#G2?W4b#yy1gN%46se-67#YXd84TR`#+;}bfNRH(PtH!Pw6rx-Qz zR1BT~c;<+YU>W5LC}tKmK+fKcbOzeLy!hkH#Yn;IEBFx#Kv2^S24mzdez9N1K_!l#GhdC6^IlIB9ZVd_4I>Nyr`Z^w9Tp>rHTm} zxOzgpiWyIr`bBw9C{*tWn4v$^C*cLMxG^qSs#)cL6>E?0D(Q&7gLZB3FXe(Tw&49; z&RIS|u9#do99ud3+p?M~ruXeXw#!?6@!F&E$tSLrJ$VaWBfBX0hz7qv`xlGGiUz73 zO&9yAPA%6-o%P1bo{W@0AwZ+u!WL(8qwY9U@P=Pr-5OuoCKt9%+jGt~J=Y``JruV; z94Y>SqNzR9lsS9STpBZ%j`hjrQrTQRX|9Wz>*D4Ok(HlY(qHZzON-}K%Nf-pdB3+8 zjFM+sBE@p~`pZYJ7RNR;ea6Mh+0hLXoi5q!m5=c8&R(!3P6WhwASm|1lYR04y28M)%_iad8C?}P)ylt8wH~-FLpEB7`KYr@*r=N_h07Kfyfu~(Bbxcu~ z)2mOd9`R04IaBuRN&B*xec9NGxV<96|Ng#8nW~JehG@h;x@9^&C*qw>%Z%o}TKH1o z=&G^oc-D$|+RDka)v>hI@wBSQ#?P%eU^LqkFQ|TN__vnqXx*qKZYhtfpEg>-2>0Nb z2cx^eK23 zB!r!mpyh&EE~tSV$6WRatTGH!f6O5=a=*wW==4b{H%8@-wqN#+Z~uw!UFuWz{}fE_ zK74KWVX*SXcY7yy55#s4#2fvSjgmYh!KmA)gvo3#_PmYSOT`{Ti`qWeN9cVY)rCW0 z0)^d_3!(OD*4d-f5EkyB_o3y_O%G!7IK7*Jq?75w#Lo<3@+k8-1HI%w!NBC{CmA$x z?tI?jQ{@7HY4onUkZ!2P<>xgYBurJ8YeEwqz{x<@INhQfY&K(l#DSlCYo zG1tWG!&x3-JUDYN<71%i2y+w*>kSPCC~P$BF+gFxVK3$m8+r^t1*<%UKUWauv{Rw4 z?jx-Bvn9`!JijDTaDjT$_Fdce(xSsxa^7G1A|m<-N>Y{#Snb`wB4fZ$+YOaP>h3SjP0>NpMU zI;ayk8IC%Db;ER+h6cm*4%|jepqF8K7tU}uvzLLo-3*6ym_Xrv*nR-n&2(e#5aUIV zL$DSAf(aBJWd;!B5aY+(AQM86K_-kKm_Xs9%wq^L2-K6b&~E#5-PIGn+%nO|%I#h8 zw(be?&{Y0vd2@?=7;V$rfo!1z(|$zzfJVHgZYq*HZ7v?I8)L=}TjemuVB3Fsif(<=5*e}^2@p=Xjc2jJ-K)P-t!?eTc| z38;Aj4DFCC*{;Gt-QOg$0<9TPRG*?@QqSIGN4}($`rTW13SvStj8(+I6&OZlFZWhGZb#|dh~Oq zz&cFd6~YAddJkZOSQvDj82zpn`$5bxa2JIdl39p;e1Tr7c_M*dR~l-!dYaOJQ=SH0 zC3+k9F$Rj{3j(Zb`o^SB`fX!mfJaRh^qq}kusKy$^l4&qp(GDW;5#3@MDB`1UmFGqKt?Q-ZSLm!!F_q@Fi@IgXKh2S8{ zh`yYp%WO$YI0Ih(6OOO}p~~lNN1ZdF<9fqsRaKfUTNOMxgr4Kn(2!!*UqKavrgl^C zgxeqR!Ed9){|kf6YlXUt_#aRv{wG9=HSt?Wkbmd8p&(QXk*3SceEs~)v!G3&*XYdI zQO!qn=4bECjGUf*@w+o0ePia#i#N`UfUl`y6~K!fba)V}0a*h`<^Ge31>TH7pa;=) z4)uZ%l85Fto9Ie)5&(w*#jZIVXdS3(dkE~*s%wDmRqz6;ULc&RV}xK=q4tB1v^RLg zsCnV5o*p6iVGi+!4_}{s^-XP<>mR&$%~6++^b^m?&R^}ubztf<&QV*!U)txU11K$w?Aq-1d= zM9E*K;kK!ssp1&cp^Pk=F}fck79(U9N?M8tz#l}@m^@GcKUY9S28U(TA-;u8P>WE1 zc!(0YCX6$KZl55+bwLEN(hN)yWN>rlq`B;xxh!s87RjG1Umq)9FK^f>@8dwUjg{}0 z4;-32a5Q!R-b%y3s~f08+O0Q0$p?gMn4m`*PlZI~=w6DwU4Pp^s` zpQf^+t0t(TH&%>3c5$0r^+JdJ%*L zaTN-kR6i#D@KbI`+DQqMAQi^sQEDfyWsus1YqFbe#iR|M=%H{oeF!C~hhbsU35)eF zU!@V2f?EVd*`_IDWH?GjIruGJ$vQc^Zh~4rZL$7++MG4NUe4Y#LGAsm#rlN-8vRYx z=ssD#?ZJjp;?q)VV8lwt~c4Ne2)Qr&)h!iva0vgXX>d1aS`u>9fP(;=^&k&KP zF+3L-K-3=^`|wxOK~?V-k*TY$54k95ZAg7Zh;%Kc$} Pd}*y*SS!;7Dw_Wf(Ui>( literal 0 HcmV?d00001 diff --git a/scripts/__pycache__/source_catalog_import.cpython-313.pyc b/scripts/__pycache__/source_catalog_import.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e1ef0ec57238e582d166ae78e9aa3637ad83a25 GIT binary patch literal 35808 zcmcJ&d3YPwc{hr|LVy6c@02Kki$qc4rd^~&*%B9#lDLQiNR%jv0tt|UL;}Hdv=sE;4%yh(l zI88`9oGzpzJVnUZs^iTUIU$o@&s&Oh7mPv{@8DCJFPksr%XlmKFK|K*QrK8{E_`<8 z%j3)WH0I0aEBFn3dMs@Lzmd;i;e~u9pUHeh{3bq&`HJ}}KAZX0@SFK9d=B|h!&<(Y z&t>5yd<~z+eCzmHKA-v4qt*iEbMSS1A@h~;_3#&wANk9WtC)qC^IQ2f%vZs0Ai?fB(h8SAX%>FTeNY&$57De&yD$KKGXi0q=eBwO{@9HxdJ0`s%O# z>e~qczk2!W@BP8IfB7e`|MJDDi4-dIi{E_jnFZ$k)%RZc#drT|A`PMc{n;0OIr-U* zzy9JMA!NS|As-uR$2=b!YlniD;d%l?aK}Bv~iDIwfbs-q=@95LU)_% zsxn5+=iR~cqrSmtO7OgU+s<9l6t90M;PFON&wHMr^NS4O>_5pT^%0D$Fp`-i-B!6Csn9tzeeAUQAu&>9Vl4EV;z z145{F{8}_cDP35DLi6w!bikR^-M3{wv;Rx`XEwZ0eXIIwwX&@$V%sj+w#&AiqH*Vk zNTVA{91|J`HgfgwV8i12wQ+Unczq2*T^zaUsPH=8VAN4ag|5oj6*dI@HBnCYYX?GZ zbF4XlJ8P`n7rob_W`O$uJVC10;Sh@Pikkfxr2(HuD95iN`~}P4OzM_wna}3WSZ8`aP{d~@RnV7XxVm%PnBH6MYXmfuDF=zFd{e&L$Ykl1r^KC*ZdOS!qf~}c1 z@J8NLYZ%k>Tr#L|>348^3N|5&OP8Zv*ST4zv!V16{cgiB zS7i{^BRgx|zrXptE=;>bnl0)huk)z{WX^+V^Q=HRul!N6!RNYyzUlgj+3?i=y? zy;sMD{n52S_po;$gzc$z-)I2aYH)w87VAK!;1)Rl`D;LjnAPBc?)Hc>?Se0=_jU-f&>zuuiws9nW9!-(B}_^XiY^8!H^I&UI_U7 zQ4_qr@u*?c>qj>T{-WkF_f_B6#F$V^RVs~rJl0sPg~Dz`mQaEoIR8oOEpOWUd~?L+ zkZg{*?4|628P}Xive!-?TC&=osk~9SnAP=`NzUq$tz8kTQ?fc`>+#9fcLC?kGo~*e zp1C6DZk!VU_mf8z;7*(@+E&pN?!*5H4^3*aspPfk^Xn2ORRc0!PeA4=00?T^9&adS zAfj$)DgbBzi;cQp+W>i^8t>&2$}mwGy&71fiLoc%qSZ_0(`#T0A*CzI)%kRpSnY`e zLQ~6NeLp-2ke)-3p0M(@Apt}KhGX!HjgrAH$KY4b;1|cGKmb;y7pn0RXlqN{s)RZO zL^m)H5WJ%}V=sFL=y*;7t(BCw5Do*@l7_$BIoJFA&}`Y)MrH^9^uk~6R6z4{Yf89q z&V-+e1RBA1c6Q(|>nTAX>nRppVox#Lfh=CP7oo9UVX6{)g>}f)hV`|zUAJ|&l}<&C zKt4jNdL>XHG#^1aL0Frud*9yNI9oB@c+XY>-$x(PfPXG42Wv^#0k1#+6E(Q~*P_JX#^e(*$*&YB~ zC$OyeN^$Ac_2W0V3|hM7YfMYy#m1X#Rk;jmDcFklDU;rhGL#3$SDNaNzvElGniq$R zp-X5$lJL41!b{K$1a)K#xF}()K!Dw&K2McdnerB-WgDN+3|}39KCULrr@5v^!-#_R`o-x&c1GD=!NsQ&d*MKEg;)!MPu#ijaCDTMKpjmTAriD z!A|B+!6%bp4e_lre%1qKUADde!Z_7x=}>hcFv8FDBj8 zbz9#h(8}}u3IH8W*-~HMSYIFIPE}c=dQT{7@qj!D`Nq7_G$M=0hl6trr=-u1So$3f z_;HL*godK2zF=UO<}w8Q&O1Kp9`deQ{TWCzKt&CV`9?>5LE$(m--W;65jdD-vwb>M zHW$rS+_$AYv-if{h^<1hRlLN{<$e9syz#|>@ASUb|7!nh1FsIqw$_NPTe5Y_wqp_7 zNy&CnJT)i|4a>F>(Kw<^Csv&H&~TuW+=qXQ!($mrsGVefqszE#C2FfYzP7OvZE6}f zU^HPx+tl^+i722g0qN6n;5@z(JCEb5m(F9`r<|7Y7HYk~d91DaJ`G)S=?O*Vb=}s4 zSiK4ieLMO!aB-Q`ez>enoZ44U7Em=^KV0R(axl@kxSSMlrMhehc~B#`5;|83pTd@o z+5#NSu~JgBy3DKd^~cj$)O5U={j=1j1AtSz!X@fyNz{V_v=^vh2Rlan?lJGLA4fgE z*BJ`nNOp$a)<-kzw{F|MW9P2jdm7w>LmuyNmp~YYdY*Das}9rX8o&3-05vQLVn8hm z=E7lzY}MJHeCBScxV`n>=GLEl<|E+%BEq?`UDq)b@P}w4Cel(^hbZAFIT%h|ReIDw zB$X9AEs_vb6-{N7ROs5cH)^C&h^8&WQldGreU?ybFf|6K*l1=f&=>NKDIw{xY6oJ3 ztTl*DdSLKc$Qz8>#sVJVQFsQh+(Ft2p#k9dULl&T@Kgqd&Vw~FFcg@;Mv)zy7z_zs z@4&Ec)Ef&+Q^Qz!Ro3`vB5XsqqNdTn6|cZ>3*jJjww#<+a@LRoHk+=>E|ikL3{EsX zHUKfwQgBnuAUQ+ijF5Al93MGeay;Y^e;`O31szH1qZ~~Zb^3$=;t7I-_uv4c=ZdNbb z%3d0hs?NUk8M*3gWaC+B zjeQc-23sBUh@ zQc2}pgIux$nadY0e79~Xzci9xHS7FNaG~+rPc4J{eF(sp7KCvHKnXR0BIM& z{=$9O2={F2S?gKbS-Wx&(%Bfu%F!Ly2le&9jX_vi5K_kLDaNHA(0Ah4A{5kb03KvO zN~4w%Ndhf(I-~(LL1}_|zi})Tc$dk=p`=3!?!+6YtRd_HE>kiZCyqe@_W=*U1;U+j zI8)hP0}9TWs)kYcr1PrQN$0XSy5Z6)`yholu~USpFurGXeP_AAuYX$XRy*~U70T7 zwBd-0%x58v(4L90qF!zuBMCqwm}r`)#D(em$Sd=kfV`L4(n6Np|j_XN9c4V95ma3}*Gi zs?Qknv9e?SSh>Txs-mu_iFV|mFo4Dj=ir3buR6BId_nLfMnLt^KK^TtlA`@Owvk0s zh6BWD4r|1r;Jq{f?sd=s{L7(4I)p$VwCu1WUIEq*_u*flxc$cX+I5=P8o`=%n$R!L#r3<@m`ylAzgaxrEmyUOGc1k+XPi5F7+u9jW2o>`c;_jo;w$8gAkspw{&PBSsqQ@(49e(k` zzZ4Wr9e$^1{j@3K*eW@;zFILq@$D`59Q$wO-!I=d8~*xZQ${hZRI-%)Ea&d2NatCp z^Q_!?PVR7{_6%uzJ2flgM#hYB#&yfKXsMVF&pPKS<|b}$kv8nQTPz;r*-;xV(k2RmMv^LR<_f^u7{@_#}h4WfIl`{;smip#5J~S zxNN!tWBwiE{q-70m@|CONwa>hLpK0}C3` ztf)I^(qFv?wwa+jm2 zLvDY-?*rppc#4uwlJi+|Xh{ZXQ=*m1p2zVh=x)Y}krKX+bX}C;-*E)|-^7E-oG!Rg zH`#i}B$u`Qz_ehTGrhQXF7$T!_V1f!c8O(eKX+`D(<=WgWhQ63>`TKhrbua(lkJRk z^G;^ol#$k;U9y!(whglrFF*O>lk+Y2DjSy4vZkx0wDq%l)8T+TfprXP6Utjo%OXwg}0j5E0P^VergGwxfK*)>vO&D>SF zaQ7Qw@$jhp*qGQbc0VoiY1d@?O!jokZ}q7rC-*={?)$kqQ|eUS)TJARU+5wpG(m-dfIBct;EF?sLUY}?#^ zsiF}mdyAf7?uYe9^zYNJqlCwI>fOq(LA~e?A>cB1kOBf+v;Ka1_A?j0bYbz#km&JA z{--_whRq$&|A$Ujd{7_zA-Yb!vibJrZ&oiB_xySFi`!?nE|iK*-BM%s*S61W{p$AH z)&E%BgYqcjhb$xN=mFOeuXB12y~g#|sK$)2ezN2Kx{BG8^18aoHqo|LGL|gavL-t} z`e3WB{1Eq1G_CTZ4>#-99OFKE$chSX!o>ekMq7&U&+KhBD7Y z6mlyp2>IL06u8S)ETlyW+0bG{+tSZwoXtF&rLa}ws00|Sw1=&D;%t_c2YvnTB%jUE z9pc*mDTW4?^B9U6)ZjP95pI_uX}zlu1Hb80^Jokg^&t1BxJ)!^^tjDHUQ*Ow9_%g& zD78jFfSHuMJsMl&_ZtTw#OC7`4IzePiXp^TU6!Q1HmfaH+euQsPROzzkzQlMEK3jU zr|&8dZ19g`2)4y<9?Jw_kXp};rN`OPDw;HwMC)U$%xNJ##ny&Vgw=vWb{p4-k9G~I0KO#q_n!}&g_m7N_ z%EGv$IWzZSFJz>hh7|HyYFf-^YBIiSLhlau-G4-l*=i}y44|qSVdt{*8LZ7zfAZet zs5#N2L>680)VXTv1lU5^jz^WAr>58576#&)FcIYU50trwkV0c4Drv|~Ag7FcexHvo zs9m*o99X%X|Du)>Be4vAKoSV3!Y<(jloL+%0NaFY>|#wc8@#Du-v|z2&j6i#&>M(m zjFKEKI1a}10I|Koc`E;`1^JxF$0#Irf<|q=;DFCR5E6*@Wd<|aJA%`41FBiJnPC|# z=!bx5S170$?F1Z)uy(u{)#S7gMdbm-<@HEE>h)HzaIZNtGA))mz<4u6Ml6+BAV%5C?ZtG8%{!BKmgh02Qw;OATCN z<R+=NOy2rd%jSxo4)Y{IcKD3~=#CCYUgW(~LwWU2ce6X#)1cX2UZG1d?5 zeIq)OA`8G*b&NMGJOnC)H`<3m^VPW zYp&?s9ESFF%H>^RN!R_gwR6E&uHL>X?e^W>B6XaTcl+czpSKNcm$6y^*pusjN*dYmcny zxREhsUCJ(*xgce4no7N&RXnrfg*~_Sh!uO}H4SoBU7r>YJpyWq-M|*EU;7m<%a3)f7~F|c7skp zDO0ISxoc;7r<Qhn)`pA8luFEW(Nd+^-vXHwlc=w=`(JiL;OqrGn zOJB&im2t1I>PGui^HNFa3w^ixX2&2em_ILfQg#G!#*{b=Jg_bwlUvIy2T;6x|tv>Pi$)BDP zLs!v(Clo4W0xz+DAksf1^@H{aEcTNC4m@!!wCaYK5pov*HQgRdKI5C{6?os>o-6JplI_0 zt1dO;nY0^e&j#i@?j9D;4oVv@$@xOW5`4c(xArjiV6)Di4f5#2QW|0gcR$#>wtcPT zhih`l-JSw>2n3Gy5aI)phWpTo(64~U=VB_6AU0314(xi=?1I;wBkh-bO1R2HA*va`3jKTk~;FR2&fxbi`44C>S#1DzLR zhzIzhLM|nxY-R zs_&ds~s$39OLm10) zzA9lT3STu_K}=XDW4ONXbEN+Q<=KJ#0ys%}{&ZMMFP-do>Dqkm*Pnv^Q%=G2h8NPm znm+r4T)17%*)iGmM(9V)f7kVcE|Kq(TTaT4oti)SuB~Y1?A!sVq&8BrO)A+oze_G@ zTxga{9{bS+@mN4^8y6oRzi-Q#?wA>YNM+A_%R=@YL9!hdtw*3vmsdF5I@!IHQ8MG6 z8I&_hrwmKB^k;V8*gd^Rj!8=946m4Po9FG3O-<6KrXL*^yZv%YK-?c#O3Pf#shTT) zrSf*={KkcBdGkIwr%6g{0-HA{|GC{ach5A*IaQII`Z<2edUyMGpL*ROI!`S2`NhCB zxi2g^!}Iw#yPxE`>6E7z&h8$)lMsO?IhTIPl1Ad4XZvP+b18G7+qU_(gn~9|kKiSajDdM7+cteOEH=92y5nNi z@x<1e=Z&}R^T!w3UOjVX*WL1esC=tZ?Ctw$op|bOWMEht7)Al|0LDRx1TIT~%c$fk zusl>U4%RSU`7L@NE2*t|aJn;#!0EQHnJcDQjCmlz7V%pu2t=(m2tR&2BXD$#QbFwtVc7hT^MQl9NQBsyVVy<+|S4L336|4u08u+>X zj07HBM$%SlwX3c0={wger9(X@gEwvEYA`2itUuG0siIt-KS|{AB_JNj$0HqRV+?N9 z(vl%2YMejI73aISfZ}BEnPl|n&pw>)&v7Jh7ZYmE2HKnhH4_>p_B>*4vR1c6t+$u- z>{dK#ocjJ<+wI1>bY<1Seg;ZTi6aN3rN30Igikm!p*f*^HOV!ZPj^!~hr6fodimcm3SR-wQ;e}0lRUO+k9pCXuaC5J=}QCeAI z9gz)Zk+U!XnhTRcbD=sz>f3Mq)I3FMTJ`Nub6Mm2khUh9F_&o??9(beijab2J5k@a zc4(|w>?!Az_0;)U<^8EMTO5bZm17Ynz0{Vtw_z4P4c>pJ5_^eDyW&xy0TKq2zsOaf z*6P`UzJJOd3njcYu3REZu-C42WyVy+)OVgM-<6AhMSMv^!p@zrpB3=ypdz-ORK(i9 z96JNP#XDk%dnqe9ja+}RtJq~>BmwqsNU_pZbc5PZtj@3FcWAk*SUqcSN<{gxcwMW> zCbgHm(jtW*OKjbmPai4Q8q^pcBWq%fF@J2_SW6x;cAZ~QAncH#vnK{?Rq9A!Hp=^U z@)e1sMNFYBMN?=?(|j3_zijCGID34&#uK>W9}U2EdI(H*@GNmS>3Vv8_20hm%RjmG zi-o6u^}X-?V&Plwz4>|J`-n(NZ4hHIs`q)qCgMIelAQqY3`uW7L6{C2dfnlETUz!Mj;cuA`M!_KslJS8s8eZGe1C#8*QQwe*ksu*o2!u$C0E(KJ zlJPP=KM}p#iCuFZo&-KW0DHOBSknDPI|o;-OO3@7lMH}jVUmHEeWcPm9^6~jBUnf> zO=d>esG8&{P_G(<^*0@oF&aX#I4e^KiShAOs#KbI;AP}tqI;%QUV8HFV|{lnzf?Xu z{PKktFU%Lq8}~&v9=O9r$_|QU2N4-FKdaxyL50Q)rFJ*wBl$obUp`Pk4t}43QWsrp zSa9gEVTqaDMXd_c+6jvi8k60qOCk1tMIBl>62kvL=p4n+;mNw1*Ygwe!tk@c*_4;9 zFIwmJ$&Njd{01?<;V0%;A1l$r)Nz=^k2>8W!KgXp9&wV2ya65|59J6yC1(N-3_!i) z_mDFTXW57t5s*Wmvk#tSrsX*1S3WDX{qURVNbevWKUiH+*BBKJ8$C-;&p>Z?x08qJ zkKg0HiUxzm_X!D#O~U_*^wH8s142xv=~W~P0`miH`Np0%_rJdXZpGb8Z&k{Bdqw_) z*mqLmPf3maifCdQ$-+jALZg7)RmV6AQT_P1LXhVvtbf3WH>z^3IJp$MV zKA@P(|FLo5~; z_9`n(_%;1XVVebVv3ID%e$RK!mwI3|8N&h_c}#hC zFZAE)pUYV+shRV>GJ1RTjXifdpyMYJH-8i@IH*5Fbn?D-B4O(~^zR#VYij=`yZE{4 zo7FSd=CUK%)l(@;^}AlPzG_|AcUQkye?-n4db|F}?8MK4U~T*~ES?#Zx(DT~vgy2M zgEKq79A3;Slj@I5nZ?YZ`x$xPzar;%|KQ5P@n^5i=D%G0V)5Kzxpeovf<52ABIS3B z$9N)>^PO^rYbxd4jJ*5VYo2e3B|Hi?Md?0yZBu0JL22zlIlFnP_1&Dpf6i>UpTGVk zfC2u$`J5EF#rzOW}IseSG>D_hZ z)2%mm&t*@wp`zyxi0d^rSdAB$aq(jQ6v0rM#ZE%yM3jc(Py0IvvUKi03>c zbb>D1l<}SX5(r}oN}dbf49{+n3u+?;JLb#YF4!?`xL;H@yZz@^y;RCc5&~Kd)tnFXwnt# z#Do>B1J9wEYvb+=|J{Y}gOdc?y3=yonMm7^=pKq3^hgIiuvJfa!FtO&y9XUCtcw)x zoZlwr?uz8@lXCYh=I$3Sj856!&B#d*4$U2!KXki$0m7lXZ6fb{t3y0~T0C=BIu2D` zkLVo(^gIse!Rs`v)$j`HKSQuo{1mZc(jGUE3Y$}^?O$AX`bO7OJBDE9{LQCkgHplf zxt*}LnL50bQ8Zg1XKY%^EMS(RhZi#|=S;8IZ`;4qx8S>L6?uYGi6>wc#{}`or=*~= zirT2A5Z6ZhmtBXb!m4)tQhv$I$j!j)kd$9Fw@J$1HoslU-!(rW(w*LkV;yyy#vO7Q1@xEc)n+(G zf~?|RCamHO4{f^jXSnyb=yGZv?ozlwxp%xhe(lBv3=bO(4?+Tr?$8Z>2<|5^>nj0UvY zO$mmfII%mQ1g)jIkk9OE*@X=)W20yrwy)ruBd-1{G(i!N@`csg>brM;V&IuloQ}*?9XA^I?lW^>C`+>d+-{lu}72 z4}ThzeG>0So!?%FS64ot(!=DVs&VJ|bS4l5eg)B1vUR2jJXenYBi6De!sN49$=DZM zmVUaBhdqpz#>#_s-j2l4Vj8W4U$Q5WbLC3u1!=JIaV4(bOZzM#6mKr-RC5;S{f=OV<9lnPWnJND?T2^7Qd>L&?f+^~dOEIhw#8mnOXLdqWu; zwxhkIw7j2wum|@_#d#98Uht7tKI-nPjz~RJD%3gCjPOi~e&|0Tv+xUYXqXjp4$}AK zgd3e8jOHg|CV}S1PX0;vrwFI0NHQ}KPQzJ6jYcaq{9VyNWxR|ex_jTJ+_`WRlka0c z*?Gq;)*ZOF`M^(hDkk3o|I4RJEAmbv1CboxIj{$I)YH*x^S3e^XhGwW6~ z7c_5=>)_Fj=7H8eXKNSV(cQ%>G~ppid<+f*(+~lJb%Ki{$jpY(N5Vf)^ffqD=?Z!H zF(pkQV|FrHV~U_kPl)Cd7=#eDC}eYRv^GV-ZV^Vj9^npUx2QM&1=I0!ImM_VmfNC6GO}GlP+=4N}&IS>L?=Ue@+06R4|c({rgeQ)f0P z_n6Ecl+^TE@y9uWN(?vmeZ>v>D#3AZF2gKNP44` z-YBQpanT?U+EmHB8x#mc5y;NMkU|48- z-8!ES^5{>?|JlA1Qteyj7cFx+Ur&oTwn>g{iflWQvscR58_78!W0SAu;FBk|pDr^c(3jL*FsXx4hQ=YWsro+g);fvsl;iGag1p zkz*jkKLhY%yhiuncMYB)9t2jJSK14E3VnMa8kA%5{-vG`oN~E2y#K3HM*as z**c3;mT|u*7#k_~;S}7n1~SP@dQfPLP+m_+ga=4L0(zP}&njIt4TS?L9T#rI4=rf; zBri$ROiMV{ureL&cBj>1sWAhy9>#7*w>Bv@$4&!@e3(8N4pimgH4!ni_zlT4o7CLu zp@oA~&s(SfK9!9_^|En@-C3ZfR{QJ5mW6y)%9m8zUPEuR9Lq;YJA@N&gHcOVYjY&p z;p_P{s!fek5X}|6*N{t$ds4KD*U(9|U5vEBwws`Pr{^qe?5$Wx%D4bTh3b*=+`DYghDFLoXi*XPQ=6LjN?#X9)EM%w%#H z)ilQ9qDi_DH(C3rq!n$`M4Vo3gzgv+{sw~w#`PO`34c!x_-_ikG@Py)!X(;|4j?+5 z@0q|gZA0MWYBvjz8b31-1|L31MNE)0L{0-`iQh^SGfZb0JQPnDS2%xpWe$~zI*iC= zI|=rWPejMCYb3jh=Tp(KGImuNy7Xu<$$O|eSPq9UG3@ER&_4Qrv~argq7W4O$L{rx z#eT=`;?a5)P(9=-Ogz5wP2$CE;XSWvtU!5Q!oZ z;T#eu=GvHL>>@EMU?X8fvCAoAcK3A@S%TsFB|O1eILcL-)`7QAp2bPWY;b3tBHaXg zbj~xsPpWFQ0`ogv!VBul_y?#0-e0h zb*gI+J*O23#Ht%F{Gu1oyA!aeM#@2bm;z70A^^XR;RU<~7U7w&2w$`gNGH$A)`5t5 zKr#=g_bV}!Z3HMAGPv++H7a@}>*-l~tOD4wU?zKt7wfLCR>w19^0C8lAFcxcJPJ!9NTVlEnhb|h zV{t=dQZ+O+UOl~-GR7wwN~{5A=EIws&@wFxK(kQG^{dD()6j>_%ajq!jOAS_|r`j%n_uQW*nuH-oSR zv-m%dJ?Msm<(!uF%;_7apFTU;9x2@^mF|QHYNln$Z2QBfKKoR}Tqv0fXN@o0U$noR z{$l$4`N*zLX%}uAiR|K~UHsxHNClphPhA(C*YAVUY5Y>-wC6ehO~0JKQMOfz#!3a8 zP^I>0;8cMi((dfnV&jIR(CyS~H*p*2ChnmccB8G}KrY;BTZ&hLG&!0JQXRLA;35vc zAJ@L%dV0Emy9PxAv~@uuoY7b10&Y30e71SESIpWZ**1yBO{?2TM;HTnVhvCmAA$IA zFkl5TP7iUscy^mO6Ng8+%8&JxkkktEfLI;zZ`H&Ao&i``O`$gIP_ zo0Nfv(rrotorAACusrM{0u|yDQX4+i`bWD{jM>s8r8$Ugtqv$y^aLvGNp4TXr}620 zMy(#@Wj5%O(J;%(Gx1r;D3Chb+0~P9(`4MXDP7kCOnX_Jj?X37Q2U=WMc%3u`8}M= z#L^ZlD@_w6B`HVSosz`G=I!@v1$u%RFQnF~W#V_yir>ZB?={QXy1YGWm&GM(PYG%> zvU=7nOS`V49b0W*f@RS1e2!)LmdBNTLR{IhxaH$tzASEeTPl{tEw69GC&YdF zIBo3FO2fucNn2g8MzQx;hnW(dBZuS=Vc3w=Fx;(8xQX8&YM!OIKOv_MyNFN^2RGe% zM+BIdc%o_Q0W?6L2to3@NJ-YMZX5&Hm{`UQ2n&DF+R??g_BtIMUC!><*EZadfp3&B6P)v2_h{&RHBy0= zp^G-FRm)y?sLFc$z)@E#@7N^#G4+IqKc*TJE~>7qr2o!dS8H|U6W1#n8|eTWtgZ}y zp!vJHPi(5HW^R?WyUWqs-PHybtdp8tQt-AV%H_9BRA?E^o~{&jj3E0Rh@WuR5fwR zW8s0ppjWue1}b56zCZ&*#ES4VobX>H?R0FCKCwM)Sh~8M?Hyf*(BBgs&UVMpgupIc zRh#ZOfC*u}`sY*m$;l^WTc4}=1MuOVnd;ijogmF{NT`+K+@8vQA~L>)&Q2T%3AeF z@J^!@gLM@!fK@3VjCX8s*_f)qG`7n2uxv!NHIQ&oZ)^w>2a&cJZKSFbMh5xSK}j+g z)oKUheb<6mpR1S8NtHDQ)oQP5W6iGxpj3Ccjvno3+eF}F$SyjalkL9?>;ziOjUhtSd7+s*Ti!`&TSvGq)4SdX^0IhDYKdB!dLB)Y?P&Bh=W zvux993jc-;AZ;2-`(k?4$jB~r-nYN9l5Huh2<@6U97sME-;^}h@*_fRdje3e*c_9c zi?OW_m$&#wz?t)gf{#22*ah*4wO|`d{24zHHINMdtv0XC-3N}g^3ARBea+E)fNynR zo9%MoO!d?zl(la^WR>_t4Is+lOqRgW+SQ^JjngZ(1jYp)u4nnQ?cKqWsUVF#QQ7L1 z%t`0ZT>xx^O^{kcLJJcLawMclIQaH8#k!LamXNEvx23h$aqy(luG)k|jus4AwOS|p zGvzc@U|j9b$}`TPq+SivG1lAw%}J2W48gWCLkifgfbA@+ zGJab1P2tYvpt&4SRzPKJ>d1N-`;+zotvEz(3ZJg}WK>3J>L#_TiDz{jf5EO8``lX$ z8HrUKZmvcYLzq4VEBGXg;L7qICu_<|3@!oQ`|#?>wJY&vTOdbxW8N4 z-6NIs$oa>_UYE$b#Qft@=JClR_%PG+DKppSoYMNO^IN3#O>*J>h2R~%SlBFOH&1rn zw`V?c_)CYUFGB^PDN~k zl8rt^A&y*>ZKIQIzt!)w8$Z;+d7#s0ramCwFH*C>3r=;2meL0%{Qj5HGWPyO`Ifo1 z`6j9QfLsn6jbl~%bhq$uMb2W159?{XD#x?F$n~~vF6u`${}`^KM|4R4IBC$1=?~CV zFr)f|hFEE)g9ZolJ}lIgR?U^j4%{rf=RpyTM4_&si(_6yc5^2<@dSAC0qMlJ+;vIp z6k@6LL9SBquGMX-oAo^?q0DR3>l>Nx;U3-IX6{at+;~iE=zY*Yk$X%$XJOul2RR+) zyhOIF7tM|b%^XD|4D-(X6py)YlFJ&z(nc-bxVJhMf0)zN?3_O%S2v4WS{@u>`44j` zt=vIQ`StKP2c_*J+}$Sm;7ReosRt)mn&TXJg;~mvhkcwbyLe`coV8KRtbA~iCFtV} zo5~gc!?T>OqIwROU^R+mdmjw2__Lg0pGEOM#5eL1I^kvU9*!$r`+)f$4s*Ja4YOzD zwcEutJ06U%c!cf5jXLZHB{;(I97G@Dsf!XnDjyybJN#NoeSqU0D%o^9O-6-zhc&J} z#Am^8e3Ss5uMnFKTY>V#KX#(9+*Ag(8w@e!yM-0_QX$2GWnk?3c)4 zN4uL39Ccua@q63?eYe6-pR&QVfdlkeIR8knDd6904O^PKJ8>w&XYK7idi3Cd<|Eh{ zIkq1`D#zqRyU-w9X)!)NdMzfFi*b`}+D3M1RCo-jC}nUL+Cx0n#}^OqqVtq|z$e*! z_rgz!7cT>j?M?z}6LZ_JG!lbWb2hcXK^_cFa`vmre^-acbBIZ(69-utPt62aO-UxE zD3fYg@u+1cl_(^=hfvL-a^uuG9xZJ^&CqdJ`6!=jY#?hYQ&+f1l|47}$2F>%NfN@T z6QSXnJvBk!2vZRxb1U{?PE0O7J?Xtd+H^R+DljL)26D>D*+@<^DvcTg_{0_1K_o*E zst}wgq=YfhdGF9gocFk~L})>5Bc<>lEs6e;O9ecyjlMeiZNDND5S`~D&hwJzFcb#YO664Uq@?j%W@f*dmw7{iMMYYjIVW-iW`O*M!KpTJBU8X76KOzxu{i9GK3 zJ(Tz|AmWrK;YP`X1r=A#_v_<4?YOoDal+{XR&ljVbvAMBgb9kX&X}=oYh2@|uYz*$ zDJ-8R2GG-$62zBhPb`gjRT?u?|Dmk8@`)?j_qFzo?8I`XM{Nsu`Rt#{Az%Hw^SB~v ze?;kNN_mNTCUmzIYn}FerG!!8_|)ZbYR&N$xeOJ$QheS_gL=%wu->6tuXEsE*RdwU zdfhPh=oFo}Z$?Q|W$g;@UHSc3_0GS6i-QBI)ovXw|997^F*U>^!G~o>!y8v+4h#lD zHS|$g_h?PV z-KA(&5Pe+I6dauxfpnADqQcu$iVR2l`a&FU&QTOq6&vbUZs90uq@yQjfdj-jJ6|kt z%GoZ4?uuCrvb_;VZAOlm+aPB&PQntf?4^deGveN3QVsCu0l8ekdq33c3jF#TRw$6B z70wjjS}#^LN=1zejf-h*cegAacZ-9g(%>agxGoJ|$2Rs9zG(z`-3djXp$jJ&3<{s= zy3w_`re@Bwutr{UWYK=~w+5CXlGY}rwUHf~Uk^=Mll>37KHK%|V`#_;aVJ^giO0{# z?KDkm-x#EyB6J58_LGgPnXEVj+;$*$j;Sq1;0wB%z{bT6&# z<6s3w@cY@>&1+5HuUOk`Gygzmf&Yg#Yjct9hk0i5ueCOBS^kY(3o@}Sb_yQ4-<;=; zaWIL-FXict$vD;D^e)r&W%By?X3vjW((KK(9+{)e|L_$3AmiIiLGR#T!?@TN`QYks5pC#@BoYCNyqBM${eLvOYk9)I`d#r>*JZ9!a1jgT^juD?J{ z5m$rt<6}**MWLHHfXKpHEk#v!RWCnu;qt?HPQ){%Ru$U)X55KEY6dHwequAQ2M2m= z{M9_TNo9|6V3VM_w?s`Df5$EI7$v|ZyURx#F_g}y;a-z;N{c(FwBz2TSMzOu31LSq{*OC9BO9iIa~6RyLb2S2kXg1-R% zn9!OpfGYnRn%D@D*whiikqaLutaYv$}@8f_aOq@yO*yQoz8$6>y2U)G0_l|n1 z1k&D9!;}~=gT})DO*aY!~w@KKKy254;P$}%0QO===cP)*IoS;eg0&7O>+h4&ZCgcp8 z+4U#*^)$t%hv>SCm>O)bDV*j|0*J{UY)Y=aN&1lq>YFj!D5Ck2DdU15qISgeQe&T% z=7P(1ZBj>#7L4K2_9Qk@1L$SkCaR(TZT{3<_rlS?se?yRy{*cLTEL^n`GA8g{eUpS zaIqfxi4$%B2=36p0JOS15Ru0s+0~2?v*UM=p*-x+h*}|UpAc|qJU(<~<*x<7O>DjD z3(@_T9-LN+F%v02DJU8-8QIs2*;GVL;Ev;RdD

M|-NQ?6P!fg^^X%f%MUo(LmrL z%&^jc#Kqx@*+i+#ON~n)oTeJ?puiw$(h=&WJc^}}k%mkNAx#hdfzKUWFBbXl8&jt$XFM-lc;NyrV(>}UJ&VQ$*1G36% zGQ5M3ADY~O0VpNX>8E`26iOxppX>)%5v6K`P^S4V0|=CO&Zjdnb(VOLULgO+*j)w3 zla4N2=314~6}w_)0~Mh;(1T7^x}OqVp(@;3jYFzW)q~EZcvdO-5c(#i&_X}9($%f~ zWU+TDEh%X~u1m=qum?C_v6}Y@;rIH-PVszqBlst?JUWY=lUN37MTWwnF zqDD3>iYll=oe?jG`OuY8su^6aSO>`$6KN=Q@1nFT;<+|#f~@&hX*~4Gs%BX?3Fi?M zwF1?hU^wxB0Nhe1h~#vW(?w1$Ik(6GcNDkNkTM0X%TbyKjxBZ-oONJS3A7C>60z zMg{Q>!AFowDnJ7RinczC6i^hLsAV6HE$A-o7e0e@_^%7n6g@C-9QVH7$Ql2`s^iN4 zO=o>am-mh?{T-d{9bLvh=?dS`W&N|R{9kO>CN0stVNsX+FoolGJ+#32jm5-eJ>pDs0O`+zV&?&68*iZkV;`QH3>?%XwI+=Yre^)(mcddr+Y>+NVy*hJv3PEnjSV MV9*&0*>e4V0U+6-FaQ7m literal 0 HcmV?d00001 diff --git a/scripts/__pycache__/test_import_display_metadata.cpython-313-pytest-8.3.4.pyc b/scripts/__pycache__/test_import_display_metadata.cpython-313-pytest-8.3.4.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29ca051e0ed241935c1fbb4a9fccc3c0d3f0db99 GIT binary patch literal 3842 zcmbtXUu+yl8K3=gcYF4o?IxGR=alwHY3xhsok@>21@H@mk! z+i^5f{jqgz<%+uy`DIDuLi;%tx zA@J>J_(Zt_=x9BgLlvna<>Plmc{nfgrU}oq%7i)>8E^7xW!*IhyI_@~uHq}z3+rEp z%vE$Q*awxdL=`M|AgYRW(im33#&Ha60>`OVN#r%29x_}VM%70R=CV8DSW3}&V%P-y z#7@@#3g#*r-)O@Uma$Tlu!>^^_;yP;UPRr}Q5hT|+0qyqsh>eu6WTEeC&VmR3v4p- z&!PY0-xT;aZ}AsZ+Y(NVi!}#nOgdU$Z=+WQ<(!cN z6ijzy!bc@tC)3ppw0J1RvPnJz;lw60-aP*lLO_W!uI5^&{GiFe? z|AMDmQDN9|pS8KdTzXyRDl@D}t_pB4F-tDT2a8iq)hNzR8Puto=e^>2+qq~N<%;om z#hEs!t=lD|$V$|#xvc2Ir4zGSbEr$oCaYQc45`)uQK-%E?*tB2lQGk-5I7pc7K&D@E5c~oI3t!BaAp|L2ikHMQ*7W0 zR&WAqI0?LIDk!)ar*OI;7v*lrmK!WPyDLJjt?bN?@eK})DG}OnOVsP0tWt1mz^Asr zChpt+fxv$&Yd82NP9jWr*LX6@H`cXfmYsU-{`_9R-OMl*15-gHE`4J*6OpFQPF)a_ zri4oy7_Sw~>N_`XUt9d>#*Nj>KfV3>n_Mdan=620PxyWO_TuW?AHpVJv}~5SCh$BA zc;@MvjVCJ^{u(pcVAuo`5UXhd@nE4N5q6*19L|RPC$A3fXIR7BPyPU zSuIkUd@P_h-2=5;0pifT!f78EV7^csu2ps0oCI?6oyhsc#u5j~{to66TFHFH@9OhMzw6_({?sc=xgP=T za=n6lnUnIIwzB8Jg_8fRXZ&-M{?xQTesSsHm!a*X+%H_PToyF`fnyZ0{i{jujSp5HUHlp5YvT#Ly}dQMtHvMHUB){)#cBF(95agT^^U3)jTIC)czYPd!ze zAseMEsm%!76hbHfY6{f2NFiS5$&D)UCn2nBg3h7vQLcHm>52m1caOk1)XxxVqu|)5 zLqr^Gs#x8&eRAzYc$QDnQ{W?F2qsQqO_3z&Lxg^UUTysl^?VwCNb0)s%m~$QHX&#iuDtisJ`JINib32Gwa)r9bgm+duyh(x@hhK} zPRUe0rBEf0+R(!YD;)@{=g|nnXlzH7Xoe=vCvl?p974u!guu6@(Gx+WqqWIADobUl zkh~+t!v&c)P572wBGkRac#~f(8JwgAw6^(DKVF}Av(Iu?n zL^u5QNI0pZ9_gqIj?f%w42{(8AexEnn1oZ}6s(0d9sB3e|M71M{hPP=i>Zc&GvgxX zu#HJaYx&l9l~Ex(%v024zU{Hk+NV6P%6fI(eA%c}ZL`~QrVZOF=?0gNANSiKj$WnK zG~ByBY1t;z|MlzNuFk*NT`f&sSGeL+n>R7nr$v)kC3r`+TQqD^HM}XFuspNEsJIZG z@GQ?ZeF3|A0G=*c)GT@~o#E&+Ek3BAQgLy-*pkmmg?aS4%vEOE zlUxKIPZ)LD^QRgdW&ymVq! zsxI|N$zoO8m?4!KAiAqF{9B)bVRbmXC@=AW6zN2amCFzM#UtK>k-Hq z4vMZ|XbZf$zht;1JR^Gp%w_cV*7o`1fqp6&C&3HfTSMrgG${WYp{^kr2A#vg9ylZH zi2S_x8JE8+e$L7-%IgU2BJ!NNqGdkR+LpAo`P{s5ZU4glrS`(&iDm6r(D4hrn0x>ETbd5h8aN};3^_9l=mQ10izyCp1uHm(HJk?CG<7Su8E0^|Th`?s z$&u?UJGVPVuA^+vkMRu-j43hNaZ9}Go~%-EYsjZvp-t`B|Gv{nQ_I&;>;BIc1iojG5iA!If&BdgtvC|O5q#5C|3yj|iX7$}0x34XJbmPYA zm7m;x?G3IKfz4&Wu`m2SetU6s?sw4=Fj}@sToZU620Zg@)%6U!zvj=S!?l3FTzzwH z^}Qdj{_OSDpTA#_=_BI6uY$QF1Sh0Cy-)N&^AXK~nbl&YDI`LA)4edu6(A1XC!F?! z0p<(E;abITtVtjzPj8TpcEAc#!Q`RQ1H$h_L@2{6PQ`?bcapj&5yL4(l%*zUQ~dP& z)OO_jVzI*S0f1wzBzqdIp51n$hmB2H!&O7(7K5YG!NO(gJ1xUSK12Pf3P+aOP1qaQbV=}O+D|_}W^#8v1Pkzulw3Hcc z7_KE`COs#uA=#47O6y48H6qQaYe|&OZc+V6+;3@z?|#u6LjGS!X1fpM{q#Aq#9vT|fcHooVflt$B9{^hOF#opkCEKiGbWZrxY`V`C#Hwvgz3EYM(D2?tVy}F)%=imbQf_J{b6m zw$yPpXdk)Rg69-auAgSFWP^Q&7Wy5?#?2;Vm+(7?wi_{CA`Ij^zn}@cs9wawD;ea4`yw{RWh;;i=yg!Bi2d zAt|n%h#veYIsiW6gU_A-v!+Or^f%P{57hf<@&T#qt!F;WJ-U>8^iu?bwUZK3Gjp>y WmG%`i^ZL+_hF(3hrXaN=%=~{BK%r#- literal 0 HcmV?d00001 diff --git a/scripts/__pycache__/test_live_catalog_refresh_progress.cpython-313-pytest-8.3.4.pyc b/scripts/__pycache__/test_live_catalog_refresh_progress.cpython-313-pytest-8.3.4.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ddf80e28ada5473c39b910919af7281948166521 GIT binary patch literal 10838 zcmeHNZ%|a%6~Fu5+ug^q{1JkJ1zDBGZY!(?(KN9nK^=nx#C?#cps$bJx66x60teI~*nv6)2j#;~FK;du*0s_|n991|D)p)nDcC0KhCYQ_2{5UNv2wsKZhZP# zSnf3Gz&@+^XG^8}X11*{9bBrD_;HSTWd&(x>QV<_h=;V>X2x172esFwRsb&J&zUy? ze5O_8&%HOVm^=$t=_5K*OrG`;5~=m)-CO!ADOM0xZZ0FFBR?(cFHE;)YB&R%K0_vjbJ2Zn0#0^_DgC^HeGsDRX|9SbeGA(0vA2iKr_3WicQbk{St%R1Sd^b&JuIIn_jaO=lpYs<2^m*}`U4 zlj5Uc=n9a-sO`hPKlFWgC|>CKK6l1-$~Cy|kAZZ|x&Jn)z zQ#%ZXnf~+Ff=+!2to9SWp@h9!UfS?5clFVwK!0E2YFOzEz&&LOI)Y{Q??dA>X(#^K z+?1GLTT0RvpxN4O0)xXe2BdN9ywZ^lIkRkkErg4Gwpo_G;K+7H;Bc^DpCjpyMpjRyw16C1C6>Eey8h!+~1`IIF`T)oM0Cm7Ig}|kR!Oho?At{xh*EI2P91c zvzx--X56jKj~Tfi08)eTa&N8 zbmPje66b&Y$-4uI_m3vdAGvw*n5Pz5Q*M}KssN{348(WiHy0AGeCO}KJTm#}YrakX zo4>e{=sz=c3s?#duG;2xn2Q22N@dLuldck>y=JksF*%s&Qou_V2NgYlc0I2@FD)EW zR1hlxJ6C!n1Eh=y7`mA3UX=V$bTQ4p#F);M3z(eTqZpUSH0+Dehk%&;Mpf$R3QF~6 z_#U2MIaJzrz_Y-UMTyGb>Qn)b4=J)5gdQyj4o7LPnBHa}64MO5W}_MjNUB~BjpRZY z-Sk!|95zy2VPcc9nn9W#iVYXtmo1s{3WgA(jBsPE*Wn)-9!)@oF z>f%D&Z9=NT9L3wE7?SHT-eS0s!m??%G-5h?;3dItK7AH52U-E>m4kFS4#j7Vu0&Fa zWCaor5{p}bE#!JRS^|}sE5T4|Z24A9QcYHu4bzqEX`?aO%hEy=vL{2IGn#zEbo4q1 zYDwkSKbUUK_Alhh(6sd~Acx6Be$ff_xcU!5%EKI3O+h!;Z=WbEIVru&_d6$e_c&iV z%9oxi9(?X>Io!gko$MV#a(1y9*5Hz%%F(j5msgJQ>!I^R!SRAY=ZL3vv~=BdzRtR` zIZ+KG2I^?~6hyOoMYRsRWUKh`m6%uHj~rV%x(Ljic(g!-yzg9@7gXRLcrJTpUk^SP z_c`Xu#Nqp%){&kyT?@{cQ$wWYTVfcUQvuJJjCe8h={Yg5Z)vw-MD@pfJ_Y_tfG1k5 zkq=5eQC0RGG)doLAB|}~JrbhT!q3T6!~kmZuW{#l%0b_T%0=LwVLfY2?pix5d>o;+-Ge7=Hc#tylU4EO1Xn(FWjp;5WVZ0~ZdxdFb-CVLH;#IkIQ(i1^L1Rg$%L zuRR%o+~Vvr%tqT=B1WTT#i|U#9#tg;@`+H@!MXVQ)?{>g;HxnAez8}*6qTa`ftXcM zIp7@Q%Tvhv=$H1y^Y;Sgo(^S$6+^L$JKi`r#;=*E_?jhpi{WyrCW;oHS^e_r(Ml;^ z)YZRb!d*D-E+2K554I1jeQoDZ&uGQE%X`P%n?GK@a$xIqx949&MOOW99djjq`_-rF z$kn>M28Ojv9n-ydF=@S`=@P}BNHC_#qG-BB5%V1^yB5s?5%NE%Zsm7r5mB@fBPgc5 z(Z`WA0x?TOcorl>35b#bIU`*$xD^nc=TaQXBSELLEae!v&2aWTw%cyIef{lxyS>Tw z&-@&FRniYp6!+KzmoJLm!lYvxil&?HC!@V0y1DgLCPsVF4r-Fg8A?Q}q4EK$7_ym@ z>B1D5WS}D&lZ`wDEgK-l*lc|Dp%uvSfbT`Z<2Q&WW68+Rn)) zcIUSV3h(8%nPrPt&#}r@mSdA`xbxfX+Z?ik&93a(aYN#Lt-AHMs;@z?&O z=;{kEjh}kiQ6CHRBxYul|MT?s^o;*J{Q_mx@>;P^QG#-O)6)rGSf&L_Us4H%dlYKM zt8HuOiw*=El6^rrIuPnlH1tQJ$HKutZ*XaE^k`6y_#*ybgW{J%vAEI@#~3r;PBOk! zh>gBJHmGf4+5TuGE=R+Xzt1Q8{PCcyNIp3z9Std=?r=~FMU;5Z7m%VoQnwuR^<$w( z@xGwc6GA4fB<;{tlUsjZ>bGOuN)^cS+-1A-lD#rzuRK$wE_^Iq^|-o5{I$L97BVM( zL?;s1M~q|Eu%AsLFAAbrw18Sf+PyZ>UT+m0cyfqEVzKCC&#ro2EU6b7O^v3AsVn>8 z+jAe;P3t4V=PiKJ*<3p>mQlJVyBm!~5fSoAmQP#KoLAB_ZAl9jrGZ#z)wtTtVg)6) z3*rpA=a%CX@==DRE}h%>2PaaOM8F4{lrYUhnHv*lmeqFLl+`iwtUJ zdQQwk>G5y=X#AD2Yw!N>>Wj}`J@b>RfAN#6FaOE4x4!?;Pfm@$`3J)z!& zjJIh>5<1}4Q8MQ92PM{&B@zt;6`49a0bZ!S75I1S5s-zPmqCn&+gieL(K3;tPd8rL ziN3U(j&iSq=j<%yxMGy-H9acyn9lNAQD(IL!NHcZ91{5YMvRaer1rdCnUF_wJkcAB z1fPk?YqhTFw`dmqV{#Dab2O+(*y>>^(~iH7*aoY={)uqZ?+Yty>$8=6Xf(4xu49L) zWxL;AH~h%h6RA~O-&pkOqPN$5c8fOf*Bq6mSMX1M2oHrPgzz?RLh$j82@y@i1OqfJ zuDoeM!)#i3nwDBY_M!rL4oHh!Pd^(#@+U-IPN_8c6_isA!n*9)|F$&THn#U?PIYr< zYIWxu8(-b{wv_3&bB^iH9dk@yWtSJBBX{ib4C*osE`OF+{WN@jM`~4TW~SvwkuNU= z(FCjwtz1cT$K_xKtNqbLBz~PRt=24CsgN0~BQ9fgfss?E!_}_OeS~X*!>|`&FcEyK#^AD<>lG>{veFNW2*&^RY*_G}5#}?rkQ_07i;C0{N zgc+qb1dFF=$nlAjD_t@YO1m4+m8IM(hOAdyWvQ~=BdbPh-d>k3+nsjp9x`8Xds3eL zBYQ`;ye*|Y`_u0Ih0A3b+3?H}XR2!HXyDw+v;7zCkLNNfhQq08w+ z)rxZ)F4|XJvCkT|{@OlYrsXkVjnXG#9Pn{`@z=R7JnGjTf`~~MVxa%l>G$h<2(Q{Lex6 z9CwdFvYM0?|mYZW(?wf-Y;UD{5$5 z=GED!&JODukikl2O6bKBgM~UxoT-l3U)?}75b6m919d=vU_=Q;Be&Ju;3FE=UjMsK~;;~dM`!5BNSoGqzU z+iznz@?$8j6)_2mO@h1z2@uryP*k&vSOSsWr*_G-@#H6nloHA3JH>hXN3*bfw8#hw3hq(wP}u&H$k4~u2jr?WBZ?P2Sa&auW*awX6zG&im#NEzS93< zKN9D3@JMub@(Pf&oq_Wxx#?e&|*B?UbDhCK$3%$?1fOc zQP)D+tl}(MOj*$o>j@m7k|BFlHWl9nsk14ymEXBQ7a1d#hQ}q<5tBG)Hh09H8;4=3$<{`)7%o;Db#T&!5mvL&%$?9~fs$xr z-sF;91cB_D$kE7_rSK7R4_7-MI$9_m98fgVvIH%N<~JAUf#u73MxSO0n_L4wJ(Z@hT* zTi-ux0X76QJMHuqQqR~gk+icST{~+d-OLe-!o<|(>qR%0e@dgedgh(+p;Om>`rg%V z|LN7g{_b@W=aae4F3)yWvq7IpgyYKRGyCH4nDTf-1DUAmA>fjEs=(J+E)D$f-Jw(AS$GA~2)Y})=l{L(^M7pldk@|zdm?A;q_x1Kd zPmLwI!y!L2+8`fqKyNL;q*$+LOfoH2rmQi2LI-yTser5v9g@8$Ghtdt_#^%5)?Sdc zriEDAWI@Bmh1Q>^&28nlCruBc3|&5KBy0hgXlMzy!B9i@$5AGu{(3Wdo|)-+78%c+ ze2@sKI;?9tFM|%)NGZ#RkjkPE!g;;VGc83||9X(Z6hj)|@0l_+=fwNr<7%L9WagVY z)Q8rjs@Ghqmd=~cA4sj=qqcRZorh9wQfj?)v06gG&lDQv*O%0qWD-?mQX}O}AhmXx z03*lTG&7mUWGb!|>1Mq)Ie~ovV~Ec9Y)s*jpQIA@0beB41EERhc{B!Mf-(}=P)MBC zb*}Zg()ASMefMJ_Xx(4^Vq1*vD~Ex2zlRymGy8J+jFCBGyt?%uxZ*L>CQ`w6@cT#w z3!6IVDZtBg8{{_#NbKaB1U&Vdn(4{4Q9uRi1Ix#{)fT@RlT%LRhDE3@g2Lu09&+5U zbDnB-(GwR|Tv(i%+p2cN@a}qD(K~6{NUiMA+nl?Z8Y$nx581D{=M6uTb}yeoV|%cE z^w>M@bjk5|+#}}m&eWQn7vgGL=f53Kt=a$5303l|foMvKsd6$UC0{x*6hFQBV3{xCvt%x~KNra&u?AS>={C1`vAq6k1Hlj%8 zf`aFwH(-X%D6d`)Ju9J}iEaoe)V4KeKJDl>%0+C2RlvL+k@?m(Rgd&`4TjBHEn}^g zXUA-xLKGlA$u8q<9w?XB$FCWEM6K+<+;cDY09VHu_!z)9>H*W9kN3&6`lK+QgT2es zqOJFR$xGge9?C5sOko-y`@#5YL*rvF>q3CMi*j3u5T45tk;5Rs*>3vn2Z4#G&$oQ# zstN0h))wOfUrw1jE6Ban&O(ssv=y!Ij`#%yEh+~xW=-;VkhQqar}SmiwJ7@Wj0G>Cp zP+it^p<6vt1}=58eZQ_c9FXEQ(KxLiIb!?@JW1D86xQ=Q%E&OmBzP_6D!%X)8P zQ>})2*qU~=PEZfkdG)EPhI7J2`y-djW~&R=Tv&KvMyh(3I;T}F-92QxQaXRcmM&dB zWMgclDrK)avnVz1&^gr#frD z>g>X}TxFMB^HQ#PXOh2mJ#@u6XSn&I^P%4fT*W)QE|mp3pOzja0C9(u0_;wS}sj@RjZppx+6hwD=wO zVn&f+YO8cC6z_{B;<_)IjGI!AFC3P-eg1wa9%XK79Xli_dGPeb!@&ZezJ=!IC=upA zA>d{@Mshah-b*v@y=~s%LLosVC+zD2y*Ve)=a7sk3dOx0PDV?p24A*` zCX7*e2P7-BmygQox;Ay-IAE)pZy{0IZ`ws5)@@>bmi>GeJ%#uP-3EDuPBU&T z=Dl>=VBTlOd#(=i4S^@2ohPCF6DwD`Xk_1L=ec94rR&s1>(lNH8CiU4kQ?rKvszu! zbkW{CwW*7284P03bODF~s4##bIZ|}i+aU|PAPtwWZ~-@&!WvDGi76+~s3n$ml#`*$ zknSP`Bh<>ID+?oe(II)#XE`&1d8U9SGEx<)GvnwNg@VvD(Sb3eg$Z~JY?zdH^^KA7 z)8D%G^XK!Wygx;^lZ{iO(Be(27O#G6@#?0`;)^ML2oOLp1AGIDly$a9J&4|6pmeM+7}4=dVOT7MP;4L;A1nY= ztu$W+fa+`pj7lm02_nQ*Yc0Bf2Tru~2%IwLw2h(DeHnq~{{u|Z zc649)4-g>KoOz=rb>(JtPn+5uz-Et`x(S1x=1DaUapumQZ^d7z^fhmk=0{jmV$QvFxBPbLv(B7kI?NHKS(&}PBOFp zLSR@}UAjSC)RcBN4_ON|wdu@YJ-`9W@ZFh&xVFr|bwHH_&JR=yvWxGK9QTCHo#^I%LSw9>WH zA^b{K7X*tyA~p%5{(@d(ctcj?@zXDVIQIO9@BKo@K?M2aWG1E;|Iy$je2@EPASm|& zT`*wScq!&G!}cNcD2U&KBm_;2#qSgV6($}Q02TQS%uxnI$v5D;6Um7%Id~vwz54BQACA5F;m=2y(FP{396B%whOA+*z;%DQ{|r-_ zOto295Qr|@FfF8)I=GOCm&h>?L-8o3gcC#tys$VxSj=>%#XaxbyRVu&J<2r@?A#fZBM*!| zccC0U8EnxbrY72{yZHljJIDv=*2?$LP2m%?eM^t>*qbF?r)R1qUBlw&cC~5$`z2l2 z(#+0szE|Dbk(%47R_|BKy2!Xf+q#Es>oGXtH{<1omFIfjIhig!@XpDR`1#|hCQ)tg zN;P$*%NCrjINkGV^>E<*vIS>P4%yVw136|tYh+%ks!?6GAyw6M9-Mr8*2*$mGxBh1 zcD>rLE;W1oMf(PW#ea6w#+4m3DfB`A$=Ot5{-v!5bb^GN)n)3ZfP=}pOosV0*FAKZ zQfND6^=1NxOino*_Z8v{aVAR@@b0WlKDA-N8IYUhr8JPR|!}q>> z?ewYf*H7sYSPUPSW|hqtf8{4v-}*kb>Lkn{tw%GhV3>662k%{b_g^wq3{rap!?(Zl z5@p>%9%)3Gg`5}sfjL`YaGU6C%{Wp^3I)v4L`-X3kK7|q);-U#G7UoPAXAeejwmn} zw&00F#rpEp*SfLHI!K(x^K4qSJGG;d1goJKd4^)D}?2mk@vNVnbU<5Pd6WA zbem>AAO`*jCFCE2OgAm^ITVQCUoJwbI{f~Depp9Gmb`gDUApUoxx4WEnfy}}f*eJd ztH(SYpvf@&WD? z8Gf50crTR``%-=na!<&8lHW<1S(_k{W)|lU)04vc=&7_@7_#2rxw75-=~YO^eOOWJ z*QwjO)u&G&;W^zF^)zfzH(55U+ok=dSE;4_!^&tx-6E->Bkz~?!~Cs7{n$Qr=_Ym2 z=Cpgu6ha(uTX)o^=Cq&RdtuA3r1YHjbY=TR``#?tpYf!s!|H)<)qe!vJY5}5SA~CV zADAXAm(~};a&zJsY)rsr1Udn44S~!g3NUS(nHXeY)LMi03R{J`Xk)O;Ku_jzq;IILqI>3|a;(o6Va~d~C}Lz;=%u!KdT6@adScX#tsscsbI%)b zj4l{{ZuHnWc{H4MufeLQ0~tE@f_vDMvd9 zunNDl7Hz1Q=<%TWASQb20O*~UK+a{1rNGWGJtjn;uoUuH7!LUeqOB2hQvrk`-8;Jk z&}!!DXf_rZ&X>&8`9fEg(RPuT!7gJd?Qt0?xx3NY9`KdPmuO?l)xH!nv8QZuE@HGZ zNz;wuE9j5o%k*a{`W&%{MVXnkPBF8=+hxp%K^e<&&G#dlFKjk$PCmaE3Rz8u&R5mia?M`J-s4`4dt_*GmJiDqi3 zsdXehH50T=dZ#8iYhofwKrZsfYm%-S;swU*c%<_U*~ikF%y-QM6VxP2YFPW(5?0o! znq*N8TNVg7sL?#|GeqDn93;KBhaUXFf!k9}eW!x^#|8V69og(PKebvu8psA~G=+%`WL4 z1&PJKM#D!TN9t!+6wQHyeK_<;XSD4{qM-;)o)Mk6Bd92d2#y4T&lsXfE~9dHD1h-7ir#2uJm1(0 zsGVFO_!dE&t6*9L9k8GpNg$@+Rw*^{{9Mrk4c1T@0m{=Lj(M0 z^h7tjevt3b6A$T$hxEk5^yE2g9&-F<=H8*r#z>dwBb~L+1a)oJdLb|8M;rgX@^4x$ z)itH+n$mR}FV*cx)$O>jJ5{&qQe8)?t^@tpZ|b8SFAY(Lmnzg>P)bR#l;bSrI7@k! zelL8M|6ll%Px6>uwDwzFVjs4><$BYlE@??u?owO#U1~j;YNc>f$+Ur9-3{vz=FHK| z|E6mD&{OF8C1-WYSv}nOfwSSJz|G!ax?O%J!Hx-U@8HJ-bF6W+oDP;-$frE*F>-bQ z4PsnD|8AaRh2kao>7x@xYmWKl>>`r_R7zUpS?IA`NrX5ki}%kSxn}X)36lE8|Lm2I ze*EJD;>PVPvKd3SsRpbjDaP~t{nEHV89Yf;>r{C)tz^8B!is)k) z9`}VJ_+tfNlsMH)yi*Ayzz_LG1W7ks8w0?8DW_c&@RhuX5$9|ZbA8Mrcp}x*_EzA{ zz&j^D%Zr-#q9~LfipkNn{+wS{m=q+EiBiuvV0SV&ykCRRdj2qz^-Y=?GDkzz_skxC zYHYLGe)zJpVr1+2CF(vLwT)xrZQ?i6Zk@S>N8HB|Q_;s9UsH_ek7_4{fVlJ!kO^mO za|(7vYOZheD6_=Q^`)zPYIk3%Dx@AccG3QfVL)Zul?6by)Rq$$WFbM4RwPL{@0h^f ztB^FOB+;MN$mF<*QzFEkOe~N=B-umXYaNkoM99D+f1SvyM7~30gvePU|48IrB55L* zL9`i?M2A9g*1{LZnYZo)@hnM?$|MQq z;R2cGv^;#X+J;}{Il^*nmWb(QxnS97x>;0WY5de{wRrUzpxSBp1GMU;+F6>N{%lJe zQmW=$9~+diPgPQkj-52yIu|+ld(@UM>dIs2tBO`FSFI@lV$3&W=}*yaY`Zi zBHkAcb?XAijLh;i*slI3N7zXSne;+^ThQ2ZvJCrUC(Jsa+1pTMvky`?n>8+ns= p_?b)d8&mTeKjG-{mcEY+D71%{4r<71o>-`$d3BQp1t=~S9b@J z=&ra$xLx($ec$)p-@W&}F`v)F!1MG+XZD?+!7zWt59_h#UdCR9my?XZ2<&Etn7=Gx z;hWpcZQ=>P$wq9O?8LswK^&W$#JR~uT$|j)y~#s70j8Rn#|ZpPMzHN+w^vz}7k-!V z<~#B3@d@@7oeU$DGYs(76?m+-dCaohYk-NeQ8wtff#0n`m*Hw_6Sr^O+AbL7U8z_+ zB1-X?s)=eMMZ&UCWu`@9N>r2Ly}4x7s5H})NiiPVFI(LHd}^+ENmznniKwVP6F1zd z)Gdped{8s|`=OPHq**1yR!8xd&3%lmg_o1e9tP)(Z~{wsft$$?8_o)`3pSt*!4A|Z zIDonYCs4QG0_qXmKuZJ<@kUF6UZYx|<5^>NXJcP7L9}*R)zljfXe4Yki7n&u;na?; z0CJKEG24rCAq}71xyff>ahK)t1#TH<=E5X1LB2Hw>%n~f{@Ww3{M0jg?8T9zFS#3% zk?s^OrkVVg6TdYx{&M0NW(7H;L{e3mXzQLyNpa!^&LC^pR5{+QVlx3_Q)5q}Pi`FO zkx8O2)|+bVRT2l{awIA*iYE5UM3I!R+^B|0EUBrD8jLZwI-<1}F*^E&WWTYFW``4s zMiOx`+#?YwtjR3Zm8{i`_BEL|HMOo@{a( z?n(Vl7`Ivji*XZ{>lC4>xuM|FYwU0sO_(k+W0$oH6W)1crD{ulY z*aSOJyWoI5=oDNH4#5pyZowmz2wwW_Yv6=w4SW;Z#47CW{KAI{i|j}FP`DxiD4ofK zIH3&FEAsoy5(Y(ga_hj?K+{uab>5`KG*-}XKE zd7%pH6=oD_-i@n6pAct_nTmgPt6?XJgr*wKaH20MDgA~Gzf-e;d*u1wjU2yl{r$H` zUpO*)`rXlges}bxpI(3CtuNj^I`aN2hNC+XkI3Y98L&1j{?9+1Hh0Sx|N4v3S5Mu* zO%`+*B@r3qwqH!8ViBV}2_ma#VsV13=z!nUgGnhYi?k`bl8DGE z!Oo6TFIeAF_&@d#ko%YbRWTLJhG`6wC=;mm>4HKn(3cRqpE+p{FfFAFQv#e(_9uKd zdzLdixzYB@{jFyiNZ=ZpV1(EpwigHxoH@h2Dk>}T!6bRi*gf?Y&Fl9hkwN?HmsJtA zdR)x46YfD^p!GMbiYLNSTz#w|UpZ>IF#tj42JBG1Y}=W~2OqkyDzkjUYx7^8f9COT z#$1s1Gj}z69R8Cad`vtcsBeoV1cqQuh`=JeMQ~btMblD|n-&hICCHOGPyv|JlJOl2H*SX_qT@oe>AyFLeCDt2gh=JiPO@wJ)zdBj)<;t!01du4RAB zF5d?o`OYp^VwY>JF6U->`Frt;TQbYrax+bqLq4eoV(=gvM!A~m(ukbX)!{@+(Qcql z3)%?oV-OhtW8%j`DEdJM0Z9Y7i7ESGGbKpB;V^-mR3lG*52y&50s5GdX5!5Emm*`s zy^=@lxBA~ejxc}rR-D**zC&+SuHN5tv0L9A)_b43$uL{EHV%IFauW9?!_4gBt}dv* zc)z|=)T1h-ujbZZw!pPxc6EsR(#{`br+vxr0sl>gpI&;Cw^w-5?ytN|>4KqH*57o# zEaP9Ac3kt7Wy-bG{={T+6SyW(+z$b+L-GF~?{4JocSKK|Z0Rz`47`sx-up!!d7b z>O5p6C9S7Erw-|RL{37As!3W(t&hZr90v5EKM4|(^GS#+`H!xCFo^1lH)Y|=%!^B4 zugQcYh^MhWP~TJ^NhostVt8o+y11#{)7Z2)V8W*fhxLt+LAi1qCR?h1qD~_+)g%1X z_a!2+?wA~@2Mr)AYAm7LRdb_+C|G}2gJv@HWl7VL<%^mY!(V$-(+c{xs~*dE1}Y+8 zq^ephyeOtbQmRH`QaputI`aCTMvgu|@^5c_e&JW+X*rwDU83z)Mb^YD%w=9n7%N5;qj zJ6*!5=7T&5y+u4Nf}AZB*b#EpXO?fi3v$TAz;1Y`g+(WU;7Wp_#>EnbO8^m2q8qmo z)DL7OlDlxHq-g&bcDi%OqsTcq@GPSq#{YJ8jMjU*+GumAr+^$`u2ojOw)vNv0Z`_4 z@ZgtB-@&I#u1zaF-upr?B-VCvkm%pZrR~@JbM<*YxiIbGvswR6KnX;SP766Yv;NK; za%`~>W=qz$C2jj20Fi?f>5{8ya(qBNO7Y>p03V)qjPsEk)H~v+XXI+&8abh)!44jh zPnah78$oY2*c&p&b0C3c+i>(LAo;J$25t^z1lqsFLb>>Z#m~3VjaC zi>Y8bY%13XmK<2g0Wb>JnqEk|LzsbDJZ~D7QUs``n!P5U3MLca!c0tU<67pKCZl3k zY?Jv!1vJ`?hAVw6M6kvYz_36bF)S_!3xIiQJ|V|LtWZ0X+2<;Z!-~P=YeQQM9$FiA z@Qw#YSVu=QYh1e%a6%ii@8sN#`-?u~5l!U+Fqew0n+eVVA1$WIeX7AOPN6`o*}mHJ ze{P^%HBeE|@BvH1-=TlC4yPYd|{r4Y?q+bJP#HmTqcJ#S;c%e%Fi?`n#Ir{t2 zSN|~jn{)~dlk2~DVf3Hg3fe(8L<|@1^j37w=$nAvnaA|b+R-<2Cle4vYL}wW%_X;S zRHLWQjiirWfA53QSN~!3H*elRW0_!Nm79@EYRPi`{Su8Gm=|LVn&2cgeXxG6?zY83^MH;P7bEZU=)&Kk_Nh`z3g zNWHS0R7J4(rDzoV)MTnF9t+c;4dg=!@LMMUDcY;0*fks&b=IgqVJdewx`4C|Qzaun zjGGoz?pS)P1IT0SeIVKxL4%D8=vU;;ZD5#3+53S99|<-RYyno_P=wndP=miKfG6kv z26BF$jrw_Z)6d%lJcX{ASOz{|BPKT?L6=2E4HpO;p?1?yWZZZ?|eGbE@qw(FV%^__>GFA{P}_)OCEt2 z0t$v9Yk>q^1Wn!oxEVGKj}h#H;W5K{?IZ>J0)io?!{ww77g>cIu0BbLb%W7lVjhkG zL6ATK9SYWM`dm+#PS-|?efL5TjIJMF*ycm`)xDtczJ@Sf#muYa(}!v=aQcR)02L3g z>(B+;!tFp8EY5b~SA-+@)6cEuA#o@7BoAM`>>B*?t>vKteeRMAU3zO+Pm+vRy=mv` zJm9cZl%(A^T}(xtK7ZB6OF#a*%&a!OGYMs%Fde{@JPyV>TfW^ANY?YRG#|1^K8_K+SYx;g;Dm90ILtv+XN%sZqO0OOz(ILaDq>sW=$<&*>bR|?aGDw@g{z+afrSsvmWA*5=- z9)(_P{f-A<+63>SIQqmOU4Uqz28@Utyx(m^SS?b3MVPwdDXIsc`uo%aas9eh(gO8R z>(&HA2xk8Ah>4q|6|xK-d?4}^zPY0$kxC-CQQSkv?T<&uRxIC!WE&9B8|}#JNGS1` zB9jnipaM0GX(83!9Xn_^5hX(iRTK`_W4)vkS$6`Vu=x20Z(cufbmZhw(=#M{A%i+X z`HYd{?~cCl7KDGWL9-!(Zz>RjT}p>@{a-$~{{Fw4RSZlFn_(K(yt|qt*0lu*>cepo zhwDv|p$T4!Z~?4|@wkxF`4A4JAMiSvvA4J0@a_Lb2tNF-hZRjKMLOX)UaKq)ghd+V{A|l1lDyyc7 zdQLST=q61u9}t$-fP?G@GS#$@gTR2!SizaAi+{4G7qD$;!Kpp^qOF%_ZH4b|NIx(I z?F3CU(>?3PwPd=M zKWGiUD~|nxDo62DzXk#THNARh?(lOTmjm|SWSD*IlZX=AxIOsu6er_P8`q6Lhq)F6 zrgmOLVA8m~_@#0ZewB9dX~#{DDci=KSPseXJE-ankL#Pd^k)u1!gK6atm7HBjk`s& z)6T11sb?(vF#^5n<^m9|2*AH6^C&-uf%5=oE7=3}Q++hxGfz!(a%$SO*loi{wq1%r#K-V-K=4Xlhfz7xY~xI`jS4x|VjZ|Fq7t4v+lG)D*yZ~m zE~ttLlI)Qb8Y${YK!lG5j*RB1wbl2n2dzgHtYwf}Fkyz=!}A888$NKJ49Bznl^`Yc zK+*?}`3KpIYv$nN8P|hXePvgCvopTgrw2auJ@C7&7n(2CuesuT0RArf9{5&8&-Oj< zY6t(J!?U{5IZl!1bM4SE9m2(NG1bwG38?gDL;iH%!ZvjP zvM1%DXL;W)n1Tat11M3rpCN1Nz{2n5qhW+^r)shXE)l4P3vL?3yQFY$&^};aC`BR* z2V4tHXMf>jC)*wUtg2>OLAo3qE9B_RCwmlp(CrDQ;0*BEmwn#D1(h+N@ zj1zN=e!|#xXTApWblQTY9|ZSj*u3m%R^u$z$4It2)xn zR`P<&-XJ8;*unnU^neDPZ}?96^aZWi>aBX)jw@|XW!lgy6xr1nEZqj+`xV2u*KnWJ zY))^4YOZ+eGTyquotM3hw|Hje7WQu7LJfwXGj( z7{UE5{>3}*>=D*(rU3}B$m+1jEF^f+Up#JlXz6a85waB+dY15U&> z*m&!>W{i+jt>%aTH##EGsj!q~ZPNqvsB1XGz%_ zN$={()Wr0C2QIk|jysH?vCr0^&v1$K!i*>y9#Mp2$rM~G6AiB@;te>I4*2qm5V?d% z82T0f1tP&12YC$1MkM=?97ggyl2?(u4#b!)ig-i|kxxm36R55f>QvE;yMleen1MkI zX%i%9+^CCy9A$2CtUbiua!j)aZUtOGDt$m^JM9X4tDLv5Wp8;r_NLp_4ts#AA6gy< z(}yxF2K{6m)ML0(N=!phF{bBa#Zu1ZH9Q&}G!Mn73}O!oFhwr_Qk&CowL>LqBtUf9 z+h1elHf-yT!5FSp<|IBw-Uc2p(in9;kefWqvY#>T&lvyTIEIaKe}l{@Dp;UjIa*jY qFnI9FoTkj2rmqX8 literal 0 HcmV?d00001 diff --git a/scripts/accept_catalog_controls.py b/scripts/accept_catalog_controls.py new file mode 100644 index 0000000..ed4b9d0 --- /dev/null +++ b/scripts/accept_catalog_controls.py @@ -0,0 +1,32 @@ +"""Browser acceptance for catalog navigation, routing, and refresh status.""" +import json,pathlib,urllib.request +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.support.ui import WebDriverWait,Select +BASE='http://127.0.0.1:8088';ROOT=pathlib.Path(__file__).resolve().parents[1];TARGET='dobot_xtrainer-软件版本区别' +options=Options();options.add_argument('--headless');options.binary_location='/snap/firefox/current/usr/lib/firefox/firefox' +driver=webdriver.Firefox(options=options,service=Service('/snap/bin/geckodriver'));driver.set_window_size(1440,1000);wait=WebDriverWait(driver,30,poll_frequency=.1) +def body():return driver.find_element(By.TAG_NAME,'body').text +def source(name):driver.find_element(By.XPATH,f"//aside[@aria-label='内容来源']//button[.//b[.='{name}']]").click();wait.until(lambda d:d.find_element(By.CSS_SELECTOR,'.catalog-summary strong').text==name) +def folder(name):driver.find_element(By.XPATH,f"//button[contains(@class,'catalog-name') and contains(.,'{name}')]").click() +def fits():assert driver.execute_script('return document.documentElement.scrollWidth<=window.innerWidth') +try: + driver.get(BASE);wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='来源与目录');wait.until(lambda d:len(d.find_elements(By.CSS_SELECTOR,'.catalog-source'))==7);wait.until(lambda d:d.find_elements(By.CSS_SELECTOR,'.snapshot-timestamp')) + assert '实时刷新目录' in body() and '快照时间' in body() + assert '实时快照已完成' in body() and '实时快照完成：1079 个目录，3444 份文件' in body() + folder('02 产品资料');wait.until(lambda d:'返回上一级' in body());driver.find_element(By.XPATH,"//button[contains(.,'返回上一级')]").click();wait.until(lambda d:len(d.find_elements(By.CSS_SELECTOR,'.catalog-table tbody tr'))==8) + source('PP资料下载中心');folder('具身产品');wait.until(lambda d:'Dobot X-Trainer' in body());folder('Dobot X-Trainer');wait.until(lambda d:TARGET in body()) + operator=driver.find_element(By.CSS_SELECTOR,'.routing-fields input[placeholder="姓名或工号"]');operator.send_keys('browser-routing') + reason=driver.find_elements(By.CSS_SELECTOR,'.routing-fields input')[1];reason.clear();reason.send_keys('网页单条与批量分流验收') + row=driver.find_element(By.XPATH,f"//tr[.//*[contains(.,'{TARGET}')]]");choice=Select(row.find_element(By.TAG_NAME,'select'));choice.select_by_value('external');wait.until(lambda d:'已标注 1 份' in body() and '已在目标队列 1 份' in body()) + wait.until(lambda d:'对外' in d.find_element(By.XPATH,f"//tr[.//*[contains(.,'{TARGET}')]]/td[3]").text) + # Current-page bulk selection is idempotent and should not create another publication. + row=driver.find_element(By.XPATH,f"//tr[.//*[contains(.,'{TARGET}')]]");row.find_element(By.CSS_SELECTOR,'input[type="checkbox"]').click();driver.find_element(By.XPATH,"//button[.='批量标注对外并入队']").click();wait.until(lambda d:'已在目标队列 1 份' in body()) + assert '新入队 0 份' in body() + for width in (1440,1024,768):driver.set_window_size(width,900);fits() + driver.set_window_size(1440,1000);driver.get_full_page_screenshot_as_file(str(ROOT/'tmp/CATALOG_ROUTING_REFRESH_FIREFOX.png')) + result={'back':'pass','snapshot_time':'pass','refresh_completed':'pass','single_route':'pass','bulk_route':'pass','idempotent':True,'final_visibility':'external','root_overflow':False} + (ROOT/'tmp/catalog-routing-refresh-acceptance.json').write_text(json.dumps(result,ensure_ascii=False,indent=2));print(json.dumps(result,ensure_ascii=False)) +finally:driver.quit() diff --git a/scripts/accept_catalog_route_canary.py b/scripts/accept_catalog_route_canary.py new file mode 100644 index 0000000..210b36d --- /dev/null +++ b/scripts/accept_catalog_route_canary.py @@ -0,0 +1,23 @@ +"""Route one verified PP public canary to the local external WeKnora KB.""" +import argparse,json,pathlib,time,urllib.parse,urllib.request +GOV='http://127.0.0.1:8088';WEKNORA='http://127.0.0.1:8080/api/v1';KBS={'internal':'1b52c140-2e18-4676-afc5-4c7d8d190a8c','external':'fd7db6aa-763a-46f1-84d2-ecc7c952a5ea'};TITLE='dobot_xtrainer-软件版本区别' +def gov(path): + with urllib.request.urlopen(GOV+path,timeout=20) as response:return json.load(response) +def nodes(connection,parent):return gov('/api/v1/source-catalog/nodes?'+urllib.parse.urlencode({'connection_id':connection,'parent_id':parent,'limit':100,'offset':0}))['items'] +def weknora(path): + key=pathlib.Path(__file__).resolve().parents[1].joinpath('secrets/test_weknora_api_key').read_text().strip();request=urllib.request.Request(WEKNORA+path,headers={'X-API-Key':key,'Accept':'application/json'});return json.load(urllib.request.urlopen(request,timeout=20)) +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--visibility',choices=('internal','external'),default='external');args=parser.parse_args();kb=KBS[args.visibility] + pp=next(item for item in gov('/api/v1/source-catalog/sources')['data'] if item['source_type']=='pp');top=next(item for item in nodes(pp['connection_id'],pp['root_node_id']) if item['name']=='具身产品');folder=next(item for item in nodes(pp['connection_id'],top['node_id']) if item['name']=='Dobot X-Trainer');file=next(item for item in nodes(pp['connection_id'],folder['node_id']) if item['name']==TITLE) + assert file['routable_count']==1 + payload={'connection_id':pp['connection_id'],'node_ids':[file['node_id']],'visibility':args.visibility,'operator_ref':'routing-canary','reason':'验证标注后进入本地解析队列'};request=urllib.request.Request(GOV+'/api/v1/source-catalog/route',data=json.dumps(payload,ensure_ascii=False).encode(),method='POST',headers={'Content-Type':'application/json','Accept':'application/json'}) + with urllib.request.urlopen(request,timeout=20) as response:result=json.load(response)['data'] + assert result['selected_files']==1 and result['unbound_files']==0 and result['hash_unknown_files']==0 + deadline=time.monotonic()+180;matched=None + while time.monotonic()= page['summary']['total']: + break + assert page['data'] +assert len(items) == len({x['id'] for x in items}) == page['summary']['total'] +assert all(x['title'] for x in items) +assert not any(x['directory_path'].startswith('/') or '://' in x['directory_path'] for x in items) +(ROOT/'tmp/document-ui-catalog.json').write_text(json.dumps(items,ensure_ascii=False)) +options = Options() +options.add_argument('--headless') +options.binary_location = '/snap/firefox/current/usr/lib/firefox/firefox' +driver = webdriver.Firefox(options=options, service=Service('/snap/bin/geckodriver')) +wait = WebDriverWait(driver,30,poll_frequency=.15) +driver.set_window_size(1440,1100) + +def text(): + return driver.find_element(By.TAG_NAME,'body').text + +def nav(label): + driver.find_element(By.XPATH,f"//nav//button[contains(.,'{label}')]").click() + +def loaded(): + wait.until(lambda d: not d.find_element(By.XPATH,"//header//button[contains(.,'刷新')]").get_attribute('disabled')) + +def root_fits(): + assert driver.execute_script('return document.documentElement.scrollWidth <= window.innerWidth'), 'root overflow' + +try: + driver.get(BASE) + wait.until(lambda d: d.find_element(By.TAG_NAME,'h1').text == '来源与目录') + loaded() + assert not driver.find_elements(By.CSS_SELECTOR,'[role="alert"]') + assert '未接通目录' in text() + assert '只走这 8 步' not in text() + expected_external = sum(x['visibility']=='external' for x in items) + counts = driver.find_element(By.CSS_SELECTOR,'.document-counts').text + assert str(len(items)) in counts and str(expected_external) in counts + first = next(x for x in items if x['directory_path'] and x['source_type']=='pp') + folder = next(e for e in driver.find_elements(By.CSS_SELECTOR,'.folder.child') if first['directory_path'] in e.text) + folder.click() + wait.until(lambda d: first['directory_path'] in d.find_element(By.CSS_SELECTOR,'.document-location').text) + rows = driver.find_elements(By.CSS_SELECTOR,'.document-table tbody tr') + assert rows and all(first['directory_path'] in r.text for r in rows) + driver.find_element(By.CSS_SELECTOR,'.folder').click() + search = driver.find_element(By.CSS_SELECTOR,'input[type="search"]') + search.send_keys(first['title']) + wait.until(lambda d: first['title'] in d.find_element(By.CSS_SELECTOR,'.document-table').text) + driver.find_element(By.CSS_SELECTOR,'.file-link').click() + detail = wait.until(lambda d:d.find_element(By.CSS_SELECTOR,'.document-inspector')) + assert detail.is_displayed() and first['title'] in detail.text + assert '原文预览尚未接通' in detail.text + driver.find_element(By.XPATH,"//button[.='关闭详情']").click() + search.clear() + search.send_keys(' ') + selects = driver.find_elements(By.CSS_SELECTOR,'.document-filters select') + Select(selects[0]).select_by_value('internal') + wait.until(lambda d: '筛选结果 5 份' in d.find_element(By.CSS_SELECTOR,'.document-location').text) + assert len(driver.find_elements(By.CSS_SELECTOR,'.document-table tbody tr')) == 5 + Select(selects[0]).select_by_value('') + root_fits() + assert driver.execute_script('return document.documentElement.scrollHeight < window.innerHeight + 300'), 'document table should scroll inside its panel' + driver.get_full_page_screenshot_as_file(str(ROOT/'tmp/DOCUMENT_WORKBENCH_FIREFOX.png')) + nav('解析监控') + wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='解析监控') + loaded() + before = driver.execute_script("return performance.getEntriesByType('resource').filter(x=>x.name.includes('/source-items?')).length") + WebDriverWait(driver,25).until(lambda d:d.execute_script("return performance.getEntriesByType('resource').filter(x=>x.name.includes('/source-items?')).length") > before) + loaded() + assert '解析完成不代表已上线' in text() + nav('待处理文件') + wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='待处理文件') + expected_attention = sum(x['content_state']=='over_limit' or x['processing_state'] in ('failed','stalled') for x in items) + assert f'筛选结果 {expected_attention} 份' in text() + nav('来源连接') + wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='来源连接') + assert not driver.find_element(By.CSS_SELECTOR,'.scan-audit-panel').get_attribute('open') + nav('发布管理') + wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='从审核到安全发布') + assert not driver.find_element(By.CSS_SELECTOR,'.release-help').get_attribute('open') + assert '只走这 8 步' not in text() + assert not driver.find_elements(By.XPATH,"//button[.='授权生产发布']") + nav('来源与目录') + loaded() + for width in (1440,1024,768): + driver.set_window_size(width,1000) + root_fits() + driver.refresh() + wait.until(lambda d:d.find_element(By.TAG_NAME,'h1').text=='来源与目录') + assert driver.find_element(By.CSS_SELECTOR,'.tab-list').text.count('解析监控')==1 + result = {'catalog':len(items),'titles':sum(bool(x['title']) for x in items),'categories':sum(bool(x['directory_path']) for x in items),'external':expected_external,'attention':expected_attention,'search':'pass','directory_filter':'pass','scope_filter':'pass','file_drawer':'pass','automatic_refresh':'pass','tabs_restore':'pass','root_overflow':False,'write_requests':0} + (ROOT/'tmp/document-ui-acceptance.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)) + print(json.dumps(result,ensure_ascii=False)) +finally: + driver.quit() diff --git a/scripts/accept_document_workbench_v2.py b/scripts/accept_document_workbench_v2.py new file mode 100644 index 0000000..62f83e5 --- /dev/null +++ b/scripts/accept_document_workbench_v2.py @@ -0,0 +1,141 @@ +"""Live browser acceptance for document workbench V2. + +Writes one PP category rule and one file override through the UI, verifies each +read-back, then clears both. It never uploads, retries, reviews or publishes. +""" +import json +from pathlib import Path +import time +import urllib.request +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.support.ui import WebDriverWait, Select + +BASE='http://127.0.0.1:8088' +ROOT=Path(__file__).resolve().parents[1] + +def request_json(path,method='GET',body=None): + data=None if body is None else json.dumps(body,ensure_ascii=False).encode() + request=urllib.request.Request(BASE+path,data=data,method=method,headers={'Accept':'application/json','Content-Type':'application/json'}) + with urllib.request.urlopen(request,timeout=10) as response:return json.load(response) + +def catalog(): + items=[] + while True: + page=request_json(f'/api/v1/source-items?limit=100&offset={len(items)}') + items.extend(page['data']) + if len(items)>=page['summary']['total']:break + assert page['data'] + assert len(items)==len({item['id'] for item in items})==page['summary']['total'] + return items + +def rules(connection):return request_json('/api/v1/scope-rules?connection_id='+connection)['data'] +def eventually(check,seconds=25): + deadline=time.monotonic()+seconds + while time.monotonic()

+ + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000..613dc28 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,2317 @@ +{ + "name": "knowledge-governance-web", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "knowledge-governance-web", + "version": "0.1.0", + "dependencies": { + "react": "19.1.1", + "react-dom": "19.1.1" + }, + "devDependencies": { + "@types/react": "19.1.10", + "@types/react-dom": "19.1.7", + "@vitejs/plugin-react": "5.0.1", + "typescript": "5.9.2", + "vite": "7.1.3", + "vitest": "3.2.4" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.32", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.32.tgz", + "integrity": "sha512-QReCdvxiUZAPkvp1xpAg62IeNzykOFA6syH2CnClif4YmALN1XKpB39XneL80008UbtMShthSVDKmrx05N1q/g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.63.1.tgz", + "integrity": "sha512-UZ8sUxPTiHWYX9QNdJedb1kDZSpS1t/VPWBWGSgqHNi9w3Cu6IXvu2mzbhiTiPvtrqgTQJ+zqiAq2iPIPilpaQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.63.1.tgz", + "integrity": "sha512-cQ4nFQABN5cDvDpbvJ7bMStCpnaVxynZrRMfUJYgxcIk9Sh54FIO1vtfkg0B69REjER77ioZ/ov+eAApx/KmLQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.1.tgz", + "integrity": "sha512-FQNqd1lRy/0QhDk3xeRIkSBiCpXCiDnZO3YLVdcDKN1UBiKToNftCzcXYNLshmPDUMlu2TdeS8tGcsU6f3YF1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.63.1.tgz", + "integrity": "sha512-pvD16V939D3CloK0+qikpGaxiPrDUXTe7Y5cWOMkMSy7m1cawa8EGy/kXYi/G/cKAC4HDAbSnzCIk1WmsoOKXg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.63.1.tgz", + "integrity": "sha512-pcFGeL2345VwdTnJhA6zLbew+YgWB0qBG2+dMtXjCicf6+rm6kO6cOoh5VnTe0ZMrMRgRyuHmCJxZWrIdzYuOw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.63.1.tgz", + "integrity": "sha512-mRJlqSRulVzcKq/LKA6ICSIc3K/l4fzlVn/gePn2nXIHy8seRi5z/eeRE0d/XMBxcMldiXtQTSpRj0tkkC3g8Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.63.1.tgz", + "integrity": "sha512-YDUNvVM85TI3g/1OpnqKP1h4NeW/j64DfWMf+G3M809xNk1bJSnpFp4sh83NpmVE5DXnkh8ULor4LTVZKoYLHw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.63.1.tgz", + "integrity": "sha512-7Mcn71p9ZuQFAj+h+dhQXy/yeLePRS2yKRnmW1DijA9thKO5qap0GNOIQK4yQ6iP3SU0Mrb/yWo8h8vgRba8lw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.63.1.tgz", + "integrity": "sha512-4YiLQTX6U4CSl0L9cluep9A9W6UmTfqBDc2/CH6wlu54pl4E7Jn3cOD8oxzvBDEGk/JMKgJ47C8g+radF7mwvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.63.1.tgz", + "integrity": "sha512-2ra8F7w8OquwZN9z2/fKFnli69wa8PLwaVzRMIPGb13ByMJwC28Fbp8YcVGoUhlYMTt7j5j9bNgpysrN2UM+vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.63.1.tgz", + "integrity": "sha512-Sy20ncyhjmBP0Ml+UvQbimjlk6VFgjW5uNP+qqwHB00mTE8Bl2C1TuHTlRwK2YoXeZbee5lP2XevBWVkAQAtSQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.63.1.tgz", + "integrity": "sha512-noITLp8oNjYliPnGWmLyelIHwULGqbHloQHGw1rtxbWhTuWooRpnZarZQJ1y9EUC4szuCusCc+HEpUtxpIwYvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.63.1.tgz", + "integrity": "sha512-hlxxXd+F1mWiAcaFR7Sv9ZQT6m6UfI8+Vy/kFJzztq2pDMU/0wZ9sish0iszNZvsQDo8Gc0i5yuFEOz5dDf6fA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.63.1.tgz", + "integrity": "sha512-EF7OpqQTQ/BvGqLzUi4rEHuagCV9MugAUXSHemwPW5vxZ75RR+jxO/2j95Ph2dalMpFHSVECjRoioHZgA9zOYA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.63.1.tgz", + "integrity": "sha512-wQO3JesW9PRkwlabQ27y7sPfVOOTLRG73I4F2UYHG5PXun3J9U3y+b7ezVKSYbsvSKGQ1k1cq8Qlun4C9kLt3w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.63.1.tgz", + "integrity": "sha512-ouAGwhO6wHRXdnOVCOsB0tRFkA7nhNB2Nwax6oECXN0YiN8EYUTBAOudADOB1PI+yDL61TeNx/u7MVCzksNbkQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.63.1.tgz", + "integrity": "sha512-q2R38Sn+1J8RxhfJ+T54wSWmyKXWec+9jgDfqO2AtArEqHO5R2aeayp5H5OYLr5UYDVGsVaZPEFUooMhYCdz5A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.63.1.tgz", + "integrity": "sha512-gfI5T24WLLuFfSKw7Go/zDXjAAV0fny0swTaDv+WjK7vqcw4cRhFfdsyKL1n+ukI+ooBxn3bVQnyrn06WpI50w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.63.1.tgz", + "integrity": "sha512-4h6XqthmB4Hspji84wvgk+ElodTsGj+dbZqHJHHtKxj4mYq0ANSEEPX9ys3moJueqsRjwpaJYH7874Itwnj2ow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.63.1.tgz", + "integrity": "sha512-dlfCOa87o1VAYegLQ9EKilx2JCeRofiyPGhTCmqnuXZ6bMPiycO1rq1+sKoulAp7pGLIsTIw+1x5R+zgh5LhhA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.63.1.tgz", + "integrity": "sha512-cjkLbOlfcm3QGhMM1J5zaZjsw1GggbN6rw9UTSSRrPrR1KkcXnN7Uq9rPw34xImQ9VOY9GN+6u2Zj80B9ptkcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.63.1.tgz", + "integrity": "sha512-Li1KdUnWGE4N3e1F/B4RTB1ms+nG4WBgjByO46pkeBVX/2UBsY53xf5vK9WygVmnH3RwncIST7lkSdLSY6P9lg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.63.1.tgz", + "integrity": "sha512-t4ZYOSoLTgwhuFMrmTMLx/+i1DQVK7HYqMc6kY46EApwi8X0nIVphzdNoThU3xt6n+N5urG1/gxBdCaKDLavfg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.63.1.tgz", + "integrity": "sha512-RgroPfMmKlD1RzSDxvwgcPiy2HNQKoYV7OmwIXDsk73uKW5t6B/V8KIy27SMv/FNXFo/oSBtWc9J0X7t91ezZg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.63.1.tgz", + "integrity": "sha512-at8QVep6S3h5Y6gSbdGU06bRY5WJkf6WUduM9YtvYMbYhB1MOFfUgc6kehitQXzOtMSaT70q7f9ydPhpqu821w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.1.10", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.10.tgz", + "integrity": "sha512-EhBeSYX0Y6ye8pNebpKrwFJq7BoQ8J5SO6NlvNwwHjSj6adXJViPQrKlsyPw7hLBLvckEMO1yxeGdR82YBBlDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.1.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.7.tgz", + "integrity": "sha512-i5ZzwYpqjmrKenzkoLM2Ibzt6mAsM7pxB6BCIouEVVmgiqaMj1TjaK7hnA36hbW5aZv20kx7Lw6hWzPWg0Rurw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.0.1.tgz", + "integrity": "sha512-DE4UNaBXwtVoDJ0ccBdLVjFTWL70NRuWNCxEieTI3lrq9ORB9aOCQEKstwDXBl87NvFdbqh/p7eINGyj0BthJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.3", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.32", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot/node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils/node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.20", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz", + "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", + "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.420", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.420.tgz", + "integrity": "sha512-2yD6XreGusOfNV+dUcvipJEXc3n/n7fgr7996aszTG+YY5E4mqM4tOq/3uhP129cazL9YHbVWSpc79ePotWtPA==", + "dev": true, + "license": "ISC" + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.17", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.1.1", + "resolved": "https://registry.npmjs.org/react/-/react-19.1.1.tgz", + "integrity": "sha512-w8nqGImo45dmMIfljjMwOGtbmC/mk4CMYhWIicdSflH91J9TyCyczcPFXJzrZ/ZXcgGRFeP6BU0BEJTw6tZdfQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.1.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.1.tgz", + "integrity": "sha512-Dlq/5LAZgF0Gaz6yiqZCf6VCcZs1ghAJyrsu84Q/GT0gV+mCxbfmKNoGRKBYMJ8IEdGPqu49YWXD02GCknEDkw==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.26.0" + }, + "peerDependencies": { + "react": "^19.1.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.1.tgz", + "integrity": "sha512-3Df9jsstwhccuEfmAMi9l8XUh/GOkVObmFTU7CCVBysEbcOZLl84jCtaAZMcPiMz2EGKsATzQcU+Xr3n/wU6cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.63.1", + "@rollup/rollup-android-arm64": "4.63.1", + "@rollup/rollup-darwin-arm64": "4.63.1", + "@rollup/rollup-darwin-x64": "4.63.1", + "@rollup/rollup-freebsd-arm64": "4.63.1", + "@rollup/rollup-freebsd-x64": "4.63.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.63.1", + "@rollup/rollup-linux-arm-musleabihf": "4.63.1", + "@rollup/rollup-linux-arm64-gnu": "4.63.1", + "@rollup/rollup-linux-arm64-musl": "4.63.1", + "@rollup/rollup-linux-loong64-gnu": "4.63.1", + "@rollup/rollup-linux-loong64-musl": "4.63.1", + "@rollup/rollup-linux-ppc64-gnu": "4.63.1", + "@rollup/rollup-linux-ppc64-musl": "4.63.1", + "@rollup/rollup-linux-riscv64-gnu": "4.63.1", + "@rollup/rollup-linux-riscv64-musl": "4.63.1", + "@rollup/rollup-linux-s390x-gnu": "4.63.1", + "@rollup/rollup-linux-x64-gnu": "4.63.1", + "@rollup/rollup-linux-x64-musl": "4.63.1", + "@rollup/rollup-openbsd-x64": "4.63.1", + "@rollup/rollup-openharmony-arm64": "4.63.1", + "@rollup/rollup-win32-arm64-msvc": "4.63.1", + "@rollup/rollup-win32-ia32-msvc": "4.63.1", + "@rollup/rollup-win32-x64-gnu": "4.63.1", + "@rollup/rollup-win32-x64-msvc": "4.63.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.26.0.tgz", + "integrity": "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/strip-literal/node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/typescript": { + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", + "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.3.tgz", + "integrity": "sha512-OOUi5zjkDxYrKhTV3V7iKsoS37VUM7v40+HuwEmcrsf11Cdx9y3DIr2Px6liIcZFwt3XSRpQvFpL3WVy7ApkGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.14" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..4cba836 --- /dev/null +++ b/web/package.json @@ -0,0 +1,24 @@ +{ + "name": "knowledge-governance-web", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite --host 0.0.0.0", + "test": "vitest run", + "build": "tsc -b && vite build", + "typecheck": "tsc -b --pretty false" + }, + "dependencies": { + "react": "19.1.1", + "react-dom": "19.1.1" + }, + "devDependencies": { + "@types/react": "19.1.10", + "@types/react-dom": "19.1.7", + "@vitejs/plugin-react": "5.0.1", + "typescript": "5.9.2", + "vite": "7.1.3", + "vitest": "3.2.4" + } +} diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx new file mode 100644 index 0000000..d899487 --- /dev/null +++ b/web/src/App.test.tsx @@ -0,0 +1,483 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it, vi } from 'vitest' + +import App, { JobDashboard, KnowledgeInventory, ModulePage, ReleaseManagement, SourceConnections, WorkspaceShell, reviewGateLabel } from './App' +import type { SourceAssetPage } from './assets' +import type { Job } from './jobs' +import type { ReleaseBatchDetail, ReleaseBatchPage, ReleaseExecutionDetail, ReleaseExecutionPage, ReleasePlan, ReviewCandidatePage } from './reviewRelease' +import type { SourceConnectionPage, SourceScanDetail, SourceScanPage } from './sourceConnections' + +const baseJob: Job = { + id: 'job-1', + type: 'demo', + payload: {}, + state: 'pending', + priority: 0, + available_at: '2026-09-03T13:00:00Z', + attempt: 0, + max_attempts: 5, + idempotency_key: 'demo:1', + created_at: '2026-09-03T13:00:00Z', + updated_at: '2026-09-03T13:00:00Z', +} + +describe('WorkspaceShell', () => { + it('renders the approved modules in the left navigation', () => { + const html = renderToStaticMarkup( + +
工作区内容
+ , + ) + + expect(html).toContain('aria-label="功能导航"') + for (const label of ['来源与目录', '解析监控', '发布管理', '系统设置']) { + expect(html).toContain(label) + } + expect(html).not.toContain('来源链接') + expect(html).not.toContain('待处理文件') + expect(html).toContain('本地治理环境') + expect(html).not.toContain('P2 本地治理环境') + }) + + it('renders open tabs and marks the active tab', () => { + const html = renderToStaticMarkup( + +
工作区内容
+ , + ) + + expect(html).toContain('aria-label="已打开页面"') + expect(html).toContain('来源与目录') + expect(html).toContain('解析监控') + expect(html).toContain('发布管理') + expect(html).toContain('aria-current="page"') + }) + + it('offers close controls only for unpinned tabs', () => { + const html = renderToStaticMarkup( + +
工作区内容
+ , + ) + + expect(html).toContain('aria-label="关闭解析监控标签"') + expect(html).not.toContain('aria-label="关闭来源与目录标签"') + }) + + it('renders a collapsed mobile navigation control', () => { + const html = renderToStaticMarkup( + +
工作区内容
+ , + ) + + expect(html).toContain('aria-label="切换功能导航"') + expect(html).toContain('aria-expanded="false"') + }) +}) + +describe('App', () => { + it('renders the dashboard inside the multi-tab workspace by default', () => { + const html = renderToStaticMarkup() + + expect(html).toContain('aria-label="功能导航"') + expect(html).toContain('aria-label="已打开页面"') + expect(html).toContain('

来源与目录

') + expect(html).not.toContain('最近任务') + }) +}) + +describe('ModulePage', () => { + it('renders the governance overview for the dashboard module', () => { + const html = renderToStaticMarkup( + , + ) + + expect(html).toContain('治理工作台') + expect(html).toContain('P2 · 知识治理总览') + expect(html).toContain('PP历史来源清单已接管') + expect(html).toContain('本地测试WeKnora已连接') + expect(html).toContain('生产WeKnora仍未连接') + expect(html).not.toContain('尚未连接测试或生产WeKnora') + expect(html).toContain('任务概览') + expect(html).toContain('Release Worker 默认关闭') + expect(html).toContain('最近任务') + expect(html).toContain('job-2') + }) + + it('renders the test publication table for the tasks module', () => { + const html = renderToStaticMarkup( + , + ) + + expect(html).toContain('

解析监控

') + expect(html).toContain('总体概览') + expect(html).toContain('失败或停滞') + expect(html).toContain('pub-1') + expect(html).toContain('remote_parse_failed') + expect(html).toContain('重试') + }) + + it('renders the real inventory for the knowledge module', () => { + const html = renderToStaticMarkup( + , + ) + + expect(html).toContain('统一治理清单') + expect(html).toContain('613') + expect(html).not.toContain('功能将在后续阶段接入') + }) + + it('renders the real source connection workbench', () => { + const html = renderToStaticMarkup( + , + ) + + expect(html).toContain('

来源连接

') + expect(html).not.toContain('conn-pp') + for (const text of ['来源连接', 'PP历史公开知识', 'AList只读源', '613', '已配置', '健康', '最近扫描', '未变化 559', '超过上限 54', '检查快照变化']) { + expect(html).toContain(text) + } + expect(html).toContain('class="source-connections-table"') + expect(html).not.toContain('将在后续阶段接入') + }) +}) + +describe('KnowledgeInventory', () => { + it('renders real source summary and safe paginated asset fields', () => { + const page: SourceAssetPage = { + items: [{ + id: 'si-safe', source_type: 'pp', connection_name: 'PP历史公开知识', size_bytes: 52428801, + visibility: 'external', source_change: 'unchanged', content_state: 'over_limit', legacy_status: 'over_limit', + has_publication: false, processing_state: '', retrieval_state: '', updated_at: '2026-09-04T00:00:00Z', + }, { + id: 'si-manual', source_type: 'manual' as never, connection_name: '本地验收', size_bytes: 128, + visibility: 'internal', source_change: 'added', content_state: 'verified', legacy_status: '', + has_publication: true, processing_state: 'pending', retrieval_state: 'unpublished', updated_at: '2026-09-05T00:00:00Z', + }], + summary: { total: 613, completed: 530, content_alias: 29, over_limit: 54 }, + limit: 50, + offset: 0, + } + const html = renderToStaticMarkup( + , + ) + + for (const text of ['知识资产', '613', '530', '29', '54', 'PP', '手工验收', '50.0 MiB', '超过旧同步上限']) { + expect(html).toContain(text) + } + expect(html).toContain('统一来源清单') + expect(html).not.toContain('当前展示PP历史公开来源') + }) +}) + +describe('SourceConnections scan workbench', () => { + it('renders scan history and selected safe change records', () => { + const now = '2026-09-06T06:00:00Z' + const page: SourceConnectionPage = { + items: [{ id: 'conn-pp', source_type: 'pp', name: 'PP历史公开知识', enabled: false, schedule_configured: false, health_status: 'healthy', item_count: 613, updated_at: now }], + summary: { total: 1, enabled: 0, disabled: 1, healthy: 1, degraded: 0, unhealthy: 0, unknown: 0 }, + } + const scanPage: SourceScanPage = { + items: [{ id: 'scan-1', state: 'completed', observed_items: 613, unchanged_items: 558, added_items: 0, updated_items: 1, over_limit_items: 54, missing_items: 0, planned_uploads: 1, error_code: '', created_at: now, completed_at: now }], + total: 1, limit: 20, offset: 0, + } + const detail: SourceScanDetail = { + ...scanPage.items[0], changes: [{ id: 'chg-1', source_item_id: 'si-safe', change_type: 'updated', observed_size_bytes: 2048, created_at: now }], + } + const html = renderToStaticMarkup() + for (const text of ['扫描历史与变化记录', 'scan-1', '已选择', '变化记录', '内容更新', 'si-safe', '2.0 KiB']) { + expect(html).toContain(text) + } + expect(html).not.toContain('stable_external_id') + expect(html).not.toContain('observed_revision') + }) + + it('states explicitly when a selected scan has no changes', () => { + const now = '2026-09-06T06:00:00Z' + const scan = { id: 'scan-1', state: 'completed' as const, observed_items: 613, unchanged_items: 559, added_items: 0, updated_items: 0, over_limit_items: 54, missing_items: 0, planned_uploads: 0, error_code: '', created_at: now, completed_at: now } + const html = renderToStaticMarkup() + expect(html).toContain('本次扫描未发现新增、更新或不可用变化') + }) +}) + +describe('JobDashboard', () => { + it('renders summary counts and task states', () => { + const html = renderToStaticMarkup( + , + ) + + expect(html).toContain('同步任务') + expect(html).toContain('任务总数') + expect(html).toContain('>3<') + expect(html).toContain('运行中') + expect(html).toContain('已完成') + expect(html).toContain('job-2') + }) +}) + +describe('ReleaseManagement', () => { + it('renders human review, draft and seal gates without implying production release', () => { + const now = '2026-09-06T10:00:00Z' + const priorNow = '2026-09-06T09:00:00Z' + const reviewPage: ReviewCandidatePage = { + items: [{ + publication_id: 'tp-safe', title: 'Synthetic', source_type: 'manual', current_visibility: 'internal', + knowledge_exists: 'passed', chunks_present: 'passed', retrieval_hit: 'manual_required', + gate_status: 'ready', gate_code: 'ready', review_state: 'pending', decided_visibility: '', decision_code: '', + reviewer_ref: '', reviewed_at: null, updated_at: now, + }], total: 1, limit: 20, offset: 0, + } + const batchPage: ReleaseBatchPage = { + items: [{ id: 'rb-safe', name: 'Synthetic batch', state: 'draft', policy_version: 'release-gate-v1', created_by: 'engineer-1', sealed_by: '', item_count: 1, created_at: now, sealed_at: null }], + total: 1, limit: 20, offset: 0, + } + const batchDetail: ReleaseBatchDetail = { ...batchPage.items[0], items: [{ publication_id: 'tp-safe', title: 'Synthetic', decided_visibility: 'internal', action: 'delete' }] } + const executionPage: ReleaseExecutionPage = { + items: [{ id: 'rx-auto', release_batch_id: 'rb-safe', operation: 'rollback', prior_execution_id: 'rx-safe', state: 'running', target_environment: 'production', authorized_by: 'system:auto-rollback', job_id: 'job-auto', item_count: 1, batch_size: 50, current_batch: 1, total_batches: 1, completed_item_count: 0, failed_item_count: 0, error_code: '', created_at: now, started_at: now, completed_at: null }, { id: 'rx-safe', release_batch_id: 'rb-safe', operation: 'publish', prior_execution_id: '', state: 'failed', target_environment: 'production', authorized_by: 'owner', job_id: 'job-safe', item_count: 2, batch_size: 50, current_batch: 1, total_batches: 1, completed_item_count: 1, failed_item_count: 1, error_code: 'release_item_failed', created_at: priorNow, started_at: priorNow, completed_at: priorNow }], + total: 2, limit: 20, offset: 0, + } + const executionDetail: ReleaseExecutionDetail = { ...executionPage.items[0], items: [{ publication_id: 'tp-safe', title: 'Synthetic', action: 'delete', state: 'verified', previous_production_revision: 'prod-v0', production_revision: 'absent', error_code: '', created_at: now, completed_at: now }, { publication_id: 'tp-restored', title: 'Restored', action: 'delete', state: 'rolled_back', previous_production_revision: 'prod-v0', production_revision: 'prod-v0', error_code: '', created_at: now, completed_at: now }] } + const releasePlan: ReleasePlan = { items: [{ publication_id: 'tp-safe', title: 'B.pdf', target_scope: 'external', action: 'add', test_state: 'ready', production_state: 'absent', can_release: true, blocker: '' }, { publication_id: 'tp-delete', title: 'C.pdf', target_scope: 'internal', action: 'delete', test_state: 'not_required', production_state: 'present', can_release: true, blocker: '' }, { publication_id: '', title: 'D.pdf', target_scope: 'internal', action: 'blocked', test_state: 'ready', production_state: 'present', can_release: false, blocker: 'source_unavailable' }], summary: { add: 1, update: 0, delete: 1, blocked: 1, unchanged: 11284 } } + const html = renderToStaticMarkup() + for (const text of ['来源与目录 → 测试知识库 → 生产知识库','自动发布计划','新增 1','更新 0','删除 1','未变化 11284','B.pdf','C.pdf','D.pdf','将从生产检索中移除','我确认所选删除项会停止生产检索','已选择删除：1（待确认）','已选 1 项（删除待确认）','为什么按钮不能点：请先填写操作人姓名或工号；请确认所选删除项会停止生产检索；至少 1 项动作通过确认；请填写批次名称','至少 1 项动作通过确认','来源当前不可用，保留生产知识','正在执行自动回滚','批次 1 / 1','每批最多 50 项','Snapshot #20260906-090000Z','审核要发布的知识', '需人工确认', '批准', '驳回', '创建草稿批次', '锁定清单（不会上线）', 'Synthetic batch', '删除生产知识', '已从生产检索中移除', '已恢复生产知识', '仍可检查调整', 'Release Worker保持关闭', '锁定清单 ≠ 上线', '生产确认、发布验证与回滚', '生产操作权限：已阻断', '生产请求门禁已开启', '快照恢复验证未通过', '生产发布已阻断']) { + expect(html).toContain(text) + } + for (const text of [ + '从检查到发布，只走这 8 步', + '1. 查看来源变化', + '2. 确认测试结果', + '3. 审核发布内容', + '4. 创建发布批次', + '5. 锁定发布清单', + '6. 生产负责人确认', + '7. 发布并验证', + '8. 必要时回滚', + '当前在第 3 步，共 8 步', + '现在要做：审核 1 项知识', + '还有 1 项等待审核；已批准文件可先创建批次', + '完成本步的 4 件事', + '填写操作人姓名或工号', + '只审核本次准备发布的文件', + '勾选要发布的已批准知识', + '填写批次名称并创建草稿', + '现在先做：填写操作人姓名或工号，逐条选择“批准”或“驳回”', + '完成后进入：第 4 步，创建发布批次', + '为什么按钮不能点：请先填写操作人姓名或工号', + '如何决定批准或驳回', + '可以批准：基础测试通过，你已确认搜索结果相关，而且发布范围正确', + '应该驳回：内容错误、过期、敏感、搜索结果不相关，或者你无法确认', + '创建批次还需要', + '确认所选发布动作', + '查看审核与发布说明', + '至少 1 项动作通过确认', + '填写批次名称', + '普通业务操作，不会影响生产', + '生产危险操作', + ]) { + expect(html).toContain(text) + } + expect(html).not.toContain('>授权生产发布') + expect(html).not.toContain('knowledge_id') + expect(html).not.toContain('acceptance_run_id') + expect(html).toContain('aria-label="选择删除C.pdf"') + expect(html).toContain('筛选 3 / 全部 3') + expect(html).toContain('aria-label="确认删除生产知识"') + expect(html).toContain('class="plan-delete"') + expect(html).toContain('至少 1 项动作通过确认') + expect(reviewGateLabel({ review_state: 'approved', gate_status: 'ready' })).toBe('测试已通过') + expect(reviewGateLabel({ review_state: 'pending', gate_status: 'ready' })).toBe('可以审核') + for (const technicalValue of ['tp-safe', 'rb-safe', 'rx-safe', 'job-safe', 'release-gate-v1', 'prod-v0', 'prod-v1', '>verified<']) { + expect(html).not.toContain(technicalValue) + } + expect(html).not.toContain('工程师标识') + }) +}) + +describe('release plan pagination (R3)', () => { + const baseProps = { + reviewPage: { items: [], total: 0, limit: 20, offset: 0 } as any, + reviewLoading: false, reviewError: null, reviewingId: null, + batchPage: { items: [], total: 0, limit: 20, offset: 0 } as any, + batchDetail: null, batchLoading: false, batchError: null, mutatingBatchId: null, + onReview: vi.fn(), onReviewRefresh: vi.fn(), onCreateBatch: vi.fn(), onSealBatch: vi.fn(), onSelectBatch: vi.fn(), onBatchRefresh: vi.fn(), + releaseCapabilities: { production_release_requests_enabled: false, production_backup_restore_verified: false }, + executionPage: { items: [], total: 0, limit: 20, offset: 0 } as any, + executionDetail: null, executionLoading: false, executionError: null, onSelectExecution: vi.fn(), onExecutionRefresh: vi.fn(), + selectedDeleteIDs: [] as string[], onToggleDelete: vi.fn(), + } + const planWith = (count: number): ReleasePlan => ({ + items: Array.from({ length: count }, (_, i) => ({ publication_id: `tp-${i}`, title: `文档${i}.pdf`, target_scope: 'internal' as const, action: 'add' as const, test_state: 'ready', production_state: 'absent', can_release: true, blocker: '' })), + summary: { add: count, update: 0, delete: 0, blocked: 0, unchanged: 0 }, + }) + it('renders at most one page (25 rows) instead of the full 618-row plan', () => { + const html = renderToStaticMarkup() + expect(html).toContain('筛选 618 / 全部 618') + expect(html).toContain('第 1 / 25 页 · 每页 25 条') + expect(html.match(/class="release-plan-table"/g)?.length ?? 0).toBeLessThanOrEqual(1) + // Only page-1 titles render; a page-2 row and a late row are absent from the DOM. + expect(html).toContain('文档0.pdf') + expect(html).not.toContain('文档25.pdf') + expect(html).not.toContain('文档617.pdf') + expect(html).toContain('仅可发布') + expect(html).toContain('下一页') + }) + it('keeps small plans pagination-free and pages exactly at the 25-item boundary', () => { + const small = renderToStaticMarkup() + expect(small).toContain('筛选 3 / 全部 3') + expect(small).not.toContain('上一页') + const exact = renderToStaticMarkup() + expect(exact).not.toContain('上一页') + expect(exact).toContain('文档24.pdf') + const over = renderToStaticMarkup() + expect(over).toContain('第 1 / 2 页 · 每页 25 条') + expect(over).not.toContain('文档25.pdf') + const huge = renderToStaticMarkup() + expect(huge.match(/ { + const baseProps = { + reviewPage: { items: [], total: 0, limit: 20, offset: 0 } as any, + reviewLoading: false, reviewError: null, reviewingId: null, + batchPage: { items: [], total: 0, limit: 20, offset: 0 } as any, + batchDetail: null, batchLoading: false, batchError: null, mutatingBatchId: null, + onReview: vi.fn(), onReviewRefresh: vi.fn(), onCreateBatch: vi.fn(), onSealBatch: vi.fn(), onSelectBatch: vi.fn(), onBatchRefresh: vi.fn(), + releaseCapabilities: { production_release_requests_enabled: true, production_backup_restore_verified: false }, + releasePlan: { items: [], summary: { add: 0, update: 0, delete: 0, blocked: 0, unchanged: 0 } } as ReleasePlan, + executionDetail: null, executionLoading: false, executionError: null, onSelectExecution: vi.fn(), onExecutionRefresh: vi.fn(), + selectedDeleteIDs: [] as string[], onToggleDelete: vi.fn(), + } + const exec = (over: Record) => ({ + id: 'rx-1', release_batch_id: 'rb-1', operation: 'rollback', prior_execution_id: 'rx-0', + state: 'rolling_back', target_environment: 'production', authorized_by: 'system:auto-rollback', job_id: 'job-1', + item_count: 2, batch_size: 50, current_batch: 1, total_batches: 1, completed_item_count: 1, failed_item_count: 0, + error_code: '', failure_level: '', restore_trigger: 'auto_rollback_after_failure', + created_at: '2026-09-12T09:00:00Z', started_at: '2026-09-12T09:01:00Z', completed_at: null, ...over, + }) as any + it('labels the compensating stage and the escalation after a failed rollback', () => { + const html = renderToStaticMarkup() + expect(html).toContain('正在回滚') + expect(html).toContain('正在执行自动回滚') + const failed = renderToStaticMarkup() + expect(failed).toContain('回滚失败') + expect(failed).toContain('整体恢复') + const done = renderToStaticMarkup() + expect(done).toContain('已回滚') + expect(done).toContain('已恢复') + const graded = renderToStaticMarkup() + expect(graded).toContain('本批后续已停止') + }) +}) diff --git a/web/src/App.tsx b/web/src/App.tsx new file mode 100644 index 0000000..cef671a --- /dev/null +++ b/web/src/App.tsx @@ -0,0 +1,1576 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { DocumentWorkbench } from './DocumentWorkbench' +import { SourceCatalogBrowser } from './SourceCatalogBrowser' +import { WeKnoraMonitor } from './WeKnoraMonitor' +import { SourceLinkManager } from './SourceLinkManager' +import { SystemSettings } from './SystemSettings' +import { WorkspaceContextTabsBar, WorkspaceContextTabsSetter } from './workspaceContext' +import type { WorkspaceContextTab } from './workspaceContext' + +import { fetchSourceAssets } from './assets' +import type { LegacyStatus, SourceAssetPage } from './assets' +import { fetchJobs } from './jobs' +import type { Job, JobState } from './jobs' +import { confirmReleaseChangeSet, createReleaseBatch, fetchLatestReleaseChangeSet, fetchReleaseAuditChain, fetchReleaseBatch, fetchReleaseBatches, fetchReleaseCapabilities, fetchReleaseExecution, fetchReleaseExecutions, fetchReleasePlan, fetchReviewCandidates, generateReleaseChangeSet, reviewPublication, sealReleaseBatch } from './reviewRelease' +import type { ReleaseAuditChain, ReleaseBatchDetail, ReleaseBatchPage, ReleaseCapabilities, ReleaseChangeSetDetail, ReleaseExecutionDetail, ReleaseExecutionPage, ReleasePlan, ReleasePlanItem, ReviewCandidatePage, Visibility } from './reviewRelease' +import { fetchSourceConnections, fetchSourceScanDetail, fetchSourceScans, triggerSourceScan } from './sourceConnections' +import type { SourceConnectionPage, SourceScanDetail, SourceScanPage } from './sourceConnections' +import { fetchTestPublications, retryPublication } from './testPublications' +import type { ProcessingState, TestPublicationPage } from './testPublications' +import { closeModule, initialWorkspace, openModule, restoreWorkspace } from './workspace' +import type { ModuleId, WorkspaceState } from './workspace' + +const workspaceStorageKey = 'knowledge-governance.workspace.v1' + +const modules: ReadonlyArray<{ id: ModuleId; label: string; icon: string }> = [ + { id: 'dashboard', label: '来源与目录', icon: '◫' }, + { id: 'tasks', label: '解析监控', icon: '↻' }, + { id: 'releases', label: '发布管理', icon: '⇧' }, + { id: 'settings', label: '系统设置', icon: '⚙' }, +] + +interface WorkspaceShellProps { + workspace: WorkspaceState + onOpen: (moduleId: ModuleId) => void + onClose: (moduleId: ModuleId) => void + children: React.ReactNode +} + +export function WorkspaceShell(props: WorkspaceShellProps) { + const [navigationOpen, setNavigationOpen] = useState(false) + const [contextTabs, setContextTabs] = useState([]) + + const activeModule = modules.find((item) => item.id === props.workspace.activeTab) + + return ( + +
+ + {navigationOpen ? ( + +
+ 知识治理 + + {activeModule?.label ?? '工作台'} +
+
+ {props.workspace.openTabs.map((tabId) => { + const module = modules.find((item) => item.id === tabId) + if (!module) return null + const active = props.workspace.activeTab === tabId + return ( +
+ + {tabId !== 'dashboard' ? ( + + ) : null} +
+ ) + })} +
+ + +
{props.children}
+ +
+ + ) +} + +const stateLabels: Record = { + pending: '待处理', + running: '运行中', + retry: '待重试', + completed: '已完成', + failed: '失败', +} + +interface JobDashboardProps { + jobs: Job[] + loading: boolean + error: string | null + onRefresh: () => void +} + +interface ModulePageProps extends JobDashboardProps { + moduleId: ModuleId + connectionPage?: SourceConnectionPage + connectionLoading?: boolean + connectionError?: string | null + onConnectionRefresh?: () => void + scanningConnectionId?: string | null + onTriggerSourceScan?: (id: string) => void + scanPage?: SourceScanPage + scanDetail?: SourceScanDetail | null + scanLoading?: boolean + scanError?: string | null + selectedScanId?: string | null + onSelectScan?: (connectionID: string, scanID: string) => void + assetPage?: SourceAssetPage + assetLoading?: boolean + assetError?: string | null + assetState?: LegacyStatus | '' + onAssetStateChange?: (state: LegacyStatus | '') => void + onAssetPageChange?: (offset: number) => void + onAssetRefresh?: () => void + publicationPage?: TestPublicationPage + publicationLoading?: boolean + publicationError?: string | null + publicationState?: ProcessingState | '' + retryingId?: string | null + onPublicationStateChange?: (state: ProcessingState | '') => void + onPublicationPageChange?: (offset: number) => void + onPublicationRefresh?: () => void + onRetry?: (id: string) => void + reviewPage?: ReviewCandidatePage + reviewLoading?: boolean + reviewError?: string | null + reviewingId?: string | null + batchPage?: ReleaseBatchPage + batchDetail?: ReleaseBatchDetail | null + batchLoading?: boolean + batchError?: string | null + mutatingBatchId?: string | null + onReview?: ReleaseManagementProps['onReview'] + onReviewRefresh?: () => void + onCreateBatch?: ReleaseManagementProps['onCreateBatch'] + onSealBatch?: ReleaseManagementProps['onSealBatch'] + onSelectBatch?: ReleaseManagementProps['onSelectBatch'] + onBatchRefresh?: () => void + releaseCapabilities?: ReleaseCapabilities + releasePlan?: ReleasePlan + releaseChangeSet?: ReleaseChangeSetDetail | null + changeSetBusy?: boolean + changeSetError?: string | null + onGenerateChangeSet?: (name: string, createdBy: string) => void + onConfirmChangeSet?: (changeSetID: string, confirmedBy: string) => void + selectedDeleteIDs?: string[] + onToggleDelete?: (publicationID: string) => void + executionPage?: ReleaseExecutionPage + executionDetail?: ReleaseExecutionDetail | null + executionChain?: ReleaseAuditChain | null + executionLoading?: boolean + executionError?: string | null + onSelectExecution?: (executionID: string) => void + onExecutionRefresh?: () => void +} + +const emptyAssetPage: SourceAssetPage = { items: [], summary: { total: 0, completed: 0, content_alias: 0, over_limit: 0 }, limit: 50, offset: 0 } + +const emptyConnectionPage: SourceConnectionPage = { items: [], summary: { total: 0, enabled: 0, disabled: 0, healthy: 0, degraded: 0, unhealthy: 0, unknown: 0 } } + +const emptyScanPage: SourceScanPage = { items: [], total: 0, limit: 20, offset: 0 } + +const emptyPublicationPage: TestPublicationPage = { items: [], total: 0, limit: 50, offset: 0 } + +const emptyReviewPage: ReviewCandidatePage = { items: [], total: 0, limit: 20, offset: 0 } + +const emptyBatchPage: ReleaseBatchPage = { items: [], total: 0, limit: 20, offset: 0 } +const emptyExecutionPage: ReleaseExecutionPage = { items: [], total: 0, limit: 20, offset: 0 } +const disabledReleaseCapabilities: ReleaseCapabilities = { production_release_requests_enabled: false, production_backup_restore_verified: false } +const emptyReleasePlan: ReleasePlan = { items: [], summary: { add: 0, update: 0, delete: 0, blocked: 0, unchanged: 0 } } + +export function ModulePage({ moduleId, jobs, loading, error, onRefresh, connectionPage = emptyConnectionPage, connectionLoading = false, connectionError = null, onConnectionRefresh = () => undefined, scanningConnectionId = null, onTriggerSourceScan = () => undefined, scanPage = emptyScanPage, scanDetail = null, scanLoading = false, scanError = null, selectedScanId = null, onSelectScan = () => undefined, assetPage = emptyAssetPage, assetLoading = false, assetError = null, assetState = '', onAssetStateChange = () => undefined, onAssetPageChange = () => undefined, onAssetRefresh = () => undefined, publicationPage = emptyPublicationPage, publicationLoading = false, publicationError = null, publicationState = '', retryingId = null, onPublicationStateChange = () => undefined, onPublicationPageChange = () => undefined, onPublicationRefresh = () => undefined, onRetry = () => undefined, reviewPage = emptyReviewPage, reviewLoading = false, reviewError = null, reviewingId = null, batchPage = emptyBatchPage, batchDetail = null, batchLoading = false, batchError = null, mutatingBatchId = null, onReview = () => undefined, onReviewRefresh = () => undefined, onCreateBatch = () => undefined, onSealBatch = () => undefined, onSelectBatch = () => undefined, onBatchRefresh = () => undefined, releaseCapabilities = disabledReleaseCapabilities, releasePlan = emptyReleasePlan, releaseChangeSet = null, changeSetBusy = false, changeSetError = null, onGenerateChangeSet = () => undefined, onConfirmChangeSet = () => undefined, selectedDeleteIDs = [], onToggleDelete = () => undefined, executionPage = emptyExecutionPage, executionDetail = null, executionChain = null, executionLoading = false, executionError = null, onSelectExecution = () => undefined, onExecutionRefresh = () => undefined }: ModulePageProps) { + if (moduleId === 'tasks') { + return + } + if (moduleId === 'knowledge') { + return + } + if (moduleId === 'sources') { + return + } + if (moduleId === 'releases') { + return + } + if (moduleId === 'settings') { + return + } + const counts = { + total: jobs.length, + running: jobs.filter((job) => job.state === 'running').length, + retry: jobs.filter((job) => job.state === 'retry').length, + failed: jobs.filter((job) => job.state === 'failed').length, + completed: jobs.filter((job) => job.state === 'completed').length, + } + + return ( +
+
+
+

P2 · 知识治理总览

+

治理工作台

+

集中查看任务状态与平台安全边界。

+
+ +
+ {error ?
{error}
: null} +
+ + + + + +
+
+
+ 运行安全 + PP历史来源清单已接管；本地测试WeKnora已连接，生产WeKnora仍未连接。 +
+ Release Worker 默认关闭 +
+
+
+
+

最近任务

+

显示最近5条真实队列记录

+
+
+ {!loading && !error && jobs.length === 0 ? ( +
暂无任务。可前往“同步任务”查看队列详情。
+ ) : null} + {jobs.length > 0 ? ( +
+ {jobs.slice(0, 5).map((job) => ( +
+
+ {job.type} + {job.id} +
+ {stateLabels[job.state]} + +
+ ))} +
+ ) : null} +
+
+ ) +} + +interface KnowledgeInventoryProps { + page: SourceAssetPage + loading: boolean + error: string | null + state: LegacyStatus | '' + onStateChange: (state: LegacyStatus | '') => void + onPageChange: (offset: number) => void + onRefresh: () => void +} + +const legacyStatusLabels: Record = { + completed: '历史已同步', + content_alias: '内容别名', + over_limit: '超过旧同步上限', +} + +const sourceTypeLabels: Record = { + pp: 'PP', + alist: 'AList', + dingtalk: '钉钉', + manual: '手工验收', +} + +const sourceHealthLabels: Record = { + unknown: '未检查', + healthy: '健康', + degraded: '降级', + unhealthy: '异常', +} + +interface SourceConnectionsProps { + page: SourceConnectionPage + loading: boolean + error: string | null + scanningConnectionId: string | null + scanPage: SourceScanPage + scanDetail: SourceScanDetail | null + scanLoading: boolean + scanError: string | null + selectedScanId: string | null + onRefresh: () => void + onTriggerScan: (id: string) => void + onSelectScan: (connectionID: string, scanID: string) => void +} + +const sourceScanLabels: Record = { + pending: '待扫描', + running: '扫描中', + completed: '已完成', + failed: '失败', +} + +const sourceScanChangeLabels: Record = { + added: '新增内容', + updated: '内容更新', + unavailable: '来源不可用', +} + +export function SourceConnections({ page, loading, error, scanningConnectionId, scanPage, scanDetail, scanLoading, scanError, selectedScanId, onRefresh, onTriggerScan, onSelectScan }: SourceConnectionsProps) { + const attention = page.summary.degraded + page.summary.unhealthy + const scannableConnectionID = page.items.find((item) => item.source_type === 'pp')?.id ?? '' + return ( +
+
+
+

连接情况

+

来源连接

+

查看来源是否可读、多久检查一次。日常文件操作请进入“来源与目录”。

+
+ +
+ {error ?
{error}
: null} +
+ + + + + +
+
+
+

来源连接

PP检查的是本机已保存快照，不是实时拉取网站。检查不会修改来源，也不会自动上传或删除文件。

+ 来源只读 +
+ {!loading && !error && page.items.length === 0 ?
尚无来源连接。
: null} + {page.items.length > 0 ? ( +
+ {page.items.map((item) => { + const activeScan = item.last_scan?.state === 'pending' || item.last_scan?.state === 'running' + return + + + + + + + + + + })} +
连接类型状态调度健康对象数最近扫描操作
{item.name.startsWith('PP legacy ') ? 'PP公开文档' : item.name}{sourceTypeLabels[item.source_type] ?? item.source_type}{item.enabled ? '已启用' : '已停用'}{item.schedule_configured ? '已配置' : '未配置'}{sourceHealthLabels[item.health_status] ?? item.health_status}{item.item_count}{item.last_scan ?
+ 最近扫描 · {sourceScanLabels[item.last_scan.state] ?? item.last_scan.state} + 观察 {item.last_scan.observed_items} · 未变化 {item.last_scan.unchanged_items} + 新增 {item.last_scan.added_items} · 更新 {item.last_scan.updated_items} · 缺失 {item.last_scan.missing_items} + 超过上限 {item.last_scan.over_limit_items} · 待上传 {item.last_scan.planned_uploads} + {item.last_scan.error_code ? {item.last_scan.error_code} : null} +
: '尚未扫描'}		{item.source_type === 'pp' ? : '暂未支持检查'}
+ ) : null} +
+
查看检查历史与审计记录 +
+

扫描历史与变化记录

展示安全聚合和治理侧变化标识；不展示来源路径、修订值或快照指纹。

+ 审计只读 +
+ {scanError ?
{scanError}
: null} + {!scanLoading && !scanError && scanPage.items.length === 0 ?
当前来源尚无扫描历史。
: null} + {scanPage.items.length > 0 ?
+
+ {scanPage.items.map((scan) => + + + + + + + )} +
扫描状态观察变化上限阻断操作
{scan.id}{sourceScanLabels[scan.state] ?? scan.state}{scan.observed_items}新增 {scan.added_items} · 更新 {scan.updated_items} · 缺失 {scan.missing_items}{scan.over_limit_items}
+
+

变化记录

+ {scanLoading ?
加载扫描记录中…
: null} + {!scanLoading && !scanDetail ?
选择一条扫描查看变化。
: null} + {!scanLoading && scanDetail?.changes.length === 0 ?
本次扫描未发现新增、更新或不可用变化。
: null} + {!scanLoading && scanDetail && scanDetail.changes.length > 0 ?
+ {scanDetail.changes.map((change) => + + + + + )} +
类型治理资产大小记录时间
{sourceScanChangeLabels[change.change_type] ?? change.change_type}{change.source_item_id ?? change.id}{change.observed_size_bytes === null ? '—' : formatBytes(change.observed_size_bytes)}{formatTime(change.created_at)}
: null} +
+
: null} +
+
+ ) +} + +function sourceHealthClass(status: string) { + if (status === 'healthy') return 'completed' + if (status === 'unhealthy') return 'failed' + if (status === 'degraded') return 'retry' + return 'pending' +} + +export function KnowledgeInventory({ page, loading, error, state, onStateChange, onPageChange, onRefresh }: KnowledgeInventoryProps) { + return ( +
+
+
+

P2 · 统一治理清单

+

知识资产

+

统一展示各类治理来源；清单只读，不会修改或重新上传来源文件。

+
+ +
+ {error ?
{error}
: null} +
+ + + + +
+
+
+

统一来源清单

按当前治理版本分页读取，超限项保持阻断。

+ +
+ {!loading && !error && page.items.length === 0 ?
当前筛选没有资产。
: null} + {page.items.length > 0 ? ( +
+ {page.items.map((item) => + + + + )} +
资产ID来源大小历史状态可见性测试处理更新时间
{item.id}{sourceTypeLabels[item.source_type] ?? item.source_type}{formatBytes(item.size_bytes)}{item.legacy_status ? legacyStatusLabels[item.legacy_status] : item.content_state}{item.visibility === 'external' ? '对外' : '对内'}{item.has_publication ? item.processing_state : '阻断'}{formatTime(item.updated_at)}
+ ) : null} +
第 {Math.floor(page.offset / page.limit) + 1} 页
+ + +
+
+
+ ) +} + +function formatBytes(value: number) { + if (value >= 1024 * 1024 * 1024) return `${(value / (1024 * 1024 * 1024)).toFixed(1)} GiB` + if (value >= 1024 * 1024) return `${(value / (1024 * 1024)).toFixed(1)} MiB` + if (value >= 1024) return `${(value / 1024).toFixed(1)} KiB` + return `${value} B` +} + +export function JobDashboard({ jobs, loading, error, onRefresh }: JobDashboardProps) { + const counts = useMemo( + () => ({ + total: jobs.length, + running: jobs.filter((job) => job.state === 'running').length, + retry: jobs.filter((job) => job.state === 'retry').length, + failed: jobs.filter((job) => job.state === 'failed').length, + completed: jobs.filter((job) => job.state === 'completed').length, + }), + [jobs], + ) + + return ( +
+
+
+

P1 · 任务编排基础

+

同步任务

+

查看持久队列、执行状态、重试次数和最近错误。

+
+ +
+ +
+ + + + + +
+ +
+
+
+

最近任务

+

按创建时间倒序显示最近50条任务

+
+ Release Worker 默认关闭 +
+ + {error ?
{error}
: null} + {!loading && !error && jobs.length === 0 ? ( +
暂无任务。P1可通过API创建demo任务验证队列。
+ ) : null} + {jobs.length > 0 ? ( +
+ + + + + + + + + + + + + {jobs.map((job) => ( + + + + + + + + + ))} + +
任务ID类型状态尝试更新时间错误
{job.id}{job.type}{stateLabels[job.state]}{job.attempt}/{job.max_attempts}{formatTime(job.updated_at)}{job.last_error || '—'}
+
+ ) : null} +
+
+ ) +} + +const processingLabels: Record = { + pending: '待处理', + processing: '解析中', + finalizing: '定稿中', + completed: '已完成', + failed: '失败', + stalled: '停滞', +} + +interface TestPublicationsProps { + page: TestPublicationPage + loading: boolean + error: string | null + state: ProcessingState | '' + retryingId: string | null + onStateChange: (state: ProcessingState | '') => void + onPageChange: (offset: number) => void + onRefresh: () => void + onRetry: (id: string) => void +} + +interface ReleaseManagementProps { + reviewPage: ReviewCandidatePage + reviewLoading: boolean + reviewError: string | null + reviewingId: string | null + batchPage: ReleaseBatchPage + batchDetail: ReleaseBatchDetail | null + batchLoading: boolean + batchError: string | null + mutatingBatchId: string | null + onReview: (publicationID: string, decision: 'approved' | 'rejected', visibility: Visibility, reviewerRef: string, decisionCode: string) => void + onReviewRefresh: () => void + onCreateBatch: (name: string, createdBy: string, publicationIDs: string[], deletePublicationIDs: string[]) => void + onSealBatch: (batchID: string, sealedBy: string) => void + onSelectBatch: (batchID: string) => void + onBatchRefresh: () => void + releaseCapabilities: ReleaseCapabilities + releasePlan: ReleasePlan + releaseChangeSet?: ReleaseChangeSetDetail | null + changeSetBusy?: boolean + changeSetError?: string | null + onGenerateChangeSet?: (name: string, createdBy: string) => void + onConfirmChangeSet?: (changeSetID: string, confirmedBy: string) => void + selectedDeleteIDs: string[] + onToggleDelete: (publicationID: string) => void + executionPage: ReleaseExecutionPage + executionDetail: ReleaseExecutionDetail | null + executionChain?: ReleaseAuditChain | null + executionLoading: boolean + executionError: string | null + onSelectExecution: (executionID: string) => void + onExecutionRefresh: () => void +} + +const acceptanceLabels: Record = { + pending: '待验收', + passed: '通过', + failed: '失败', + manual_required: '需人工确认', +} + +const reviewLabels: Record = { + pending: '待审核', + approved: '已批准', + rejected: '已驳回', +} + +const batchLabels: Record = { + draft: '草稿', + sealed: '清单已锁定', + cancelled: '已取消', +} + +const executionLabels: Record = { + pending: '待执行', + running: '执行中', + completed: '已完成', + failed: '失败', + rolling_back: '正在回滚', + rolled_back: '已回滚', + rollback_failed: '回滚失败', +} + +const snapshotLabel = (createdAt: string) => { + const value = new Date(createdAt) + if (Number.isNaN(value.getTime())) return 'Snapshot #已记录' + return `Snapshot #${value.toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15)}Z` +} + +const executionStateDetail = (execution: ReleaseExecutionPage['items'][number], executions: ReleaseExecutionPage['items']) => { + const automaticRollback = execution.operation === 'rollback' && execution.authorized_by === 'system:auto-rollback' + const snapshotSource = automaticRollback ? executions.find((candidate) => candidate.id === execution.prior_execution_id) : execution + const snapshot = snapshotLabel(snapshotSource?.created_at ?? execution.created_at) + if (automaticRollback && (execution.state === 'rolled_back' || execution.state === 'completed')) return `已恢复 ${snapshot}` + if (automaticRollback && (execution.state === 'pending' || execution.state === 'running' || execution.state === 'rolling_back')) return `正在执行自动回滚 · ${snapshot}` + if (automaticRollback && (execution.state === 'failed' || execution.state === 'rollback_failed')) return `自动回滚失败 · ${snapshot}（发布已升级为整体恢复，需快照还原授权）` + if (execution.operation === 'publish' && execution.state === 'pending') return '等待创建生产快照' + if (execution.operation === 'publish' && (execution.state === 'running' || execution.state === 'rolling_back')) return `正在同步 ${execution.completed_item_count} / ${execution.item_count}` + if (execution.operation === 'publish' && execution.failure_level === 'batch') return '本批后续已停止，正在补偿本批已发布项' + if (execution.operation === 'publish' && execution.failure_level === 'single_item') return '单项失败，后续项未推进' + if (execution.operation === 'publish' && execution.failure_level === 'overall') return '需整体快照恢复（等待授权）' + return '' +} + +const releaseItemLabels: Record = { + pending: '等待开始', + staged: '内容已准备', + activated: '已切换，等待验证', + verified: '发布并验证通过', + failed: '发布失败', + rolled_back: '已恢复上一版本', +} + +const releaseItemResult = (item: ReleaseExecutionDetail['items'][number]) => { + if (item.action === 'delete' && item.state === 'verified' && item.production_revision === 'absent') return { detail: '删除并验证通过', outcome: '已从生产检索中移除' } + if (item.action === 'delete' && item.state === 'rolled_back') return { detail: '删除操作已回滚', outcome: '已恢复生产知识' } + return { detail: releaseItemLabels[item.state] ?? item.state, outcome: item.state === 'verified' ? '生产可用' : item.state === 'rolled_back' ? '已恢复' : '处理中' } +} + +export const reviewGateLabel = (item: Pick) => item.review_state === 'approved' && item.gate_status === 'ready' ? '测试已通过' : item.gate_status === 'ready' ? '可以审核' : '暂时不能审核' + +function releaseFailureMessage(code: string) { + if (!code) return '' + const labels: Record = { + release_contract_mismatch: '目标环境与测试环境不一致，已停止发布', + release_verification_failed: '发布后的检索验证未通过', + release_item_failed: '部分知识发布失败', + release_retryable: '遇到临时故障，系统将自动重试', + release_manifest_mismatch: '发布结果与冻结的发布清单不一致，已降级为失败', + release_rollback_failed: '自动回滚失败，需整体快照恢复（等待授权）', + } + return labels[code] ?? '执行失败，请联系管理员查看技术记录' +} + +const releaseFlowSteps = [ + { title: '1. 查看来源变化', kind: '只读查看' }, + { title: '2. 确认测试结果', kind: '只读查看' }, + { title: '3. 审核发布内容', kind: '普通业务操作' }, + { title: '4. 创建发布批次', kind: '普通业务操作' }, + { title: '5. 锁定发布清单', kind: '普通业务操作' }, + { title: '6. 生产负责人确认', kind: '生产危险操作' }, + { title: '7. 发布并验证', kind: '生产危险操作' }, + { title: '8. 必要时回滚', kind: '生产危险操作' }, +] as const + +function releaseGuidance(props: ReleaseManagementProps, pending: number) { + const approved = props.reviewPage.items.filter((item) => item.review_state === 'approved' && item.gate_status === 'ready').length + const drafts = props.batchPage.items.filter((batch) => batch.state === 'draft').length + const sealed = props.batchPage.items.filter((batch) => batch.state === 'sealed').length + const activeExecutions = props.executionPage.items.filter((execution) => execution.state === 'pending' || execution.state === 'running').length + const completedExecutions = props.executionPage.items.filter((execution) => execution.state === 'completed').length + + if (pending > 0) return { + step: 3, + title: `现在要做：审核 ${pending} 项知识`, + reason: `还有 ${pending} 项等待审核；已批准文件可先创建批次`, + action: '填写操作人姓名或工号，逐条选择“批准”或“驳回”', + after: '第 4 步，创建发布批次', + impact: '普通业务操作，不会影响生产', + href: '#review-actions', + button: '前往审核列表', + } + if (drafts > 0) return { + step: 5, + title: `现在要做：锁定 ${drafts} 个草稿批次`, + reason: '暂时不能继续：草稿内容还可以变化，发布清单尚未锁定', + action: '核对批次内容，然后点击“锁定清单（不会上线）”', + after: '第 6 步，等待生产负责人确认', + impact: '普通业务操作，不会影响生产', + href: '#batch-actions', + button: '查看草稿批次', + } + if (approved > 0 && props.batchPage.total === 0) return { + step: 4, + title: `现在要做：把 ${approved} 项已批准知识加入批次`, + reason: '暂时不能继续：已批准内容还没有组成一次发布清单', + action: '勾选已批准内容，填写批次名称并创建草稿', + after: '第 5 步，锁定本次发布清单', + impact: '普通业务操作，不会影响生产', + href: '#review-actions', + button: '创建发布批次', + } + if (sealed > 0 && props.executionPage.total === 0) return { + step: 6, + title: `等待生产负责人确认 ${sealed} 个已锁定批次`, + reason: !props.releaseCapabilities.production_release_requests_enabled + ? '暂时不能继续：生产发布功能处于安全锁定状态' + : !props.releaseCapabilities.production_backup_restore_verified + ? '暂时不能继续：生产快照恢复验证尚未通过' + : '暂时不能继续：生产负责人还没有确认本次上线', + action: '普通用户无需操作；生产负责人核对影响范围后单独确认', + after: '第 7 步，系统发布并验证', + impact: '生产危险操作，当前页面不能执行', + href: '#production-status', + button: '查看安全状态', + } + if (activeExecutions > 0) return { + step: 7, + title: `正在发布并验证 ${activeExecutions} 个批次`, + reason: '暂时不能继续：系统仍在复制并核对生产知识', + action: '等待系统完成；如果失败，请查看人类可读的失败原因', + after: '第 8 步，仅在发现问题时申请回滚', + impact: '生产危险操作，只读查看进度', + href: '#production-status', + button: '查看发布进度', + } + if (completedExecutions > 0) return { + step: 8, + title: '发布已经完成并验证', + reason: '当前没有必须处理的阻断', + action: '正常情况下无需操作；确认发布错误时再申请回滚', + after: '流程完成', + impact: '回滚是生产危险操作，必须由生产负责人再次确认', + href: '#production-status', + button: '查看发布结果', + } + return { + step: 2, + title: '现在要做：确认测试结果', + reason: '暂时不能继续：目前没有可以审核的已通过知识', + action: '前往“解析监控”查看处理结果，并核对搜索效果', + after: '第 3 步，审核发布内容', + impact: '只读查看，不会影响生产', + href: '#review-actions', + button: '查看当前结果', + } +} + +export function ReleaseManagement(props: ReleaseManagementProps) { + /* R3 pagination probe */ + const changeSet = props.releaseChangeSet ?? null + const changeSetBusy = props.changeSetBusy ?? false + const changeSetError = props.changeSetError ?? null + const onGenerateChangeSet = props.onGenerateChangeSet ?? (() => undefined) + const onConfirmChangeSet = props.onConfirmChangeSet ?? (() => undefined) + const [reviewerRef, setReviewerRef] = useState('') + const [batchName, setBatchName] = useState('') + const [selectedIDs, setSelectedIDs] = useState([]) + const [deleteConfirmed, setDeleteConfirmed] = useState(false) + const [visibilityByID, setVisibilityByID] = useState>({}) + const [planActionFilter, setPlanActionFilter] = useState<'' | 'add' | 'update' | 'delete' | 'blocked'>('') + const [planScopeFilter, setPlanScopeFilter] = useState<'' | Visibility>('') + const [planOnlyReleasable, setPlanOnlyReleasable] = useState(false) + const [planPage, setPlanPage] = useState(0) + const planPageSize = 25 + const ready = props.reviewPage.items.filter((item) => item.gate_status === 'ready').length + const pending = props.reviewPage.items.filter((item) => item.review_state === 'pending').length + const selectedCount = selectedIDs.length + props.selectedDeleteIDs.length + const confirmedSelection = selectedIDs.length > 0 || (props.selectedDeleteIDs.length > 0 && deleteConfirmed) + const guidance = releaseGuidance(props, pending) + const productionStatus = !props.releaseCapabilities.production_release_requests_enabled + ? '生产操作权限：已关闭' + : !props.releaseCapabilities.production_backup_restore_verified + ? '生产操作权限：已阻断' + : '生产操作权限：等待负责人确认' + const batchBlocker = [ + !reviewerRef ? '请先填写操作人姓名或工号' : '', + selectedCount === 0 ? '请先勾选已批准或待删除的知识' : '', + props.selectedDeleteIDs.length > 0 && !deleteConfirmed ? '请确认所选删除项会停止生产检索' : '', + selectedCount > 0 && !confirmedSelection ? '至少 1 项动作通过确认' : '', + !batchName.trim() ? '请填写批次名称' : '', + ].filter(Boolean).join('；') + + const toggleSelected = (publicationID: string) => { + setSelectedIDs((current) => current.includes(publicationID) ? current.filter((id) => id !== publicationID) : [...current, publicationID]) + } + + return
+
+
+

发布管理

+

测试知识库与生产知识库的同步。生产发布仍由独立门禁、备份和负责人确认保护。

+
+
+
+ +
{props.releaseCapabilities.production_backup_restore_verified ? '发布前将创建可恢复快照' : '生产发布当前不可执行'}{props.releaseCapabilities.production_backup_restore_verified ? '发布失败时系统会按已验证的快照执行回滚。' : '尚未通过生产备份恢复验证，当前只能审核和锁定发布清单。'}
+
查看审核与发布说明 +
+
操作顺序

从检查到发布，只走这 8 步

蓝色是当前步骤；绿色表示已完成；红色步骤会修改生产环境。

+
    + {releaseFlowSteps.map((item, index) => { + const number = index + 1 + const state = number < guidance.step ? 'done' : number === guidance.step ? 'current' : 'upcoming' + const danger = number >= 6 + return
  1. + {number < guidance.step ? '✓' : number} + {item.title} + {item.kind} +
    2. + })} +
+
+
+
当前在第 {guidance.step} 步，共 8 步
+
+

{guidance.title}

+ {guidance.reason} + {guidance.step === 3 ?
完成本步的 4 件事
  1. 填写操作人姓名或工号
  2. 只审核本次准备发布的文件
  3. 勾选要发布的已批准知识
  4. 填写批次名称并创建草稿
: null} +

{`现在先做：${guidance.action}`}

+

{`完成后进入：${guidance.after}`}

+ {guidance.impact} +
+ {guidance.step === 3 ? '滚动到审核列表' : guidance.button} +
+
+
+
{productionStatus}{props.releaseCapabilities.production_release_requests_enabled && !props.releaseCapabilities.production_backup_restore_verified ? '生产发布已阻断：先完成生产快照并通过恢复验证。' : '锁定清单只表示内容不再变化，不会自动上线；普通业务操作不会改动生产知识。'}
+ 修改生产内容：关闭 +
查看技术状态

生产请求门禁{props.releaseCapabilities.production_release_requests_enabled ? '已开启' : '关闭'}；快照恢复验证{props.releaseCapabilities.production_backup_restore_verified ? '已通过' : '未通过'}；Release Worker保持关闭。

+
+
+

自动发布计划

来源与目录 → 测试知识库 → 生产知识库。来源标注是最终 Truth Source。

按知识级别同步
+
{`新增 ${props.releasePlan.summary.add}`}{`更新 ${props.releasePlan.summary.update}`}{`删除 ${props.releasePlan.summary.delete}`}{`阻断 ${props.releasePlan.summary.blocked}`}{`未变化 ${props.releasePlan.summary.unchanged}`}
+ {(() => { + const filtered = props.releasePlan.items.filter((item) => { + if (planActionFilter === 'blocked' ? item.can_release : planActionFilter ? item.action !== planActionFilter : false) return false + if (planScopeFilter && item.target_scope !== planScopeFilter) return false + if (planOnlyReleasable && !item.can_release) return false + return true + }) + const pageCount = Math.max(1, Math.ceil(filtered.length / planPageSize)) + const safePage = Math.min(planPage, pageCount - 1) + const pageItems = filtered.slice(safePage * planPageSize, safePage * planPageSize + planPageSize) + const actionLabel = (item: ReleasePlanItem) => item.action === 'add' ? '新增' : item.action === 'update' ? '更新' : item.action === 'delete' ? '删除' : '等待测试' + return <> +
+ + + + {`筛选 ${filtered.length} / 全部 ${props.releasePlan.items.length}`} +
+ {pageItems.length ?
{pageItems.map((item, index) => )}
选择知识目标计划动作测试状态生产状态能否发布
{item.action === 'delete' && item.can_release ? { props.onToggleDelete(item.publication_id); setDeleteConfirmed(false) }} /> : '—'}{item.title}{item.target_scope === 'internal' ? '对内知识库' : '对外知识库'}{actionLabel(item)}{item.test_state === 'ready' ? '已通过' : item.test_state === 'not_required' ? '无需测试' : '未就绪'}{item.production_state === 'present' ? '已存在' : '不存在'}{item.can_release ? item.action === 'delete' ? '将从生产检索中移除' : '可加入发布批次' : item.blocker === 'source_unavailable' ? '来源当前不可用，保留生产知识' : '测试未就绪，当前阻断'}
: filtered.length ?
当前筛选没有发布变更。
:
来源、测试与生产状态一致，当前没有发布变更。
} + {pageCount > 1 ?
{`第 ${safePage + 1} / ${pageCount} 页 · 每页 ${planPageSize} 条`}
: null} + + })()} + {(() => { + const manifestShort = changeSet ? `${changeSet.manifest_sha256.slice(0, 8)}…${changeSet.manifest_sha256.slice(-6)}` : '' + const frozenActionLabel = (action: string) => action === 'add' ? '新增' : action === 'update' ? '更新' : action === 'delete' ? '删除' : action === 'blocked' ? '阻断' : '不变' + return
+

冻结发布计划集（R1）

生成即固化：来源刷新不会改变已确认清单；正式批次只能引用已确认版本。

+ {changeSet ? v{changeSet.number} · {changeSet.state === 'confirmed' ? '已确认' : changeSet.state === 'consumed' ? '已消费' : changeSet.state === 'cancelled' ? '已取消' : '待确认'} : 尚未生成} +
+ {changeSet ?
+ {`清单 ${changeSet.items.length} 项（新增${changeSet.add_count}·更新${changeSet.update_count}·删除${changeSet.delete_count}·阻断${changeSet.blocked_count}）`} + {`摘要 ${manifestShort}`} + {`绑定快照 ${changeSet.source_snapshot_ids.length} 个`} + {`生成人 ${changeSet.created_by} · ${new Date(changeSet.created_at).toLocaleString('zh-CN', { hour12: false })}`} + {changeSet.confirmed_by ? {`确认人 ${changeSet.confirmed_by} · ${changeSet.confirmed_at ? new Date(changeSet.confirmed_at).toLocaleString('zh-CN', { hour12: false }) : ''}`} : null} +
: null} + {changeSet && changeSet.state === 'planned' ?
查看冻结清单（{changeSet.items.length}项）
    {changeSet.items.map((item) =>
  • {`${item.seq}. ${item.title} · ${item.target_scope === 'internal' ? '对内' : '对外'} · ${frozenActionLabel(item.action)}${item.can_release ? '' : '（阻断：' + (item.blocker === 'source_unavailable' ? '来源不可用' : '测试未就绪') + '）'}`}
    • )}
: null} + {changeSetError ?

{changeSetError}

: null} +
+ + {changeSet && changeSet.state === 'planned' ? : null} + {changeSet && changeSet.state === 'confirmed' ? 创建批次将自动绑定 v{changeSet.number} 并校验清单 : 草稿批次需先由上方“确认冻结”固定目标} +
+
+ })()} + {props.selectedDeleteIDs.length > 0 ? : null} +
+
+ + + + 0 && !deleteConfirmed ? `已选择删除：${props.selectedDeleteIDs.length}（待确认）` : '已选入批次'} value={selectedCount} tone="cyan" /> +
+
+
+

第 3 步：审核要发布的知识

先确认测试结果和对内/对外范围，再批准或驳回。批准后才能勾选加入批次。

+ +
+ {batchBlocker ?
{`为什么按钮不能点：${batchBlocker}`}
: null} +
+

如何决定批准或驳回

+

可以批准：基础测试通过，你已确认搜索结果相关，而且发布范围正确

+

应该驳回：内容错误、过期、敏感、搜索结果不相关，或者你无法确认

+
+ {props.reviewError ?
{props.reviewError}
: null} + {!props.reviewLoading && !props.reviewError && props.reviewPage.items.length === 0 ?
当前没有带最新验收结果的审核候选。
: null} + {props.reviewPage.items.length > 0 ?
+ {props.reviewPage.items.map((item) => { + const visibility = visibilityByID[item.publication_id] ?? (item.decided_visibility || item.current_visibility) + const selectable = item.review_state === 'approved' && item.gate_status === 'ready' + return + + + + + + + + + })} +
选择知识内容测试结果能否继续发布范围审核状态操作
toggleSelected(item.publication_id)} />{item.title}{sourceTypeLabels[item.source_type] ?? item.source_type} + {item.knowledge_exists === 'passed' && item.chunks_present === 'passed' ? '基础测试通过' : '基础测试未通过'} + {item.retrieval_hit === 'manual_required' ? '搜索效果需人工确认' : `搜索测试${acceptanceLabels[item.retrieval_hit]}`} +
查看测试详情知识读取：{acceptanceLabels[item.knowledge_exists]}；内容处理：{acceptanceLabels[item.chunks_present]}；搜索验证：{acceptanceLabels[item.retrieval_hit]}
+ 		{reviewGateLabel(item)}{reviewLabels[item.review_state]}{item.reviewer_ref ? {item.reviewer_ref} : null}
: null} +
创建批次还需要填写操作人确认所选发布动作至少 1 项动作通过确认填写批次名称
+
已选 {selectedCount} 项{props.selectedDeleteIDs.length > 0 && !deleteConfirmed ? '（删除待确认）' : ''}
+
+
+

第 4–5 步：创建并锁定发布清单

草稿可以继续核对；点击“锁定清单（不会上线）”后，本次内容不再变化，但仍不会发布到生产。

锁定清单 ≠ 上线
+ {props.batchError ?
{props.batchError}
: null} + {!props.batchLoading && !props.batchError && props.batchPage.items.length === 0 ?
尚无草稿或已锁定的发布清单。
: null} + {props.batchPage.items.length > 0 ?
+ {props.batchPage.items.map((batch) => )} +
批次名称状态知识数量内容状态操作
{batch.name}{batchLabels[batch.state]}{batch.item_count}{batch.state === 'sealed' ? '内容已锁定' : '仍可检查调整'}
{batch.state === 'draft' ? : null}

本批次包含

{props.batchDetail ? <>{props.batchDetail.items.map((item) =>
{item.title}{item.action === 'delete' ? '删除生产知识' : '新增或更新生产知识'}{item.decided_visibility === 'internal' ? '仅内部员工' : '允许外部用户'}
)}{props.batchDetail.items.length === 0 ?
批次没有知识。
: null} :
点击“查看内容”核对本次发布清单。
}
: null} +
+
+

第 6–8 步：生产确认、发布验证与回滚

这里仅供查看生产负责人确认后的发布结果。普通用户不能在本页面发起生产发布或回滚。

{props.releaseCapabilities.production_release_requests_enabled ? '等待生产负责人确认' : '生产操作已锁定'}
+ {props.executionError ?
{props.executionError}
: null} + {!props.executionLoading && !props.executionError && props.executionPage.items.length === 0 ?
尚无生产发布或回滚执行记录。
: null} + {props.executionPage.items.length > 0 ?
+ {props.executionPage.items.map((execution) => )} +
开始时间操作状态批次进度完成进度授权人操作
{formatTime(execution.created_at)}{execution.operation === 'publish' ? '发布' : '回滚'}{executionLabels[execution.state]}{executionStateDetail(execution, props.executionPage.items) ? {executionStateDetail(execution, props.executionPage.items)} : null}{execution.error_code ? {releaseFailureMessage(execution.error_code)} : null}{execution.total_batches>0?`批次 ${execution.current_batch} / ${execution.total_batches}`:'等待开始'}每批最多 {execution.batch_size} 项{execution.completed_item_count}/{execution.item_count}{execution.failed_item_count ? 失败 {execution.failed_item_count} : null}{execution.authorized_by}

每项知识的结果

{props.executionDetail ? <>{props.executionDetail.items.map((item) => { const result = releaseItemResult(item); return
{item.title}{result.detail}{item.error_code ? ` · ${releaseFailureMessage(item.error_code)}` : ''}{result.outcome}
})}{props.executionDetail.items.length === 0 ?
发布尚无逐项结果。
: null} :
点击“查看结果”核对每项知识。
}
: null} + {props.executionChain ?
+

发布审计链

+
批次{props.executionChain.release_batch_name || '未绑定批次'}{`创建 ${props.executionChain.created_by || '—'} · 锁定 ${props.executionChain.sealed_by || '—'} · 授权 ${props.executionChain.authorized_by || '—'}`}
+ {props.executionChain.release_change_set_id ?
发布计划集{props.executionChain.release_change_set_name}{`${props.executionChain.release_change_set_state === 'confirmed' ? '已确认' : props.executionChain.release_change_set_state} · 确认人 ${props.executionChain.release_change_set_confirmed_by || '—'} · 摘要 ${props.executionChain.manifest_sha256.slice(0, 8)}…${props.executionChain.manifest_sha256.slice(-6)}`}
: null} + {props.executionChain.snapshots.length ?
来源快照{props.executionChain.snapshots.map((snap) => `${snap.source_name}(${snap.file_count})`).join('、')}
: null} + {props.executionChain.catalog_change_sets.length ?
来源变更集{props.executionChain.catalog_change_sets.map((set) => `${set.id}·${set.state}`).join('、')}
: null} + {props.executionChain.rollback_execution_id ?
补偿回滚已关联回滚执行
: null} + {props.executionChain.prior_execution_id ?
上游发布由发布执行补偿而来
: null} +
: null} +
+
+} + +function TestPublications({ page, loading, error, state, retryingId, onStateChange, onPageChange, onRefresh, onRetry }: TestPublicationsProps) { + const [view, setView] = useState<'overview' | 'internal' | 'external' | 'failed'>('overview') + const visibleItems = page.items.filter((item) => view === 'internal' ? item.target_scope === 'internal' : view === 'external' ? item.target_scope === 'external' : view === 'failed' ? item.processing_state === 'failed' || item.processing_state === 'stalled' : true) + const internalCount = page.items.filter((item) => item.target_scope === 'internal').length + const externalCount = page.items.filter((item) => item.target_scope === 'external').length + const processingCount = page.items.filter((item) => item.processing_state === 'processing' || item.processing_state === 'finalizing').length + const problemCount = page.items.filter((item) => item.processing_state === 'failed' || item.processing_state === 'stalled').length + return ( +
+
+
+

解析监控

+

查看测试知识库中每份资料的上传、解析和验收状态。

+
+ +
+ + {error ?
{error}
: null} +
+
+
+

{view === 'failed' ? '失败与停滞文件' : '解析队列'}

当前筛选共 {visibleItems.length} 条；仅返回治理安全投影，不含来源地址或凭据。

+ +
+ {!loading && !error && visibleItems.length === 0 ?
当前筛选没有测试发布记录。
: null} + {visibleItems.length > 0 ? ( +
+ {visibleItems.map((item) => ( + + + + + + + + + + ))} +
发布标题处理状态重试次数错误码更新时间操作
{item.id}{item.title}{processingLabels[item.processing_state]}{item.retry_count}{item.last_error || '—'}{formatTime(item.updated_at)}{item.can_retry ? : '—'}
+ ) : null} +
第 {Math.floor(page.offset / page.limit) + 1} 页
+ + +
+
+
+ ) +} + +function SummaryCard({ label, value, tone }: { label: string; value: number; tone: string }) { + return ( +
+ {label} + {value} +
+ ) +} + +function formatTime(value: string) { + if (!value) return '—' + const date = new Date(value) + return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false }) +} + +export default function App() { + const [jobs, setJobs] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [connectionPage, setConnectionPage] = useState(emptyConnectionPage) + const [connectionLoading, setConnectionLoading] = useState(true) + const [connectionError, setConnectionError] = useState(null) + const [scanningConnectionId, setScanningConnectionId] = useState(null) + const [scanPage, setScanPage] = useState(emptyScanPage) + const [scanDetail, setScanDetail] = useState(null) + const [scanLoading, setScanLoading] = useState(false) + const [scanError, setScanError] = useState(null) + const [selectedScanId, setSelectedScanId] = useState(null) + const [assetPage, setAssetPage] = useState(emptyAssetPage) + const [assetLoading, setAssetLoading] = useState(true) + const [assetError, setAssetError] = useState(null) + const [assetState, setAssetState] = useState('') + const [assetOffset, setAssetOffset] = useState(0) + const [publicationPage, setPublicationPage] = useState(emptyPublicationPage) + const [publicationLoading, setPublicationLoading] = useState(true) + const [publicationError, setPublicationError] = useState(null) + const [publicationState, setPublicationState] = useState('') + const [publicationOffset, setPublicationOffset] = useState(0) + const [retryingId, setRetryingId] = useState(null) + const [reviewPage, setReviewPage] = useState(emptyReviewPage) + const [reviewLoading, setReviewLoading] = useState(true) + const [reviewError, setReviewError] = useState(null) + const [reviewingId, setReviewingId] = useState(null) + const [batchPage, setBatchPage] = useState(emptyBatchPage) + const [batchDetail, setBatchDetail] = useState(null) + const [batchLoading, setBatchLoading] = useState(true) + const [batchError, setBatchError] = useState(null) + const [mutatingBatchId, setMutatingBatchId] = useState(null) + const [releaseCapabilities, setReleaseCapabilities] = useState(disabledReleaseCapabilities) + const [releasePlan, setReleasePlan] = useState(emptyReleasePlan) + const [releaseChangeSet, setReleaseChangeSet] = useState(null) + const [changeSetBusy, setChangeSetBusy] = useState(false) + const [changeSetError, setChangeSetError] = useState(null) + const [selectedDeleteIDs, setSelectedDeleteIDs] = useState([]) + const [executionPage, setExecutionPage] = useState(emptyExecutionPage) + const [executionDetail, setExecutionDetail] = useState(null) + const [executionChain, setExecutionChain] = useState(null) + const [executionLoading, setExecutionLoading] = useState(true) + const [executionError, setExecutionError] = useState(null) + const [workspace, setWorkspace] = useState(() => { + if (typeof window === 'undefined') return initialWorkspace + try { + return restoreWorkspace(window.localStorage.getItem(workspaceStorageKey)) + } catch { + return initialWorkspace + } + }) + + const load = useCallback(async (signal?: AbortSignal) => { + setLoading(true) + setError(null) + try { + setJobs(await fetchJobs(signal)) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setError(cause instanceof Error ? cause.message : '加载任务失败') + } finally { + setLoading(false) + } + }, []) + + const loadAssets = useCallback(async (signal?: AbortSignal) => { + setAssetLoading(true) + setAssetError(null) + try { + setAssetPage(await fetchSourceAssets({ state: assetState, limit: 50, offset: assetOffset }, signal)) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setAssetError(cause instanceof Error ? cause.message : '加载知识资产失败') + } finally { + setAssetLoading(false) + } + }, [assetOffset, assetState]) + + const loadConnections = useCallback(async (signal?: AbortSignal) => { + setConnectionLoading(true) + setConnectionError(null) + try { + setConnectionPage(await fetchSourceConnections(signal)) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setConnectionError(cause instanceof Error ? cause.message : '加载来源连接失败') + } finally { + setConnectionLoading(false) + } + }, []) + + const loadSourceScans = useCallback(async (connectionID: string, signal?: AbortSignal) => { + setScanLoading(true) + setScanError(null) + try { + const page = await fetchSourceScans(connectionID, 20, 0, signal) + setScanPage(page) + const latest = page.items[0] + setSelectedScanId(latest?.id ?? null) + setScanDetail(latest ? await fetchSourceScanDetail(connectionID, latest.id, signal) : null) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setScanError(cause instanceof Error ? cause.message : '加载扫描记录失败') + } finally { + setScanLoading(false) + } + }, []) + + const handleSelectScan = useCallback(async (connectionID: string, scanID: string) => { + setSelectedScanId(scanID) + setScanLoading(true) + setScanError(null) + try { + setScanDetail(await fetchSourceScanDetail(connectionID, scanID)) + } catch (cause) { + setScanError(cause instanceof Error ? cause.message : '加载扫描详情失败') + } finally { + setScanLoading(false) + } + }, []) + + const handleTriggerSourceScan = useCallback(async (connectionID: string) => { + setScanningConnectionId(connectionID) + setConnectionError(null) + try { + await triggerSourceScan(connectionID) + await loadConnections() + } catch (cause) { + setConnectionError(cause instanceof Error ? cause.message : '触发来源扫描失败') + } finally { + setScanningConnectionId(null) + } + }, [loadConnections]) + + const loadPublications = useCallback(async (signal?: AbortSignal) => { + setPublicationLoading(true) + setPublicationError(null) + try { + setPublicationPage(await fetchTestPublications({ processing_state: publicationState, limit: 50, offset: publicationOffset }, signal)) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setPublicationError(cause instanceof Error ? cause.message : '加载测试发布失败') + } finally { + setPublicationLoading(false) + } + }, [publicationOffset, publicationState]) + + const handleRetry = useCallback(async (id: string) => { + setRetryingId(id) + try { + await retryPublication(id) + await loadPublications() + } catch (cause) { + setPublicationError(cause instanceof Error ? cause.message : '重试失败') + } finally { + setRetryingId(null) + } + }, [loadPublications]) + + const loadReviews = useCallback(async (signal?: AbortSignal) => { + setReviewLoading(true) + setReviewError(null) + try { + setReviewPage(await fetchReviewCandidates('', 20, 0, signal)) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setReviewError(cause instanceof Error ? cause.message : '加载审核队列失败') + } finally { + setReviewLoading(false) + } + }, []) + + const loadBatches = useCallback(async (signal?: AbortSignal) => { + setBatchLoading(true) + setBatchError(null) + try { + const page = await fetchReleaseBatches('', 20, 0, signal) + setBatchPage(page) + setBatchDetail(page.items[0] ? await fetchReleaseBatch(page.items[0].id, signal) : null) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setBatchError(cause instanceof Error ? cause.message : '加载封批列表失败') + } finally { + setBatchLoading(false) + } + }, []) + + const handleReview = useCallback(async (publicationID: string, decision: 'approved' | 'rejected', visibility: Visibility, reviewerRef: string, decisionCode: string) => { + setReviewingId(publicationID) + setReviewError(null) + try { + await reviewPublication(publicationID, decision, visibility, reviewerRef, decisionCode) + await loadReviews() + } catch (cause) { + setReviewError(cause instanceof Error ? cause.message : '提交审核决定失败') + } finally { + setReviewingId(null) + } + }, [loadReviews]) + + const reloadChangeSet = useCallback(async () => { + setReleaseChangeSet(await fetchLatestReleaseChangeSet().catch(() => null)) + }, []) + + const handleGenerateChangeSet = useCallback(async (name: string, createdBy: string) => { + setChangeSetBusy(true) + setChangeSetError(null) + try { + await generateReleaseChangeSet(name, createdBy) + await reloadChangeSet() + } catch (cause) { + setChangeSetError(cause instanceof Error ? cause.message : '生成发布计划集失败') + } finally { + setChangeSetBusy(false) + } + }, [reloadChangeSet]) + + const handleConfirmChangeSet = useCallback(async (changeSetID: string, confirmedBy: string) => { + setChangeSetBusy(true) + setChangeSetError(null) + try { + await confirmReleaseChangeSet(changeSetID, confirmedBy) + await reloadChangeSet() + } catch (cause) { + setChangeSetError(cause instanceof Error ? cause.message : '确认发布计划集失败') + } finally { + setChangeSetBusy(false) + } + }, [reloadChangeSet]) + + const handleCreateBatch = useCallback(async (name: string, createdBy: string, publicationIDs: string[], deletePublicationIDs: string[]) => { + setMutatingBatchId('creating') + setBatchError(null) + try { + const batch = await createReleaseBatch(name, createdBy, publicationIDs, deletePublicationIDs, releaseChangeSet?.state === 'confirmed' ? releaseChangeSet.id : '') + setSelectedDeleteIDs([]) + await loadBatches() + setBatchDetail(await fetchReleaseBatch(batch.id)) + } catch (cause) { + setBatchError(cause instanceof Error ? cause.message : '创建封批失败') + } finally { + setMutatingBatchId(null) + } + }, [loadBatches, releaseChangeSet]) + + const handleSealBatch = useCallback(async (batchID: string, sealedBy: string) => { + setMutatingBatchId(batchID) + setBatchError(null) + try { + await sealReleaseBatch(batchID, sealedBy) + await loadBatches() + setBatchDetail(await fetchReleaseBatch(batchID)) + } catch (cause) { + setBatchError(cause instanceof Error ? cause.message : '封批失败') + } finally { + setMutatingBatchId(null) + } + }, [loadBatches]) + + const handleSelectBatch = useCallback(async (batchID: string) => { + setBatchLoading(true) + setBatchError(null) + try { + setBatchDetail(await fetchReleaseBatch(batchID)) + } catch (cause) { + setBatchError(cause instanceof Error ? cause.message : '加载封批详情失败') + } finally { + setBatchLoading(false) + } + }, []) + + const loadReleaseExecutions = useCallback(async (signal?: AbortSignal) => { + setExecutionLoading(true) + setExecutionError(null) + try { + const [capabilities, page, plan, changeSet] = await Promise.all([ + fetchReleaseCapabilities(signal), + fetchReleaseExecutions('', '', 20, 0, signal), + fetchReleasePlan(signal), + fetchLatestReleaseChangeSet(signal).catch(() => null), + ]) + setReleaseCapabilities(capabilities) + setExecutionPage(page) + setReleasePlan(plan) + setReleaseChangeSet(changeSet) + setSelectedDeleteIDs((current) => current.filter((publicationID) => plan.items.some((item) => item.publication_id === publicationID && item.action === 'delete' && item.can_release))) + setExecutionDetail(page.items[0] ? await fetchReleaseExecution(page.items[0].id, signal) : null) + } catch (cause) { + if (cause instanceof DOMException && cause.name === 'AbortError') return + setExecutionError(cause instanceof Error ? cause.message : '加载生产发布审计失败') + } finally { + setExecutionLoading(false) + } + }, []) + + const handleSelectExecution = useCallback(async (executionID: string) => { + setExecutionLoading(true) + setExecutionError(null) + try { + const [detail, chain] = await Promise.all([fetchReleaseExecution(executionID), fetchReleaseAuditChain(executionID)]) + setExecutionDetail(detail) + setExecutionChain(chain) + } catch (cause) { + setExecutionError(cause instanceof Error ? cause.message : '加载生产发布详情失败') + } finally { + setExecutionLoading(false) + } + }, []) + + const ppConnection = connectionPage.items.find((item) => item.source_type === 'pp') + const ppConnectionID = ppConnection?.id ?? '' + const ppLastScanID = ppConnection?.last_scan?.id ?? '' + const ppLastScanState = ppConnection?.last_scan?.state ?? '' + + useEffect(() => { + const controller = new AbortController() + void load(controller.signal) + return () => controller.abort() + }, [load]) + + useEffect(() => { + const controller = new AbortController() + void loadAssets(controller.signal) + return () => controller.abort() + }, [loadAssets]) + + useEffect(() => { + const controller = new AbortController() + void loadConnections(controller.signal) + return () => controller.abort() + }, [loadConnections]) + + useEffect(() => { + if (!ppConnectionID) return + const controller = new AbortController() + void loadSourceScans(ppConnectionID, controller.signal) + return () => controller.abort() + }, [loadSourceScans, ppConnectionID, ppLastScanID, ppLastScanState]) + + useEffect(() => { + if (!connectionPage.items.some((item) => item.last_scan?.state === 'pending' || item.last_scan?.state === 'running')) return + const timer = window.setTimeout(() => void loadConnections(), 1000) + return () => window.clearTimeout(timer) + }, [connectionPage, loadConnections]) + + useEffect(() => { + const controller = new AbortController() + void loadPublications(controller.signal) + return () => controller.abort() + }, [loadPublications]) + + useEffect(() => { + const controller = new AbortController() + void loadReviews(controller.signal) + return () => controller.abort() + }, [loadReviews]) + + useEffect(() => { + const controller = new AbortController() + void loadBatches(controller.signal) + return () => controller.abort() + }, [loadBatches]) + + useEffect(() => { + const controller = new AbortController() + void loadReleaseExecutions(controller.signal) + return () => controller.abort() + }, [loadReleaseExecutions]) + + useEffect(() => { + try { + window.localStorage.setItem(workspaceStorageKey, JSON.stringify(workspace)) + } catch { + // Storage can be unavailable in privacy modes; tab navigation still works in memory. + } + }, [workspace]) + + return ( + setWorkspace((current) => openModule(current, moduleId))} + onClose={(moduleId) => setWorkspace((current) => closeModule(current, moduleId))} + > + {workspace.activeTab === 'dashboard' ? : workspace.activeTab === 'tasks' ? : workspace.activeTab === 'sources' ? : workspace.activeTab === 'settings' ? : workspace.activeTab === 'knowledge' ? : void load()} + connectionPage={connectionPage} + connectionLoading={connectionLoading} + connectionError={connectionError} + onConnectionRefresh={() => { + void loadConnections() + if (ppConnectionID) void loadSourceScans(ppConnectionID) + }} + scanningConnectionId={scanningConnectionId} + onTriggerSourceScan={(id) => void handleTriggerSourceScan(id)} + scanPage={scanPage} + scanDetail={scanDetail} + scanLoading={scanLoading} + scanError={scanError} + selectedScanId={selectedScanId} + onSelectScan={(connectionID, scanID) => void handleSelectScan(connectionID, scanID)} + assetPage={assetPage} + assetLoading={assetLoading} + assetError={assetError} + assetState={assetState} + onAssetStateChange={(state) => { + setAssetState(state) + setAssetOffset(0) + }} + onAssetPageChange={setAssetOffset} + onAssetRefresh={() => void loadAssets()} + publicationPage={publicationPage} + publicationLoading={publicationLoading} + publicationError={publicationError} + publicationState={publicationState} + retryingId={retryingId} + onPublicationStateChange={(state) => { + setPublicationState(state) + setPublicationOffset(0) + }} + onPublicationPageChange={setPublicationOffset} + onPublicationRefresh={() => void loadPublications()} + onRetry={(id) => void handleRetry(id)} + reviewPage={reviewPage} + reviewLoading={reviewLoading} + reviewError={reviewError} + reviewingId={reviewingId} + batchPage={batchPage} + batchDetail={batchDetail} + batchLoading={batchLoading} + batchError={batchError} + mutatingBatchId={mutatingBatchId} + onReview={(publicationID, decision, visibility, reviewerRef, decisionCode) => void handleReview(publicationID, decision, visibility, reviewerRef, decisionCode)} + onReviewRefresh={() => void loadReviews()} + onCreateBatch={(name, createdBy, publicationIDs, deletePublicationIDs) => void handleCreateBatch(name, createdBy, publicationIDs, deletePublicationIDs)} + onSealBatch={(batchID, sealedBy) => void handleSealBatch(batchID, sealedBy)} + onSelectBatch={(batchID) => void handleSelectBatch(batchID)} + onBatchRefresh={() => void loadBatches()} + releaseCapabilities={releaseCapabilities} + releasePlan={releasePlan} + releaseChangeSet={releaseChangeSet} + changeSetBusy={changeSetBusy} + changeSetError={changeSetError} + onGenerateChangeSet={handleGenerateChangeSet} + onConfirmChangeSet={handleConfirmChangeSet} + selectedDeleteIDs={selectedDeleteIDs} + onToggleDelete={(publicationID) => setSelectedDeleteIDs((current) => current.includes(publicationID) ? current.filter((id) => id !== publicationID) : [...current, publicationID])} + executionPage={executionPage} + executionDetail={executionDetail} + executionChain={executionChain} + executionLoading={executionLoading} + executionError={executionError} + onSelectExecution={(executionID) => void handleSelectExecution(executionID)} + onExecutionRefresh={() => void loadReleaseExecutions()} + />} + + ) +} diff --git a/web/src/DocumentWorkbench.test.tsx b/web/src/DocumentWorkbench.test.tsx new file mode 100644 index 0000000..caa456e --- /dev/null +++ b/web/src/DocumentWorkbench.test.tsx @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from 'vitest' +import { renderToStaticMarkup } from 'react-dom/server' +import { DocumentWorkbenchView } from './DocumentWorkbench' +import { document } from './documentFixture' + +describe('document engineer workbench', () => { + it('shows real file names, source categories and scope without technical identifiers', () => { + const manual={...document,id:'manual',title:'P3 验收文档',source_type:'manual' as const,connection_id:'manual-1'} + const html = renderToStaticMarkup() + for (const text of ['来源与目录','手册.pdf','使用指南','对外','解析完成','PP分类','未接通目录','文件名','显示 1 份验收文档']) expect(html).toContain(text) + expect(html).not.toContain('P3 验收文档') + expect(html).not.toContain('a') + expect(html).not.toContain('只走这 8 步') + }) + it('shows monitoring and no fabricated page progress or production availability', () => { + const html = renderToStaticMarkup() + expect(html).toContain('解析监控') + expect(html).toContain('来源处理概览') + expect(html).toContain('已就绪 1 / 2') + expect(html).toContain('独立解析') + expect(html).toContain('内容复用') + expect(html).toContain('处理中 1') + expect(html).toContain('解析完成不代表已上线') + expect(html).not.toContain('100%') + expect(html).not.toContain('生产可用') + }) + it('does not display zero as a successful load when the request failed', () => { + const html = renderToStaticMarkup() + expect(html).toContain('加载失败') + expect(html).not.toContain('aria-label="当前目录统计"') + expect(html).not.toContain('没有符合条件的文件') + }) +}) diff --git a/web/src/DocumentWorkbench.tsx b/web/src/DocumentWorkbench.tsx new file mode 100644 index 0000000..f6afc34 --- /dev/null +++ b/web/src/DocumentWorkbench.tsx @@ -0,0 +1,160 @@ +import { useContext, useEffect, useMemo, useState } from 'react' +import type { SourceAsset } from './assets' +import type { SourceConnectionPage } from './sourceConnections' +import { retryPublication } from './testPublications' +import { clearScopeRule, fetchScopeRules, setScopeRule } from './scopeRules' +import type { ClearScopeRuleDecision, ScopeRule, ScopeRuleDecision } from './scopeRules' +import { ScopeRuleEditor } from './ScopeRuleEditor' +import { WorkspaceContextTabsSetter } from './workspaceContext' +import { connectionKey, directoryGroups, documentPipeline, documentProblem, documentStage, fetchDocumentCatalog, filterDocuments, operationalDocuments, sourceProgress, stageLabels, visibilityOrigin } from './documentWorkbench' +import type { DocumentStage } from './documentWorkbench' +import './documentWorkbench.css' + +type Mode = 'directory' | 'monitor' | 'attention' +const sourceNames: Record = {pp:'PP公开文档', alist:'AList文件源', dingtalk:'钉钉知识库', manual:'手工验收文档'} +const activeStages: DocumentStage[] = ['downloading','uploading','parsing','indexing'] +interface ViewProps { + items: SourceAsset[] + connections: SourceConnectionPage['items'] + mode: Mode + loading: boolean + error: string | null + onRefresh: () => void + onRetry: (id:string) => void + retryingId: string | null +} +const time = (value:string) => new Date(value).toLocaleString('zh-CN', {hour12:false}) +const bytes = (value:number) => value >= 1048576 ? `${(value/1048576).toFixed(1)} MB` : `${(value/1024).toFixed(1)} KB` + +export function DocumentWorkbenchView({items, connections, mode, loading, error, onRefresh, onRetry, retryingId}: ViewProps) { + const [connection, setConnection] = useState('') + const [directory, setDirectory] = useState(null) + const [query, setQuery] = useState('') + const [visibility, setVisibility] = useState('') + const [stage, setStage] = useState('') + const [page, setPage] = useState(0) + const [selectedID, setSelectedID] = useState(null) + const [showTestDocuments,setShowTestDocuments]=useState(false) + const [scopeRules,setScopeRules]=useState([]) + const [scopeLoading,setScopeLoading]=useState(false) + const [scopeError,setScopeError]=useState(null) + const setWorkspaceContextTabs=useContext(WorkspaceContextTabsSetter) + const visibleItems=useMemo(()=>operationalDocuments(items,showTestDocuments),[items,showTestDocuments]) + const groups = useMemo(()=>directoryGroups(visibleItems),[visibleItems]) + const sources = useMemo(()=> { + const map = new Map(connections.filter(c=>showTestDocuments||c.source_type!=='manual').map(c=>[c.id, {id:c.id,type:c.source_type,name:c.name}])) + for (const item of visibleItems) if (!map.has(connectionKey(item))) map.set(connectionKey(item), {id:connectionKey(item),type:item.source_type,name:item.connection_name}) + return [...map.values()] + },[visibleItems,connections,showTestDocuments]) + const scoped = filterDocuments(visibleItems,{connection,directory,query:'',visibility:'',stage:''}) + const filtered = filterDocuments(visibleItems,{connection,directory,query,visibility,stage}).filter(item => mode !== 'attention' || ['blocked','failed'].includes(documentStage(item))) + const shownPage = Math.min(page,Math.max(0,Math.ceil(filtered.length/40)-1)) + const selected = filtered.find(item=>item.id===selectedID) + const reset = () => {setPage(0);setSelectedID(null)} + const selectFolder = (id:string,path:string|null) => {setConnection(id);setDirectory(path);reset()} + const title = mode === 'monitor' ? '解析监控' : mode === 'attention' ? '待处理文件' : '来源与目录' + const unavailable = items.length === 0 && Boolean(loading || error) + const progress=sourceProgress(visibleItems) + const manualCount=items.filter(item=>item.source_type==='manual').length + const directDirectoryRule=scopeRules.find(rule=>rule.target_type==='directory'&&rule.connection_id===connection&&rule.directory_path===(directory??'')) + const selectedItemRule=scopeRules.find(rule=>rule.target_type==='item'&&rule.source_item_id===selectedID) + const selectedSourceLabel=sourceNames[sources.find(item=>item.id===connection)?.type||'']||'当前来源' + useEffect(()=>{ + if(!connection){setScopeRules([]);setScopeError(null);return} + const controller=new AbortController() + setScopeLoading(true) + void fetchScopeRules(connection,controller.signal).then(setScopeRules).catch(cause=>{if(!controller.signal.aborted)setScopeError(cause instanceof Error?cause.message:'加载范围规则失败')}).finally(()=>{if(!controller.signal.aborted)setScopeLoading(false)}) + return()=>controller.abort() + },[connection]) + useEffect(()=>{ + if(!connection){setWorkspaceContextTabs([]);return} + const tabs=[{key:'source',label:selectedSourceLabel,onActivate:()=>{setDirectory(null);setSelectedID(null);setPage(0)}}] + if(directory!==null)tabs.push({key:'directory',label:directory||'分类待补齐',onActivate:()=>setSelectedID(null)}) + if(selected)tabs.push({key:'file',label:selected.title||'当前文件',onActivate:()=>setSelectedID(selected.id)}) + setWorkspaceContextTabs(tabs) + return()=>setWorkspaceContextTabs([]) + },[connection,directory,selected,selectedSourceLabel,setWorkspaceContextTabs]) + const mutateScope=async(decision:ScopeRuleDecision|ClearScopeRuleDecision,clear:boolean)=>{ + if(scopeLoading)return + setScopeLoading(true);setScopeError(null) + try{if(clear)await clearScopeRule(decision as ClearScopeRuleDecision);else await setScopeRule(decision as ScopeRuleDecision);setScopeRules(await fetchScopeRules(connection));onRefresh()}catch(cause){setScopeError(cause instanceof Error?cause.message:'保存范围规则失败')}finally{setScopeLoading(false)} + } + return
+

文档工作台

{title}

{mode === 'monitor' ? '看每份文件处理到哪一步。解析完成不代表已上线。' : mode === 'attention' ? '只列出超限、失败或来源不可访问的文件。内容审核请进入发布管理。' : '先选来源和目录，再看文件、对内/对外和处理状态。'}

+ {error ?
{error}（旧数据仅供参考）
: null} + {mode==='monitor'&&!unavailable?
来源处理概览直接看各来源完成、处理中和异常数量
{progress.map(item=>{const source=sources.find(value=>value.id===item.connection);return })}
:null} +
+ +
+ + {!unavailable ?
+ 文件 {scoped.length} + {mode==='monitor'?<>独立解析 {scoped.filter(x=>documentStage(x)==='completed').length}内容复用 {scoped.filter(x=>documentStage(x)==='reused').length}处理中 {scoped.filter(x=>activeStages.includes(documentStage(x))).length}等待 {scoped.filter(x=>documentStage(x)==='waiting').length}需处理（含超限） {scoped.filter(x=>['blocked','failed'].includes(documentStage(x))).length}:<>对内 {scoped.filter(x=>x.visibility==='internal').length}对外 {scoped.filter(x=>x.visibility==='external').length}处理中 {scoped.filter(x=>activeStages.includes(documentStage(x))).length}已解析 {scoped.filter(x=>documentStage(x)==='completed').length}需处理 {scoped.filter(x=>['blocked','failed'].includes(documentStage(x))).length}} +
:

文件数量尚未核实

} + {mode==='directory'&&connection&&directory!==''?void mutateScope(decision,false)} onClear={decision=>void mutateScope(decision,true)} />:mode==='directory'&&connection&&directory===''?

“分类待补齐”不是可靠目录，不能单独设置范围。请先补齐来源目录，或返回来源级设置。

:null} +
+ + + +
+
{!connection?'全部来源':sourceNames[sources.find(s=>s.id===connection)?.type || ''] || '当前来源'} / {directory === null?'全部文件':directory || '目录待补齐'} {!unavailable ? 筛选结果 {filtered.length} 份 : null}
+ {loading && !items.length ?
正在核对完整文档清单…
: null} + {!loading && !error && !filtered.length ?
没有符合条件的文件。
: null} + {filtered.length ?
+ {filtered.slice(shownPage*40,(shownPage+1)*40).map(item=> + + + + )} +
文件名对内/对外处理状态大小状态更新时间
{sourceNames[item.source_type]} / {item.directory_path || '目录待补齐'}{item.visibility==='external'?'对外':'对内'}{visibilityOrigin(item)}{stageLabels[documentStage(item)]}{bytes(item.size_bytes)}
: null} +
第 {shownPage+1} / {Math.max(1,Math.ceil(filtered.length/40))} 页
+ {mode === 'monitor' ?

每15秒自动刷新。当前接口未提供页数或逐页百分比，因此只显示真实阶段。时间为状态更新时间，不是源文件修改时间。

: null} +
+
+ {selected ?

{selected.title || '文件名待补齐'}

+
来源位置
{sourceNames[selected.source_type]} / {selected.directory_path || '目录待补齐'}
对内/对外
{selected.visibility==='external'?'对外':'对内'} · {visibilityOrigin(selected)}
当前进度
{stageLabels[documentStage(selected)]}
+ {documentProblem(selected) ?

{documentProblem(selected)}

: null} +
文件大小 {bytes(selected.size_bytes)}状态更新时间 {time(selected.updated_at)}来源变化 {selected.source_change==='unchanged'?'无变化':'需要核对'}检索状态 {selected.retrieval_state==='enabled'?'测试环境已启用':'未证明已启用'}
+
{documentPipeline(selected).map(step=>
)}
+

原文与版本差异尚未接通，请在源系统核对内容；不要仅凭“解析完成”批准发布。

+ {mode==='directory'?void mutateScope(decision,false)} onClear={decision=>void mutateScope(decision,true)} />:null} + {selected.can_retry && selected.publication_id ? : null} + {selected.last_error ?
错误详情{selected.last_error}
: null} +
: null} +
+} + +export function DocumentWorkbench({mode,connections}: {mode:Mode;connections:SourceConnectionPage['items']}) { + const [items,setItems] = useState([]) + const [loading,setLoading] = useState(true) + const [error,setError] = useState(null) + const [revision,setRevision] = useState(0) + const [retryingId,setRetryingId] = useState(null) + useEffect(()=>{ + const controller = new AbortController() + setLoading(true) + void fetchDocumentCatalog(controller.signal).then(data=>{if (!controller.signal.aborted) {setItems(data);setError(null)}}).catch(cause=>{if (!controller.signal.aborted) setError(cause instanceof Error?cause.message:'加载文件失败')}).finally(()=>{if (!controller.signal.aborted) setLoading(false)}) + return ()=>controller.abort() + },[revision]) + useEffect(()=>{ + if (mode!=='monitor' || loading) return + const timer = window.setTimeout(()=>setRevision(r=>r+1),15000) + return ()=>window.clearTimeout(timer) + },[mode,loading,revision]) + const retry = async(id:string)=>{ + if (retryingId) return + if (!window.confirm('仅重试这份失败文件，不影响已成功文件或生产。继续吗？')) return + setRetryingId(id) + try {await retryPublication(id);setRevision(r=>r+1)} catch(cause) {setError(cause instanceof Error?cause.message:'重试失败')} finally {setRetryingId(null)} + } + return setRevision(r=>r+1)} onRetry={id=>void retry(id)} retryingId={retryingId} /> +} diff --git a/web/src/ReleasePlanner.test.tsx b/web/src/ReleasePlanner.test.tsx new file mode 100644 index 0000000..87c3ff6 --- /dev/null +++ b/web/src/ReleasePlanner.test.tsx @@ -0,0 +1,20 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { ReleasePlannerView } from './ReleasePlanner' +import { simulatedReleaseCandidates } from './releasePlanner' + +describe('ReleasePlannerView',()=>{ + it('shows a four-step document release plan with explicit closure and production gates',()=>{ + const html=renderToStaticMarkup() + for(const text of ['本地模拟发布准备','1. 选择文档','2. 检查数据闭包','3. 锁定清单','4. 生产负责人授权','[模拟] CR5用户手册.pdf','切片 186','向量 186','Wiki 完整','图谱 完整','闭包完整','文档级：','闭包可独立验证','整知识库：','Wiki或图谱共享','推荐方式','文档级同步','生产发布已关闭','生产备份到OSS：尚未配置'])expect(html).toContain(text) + expect(html).not.toContain('>授权生产发布') + }) + it('forces whole knowledge-base mode for shared graph data and blocks failed parsing',()=>{ + const whole=renderToStaticMarkup() + expect(whole).toContain('整知识库同步') + expect(whole).toContain('图谱 共享，需整库') + const failed=renderToStaticMarkup() + expect(failed).toContain('当前清单不能同步') + expect(failed).toContain('闭包不完整') + }) +}) diff --git a/web/src/ReleasePlanner.tsx b/web/src/ReleasePlanner.tsx new file mode 100644 index 0000000..d7fe553 --- /dev/null +++ b/web/src/ReleasePlanner.tsx @@ -0,0 +1,26 @@ +import { useState } from 'react' +import { evaluateReleaseSelection,simulatedReleaseCandidates } from './releasePlanner' +import type { ClosureState,ReleaseCandidate } from './releasePlanner' +import './releasePlanner.css' +const closure=(value:ClosureState)=>value==='ready'?'完整':value==='shared'?'共享，需整库':value==='missing'?'缺失':'不适用' +const modeLabel={none:'尚未选择',document:'文档级同步',knowledge_base:'整知识库同步',blocked:'当前清单不能同步'} as const +interface Props{candidates:ReleaseCandidate[];selectedIDs:string[];onToggle:(id:string)=>void;onClear:()=>void;onGenerate?:()=>void;generated?:boolean} +export function ReleasePlannerView({candidates,selectedIDs,onToggle,onClear,onGenerate=()=>undefined,generated=false}:Props){ + const selected=candidates.filter(item=>selectedIDs.includes(item.id));const plan=evaluateReleaseSelection(selected);const totals=selected.reduce((sum,item)=>({chunks:sum.chunks+item.chunks,vectors:sum.vectors+item.vectors,assets:sum.assets+item.assets}),{chunks:0,vectors:0,assets:0}) + return
+

同步准备

发布管理

确认测试环境哪些文档具备完整数据，再决定文档级或整知识库同步。

生产发布已关闭
+
本地模拟发布准备候选文档和闭包数量均为模拟，用于确认工作流程；不会连接生产。
+
  1. 1. 选择文档
  2. 2. 检查数据闭包
  3. 3. 锁定清单
  4. 4. 生产负责人授权
+
+

测试环境候选文档

解析失败或核心数据缺失的文件不可加入
+
{candidates.map(item=>{const blocked=item.parse_status!=='completed'||item.chunks<1||item.vectors!==item.chunks||item.assets<1||item.wiki==='missing'||item.graph==='missing';const shared=item.wiki==='shared'||item.graph==='shared';return })}
+
+ +
+
+} +export function ReleasePlanner(){ + const [selectedIDs,setSelectedIDs]=useState([]);const [generated,setGenerated]=useState(false) + const toggle=(id:string)=>{setGenerated(false);setSelectedIDs(current=>current.includes(id)?current.filter(value=>value!==id):[...current,id])} + return {setSelectedIDs([]);setGenerated(false)}} onGenerate={()=>setGenerated(true)} generated={generated} /> +} diff --git a/web/src/ScopeRuleEditor.test.tsx b/web/src/ScopeRuleEditor.test.tsx new file mode 100644 index 0000000..ca98320 --- /dev/null +++ b/web/src/ScopeRuleEditor.test.tsx @@ -0,0 +1,18 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it, vi } from 'vitest' +import { ScopeRuleEditor } from './ScopeRuleEditor' + +describe('ScopeRuleEditor',()=>{ + it('explains directory inheritance and keeps production publication separate',()=>{ + const html=renderToStaticMarkup() + for(const text of ['此目录及子目录','当前设置：对内','人工：engineer','移除本级设置，恢复继承','不会自动发布到生产']) expect(html).toContain(text) + }) + it('shows an honest inherited state when no direct rule exists',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('整个来源') + expect(html).toContain('当前没有本级设置') + expect(html).toContain('拟修改为') + expect(html).toContain('选择新范围') + expect(html).not.toContain('移除本级设置') + }) +}) diff --git a/web/src/ScopeRuleEditor.tsx b/web/src/ScopeRuleEditor.tsx new file mode 100644 index 0000000..559485a --- /dev/null +++ b/web/src/ScopeRuleEditor.tsx @@ -0,0 +1,33 @@ +import { useState } from 'react' +import type { ClearScopeRuleDecision, ScopeRule, ScopeRuleDecision, ScopeRuleTarget, Visibility } from './scopeRules' + +interface Props { + target: ScopeRuleTarget + currentRule?: ScopeRule + saving: boolean + error?: string|null + onSave: (decision:ScopeRuleDecision)=>void + onClear: (decision:ClearScopeRuleDecision)=>void +} + +export function ScopeRuleEditor({target,currentRule,saving,error,onSave,onClear}:Props){ + const [visibility,setVisibility]=useState(currentRule?.visibility||'') + const [operator,setOperator]=useState('') + const [reason,setReason]=useState('') + const identityReady=operator.trim()!==''&&reason.trim()!==''&&!saving + const saveReady=identityReady&&visibility!=='' + const label=target.target_type==='item'?'仅此文件':target.directory_path?'此目录及子目录':'整个来源' + const base={...target,operator_ref:operator.trim(),reason:reason.trim()} + return
+
对内/对外设置{label}
{currentRule?当前设置：{currentRule.visibility==='external'?'对外':'对内'}:当前没有本级设置}
+ {currentRule?

人工：{currentRule.updated_by} · 原因：{currentRule.reason}

:

当前继承上级规则或来源原始标记。

} +
+ + + +
+ {error?

{error}

:null} +
{currentRule?:null}
+ 这里只更新治理规则并记录审计，不会修改源文件，也不会自动发布到生产。 +
+} diff --git a/web/src/SnapshotHistory.test.tsx b/web/src/SnapshotHistory.test.tsx new file mode 100644 index 0000000..5693d02 --- /dev/null +++ b/web/src/SnapshotHistory.test.tsx @@ -0,0 +1,34 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { SnapshotHistoryView } from './SnapshotHistory' +import type { CatalogSnapshot } from './sourceCatalog' + +const head:CatalogSnapshot={snapshot_id:'ss-new',connection_id:'conn-1',root_node_id:'root',source_name:'国内工业',node_count:2,folder_count:1,file_count:1,observed_at:'2026-09-11T00:00:00Z',created_at:'2026-09-11T00:00:05Z',is_head:true} +const older:CatalogSnapshot={...head,snapshot_id:'ss-old',observed_at:'2026-09-10T00:00:00Z',is_head:false} +const fileNode={node_id:'file-old',parent_node_id:'root',node_type:'file' as const,name:'手册.pdf',direct_child_count:0,subtree_file_count:1,subtree_bytes:20,size_bytes:20,modified_at:'2026-09-09T00:00:00Z',content_sha256:'ab'.repeat(32),visibility:'unknown' as const,imported_count:0,parsed_count:0,active_count:0,failed_count:0,routable_count:0} +const page={current:{node_id:'root',node_type:'folder' as const,name:'国内工业',direct_child_count:1,subtree_file_count:1,subtree_bytes:20,visibility:'unknown' as const,imported_count:0,parsed_count:0,active_count:0,failed_count:0,routable_count:0},breadcrumbs:[{node_id:'root',name:'国内工业'}],items:[fileNode],total:1,limit:50,offset:0} + +describe('SnapshotHistoryView',()=>{ + it('lists snapshot history, flags head and opens a historical snapshot read-only',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('历史快照') + expect(html).toContain('当前') + expect(html).toContain('只读') + expect(html).toContain('手册.pdf') + expect(html).toContain('内容Hash') + expect(html).toContain('abababababab') + // hash is displayable but never turns into an interactive policy checkbox/button + expect(html).not.toContain('checkbox') + expect(html).not.toContain('policy-button') + }) + it('shows 内容Hash未获取 instead of faking a hash',()=>{ + const missing={...page,items:[{...fileNode,content_sha256:undefined}]} + const html=renderToStaticMarkup() + expect(html).toContain('内容Hash未获取') + }) + it('lists snapshots without an active page and offers opening each one',()=>{ + const html=renderToStaticMarkup() + expect((html.match(/查看该快照/g)||[]).length).toBe(2) + expect(html).toContain('份文件') + }) +}) diff --git a/web/src/SnapshotHistory.tsx b/web/src/SnapshotHistory.tsx new file mode 100644 index 0000000..4ff1513 --- /dev/null +++ b/web/src/SnapshotHistory.tsx @@ -0,0 +1,32 @@ +import type { CatalogNodePage, CatalogSnapshot } from './sourceCatalog' +import './sourceCatalogBrowser.css' + +const stamp=(value:string)=>new Date(value).toLocaleString('zh-CN',{hour12:false}) +const shortHash=(value:string)=>`${value.slice(0,12)}…${value.slice(-8)}` + +interface Props{snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null + onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void} + +export function SnapshotHistoryView({snapshots,active,page,loading,error,onClose,onOpenSnapshot,onOpenFolder}:Props){ + return
+

历史快照

只读查看不可变快照；切换查看不会改变当前生效版本。

+ {error?
{error}
:null} +
    {snapshots.map(snapshot=>
  • +
    {stamp(snapshot.observed_at)}{snapshot.is_head?当前:null}{snapshot.node_count} 个节点 · {snapshot.file_count} 份文件 · 入库 {stamp(snapshot.created_at)}
    + +
    • )}
+ {loading?
正在读取历史快照目录…
:null} + {active&&page?
{active.source_name}只读历史快照
+ + {page.items.map(node=> + + + + + + )}
名称类型大小更新时间内容Hash
{node.node_type==='folder'?:<>📄 {node.name}}{node.node_type==='folder'?'文件夹':'文件'}{typeof node.size_bytes==='number'?`${node.size_bytes} B`:'—'}{node.modified_at?stamp(node.modified_at):'—'}{node.content_sha256?shortHash(node.content_sha256):'内容Hash未获取'}
+ {page.total>page.limit?

共 {page.total} 项 · 每页 {page.limit} 项

:null} +
:null} + {!loading&&active&&!page&&!error?
该快照暂无可见内容。
:null} +
+} diff --git a/web/src/SourceCatalogBrowser.test.tsx b/web/src/SourceCatalogBrowser.test.tsx new file mode 100644 index 0000000..e25c878 --- /dev/null +++ b/web/src/SourceCatalogBrowser.test.tsx @@ -0,0 +1,127 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { SourceCatalogBrowserView } from './SourceCatalogBrowser' +import type { CatalogNode,CatalogNodePage,CatalogSource } from './sourceCatalog' +import type { Job } from './jobs' + +const source:CatalogSource={connection_id:'conn-1',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'root-1',folder_count:1080,file_count:3468,observed_at:'2026-09-08T00:00:00Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:12000} +const node=(name:string,type:'folder'|'file',id:string)=>({node_id:id,parent_node_id:'root-1',node_type:type,name,direct_child_count:type==='folder'?2:0,subtree_file_count:type==='folder'?10:1,subtree_bytes:1024,size_bytes:type==='file'?1024:undefined,visibility:type==='folder'?'internal' as const:'unknown' as const,imported_count:type==='folder'?8:0,parsed_count:type==='folder'?7:0,active_count:1,failed_count:0,routable_count:type==='file'?1:8}) +const page:CatalogNodePage={current:{...node('公司级技术支持知识库','folder','root-1'),parent_node_id:undefined},breadcrumbs:[{node_id:'root-1',name:'公司级技术支持知识库'}],items:[node('01｜产品总览与定位','folder','folder-1'),node('根目录说明.pdf','file','file-1')],total:2,limit:50,offset:0} + +describe('SourceCatalogBrowserView',()=>{ + + it('uses a Windows-style path bar instead of a directory tree',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('aria-label="当前目录路径"') + expect(html).toContain('aria-label="目录面包屑"') + expect(html).toContain('aria-label="返回上一级"') + expect(html).not.toContain('role="tree"') + expect(html).not.toContain('目录树') + }) + + it('looks like a folder browser and shows only direct children',()=>{ + const html=renderToStaticMarkup() + for(const text of ['各个知识源头的资料总览与标注','公司级技术支持知识库','01｜产品总览与定位','文件夹','已解析 7 / 已导入 8','本地只读快照','含所有子目录文件'])expect(html).toContain(text) + expect(html).not.toContain('真实目录') + expect(html).toContain('根目录说明.pdf') + expect(html).not.toContain('显示根目录文件') + expect(html).toContain('aria-label="当前目录路径"') + expect(html).not.toContain('第 1 / 1 页') + expect(html).not.toContain('PP分类 · 不代表完整原始目录') + expect(html).toContain('快照时间') + expect(html).toContain('快照就绪') + expect(html).toContain('上次刷新耗时 12秒') + expect(html).toContain('实时刷新目录') + expect(html).not.toContain('aria-label="处理策略01｜产品总览与定位"') + expect(html).not.toContain('递归作用于全部子目录') + }) + it('shows current directory, discovered file counts and the estimate marker during refresh',()=>{ + const job:Job={id:'job-scan',type:'source.catalog.refresh',payload:{connection_id:'conn-1',stage:'scanning',progress:43,message:'正在扫描目录：已完成 125 个文件夹，待处理 55 个',processed:125,total:180,current_directory:'产品资料/CR系列',scanned_files:2438,estimated_total_files:5662,total_is_estimate:true},state:'running',priority:20,available_at:'2026-09-09T00:00:00Z',attempt:1,max_attempts:3,idempotency_key:'safe',created_at:'2026-09-09T00:00:00Z',updated_at:'2026-09-09T00:00:00Z'} + const html=renderToStaticMarkup() + expect(html).toContain('当前目录：产品资料/CR系列') + expect(html).toContain('已发现 2438 份文件') + expect(html).toContain('（估算）') + }) + it('hides the running refresh to a compact pill that expands back on click without blocking browsing',()=>{ + const job:Job={id:'job-hide',type:'source.catalog.refresh',payload:{connection_id:'conn-1',stage:'scanning',progress:60,message:'正在扫描目录：已完成 1000 个文件夹',processed:1000,total:1339,current_directory:'产品资料/CR系列',scanned_files:2438},state:'running',priority:20,available_at:'2026-09-09T00:00:00Z',attempt:1,max_attempts:3,idempotency_key:'safe',created_at:'2026-09-09T00:00:00Z',updated_at:'2026-09-09T00:00:00Z'} + const common={sources:[source],selectedSource:source,page,loading:false,error:null,refreshJob:job,onSelectSource:vi.fn(),onOpenFolder:vi.fn(),onNavigateTo:vi.fn(),onBack:vi.fn(),onPageChange:vi.fn(),onRefresh:vi.fn()} + const full=renderToStaticMarkup() + expect(full).toContain('隐藏到后台') + expect(full).toContain('实时快照进度') + expect(full).toContain('产品资料/CR系列') + const hidden=renderToStaticMarkup() + expect(hidden).not.toContain('实时快照进度') + expect(hidden).toContain('catalog-refresh-pill') + expect(hidden).toContain('60%') + expect(hidden).toContain('公司级技术支持知识库') // pill still names the source + // browsing and annotation stay interactive in hidden mode (only a duplicate refresh is blocked) + expect(hidden).toContain('aria-label="选择01｜产品总览与定位"') + expect(hidden).toContain('aria-label="选择根目录说明.pdf"') + expect(hidden).toContain('全选当前页') + expect(hidden).toContain('实时刷新中…') + }) + it('announces failure safely and keeps old snapshot usable',()=>{ + const job:Job={id:'job-fail',type:'source.catalog.refresh',payload:{connection_id:'conn-1',stage:'failed',progress:0,message:'AList API Token 未配置，请联系管理员更新来源凭据；旧快照仍然保留',processed:0,total:0},state:'failed',priority:20,available_at:'2026-09-09T00:00:00Z',attempt:3,max_attempts:3,idempotency_key:'safe',created_at:'2026-09-09T00:00:00Z',updated_at:'2026-09-09T00:00:00Z'} + const html=renderToStaticMarkup() + expect(html).toContain('实时刷新失败') + expect(html).toContain('刷新失败；旧快照仍然保留，目录浏览与标注不受影响。') + expect(html).toContain('快照就绪') + expect(html).toContain('01｜产品总览与定位') + }) + it('keeps the progress bar free of directory details when the scanner reports none',()=>{ + const job:Job={id:'job-plain',type:'source.catalog.refresh',payload:{connection_id:'conn-1',stage:'building',progress:80,message:'正在校验目录结构',processed:10,total:10},state:'running',priority:20,available_at:'2026-09-09T00:00:00Z',attempt:1,max_attempts:3,idempotency_key:'safe',created_at:'2026-09-09T00:00:00Z',updated_at:'2026-09-09T00:00:00Z'} + const html=renderToStaticMarkup() + expect(html).toContain('正在校验目录结构') + expect(html).not.toContain('刷新明细') + }) + it('uses durable background refresh progress and the current stage message',()=>{ + const job:Job={id:'job-refresh',type:'source.catalog.refresh',payload:{connection_id:'conn-1',stage:'scanning',progress:50,message:'正在扫描目录：已完成 50 个',processed:50,total:100},state:'running',priority:20,available_at:'2026-09-09T00:00:00Z',attempt:1,max_attempts:3,idempotency_key:'safe',created_at:'2026-09-09T00:00:00Z',updated_at:'2026-09-09T00:00:00Z'} + const html=renderToStaticMarkup() + expect(html).toContain('实时刷新中') + expect(html).toContain('正在扫描目录：已完成 50 个') + expect(html).toContain('value="50"') + }) + it('uses clickable breadcrumbs and an explicit back action below the source root',()=>{ + const nested={...page,current:{...page.current,node_id:'folder-1',parent_node_id:'root-1',name:'01｜产品总览与定位'},breadcrumbs:[...page.breadcrumbs,{node_id:'folder-1',name:'01｜产品总览与定位'}]} + const html=renderToStaticMarkup() + expect(html).toContain('aria-label="返回上一级"') + expect(html).toContain('公司级技术支持知识库') + expect(html).toContain('01｜产品总览与定位') + }) + it('shows policy as a read-only table state and offers top-level policy buttons',()=>{ + const nested={...page,current:{...page.current,node_id:'folder-1',parent_node_id:'root-1',name:'01｜产品总览与定位'},breadcrumbs:[...page.breadcrumbs,{node_id:'folder-1',name:'01｜产品总览与定位'}],items:[node('可入队.pdf','file','file-ready'),node('已知目录','folder','folder-known'),{...node('大小未知目录','folder','folder-unknown'),subtree_bytes:0},{...node('只有元数据.pdf','file','file-meta'),size_bytes:undefined,routable_count:0}],total:4} + const html=renderToStaticMarkup() + for(const text of ['名称','处理策略','全选当前页','对内','对外','禁止导入','禁止导入只保存期望策略；经“快照差异”确认后才会从测试知识库移除','aria-label="选择可入队.pdf"','大小未提供','已知合计 1.0 KB'])expect(html).toContain(text) + for(const text of ['class="catalog-policy-actions"','class="internal-policy-button"','class="external-policy-button"','class="blocked-policy-button"','class="selected-count"'])expect(html).toContain(text) + for(const text of ['操作人','判断原因','名称与处理策略','当前策略','选择策略','批量处理策略','应用策略','对内 + 对外','{ + const nested={...page,current:{...page.current,node_id:'folder-1'},breadcrumbs:[...page.breadcrumbs,{node_id:'folder-1',name:'01｜产品总览与定位'}],items:[node('可入队.pdf','file','file-ready'),node('已知目录','folder','folder-known')],total:2} + const html=renderToStaticMarkup() + // exactly three top-level policy buttons: 对内 / 对外 / 禁止导入 (docs/SOURCE_SCOPE_POLICY.md rules 4-6) + expect((html.match(/class="[a-z-]+-policy-button"/g)||[]).length).toBe(3) + // rule 2: only current-page select-all exists + expect(html).toContain('全选当前页') + expect(html).not.toContain('递归全选') + // rule 1: both file and folder rows expose selection checkboxes + expect(html).toContain('aria-label="选择可入队.pdf"') + expect(html).toContain('aria-label="选择已知目录"') + // rules 5-6: no batch entry for historical 对内 + 对外 or for unset state + expect(html).not.toContain('对内 + 对外') + expect(html).not.toContain('批量未选择') + }) + it('lets an engineer select a directory without opening it',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('aria-label="选择01｜产品总览与定位"') + }) + it('does not claim an empty source loaded when the request failed',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('加载失败') + expect(html).not.toContain('尚无来源目录') + }) +}) diff --git a/web/src/SourceCatalogBrowser.tsx b/web/src/SourceCatalogBrowser.tsx new file mode 100644 index 0000000..3c32f98 --- /dev/null +++ b/web/src/SourceCatalogBrowser.tsx @@ -0,0 +1,111 @@ +import { useContext,useEffect,useState } from 'react' +import { fetchCatalogNodes,fetchCatalogRoutingCapabilities,fetchCatalogSnapshotNodes,fetchCatalogSnapshots,fetchCatalogSources,routeCatalogFiles } from './sourceCatalog' +import type { CatalogNode,CatalogNodePage,CatalogPolicy,CatalogRoutingCapabilities,CatalogSnapshot,CatalogSource } from './sourceCatalog' +import { SnapshotHistoryView } from './SnapshotHistory' +import { triggerCatalogRefresh } from './sourceCatalog' +import { fetchJobs } from './jobs' +import type { Job } from './jobs' +import { WorkspaceContextTabsSetter } from './workspaceContext' +import './sourceCatalogBrowser.css' + +const sourceLabels:Record={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'} +const bytes=(value:number)=>value>=1073741824?`${(value/1073741824).toFixed(1)} GB`:value>=1048576?`${(value/1048576).toFixed(1)} MB`:value>=1024?`${(value/1024).toFixed(1)} KB`:`${value} B` +const duration=(milliseconds:number)=>milliseconds<=0?'未记录':milliseconds<1000?`${milliseconds}毫秒`:milliseconds<60000?`${Math.round(milliseconds/1000)}秒`:`${Math.floor(milliseconds/60000)}分${Math.round(milliseconds%60000/1000)}秒` +const scope=(value:string)=>value==='internal'?'对内':value==='external'?'对外':value==='both'?'对内 + 对外':value==='blocked'?'禁止导入':value==='mixed'?'混合':'未选择' +const progress=(node:CatalogNode)=>node.imported_count?`已解析 ${node.parsed_count} / 已导入 ${node.imported_count}`:'尚未导入' + +interface ViewProps{ + sources:CatalogSource[];selectedSource:CatalogSource|null;page:CatalogNodePage|null;loading:boolean;error:string|null + routingCapabilities?:CatalogRoutingCapabilities;routingMessage?:string;routingLoading?:boolean + refreshJob?:Job|null + onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise + history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void} + refreshHidden?:boolean;onToggleRefreshHidden?:()=>void +} +export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden}:ViewProps){ + const [selectedIDs,setSelectedIDs]=useState([]) + const [appliedPolicies,setAppliedPolicies]=useState>({}) + const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1) + const visibleItems=page?.items||[] + useEffect(()=>{setSelectedIDs([]);setAppliedPolicies({})},[selectedSource?.connection_id,page?.current.node_id]) + useEffect(()=>{if(routingMessage)setSelectedIDs([])},[routingMessage]) + const selectableNodes=visibleItems + const selectedNodeIDs=selectedIDs + const selectedCount=selectedIDs.length + const routeReady=!routingLoading + const policyEnabled=(policy:CatalogPolicy)=>policy==='blocked'||policy==='both'?(policy==='blocked'||routingCapabilities.internal_enabled&&routingCapabilities.external_enabled):policy==='internal'?routingCapabilities.internal_enabled:routingCapabilities.external_enabled + const toggleNode=(id:string)=>setSelectedIDs(current=>current.includes(id)?current.filter(value=>value!==id):[...current,id]) + const applyPolicy=async(policy:CatalogPolicy)=>{if(!selectedNodeIDs.length)return;const changed=await onRoute(selectedNodeIDs,policy);if(changed)setAppliedPolicies(current=>({...current,...Object.fromEntries(selectedNodeIDs.map(id=>[id,policy]))}))} + const refreshPayload=refreshJob?.payload&&typeof refreshJob.payload==='object'?refreshJob.payload as Record:{} + const refreshProgress=typeof refreshPayload.progress==='number'?refreshPayload.progress:0 + const refreshMessage=typeof refreshPayload.message==='string'?refreshPayload.message:'等待后台刷新' + const refreshDirectory=typeof refreshPayload.current_directory==='string'&&refreshPayload.current_directory?refreshPayload.current_directory:'' + const refreshFiles=typeof refreshPayload.scanned_files==='number'&&refreshPayload.scanned_files>0?refreshPayload.scanned_files:0 + const refreshEstimate=refreshPayload.total_is_estimate===true + const stageLabels:Record={queued:'排队中',connecting:'连接来源',scanning:'扫描目录',building:'构建快照',activating:'写入快照',completed:'完成',failed:'失败'} + const refreshStageLabel=typeof refreshPayload.stage==='string'?stageLabels[refreshPayload.stage]||refreshPayload.stage:'排队中' + const pillSource=typeof refreshPayload.source_name==='string'?refreshPayload.source_name:'' + const refreshPillSource=pillSource||selectedSource?.name||'' + const refreshActive=refreshJob?.state==='pending'||refreshJob?.state==='running'||refreshJob?.state==='retry' + return
+

来源与目录

各个知识源头的资料总览与标注

+ {error?
{error}
:null} + {refreshJob&&refreshActive&&refreshHidden?:null} + {refreshJob&&!(refreshActive&&refreshHidden)?<>
{refreshActive?'实时刷新中':refreshJob.state==='completed'?'实时快照已完成':'实时刷新失败'}{selectedSource?selectedSource.name+' · ':''}{refreshStageLabel}{refreshMessage}
{refreshProgress}%{refreshEstimate?'（估算）':''} · 后台运行，可以切换页面{refreshActive&&onToggleRefreshHidden?:null}
{refreshActive&&(refreshDirectory||refreshFiles)?
{refreshDirectory?当前目录：{refreshDirectory}:null}{refreshFiles?已发现 {refreshFiles} 份文件:null}
:null}{refreshJob.state==='failed'?

刷新失败；旧快照仍然保留，目录浏览与标注不受影响。

:null}:null} +
+ +
+ {selectedSource&&page?<> +
+ + +
+
{page.current.name}{selectedSource.snapshot_only?'本地只读快照，不代表实时连接':'实时目录'}
{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}含所有子目录文件 {page.current.subtree_file_count}总大小 {bytes(page.current.subtree_bytes)}
+ + {visibleItems.length?
+
+ +
已选 {selectedCount} 项
+
+ 禁止导入只保存期望策略；经“快照差异”确认后才会从测试知识库移除，不删除来源文件，也不操作生产知识库。 + {routingMessage?

{routingMessage}

:null} +
:null} + {visibleItems.length?
{visibleItems.map(node=>{const visibility=appliedPolicies[node.node_id]||node.visibility;const size=node.node_type==='folder'?(node.subtree_bytes>0?`已知合计 ${bytes(node.subtree_bytes)}`:'大小未提供'):typeof node.size_bytes==='number'?bytes(node.size_bytes):'大小未提供';return })}
名称类型处理策略文件数量导入与解析大小更新时间
{node.node_type==='folder'?
toggleNode(node.node_id)} />
:}		{node.node_type==='folder'?'文件夹':'文件'}
{scope(visibility)}
{node.node_type==='folder'?`${node.subtree_file_count} 份`:'—'}{progress(node)}{node.active_count?处理中 {node.active_count}:null}{node.failed_count?失败 {node.failed_count}:null}{size}{node.modified_at?new Date(node.modified_at).toLocaleString('zh-CN',{hour12:false}):'—'}
:
当前目录为空。
} + {page.total>page.limit?
第 {Math.floor(page.offset/page.limit)+1} / {Math.max(1,Math.ceil(page.total/page.limit))} 页
:null} +
快照就绪快照时间：{new Date(selectedSource.observed_at).toLocaleString('zh-CN',{hour12:false})}上次刷新耗时 {duration(selectedSource.last_refresh_duration_ms)}{history?:null}
+ {history&&history.open?:null} + :loading?
正在读取第一层目录…
:!error?
请选择一个内容来源。
:null} +
+
+
+} + +export function SourceCatalogBrowser(){ + const [sources,setSources]=useState([]);const [selected,setSelected]=useState(null);const [parentID,setParentID]=useState('');const [page,setPage]=useState(null);const [offset,setOffset]=useState(0);const [revision,setRevision]=useState(0);const [loading,setLoading]=useState(true);const [error,setError]=useState(null);const setContextTabs=useContext(WorkspaceContextTabsSetter) + const [routingCapabilities,setRoutingCapabilities]=useState({internal_enabled:false,external_enabled:false});const [routingMessage,setRoutingMessage]=useState('');const [routingLoading,setRoutingLoading]=useState(false) + const [refreshJob,setRefreshJob]=useState(null) + const [historySnapshots,setHistorySnapshots]=useState([]);const [historyActive,setHistoryActive]=useState(null);const [historyPage,setHistoryPage]=useState(null);const [historyLoading,setHistoryLoading]=useState(false);const [historyError,setHistoryError]=useState(null);const [historyOpen,setHistoryOpen]=useState(false) + const [refreshHidden,setRefreshHidden]=useState(false) + + const openHistory=async()=>{if(!selected||historyLoading)return;if(historyOpen){closeHistory();return}setHistoryOpen(true);setHistoryLoading(true);setHistoryError(null);try{const result=await fetchCatalogSnapshots(selected.connection_id,20,0);setHistorySnapshots(result.items);const head=result.items.find(item=>item.is_head)||result.items[0]||null;setHistoryActive(head);if(head)setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,head.snapshot_id,head.root_node_id));else setHistoryPage(null)}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载快照历史失败')}finally{setHistoryLoading(false)}} + const openHistorySnapshot=async(snapshot:CatalogSnapshot)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryActive(snapshot);setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,snapshot.root_node_id))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}} + const openHistoryFolder=async(snapshot:CatalogSnapshot,nodeID:string)=>{if(!selected||historyLoading)return;setHistoryLoading(true);setHistoryError(null);try{setHistoryPage(await fetchCatalogSnapshotNodes(selected.connection_id,snapshot.snapshot_id,nodeID))}catch(cause){setHistoryError(cause instanceof Error?cause.message:'加载历史快照目录失败')}finally{setHistoryLoading(false)}} + const closeHistory=()=>{setHistoryOpen(false);setHistoryActive(null);setHistoryPage(null);setHistoryError(null)} + + useEffect(()=>{const controller=new AbortController();setLoading(true);void fetchCatalogSources(controller.signal).then(result=>{if(controller.signal.aborted)return;setSources(result);setSelected(current=>current&&result.find(source=>source.connection_id===current.connection_id)||result[0]||null);setParentID(current=>current||result[0]?.root_node_id||'');setError(null)}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载来源目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[revision]) + useEffect(()=>{if(!selected||!parentID)return;const controller=new AbortController();setLoading(true);void fetchCatalogNodes(selected.connection_id,parentID,50,offset,controller.signal).then(result=>{if(!controller.signal.aborted){setPage(result);setError(null)}}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载目录失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[selected,parentID,offset,revision]) + + useEffect(()=>{const controller=new AbortController();void fetchCatalogRoutingCapabilities(controller.signal).then(setRoutingCapabilities).catch(()=>setRoutingCapabilities({internal_enabled:false,external_enabled:false}));return()=>controller.abort()},[revision]) + useEffect(()=>{if(!selected)return;const controller=new AbortController();void fetchJobs(controller.signal).then(items=>{if(controller.signal.aborted)return;const match=items.find(job=>job.type==='source.catalog.refresh'&&job.payload&&typeof job.payload==='object'&&(job.payload as Record).connection_id===selected.connection_id);setRefreshJob(match||null)}).catch(()=>undefined);return()=>controller.abort()},[selected]) + useEffect(()=>{if(!refreshJob||!['pending','running','retry'].includes(refreshJob.state))return;const timer=window.setTimeout(()=>void fetchJobs().then(items=>{const current=items.find(job=>job.id===refreshJob.id);if(!current)return;setRefreshJob(previous=>{if(previous?.state!=='completed'&¤t.state==='completed')setRevision(value=>value+1);return current})}).catch(()=>undefined),750);return()=>window.clearTimeout(timer)},[refreshJob]) + useEffect(()=>{if(!selected||!page){setContextTabs([]);return}setContextTabs(page.breadcrumbs.map(item=>({key:item.node_id,label:item.name,onActivate:()=>{setParentID(item.node_id);setOffset(0)}})));return()=>setContextTabs([])},[selected,page,setContextTabs]) + const selectSource=(source:CatalogSource)=>{setSelected(source);setParentID(source.root_node_id);setOffset(0);setPage(null);setRefreshJob(null);closeHistory()} + const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)} + const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,operator_ref:'web-console',reason:`目录页策略：${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份；可处理 ${result.routable_files} 份；内容未校验 ${result.hash_unknown_files} 份；未关联来源文件 ${result.unbound_files} 份；其余 ${result.unavailable_files} 份暂不可处理。变化需到“快照差异”确认后才会进入解析队列。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}} + const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}} + return {setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} /> +} diff --git a/web/src/SourceLinkManager.test.tsx b/web/src/SourceLinkManager.test.tsx new file mode 100644 index 0000000..d52f411 --- /dev/null +++ b/web/src/SourceLinkManager.test.tsx @@ -0,0 +1,26 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { SourceLinkManagerView } from './SourceLinkManager' +import type { CatalogSource } from './sourceCatalog' +const sources:CatalogSource[]=[ + {connection_id:'ding',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'r1',folder_count:8,file_count:3468,observed_at:'2026-09-08T00:00:00Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}, + {connection_id:'pp',source_type:'pp',name:'PP资料下载中心',root_node_id:'r2',folder_count:64,file_count:613,observed_at:'2026-09-08T00:00:00Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}, + {connection_id:'alist',source_type:'alist',name:'国内工业',root_node_id:'r3',folder_count:1688,file_count:4880,observed_at:'2026-09-08T00:00:00Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}, +] +describe('SourceLinkManagerView',()=>{ + it('shows understandable URL credential and login fields without accepting real secrets locally',()=>{ + const html=renderToStaticMarkup() + for(const text of ['来源链接','本地模拟配置','公司级技术支持知识库','PP资料下载中心','国内工业','知识库链接','认证方式','专用浏览器扫码登录','Key或Token','本地模拟阶段不接收真实Key','模拟登录并验证目录','不保存真实Key'])expect(html).toContain(text) + expect(html).not.toContain('secret_ref') + expect(html).not.toContain('base_url') + }) + it('separates an AList site address from its starting directory and labels simulated status honestly',()=>{ + const html=renderToStaticMarkup() + for(const text of ['AList站点地址','起始目录','/国内工业','模拟验证待执行','不下载文件内容，只读取目录和元数据'])expect(html).toContain(text) + expect(html).not.toContain('实时连接待验证') + }) + it('does not show a successful zero-source state after errors',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('加载失败');expect(html).not.toContain('尚无来源链接') + }) +}) diff --git a/web/src/SourceLinkManager.tsx b/web/src/SourceLinkManager.tsx new file mode 100644 index 0000000..50b8493 --- /dev/null +++ b/web/src/SourceLinkManager.tsx @@ -0,0 +1,35 @@ +import { useContext,useEffect,useState } from 'react' +import { fetchCatalogSources } from './sourceCatalog' +import type { CatalogSource } from './sourceCatalog' +import { WorkspaceContextTabsSetter } from './workspaceContext' +import './sourceLinkManager.css' +const typeLabel={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'} +const authLabel={dingtalk:'专用浏览器扫码登录',pp:'公开访问，无需Key',alist:'只读Token'} +const actionLabel={dingtalk:'模拟登录并验证目录',pp:'模拟验证公开目录',alist:'模拟验证只读Token'} +const placeholder={dingtalk:'粘贴钉钉知识库网页链接（模拟）',pp:'粘贴PP资料下载中心链接（模拟）',alist:'例如：https://alist.example.com（模拟）'} +const linkLabel={dingtalk:'知识库链接',pp:'资料中心地址',alist:'AList站点地址'} +interface Props{sources:CatalogSource[];selected:CatalogSource|null;validated:boolean;loading:boolean;error:string|null;onSelect:(source:CatalogSource)=>void;onValidate:()=>void;onRefresh:()=>void} +export function SourceLinkManagerView({sources,selected,validated,loading,error,onSelect,onValidate,onRefresh}:Props){ + const [link,setLink]=useState('') + return
+

连接配置

来源链接

配置钉钉、PP和AList的入口、认证方式及目录验证。

+ {error?
{error}
:null} +
本地模拟配置使用真实来源名称和快照数量，但不连接登录服务、不保存真实Key。
+
+ +
{selected?<>

{selected.name}

{typeLabel[selected.source_type]} · 本地只读快照
模拟验证待执行
+
+
当前快照文件 {selected.file_count}目录 {selected.folder_count}不下载文件内容，只读取目录和元数据不执行上传、移动、重命名或删除
+ + {validated?
网络连接模拟通过
登录或Token模拟通过
根目录模拟通过
读取第一层目录模拟通过
只读边界模拟通过
以上只是本地交互模拟；迁移到测试服务器后才执行真实验证。
:null} +

真实模式下，新凭据先验证，成功后才替换旧凭据；网页永远不能读取已保存Key。

+ :loading?
正在加载来源…
:null}
+
+
+} +export function SourceLinkManager(){ + const [sources,setSources]=useState([]);const [selected,setSelected]=useState(null);const [validated,setValidated]=useState(false);const [loading,setLoading]=useState(true);const [error,setError]=useState(null);const [revision,setRevision]=useState(0);const setContextTabs=useContext(WorkspaceContextTabsSetter) + useEffect(()=>{const controller=new AbortController();setLoading(true);void fetchCatalogSources(controller.signal).then(result=>{if(!controller.signal.aborted){setSources(result);setSelected(current=>current&&result.find(item=>item.connection_id===current.connection_id)||result[0]||null);setError(null)}}).catch(cause=>{if(!controller.signal.aborted)setError(cause instanceof Error?cause.message:'加载来源链接失败')}).finally(()=>{if(!controller.signal.aborted)setLoading(false)});return()=>controller.abort()},[revision]) + useEffect(()=>{setContextTabs(selected?[{key:selected.connection_id,label:selected.name,onActivate:()=>undefined}]:[]);return()=>setContextTabs([])},[selected,setContextTabs]) + return {setSelected(source);setValidated(false)}} onValidate={()=>setValidated(true)} onRefresh={()=>{setValidated(false);setRevision(value=>value+1)}} /> +} diff --git a/web/src/SystemSettings.test.tsx b/web/src/SystemSettings.test.tsx new file mode 100644 index 0000000..03c8e5c --- /dev/null +++ b/web/src/SystemSettings.test.tsx @@ -0,0 +1,49 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { SystemSettingsView, visibleSaveMessage } from './SystemSettings' +import type { MonitorOverview } from './monitor' +import type { CatalogSource } from './sourceCatalog' +import type { UISettings } from './uiSettings' +import type { SourceConnectionListItem } from './sourceConnections' +const monitor:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:1,memory_total_bytes:2,disk_used_bytes:1,disk_total_bytes:2,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟',latency_ms:1,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[],health:[{key:'mineru',name:'MinerU',liveness:'up',readiness:'blocked',reason:'服务可达但容量已满（429），暂时不可接单',latency_ms:30,observed_at:'2026-09-08T12:00:00Z',last_success_at:'2026-09-08T11:59:00Z',consecutive_failures:2,configured:true,applicable:true}]} +const sources:CatalogSource[]=[{connection_id:'ding-1',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'root-1',folder_count:1079,file_count:3444,observed_at:'2026-09-09T09:30:32Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}] +const connections:SourceConnectionListItem[]=[{id:'conn-ding',source_type:'dingtalk',name:'公司级技术支持知识库',enabled:true,schedule_configured:true,health_status:'healthy',item_count:3444,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:'2026-09-12T10:00:00Z'},{id:'conn-man',source_type:'manual',name:'本地手工验收',enabled:false,schedule_configured:false,health_status:'unknown',item_count:5,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:null}] +const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''} +describe('SystemSettingsView',()=>{ + it('separates document-engineer settings from locked administrator and production controls',()=>{ + const html=renderToStaticMarkup() + for(const text of ['系统设置','数据源配置','服务配置','存储与备份','刷新策略','设置服务已启用','当前活动版本','公司级技术支持知识库','钉钉知识库','认证方式','端点与凭据','文档工程师可调整','管理员设置已锁定'])expect(html).toContain(text) + const servicesHtml=renderToStaticMarkup() + for(const text of ['测试 WeKnora','生产 WeKnora','MinerU','Embedding'])expect(servicesHtml).toContain(text) + const storageHtml=renderToStaticMarkup() + expect(storageHtml).toContain('生产发布请求') + expect(storageHtml).toContain('仅保存凭据引用，不可回读') + const refreshHtml=renderToStaticMarkup() + for(const text of ['刷新与告警参数已验证并启用','停滞30分钟','finalizing 2小时','文档状态刷新间隔','15秒','解析停滞判定','30分钟','保存并应用刷新与告警参数'])expect(refreshHtml).toContain(text) + expect(visibleSaveMessage('已保存',true)).toBeNull();expect(visibleSaveMessage('已保存',false)).toBe('已保存') + expect(html).not.toContain('本地模拟');expect(html).not.toContain('保存模拟设置') + for(const forbidden of ['DATABASE_URL','RELEASE_CONFIG_FILE','API Key明文','secret_ref'])expect(html).not.toContain(forbidden) + // P10 dual-state matrix: reachable-but-not-workable must be explicit (services section). + expect(servicesHtml).toContain('可达 · 不可接单') + expect(servicesHtml).toContain('服务可达但容量已满（429）') + expect(servicesHtml).toContain('连续失败2次') + expect(servicesHtml).toContain('PostgreSQL') + expect(servicesHtml).toContain('对内解析队列：已配置') + expect(servicesHtml).toContain('对外解析队列：已配置') + // V1: left-nav section controls, per-source schedule editing, danger modal & history affordances. + expect(html).toContain('settings-nav') + expect(html).toContain('下次自动刷新') + expect(html).toContain('仅手动刷新') + expect(html).toContain('查看修订历史') + expect(html).toContain('保存并启用调度') + expect(html).toContain('停用来源') + expect(html).toContain('本地手工导入来源不支持定时刷新') + // dead buttons removed + expect(html).not.toContain('添加内容源') + expect(html).not.toContain('管理员编辑') + }) + it('does not show healthy simulation when the settings data failed to load',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('加载失败');expect(html).not.toContain('模拟运行中') + }) +}) diff --git a/web/src/SystemSettings.tsx b/web/src/SystemSettings.tsx new file mode 100644 index 0000000..d9d3416 --- /dev/null +++ b/web/src/SystemSettings.tsx @@ -0,0 +1,222 @@ +import { useCallback, useEffect, useState } from 'react' + +import { fetchMonitorOverview } from './monitor' +import type { MonitorOverview } from './monitor' +import { fetchReleaseCapabilities } from './reviewRelease' +import { fetchCatalogRoutingCapabilities, fetchCatalogSources } from './sourceCatalog' +import type { CatalogRoutingCapabilities, CatalogSource } from './sourceCatalog' +import { activateSourceSettingsRevision, createSourceSettingsRevision, fetchSourceConnectionList, fetchSourceSettingsRevisions } from './sourceConnections' +import type { SourceConnectionListItem, SourceSettingsRevision } from './sourceConnections' +import { activateUISettings, createUISettingsCandidate, fetchActiveUISettings } from './uiSettings' +import type { UISettings, UISettingsCandidateInput } from './uiSettings' +import './systemSettings.css' + +interface Props { + sources: CatalogSource[] + connections: SourceConnectionListItem[] + monitor: MonitorOverview | null + settings: UISettings | null + productionEnabled: boolean + routingCapabilities: CatalogRoutingCapabilities + loading: boolean + saving: boolean + error: string | null + saveMessage: string | null + onSave: (input: UISettingsCandidateInput) => void + onSourcesChanged: () => void + onRefresh: () => void + initialSection?: SectionKey +} + +type SectionKey = 'sources' | 'services' | 'storage' | 'refresh' + +const sections: { key: SectionKey; label: string }[] = [ + { key: 'sources', label: '数据源配置' }, + { key: 'services', label: '服务配置' }, + { key: 'storage', label: '存储与备份' }, + { key: 'refresh', label: '刷新策略' }, +] + +const sourceTypeLabels: Record = { dingtalk: '钉钉知识库', pp: 'PP 资料下载中心', alist: 'AList 文件源', manual: '本地手工导入' } +const sourceAuthLabels: Record = { dingtalk: '专用浏览器扫码会话', pp: '公开只读访问', alist: '直连 API Token', manual: '离线导入，无在线认证' } +const secondsLabel = (value: number) => value < 60 ? `${value}秒` : value < 3600 ? `${value / 60}分钟` : `${value / 3600}小时` +export const visibleSaveMessage = (message: string | null, dirty: boolean) => dirty ? null : message + +const intervalChoices: { value: number; label: string }[] = [ + { value: 0, label: '仅手动刷新' }, + { value: 3600, label: '每小时' }, + { value: 21600, label: '每6小时' }, + { value: 86400, label: '每24小时' }, +] + +function formatHorizon(value: string | null): string { + if (!value) return '仅手动刷新' + const at = new Date(value) + if (Number.isNaN(at.getTime())) return '仅手动刷新' + return `下次自动刷新 ${at.toLocaleString('zh-CN', { hour12: false })}` +} + +function SourceSettingsCard({ connection, onSourcesChanged }: { connection: SourceConnectionListItem; onSourcesChanged: () => void }) { + const [operator, setOperator] = useState('') + const [interval, setInterval] = useState(() => { + if (!connection.next_refresh_at) return 0 + const remaining = new Date(connection.next_refresh_at).getTime() - Date.now() + if (remaining <= 0 || remaining > 10 * 3600_000) return 86400 + return remaining > 3 * 3600_000 ? 21600 : 3600 + }) + const [busy, setBusy] = useState(false) + const [message, setMessage] = useState(null) + const [failure, setFailure] = useState(null) + const [showHistory, setShowHistory] = useState(false) + const [history, setHistory] = useState(null) + const [pendingDisable, setPendingDisable] = useState(false) + const scheduled = connection.source_type !== 'manual' + + const submit = async (nextEnabled: boolean) => { + setBusy(true); setMessage(null); setFailure(null) + try { + const draft = { + source_type: connection.source_type, name: connection.name, endpoint: '', credential_ref: '', + enabled: nextEnabled, schedule: { refresh_interval_seconds: nextEnabled && interval > 0 ? interval : null }, + operator: operator.trim(), + } + const created = await createSourceSettingsRevision(connection.id, draft) + const activated = await activateSourceSettingsRevision(connection.id, created.id) + setMessage(`${connection.name}：调度设置已验证并启用${activated.operation === 'replayed' ? '（沿用上次结果）' : ''}；${nextEnabled ? formatHorizon(new Date(Date.now() + interval * 1000).toISOString()) : '已停用，新的自动扫描与刷新任务不再入队，历史记录全部保留'}。`) + onSourcesChanged() + } catch (cause) { + setFailure(cause instanceof Error ? cause.message : '保存来源设置失败') + } finally { + setBusy(false) + setPendingDisable(false) + } + } + + const loadHistory = async () => { + setShowHistory(true) + if (history) return + try { setHistory(await fetchSourceSettingsRevisions(connection.id)) } + catch (cause) { setFailure(cause instanceof Error ? cause.message : '加载修订历史失败') } + } + + const canApply = operator.trim().length >= 2 && !busy + + return
+
{connection.name}{sourceTypeLabels[connection.source_type] ?? connection.source_type}{connection.enabled ? '' : ' · 已停用'}
+
+
认证方式
{sourceAuthLabels[connection.source_type] ?? '按连接配置'}
+
端点与凭据
已保存，不可回读（草稿留空即沿用现值）
+
当前调度
{connection.enabled ? formatHorizon(connection.next_refresh_at) : '停用中（不入队）'}
+
+
+ {scheduled ? + : 本地手工导入来源不支持定时刷新，只能手动触发扫描。} + + {connection.enabled + ?
+
+ :
} + +
+ {message ?

{message}

: null} + {failure ?

{failure}

: null} + {showHistory && history ?
    {history.map((item) =>
  • 修订 {item.revision_number}{item.state === 'active' ? '当前启用' : item.state === 'validated' ? '待启用' : item.state === 'retired' ? '已退休' : '失败'}{item.created_by} · {new Date(item.created_at).toLocaleString('zh-CN', { hour12: false })} · {item.refresh_interval_seconds ? `每${secondsLabel(Number(item.refresh_interval_seconds))}` : '仅手动'}
    • )}
: null} + {showHistory && !history ? 正在加载修订历史… : null} + {pendingDisable ?
+

停用「{connection.name}」？

+

停用会立即停止该来源的自动扫描与新刷新任务入队；历史快照、标注和审计全部保留，随时可以重新启用。此操作会先由后端再次校验权限。

+
+
: null} +
+} + +export function SystemSettingsView({ sources, connections, monitor, settings, productionEnabled, routingCapabilities, loading, saving, error, saveMessage, onSave, onSourcesChanged, onRefresh, initialSection = 'sources' }: Props) { + const [section, setSection] = useState(initialSection) + const [documentRefresh, setDocumentRefresh] = useState<15 | 30 | 60>(settings?.document_refresh_seconds ?? 15) + const [resourceRefresh, setResourceRefresh] = useState<30 | 60>(settings?.resource_refresh_seconds ?? 30) + const [stalled, setStalled] = useState<1800 | 3600>(settings?.stalled_after_seconds ?? 1800) + const [finalizing, setFinalizing] = useState<7200 | 14400>(settings?.finalizing_after_seconds ?? 7200) + const [operator, setOperator] = useState('') + const [dirty, setDirty] = useState(false) + + useEffect(() => { + if (!settings) return + setDocumentRefresh(settings.document_refresh_seconds) + setResourceRefresh(settings.resource_refresh_seconds) + setStalled(settings.stalled_after_seconds) + setFinalizing(settings.finalizing_after_seconds) + setDirty(false) + }, [settings]) + + const canSave = Boolean(settings && operator.trim().length >= 2 && !saving) + return
+

系统设置

查看内容源、服务与备份状态；来源调度与刷新参数采用版本化保存。

+ {error ?
{error}
: null} + {monitor && settings ?
+ +
+
文档工程师可调整：来源调度周期、刷新间隔、停滞判断管理员设置已锁定：端点、凭据、映射、生产开关
+
设置服务已启用当前活动版本 · 启用人 {settings.updated_by} · 启用时间 {settings.activated_at ? new Date(settings.activated_at).toLocaleString('zh-CN') : '待激活'}；新设置先验证，再原子切换。
+ {section === 'sources' ?
+

数据源配置

逐来源设置自动刷新周期或保持仅手动；保存会先生成修订、验证后原子启用，旧版本保留用于审计。

+ {connections.length ?
+ {connections.map((connection) => )} + {sources.length ? null : 尚无快照数据；快照将在来源首次刷新完成后出现。} +
:
尚无可显示的来源连接。
} +
: null} + {section === 'services' ?
+

服务配置

测试、生产必须独立配置端点、凭据引用和知识库 ID；凭据只写不读。

+
测试 WeKnora本地真实连接端点 / 凭据 / 知识库 ID：管理员受限配置，页面上不可回读
生产 WeKnora未连接生产写入：关闭；凭据不可回读
+
测试知识库映射对内解析队列：{routingCapabilities.internal_enabled ? '已配置' : '未配置'}对外解析队列：{routingCapabilities.external_enabled ? '已配置' : '未配置'}生产映射：未配置
+
{['PostgreSQL', 'MinerU', 'Embedding', 'LLM API', 'Redis / MQ', 'OSS'].map((name) => { const health = (monitor.health || []).find((item) => item.name === name || item.name.startsWith(name)); const service = monitor.services.find((item) => item.name === name || item.name.startsWith(name)); const liveLabel = !health ? (service ? (service.state === 'healthy' ? '正常' : service.state === 'unknown' ? '未接入' : '需关注') : '待配置') : health.readiness === 'ready' ? '可达 · 可接单' : health.readiness === 'not_configured' ? '状态未知 · 未配置' : health.readiness === 'not_applicable' ? '未部署' : (health.liveness === 'up' ? '可达 · 不可接单' : '不可达 · 不可接单'); return
{name}{liveLabel}{health ? [health.reason, health.last_success_at ? `最近成功 ${new Date(health.last_success_at).toLocaleTimeString('zh-CN', { hour12: false })}` : '', health.consecutive_failures ? `连续失败${health.consecutive_failures}次` : ''].filter(Boolean).join('；') || (health.readiness === 'ready' ? `延迟 ${health.latency_ms}ms` : '') : service?.detail || '真实 Provider 未接入'}
})}
+
: null} + {section === 'storage' ?
+

存储与备份

文件资源与生产发布前快照使用独立受限配置。

+
Endpoint / Bucket
尚未配置
AccessKey / Secret
仅保存凭据引用，不可回读
CDN / 公开访问域名
尚未配置
临时签名有效期
待配置
测试备份前缀
/backup/weknora/test/
生产备份前缀
/backup/weknora/prod/
快照保留
最近 30 个生产快照
恢复验证
由生产门禁读回
  • PostgreSQL 未配置
  • Elasticsearch 未配置
  • 对象存储 未配置
  • Neo4j 未配置
  • 生产发布请求 {productionEnabled ? '等待管理员核对' : '关闭'}
  • Release Worker 网页不可启用
+
: null} + {section === 'refresh' ?
+

刷新策略

这些值保存在服务器活动版本中；保存不会修改端点、凭据或生产开关。

当前活动版本
+
+ + + + +
+ + + {visibleSaveMessage(saveMessage, dirty) ?

{saveMessage}

: null} + 当前：文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}。 +
: null} +
+
: null} +
+} + +export function SystemSettings() { + const [sources, setSources] = useState([]) + const [connections, setConnections] = useState([]) + const [monitor, setMonitor] = useState(null) + const [settings, setSettings] = useState(null) + const [productionEnabled, setProductionEnabled] = useState(false) + const [routingCapabilities, setRoutingCapabilities] = useState({ internal_enabled: false, external_enabled: false }) + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [error, setError] = useState(null) + const [saveMessage, setSaveMessage] = useState(null) + const [revision, setRevision] = useState(0) + + useEffect(() => { + const controller = new AbortController(); setLoading(true) + void Promise.all([fetchCatalogSources(controller.signal), fetchSourceConnectionList(controller.signal), fetchMonitorOverview(controller.signal), fetchReleaseCapabilities(controller.signal), fetchCatalogRoutingCapabilities(controller.signal), fetchActiveUISettings(controller.signal)]).then(([nextSources, nextConnections, overview, capabilities, routing, active]) => { if (!controller.signal.aborted) { setSources(nextSources); setConnections(nextConnections); setMonitor(overview); setProductionEnabled(capabilities.production_release_requests_enabled); setRoutingCapabilities(routing); setSettings(active); setError(null) } }).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载系统设置失败') }).finally(() => { if (!controller.signal.aborted) setLoading(false) }) + return () => controller.abort() + }, [revision]) + + const save = async (input: UISettingsCandidateInput) => { + setSaving(true); setSaveMessage(null); setError(null) + try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用：文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)}；旧版本已保留用于审计。`) } + catch (cause) { setError(cause instanceof Error ? cause.message : '保存系统设置失败') } + finally { setSaving(false) } + } + const reloadSources = useCallback(() => { setSaveMessage(null); setRevision((value) => value + 1) }, []) + return void save(input)} onSourcesChanged={reloadSources} onRefresh={reloadSources}/> +} diff --git a/web/src/WeKnoraMonitor.test.tsx b/web/src/WeKnoraMonitor.test.tsx new file mode 100644 index 0000000..2f4b35f --- /dev/null +++ b/web/src/WeKnoraMonitor.test.tsx @@ -0,0 +1,65 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { WeKnoraMonitorView } from './WeKnoraMonitor' +import type { MonitorFilePage,MonitorOverview,RunMetrics } from './monitor' +import type { TestPublicationPage } from './testPublications' +import type { CatalogChangeSetDetail } from './sourceCatalog' + +const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2，队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],health:[]} +const files:MonitorFilePage={items:[{name:'[模拟] 产品参数总表.xlsx',directory:'02｜产品资料',visibility:'external',status:'finalizing',stage_detail:'基础检索可用，Wiki与图谱仍在生成',elapsed_seconds:2100,error_code:''}],total:1,limit:20,offset:0,sample_only:true} +const metrics:RunMetrics={observed_at:'2026-09-12T00:00:00Z',active_workers:2,worker_source:'pg_lease',queue:{pending:3,running:4,retry:1,failed:2,oldest_pending_age_seconds:120},short_window:{window_seconds:3600,jobs_started:50,jobs_completed:48,jobs_failed:6,success_rate:88.9,failure_rate:11.1,duration_avg_seconds:55,duration_p50_seconds:30,duration_p95_seconds:1800},long_window:{window_seconds:86400,jobs_started:500,jobs_completed:482,jobs_failed:18,success_rate:96.4,failure_rate:3.6,duration_avg_seconds:70,duration_p50_seconds:45,duration_p95_seconds:2400}} +const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'pending',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: 'cs-1', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: 'cs-1', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:2,limit:50,offset:0} +const changeSets:CatalogChangeSetDetail[]=[{id:'cs-1',connection_id:'conn-1',source_name:'国内工业',state:'planned',add_count:2,update_count:1,move_count:1,move_scope_count:1,delete_count:1,no_change_count:10,created_at:'2026-09-10T00:00:00Z',items:[{action:'delete',title:'旧手册.pdf',old_path:'旧目录/旧手册.pdf',new_path:'',reason_code:'source_missing',seq:7}],total:15,limit:20,offset:0}] + +const baseProps={overview,files,queue,changeSets,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()} +describe('WeKnoraMonitorView',()=>{ + it('filters the processing queue by target knowledge base and shows each task target',()=>{ + const html=renderToStaticMarkup() + expect(html).toContain('aria-label="队列目标知识库"') + expect(html).toContain('对内知识库') + expect(html).toContain('对外知识库') + expect(html).toContain('