From e6472b0ec67684a6e0e09372d67699ac9454f0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Fri, 30 Oct 2026 18:37:38 +0800 Subject: [PATCH] =?UTF-8?q?feat(library):=20=E4=BA=91=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E7=9B=AE=E6=A0=87=E6=8E=A5=E5=85=A5=E7=9A=84?= =?UTF-8?q?=E8=A7=92=E8=89=B2=E5=88=86=E7=A6=BB=E3=80=81=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E6=90=AC=E8=BF=90=E4=B8=8E=E5=9B=BE=E8=B0=B1=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么改:发布执行链是"与目标同机"假设写死的(本机 docker.sock、本机 drain.json、 本机 Redis/PG),把目标换成云测试不能只改展示 URL;直接搬整套 worker 又会抢走 本机封存/QA/周报队列并按本机配置写错目标。所以先把"谁消费哪段队列"变成显式角色, 再补齐跨主机搬运与目标侧门禁。 角色(LIBRARY_WORKER_ROLE=full/release-executor/capture)不只影响 handler 列表, 还决定是否拥有 DDL(release-executor 只读校验迁移版本,落后即拒绝启动)、 收拾哪些 library_jobs.kind、是否跑 release 接管扫描、是否起 scheduler/QA/周报/探测。 capture 保留本机原有全部业务;release-executor 只挂 sync/restore/recover/baseline。 快照跨主机:新增内容寻址的 bundle(版本清单先落、blob 按 sha256 命名、清单最后 Link 作提交点),pack/unpack 幂等且拒绝覆盖与同名不同内容;执行器使用前仍逐 blob 重算 sha256,因此传输不可变有两道证据。 目标侧门禁:目标知识库 graph_enabled=true 时拒绝发布(发布只写 PG/ES/OSS,不写 Neo4j;未知也拒绝),只读检测、不改目标配置、不做图谱迁移。 云端入口:新增 drain 感知的 ingress 闸门(建连超时与读超时分离,半关闭单向传播, SIGTERM 排空后写 drain.json 并 exit 0/2)、compose 叠加片段(app 不再发布端口、 frontend 与 adapter 的入口收敛到闸门、指向真实网络 weknora-test-v071-network)、 执行器启动脚本与治理 PG 反向隧道脚本(默认 unix socket,不占 TCP)。 验证:deploy/cloud-test/test_gate.py 13 项全 PASS(含 12 秒静默流、半关闭、2MiB 完整、 排空 exit 0/active_at_close=1、排空超时 exit 2、上游不可达不挂住);go test 相关包全绿; scripts/ci_checks.py 16 项全部 success(EXIT=0,1241 用例通过/0 失败/191 已声明可选集成跳过); 云测试只读预检 17 项 / 6 FAIL,FAIL 项即当前部署缺口(无闸门、app 仍发布端口、 adapter 与 frontend 仍直连 app、云端归档根未建),未部署、未改云端任何状态。 --- Dockerfile | 1 + cmd/libraryctl/main.go | 57 ++- cmd/worker/library_release_handlers.go | 33 ++ cmd/worker/library_role_test.go | 87 +++++ cmd/worker/main.go | 222 +++++------ deploy/cloud-test/README.md | 176 +++++++++ .../cloud-test/compose.release-overlay.yaml | 66 ++++ deploy/cloud-test/gate.py | 193 ++++++++++ deploy/cloud-test/pg-forward.sh | 63 ++++ .../cloud-test/release-executor.env.example | 61 +++ deploy/cloud-test/run-release-executor.sh | 48 +++ deploy/cloud-test/test_gate.py | 350 +++++++++++++++++ internal/appconfig/config.go | 11 +- internal/appconfig/library_role.go | 84 +++++ internal/appconfig/library_role_test.go | 83 ++++ internal/librarysnapshot/transfer.go | 354 ++++++++++++++++++ internal/librarysnapshot/transfer_test.go | 225 +++++++++++ internal/librarystack/stack.go | 6 + internal/platform/postgres/library_jobs.go | 21 +- .../postgres/library_reconcile_test.go | 18 + internal/platform/postgres/migrate.go | 44 +++ .../releaseadapter/v071/graph_gate_test.go | 26 ++ .../v071/library_target_contract.go | 45 +++ scripts/verify_cloud_release_target.py | 252 +++++++++++++ 24 files changed, 2409 insertions(+), 117 deletions(-) create mode 100644 cmd/worker/library_release_handlers.go create mode 100644 cmd/worker/library_role_test.go create mode 100644 deploy/cloud-test/README.md create mode 100644 deploy/cloud-test/compose.release-overlay.yaml create mode 100755 deploy/cloud-test/gate.py create mode 100755 deploy/cloud-test/pg-forward.sh create mode 100644 deploy/cloud-test/release-executor.env.example create mode 100755 deploy/cloud-test/run-release-executor.sh create mode 100755 deploy/cloud-test/test_gate.py create mode 100644 internal/appconfig/library_role.go create mode 100644 internal/appconfig/library_role_test.go create mode 100644 internal/librarysnapshot/transfer.go create mode 100644 internal/librarysnapshot/transfer_test.go create mode 100644 internal/platform/postgres/library_reconcile_test.go create mode 100644 internal/releaseadapter/v071/graph_gate_test.go create mode 100755 scripts/verify_cloud_release_target.py diff --git a/Dockerfile b/Dockerfile index acf3933..0cd3aa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN LDFLAGS="-X knowledge-governance/internal/version.Number=${VERSION} -X knowl && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/api ./cmd/api \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/worker ./cmd/worker \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/release-worker ./cmd/release-worker \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/libraryctl ./cmd/libraryctl \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/test-publication-seed ./cmd/test-publication-seed \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/test-weknora-e2e-report ./cmd/test-weknora-e2e-report \ && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags "$LDFLAGS" -o /out/test-weknora-probe ./cmd/test-weknora-probe diff --git a/cmd/libraryctl/main.go b/cmd/libraryctl/main.go index 7b2206b..0011357 100644 --- a/cmd/libraryctl/main.go +++ b/cmd/libraryctl/main.go @@ -60,6 +60,8 @@ type options struct { job string dryRun bool snapshot string + // bundle 是快照跨主机包路径(snapshot-pack 输出 / snapshot-unpack 输入)。 + bundle string // releasedDeadLetters 是治理平台审计过的精确解除指纹(逗号分隔)。它只让这些 // 确切的行不再阻断;实时行必须逐字节命中,否则照旧拒绝。 releasedDeadLetters string @@ -100,6 +102,10 @@ func main() { runErr = runRecover(ctx, opts) case "dead-letters": runErr = runDeadLetters(ctx, opts) + case "snapshot-pack": + runErr = runSnapshotPack(opts) + case "snapshot-unpack": + runErr = runSnapshotUnpack(opts) default: usage() os.Exit(2) @@ -111,7 +117,7 @@ func main() { } func usage() { - fmt.Fprint(os.Stderr, `libraryctl [flags] + fmt.Fprint(os.Stderr, `libraryctl [flags] status probe the real source and print what is configured / blocked preflight read the source, run settlement + candidate validation (no writes, no stop) @@ -121,6 +127,9 @@ func usage() { recover restore writers from an interrupted freeze journal dead-letters print the source's live dead letters (read-only) with a digest the governance platform binds a precise release to + snapshot-pack export one sealed snapshot into a single verifiable bundle file + snapshot-unpack verify a bundle and install it into this archive (content-addressed, + version manifest is the commit point; repeated import is idempotent) Flags (defaults target the local WKNFull test stack): -source-app WKNFull-app container whose env describes the source @@ -177,6 +186,7 @@ func parseFlags(command string, args []string) (options, []string, error) { set.BoolVar(&opts.dryRun, "dry-run", false, "export and validate without archiving or stopping") set.StringVar(&opts.snapshot, "snapshot", "", "snapshot id for verify") set.StringVar(&opts.releasedDeadLetters, "released-dead-letters", "", "audited dead-letter fingerprints (comma separated)") + set.StringVar(&opts.bundle, "bundle", "", "snapshot bundle path (snapshot-pack / snapshot-unpack)") if err := set.Parse(args); err != nil { return opts, nil, err } @@ -1151,3 +1161,48 @@ func runDeadLetters(ctx context.Context, opts options) error { printJSON(map[string]any{"observation": json.RawMessage(payload), "observation_digest": digest}) return nil } + +// runSnapshotPack 导出一份已封存版本,供另一台主机(发布执行器所在机)导入。 +// 只读本机归档:不碰来源、不碰目标、不改任何版本。 +func runSnapshotPack(opts options) error { + if opts.snapshot == "" || opts.bundle == "" { + return errors.New("snapshot-pack 需要 -snapshot 与 -bundle ") + } + archive, err := openArchive(opts) + if err != nil { + return err + } + defer archive.Close() + info, err := archive.Pack(opts.snapshot, opts.bundle) + if err != nil { + return err + } + printJSON(info) + return nil +} + +// runSnapshotUnpack 校验并装入快照包。装入是内容寻址 + 版本清单提交: +// 校验不通过就什么都不留下,重复导入是幂等成功。 +func runSnapshotUnpack(opts options) error { + if opts.bundle == "" { + return errors.New("snapshot-unpack 需要 -bundle (-snapshot 可选,用于核对身份)") + } + archive, err := openArchive(opts) + if err != nil { + return err + } + defer archive.Close() + manifest, err := archive.Unpack(opts.bundle, opts.snapshot) + if err != nil { + return err + } + printJSON(map[string]any{ + "format": librarysnapshot.Format, + "scope": manifest.Scope, + "tenant_id": manifest.TenantID, + "knowledge_base_id": manifest.KnowledgeBaseID, + "parts": len(manifest.Parts), + "created_at": manifest.CreatedAt, + }) + return nil +} diff --git a/cmd/worker/library_release_handlers.go b/cmd/worker/library_release_handlers.go new file mode 100644 index 0000000..09a0586 --- /dev/null +++ b/cmd/worker/library_release_handlers.go @@ -0,0 +1,33 @@ +package main + +import ( + "context" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// registerLibraryReleaseHandlers 注册整库发布执行器:production sync / restore / +// recover / baseline,四类且仅这四类。 +// +// 为什么单独成一个函数:这是"发布执行器"角色的边界,也是唯一会让本进程写 +// 目标整库的地方。它必须能被单独测试(注册了哪些、未准入时是不是明确失败), +// 而不是埋在 run() 里只能靠连库启动来验证。 +func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *postgresstore.Store, + config appconfig.Config, admissionErr error) { + handlers[librarySyncJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { + return runLibraryRelease(ctx, store, config, job, governance.LibraryJobSync) + }) + handlers[libraryRestoreJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { + return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) + }) + handlers[libraryRecoverJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { + return runLibraryRecovery(ctx, store, config, job) + }) + // 建立目标权威基线:目标非空又没有生效版本时的唯一入口(只读捕获 + 校验 + 登记)。 + handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { + return runLibraryBaseline(ctx, store, config, job) + }) +} diff --git a/cmd/worker/library_role_test.go b/cmd/worker/library_role_test.go new file mode 100644 index 0000000..eb3efb7 --- /dev/null +++ b/cmd/worker/library_role_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "strings" + "testing" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/versionhistory" +) + +// 发布执行器必须只挂四类 release 作业:多挂一类就会抢走本机队列里别的执行者的作业。 +func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) { + handlers := map[string]jobs.Handler{} + registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil) + want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} + if len(handlers) != len(want) { + t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers) + } + for _, jobType := range want { + if handlers[jobType] == nil { + t.Fatalf("handler %q is not registered", jobType) + } + } +} + +// 准入未通过时 handler 必须"明确失败且不做任何动作":不能靠 nil store 或进程内状态蒙过去。 +func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) { + handlers := map[string]jobs.Handler{} + registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{}) + for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} { + err := handlers[jobType](context.Background(), jobs.Job{}) + if err == nil || !strings.Contains(err.Error(), "准入未通过") { + t.Fatalf("handler %q error = %v, want admission refusal", jobType, err) + } + } +} + +type errNotAdmitted struct{} + +func (errNotAdmitted) Error() string { return "fixture: target instance unknown" } + +// capture 角色不能注册任何 source 之外的队列;release-executor 反之。 +func TestWorkerHandlersFollowWorkerRole(t *testing.T) { + release := appconfig.Config{ + LibraryWorkerRole: appconfig.LibraryWorkerRoleReleaseExecutor, + LibraryAgentSocket: "/run/library-agent/agent.sock", + TestWeKnoraEnabled: true, + TestWeKnoraAPIKeyFile: "/nonexistent/api-key", + PPScanEnabled: true, + VersionHistoryGiteaBaseURL: "https://gitea.invalid/Weknora/knowledge-governance", + VersionHistoryAllowedOrigins: []string{"https://gitea.invalid"}, + } + handlers, err := workerHandlers(postgresstore.NewStore(nil), release) + if err != nil { + t.Fatalf("release-executor workerHandlers() error = %v", err) + } + for _, forbidden := range []string{libraryCaptureJobType, "test.upload", "test.poll", "test.validate", "test.remove", "source.scan.pp", "source.catalog.refresh", "artifact.fetch", versionhistory.JobType} { + if handlers[forbidden] != nil { + t.Fatalf("release-executor registered %q", forbidden) + } + } + if handlers["demo"] == nil { + t.Fatal("demo handler must stay registered") + } + + capture := appconfig.Config{ + LibraryWorkerRole: appconfig.LibraryWorkerRoleCapture, + LibraryAgentSocket: "/run/library-agent/agent.sock", + TestWeKnoraEnabled: true, + TestWeKnoraAPIKeyFile: "/nonexistent/api-key", + } + if _, err := workerHandlers(nil, capture); err == nil || !strings.Contains(err.Error(), "test WeKnora API key") { + t.Fatalf("capture role must reach the test WeKnora wiring, got %v", err) + } + capture.TestWeKnoraEnabled = false + capture.PPScanEnabled = false + handlers, err = workerHandlers(postgresstore.NewStore(nil), capture) + if err != nil { + t.Fatalf("capture workerHandlers() error = %v", err) + } + if handlers[libraryCaptureJobType] == nil { + t.Fatal("capture role must register library.capture") + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go index bf3a9eb..62c94c0 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -129,36 +129,51 @@ func run(ctx context.Context, getenv func(string) string) error { if err := pool.Ping(ctx); err != nil { return err } - if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { - return err + // 迁移也是"后台副作用":只有控制面那个进程拥有 DDL。云端发布执行器只读校验 + // schema 版本,落后即拒绝启动——它不该成为治理库的第二个结构决策者。 + if config.LibraryWorkerRole.ExecutesMigrations() { + if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { + return err + } + } else if pending, pendingErr := postgresstore.PendingMigrations(ctx, pool); pendingErr != nil { + return fmt.Errorf("治理库 schema 版本检查失败: %w", pendingErr) + } else if len(pending) > 0 { + return fmt.Errorf("治理库 schema 落后(待应用迁移 %v):本角色不拥有 DDL,先由控制面进程完成迁移", pending) } store := postgresstore.NewStore(pool) // 启动先接管被中断的写作业(复用已持久化的预备份),再把剩下的孤儿判失败: // 顺序反了会把可接管的作业先标失败,丢掉已完成的备份。 - if resumed, failed, resumeErr := resumeInterruptedLibraryJobs(context.Background(), store, config); resumeErr != nil { - log.Printf("library job resume failed: %v", resumeErr) - } else if resumed > 0 || failed > 0 { - log.Printf("library job resume: %d resumed, %d failed without backup", resumed, failed) - } - // 运行期也要持续接管:只在启动扫一次会漏掉"worker 在两次重启之间被杀"的情况。 - go func() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - resumed, failed, err := resumeInterruptedLibraryJobs(ctx, store, config) - if err != nil { - log.Printf("library job takeover sweep: %v", err) - } else if resumed > 0 || failed > 0 { - log.Printf("library job takeover sweep: %d resumed, %d failed without backup", resumed, failed) + // + // 只由"发布执行器"角色接管:release 作业的目标来自本进程配置,capture 角色的 + // 进程按本机维护配置去接管云端目标的作业,等于把快照写到另一套目标上。 + if config.LibraryWorkerRole.ExecutesRelease() { + if resumed, failed, resumeErr := resumeInterruptedLibraryJobs(context.Background(), store, config); resumeErr != nil { + log.Printf("library job resume failed: %v", resumeErr) + } else if resumed > 0 || failed > 0 { + log.Printf("library job resume: %d resumed, %d failed without backup", resumed, failed) + } + // 运行期也要持续接管:只在启动扫一次会漏掉"worker 在两次重启之间被杀"的情况。 + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + resumed, failed, err := resumeInterruptedLibraryJobs(ctx, store, config) + if err != nil { + log.Printf("library job takeover sweep: %v", err) + } else if resumed > 0 || failed > 0 { + log.Printf("library job takeover sweep: %d resumed, %d failed without backup", resumed, failed) + } } } - } - }() - if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC()); reconcileErr != nil { + }() + } + // 只收拾本角色负责的 kind:跨域"顺手收拾"会把另一台执行器的在飞作业标失败。 + if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC(), + config.LibraryWorkerRole.LibraryJobKinds()); reconcileErr != nil { log.Printf("library job reconcile failed: %v", reconcileErr) } else if repaired > 0 { log.Printf("library job reconcile: %d orphan job(s) marked failed", repaired) @@ -167,32 +182,33 @@ func run(ctx context.Context, getenv func(string) string) error { if err != nil { return err } - if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil { - log.Print("QA directory synchronization unavailable; handler was not registered") - } - mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers) - if mirrorErr != nil { - log.Print("QA local mirror configuration invalid; handler was not registered") + if config.LibraryWorkerRole.ExecutesSourceWork() { + if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil { + log.Print("QA directory synchronization unavailable; handler was not registered") + } + mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers) + if mirrorErr != nil { + log.Print("QA local mirror configuration invalid; handler was not registered") + } + if mirrorStore != nil { + go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") }) + } } // 生产同步/恢复的执行器不依赖主机侧封存 agent:没有 agent socket 也要能跑。 - // 这里在全部注册之后无条件覆盖一次,确保 library.sync/library.restore 一定在队列类型里。 // // 执行域准入(R4)在这里统一判定:目标实况、旧 release 路由绑定、未释放维护窗口 // 任何一项未知或重叠 ⇒ 这些 handler 直接拒绝执行(明确失败,无任何维护动作, // 也不停止任何既有服务)。 - admissionErr := verifyLibraryExecutionDomain(context.Background(), config, store) - if admissionErr != nil { - log.Printf("library execution domain NOT admitted: %v", admissionErr) + // + // 只有"发布执行器"角色注册它们:capture 角色的进程按本机目标配置执行一条 + // 指向别处目标的作业,后果是跨目标写入,不是失败重试能纠正的事。 + if config.LibraryWorkerRole.ExecutesRelease() { + admissionErr := verifyLibraryExecutionDomain(context.Background(), config, store) + if admissionErr != nil { + log.Printf("library execution domain NOT admitted: %v", admissionErr) + } + registerLibraryReleaseHandlers(handlers, store, config, admissionErr) } - handlers[librarySyncJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryRelease(ctx, store, config, job, governance.LibraryJobSync) - }) - handlers[libraryRestoreJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) - }) - handlers[libraryRecoverJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryRecovery(ctx, store, config, job) - }) worker, err := jobs.NewWorker(jobs.WorkerConfig{ Queue: store, @@ -205,19 +221,20 @@ func run(ctx context.Context, getenv func(string) string) error { if err != nil { return err } - if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil { - go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") }) + if config.LibraryWorkerRole.ExecutesSourceWork() { + if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil { + go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") }) + } + go weeklyMailLoop(ctx, pool, getenv) + // 封存执行器状态:worker 是唯一能连 agent socket 的一方,它写观测、页面读观测。 + go runLibraryExecutorProbe(ctx, store, config, libraryExecutorProbeInterval) + go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) { + log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err) + }) } - if mirrorStore != nil { - go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") }) - } - go weeklyMailLoop(ctx, pool, getenv) - // 封存执行器状态:worker 是唯一能连 agent socket 的一方,它写观测、页面读观测。 - go runLibraryExecutorProbe(ctx, store, config, libraryExecutorProbeInterval) - go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) { - log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err) - }) - log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval) + log.Printf("worker %s started role=%s release=%t source=%t (source refresh scheduler every %s)", + config.WorkerID, config.LibraryWorkerRole, config.LibraryWorkerRole.ExecutesRelease(), + config.LibraryWorkerRole.ExecutesSourceWork(), config.SourceRefreshSchedulerInterval) return worker.Run(ctx, config.PollInterval) } @@ -229,40 +246,44 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st }, "demo.sleep": jobs.DemoSleepHandler, } - catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py") - if err != nil { - return nil, fmt.Errorf("create catalog refresh runner: %w", err) + // 目录刷新与附件抓取属于"源侧"作业:它们读本机 spool、调本机脚本。 + // 发布执行器角色在云端没有这些挂载,注册它们只会抢走本机队列里的作业。 + if config.LibraryWorkerRole.ExecutesSourceWork() { + catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py") + if err != nil { + return nil, fmt.Errorf("create catalog refresh runner: %w", err) + } + catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner}) + if err != nil { + return nil, fmt.Errorf("create catalog refresh handler: %w", err) + } + handlers["source.catalog.refresh"] = catalogHandler.Handle + ingestionRoot := config.IngestionSpoolRoot + if ingestionRoot == "" { + ingestionRoot = config.GovernanceSpoolRoot + } + if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" { + ingestionRoot = "/var/lib/governance/ingestion" + } + fetchMaxBytes := config.ArtifactFetchMaxBytes + if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes { + fetchMaxBytes = config.TestWeKnoraMaxUploadBytes + } + if fetchMaxBytes <= 0 { + fetchMaxBytes = 50 * 1024 * 1024 + } + artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", + ingestionRoot, "ingestion", fetchMaxBytes) + if err != nil { + return nil, fmt.Errorf("create artifact fetch runner: %w", err) + } + artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner}) + if err != nil { + return nil, fmt.Errorf("create artifact fetch handler: %w", err) + } + handlers["artifact.fetch"] = artifactHandler.Handle } - catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner}) - if err != nil { - return nil, fmt.Errorf("create catalog refresh handler: %w", err) - } - handlers["source.catalog.refresh"] = catalogHandler.Handle - ingestionRoot := config.IngestionSpoolRoot - if ingestionRoot == "" { - ingestionRoot = config.GovernanceSpoolRoot - } - if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" { - ingestionRoot = "/var/lib/governance/ingestion" - } - fetchMaxBytes := config.ArtifactFetchMaxBytes - if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes { - fetchMaxBytes = config.TestWeKnoraMaxUploadBytes - } - if fetchMaxBytes <= 0 { - fetchMaxBytes = 50 * 1024 * 1024 - } - artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", - ingestionRoot, "ingestion", fetchMaxBytes) - if err != nil { - return nil, fmt.Errorf("create artifact fetch runner: %w", err) - } - artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner}) - if err != nil { - return nil, fmt.Errorf("create artifact fetch handler: %w", err) - } - handlers["artifact.fetch"] = artifactHandler.Handle - if config.LibraryAgentSocket != "" { + if config.LibraryAgentSocket != "" && config.LibraryWorkerRole.ExecutesSourceWork() { // 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧 // agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。 agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute} @@ -341,25 +362,6 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy) return nil } - // 执行域准入(R4):注册整库发布执行器**之前**核对目标实况、旧 release 路由与 - // 未释放的维护窗口。未通过时下面这些 handler 一律直接拒绝执行——明确失败, - // 不带任何维护动作,也不停止任何既有服务。 - admissionErr := verifyLibraryExecutionDomain(context.Background(), config, store) - if admissionErr != nil { - log.Printf("library execution domain NOT admitted: %v", admissionErr) - } - // 生产同步:唯一入口在 API(闸口未齐不建作业)。这里仍实现执行器, - // 但默认 fail-closed:没有闸口证据就明确失败,绝不向生产写入半个字节。 - handlers[librarySyncJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryRelease(ctx, store, config, job, governance.LibraryJobSync) - }) - handlers[libraryRestoreJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) - }) - // 建立目标权威基线:目标非空又没有生效版本时的唯一入口(只读捕获 + 校验 + 登记)。 - handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { - return runLibraryBaseline(ctx, store, config, job) - }) // A crash can leave the source writer stopped: recover before taking work, // and keep the failure visible instead of pretending the worker is ready. go func() { @@ -372,7 +374,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st log.Printf("library agent reachable; freeze journals recovered") }() } - if config.VersionHistoryGiteaBaseURL != "" { + if config.VersionHistoryGiteaBaseURL != "" && config.LibraryWorkerRole.ExecutesSourceWork() { // 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册, // 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。 client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{ @@ -417,7 +419,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st return nil } } - if config.PPScanEnabled { + if config.PPScanEnabled && config.LibraryWorkerRole.ExecutesSourceWork() { scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{ AllowedRoot: config.PPScanAllowedRoot, LedgerPath: config.PPScanLedgerPath, @@ -432,7 +434,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st } handlers["source.scan.pp"] = handler.Handle } - if !config.TestWeKnoraEnabled { + if !config.TestWeKnoraEnabled || !config.LibraryWorkerRole.ExecutesSourceWork() { return handlers, nil } apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) diff --git a/deploy/cloud-test/README.md b/deploy/cloud-test/README.md new file mode 100644 index 0000000..3bcd20f --- /dev/null +++ b/deploy/cloud-test/README.md @@ -0,0 +1,176 @@ +# 云测试发布目标:部署与验证(父执行) + +目标:把本机治理平台的"整库发布执行"搬到云测试主机,与目标容器同机; +本机只保留封存源、平台与 capture 角色的 worker。本文所有命令都在**目标机上**执行, +除了明确标注"本机"的两步。 + +## 0. 前置(缺一条都别继续) + +- 云端 compose 项目 `weknora-test-v071` 当前健康,且已确认 `weknora-test.dobot.cn` 的维护窗口可接受。 +- 已交付镜像:`knowledge-governance-worker:<与验收提交同 sha>`(含 `/app/libraryctl`)。 +- `/srv/kgtest-gate/{gate.py,state}`、`/srv/knowledge-governance/{library,target-journal,bundles,release-executor.env}`。 +- 治理库接入方式已定:SSH 加密反向隧道(`deploy/cloud-test/pg-forward.sh`,默认 streamlocal + unix socket)。本机 PG 在 Docker 内、无 5432 宿主机发布,**不要**写 `localhost:5432`。 + +## 1. 云端入口闸门(一次) + +**先做 compose 身份核对(不能盲 `compose up`)**:父的现场记录里,某历史容器的 label +引用了已不存在的 `/tmp/weknora-test-compose.wiki-graph.yaml`。对这类容器执行 +`docker compose up` 会让 compose 按"缺失的配置链"重建它,等于在没有可验证配置的情况下 +重建业务容器。先逐容器读回配置链,只对**配置链文件都在**的服务做 `up -d`: + +```bash +for c in $(docker ps -q --filter label=com.docker.compose.project=weknora-test-v071); do + docker inspect "$c" --format '{{.Name}} files={{index .Config.Labels "com.docker.compose.project.config_files"}}' +done +``` + +规则:`app`/`frontend` 的配置链齐全才允许 `up -d`;`ingress` 是本片段新增,没有历史包袱。 +任何服务的 config_files 里有缺失文件 → **停手**,先补回可验证配置(或让父按运维流程处理), +不要用 `compose up` 把它"顺便"重建。只用显式服务名 `up -d app ingress frontend`, +不要 `up -d` 全项目(那会把无关服务一起重建)。 + +```bash +install -d -m 0755 /srv/kgtest-gate /srv/kgtest-gate/state +install -m 0644 deploy/cloud-test/gate.py /srv/kgtest-gate/gate.py +# 叠加片段来自本仓库 deploy/cloud-test/compose.release-overlay.yaml +cd /opt/weknora-test-v071 # 现有 compose.yaml 所在目录 +docker compose -f compose.yaml -f /path/to/compose.release-overlay.yaml config >/dev/null +docker compose -f compose.yaml -f /path/to/compose.release-overlay.yaml up -d app ingress frontend +curl -fsS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/health # 期望 200 +docker inspect weknora-test-v071-app-1 --format '{{json .HostConfig.PortBindings}}' # 期望 null +``` + +要点:`app` 不再发布宿主机端口(写入者发布端口 = 绕开闸门的直连,拓扑校验会拒绝); +`ingress` 是宿主机 8080 上唯一的 HTTP 入口,`SIGTERM` 后写 `drain.json` 并 `exit 0`; +`restart: "no"` 保证维护窗口停下的入口不会被 Docker 自己拉回来。 + +### 1.1 入口不止 frontend:四条路径都要落到闸门上 + +只把 frontend 改成 `APP_HOST=ingress` **不等于**入口唯一。现场实测四条能到 app 的路径: + +| 入口 | 现状 | 必须改成 | 谁做 | +|---|---|---|---| +| 宿主机 nginx `weknora-test.dobot.cn` → `127.0.0.1:8088` | frontend | 不用改;frontend 的 `/api` 由 `APP_HOST` 决定 | 父不改 | +| frontend 容器 → `app:8080` | `APP_HOST=app` | overlay 已改为 `APP_HOST=ingress` | overlay | +| knowledge-adapter 容器 → `http://app:8080`(双网络) | 直连 app | adapter 的 `WEKNORA_BASE_URL=http://ingress:8080` | 父改 adapter 部署 | +| DobotAI(host 网络)→ `http://127.0.0.1:8080` | 走宿主机发布端口 | 不用改;overlay 后该端口就是闸门 | 无 | + +`scripts/verify_cloud_release_target.py` 的 `entry/*` 三项就是在核对这张表 +(adapter 仍指 `app:8080` 时报 FAIL —— 它就是一条真实存在的绕闸通道)。 + +## 2. 快照(本机打包 → 云端导入) + +```bash +# 本机(只读归档;输出了每个 blob 的 sha256,可与云端对拍) +libraryctl snapshot-pack -snapshot -bundle /var/tmp/.tar + +# 传到云端 +scp /var/tmp/.tar weknora-test:/srv/knowledge-governance/bundles/ + +# 云端:校验 + 原子装入(版本清单是提交点;重复导入幂等) +docker run --rm \ + -v /srv/knowledge-governance/library:/var/lib/knowledge-governance/library \ + -v /srv/knowledge-governance/bundles:/bundles:ro \ + knowledge-governance-worker: \ + /app/libraryctl snapshot-unpack \ + -archive /var/lib/knowledge-governance/library \ + -bundle /bundles/.tar \ + -snapshot +``` + +导入失败时不会留下版本:blob 以 sha256 命名且先落盘,版本清单最后才 Link。 +执行器每次使用快照前还会 `Load` 逐 blob 重算 sha256,所以"传输不可变"有两道证据。 + +## 2.5 治理库连接(SSH 加密反向隧道) + +本机治理 PG 在 Docker 内且没有 5432 宿主机发布,不能写 `localhost:5432`。 +`deploy/cloud-test/pg-forward.sh`(本机运行)解析 PG 容器的真实 IP 后建反向转发: + +```bash +MODE=streamlocal bash deploy/cloud-test/pg-forward.sh # 推荐:云侧只出现 unix socket +MODE=tcp bash deploy/cloud-test/pg-forward.sh # 云侧 127.0.0.1:15433,仅宿主进程可用 +``` + +- **streamlocal(默认)**:云侧生成 `/run/kgtest-pg/pg.sock`,执行器容器把该目录挂进去, + `DATABASE_URL` 用 `host=/run/kgtest-pg/pg.sock`。没有 TCP 端口、不受"容器看不到宿主 + 127.0.0.1"影响。 +- **tcp**:`ssh -R 127.0.0.1:15433` 只在云**宿主机** loopback 上;执行器容器里的 + 127.0.0.1 是它自己,**不能**直接用。要用就必须在云侧再加 docker bridge → 127.0.0.1 + 的桥接(socat/haproxy),那是父的运维决定,脚本不替你开。 +- 隧道断 = 执行器拿不到租约 = 作业 fail-closed,目标保持维护态(不会"假装成功")。 + +## 3. 执行器 + +```bash +# 1) 填写 /srv/knowledge-governance/release-executor.env(0600 root), +# 字段见 deploy/cloud-test/release-executor.env.example。 +# 2) 先让本机变成 capture 角色(否则两边都注册 release,谁抢到按谁的配置执行): +# 本机 compose 加 LIBRARY_WORKER_ROLE=capture 重启 worker,日志应出现 +# role=capture release=false source=true。 +# 3) 起云端执行器: +RELEASE_EXECUTOR_IMAGE=knowledge-governance-worker: bash deploy/cloud-test/run-release-executor.sh +docker logs --tail 30 kgtest-release-executor # 期望:execution domain admitted / role=release-executor +``` + +## 4. 只读预检(任何时候都可跑) + +```bash +python3 scripts/verify_cloud_release_target.py --host weknora-test --snapshot +``` + +判据:ssh、项目服务清单、写入者无发布端口、闸门 healthy 且发布 8080、经闸门 /health=200、 +drain 目录可写且无残留、redis/ES/RDS 可达、归档根 0700、快照清单摘要自证、执行器角色。 +`MANUAL target-graph` 必须人工确认目标 KB `graph_enabled=false`(发布路径不写 Neo4j)。 +有 FAIL 即退出码 1。 + +## 5. 回退 + +```bash +docker rm -f kgtest-release-executor +# 本机 worker 改回 LIBRARY_WORKER_ROLE=full(或保持 capture)并重启 +cd /opt/weknora-test-v071 +docker compose -f compose.yaml -f /path/to/compose.release-overlay.yaml down # 或去掉 -f overlay 重新 up: +docker compose -f compose.yaml up -d app frontend +``` + +治理库不需要回滚:维护记录与版本证据都带 `target_fingerprint` 与实例身份, +跨目标不会被误认。云端归档里的快照与预备份保留,不自动删除。 + +## 6. 已知边界(交付时写明,不当成已完成) + +- 本机 UI 的"版本与快照"读的是本机归档:云端预备份/快照原件在页面上看不到, + 只能看到治理库里的版本证据行。需要页面展示云端原件时再单独做只读读回链路。 +- 目标 KB `graph_enabled` 没有 fail-closed 检查(当前只有源侧导出拒绝)。 +- 维护窗口内 `weknora-test.dobot.cn` 会不可用(这是语义正确的关闭)。 +- **归档自动传输未实现**:今天只有 `libraryctl snapshot-pack / snapshot-unpack`(人工两步)。 + UI 里点"发布/恢复"不会把本机快照送到云端执行器,也不会把云端预备份读回本机页面。 + 一致寻址的现状与阻塞见本文 §7。 + +## 7. 归档寻址:现状、缺口与阻塞(不许当成已完成) + +**寻址口径(现状)**:快照是内容寻址的(`snapshot_id` = 版本清单 sha256),本机与云端 +各自有独立 `LIBRARY_ARCHIVE_ROOT`;治理库的版本证据行记录 `snapshot_id` 与 `source_job_id`。 +因此"这份版本是谁"跨主机一致,而"这份版本的字节现在在哪台机器"**没有**被记录。 + +后果(真实缺口): +1. 本机页面能看到云端执行器登记的版本证据行,但**看不到云端预备份/快照原件**; + 点"按历史版本恢复"时,若该快照不在执行器所在机器上,作业会在装配阶段失败 + (`librarysnapshot.Load` 读不到),错误信息只会说"快照不可读"。 +2. UI 无法触发传输:本机 API 没有读归档/推送归档的接口,云端也**不能**主动连本机 + (公网 22 关闭、无入站);今天唯一通路是人工 `pack` → `scp` → `unpack`。 + +**为什么不用脚本硬糊**:定时 rsync/scp 无法保证"版本清单与 blob 同时可见 + 字节可校验", +而发布前的 `Load` 会逐 blob 重算 sha256,半拷贝的归档会让发布在维护窗口内失败, +正好落在最不该失败的时刻。 + +**建议的最小协议(未实现,需父批准)**:给执行器一条**出站**拉取路径 —— +`GET /api/v1/library/snapshots/{id}/bundle`(本机 API 侧,仅服务已封存且未过期的快照, +返回带 sha256 的流)+ 在版本证据里补 `archive_host` 字段。执行器在装配前若本地缺快照, +按证据里的 `archive_host` 标记走到该 API 拉取并复用 `Unpack` 的校验/原子落位。 +工作量约:API 1 个只读端点 + 证据字段 + 执行器一次拉取调用(~300 行 + 权限/审计)。 +通路本身可以复用已有的 `ssh -R`(再加一条把本机 API 映到云侧 loopback/unix socket 的 +反向转发),所以阻塞不在"连不上",而在:① 该端点等于让云侧凭据读走整库快照, +需要单独的鉴权与审计口径;② 版本证据要新增 `archive_host`,否则 UI 无法判断"该去哪台机器取"; +③ 拉取失败必须与"目标已进维护窗口"的时序绑定,不能先开窗口再发现取不到。 +这三条都要父拍板,属于新协议而非配置。 \ No newline at end of file diff --git a/deploy/cloud-test/compose.release-overlay.yaml b/deploy/cloud-test/compose.release-overlay.yaml new file mode 100644 index 0000000..86013fe --- /dev/null +++ b/deploy/cloud-test/compose.release-overlay.yaml @@ -0,0 +1,66 @@ +# 云测试用作发布目标时的 compose 叠加片段(project: weknora-test-v071) +# +# 作用:把"唯一入口"这个不变量在云测试上真正建立起来,让维护窗口有可信的排空证据。 +# 1. app 不再发布宿主机端口(写入者发布端口 = 存在绕开闸门的直连通道,拓扑校验会拒绝); +# 2. 新增 ingress 闸门,成为宿主机 8080 上唯一的 HTTP 入口,并在 SIGTERM 时写 drain.json; +# 3. frontend 的 API 目标改为 ingress(这样网页流量也走闸门,关闸门 = 站点真的关)。 +# +# 使用(父执行,先校验再应用): +# docker compose -f compose.yaml -f compose.release-overlay.yaml config >/dev/null +# docker compose -f compose.yaml -f compose.release-overlay.yaml up -d app ingress frontend +# +# !reset 需要 Docker Compose >= 2.24(云端实测 2.40.3)。 +# +# 网络:本项目实际网络名是 weknora-test-v071-network(不是 compose 默认的 *_default)。 +# 网络:本项目实际网络名是 weknora-test-v071-network(不是 compose 默认的 *_default)。 +# ingress 用独立 key `release_target` 以 external 方式加入它:既不重定义原 default, +# 也不靠"默认网络恰好同名"这种巧合。 +name: weknora-test-v071 + +services: + app: + # 写入者不发布端口;验证器/发布执行器的直连走 docker 网络里的服务名 app:8080。 + ports: !reset [] + + ingress: + # 唯一对外入口:关入口=停这个闸门。镜像只用现成的 python,不构建、不改 WeKnora 镜像。 + image: python:3.13-slim + container_name: kgtest-ingress + # restart: "no" 是硬约束:维护窗口把入口停下之后,Docker 不允许自己把它拉回来。 + restart: "no" + command: ["python3", "/gate/gate.py"] + environment: + # 闸门监听端口 = 宿主机发布端口,入口 URL 用同端口,拓扑校验才可比对。 + GATE_PORT: "8080" + GATE_UPSTREAM_HOST: app + GATE_UPSTREAM_PORT: "8080" + GATE_DRAIN_FILE: /gate-state/drain.json + GATE_DRAIN_TIMEOUT: "45" + ports: + - "127.0.0.1:8080:8080" + volumes: + # gate.py 与 state 都在宿主机固定目录;发布执行器挂同一个 state 目录, + # 所以"删除 drain 记录 / 读回排空证据"发生在同一个真实文件上。 + - /srv/kgtest-gate/gate.py:/gate/gate.py:ro + - /srv/kgtest-gate/state:/gate-state + healthcheck: + test: ["CMD", "python3", "-c", "import socket;s=socket.create_connection(('127.0.0.1',8080),2);s.close()"] + interval: 3s + timeout: 3s + retries: 30 + start_period: 5s + stop_grace_period: 60s + # 用独立 key 指向项目真实网络(external,不改原 compose 的 default 定义): + # 这样 ingress 与 app 在同一条网络里能按服务名互通,也不会凭空创建 *_default。 + networks: + - release_target + + frontend: + environment: + # 网页的 /api 也要经过闸门,否则"关入口"只关了一半。 + APP_HOST: ingress + +networks: + release_target: + external: true + name: weknora-test-v071-network diff --git a/deploy/cloud-test/gate.py b/deploy/cloud-test/gate.py new file mode 100755 index 0000000..e960079 --- /dev/null +++ b/deploy/cloud-test/gate.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""目标入口闸门:唯一去 app 的 HTTP 路径,负责"关入口 + 排空在途请求"。 + +为什么必须有一个独立闸门,而不是在被代理的 WeKnora 上排空: +v0.7.1 的 app 收到 SIGTERM 会先关监听、随后因 use of closed network connection +直接 exit(1),在途请求被切断。所以"入口已关、在途已归零"这条证据只能在真正的 +连接入口上产生,并且必须落在**目标主机自己的**文件上(治理侧的维护协调器会先删除 +这个文件、发 SIGTERM、等容器退出、再读回;读到旧文件就是伪造证据)。 + +连接语义(两条都被真实业务踩过,不是理论问题): + 1. 上游读**不设超时**:`create_connection(timeout=...)` 只对 connect 生效,连上后 + 必须 `settimeout(None)`。否则一条等待模型输出超过这个秒数的空闲流会被闸门自己 + 掐断 —— 那是"闸门把正常业务掐了",不是"连接已经排空"。 + 2. 半关闭按 TCP 语义单向传播:客户端写完请求后关自己的写端(shutdown(SHUT_WR))时, + 只把 EOF 传到上游,**不能**顺手把上游→客户端方向也拆掉,否则响应会被截断。 + 只有真正的错误(RST 等)才双向拆除,避免另一侧线程永久挂住。 + +SIGTERM 语义(顺序是证据的一部分): + 1. 立即关闭监听 —— 新连接被拒 = 入口已关; + 2. 等在途连接自然结束(上限 GATE_DRAIN_TIMEOUT); + 3. 写 drain.json(临时文件 + fsync + rename,0600)后 exit 0(drained) + / exit 2(drain_timeout,此时绝不能当成排空成功)。 +""" + +import json +import os +import signal +import socket +import sys +import threading +import time + +GATE_PORT = int(os.environ.get("GATE_PORT", "8080")) +UPSTREAM_HOST = os.environ.get("GATE_UPSTREAM_HOST", "app") +UPSTREAM_PORT = int(os.environ.get("GATE_UPSTREAM_PORT", "8080")) +DRAIN_FILE = os.environ.get("GATE_DRAIN_FILE", "/gate-state/drain.json") +DRAIN_TIMEOUT = float(os.environ.get("GATE_DRAIN_TIMEOUT", "45")) +CONNECT_TIMEOUT = float(os.environ.get("GATE_CONNECT_TIMEOUT", "10")) + +active = 0 +served = 0 +lock = threading.Lock() +listener = None +stopping = threading.Event() + + +def write_state(payload): + if not os.path.isabs(DRAIN_FILE): + print("gate: GATE_DRAIN_FILE must be an absolute path", file=sys.stderr, flush=True) + os._exit(3) + payload = dict(payload, at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), pid=os.getpid()) + directory = os.path.dirname(DRAIN_FILE) + os.makedirs(directory, exist_ok=True) + tmp = DRAIN_FILE + ".tmp" + fd = os.open(tmp, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as handle: + json.dump(payload, handle) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, DRAIN_FILE) + + +def pump(src, dst): + """单向搬运并传播半关闭。正常 EOF 只关对端写端;异常则双向拆除。""" + clean = False + try: + while True: + data = src.recv(65536) + if not data: + clean = True + break + dst.sendall(data) + except OSError: + clean = False + finally: + if clean: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + else: + for sock in (src, dst): + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def handle(client): + global active, served + with lock: + if stopping.is_set(): + client.close() + return + active += 1 + served += 1 + current = active + print(f"conn:open active={current} served={served}", flush=True) + upstream = None + try: + upstream = socket.create_connection((UPSTREAM_HOST, UPSTREAM_PORT), timeout=CONNECT_TIMEOUT) + # 建连超时只对 connect 生效;读超时必须清掉,否则长静默流会被掐断。 + upstream.settimeout(None) + client.settimeout(None) + except OSError: + with lock: + active -= 1 + remaining = active + print(f"conn:close active={remaining} upstream-unreachable", flush=True) + if upstream is not None: + upstream.close() + client.close() + return + left = threading.Thread(target=pump, args=(client, upstream), daemon=True) + right = threading.Thread(target=pump, args=(upstream, client), daemon=True) + left.start() + right.start() + + def finish(): + global active + try: + left.join() + right.join() + finally: + # 两个方向都结束才算这条连接离开:只看一个方向会把半关的连接当"已排空"。 + for sock in (client, upstream): + try: + sock.close() + except OSError: + pass + with lock: + active -= 1 + remaining = active + print(f"conn:close active={remaining}", flush=True) + + threading.Thread(target=finish, daemon=True).start() + + +def on_term(_signum, _frame): + stopping.set() + if listener is not None: + try: + listener.close() + except OSError: + pass + + +def main(): + global listener + signal.signal(signal.SIGTERM, on_term) + signal.signal(signal.SIGINT, on_term) + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("0.0.0.0", GATE_PORT)) + listener.listen(256) + write_state({"phase": "serving", "active": 0, "served": 0}) + started = time.time() + print(f"gate: listening on {GATE_PORT} -> {UPSTREAM_HOST}:{UPSTREAM_PORT} drain={DRAIN_FILE}", + flush=True) + while not stopping.is_set(): + try: + client, _ = listener.accept() + except OSError: + break + handle(client) + + with lock: + active_at_close = active + drain_started = time.time() + write_state({"phase": "draining", "active": active_at_close, "served": served}) + deadline = drain_started + DRAIN_TIMEOUT + while time.time() < deadline: + with lock: + if active == 0: + break + time.sleep(0.05) + with lock: + remaining = active + drain_ms = int((time.time() - drain_started) * 1000) + uptime_ms = int((time.time() - started) * 1000) + if remaining != 0: + write_state({"phase": "drain_timeout", "active": remaining, "served": served, + "active_at_close": active_at_close, "drain_ms": drain_ms, + "uptime_ms": uptime_ms}) + os._exit(2) + write_state({"phase": "drained", "active": 0, "served": served, + "active_at_close": active_at_close, "drain_ms": drain_ms, + "uptime_ms": uptime_ms}) + os._exit(0) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/deploy/cloud-test/pg-forward.sh b/deploy/cloud-test/pg-forward.sh new file mode 100755 index 0000000..ab3b016 --- /dev/null +++ b/deploy/cloud-test/pg-forward.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# 治理库 → 云测试发布执行器:SSH 加密反向转发(在**本机控制面**上运行)。 +# +# 为什么不能写 localhost:5432:本机治理 PG 跑在 Docker 里,没有 5432 宿主机发布, +# 也不允许为它开公网/主机端口。所以只能: +# 1. 先解析 PG 容器在它自己网络里的真实 IP; +# 2. 用 ssh -R 建反向转发,让云侧出现一个**只属于云主机/容器**的入口; +# 3. 执行器通过这个入口连库,凭据只在本机与云端 0600 env 之间传递。 +# +# 两种模式(默认 streamlocal,不占 TCP 端口、不开监听): +# streamlocal 云侧生成 unix socket(如 /run/kgtest-pg/pg.sock),执行器容器把该目录 +# 挂进去,DSN 用 host=/run/kgtest-pg/pg.sock。**推荐**:没有 TCP 暴露, +# 不受"容器看不到宿主机 127.0.0.1"影响。 +# tcp 云侧 127.0.0.1:15433 loopback。⚠️ 只有云**宿主机**上的进程能用; +# 容器里的进程看到的 127.0.0.1 是容器自己。要用必须在云侧再加一个 +# 从 docker bridge 到 127.0.0.1 的桥接(socat/haproxy),那属于父运维决定, +# 本脚本不替你开。 +# +# 用法: +# MODE=streamlocal bash deploy/cloud-test/pg-forward.sh +# MODE=tcp REMOTE_PORT=15433 bash deploy/cloud-test/pg-forward.sh +set -euo pipefail + +SSH_HOST="${SSH_HOST:-weknora-test}" +PG_CONTAINER="${PG_CONTAINER:-knowledge-governance-p1-postgres-1}" +MODE="${MODE:-streamlocal}" +REMOTE_SOCKET="${REMOTE_SOCKET:-/run/kgtest-pg/pg.sock}" +REMOTE_PORT="${REMOTE_PORT:-15433}" + +die() { echo "pg-forward: $*" >&2; exit 1; } + +command -v docker >/dev/null || die "需要 docker(解析治理 PG 容器 IP)" +docker inspect "$PG_CONTAINER" >/dev/null 2>&1 || die "找不到容器 $PG_CONTAINER" + +# 只取第一个网络里的 IP;多网络时打印提示,由人确认走哪个。 +IPS=$(docker inspect -f '{{range $name, $net := .NetworkSettings.Networks}}{{$name}}={{$net.IPAddress}} {{end}}' "$PG_CONTAINER") +PG_IP=$(echo "$IPS" | tr ' ' '\n' | sed -n 's/^[^=]*=\([0-9.]\+\)$/\1/p' | head -n1) +[ -n "$PG_IP" ] || die "解析不到 $PG_CONTAINER 的容器 IP(网络:$IPS)" +echo "pg-forward: 治理 PG 容器 $PG_CONTAINER -> $PG_IP:5432(网络:$IPS)" + +case "$MODE" in + streamlocal) + ssh -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=3 \ + "$SSH_HOST" "install -d -m 0700 -o root -g root $(dirname "$REMOTE_SOCKET")" + DSN_SHAPE="postgres://@/?host=$REMOTE_SOCKET&sslmode=disable" + echo "pg-forward: 云侧入口 unix socket = $REMOTE_SOCKET(仅云主机与挂载它的容器可见)" + echo "pg-forward: 执行器容器必须挂载 -v $(dirname "$REMOTE_SOCKET"):$(dirname "$REMOTE_SOCKET")" + echo "pg-forward: DATABASE_URL 形状 = $DSN_SHAPE" + exec ssh -N -o BatchMode=yes -o ExitOnForwardFailure=yes \ + -o ServerAliveInterval=15 -o ServerAliveCountMax=3 \ + -R "$REMOTE_SOCKET:$PG_IP:5432" "$SSH_HOST" + ;; + tcp) + echo "pg-forward: 云侧 127.0.0.1:$REMOTE_PORT -> 本机 $PG_IP:5432(仅云宿主机可用)" + echo "pg-forward: 容器内进程不能用这个 loopback;要么用 MODE=streamlocal,要么另做 bridge(父决定)" + exec ssh -N -o BatchMode=yes -o ExitOnForwardFailure=yes \ + -o ServerAliveInterval=15 -o ServerAliveCountMax=3 \ + -R "127.0.0.1:$REMOTE_PORT:$PG_IP:5432" "$SSH_HOST" + ;; + *) + die "MODE 只能是 streamlocal 或 tcp(当前 $MODE)" + ;; +esac \ No newline at end of file diff --git a/deploy/cloud-test/release-executor.env.example b/deploy/cloud-test/release-executor.env.example new file mode 100644 index 0000000..e1d9937 --- /dev/null +++ b/deploy/cloud-test/release-executor.env.example @@ -0,0 +1,61 @@ +# 云测试发布执行器(governance release-executor)配置样例 +# +# 这份文件不含任何真实凭据:值全是占位符。真实文件放在云测试主机的 +# /srv/knowledge-governance/release-executor.env,权限 0600、属主 root, +# 由父在目标机本地填写(不要把生产凭据拷过来)。 +# +# 角色是这个部署的核心:release-executor 只消费 library.sync / library.restore / +# library.recover / library.baseline,不起 scheduler、不接封存 agent、不碰 QA/测试/ +# 目录刷新队列 —— 那些归本机那个 capture 角色。 +LIBRARY_WORKER_ROLE=release-executor +WORKER_ID=kgtest-release-executor + +# ---- 治理库(决策点:推荐本机 ssh -R 把本机 5432 映射到云端 127.0.0.1:15433)---- +# 只监听 loopback,凭据不出本机。隧道断了 = 拿不到租约 = fail-closed。 +DATABASE_URL=postgres://:@127.0.0.1:15433/?sslmode=disable + +# ---- 目标:云测试 WeKnora ---- +LIBRARY_PRODUCTION_DATABASE_URL=postgres://:@pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test?sslmode=require +LIBRARY_PRODUCTION_ELASTIC_URL=http://elasticsearch:9200 +LIBRARY_PRODUCTION_INDEX=weknora_test_chunks +LIBRARY_PRODUCTION_TENANT_ID= +LIBRARY_PRODUCTION_KB_INTERNAL= +# 对外库没有对应目标时留空:API 侧不会给 external 建发布作业。 +# LIBRARY_PRODUCTION_KB_EXTERNAL= +LIBRARY_PRODUCTION_S3_PROVIDER=oss +LIBRARY_PRODUCTION_S3_ENDPOINT=https://oss-cn-shenzhen.aliyuncs.com +LIBRARY_PRODUCTION_S3_REGION=cn-shenzhen +LIBRARY_PRODUCTION_S3_BUCKET=dobot-weknora-test +LIBRARY_PRODUCTION_S3_ACCESS_KEY= +LIBRARY_PRODUCTION_S3_SECRET_KEY= +LIBRARY_EMBEDDING_MODEL= +LIBRARY_EMBEDDING_DIMENSION=1024 + +# ---- 维护目标:同机的 weknora-test-v071 容器组 ---- +LIBRARY_TARGET_INSTANCE=weknora-test-v071-app +LIBRARY_TARGET_PROJECT=weknora-test-v071 +LIBRARY_TARGET_WRITERS=app +# 项目里每个服务都必须声明成写入者或"不是写入者 + 原因",否则窗口会拒绝开。 +LIBRARY_TARGET_NON_WRITERS=frontend=UI only; no knowledge writes,elasticsearch=index store; not an initiator,redis=queue/cache store; not an initiator,neo4j=graph store; release path refuses graph writes,docreader=parse worker; driven by app only,ingress=TCP ingress gate which only drains and forwards +LIBRARY_TARGET_DOCKER_SOCKET=/var/run/docker.sock +# 入口 URL 用 docker 网络里的闸门服务名:执行器与闸门同项目网络, +# 而闸门在宿主机发布的是同一个 8080(拓扑校验比对的就是这个端口)。 +LIBRARY_TARGET_INGRESS_URL=http://ingress:8080/health +LIBRARY_TARGET_INGRESS_SERVICE=ingress +LIBRARY_TARGET_INGRESS_DRAIN_FILE=/gate-state/drain.json +LIBRARY_TARGET_REDIS_ADDR=redis:6379 +LIBRARY_TARGET_REDIS_DB=0 +LIBRARY_TARGET_JOURNAL_ROOT=/var/lib/knowledge-governance/target-journal + +# ---- 归档:本机 ip 封存 → 包 → 云端导入到这里 ---- +LIBRARY_ARCHIVE_ROOT=/var/lib/knowledge-governance/library +LIBRARY_CAPACITY_BUDGET_BYTES=53687091200 + +# 不接封存 agent(封存仍在本机由既有 systemd user unit 提供)。 +LIBRARY_AGENT_SOCKET= + +# ---- 写目标的两道闸口 ---- +# 这两个开关只是"本进程允许写"的部署声明;真正的授权是治理库里与快照绑定的 +# release 授权行。任何一道为 false,发布会在维护窗口之前被拒绝。 +PRODUCTION_RELEASE_REQUESTS_ENABLED=true +PRODUCTION_BACKUP_RESTORE_VERIFIED=true \ No newline at end of file diff --git a/deploy/cloud-test/run-release-executor.sh b/deploy/cloud-test/run-release-executor.sh new file mode 100755 index 0000000..6a8893c --- /dev/null +++ b/deploy/cloud-test/run-release-executor.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# 在云测试主机上启动/更新"发布执行器"容器(由父在目标机执行,不在本机远程包装)。 +# +# 为什么单独 docker run 而不是塞进 weknora-test-v071 这个 compose 项目: +# 维护协调器要求项目里每个服务都被声明为写入者或"非写入者",执行器不是目标的一部分, +# 放进同一个项目只会让拓扑清单多一条无关服务。它只需要挂上目标项目的网络即可。 +set -euo pipefail + +IMAGE="${RELEASE_EXECUTOR_IMAGE:-}" +NETWORK="${TARGET_NETWORK:-weknora-test-v071_default}" +ENV_FILE="${RELEASE_EXECUTOR_ENV:-/srv/knowledge-governance/release-executor.env}" +ARCHIVE="${RELEASE_ARCHIVE_ROOT:-/srv/knowledge-governance/library}" +JOURNAL="${RELEASE_JOURNAL_ROOT:-/srv/knowledge-governance/target-journal}" +GATE_STATE="${RELEASE_GATE_STATE:-/srv/kgtest-gate/state}" +NAME="${RELEASE_EXECUTOR_NAME:-kgtest-release-executor}" + +die() { echo "release-executor: $*" >&2; exit 1; } + +[ -n "$IMAGE" ] || die "RELEASE_EXECUTOR_IMAGE 未设置(必须是与已验收提交同 sha 的 worker 镜像)" +[ "$(id -u)" -eq 0 ] || die "需要 root:要读 docker.sock 并读写 0700 归档目录" +docker network inspect "$NETWORK" >/dev/null 2>&1 || die "网络 $NETWORK 不存在(先部署目标项目)" +[ -f "$ENV_FILE" ] || die "缺少 env 文件 $ENV_FILE" +[ "$(stat -c '%a' "$ENV_FILE")" = "600" ] || die "$ENV_FILE 必须是 0600(里面有目标凭据)" +grep -q '^LIBRARY_WORKER_ROLE=release-executor$' "$ENV_FILE" || die "$ENV_FILE 里 LIBRARY_WORKER_ROLE 必须是 release-executor" +[ -d "$GATE_STATE" ] || die "闸门状态目录 $GATE_STATE 不存在(先部署 ingress 闸门)" + +install -d -m 0700 "$ARCHIVE" "$JOURNAL" +install -d -m 0700 "$ARCHIVE/blobs" "$ARCHIVE/versions" + +if docker inspect "$NAME" >/dev/null 2>&1; then + echo "release-executor: 已存在,按同一 env 重建(先停,便于看清失败原因)" + docker rm -f "$NAME" >/dev/null +fi + +docker run -d --name "$NAME" \ + --restart unless-stopped \ + --network "$NETWORK" \ + --label com.nous.role=release-executor \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$ARCHIVE:/var/lib/knowledge-governance/library" \ + -v "$JOURNAL:/var/lib/knowledge-governance/target-journal" \ + -v "$GATE_STATE:/gate-state" \ + --env-file "$ENV_FILE" \ + "$IMAGE" >/dev/null + +sleep 2 +docker ps --filter "name=$NAME" --format '{{.Names}} {{.Status}} {{.Image}}' +echo "release-executor: 启动完成;日志用 docker logs -f $NAME(准入失败会在第一屏写明原因)" \ No newline at end of file diff --git a/deploy/cloud-test/test_gate.py b/deploy/cloud-test/test_gate.py new file mode 100755 index 0000000..a33b4e6 --- /dev/null +++ b/deploy/cloud-test/test_gate.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""gate.py 的真实回归(无外部依赖、无副作用,只在本机 loopback 上跑)。 + +覆盖的是"闸门会不会掐断正常业务"和"排空证据是不是真的"这两件事,而不是 /health 200: + + 1. 长静默流:上游发一半后静默 12 秒再发剩下的 —— 闸门必须照原样送达 + (旧实现 create_connection(timeout=10) 未清除读超时,会在这里失败)。 + 2. 半关闭:客户端写完请求即 shutdown(SHUT_WR),上游收到 EOF 后才应答 —— + 响应必须完整回来,且耗时证明上游确实等到了 EOF。 + 3. 大包完整性:2MB 正文穿过闸门逐字节一致。 + 4. 排空:在途连接存在时 SIGTERM —— 新连接被拒、在途连接照常收完、退出码 0、 + drain.json phase=drained 且 active_at_close>=1。 + 5. 排空超时:上游永不应答、在途连接不结束 —— 退出码 2、phase=drain_timeout(不能算排空成功)。 + 6. 上游不可达:连接被关掉而不是挂住,闸门本身仍存活并能正常排空。 + +用法:python3 deploy/cloud-test/test_gate.py (打印 PASS/FAIL,任一 FAIL 退出码 1) +""" + +import json +import os +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import traceback + +HERE = os.path.dirname(os.path.abspath(__file__)) +GATE = os.path.join(HERE, "gate.py") +SILENT_GAP = 12.0 + +failures = [] + + +def check(name, condition, detail=""): + if condition: + print(f"PASS {name} {detail}".rstrip(), flush=True) + else: + print(f"FAIL {name} {detail}".rstrip(), flush=True) + failures.append(name) + + +def free_port(): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return port + + +class FakeUpstream: + """按请求行决定行为的极简上游。""" + + def __init__(self): + self.port = free_port() + self.server = socket.socket() + self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.server.bind(("127.0.0.1", self.port)) + self.server.listen(16) + self.stopped = threading.Event() + threading.Thread(target=self._serve, daemon=True).start() + + def _serve(self): + while not self.stopped.is_set(): + try: + client, _ = self.server.accept() + except OSError: + return + threading.Thread(target=self._handle, args=(client,), daemon=True).start() + + def _handle(self, client): + try: + client.settimeout(None) + data = b"" + while b"\r\n\r\n" not in data: + chunk = client.recv(4096) + if not chunk: + return + data += chunk + head, _, rest = data.partition(b"\r\n\r\n") + path = head.split(b" ")[1].decode() if b" " in head else "/" + if path == "/slow": + client.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabc") + time.sleep(SILENT_GAP) + client.sendall(b"defghij") + elif path == "/half": + # 等客户端半关闭(EOF)再应答:证明闸门把 EOF 单向传了过来。 + while True: + chunk = client.recv(4096) + if not chunk: + break + time.sleep(0.8) + body = b"after-eof" + client.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(body) + body) + elif path == "/echo": + length = 0 + for line in head.split(b"\r\n"): + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1].strip()) + body = rest + while len(body) < length: + chunk = client.recv(65536) + if not chunk: + break + body += chunk + client.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(body) + body) + elif path == "/hold": + client.sendall(b"HTTP/1.1 200 OK\r\n\r\n") + time.sleep(3600) + else: + client.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + except OSError: + pass + finally: + try: + client.close() + except OSError: + pass + + def close(self): + self.stopped.set() + try: + self.server.close() + except OSError: + pass + + +class Gate: + def __init__(self, upstream_port, drain_timeout=45.0): + self.port = free_port() + self.dir = tempfile.mkdtemp(prefix="gate-test-") + self.drain = os.path.join(self.dir, "drain.json") + env = dict(os.environ) + env.update({ + "GATE_PORT": str(self.port), + "GATE_UPSTREAM_HOST": "127.0.0.1", + "GATE_UPSTREAM_PORT": str(upstream_port), + "GATE_DRAIN_FILE": self.drain, + "GATE_DRAIN_TIMEOUT": str(drain_timeout), + }) + self.process = subprocess.Popen([sys.executable, GATE, ], env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + deadline = time.time() + 10 + while time.time() < deadline: + try: + socket.create_connection(("127.0.0.1", self.port), timeout=0.2).close() + return + except OSError: + if self.process.poll() is not None: + raise RuntimeError("gate exited early: " + (self.process.stdout.read() or "")) + time.sleep(0.05) + raise RuntimeError("gate did not start listening") + + def connect(self): + return socket.create_connection(("127.0.0.1", self.port), timeout=30) + + def state(self): + with open(self.drain) as handle: + return json.load(handle) + + def terminate(self, timeout=20): + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(5) + return None + return self.process.returncode + + def logs(self): + try: + return self.process.stdout.read() or "" + except Exception: + return "" + + +def read_all(conn, timeout=30): + conn.settimeout(timeout) + chunks = [] + try: + while True: + data = conn.recv(65536) + if not data: + break + chunks.append(data) + except socket.timeout: + chunks.append(b"") + except ConnectionResetError: + # 对端 RST(例如上游不可达时闸门立即拆连接)也是"连接已结束",不是挂住。 + pass + return b"".join(chunks) + + +def test_long_silent_stream(): + upstream = FakeUpstream() + gate = Gate(upstream.port) + try: + conn = gate.connect() + conn.sendall(b"GET /slow HTTP/1.1\r\nHost: gate\r\n\r\n") + started = time.time() + payload = read_all(conn, timeout=30) + elapsed = time.time() - started + conn.close() + check("long-silent-stream/complete", b"abcdefghij" in payload, + f"({elapsed:.1f}s, {len(payload)}B)") + check("long-silent-stream/survived-gap", elapsed >= SILENT_GAP - 1, + f"gap={SILENT_GAP}s elapsed={elapsed:.1f}s") + finally: + gate.terminate() + upstream.close() + + +def test_half_close(): + upstream = FakeUpstream() + gate = Gate(upstream.port) + try: + conn = gate.connect() + conn.sendall(b"GET /half HTTP/1.1\r\nHost: gate\r\n\r\n") + conn.shutdown(socket.SHUT_WR) # 客户端半关闭:只关自己的写端 + started = time.time() + payload = read_all(conn, timeout=15) + elapsed = time.time() - started + conn.close() + check("half-close/response-delivered", b"after-eof" in payload, f"({elapsed:.1f}s)") + check("half-close/eof-propagated", elapsed >= 0.5, f"elapsed={elapsed:.1f}s") + finally: + gate.terminate() + upstream.close() + + +def test_large_body_integrity(): + upstream = FakeUpstream() + gate = Gate(upstream.port) + try: + body = (b"0123456789abcdef" * 131072) # 2 MiB + conn = gate.connect() + conn.sendall(b"POST /echo HTTP/1.1\r\nHost: gate\r\nContent-Length: %d\r\n\r\n" % len(body)) + conn.sendall(body) + payload = read_all(conn, timeout=30) + conn.close() + head, _, received = payload.partition(b"\r\n\r\n") + check("large-body/integrity", received == body, + f"sent={len(body)}B got={len(received)}B head={head.splitlines()[:1]}") + finally: + gate.terminate() + upstream.close() + + +def test_drain_with_inflight(): + upstream = FakeUpstream() + gate = Gate(upstream.port, drain_timeout=45) + try: + conn = gate.connect() + conn.sendall(b"GET /slow HTTP/1.1\r\nHost: gate\r\n\r\n") + time.sleep(0.5) # 确保在途连接已被计数 + sent_at = time.time() + gate.process.send_signal(signal.SIGTERM) + time.sleep(0.5) + refused = False + try: + late = socket.create_connection(("127.0.0.1", gate.port), timeout=2) + late.settimeout(2) + try: + late.sendall(b"GET /slow HTTP/1.1\r\nHost: gate\r\n\r\n") + refused = late.recv(1) == b"" + except OSError: + refused = True + late.close() + except OSError: + refused = True + check("drain/new-connections-refused", refused) + payload = read_all(conn, timeout=40) + conn.close() + check("drain/inflight-completed", b"abcdefghij" in payload, f"{len(payload)}B") + code = gate.terminate(timeout=30) + check("drain/exit-zero", code == 0, f"exit={code}") + state = gate.state() + fresh = os.path.getmtime(gate.drain) >= sent_at - 1 + check("drain/evidence", state.get("phase") == "drained" and state.get("active_at_close", 0) >= 1, + json.dumps({k: state.get(k) for k in ("phase", "active_at_close", "served", "drain_ms")})) + check("drain/evidence-fresh", fresh, f"mtime ok={fresh}") + finally: + if gate.process.poll() is None: + gate.terminate() + upstream.close() + + +def test_drain_timeout_is_failure(): + upstream = FakeUpstream() + gate = Gate(upstream.port, drain_timeout=1) + try: + conn = gate.connect() + conn.sendall(b"GET /hold HTTP/1.1\r\nHost: gate\r\n\r\n") + time.sleep(0.5) + code = gate.terminate(timeout=15) + check("drain-timeout/exit-nonzero", code == 2, f"exit={code}") + state = gate.state() + check("drain-timeout/phase", state.get("phase") == "drain_timeout", + json.dumps({k: state.get(k) for k in ("phase", "active", "active_at_close")})) + conn.close() + finally: + if gate.process.poll() is None: + gate.terminate() + upstream.close() + + +def test_upstream_unreachable(): + dead = free_port() + gate = Gate(dead) + try: + conn = gate.connect() + conn.sendall(b"GET / HTTP/1.1\r\nHost: gate\r\n\r\n") + started = time.time() + payload = read_all(conn, timeout=5) + elapsed = time.time() - started + conn.close() + check("upstream-dead/connection-closed", b"" not in payload and elapsed < 4, + f"{elapsed:.1f}s {len(payload)}B") + # 闸门本身还活着:仍能建连(会在上游阶段失败,但不挂住)。 + code = gate.terminate(timeout=10) + check("upstream-dead/gate-drains", code == 0, f"exit={code}") + finally: + if gate.process.poll() is None: + gate.terminate() + + +def main(): + for test in ( + test_long_silent_stream, + test_half_close, + test_large_body_integrity, + test_drain_with_inflight, + test_drain_timeout_is_failure, + test_upstream_unreachable, + ): + try: + test() + except Exception: + traceback.print_exc() + failures.append(test.__name__) + print() + print(f"summary: {len(failures)} failure(s)" + (":" + ",".join(failures) if failures else "")) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index 48d0f44..a1c175a 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -87,8 +87,11 @@ type Config struct { LibraryProductionS3Bucket string LibraryProductionS3AccessKey string LibraryProductionS3SecretKey string - LibraryEmbeddingModel string - LibraryEmbeddingDimension int + // LibraryEmbeddingModel 是整库发布使用的向量模型声明(快照契约缺失时的回落值)。 + LibraryEmbeddingModel string + // LibraryWorkerRole 决定这个进程消费哪一段队列(见 LibraryWorkerRole 说明)。 + LibraryWorkerRole LibraryWorkerRole + LibraryEmbeddingDimension int // LibraryCapacityBudgetBytes 是整库归档容量预算;0 表示未设置(容量门禁不放行)。 LibraryCapacityBudgetBytes int64 // LibraryRetentionVersions / Days 是保留策略;都为 0 表示未设置(不生成清理候选)。 @@ -238,6 +241,10 @@ func Load(getenv func(string) string) (Config, error) { if config.DatabaseURL == "" { return Config{}, errors.New("DATABASE_URL is required") } + // 角色非法必须在启动时失败:一个拼错的角色会让进程悄悄回到"什么都消费"。 + if config.LibraryWorkerRole, err = parseLibraryWorkerRole(getenv("LIBRARY_WORKER_ROLE")); err != nil { + return Config{}, err + } if config.PollInterval, err = durationOrDefault(getenv("WORKER_POLL_INTERVAL"), config.PollInterval); err != nil { return Config{}, fmt.Errorf("WORKER_POLL_INTERVAL: %w", err) } diff --git a/internal/appconfig/library_role.go b/internal/appconfig/library_role.go new file mode 100644 index 0000000..c91fdfa --- /dev/null +++ b/internal/appconfig/library_role.go @@ -0,0 +1,84 @@ +package appconfig + +import ( + "fmt" + "strings" + + "knowledge-governance/internal/governance" +) + +// LibraryWorkerRole 是这个进程在整库发布链上的角色(LIBRARY_WORKER_ROLE)。 +// +// 为什么要有显式角色,而不是"配了目标就算这个进程能发布":一条 release 作业 +// 只带 scope 与快照 ID,**目标实例来自本进程的部署配置**(LIBRARY_TARGET_* / +// LIBRARY_PRODUCTION_*)。两个执行器同时消费同一队列时,谁抢到就按谁自己的 +// 配置执行——本机执行器会把"发往云测试"的作业按本机 kgseal 的维护窗口去停本机 +// 容器、往本机目标写。这不是权限问题,是身份问题,只能靠"谁消费哪段队列" +// 在启动时定死。 +type LibraryWorkerRole string + +const ( + // LibraryWorkerRoleFull 是历史行为:本进程消费全部队列(单机部署)。 + LibraryWorkerRoleFull LibraryWorkerRole = "full" + // LibraryWorkerRoleReleaseExecutor 只消费整库发布(sync / restore / recover / + // baseline)与它们的接管扫描。不起 scheduler、不接封存 agent、不消费 QA/测试/ + // 目录刷新等作业:那些队列归本机那个源侧执行器。 + LibraryWorkerRoleReleaseExecutor LibraryWorkerRole = "release-executor" + // LibraryWorkerRoleCapture 消费除整库发布以外的全部队列(封存、QA、测试、 + // 目录刷新…),并且**不接管** release 作业。与 release-executor 互斥。 + LibraryWorkerRoleCapture LibraryWorkerRole = "capture" +) + +func parseLibraryWorkerRole(raw string) (LibraryWorkerRole, error) { + value := LibraryWorkerRole(strings.TrimSpace(raw)) + switch value { + case "": + return LibraryWorkerRoleFull, nil + case LibraryWorkerRoleFull, LibraryWorkerRoleReleaseExecutor, LibraryWorkerRoleCapture: + return value, nil + default: + return "", fmt.Errorf("LIBRARY_WORKER_ROLE 只能是 full/release-executor/capture,当前是 %q", string(value)) + } +} + +func (r LibraryWorkerRole) String() string { + if r == "" { + return string(LibraryWorkerRoleFull) + } + return string(r) +} + +// ExecutesRelease:本进程注册整库发布 handler,并接管中断的发布作业。 +func (r LibraryWorkerRole) ExecutesRelease() bool { + return r == "" || r == LibraryWorkerRoleFull || r == LibraryWorkerRoleReleaseExecutor +} + +// ExecutesSourceWork:本进程注册封存/QA/测试/目录刷新等非发布队列,并起后台调度。 +func (r LibraryWorkerRole) ExecutesSourceWork() bool { + return r == "" || r == LibraryWorkerRoleFull || r == LibraryWorkerRoleCapture +} + +// ExecutesMigrations:只有控制面所在的那个进程才做 DDL。 +// +// 发布执行器是"目标旁边的一个消费者",不是控制面:让它去改治理库 schema, +// 等于把"谁有权决定治理库结构"扩散到第二台机器。云侧只读校验版本,落后就拒绝启动。 +func (r LibraryWorkerRole) ExecutesMigrations() bool { + return r == "" || r == LibraryWorkerRoleFull || r == LibraryWorkerRoleCapture +} + +// LibraryJobKinds 是这个角色负责收拾孤儿/接管的 library_jobs.kind。 +// 分开是因为"收拾"会写终态和恢复责任文案:seal 的恢复责任在封存 agent, +// release 的恢复责任在目标与预备份,混着写会把责任指错人。 +func (r LibraryWorkerRole) LibraryJobKinds() []string { + switch r { + case LibraryWorkerRoleReleaseExecutor: + return []string{string(governance.LibraryJobSync), string(governance.LibraryJobRestore), + string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline)} + case LibraryWorkerRoleCapture: + return []string{string(governance.LibraryJobSeal)} + default: + return []string{string(governance.LibraryJobSeal), string(governance.LibraryJobSync), + string(governance.LibraryJobRestore), string(governance.LibraryJobRecover), + string(governance.LibraryJobBaseline)} + } +} diff --git a/internal/appconfig/library_role_test.go b/internal/appconfig/library_role_test.go new file mode 100644 index 0000000..a92fc5b --- /dev/null +++ b/internal/appconfig/library_role_test.go @@ -0,0 +1,83 @@ +package appconfig + +import "testing" + +func TestParseLibraryWorkerRole(t *testing.T) { + cases := []struct { + raw string + want LibraryWorkerRole + release bool + source bool + wantFail bool + }{ + {raw: "", want: LibraryWorkerRoleFull, release: true, source: true}, + {raw: "full", want: LibraryWorkerRoleFull, release: true, source: true}, + {raw: " release-executor ", want: LibraryWorkerRoleReleaseExecutor, release: true, source: false}, + {raw: "capture", want: LibraryWorkerRoleCapture, release: false, source: true}, + {raw: "release", wantFail: true}, + {raw: "worker", wantFail: true}, + } + for _, test := range cases { + got, err := parseLibraryWorkerRole(test.raw) + if test.wantFail { + if err == nil { + t.Fatalf("parseLibraryWorkerRole(%q) accepted an unknown role", test.raw) + } + continue + } + if err != nil { + t.Fatalf("parseLibraryWorkerRole(%q) error = %v", test.raw, err) + } + if got != test.want { + t.Fatalf("parseLibraryWorkerRole(%q) = %q, want %q", test.raw, got, test.want) + } + if got.ExecutesRelease() != test.release || got.ExecutesSourceWork() != test.source { + t.Fatalf("role %q release=%t source=%t, want release=%t source=%t", + got, got.ExecutesRelease(), got.ExecutesSourceWork(), test.release, test.source) + } + if got.ExecutesRelease() && got.ExecutesSourceWork() && got != LibraryWorkerRoleFull { + t.Fatalf("role %q claims both domains without being full", got) + } + } +} + +// 非法角色必须让进程起不来:静默回到 full 会让两套执行器抢同一批 release 作业。 +func TestLoadRejectsUnknownLibraryWorkerRole(t *testing.T) { + env := map[string]string{ + "DATABASE_URL": "postgres://user:pass@127.0.0.1:5432/governance?sslmode=disable", + "LIBRARY_WORKER_ROLE": "release", + } + if _, err := Load(func(key string) string { return env[key] }); err == nil { + t.Fatal("Load() accepted LIBRARY_WORKER_ROLE=release") + } + env["LIBRARY_WORKER_ROLE"] = "release-executor" + config, err := Load(func(key string) string { return env[key] }) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !config.LibraryWorkerRole.ExecutesRelease() || config.LibraryWorkerRole.ExecutesSourceWork() { + t.Fatalf("release-executor role mapping is wrong: %#v", config.LibraryWorkerRole) + } +} + +// 每个角色收拾哪些 kind、谁拥有 DDL,都必须互斥且覆盖完整。 +func TestLibraryWorkerRoleOwnershipMaps(t *testing.T) { + release := LibraryWorkerRoleReleaseExecutor + if got := release.LibraryJobKinds(); len(got) != 4 || got[0] != "sync" || got[1] != "restore" || got[2] != "recover" || got[3] != "baseline" { + t.Fatalf("release-executor kinds = %v", got) + } + if got := LibraryWorkerRoleCapture.LibraryJobKinds(); len(got) != 1 || got[0] != "seal" { + t.Fatalf("capture kinds = %v", got) + } + if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 5 { + t.Fatalf("full kinds = %v", got) + } + for _, role := range []LibraryWorkerRole{LibraryWorkerRoleFull, LibraryWorkerRoleCapture} { + if !role.ExecutesMigrations() { + t.Fatalf("role %q must own DDL", role) + } + } + if release.ExecutesMigrations() { + t.Fatal("release-executor must not own governance DDL") + } +} diff --git a/internal/librarysnapshot/transfer.go b/internal/librarysnapshot/transfer.go new file mode 100644 index 0000000..f641631 --- /dev/null +++ b/internal/librarysnapshot/transfer.go @@ -0,0 +1,354 @@ +package librarysnapshot + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "syscall" +) + +// 跨主机搬运一份已封存快照。 +// +// 为什么不是"把 versions/ 和 blobs/ 目录 rsync 过去": +// - 直接同步目录时,"版本清单先到、blob 还在路上"的瞬间看起来就是一份可用版本, +// 下一个发布作业会读到缺字节的档然后把失败记在目标身上; +// - 也不能只搬 blobs:清单之外的 blob 无法证明属于这份内容。 +// +// 所以搬运单位是"自描述包":清单先写、每个 blob 以 sha256 命名、下载侧边收边算, +// 只有全部齐了才把版本清单落位。版本清单文件就是提交点(见 Install 的注释)。 +const BundleFormat = "weknora-library-snapshot-bundle/v1" + +// 包内路径。清单带身份(文件名即快照 ID),blob 带内容身份(文件名即 sha256): +// 接收侧不需要额外的锁文件或元数据就能判断"这是什么、字节对不对"。 +const ( + bundleManifestPrefix = "manifest/" + bundleBlobPrefix = "blobs/" + maxBundleEntries = 200000 +) + +// BundleInfo 是一次导出的结果(CLI 打印它,父据此核对哈希)。 +type BundleInfo struct { + Format string `json:"format"` + Snapshot string `json:"snapshot"` + Parts int `json:"parts"` + Bytes int64 `json:"bytes"` + // Blobs 是逐个 blob 的 sha256 列表,便于在两台机器上对拍而不传大文件。 + Blobs []string `json:"blobs"` +} + +// Pack 把一份已封存版本导出成单文件包(tar)。 +// +// 导出前先 Load:它逐 blob 重算 sha256,所以"包里的每个字节都在这台机器上验证过"。 +// 导出后不删源,也不改归档。 +func (a *Archive) Pack(id, outPath string) (BundleInfo, error) { + if !validDigest(id) { + return BundleInfo{}, errors.New("invalid snapshot ID") + } + if !filepath.IsAbs(outPath) || filepath.Clean(outPath) != outPath { + return BundleInfo{}, errors.New("bundle path must be an absolute clean path") + } + manifest, err := a.Load(id) + if err != nil { + return BundleInfo{}, fmt.Errorf("snapshot is not verifiable on this host: %w", err) + } + manifestFile, err := regular(a.versions, id+".json", maxManifestBytes) + if err != nil { + return BundleInfo{}, err + } + manifestBytes, readErr := io.ReadAll(io.LimitReader(manifestFile, maxManifestBytes+1)) + manifestFile.Close() + if readErr != nil || int64(len(manifestBytes)) > maxManifestBytes || digest(manifestBytes) != id { + return BundleInfo{}, errors.New("snapshot manifest changed while packing") + } + if _, err := os.Lstat(outPath); err == nil { + return BundleInfo{}, errors.New("bundle already exists; refusing to overwrite") + } else if !errors.Is(err, os.ErrNotExist) { + return BundleInfo{}, errors.New("bundle path is not usable") + } + out, err := os.OpenFile(outPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return BundleInfo{}, errors.New("create bundle") + } + cleanup := func(cause error) (BundleInfo, error) { + out.Close() + os.Remove(outPath) + return BundleInfo{}, cause + } + writer := tar.NewWriter(out) + writeEntry := func(name string, size int64, body io.Reader) error { + header := &tar.Header{Name: name, Mode: 0600, Size: size, Typeflag: tar.TypeReg} + if err := writer.WriteHeader(header); err != nil { + return err + } + _, err := io.CopyN(writer, body, size) + return err + } + // 清单必须第一个写:接收侧读到清单才知道后面应该有哪些 blob。 + if err := writeEntry(bundleManifestPrefix+id+".json", int64(len(manifestBytes)), bytes.NewReader(manifestBytes)); err != nil { + return cleanup(errors.New("write bundle manifest")) + } + info := BundleInfo{Format: BundleFormat, Snapshot: id, Parts: len(manifest.Parts)} + blobs := make([]string, 0, len(manifest.Parts)) + seen := map[string]bool{} + for _, part := range manifest.Parts { + if seen[part.SHA256] { + continue + } + seen[part.SHA256] = true + file, err := regular(a.blobs, part.SHA256, a.maxBlobBytes) + if err != nil { + return cleanup(fmt.Errorf("snapshot blob %s is not readable: %w", part.SHA256, err)) + } + writeErr := writeEntry(bundleBlobPrefix+part.SHA256, part.Bytes, file) + file.Close() + if writeErr != nil { + return cleanup(errors.New("write bundle blob")) + } + info.Bytes += part.Bytes + blobs = append(blobs, part.SHA256) + } + if err := writer.Close(); err != nil { + return cleanup(errors.New("finalize bundle")) + } + if err := out.Sync(); err != nil { + return cleanup(errors.New("sync bundle")) + } + if err := out.Close(); err != nil { + return BundleInfo{}, errors.New("close bundle") + } + sort.Strings(blobs) + info.Blobs = blobs + return info, nil +} + +// Unpack 校验一个包并把它装入本归档。返回值是这份快照自己的清单。 +// +// 提交点是"版本清单落位":blob 先按内容寻址写好并 fsync,最后一步才 Link +// versions/.json。因此任何中断都只会留下不可见的 blob(可被下一次重放复用), +// 不会留下"看起来存在但缺字节"的版本。 +// +// 幂等:同一份包重复导入成功;名字相同内容不同的 blob 或清单直接拒绝。 +func (a *Archive) Unpack(bundlePath, expectID string) (Manifest, error) { + if expectID != "" && !validDigest(expectID) { + return Manifest{}, errors.New("invalid expected snapshot ID") + } + if !filepath.IsAbs(bundlePath) || filepath.Clean(bundlePath) != bundlePath { + return Manifest{}, errors.New("bundle path must be an absolute clean path") + } + file, err := os.OpenFile(bundlePath, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return Manifest{}, errors.New("open bundle") + } + defer file.Close() + if info, statErr := file.Stat(); statErr != nil || !info.Mode().IsRegular() { + return Manifest{}, errors.New("bundle must be a regular file") + } + reader := tar.NewReader(file) + var ( + id string + manifest Manifest + manifestRaw []byte + expected map[string]Part + received map[string]bool + entryCount int + ) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return Manifest{}, errors.New("bundle is not a readable archive") + } + entryCount++ + if entryCount > maxBundleEntries { + return Manifest{}, errors.New("bundle has too many entries") + } + if header.Typeflag != tar.TypeReg { + return Manifest{}, fmt.Errorf("bundle entry %q is not a regular file", header.Name) + } + switch { + case len(header.Name) == len(bundleManifestPrefix)+len(".json")+64 && header.Name[:len(bundleManifestPrefix)] == bundleManifestPrefix: + if id != "" { + return Manifest{}, errors.New("bundle contains more than one manifest") + } + id = header.Name[len(bundleManifestPrefix) : len(header.Name)-len(".json")] + if !validDigest(id) { + return Manifest{}, errors.New("bundle manifest name is not a snapshot ID") + } + if expectID != "" && id != expectID { + return Manifest{}, fmt.Errorf("bundle snapshot is %s, expected %s", id, expectID) + } + if header.Size <= 0 || header.Size > maxManifestBytes { + return Manifest{}, errors.New("bundle manifest size is not allowed") + } + manifestRaw, err = io.ReadAll(io.LimitReader(reader, maxManifestBytes+1)) + if err != nil || int64(len(manifestRaw)) != header.Size || digest(manifestRaw) != id { + return Manifest{}, errors.New("bundle manifest digest does not match its ID") + } + manifest, expected, err = decodeBundleManifest(manifestRaw, id) + if err != nil { + return Manifest{}, err + } + received = map[string]bool{} + case len(header.Name) == len(bundleBlobPrefix)+64 && header.Name[:len(bundleBlobPrefix)] == bundleBlobPrefix: + if id == "" { + return Manifest{}, errors.New("bundle blob appears before the manifest") + } + name := header.Name[len(bundleBlobPrefix):] + if !validDigest(name) { + return Manifest{}, errors.New("bundle blob name is not a sha256 digest") + } + part, ok := expected[name] + if !ok { + return Manifest{}, fmt.Errorf("bundle carries blob %s which this snapshot does not declare", name) + } + if part.Bytes != header.Size { + return Manifest{}, fmt.Errorf("bundle blob %s size %d does not match manifest %d", name, header.Size, part.Bytes) + } + if received[name] { + return Manifest{}, fmt.Errorf("bundle repeats blob %s", name) + } + if err := a.installBlob(reader, part); err != nil { + return Manifest{}, err + } + received[name] = true + default: + return Manifest{}, fmt.Errorf("bundle entry %q is not part of a snapshot bundle", header.Name) + } + } + if id == "" { + return Manifest{}, errors.New("bundle has no manifest") + } + for name := range expected { + if !received[name] { + return Manifest{}, fmt.Errorf("bundle is missing blob %s", name) + } + } + if err := syncRoot(a.blobs); err != nil { + return Manifest{}, err + } + if err := a.installManifest(id, manifestRaw); err != nil { + return Manifest{}, err + } + if err := syncRoot(a.versions); err != nil { + return Manifest{}, err + } + // 落位后用正式读取路径再验一遍:证明这份快照在本机真的可发布。 + if _, err := a.Load(id); err != nil { + return Manifest{}, fmt.Errorf("imported snapshot does not verify: %w", err) + } + return manifest, nil +} + +// installBlob 把一个 blob 从包里写到 blobs/。 +// 已存在时必须字节一致,否则拒绝——同名不同内容是身份冲突,不是"覆盖一下"。 +func (a *Archive) installBlob(body io.Reader, part Part) error { + if _, err := a.blobs.Lstat(part.SHA256); err == nil { + if err := a.verify(part); err != nil { + return fmt.Errorf("archive already holds different bytes for blob %s", part.SHA256) + } + // 已有且一致:仍然读完包里的字节,否则 tar 流会错位。 + _, err := io.CopyN(io.Discard, body, part.Bytes) + return err + } else if !errors.Is(err, os.ErrNotExist) { + return errors.New("archive blob lookup failed") + } + name, tmp, err := temporary(a.blobs) + if err != nil { + return err + } + defer a.blobs.Remove(name) + hash := sha256.New() + written, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(body, part.Bytes+1)) + if err != nil || written != part.Bytes || hex.EncodeToString(hash.Sum(nil)) != part.SHA256 { + tmp.Close() + return fmt.Errorf("bundle blob %s bytes do not match its digest", part.SHA256) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := a.blobs.Link(name, part.SHA256); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + return nil +} + +// installManifest 落位版本清单(提交点)。已存在必须是同一份字节。 +func (a *Archive) installManifest(id string, raw []byte) error { + if _, err := a.versions.Lstat(id + ".json"); err == nil { + existing, err := a.LoadManifest(id) + if err != nil { + return fmt.Errorf("archive already holds an unreadable version %s: %w", id, err) + } + _ = existing + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return errors.New("archive version lookup failed") + } + name, tmp, err := temporary(a.versions) + if err != nil { + return err + } + defer a.versions.Remove(name) + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := a.versions.Link(name, id+".json"); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + return nil +} + +// decodeBundleManifest 只接受"规范化字节"的清单:包里的清单必须与归档 Seal 时 +// 写出的字节一致,否则即使摘要对得上,也可能是另一套字段排序/空值表达。 +func decodeBundleManifest(raw []byte, id string) (Manifest, map[string]Part, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var manifest Manifest + if err := decoder.Decode(&manifest); err != nil { + return Manifest{}, nil, errors.New("bundle manifest is not a valid snapshot manifest") + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return Manifest{}, nil, errors.New("bundle manifest contains trailing data") + } + canonical, err := encodeManifest(manifest) + if err != nil { + return Manifest{}, nil, fmt.Errorf("bundle manifest is not publishable: %w", err) + } + if !bytes.Equal(canonical, raw) || digest(canonical) != id { + return Manifest{}, nil, errors.New("bundle manifest is not in canonical form") + } + expected := make(map[string]Part, len(manifest.Parts)) + for _, part := range manifest.Parts { + if existing, ok := expected[part.SHA256]; ok { + if existing.Bytes != part.Bytes { + return Manifest{}, nil, errors.New("bundle manifest declares conflicting blob sizes") + } + continue + } + expected[part.SHA256] = part + } + return manifest, expected, nil +} diff --git a/internal/librarysnapshot/transfer_test.go b/internal/librarysnapshot/transfer_test.go new file mode 100644 index 0000000..997b736 --- /dev/null +++ b/internal/librarysnapshot/transfer_test.go @@ -0,0 +1,225 @@ +package librarysnapshot + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "testing" +) + +func newTransferArchive(t *testing.T) (*Archive, string) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + archive, err := Open(root, 1024*1024) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { archive.Close() }) + return archive, root +} + +func sealFixture(t *testing.T, archive *Archive) (string, Manifest) { + t.Helper() + part, err := archive.Put(bytes.NewBufferString("[]")) + if err != nil { + t.Fatal(err) + } + manifest := testManifest(part) + id, err := archive.Seal(manifest) + if err != nil { + t.Fatal(err) + } + return id, manifest +} + +// 一份验证过的快照搬过去必须字节一致、身份不变,重复导入幂等。 +func TestPackUnpackRoundTripKeepsIdentityAndBytes(t *testing.T) { + source, _ := newTransferArchive(t) + id, manifest := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + info, err := source.Pack(id, bundle) + if err != nil { + t.Fatalf("Pack() error = %v", err) + } + if info.Snapshot != id || info.Parts != len(manifest.Parts) || info.Bytes <= 0 || len(info.Blobs) == 0 { + t.Fatalf("bundle info = %#v", info) + } + destination, _ := newTransferArchive(t) + got, err := destination.Unpack(bundle, id) + if err != nil { + t.Fatalf("Unpack() error = %v", err) + } + if got.KnowledgeBaseID != manifest.KnowledgeBaseID || len(got.Parts) != len(manifest.Parts) { + t.Fatal("imported manifest identity changed") + } + if _, err := destination.Load(id); err != nil { + t.Fatalf("imported snapshot does not load: %v", err) + } + if _, err := destination.Unpack(bundle, id); err != nil { + t.Fatalf("repeated import is not idempotent: %v", err) + } +} + +// 包不覆盖已有文件:导出和导入都不能"顺手替换"一份已存在的产物。 +func TestPackRefusesToOverwriteExistingBundle(t *testing.T) { + source, _ := newTransferArchive(t) + id, _ := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + if err := os.WriteFile(bundle, []byte("occupied"), 0600); err != nil { + t.Fatal(err) + } + if _, err := source.Pack(id, bundle); err == nil { + t.Fatal("Pack() overwrote an existing bundle") + } +} + +// 身份对不上就不许导入:调用方点错快照时,宁可失败。 +func TestUnpackRejectsUnexpectedSnapshotIdentity(t *testing.T) { + source, _ := newTransferArchive(t) + id, _ := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + if _, err := source.Pack(id, bundle); err != nil { + t.Fatal(err) + } + destination, _ := newTransferArchive(t) + if _, err := destination.Unpack(bundle, digest([]byte("another snapshot"))); err == nil { + t.Fatal("Unpack() accepted a bundle whose ID does not match the expectation") + } +} + +// 包里的字节必须与名字(sha256)一致:篡改一个 blob 就整体拒绝,且不留下版本。 +func TestUnpackRejectsTamperedBlobAndLeavesNoVersion(t *testing.T) { + source, _ := newTransferArchive(t) + id, manifest := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + info, err := source.Pack(id, bundle) + if err != nil { + t.Fatal(err) + } + tampered := filepath.Join(t.TempDir(), "tampered.tar") + rewriteBundle(t, bundle, tampered, func(name string, body []byte) (string, []byte) { + if name == bundleBlobPrefix+info.Blobs[0] && len(body) > 0 { + body = append([]byte(nil), body...) + body[0] ^= 0xff + } + return name, body + }) + destination, root := newTransferArchive(t) + if _, err := destination.Unpack(tampered, id); err == nil { + t.Fatal("Unpack() accepted a tampered blob") + } + if _, err := os.Stat(filepath.Join(root, "versions", id+".json")); !os.IsNotExist(err) { + t.Fatal("a failed import left a version behind") + } + _ = manifest +} + +// 包里出现快照不声明的 blob / 缺 blob / 非普通文件,一律拒绝。 +func TestUnpackRejectsIncompleteAndForeignBundles(t *testing.T) { + source, _ := newTransferArchive(t) + id, manifest := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + if _, err := source.Pack(id, bundle); err != nil { + t.Fatal(err) + } + missing := filepath.Join(t.TempDir(), "missing.tar") + first := manifest.Parts[0].SHA256 + rewriteBundle(t, bundle, missing, func(name string, body []byte) (string, []byte) { + if name == bundleBlobPrefix+first { + return "", nil + } + return name, body + }) + destination, _ := newTransferArchive(t) + if _, err := destination.Unpack(missing, id); err == nil { + t.Fatal("Unpack() accepted a bundle missing a declared blob") + } + foreign := filepath.Join(t.TempDir(), "foreign.tar") + extra := digest([]byte("extra blob")) + rewriteBundle(t, bundle, foreign, func(name string, body []byte) (string, []byte) { + if name == bundleBlobPrefix+first { + return bundleBlobPrefix + extra, []byte("x") + } + return name, body + }) + other, _ := newTransferArchive(t) + if _, err := other.Unpack(foreign, id); err == nil { + t.Fatal("Unpack() accepted a bundle carrying a blob the snapshot does not declare") + } + traversal := filepath.Join(t.TempDir(), "traversal.tar") + rewriteBundle(t, bundle, traversal, func(name string, body []byte) (string, []byte) { + if name == bundleBlobPrefix+first { + return "../escape", body + } + return name, body + }) + third, _ := newTransferArchive(t) + if _, err := third.Unpack(traversal, id); err == nil { + t.Fatal("Unpack() accepted an entry outside the bundle layout") + } +} + +// 目标机已经存在同名不同内容的 blob:那是身份冲突,必须拒绝而不是覆盖。 +func TestUnpackRejectsConflictingExistingBlob(t *testing.T) { + source, _ := newTransferArchive(t) + id, manifest := sealFixture(t, source) + bundle := filepath.Join(t.TempDir(), "snapshot.tar") + if _, err := source.Pack(id, bundle); err != nil { + t.Fatal(err) + } + destination, root := newTransferArchive(t) + victim := manifest.Parts[0].SHA256 + if err := os.WriteFile(filepath.Join(root, "blobs", victim), []byte("[]different"), 0600); err != nil { + t.Fatal(err) + } + if _, err := destination.Unpack(bundle, id); err == nil { + t.Fatal("Unpack() overwrote a blob whose bytes differ") + } +} + +// rewriteBundle 逐条重放一个包,按 mutate 改名字/内容;返回空名字表示删掉该条。 +func rewriteBundle(t *testing.T, source, target string, mutate func(name string, body []byte) (string, []byte)) { + t.Helper() + in, err := os.Open(source) + if err != nil { + t.Fatal(err) + } + defer in.Close() + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + writer := tar.NewWriter(out) + reader := tar.NewReader(in) + for { + header, err := reader.Next() + if err != nil { + break + } + body := make([]byte, header.Size) + if _, err := io.ReadFull(reader, body); err != nil { + t.Fatal(err) + } + name, rewritten := mutate(header.Name, body) + if name == "" { + continue + } + if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: int64(len(rewritten)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := writer.Write(rewritten); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/librarystack/stack.go b/internal/librarystack/stack.go index 6449955..f20abe5 100644 --- a/internal/librarystack/stack.go +++ b/internal/librarystack/stack.go @@ -150,6 +150,12 @@ func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, sn target.Close() return nil, nil, verifyErr } + // 图谱门禁:目标开了 graph_enabled 时,发布只写 PG/ES/OSS,图谱会静默变旧。 + // 只读检测 + 拒绝;不关闭目标图谱、不搬 Neo4j(那属于扩围)。 + if graphErr := target.VerifyGraphGate(ctx); graphErr != nil { + target.Close() + return nil, nil, graphErr + } // 生产索引存在时必须与快照契约一致;不存在或为空基线时跳过比对 // (空索引的映射没有可比性,非空之后仍然严格比对)。 if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index f0da635..18f33b4 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -521,23 +521,36 @@ WHERE id=$1 AND state IN ('running','pending') // ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿: // 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。 // 只标失败并写明恢复责任,不猜结果、不重放。 -func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time) (int, error) { +// +// kinds 是**本进程负责的那几类**:两台执行器(本机封存 / 云端发布)各自收拾自己的 +// 作业,跨域"顺手收拾"会把别人的在飞作业标失败,并把恢复责任写错(封存要 agent +// 恢复来源,发布要按预备份确认目标)。kinds 为空直接报错,不允许退化成"收拾全部"。 +func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time, kinds []string) (int, error) { + if len(kinds) == 0 { + return 0, errors.New("reconcile library jobs requires the kinds this executor owns") + } if _, err := s.db.Exec(ctx, ` UPDATE library_jobs AS lj -SET state='failed', error='队列中已无对应作业,业务作业未收到结果', recovery='封存作业由 agent 恢复来源;确认来源 healthy 后可重发', updated_at=$1, finished_at=$1 +SET state='failed', error='队列中已无对应作业,业务作业未收到结果', + recovery=CASE WHEN lj.kind IN ('sync','restore','recover','baseline') + THEN '未确认目标状态:若已进入维护窗口,按预备份恢复并核验后再重发' + ELSE '封存作业由 agent 恢复来源;确认来源 healthy 后可重发' END, + updated_at=$1, finished_at=$1 WHERE lj.state IN ('pending','running') + AND lj.kind = ANY($2) AND lj.queue_job_id <> '' AND NOT EXISTS ( SELECT 1 FROM jobs AS j WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry') - )`, now.UTC()); err != nil { + )`, now.UTC(), kinds); err != nil { return 0, fmt.Errorf("reconcile library jobs: %w", err) } // 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。 tag, err := s.db.Exec(ctx, ` UPDATE library_jobs SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1 -WHERE state IN ('pending','running') AND queue_job_id='' AND created_at < $2`, now.UTC(), now.Add(-5*time.Minute).UTC()) +WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND created_at < $2`, + now.UTC(), now.Add(-5*time.Minute).UTC(), kinds) if err != nil { return 0, fmt.Errorf("reconcile unattached library jobs: %w", err) } diff --git a/internal/platform/postgres/library_reconcile_test.go b/internal/platform/postgres/library_reconcile_test.go new file mode 100644 index 0000000..665645e --- /dev/null +++ b/internal/platform/postgres/library_reconcile_test.go @@ -0,0 +1,18 @@ +package postgres + +import ( + "context" + "strings" + "testing" + "time" +) + +// 收拾孤儿必须带"本进程负责的 kind":没有 kind 就退化成"收拾全部", +// 会把另一台执行器的在飞作业标失败,并把恢复责任写错人。 +func TestReconcileLibraryJobsRequiresOwnedKinds(t *testing.T) { + store := NewStore(nil) + _, err := store.ReconcileLibraryJobs(context.Background(), time.Now(), nil) + if err == nil || !strings.Contains(err.Error(), "kinds") { + t.Fatalf("empty kinds must be rejected before touching the database, got %v", err) + } +} diff --git a/internal/platform/postgres/migrate.go b/internal/platform/postgres/migrate.go index a41aad0..829b81f 100644 --- a/internal/platform/postgres/migrate.go +++ b/internal/platform/postgres/migrate.go @@ -3,6 +3,7 @@ package postgres import ( "context" "embed" + "errors" "fmt" "io/fs" "sort" @@ -138,3 +139,46 @@ func RollbackLatestMigration(ctx context.Context, pool *pgxpool.Pool) error { } return nil } + +// PendingMigrations 只读地回答"治理库 schema 落后了没有"。 +// +// 给"不拥有 DDL 的执行器"用(云端发布执行器):启动即校验,落后就拒绝启动, +// 绝不顺手迁移——治理库结构由控制面那个进程决定,不因为多了一台机器就多一个 DDL 源。 +func PendingMigrations(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) { + if pool == nil { + return nil, errors.New("database pool is required") + } + migrations, err := loadMigrations() + if err != nil { + return nil, err + } + var ledgerExists bool + if err := pool.QueryRow(ctx, "SELECT to_regclass('public.schema_migrations') IS NOT NULL").Scan(&ledgerExists); err != nil { + return nil, fmt.Errorf("check migration ledger: %w", err) + } + applied := map[int64]bool{} + if ledgerExists { + rows, err := pool.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return nil, fmt.Errorf("read migration ledger: %w", err) + } + defer rows.Close() + for rows.Next() { + var version int64 + if err := rows.Scan(&version); err != nil { + return nil, err + } + applied[version] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + } + pending := make([]int64, 0, 4) + for _, item := range migrations { + if !applied[item.Version] { + pending = append(pending, item.Version) + } + } + return pending, nil +} diff --git a/internal/releaseadapter/v071/graph_gate_test.go b/internal/releaseadapter/v071/graph_gate_test.go new file mode 100644 index 0000000..75ce144 --- /dev/null +++ b/internal/releaseadapter/v071/graph_gate_test.go @@ -0,0 +1,26 @@ +package v071 + +import ( + "errors" + "strings" + "testing" +) + +// 目标开图谱 = 拒绝发布(本模块只写 PG/ES/OSS,不写 Neo4j)。 +func TestVerifyTargetGraphGateRefusesGraphEnabledTarget(t *testing.T) { + if err := verifyTargetGraphGate(false, nil); err != nil { + t.Fatalf("graph disabled must allow publish, got %v", err) + } + err := verifyTargetGraphGate(true, nil) + if err == nil || !strings.Contains(err.Error(), "graph_enabled") { + t.Fatalf("graph enabled must refuse with graph_enabled reason, got %v", err) + } +} + +// 读不到开关也是未知 → 拒绝(不能把"未知"当成"没开")。 +func TestVerifyTargetGraphGateFailsClosedWhenUnknown(t *testing.T) { + err := verifyTargetGraphGate(false, errors.New("connection reset")) + if err == nil || !strings.Contains(err.Error(), "未知") { + t.Fatalf("unknown graph state must fail closed, got %v", err) + } +} diff --git a/internal/releaseadapter/v071/library_target_contract.go b/internal/releaseadapter/v071/library_target_contract.go index 1dd5697..dd9ad3d 100644 --- a/internal/releaseadapter/v071/library_target_contract.go +++ b/internal/releaseadapter/v071/library_target_contract.go @@ -118,3 +118,48 @@ func ContractFromLive(source librarysnapshot.Compatibility, live LiveTargetContr } return contract } + +// graphEnabledSQL 只读知识库自己的索引策略:图谱开关是 KB 的声明,不是部署的推测值。 +const graphEnabledSQL = ` +SELECT COALESCE((indexing_strategy->>'graph_enabled')::boolean, false) +FROM knowledge_bases +WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL +` + +// GraphEnabled 回答"这个目标知识库是否开启了图谱索引"。 +// +// 为什么要挡在发布之前:发布链路只搬 PostgreSQL / Elasticsearch / 对象存储, +// 从不动 Neo4j。目标开图谱时,写进去的文档与图谱会静默不一致——没有任何 +// 行数/摘要校验能发现"图谱还是旧的"。这里只读、只拒绝,不修改目标配置, +// 也不扩写图谱迁移。 +// +// 读不到(库/行不可读、值无法解析)按未知处理:未知即拒绝。 +func (t *ProductionLibraryTarget) GraphEnabled(ctx context.Context) (bool, error) { + if t.pool == nil { + return false, errors.New("目标数据库未接线:无法确认图谱开关") + } + var enabled bool + if err := t.pool.QueryRow(ctx, graphEnabledSQL, t.scope.KnowledgeBaseID, t.scope.TenantID).Scan(&enabled); err != nil { + return false, fmt.Errorf("读不到目标知识库 %s 的图谱开关:%w", t.scope.KnowledgeBaseID, err) + } + return enabled, nil +} + +// verifyTargetGraphGate 把"读到开 / 读到关 / 读不到"换算成发布是否允许。 +// 纯函数:决定逻辑必须能被单测固定,而不是只能靠连真库才看得到。 +func verifyTargetGraphGate(enabled bool, readErr error) error { + if readErr != nil { + return fmt.Errorf("目标图谱开关未知,拒绝发布:%w", readErr) + } + if enabled { + return errors.New("目标知识库开启了图谱索引(graph_enabled=true):发布链路只写 PostgreSQL/Elasticsearch/对象存储," + + "不会同步 Neo4j,写进去的内容会与图谱静默不一致;拒绝发布(在目标侧关闭图谱或先提供图谱迁移)") + } + return nil +} + +// VerifyGraphGate 是发布装配处调用的门禁入口(读实时开关 + 换算成允许/拒绝)。 +func (t *ProductionLibraryTarget) VerifyGraphGate(ctx context.Context) error { + enabled, err := t.GraphEnabled(ctx) + return verifyTargetGraphGate(enabled, err) +} diff --git a/scripts/verify_cloud_release_target.py b/scripts/verify_cloud_release_target.py new file mode 100755 index 0000000..3f2446c --- /dev/null +++ b/scripts/verify_cloud_release_target.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""云测试发布目标:只读预检。 + +只做 ssh 上的只读命令(docker inspect/ps/exec 的只读查询、curl GET、test、stat), +不启停容器、不写文件、不改配置、不连治理库。每行一个判据:PASS / FAIL / WARN / MANUAL。 + +用法: + python3 scripts/verify_cloud_release_target.py --host weknora-test + python3 scripts/verify_cloud_release_target.py --host weknora-test --snapshot <64位ID> + +退出码:有 FAIL 即 1,否则 0(WARN/MANUAL 不影响退出码,但必须人读)。 +""" + +import argparse +import json +import shlex +import subprocess +import sys + +PROJECT_DEFAULT = "weknora-test-v071" +APP_DEFAULT = "app" +GATE_SERVICE_DEFAULT = "ingress" +DRAIN_DIR_DEFAULT = "/srv/kgtest-gate/state" +ARCHIVE_DEFAULT = "/var/lib/knowledge-governance/library" +GATE_URL_DEFAULT = "http://127.0.0.1:8080/health" +EXECUTOR_DEFAULT = "kgtest-release-executor" +RDS_HOST_DEFAULT = "pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com" + +results = [] + + +def record(level, name, detail): + results.append((level, name, detail)) + print(f"{level:<6} {name}: {detail}", flush=True) + + +def ssh(host, command, timeout=25): + argv = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8", host, command] + try: + completed = subprocess.run(argv, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return 124, "", "ssh timeout" + return completed.returncode, completed.stdout.strip(), completed.stderr.strip() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="weknora-test") + parser.add_argument("--project", default=PROJECT_DEFAULT) + parser.add_argument("--app-service", default=APP_DEFAULT) + parser.add_argument("--gate-service", default=GATE_SERVICE_DEFAULT) + parser.add_argument("--drain-dir", default=DRAIN_DIR_DEFAULT) + parser.add_argument("--archive", default=ARCHIVE_DEFAULT) + parser.add_argument("--gate-url", default=GATE_URL_DEFAULT) + parser.add_argument("--executor", default=EXECUTOR_DEFAULT) + parser.add_argument("--rds-host", default=RDS_HOST_DEFAULT) + parser.add_argument("--snapshot", default="") + args = parser.parse_args() + host = args.host + + code, _, err = ssh(host, "true") + if code != 0: + record("FAIL", "ssh", f"{host} 不可达({err or code})") + return finish() + record("PASS", "ssh", f"{host} 可达") + + # 1) 项目容器清单:每个服务都必须是写入者或"声明为非写入者",否则窗口拒绝开。 + code, out, err = ssh( + host, + "docker ps -a --filter label=com.docker.compose.project=%s " + "--format '{{.Label \"com.docker.compose.service\"}}'" % shlex.quote(args.project), + ) + services = sorted({line.strip() for line in out.splitlines() if line.strip()}) + if code != 0: + record("FAIL", "project-containers", f"读不到项目容器({err or code})") + services = [] + else: + record("PASS" if services else "FAIL", "project-containers", + f"project={args.project} services={','.join(services) or ''}") + + # 2) 写入者不得发布宿主机端口(否则存在绕开入口闸门的直连通道)。 + containers = {service: f"{args.project}-{service}-1" for service in services} + app_container = containers.get(args.app_service, "") + if app_container: + code, out, err = ssh(host, "docker inspect %s --format '{{json .HostConfig.PortBindings}}'" + % shlex.quote(app_container)) + if code != 0: + record("FAIL", "writer-no-publish", f"inspect {app_container} 失败({err or code})") + else: + bindings = json.loads(out) if out else None + if bindings in (None, {}, "null"): + record("PASS", "writer-no-publish", f"{app_container} 未发布宿主机端口") + else: + record("FAIL", "writer-no-publish", + f"{app_container} 仍发布端口 {out}:拓扑校验会拒绝开维护窗口") + else: + record("FAIL", "writer-no-publish", f"项目里找不到写入者服务 {args.app_service}") + + # 3) 入口闸门存在、健康、发布端口与入口地址端口一致。 + code, out, err = ssh(host, "docker inspect %s --format '{{json .State.Health.Status}} {{json .HostConfig.PortBindings}} {{json .State.Running}}'" + % shlex.quote(f"{args.project}-{args.gate_service}-1")) + if code != 0: + record("FAIL", "gate", f"找不到闸门容器 {args.project}-{args.gate_service}-1({err or code})") + else: + parts = out.split(" ", 2) + health = parts[0] if parts else "" + bindings = parts[1] if len(parts) > 1 else "" + running = parts[2] if len(parts) > 2 else "" + if running.strip() == "true" and health.strip('"') == "healthy" and ":8080->" in bindings: + record("PASS", "gate", f"healthy,发布端口 {bindings}") + else: + record("FAIL", "gate", f"running={running} health={health} bindings={bindings}") + + # 4) 经闸门的健康检查:这是"入口在服务"的实际证据,而不是容器状态。 + code, out, err = ssh(host, "curl -s -o /dev/null -w '%%{http_code}' -m 8 %s" % shlex.quote(args.gate_url)) + if code == 0 and out.strip() == "200": + record("PASS", "gate-health", f"{args.gate_url} -> 200") + else: + record("FAIL", "gate-health", f"{args.gate_url} -> {out or err or code}") + + # 5) 闸门写排空证据的目录必须存在且可写(协调器要在同一真实文件上删/读)。 + code, out, err = ssh(host, "test -d %s && test -w %s && echo ok" % (shlex.quote(args.drain_dir), shlex.quote(args.drain_dir))) + if code == 0 and out.strip() == "ok": + record("PASS", "drain-dir", f"{args.drain_dir} 存在且可写") + else: + record("FAIL", "drain-dir", f"{args.drain_dir} 不存在或不可写({err or code})") + code, out, _ = ssh(host, "stat -c '%%y %%s' %s 2>/dev/null" % shlex.quote(args.drain_dir + "/drain.json")) + if code == 0 and out: + record("WARN", "drain-file", f"已存在 drain.json({out}):协调器开窗口前会先删除并验证不存在") + else: + record("PASS", "drain-file", "无残留 drain.json") + + # 6) 队列:redis 在本项目网络内可达(只读 PING;有口令时 NOAUTH 也算可达)。 + redis_container = containers.get("redis", "") + if redis_container: + code, out, err = ssh(host, "docker exec %s redis-cli ping 2>&1" % shlex.quote(redis_container)) + text = (out or err).strip() + if "PONG" in text or "NOAUTH" in text: + record("PASS", "redis", f"{redis_container}: {text}") + else: + record("FAIL", "redis", f"{redis_container}: {text or code}") + else: + record("FAIL", "redis", "项目里没有 redis 服务") + + # 7) 索引:ES 可达且目标索引存在(只读 HEAD/GET)。 + es_container = containers.get("elasticsearch", "") + if es_container: + code, out, err = ssh(host, "docker exec %s curl -s -o /dev/null -w '%%{http_code}' -m 8 http://localhost:9200/_cluster/health" + % shlex.quote(es_container)) + record("PASS" if out.strip() == "200" else "FAIL", "elasticsearch", + f"{es_container} cluster/health -> {out.strip() or err or code}") + else: + record("FAIL", "elasticsearch", "项目里没有 elasticsearch 服务") + + # 8) RDS:宿主机到 5432 的 TCP 可达(不登录、不带凭据)。 + code, out, err = ssh(host, "timeout 5 bash -c 'cat < /dev/null > /dev/tcp/%s/5432' && echo open" + % shlex.quote(args.rds_host)) + record("PASS" if out.strip() == "open" else "FAIL", "rds-reachable", + f"{args.rds_host}:5432 {'可达' if out.strip() == 'open' else '不可达'}") + + # 9) 归档根:执行器要在这里读快照、写预备份与写集合台账。 + code, out, err = ssh(host, "test -d %s && stat -c '%%a' %s" % (shlex.quote(args.archive), shlex.quote(args.archive))) + if code == 0: + record("PASS" if out.strip() == "700" else "WARN", "archive-root", + f"{args.archive} 权限 {out.strip()}(要求 0700)") + else: + record("FAIL", "archive-root", f"{args.archive} 不存在") + + # 10) 可选的快照包:给出 ID 时只核对版本清单是否已落位(不读 blob)。 + if args.snapshot: + version = f"{args.archive}/versions/{args.snapshot}.json" + code, out, err = ssh(host, "test -f %s && sha256sum %s | cut -d' ' -f1" + % (shlex.quote(version), shlex.quote(version))) + if code == 0 and out.strip() == args.snapshot: + record("PASS", "snapshot", f"{args.snapshot} 已导入且清单摘要自证一致") + else: + record("FAIL", "snapshot", f"{args.snapshot} 未导入或清单摘要不符({out.strip() or err or code})") + + # 11) 其它入口:adapter / DobotAI / frontend 都能到 app。只改 frontend 不够, + # "唯一入口"必须逐条核对:谁能绕过闸门直连 app,就等于闸门关不干净。 + code, out, _ = ssh(host, "docker network ls --format '{{.Name}}'") + networks = set(out.split()) + record("PASS" if "weknora-test-v071-network" in networks else "FAIL", "target-network", + f"weknora-test-v071-network {'存在' if 'weknora-test-v071-network' in networks else '不存在'}") + + def inspect_env(container): + code, env_raw, _ = ssh(host, "docker inspect %s --format '{{json .Config.Env}}'" % shlex.quote(container)) + if code != 0 or not env_raw: + return None + values = {} + for entry in json.loads(env_raw): + key, _, value = entry.partition("=") + values[key] = value + return values + + code, out, _ = ssh(host, "docker ps --format '{{.Names}}'") + running = set(out.split()) + adapter = next((name for name in running if name.startswith("knowledge-adapter-adapter")), "") + if adapter: + env = inspect_env(adapter) or {} + base = env.get("WEKNORA_BASE_URL", "") + ok = base.startswith("http://ingress:") or base.startswith("http://kgtest-ingress:") + record("PASS" if ok else "FAIL", "entry/adapter", + f"{adapter} WEKNORA_BASE_URL={base or ''}(要求指向 ingress,否则 adapter 绕过闸门直连 app)") + else: + record("WARN", "entry/adapter", "未找到 adapter 容器(若它就是知识问答入口,必须改成指向 ingress)") + + dobotai = next((name for name in running if name.startswith("dobotai-dingtalk")), "") + if dobotai: + env = inspect_env(dobotai) or {} + base = env.get("WEKNORA_BASE_URL", "") + code, mode, _ = ssh(host, "docker inspect %s --format '{{.HostConfig.NetworkMode}}'" % shlex.quote(dobotai)) + host_network = mode.strip() == "host" + ok = host_network and base in ("http://127.0.0.1:8080", "http://localhost:8080") + record("PASS" if ok else "FAIL", "entry/dobotai", + f"{dobotai} net={mode.strip()} WEKNORA_BASE_URL={base or ''}(host 网络 + 127.0.0.1:8080 才恰好走闸门)") + else: + record("WARN", "entry/dobotai", "未找到 dobotai-dingtalk 容器") + + if "frontend" in containers: + env = inspect_env(containers["frontend"]) or {} + app_host = env.get("APP_HOST", "") + record("PASS" if app_host == "ingress" else "FAIL", "entry/frontend", + f"frontend APP_HOST={app_host or ''}(要求 ingress)") + + # 11) 发布执行器:在不在跑、角色对不对(只读这一个变量,不打印 env)。 + code, out, err = ssh(host, "docker ps --filter name=%s --format '{{.Status}}'" % shlex.quote(args.executor)) + if code == 0 and out.strip(): + record("PASS", "executor", f"{args.executor} {out.strip()}") + code, role, _ = ssh(host, "docker exec %s printenv LIBRARY_WORKER_ROLE" % shlex.quote(args.executor)) + if code == 0 and role.strip() == "release-executor": + record("PASS", "executor-role", "LIBRARY_WORKER_ROLE=release-executor") + else: + record("FAIL", "executor-role", f"LIBRARY_WORKER_ROLE={role.strip() or ''}(要求 release-executor)") + else: + record("WARN", "executor", f"{args.executor} 未运行(父部署后才会出现)") + + # 12) 图谱:目标 KB 若开了 graph_enabled,发布只写 PG/ES/OSS,图谱会静默变旧。 + record("MANUAL", "target-graph", + "需人工确认目标 KB 的 indexing_strategy.graph_enabled=false(发布路径不写 Neo4j)") + return finish() + + +def finish(): + failures = [row for row in results if row[0] == "FAIL"] + print() + print(f"summary: {len(results)} checks, {len(failures)} FAIL") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file