diff --git a/docs/library-sync-acceptance.md b/docs/library-sync-acceptance.md index fe491d1..c22170d 100644 --- a/docs/library-sync-acceptance.md +++ b/docs/library-sync-acceptance.md @@ -94,6 +94,18 @@ 4. 生产预备份、独立恢复演练、持久化协调作业、授权 API、真实进度、发布页面与完整存储治理仍未交付;以上验收项目保持待验证。 5. 基础模块已随 `0.1.96 / 9105f8316cc870a62b5ddacd1496f0cb0f798d87` 提交、推送并部署到本地测试治理平台;运行 API/worker 的二进制摘要已读回校验。该部署不等于业务功能接线或完整验收通过;未执行生产同步、恢复或真实业务清理。完整验收仍受本节及下述缺失接线阻塞。 +## 联合隔离捕获验收 + +新增 `TestLibraryCaptureIsolatedStack`,在同一次 `CaptureLibrary` 调用中连接真实隔离 PostgreSQL、随机 Elasticsearch 索引、真实 Docker 暂停/恢复控制器和本地资源文件读取器。写入服务使用无网络、无宿主挂载的 busybox 夹具,不是实际 WeKnora;资源读取不是实际 OSS/S3 验收。 + +- 非空文档、分块、资源、Wiki 正文及字符串状态归档后读回;检查无跨租户记录或会话数据泄漏。 +- 删除真实夹具索引记录后,新捕获必须失败,原运行容器仍恢复健康。 +- 删除源资源文件、修改源 PostgreSQL、删除源 ES 文档后,成功作业重放仍返回原快照,不再调用源端就绪检查或资源读取;归档重新加载通过。 +- 反向自检使用 Go overlay 仅关闭 legacy 索引覆盖检查:同一联合用例退出码 1,明确报 `missing real index accepted`;未修改的实现复跑退出码 0。 +- `make verify` 与三包 race 回归退出码 0;新用例在默认 CI 中显式声明环境跳过,本地单独实际执行。夹具索引和容器删除后读回不存在,随机隔离数据库已删除。 + +证据:`backups/library-stack-acceptance-20261028/`。该联合用例补充捕获协调证据,不证明真实源全部写入者受控、Wiki 撤销语义、内容/问答/性能验收绑定或生产同步恢复已经完成。 + ## 方案 A:暂停与恢复原语的实测证据 新增 `internal/libraryfreeze/`,包含持久化作业日志、源服务组互斥、暂停前原运行状态、不可变容器 ID、恢复读回,以及本地 Unix socket Docker 控制器。Docker 控制器只接受对应 source 且 `environment=test`、`freeze-role=writer` 的容器;不按模糊名称选取,不控制数据库/索引存储,不使用远程默认 Docker context。运行容器必须有健康检查。 diff --git a/internal/releaseadapter/v071/library_capture_integration_test.go b/internal/releaseadapter/v071/library_capture_integration_test.go new file mode 100644 index 0000000..312b552 --- /dev/null +++ b/internal/releaseadapter/v071/library_capture_integration_test.go @@ -0,0 +1,241 @@ +package v071 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/librarysnapshot" +) + +// The writer is a real disposable Docker process, not a deployed WeKnora. +// This tests capture orchestration, not production readiness or Wiki semantics. +func TestLibraryCaptureIsolatedStack(t *testing.T) { + dsn, base := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL") + if dsn == "" || base == "" || os.Getenv("KG_FREEZE_DOCKER_ISOLATED") != "1" { + t.Skip("requires isolated PG, ES and disposable Docker opt-in") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" { + t.Fatal("isolated database required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + db, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("open isolated PG") + } + defer db.Close() + if _, err = db.Exec(ctx, libraryTestSchema); err != nil { + t.Fatal(err) + } + if _, err = db.Exec(ctx, `ALTER TABLE resources ADD COLUMN provider text DEFAULT 's3', ADD COLUMN size bigint DEFAULT 8; + ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'; + ALTER TABLE wiki_page_issues ADD COLUMN status text DEFAULT 'open';`); err != nil { + t.Fatal(err) + } + token := make([]byte, 8) + if _, err = rand.Read(token); err != nil { + t.Fatal(err) + } + source := "kg-freeze-test-" + hex.EncodeToString(token) + index := "kg_library_test_" + hex.EncodeToString(token) + client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + request := func(method, path, body string, want int) { + t.Helper() + r, e := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body)) + if e != nil { + t.Fatal("fixture request") + } + r.Header.Set("Content-Type", "application/json") + resp, e := client.Do(r) + if e != nil { + t.Fatal("fixture HTTP failed") + } + defer resp.Body.Close() + if resp.StatusCode != want { + t.Fatalf("fixture HTTP %d, expected %d", resp.StatusCode, want) + } + } + request("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}`, 200) + defer func() { request("DELETE", "/"+index, "", 200); request("HEAD", "/"+index, "", 404) }() + body := `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"chunk-safe","source_id":"chunk-safe","content":"Temporary context\nchunk","embedding":[0.1,0.2]}` + request("PUT", "/"+index+"/_doc/chunk-safe?refresh=true", body, 201) + search, err := NewSearchExporter(SearchExporterConfig{BaseURL: base, Index: index, PageSize: 1, HTTPClient: client}) + if err != nil { + t.Fatal(err) + } + mapping, err := search.inspectMapping(ctx) + if err != nil { + t.Fatal(err) + } + docker := func(args ...string) string { + t.Helper() + c, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + out, e := exec.CommandContext(c, "docker", append([]string{"--host", "unix:///var/run/docker.sock"}, args...)...).CombinedOutput() + if e != nil { + t.Fatalf("fixture Docker failed: %v", e) + } + return strings.TrimSpace(string(out)) + } + image := docker("image", "inspect", "busybox:1.36", "--format", "{{.Id}}") + id := docker("create", "--name", source, "--network", "none", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1m", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--memory", "32m", "--cpus", "0.1", "--label", "io.knowledge-governance.freeze-source="+source, "--label", "io.knowledge-governance.environment=test", "--label", "io.knowledge-governance.freeze-role=writer", "--health-cmd", "test -f /tmp/ready", "--health-interval", "1s", "--health-timeout", "1s", "--health-retries", "3", image, "sh", "-c", "trap 'exit 0' TERM; touch /tmp/ready; while :; do sleep 1 & wait $!; done") + defer func() { + docker("rm", "-f", id) + if docker("ps", "-aq", "--filter", "label=io.knowledge-governance.freeze-source="+source) != "" { + t.Error("Docker fixture remains") + } + }() + controller, err := libraryfreeze.NewDockerController("/var/run/docker.sock", source, 10*time.Second) + if err != nil { + t.Fatal(err) + } + defer controller.Close() + if err = controller.Start(ctx, id); err != nil { + t.Fatal(err) + } + privateDir := func() string { + t.Helper() + p := t.TempDir() + if e := os.Chmod(p, 0700); e != nil { + t.Fatal(e) + } + return p + } + freeze, err := libraryfreeze.Open(privateDir(), source, controller) + if err != nil { + t.Fatal(err) + } + defer freeze.Close() + archive, err := librarysnapshot.Open(privateDir(), 1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + objectPath := filepath.Join(privateDir(), "source.txt") + if err = os.WriteFile(objectPath, []byte("resource"), 0600); err != nil { + t.Fatal(err) + } + objects := &captureFileReader{path: objectPath} + checks := 0 + ready := func(ctx context.Context) error { + items, e := controller.Inventory(ctx) + if e != nil { + return e + } + if len(items) != 1 { + return fmt.Errorf("unexpected fixture group") + } + checks++ + if items[0].Running != (checks%2 == 1) { + return fmt.Errorf("wrong writer state at readiness boundary") + } + return nil + } + req := LibraryCaptureRequest{JobID: "stack-capture", Scope: "internal", Identity: PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: mapping.digest}, CreatedAt: time.Now().UTC(), MaxDocuments: 100} + result, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req) + if err != nil || result.State != "succeeded" { + t.Fatalf("capture failed: %v state=%s", err, result.State) + } + assertHealthy := func() { + t.Helper() + items, e := controller.Inventory(ctx) + if e != nil || len(items) != 1 || !items[0].Running || !items[0].Healthy { + t.Fatal("fixture writer not restored healthy") + } + } + assertHealthy() + saved, err := archive.Load(result.SnapshotID) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, part := range saved.Parts { + r, e := archive.OpenPart(part) + if e != nil { + t.Fatal(e) + } + data, e := io.ReadAll(r) + r.Close() + if e != nil { + t.Fatal(e) + } + if strings.Contains(string(data), "private-session-marker") || strings.Contains(string(data), "other-scope") { + t.Fatal("unrelated data leaked") + } + switch { + case part.Kind == "index": + var docs []SearchDocument + if e = json.Unmarshal(data, &docs); e != nil { + t.Fatal(e) + } + if len(docs) != 1 || !strings.Contains(string(docs[0].Source), "Temporary context") { + t.Fatal("captured index lost") + } + seen["index"] = true + case part.Kind == "resource": + if string(data) != "resource" { + t.Fatal("resource mismatch") + } + seen["object"] = true + case part.Name == "wiki_pages": + if !strings.Contains(string(data), "old-content") || !strings.Contains(string(data), "published") { + t.Fatal("Wiki content/status lost") + } + seen["wiki"] = true + } + } + if len(seen) != 3 || saved.ValidationPolicy != librarysnapshot.ArtifactAcceptancePolicy { + t.Fatalf("incomplete captured evidence: %v", seen) + } + request("DELETE", "/"+index+"/_doc/chunk-safe?refresh=true", "", 200) + req.JobID = "stack-missing-index" + failed, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req) + if err == nil || failed.State == "succeeded" { + t.Fatal("missing real index accepted") + } + assertHealthy() + if err = os.Remove(objectPath); err != nil { + t.Fatal(err) + } + if _, err = db.Exec(ctx, `UPDATE knowledges SET parse_status='processing'; UPDATE wiki_pages SET content='later-content'`); err != nil { + t.Fatal(err) + } + req.JobID = "stack-capture" + beforeCalls, beforeChecks := objects.calls, checks + replay, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req) + if err != nil || replay.SnapshotID != result.SnapshotID || objects.calls != beforeCalls || checks != beforeChecks { + t.Fatal("sealed replay touched changed source") + } + assertHealthy() + if _, err = archive.Load(replay.SnapshotID); err != nil { + t.Fatal("archive depended on deleted source object") + } + t.Log("real PG + ES + Docker capture and failed-capture restoration passed; source object removed, PG changed, ES document deleted; immutable replay verified") +} + +type captureFileReader struct { + path string + calls int +} + +func (r *captureFileReader) Open(_ context.Context, provider, key string) (io.ReadCloser, error) { + r.calls++ + if provider != "s3" || key != "source.txt" { + return nil, fmt.Errorf("unexpected synthetic resource") + } + return os.Open(r.path) +} diff --git a/scripts/test-scope.json b/scripts/test-scope.json index 9ed9bc5..d2006bc 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -9,6 +9,11 @@ "⚠️ 文件缺失或解析不了 ⇒ 门禁红:范围声明不出来时,不许默认放行。" ], "optional": [ + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryCaptureIsolatedStack", + "reason": "显式隔离 PG、随机 ES 索引和无网络 Docker 夹具的联合捕获验收;默认 CI 不连接这些服务,本地另行运行并做索引覆盖变异反向验证;不代表真实 WeKnora 写入隔离或 Wiki 语义验收" + }, { "package": "knowledge-governance/internal/libraryfreeze", "test": "TestFreezeIsolatedDocker",