Files
knowledge-governance/internal/releaseadapter/v071/library_capture_integration_test.go
T
2026-10-28 08:22:21 +08:00

242 lines
8.8 KiB
Go

package v071
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarysnapshot"
)
// The writer is a real disposable Docker process, not a deployed WeKnora.
// This tests capture orchestration, not production readiness or Wiki semantics.
func TestLibraryCaptureIsolatedStack(t *testing.T) {
dsn, base := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
if dsn == "" || base == "" || os.Getenv("KG_FREEZE_DOCKER_ISOLATED") != "1" {
t.Skip("requires isolated PG, ES and disposable Docker opt-in")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Fatal("isolated database required")
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
db, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal("open isolated PG")
}
defer db.Close()
if _, err = db.Exec(ctx, libraryTestSchema); err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, `ALTER TABLE resources ADD COLUMN provider text DEFAULT 's3', ADD COLUMN size bigint DEFAULT 8;
ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published';
ALTER TABLE wiki_page_issues ADD COLUMN status text DEFAULT 'open';`); err != nil {
t.Fatal(err)
}
token := make([]byte, 8)
if _, err = rand.Read(token); err != nil {
t.Fatal(err)
}
source := "kg-freeze-test-" + hex.EncodeToString(token)
index := "kg_library_test_" + hex.EncodeToString(token)
client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
request := func(method, path, body string, want int) {
t.Helper()
r, e := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
if e != nil {
t.Fatal("fixture request")
}
r.Header.Set("Content-Type", "application/json")
resp, e := client.Do(r)
if e != nil {
t.Fatal("fixture HTTP failed")
}
defer resp.Body.Close()
if resp.StatusCode != want {
t.Fatalf("fixture HTTP %d, expected %d", resp.StatusCode, want)
}
}
request("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}`, 200)
defer func() { request("DELETE", "/"+index, "", 200); request("HEAD", "/"+index, "", 404) }()
body := `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"chunk-safe","source_id":"chunk-safe","content":"Temporary context\nchunk","embedding":[0.1,0.2]}`
request("PUT", "/"+index+"/_doc/chunk-safe?refresh=true", body, 201)
search, err := NewSearchExporter(SearchExporterConfig{BaseURL: base, Index: index, PageSize: 1, HTTPClient: client})
if err != nil {
t.Fatal(err)
}
mapping, err := search.inspectMapping(ctx)
if err != nil {
t.Fatal(err)
}
docker := func(args ...string) string {
t.Helper()
c, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
out, e := exec.CommandContext(c, "docker", append([]string{"--host", "unix:///var/run/docker.sock"}, args...)...).CombinedOutput()
if e != nil {
t.Fatalf("fixture Docker failed: %v", e)
}
return strings.TrimSpace(string(out))
}
image := docker("image", "inspect", "busybox:1.36", "--format", "{{.Id}}")
id := docker("create", "--name", source, "--network", "none", "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1m", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--memory", "32m", "--cpus", "0.1", "--label", "io.knowledge-governance.freeze-source="+source, "--label", "io.knowledge-governance.environment=test", "--label", "io.knowledge-governance.freeze-role=writer", "--health-cmd", "test -f /tmp/ready", "--health-interval", "1s", "--health-timeout", "1s", "--health-retries", "3", image, "sh", "-c", "trap 'exit 0' TERM; touch /tmp/ready; while :; do sleep 1 & wait $!; done")
defer func() {
docker("rm", "-f", id)
if docker("ps", "-aq", "--filter", "label=io.knowledge-governance.freeze-source="+source) != "" {
t.Error("Docker fixture remains")
}
}()
controller, err := libraryfreeze.NewDockerController("/var/run/docker.sock", source, 10*time.Second)
if err != nil {
t.Fatal(err)
}
defer controller.Close()
if err = controller.Start(ctx, id); err != nil {
t.Fatal(err)
}
privateDir := func() string {
t.Helper()
p := t.TempDir()
if e := os.Chmod(p, 0700); e != nil {
t.Fatal(e)
}
return p
}
freeze, err := libraryfreeze.Open(privateDir(), source, controller)
if err != nil {
t.Fatal(err)
}
defer freeze.Close()
archive, err := librarysnapshot.Open(privateDir(), 1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
objectPath := filepath.Join(privateDir(), "source.txt")
if err = os.WriteFile(objectPath, []byte("resource"), 0600); err != nil {
t.Fatal(err)
}
objects := &captureFileReader{path: objectPath}
checks := 0
ready := func(ctx context.Context) error {
items, e := controller.Inventory(ctx)
if e != nil {
return e
}
if len(items) != 1 {
return fmt.Errorf("unexpected fixture group")
}
checks++
if items[0].Running != (checks%2 == 1) {
return fmt.Errorf("wrong writer state at readiness boundary")
}
return nil
}
req := LibraryCaptureRequest{JobID: "stack-capture", Scope: "internal", Identity: PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: mapping.digest}, CreatedAt: time.Now().UTC(), MaxDocuments: 100}
result, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req)
if err != nil || result.State != "succeeded" {
t.Fatalf("capture failed: %v state=%s", err, result.State)
}
assertHealthy := func() {
t.Helper()
items, e := controller.Inventory(ctx)
if e != nil || len(items) != 1 || !items[0].Running || !items[0].Healthy {
t.Fatal("fixture writer not restored healthy")
}
}
assertHealthy()
saved, err := archive.Load(result.SnapshotID)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, part := range saved.Parts {
r, e := archive.OpenPart(part)
if e != nil {
t.Fatal(e)
}
data, e := io.ReadAll(r)
r.Close()
if e != nil {
t.Fatal(e)
}
if strings.Contains(string(data), "private-session-marker") || strings.Contains(string(data), "other-scope") {
t.Fatal("unrelated data leaked")
}
switch {
case part.Kind == "index":
var docs []SearchDocument
if e = json.Unmarshal(data, &docs); e != nil {
t.Fatal(e)
}
if len(docs) != 1 || !strings.Contains(string(docs[0].Source), "Temporary context") {
t.Fatal("captured index lost")
}
seen["index"] = true
case part.Kind == "resource":
if string(data) != "resource" {
t.Fatal("resource mismatch")
}
seen["object"] = true
case part.Name == "wiki_pages":
if !strings.Contains(string(data), "old-content") || !strings.Contains(string(data), "published") {
t.Fatal("Wiki content/status lost")
}
seen["wiki"] = true
}
}
if len(seen) != 3 || saved.ValidationPolicy != librarysnapshot.ArtifactAcceptancePolicy {
t.Fatalf("incomplete captured evidence: %v", seen)
}
request("DELETE", "/"+index+"/_doc/chunk-safe?refresh=true", "", 200)
req.JobID = "stack-missing-index"
failed, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req)
if err == nil || failed.State == "succeeded" {
t.Fatal("missing real index accepted")
}
assertHealthy()
if err = os.Remove(objectPath); err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, `UPDATE knowledges SET parse_status='processing'; UPDATE wiki_pages SET content='later-content'`); err != nil {
t.Fatal(err)
}
req.JobID = "stack-capture"
beforeCalls, beforeChecks := objects.calls, checks
replay, err := CaptureLibrary(ctx, freeze, archive, db, search, objects, ready, req)
if err != nil || replay.SnapshotID != result.SnapshotID || objects.calls != beforeCalls || checks != beforeChecks {
t.Fatal("sealed replay touched changed source")
}
assertHealthy()
if _, err = archive.Load(replay.SnapshotID); err != nil {
t.Fatal("archive depended on deleted source object")
}
t.Log("real PG + ES + Docker capture and failed-capture restoration passed; source object removed, PG changed, ES document deleted; immutable replay verified")
}
type captureFileReader struct {
path string
calls int
}
func (r *captureFileReader) Open(_ context.Context, provider, key string) (io.ReadCloser, error) {
r.calls++
if provider != "s3" || key != "source.txt" {
return nil, fmt.Errorf("unexpected synthetic resource")
}
return os.Open(r.path)
}