package objectgateway_test import ( "context" "crypto/ed25519" "crypto/rand" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/http/httptest" "net/http/httputil" "net/url" "os" "os/exec" "path/filepath" "strings" "sync/atomic" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/jackc/pgx/v5/pgxpool" g "knowledge-governance/internal/objectgateway" ) func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) { t.Helper() dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL") if dsn == "" { t.Skip("isolated gateway PG/MinIO opt-in required") } u, e := url.Parse(dsn) if e != nil || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "kg_library_test_gateway_") || u.Hostname() != "127.0.0.1" { t.Fatal("unsafe gateway test database") } ctx := context.Background() pool, e := pgxpool.New(ctx, dsn) if e != nil { t.Fatal("PG unavailable") } t.Cleanup(pool.Close) if _, e = pool.Exec(ctx, g.Schema); e != nil { t.Fatal(e) } endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT") if endpoint != "http://127.0.0.1:19100" { t.Fatal("unsafe test MinIO endpoint") } bucket := "kg-library-test-gateway-" + g.Digest([]byte(t.Name() + time.Now().String()))[:16] client := s3.New(s3.Options{Region: "us-east-1", BaseEndpoint: aws.String(endpoint), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "")}) if _, e = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); e != nil { t.Fatal("create isolated bucket failed") } if _, e = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}}); e != nil { t.Fatal(e) } t.Cleanup(func() { list, e := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)}) if e != nil { t.Error("residue list unavailable") return } if len(list.Versions) != 0 || len(list.DeleteMarkers) != 0 { t.Errorf("preserving residue bucket %s: versions=%d markers=%d", bucket, len(list.Versions), len(list.DeleteMarkers)) return } if _, e = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); e != nil { t.Error("delete empty test bucket failed") } t.Logf("exact sandbox %s: versions=0 markers=0; empty bucket deleted", bucket) }) return ctx, pool, &g.S3Backend{Client: client, Bucket: bucket, MaxBytes: 1 << 20} } func planFor(grant g.Grant, requests ...g.Request) g.Plan { p := g.Plan{Digest: grant.Plan, Owner: grant.Owner} for _, r := range requests { p.Objects = append(p.Objects, g.ObjectBinding{Operation: r.Operation, Key: r.Key, SHA256: r.SHA256, Bytes: r.Bytes}) } return p } func request(label string, grant g.Grant) g.Request { body := []byte("gateway fixture " + label) return g.Request{Grant: grant, Operation: g.Digest([]byte(label)), Key: "unchanged/" + label, Action: "create", SHA256: g.Digest(body), Bytes: int64(len(body))} } func cleanup(t *testing.T, ctx context.Context, gw *g.Gateway, receipt g.Receipt) { t.Helper() r := receipt.Request r.Operation = g.Digest([]byte(r.Operation + "delete")) r.Action = "delete" r.CreatedBy = receipt.Request.Operation r.Version = receipt.Version if got, e := gw.Execute(ctx, r, nil); e != nil || got.State != "removed" { t.Fatalf("exact cleanup: %v %s", e, got.State) } } func TestGatewayReadinessRequiresJournal(t *testing.T) { ctx, pool, backend := fixture(t) // An isolated search_path lacks the journal while PG settings remain healthy. conn, e := pool.Acquire(ctx) if e != nil { t.Fatal(e) } conn.Release() cfg := pool.Config() cfg.ConnConfig.RuntimeParams["search_path"] = "pg_catalog" other, e := pgxpool.NewWithConfig(ctx, cfg) if e != nil { t.Fatal(e) } defer other.Close() gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, MaxBytes: 1 << 20} if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) { t.Fatalf("missing durable journal accepted: %v", e) } } func TestGatewayRealReplayAndImmutableLedger(t *testing.T) { ctx, pool, backend := fixture(t) grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))} gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20} if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists { t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists) } if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil { t.Fatal(e) } if e := gw.Advance(ctx, 0, grant); e != nil { t.Fatal(e) } req := request(t.Name(), grant) body := []byte("gateway fixture " + t.Name()) one, e := gw.Execute(ctx, req, body) if e != nil || one.State != "committed" { t.Fatalf("create %v %s", e, one.State) } two, e := gw.Execute(ctx, req, body) if e != nil || two != one { t.Fatal("replay changed receipt", e) } bad := req bad.Key += "-different" if _, e = gw.Execute(ctx, bad, body); !errors.Is(e, g.ErrConflict) { t.Fatal("operation rebound", e) } for _, table := range []string{"intents", "receipts", "grants", "plans"} { if _, e = pool.Exec(ctx, "DELETE FROM kg_object_gateway_"+table+" WHERE scope=$1", gw.Scope); e == nil { t.Fatal("immutable evidence deleted", table) } } for _, table := range []string{"plans", "intents", "receipts", "grants"} { tx, e := pool.Begin(ctx) if e != nil { t.Fatal(e) } _, e = tx.Exec(ctx, "TRUNCATE kg_object_gateway_"+table+" CASCADE") _ = tx.Rollback(ctx) if e == nil { cleanup(t, ctx, gw, one) t.Fatal("immutable evidence truncation accepted", table) } } cleanup(t, ctx, gw, one) next := grant next.Epoch = 2 if e = gw.Advance(ctx, 1, next); e != nil { t.Fatal(e) } late := request("late", grant) if _, e = gw.Execute(ctx, late, []byte("gateway fixture late")); !errors.Is(e, g.ErrEpoch) { t.Fatal("old epoch sent", e) } } func signedAudit(t *testing.T, scope string) (string, string) { t.Helper() pub, priv, e := ed25519.GenerateKey(rand.Reader) if e != nil { t.Fatal(e) } // This is a test authority. It exercises verification, NOT production exclusivity. a := g.Attestation{Protocol: g.Protocol, Scope: scope, GatewayID: "isolated-fixture", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("fixture inventory")), PolicyEvidence: g.Digest([]byte("fixture policy")), DenialEvidence: g.Digest([]byte("fixture denial")), DrainEvidence: g.Digest([]byte("fixture drain"))} raw, _ := json.Marshal(a) signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)}) path := filepath.Join(t.TempDir(), "audit.json") if os.WriteFile(path, signed, 0600) != nil { t.Fatal("audit write failed") } return path, base64.StdEncoding.EncodeToString(pub) } func startGateway(t *testing.T, env []string, client *g.Client) *exec.Cmd { t.Helper() binary := os.Getenv("KG_GATEWAY_TEST_BINARY") if binary == "" { t.Fatal("actual gateway binary is required") } cmd := exec.Command(binary) cmd.Env = env log, e := os.Create(filepath.Join(t.TempDir(), "service.log")) if e != nil { t.Fatal(e) } cmd.Stdout = log cmd.Stderr = log if e = cmd.Start(); e != nil { t.Fatal(e) } t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); _ = log.Close() }) for i := 0; i < 100; i++ { ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) e = client.Ready(ctx) cancel() if e == nil { return cmd } time.Sleep(30 * time.Millisecond) } raw, _ := os.ReadFile(log.Name()) t.Fatalf("actual gateway not ready: %s", raw) return nil } func TestGatewayActualServiceTwoInterruptions(t *testing.T) { ctx, pool, backend := fixture(t) upstream, _ := url.Parse(os.Getenv("KG_GATEWAY_TEST_ENDPOINT")) proxy := httputil.NewSingleHostReverseProxy(upstream) beforeSend := make(chan struct{}, 2) allowSend := make(chan struct{}, 2) committed := make(chan struct{}, 2) release := make(chan struct{}, 2) var puts atomic.Int32 proxy.ModifyResponse = func(resp *http.Response) error { if resp.Request.Method == "PUT" && !strings.Contains(resp.Request.URL.RawQuery, "versioning") { puts.Add(1) if resp.StatusCode >= 200 && resp.StatusCode < 300 { committed <- struct{}{} select { case <-release: case <-resp.Request.Context().Done(): } return errors.New("injected lost response after actual MinIO commit") } } return nil } proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) { http.Error(w, "injected_transport_loss", 502) } p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == "PUT" && !strings.Contains(r.URL.RawQuery, "versioning") { beforeSend <- struct{}{} select { case <-allowSend: case <-r.Context().Done(): return } } proxy.ServeHTTP(w, r) })) defer p.Close() scope := g.Digest([]byte("minio\x00" + p.URL + "\x00" + backend.Bucket)) audit, pub := signedAudit(t, scope) listener, e := net.Listen("tcp", "127.0.0.1:0") if e != nil { t.Fatal(e) } addr := listener.Addr().String() listener.Close() data := g.Digest([]byte("data " + scope)) control := g.Digest([]byte("control " + scope)) client, e := g.NewClient("http://"+addr, data, scope) if e != nil { t.Fatal(e) } admin, _ := g.NewClient("http://"+addr, control, scope) env := append(os.Environ(), "OBJECT_GATEWAY_LISTEN="+addr, "OBJECT_GATEWAY_DATABASE_URL="+os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"), "OBJECT_GATEWAY_PROVIDER=minio", "OBJECT_GATEWAY_ENDPOINT="+p.URL, "OBJECT_GATEWAY_BUCKET="+backend.Bucket, "OBJECT_GATEWAY_ACCESS_KEY="+os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), "OBJECT_GATEWAY_SECRET_KEY="+os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY="+pub, "OBJECT_GATEWAY_AUDIT_FILE="+audit, "OBJECT_GATEWAY_ID=isolated-fixture", "OBJECT_GATEWAY_DATA_TOKEN="+data, "OBJECT_GATEWAY_CONTROL_TOKEN="+control) cmd := startGateway(t, env, client) grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("actual-worker")), Plan: g.Digest([]byte("frozen-actual-plan"))} if e = client.InstallPlan(ctx, planFor(grant, request("actual-0", grant))); e == nil { t.Fatal("data client installed plan") } if e = admin.InstallPlan(ctx, planFor(grant, request("actual-0", grant), request("actual-1", grant))); e != nil { t.Fatal(e) } if e = admin.Advance(ctx, 0, grant); e != nil { t.Fatal(e) } if e = client.Advance(ctx, 0, grant); e == nil { t.Fatal("data client acquired control authority") } for iteration := 0; iteration < 2; iteration++ { label := fmt.Sprintf("actual-%d", iteration) r := request(label, grant) body := []byte("gateway fixture " + label) done := make(chan error, 1) go func() { _, e := client.Execute(ctx, r, body); done <- e }() select { case <-beforeSend: case <-time.After(15 * time.Second): t.Fatal("gateway did not send to proxy") } if _, e = client.Reconcile(ctx, r); !errors.Is(e, g.ErrHeld) { t.Fatal("not-yet-visible PUT treated as terminal", e) } blockedNext := grant blockedNext.Epoch++ if e = admin.Advance(ctx, grant.Epoch, blockedNext); !errors.Is(e, g.ErrHeld) { t.Fatal("epoch crossed delayed precommit request", e) } allowSend <- struct{}{} select { case <-committed: case <-time.After(15 * time.Second): t.Fatal("actual MinIO PUT did not commit") } next := grant next.Epoch++ if e = admin.Advance(ctx, grant.Epoch, next); !errors.Is(e, g.ErrHeld) { t.Fatal("epoch crossed in-flight write", e) } sameKey := r sameKey.Operation = g.Digest([]byte(label + "other")) if _, e = client.Execute(ctx, sameKey, body); !errors.Is(e, g.ErrHeld) { t.Fatal("same-key competitor admitted", e) } if e = cmd.Process.Kill(); e != nil { t.Fatal(e) } _ = cmd.Wait() release <- struct{}{} select { case <-done: case <-time.After(5 * time.Second): t.Fatal("dead gateway call did not end") } t.Logf("actual gateway SIGKILL %d after MinIO commit before receipt", iteration+1) cmd = startGateway(t, env, client) receipt, e := client.Reconcile(ctx, r) if e != nil || receipt.State != "committed" { t.Fatalf("restart reconcile %v %s", e, receipt.State) } repeated, e := client.Execute(ctx, r, body) if e != nil || receipt != repeated { t.Fatal("restart duplicate not idempotent", e) } del := r del.Action = "delete" del.CreatedBy = r.Operation del.Operation = g.Digest([]byte(label + "cleanup")) del.Version = receipt.Version cleaned, e := client.Execute(ctx, del, nil) if e != nil || cleaned.State != "removed" { t.Fatal("actual exact cleanup failed", e) } if e = admin.Advance(ctx, grant.Epoch, next); e != nil { t.Fatal(e) } late := request(label+"late", grant) if _, e = client.Execute(ctx, late, []byte("gateway fixture "+label+"late")); !errors.Is(e, g.ErrEpoch) { t.Fatal("late stale epoch admitted", e) } grant = next } if puts.Load() != 2 { t.Fatalf("blind PUT replay count=%d", puts.Load()) } var n int if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", scope).Scan(&n) != nil || n != 4 { t.Fatalf("durable receipts=%d", n) } _ = cmd.Process.Kill() _ = cmd.Wait() if e = client.Ready(ctx); e == nil { t.Fatal("unavailable gateway accepted") } t.Log("actual binary + HTTP + durable PG + MinIO: two kills/restarts, 2 PUTs, 4 receipts, exact cleanup") } func TestGatewayClientRejectsBadProtocolAndLeaks(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("secret") != "" { t.Error("secret in URL") } _, _ = io.WriteString(w, `{"protocol":"other","scope":"scope","ready":true}`) })) defer server.Close() c, e := g.NewClient(server.URL, strings.Repeat("x", 32), "scope") if e != nil { t.Fatal(e) } if c.Ready(context.Background()) == nil { t.Fatal("wrong protocol accepted") } }