a55319d972
build / build (push) Failing after 2m29s
现象:共享桶部署下网关启动即报 gateway_auditor_key_required,残留过期 审计文件则每次 Execute 报 gateway_exclusive_writer_evidence_unverified; 用户已改为“停目标写入容器→更新→启动”的维护式更新。 根因:Gateway.Ready 强制 Isolation.Verify,main 强制 Ed25519 公钥,等于 把整桶独占证明当作发布前置;这与共享桶、沿用原 AK 的实际部署矛盾。 取舍:删除审计校验与相关 env 依赖,保留 journal、治理 PG 只读 authority、 plan/owner/attempt/epoch 绑定、intent-before-send、单次写与精确版本回滚。 create 仅在精确版本读回且仍为 current 时提交;发现外部 latest 只返回冲突, 不在 confirm 阶段删除(journal create intent 已提交,删除会变成无 delete intent/receipt 的未记账副作用并让 intent 永久 held)。同 key 不自动合并。 证据:真实二进制无审计/残留审计两环境通过;kg-pgtest-writeset 一次性库 + 随机 MinIO 桶跑 14 个网关用例(含共享桶共存写者、同 key 既有对象拒绝、 精确回滚)全过;umask 022 下 make verify 本地必需项全 success。
397 lines
14 KiB
Go
397 lines
14 KiB
Go
package objectgateway_test
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
g "knowledge-governance/internal/objectgateway"
|
|
)
|
|
|
|
func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) {
|
|
t.Helper()
|
|
dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL")
|
|
if dsn == "" {
|
|
t.Skip("isolated gateway PG/MinIO opt-in required")
|
|
}
|
|
u, e := url.Parse(dsn)
|
|
if e != nil || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "kg_library_test_gateway_") || u.Hostname() != "127.0.0.1" {
|
|
t.Fatal("unsafe gateway test database")
|
|
}
|
|
ctx := context.Background()
|
|
pool, e := pgxpool.New(ctx, dsn)
|
|
if e != nil {
|
|
t.Fatal("PG unavailable")
|
|
}
|
|
t.Cleanup(pool.Close)
|
|
if _, e = pool.Exec(ctx, g.Schema); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT")
|
|
if endpoint != "http://127.0.0.1:19100" {
|
|
t.Fatal("unsafe test MinIO endpoint")
|
|
}
|
|
bucket := "kg-library-test-gateway-" + g.Digest([]byte(t.Name() + time.Now().String()))[:16]
|
|
client := s3.New(s3.Options{Region: "us-east-1", BaseEndpoint: aws.String(endpoint), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "")})
|
|
if _, e = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); e != nil {
|
|
t.Fatal("create isolated bucket failed")
|
|
}
|
|
if _, e = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
t.Cleanup(func() {
|
|
list, e := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
|
|
if e != nil {
|
|
t.Error("residue list unavailable")
|
|
return
|
|
}
|
|
if len(list.Versions) != 0 || len(list.DeleteMarkers) != 0 {
|
|
t.Errorf("preserving residue bucket %s: versions=%d markers=%d", bucket, len(list.Versions), len(list.DeleteMarkers))
|
|
return
|
|
}
|
|
if _, e = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); e != nil {
|
|
t.Error("delete empty test bucket failed")
|
|
}
|
|
t.Logf("exact sandbox %s: versions=0 markers=0; empty bucket deleted", bucket)
|
|
})
|
|
return ctx, pool, &g.S3Backend{Client: client, Bucket: bucket, MaxBytes: 1 << 20}
|
|
}
|
|
func planFor(grant g.Grant, requests ...g.Request) g.Plan {
|
|
p := g.Plan{Digest: grant.Plan, Owner: grant.Owner}
|
|
for _, r := range requests {
|
|
p.Objects = append(p.Objects, g.ObjectBinding{Operation: r.Operation, Key: r.Key, SHA256: r.SHA256, Bytes: r.Bytes})
|
|
}
|
|
return p
|
|
}
|
|
func request(label string, grant g.Grant) g.Request {
|
|
body := []byte("gateway fixture " + label)
|
|
return g.Request{Grant: grant, Operation: g.Digest([]byte(label)), Key: "unchanged/" + label, Action: "create", SHA256: g.Digest(body), Bytes: int64(len(body))}
|
|
}
|
|
func cleanup(t *testing.T, ctx context.Context, gw *g.Gateway, receipt g.Receipt) {
|
|
t.Helper()
|
|
r := receipt.Request
|
|
r.Operation = g.Digest([]byte(r.Operation + "delete"))
|
|
r.Action = "delete"
|
|
r.CreatedBy = receipt.Request.Operation
|
|
r.Version = receipt.Version
|
|
if got, e := gw.Execute(ctx, r, nil); e != nil || got.State != "removed" {
|
|
t.Fatalf("exact cleanup: %v %s", e, got.State)
|
|
}
|
|
}
|
|
func TestGatewayReadinessRequiresJournal(t *testing.T) {
|
|
ctx, pool, backend := fixture(t)
|
|
// An isolated search_path lacks the journal while PG settings remain healthy.
|
|
conn, e := pool.Acquire(ctx)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
conn.Release()
|
|
cfg := pool.Config()
|
|
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_catalog"
|
|
other, e := pgxpool.NewWithConfig(ctx, cfg)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer other.Close()
|
|
gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, MaxBytes: 1 << 20}
|
|
if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
|
|
t.Fatalf("missing durable journal accepted: %v", e)
|
|
}
|
|
}
|
|
func TestGatewayRealReplayAndImmutableLedger(t *testing.T) {
|
|
ctx, pool, backend := fixture(t)
|
|
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
|
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
|
if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists {
|
|
t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists)
|
|
}
|
|
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e := gw.Advance(ctx, 0, grant); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
req := request(t.Name(), grant)
|
|
body := []byte("gateway fixture " + t.Name())
|
|
one, e := gw.Execute(ctx, req, body)
|
|
if e != nil || one.State != "committed" {
|
|
t.Fatalf("create %v %s", e, one.State)
|
|
}
|
|
two, e := gw.Execute(ctx, req, body)
|
|
if e != nil || two != one {
|
|
t.Fatal("replay changed receipt", e)
|
|
}
|
|
bad := req
|
|
bad.Key += "-different"
|
|
if _, e = gw.Execute(ctx, bad, body); !errors.Is(e, g.ErrConflict) {
|
|
t.Fatal("operation rebound", e)
|
|
}
|
|
for _, table := range []string{"intents", "receipts", "grants", "plans"} {
|
|
if _, e = pool.Exec(ctx, "DELETE FROM kg_object_gateway_"+table+" WHERE scope=$1", gw.Scope); e == nil {
|
|
t.Fatal("immutable evidence deleted", table)
|
|
}
|
|
}
|
|
for _, table := range []string{"plans", "intents", "receipts", "grants"} {
|
|
tx, e := pool.Begin(ctx)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
_, e = tx.Exec(ctx, "TRUNCATE kg_object_gateway_"+table+" CASCADE")
|
|
_ = tx.Rollback(ctx)
|
|
if e == nil {
|
|
cleanup(t, ctx, gw, one)
|
|
t.Fatal("immutable evidence truncation accepted", table)
|
|
}
|
|
}
|
|
cleanup(t, ctx, gw, one)
|
|
next := grant
|
|
next.Epoch = 2
|
|
if e = gw.Advance(ctx, 1, next); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
late := request("late", grant)
|
|
if _, e = gw.Execute(ctx, late, []byte("gateway fixture late")); !errors.Is(e, g.ErrEpoch) {
|
|
t.Fatal("old epoch sent", e)
|
|
}
|
|
}
|
|
|
|
func signedAudit(t *testing.T, scope string) (string, string) {
|
|
t.Helper()
|
|
pub, priv, e := ed25519.GenerateKey(rand.Reader)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
// This is a test authority. It exercises verification, NOT production exclusivity.
|
|
a := g.Attestation{Protocol: g.Protocol, Scope: scope, GatewayID: "isolated-fixture", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("fixture inventory")), PolicyEvidence: g.Digest([]byte("fixture policy")), DenialEvidence: g.Digest([]byte("fixture denial")), DrainEvidence: g.Digest([]byte("fixture drain"))}
|
|
raw, _ := json.Marshal(a)
|
|
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
|
|
path := filepath.Join(t.TempDir(), "audit.json")
|
|
if os.WriteFile(path, signed, 0600) != nil {
|
|
t.Fatal("audit write failed")
|
|
}
|
|
return path, base64.StdEncoding.EncodeToString(pub)
|
|
}
|
|
func startGateway(t *testing.T, env []string, client *g.Client) *exec.Cmd {
|
|
t.Helper()
|
|
binary := os.Getenv("KG_GATEWAY_TEST_BINARY")
|
|
if binary == "" {
|
|
t.Fatal("actual gateway binary is required")
|
|
}
|
|
cmd := exec.Command(binary)
|
|
cmd.Env = env
|
|
log, e := os.Create(filepath.Join(t.TempDir(), "service.log"))
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
cmd.Stdout = log
|
|
cmd.Stderr = log
|
|
if e = cmd.Start(); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); _ = log.Close() })
|
|
for i := 0; i < 100; i++ {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
e = client.Ready(ctx)
|
|
cancel()
|
|
if e == nil {
|
|
return cmd
|
|
}
|
|
time.Sleep(30 * time.Millisecond)
|
|
}
|
|
raw, _ := os.ReadFile(log.Name())
|
|
t.Fatalf("actual gateway not ready: %s", raw)
|
|
return nil
|
|
}
|
|
func TestGatewayActualServiceTwoInterruptions(t *testing.T) {
|
|
ctx, pool, backend := fixture(t)
|
|
upstream, _ := url.Parse(os.Getenv("KG_GATEWAY_TEST_ENDPOINT"))
|
|
proxy := httputil.NewSingleHostReverseProxy(upstream)
|
|
beforeSend := make(chan struct{}, 2)
|
|
allowSend := make(chan struct{}, 2)
|
|
committed := make(chan struct{}, 2)
|
|
release := make(chan struct{}, 2)
|
|
var puts atomic.Int32
|
|
proxy.ModifyResponse = func(resp *http.Response) error {
|
|
if resp.Request.Method == "PUT" && !strings.Contains(resp.Request.URL.RawQuery, "versioning") {
|
|
puts.Add(1)
|
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
committed <- struct{}{}
|
|
select {
|
|
case <-release:
|
|
case <-resp.Request.Context().Done():
|
|
}
|
|
return errors.New("injected lost response after actual MinIO commit")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) { http.Error(w, "injected_transport_loss", 502) }
|
|
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == "PUT" && !strings.Contains(r.URL.RawQuery, "versioning") {
|
|
beforeSend <- struct{}{}
|
|
select {
|
|
case <-allowSend:
|
|
case <-r.Context().Done():
|
|
return
|
|
}
|
|
}
|
|
proxy.ServeHTTP(w, r)
|
|
}))
|
|
defer p.Close()
|
|
scope := g.Digest([]byte("minio\x00" + p.URL + "\x00" + backend.Bucket))
|
|
audit, pub := signedAudit(t, scope)
|
|
listener, e := net.Listen("tcp", "127.0.0.1:0")
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
addr := listener.Addr().String()
|
|
listener.Close()
|
|
data := g.Digest([]byte("data " + scope))
|
|
control := g.Digest([]byte("control " + scope))
|
|
client, e := g.NewClient("http://"+addr, data, scope)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
admin, _ := g.NewClient("http://"+addr, control, scope)
|
|
env := append(os.Environ(), "OBJECT_GATEWAY_LISTEN="+addr, "OBJECT_GATEWAY_DATABASE_URL="+os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"), "OBJECT_GATEWAY_PROVIDER=minio", "OBJECT_GATEWAY_ENDPOINT="+p.URL, "OBJECT_GATEWAY_BUCKET="+backend.Bucket, "OBJECT_GATEWAY_ACCESS_KEY="+os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), "OBJECT_GATEWAY_SECRET_KEY="+os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY="+pub, "OBJECT_GATEWAY_AUDIT_FILE="+audit, "OBJECT_GATEWAY_ID=isolated-fixture", "OBJECT_GATEWAY_DATA_TOKEN="+data, "OBJECT_GATEWAY_CONTROL_TOKEN="+control)
|
|
cmd := startGateway(t, env, client)
|
|
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("actual-worker")), Plan: g.Digest([]byte("frozen-actual-plan"))}
|
|
if e = client.InstallPlan(ctx, planFor(grant, request("actual-0", grant))); e == nil {
|
|
t.Fatal("data client installed plan")
|
|
}
|
|
if e = admin.InstallPlan(ctx, planFor(grant, request("actual-0", grant), request("actual-1", grant))); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = admin.Advance(ctx, 0, grant); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = client.Advance(ctx, 0, grant); e == nil {
|
|
t.Fatal("data client acquired control authority")
|
|
}
|
|
for iteration := 0; iteration < 2; iteration++ {
|
|
label := fmt.Sprintf("actual-%d", iteration)
|
|
r := request(label, grant)
|
|
body := []byte("gateway fixture " + label)
|
|
done := make(chan error, 1)
|
|
go func() { _, e := client.Execute(ctx, r, body); done <- e }()
|
|
select {
|
|
case <-beforeSend:
|
|
case <-time.After(15 * time.Second):
|
|
t.Fatal("gateway did not send to proxy")
|
|
}
|
|
if _, e = client.Reconcile(ctx, r); !errors.Is(e, g.ErrHeld) {
|
|
t.Fatal("not-yet-visible PUT treated as terminal", e)
|
|
}
|
|
blockedNext := grant
|
|
blockedNext.Epoch++
|
|
if e = admin.Advance(ctx, grant.Epoch, blockedNext); !errors.Is(e, g.ErrHeld) {
|
|
t.Fatal("epoch crossed delayed precommit request", e)
|
|
}
|
|
allowSend <- struct{}{}
|
|
select {
|
|
case <-committed:
|
|
case <-time.After(15 * time.Second):
|
|
t.Fatal("actual MinIO PUT did not commit")
|
|
}
|
|
next := grant
|
|
next.Epoch++
|
|
if e = admin.Advance(ctx, grant.Epoch, next); !errors.Is(e, g.ErrHeld) {
|
|
t.Fatal("epoch crossed in-flight write", e)
|
|
}
|
|
sameKey := r
|
|
sameKey.Operation = g.Digest([]byte(label + "other"))
|
|
if _, e = client.Execute(ctx, sameKey, body); !errors.Is(e, g.ErrHeld) {
|
|
t.Fatal("same-key competitor admitted", e)
|
|
}
|
|
if e = cmd.Process.Kill(); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
_ = cmd.Wait()
|
|
release <- struct{}{}
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("dead gateway call did not end")
|
|
}
|
|
t.Logf("actual gateway SIGKILL %d after MinIO commit before receipt", iteration+1)
|
|
cmd = startGateway(t, env, client)
|
|
receipt, e := client.Reconcile(ctx, r)
|
|
if e != nil || receipt.State != "committed" {
|
|
t.Fatalf("restart reconcile %v %s", e, receipt.State)
|
|
}
|
|
repeated, e := client.Execute(ctx, r, body)
|
|
if e != nil || receipt != repeated {
|
|
t.Fatal("restart duplicate not idempotent", e)
|
|
}
|
|
del := r
|
|
del.Action = "delete"
|
|
del.CreatedBy = r.Operation
|
|
del.Operation = g.Digest([]byte(label + "cleanup"))
|
|
del.Version = receipt.Version
|
|
cleaned, e := client.Execute(ctx, del, nil)
|
|
if e != nil || cleaned.State != "removed" {
|
|
t.Fatal("actual exact cleanup failed", e)
|
|
}
|
|
if e = admin.Advance(ctx, grant.Epoch, next); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
late := request(label+"late", grant)
|
|
if _, e = client.Execute(ctx, late, []byte("gateway fixture "+label+"late")); !errors.Is(e, g.ErrEpoch) {
|
|
t.Fatal("late stale epoch admitted", e)
|
|
}
|
|
grant = next
|
|
}
|
|
if puts.Load() != 2 {
|
|
t.Fatalf("blind PUT replay count=%d", puts.Load())
|
|
}
|
|
var n int
|
|
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", scope).Scan(&n) != nil || n != 4 {
|
|
t.Fatalf("durable receipts=%d", n)
|
|
}
|
|
_ = cmd.Process.Kill()
|
|
_ = cmd.Wait()
|
|
if e = client.Ready(ctx); e == nil {
|
|
t.Fatal("unavailable gateway accepted")
|
|
}
|
|
t.Log("actual binary + HTTP + durable PG + MinIO: two kills/restarts, 2 PUTs, 4 receipts, exact cleanup")
|
|
}
|
|
|
|
func TestGatewayClientRejectsBadProtocolAndLeaks(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Query().Get("secret") != "" {
|
|
t.Error("secret in URL")
|
|
}
|
|
_, _ = io.WriteString(w, `{"protocol":"other","scope":"scope","ready":true}`)
|
|
}))
|
|
defer server.Close()
|
|
c, e := g.NewClient(server.URL, strings.Repeat("x", 32), "scope")
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if c.Ready(context.Background()) == nil {
|
|
t.Fatal("wrong protocol accepted")
|
|
}
|
|
}
|