a55319d972
build / build (push) Failing after 2m29s
现象:共享桶部署下网关启动即报 gateway_auditor_key_required,残留过期 审计文件则每次 Execute 报 gateway_exclusive_writer_evidence_unverified; 用户已改为“停目标写入容器→更新→启动”的维护式更新。 根因:Gateway.Ready 强制 Isolation.Verify,main 强制 Ed25519 公钥,等于 把整桶独占证明当作发布前置;这与共享桶、沿用原 AK 的实际部署矛盾。 取舍:删除审计校验与相关 env 依赖,保留 journal、治理 PG 只读 authority、 plan/owner/attempt/epoch 绑定、intent-before-send、单次写与精确版本回滚。 create 仅在精确版本读回且仍为 current 时提交;发现外部 latest 只返回冲突, 不在 confirm 阶段删除(journal create intent 已提交,删除会变成无 delete intent/receipt 的未记账副作用并让 intent 永久 held)。同 key 不自动合并。 证据:真实二进制无审计/残留审计两环境通过;kg-pgtest-writeset 一次性库 + 随机 MinIO 桶跑 14 个网关用例(含共享桶共存写者、同 key 既有对象拒绝、 精确回滚)全过;umask 022 下 make verify 本地必需项全 success。
334 lines
11 KiB
Go
334 lines
11 KiB
Go
package objectgateway
|
|
|
|
import (
|
|
"context"
|
|
_ "embed"
|
|
"encoding/json"
|
|
"errors"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
// Schema is applied explicitly by the migration identity, never on server startup.
|
|
//
|
|
//go:embed schema.sql
|
|
var Schema string
|
|
|
|
type Gateway struct {
|
|
Authority PlanAuthority
|
|
Pool *pgxpool.Pool
|
|
Scope string
|
|
Backend Backend
|
|
MaxBytes int64
|
|
}
|
|
|
|
// Ready gates admission on the local journal and the object backend only. A
|
|
// shared bucket is the supported deployment shape: unrelated writers on other
|
|
// keys do not need to be drained, and no whole-bucket exclusivity evidence is
|
|
// required. Per-key competition is handled at execute time, not by a global
|
|
// "only writer" attestation.
|
|
func (g *Gateway) Ready(ctx context.Context) error {
|
|
if g.Pool == nil || g.Scope == "" || g.Backend == nil || g.MaxBytes < 1 {
|
|
return ErrPersistence
|
|
}
|
|
var sync, fsync, full string
|
|
if g.Pool.QueryRow(ctx, "SELECT current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes')").Scan(&sync, &fsync, &full) != nil || sync != "on" || fsync != "on" || full != "on" {
|
|
return ErrPersistence
|
|
}
|
|
var journal bool
|
|
if g.Pool.QueryRow(ctx, `SELECT NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off'
|
|
AND to_regclass('kg_object_gateway_plans') IS NOT NULL
|
|
AND to_regclass('kg_object_gateway_scopes') IS NOT NULL
|
|
AND to_regclass('kg_object_gateway_grants') IS NOT NULL
|
|
AND to_regclass('kg_object_gateway_intents') IS NOT NULL
|
|
AND to_regclass('kg_object_gateway_receipts') IS NOT NULL
|
|
AND to_regclass('kg_object_gateway_deliveries') IS NOT NULL
|
|
AND (SELECT count(*)=10 FROM pg_trigger WHERE tgname IN ('kg_object_gateway_deliveries_immutable','kg_object_gateway_deliveries_no_truncate','kg_object_gateway_plans_no_truncate','kg_object_gateway_intents_no_truncate','kg_object_gateway_receipts_no_truncate','kg_object_gateway_grants_no_truncate','kg_object_gateway_plans_immutable','kg_object_gateway_intents_immutable','kg_object_gateway_receipts_immutable','kg_object_gateway_grants_immutable') AND tgenabled='O')`).Scan(&journal) != nil || !journal {
|
|
return ErrPersistence
|
|
}
|
|
if g.Backend.Ready(ctx) != nil {
|
|
return errors.New("gateway_backend_not_ready")
|
|
}
|
|
return nil
|
|
}
|
|
func (g *Gateway) Advance(ctx context.Context, expected int64, next Grant) error {
|
|
if !next.valid() || next.Epoch != expected+1 {
|
|
return ErrEpoch
|
|
}
|
|
if err := g.Ready(ctx); err != nil {
|
|
return err
|
|
}
|
|
var frozen []byte
|
|
var plan Plan
|
|
if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, next.Plan).Scan(&frozen) != nil {
|
|
return ErrConflict
|
|
}
|
|
if json.Unmarshal(frozen, &plan) != nil || !plan.valid() || plan.Owner != next.Owner {
|
|
return ErrConflict
|
|
}
|
|
tx, err := g.Pool.Begin(ctx)
|
|
if err != nil {
|
|
return ErrPersistence
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); err != nil {
|
|
return ErrPersistence
|
|
}
|
|
var epoch int64
|
|
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
|
|
return ErrPersistence
|
|
}
|
|
if err = g.advanceLocked(ctx, tx, epoch, next); err != nil {
|
|
return err
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
return ErrPersistence
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (g *Gateway) advanceLocked(ctx context.Context, tx pgx.Tx, epoch int64, next Grant) error {
|
|
var err error
|
|
if epoch == next.Epoch {
|
|
var owner, plan string
|
|
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
|
return ErrPersistence
|
|
}
|
|
if owner != next.Owner || plan != next.Plan {
|
|
return ErrEpoch
|
|
}
|
|
return nil
|
|
}
|
|
if epoch != next.Epoch-1 {
|
|
return ErrEpoch
|
|
}
|
|
var pending bool
|
|
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts r WHERE r.scope=i.scope AND r.operation=i.operation))`, g.Scope).Scan(&pending) != nil {
|
|
return ErrPersistence
|
|
}
|
|
if pending {
|
|
return ErrHeld
|
|
}
|
|
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_grants(scope,epoch,owner,plan) VALUES($1,$2,$3,$4)", g.Scope, next.Epoch, next.Owner, next.Plan); err != nil {
|
|
return ErrPersistence
|
|
}
|
|
if _, err = tx.Exec(ctx, "UPDATE kg_object_gateway_scopes SET epoch=$2 WHERE scope=$1", g.Scope, next.Epoch); err != nil {
|
|
return ErrPersistence
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reserve commits an immutable may-send intent. Even a commit timeout cannot be
|
|
// followed by a send. Another process can reconcile, but cannot retry this PUT.
|
|
func (g *Gateway) reserve(ctx context.Context, r Request, authorization *Grant) (bool, error) {
|
|
tx, err := g.Pool.Begin(ctx)
|
|
if err != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
var epoch int64
|
|
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
|
|
return false, ErrEpoch
|
|
}
|
|
if authorization != nil {
|
|
var owner, plan string
|
|
if epoch != authorization.Epoch {
|
|
return false, ErrEpoch
|
|
}
|
|
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
if owner != authorization.Owner || plan != authorization.Plan {
|
|
return false, ErrEpoch
|
|
}
|
|
}
|
|
raw, _ := json.Marshal(r)
|
|
var previous []byte
|
|
err = tx.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&previous)
|
|
if err == nil {
|
|
if string(previous) != string(raw) {
|
|
return false, ErrConflict
|
|
}
|
|
return false, nil
|
|
}
|
|
if !errors.Is(err, pgx.ErrNoRows) {
|
|
return false, ErrPersistence
|
|
}
|
|
var owner, plan string
|
|
if epoch != r.Grant.Epoch {
|
|
return false, ErrEpoch
|
|
}
|
|
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
if owner != r.Grant.Owner || plan != r.Grant.Plan {
|
|
return false, ErrEpoch
|
|
}
|
|
var pending bool
|
|
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND i.key=$2 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts x WHERE x.scope=i.scope AND x.operation=i.operation))`, g.Scope, r.Key).Scan(&pending) != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
if pending {
|
|
return false, ErrHeld
|
|
}
|
|
var frozen []byte
|
|
var allowed Plan
|
|
if tx.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, r.Grant.Plan).Scan(&frozen) != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
if json.Unmarshal(frozen, &allowed) != nil || !allowed.valid() || !allowed.allows(r) {
|
|
return false, ErrConflict
|
|
}
|
|
if r.Action == "delete" {
|
|
previous, err := g.receipt(ctx, r.CreatedBy)
|
|
if err != nil || previous.State != "committed" || previous.Request.Key != r.Key || previous.Version != r.Version {
|
|
return false, ErrConflict
|
|
}
|
|
}
|
|
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_intents(scope,operation,key,request) VALUES($1,$2,$3,$4)", g.Scope, r.Operation, r.Key, raw); err != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
return false, ErrPersistence
|
|
}
|
|
return true, nil
|
|
}
|
|
func (g *Gateway) receipt(ctx context.Context, id string) (Receipt, error) {
|
|
var raw []byte
|
|
var r Receipt
|
|
err := g.Pool.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE scope=$1 AND operation=$2", g.Scope, id).Scan(&raw)
|
|
if err != nil {
|
|
return r, err
|
|
}
|
|
if json.Unmarshal(raw, &r) != nil {
|
|
return r, ErrPersistence
|
|
}
|
|
return r, nil
|
|
}
|
|
func (g *Gateway) finish(ctx context.Context, r Request, state string, v Version) (Receipt, error) {
|
|
receipt := Receipt{r, state, v}
|
|
raw, _ := json.Marshal(receipt)
|
|
_, err := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_receipts(scope,operation,receipt) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, r.Operation, raw)
|
|
if err != nil {
|
|
return Receipt{}, ErrPersistence
|
|
}
|
|
got, err := g.receipt(ctx, r.Operation)
|
|
if err != nil {
|
|
return Receipt{}, ErrPersistence
|
|
}
|
|
if got != receipt {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
return got, nil
|
|
}
|
|
func (g *Gateway) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) {
|
|
return g.execute(ctx, r, body, nil)
|
|
}
|
|
func (g *Gateway) execute(ctx context.Context, r Request, body []byte, authorization *Grant) (Receipt, error) {
|
|
if err := r.validate(); err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
if r.Bytes > g.MaxBytes || r.Action == "create" && (int64(len(body)) != r.Bytes || Digest(body) != r.SHA256) || r.Action == "delete" && len(body) != 0 {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
if err := g.Ready(ctx); err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
fresh, err := g.reserve(ctx, r, authorization)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
if !fresh {
|
|
return g.Reconcile(ctx, r)
|
|
}
|
|
if r.Action == "create" {
|
|
_, exists, err := g.Backend.Observe(ctx, r.Key, "")
|
|
if err != nil {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
if exists {
|
|
return g.finish(ctx, r, "rejected", Version{})
|
|
}
|
|
v, err := sendCreate(ctx, g.Backend, r, body)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
return g.finish(ctx, r, "committed", v)
|
|
}
|
|
v, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
|
|
if err != nil {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
if exists && v != r.Version {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
if exists {
|
|
if g.Backend.Delete(ctx, r.Key, r.Version) != nil {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
}
|
|
return g.Reconcile(ctx, r)
|
|
}
|
|
func (g *Gateway) Reconcile(ctx context.Context, r Request) (Receipt, error) {
|
|
if err := r.validate(); err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
if err := g.Ready(ctx); err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
var raw []byte
|
|
if g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&raw) != nil {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
want, _ := json.Marshal(r)
|
|
if string(raw) != string(want) {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
got, err := g.receipt(ctx, r.Operation)
|
|
if err == nil {
|
|
return got, nil
|
|
}
|
|
if !errors.Is(err, pgx.ErrNoRows) {
|
|
return Receipt{}, ErrPersistence
|
|
}
|
|
if r.Action == "delete" {
|
|
_, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
|
|
if err != nil || exists {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
return g.finish(ctx, r, "removed", r.Version)
|
|
}
|
|
versions, err := g.Backend.List(ctx, r.Key)
|
|
if err != nil {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
var found []Version
|
|
for _, v := range versions {
|
|
if v.Owner == r.token() {
|
|
if !r.matches(v) {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
found = append(found, v)
|
|
}
|
|
}
|
|
if len(found) != 1 {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
exact, exists, err := g.Backend.Observe(ctx, r.Key, found[0].ID)
|
|
if err != nil || !exists || exact != found[0] {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
latest, current, err := g.Backend.Observe(ctx, r.Key, "")
|
|
if err != nil || !current {
|
|
return Receipt{}, ErrHeld
|
|
}
|
|
if latest.ID != exact.ID {
|
|
// Another writer owns the current version of this key. Do not delete
|
|
// anything during reconcile and do not report a committed create for a
|
|
// key this plan no longer controls.
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
return g.finish(ctx, r, "committed", exact)
|
|
}
|