diff --git a/cmd/object-gateway-control/main.go b/cmd/object-gateway-control/main.go new file mode 100644 index 0000000..e56c152 --- /dev/null +++ b/cmd/object-gateway-control/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "time" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +// This command belongs to the separately authenticated controller. It must not +// be installed with control credentials in an ordinary data worker container. +func main() { + if e := run(); e != nil { + fmt.Fprintln(os.Stderr, e) + os.Exit(1) + } +} +func run() error { + path := flag.String("plan", "", "exact frozen write-set JSON") + directory := flag.String("grant-directory", "", "private durable grant output directory") + expected := flag.Int64("expected-epoch", -1, "previous epoch from the frozen controller intent; never guessed") + flag.Parse() + if *path == "" || *directory == "" || *expected < 0 || flag.NArg() != 0 { + return errors.New("exact plan, grant directory and expected epoch required") + } + f, e := os.Open(*path) + if e != nil { + return errors.New("frozen gateway plan unavailable") + } + defer f.Close() + var set libraryrelease.WriteSet + d := json.NewDecoder(io.LimitReader(f, 16<<20)) + d.DisallowUnknownFields() + if d.Decode(&set) != nil || d.Decode(&struct{}{}) != io.EOF { + return errors.New("frozen gateway plan invalid") + } + provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET") + if provider == "" || endpoint == "" || bucket == "" { + return errors.New("exact gateway target configuration required") + } + scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket)) + client, e := objectgateway.NewClient(os.Getenv("OBJECT_GATEWAY_URL"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN"), scope) + if e != nil { + return e + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if e = v071.PrepareObjectGatewayGrant(ctx, client, set, *expected, provider, bucket, *directory); e != nil { + return e + } + fmt.Printf("gateway grant persisted: plan=%s epoch=%d\n", set.Digest, *expected+1) + return nil +} diff --git a/cmd/object-gateway/main.go b/cmd/object-gateway/main.go new file mode 100644 index 0000000..c10b5d2 --- /dev/null +++ b/cmd/object-gateway/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "errors" + "log" + "net" + "net/http" + "os" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/objectgateway" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +func main() { + if err := run(); err != nil { + log.Print(err) + os.Exit(1) + } +} +func run() error { + ctx := context.Background() + addr := os.Getenv("OBJECT_GATEWAY_LISTEN") + host, _, e := net.SplitHostPort(addr) + // A reviewed TLS terminator may expose this loopback-only listener. Never bind + // authenticated plaintext to a LAN address by an accidental config default. + if e != nil || (host != "127.0.0.1" && host != "::1") { + return errors.New("gateway_requires_loopback_listener") + } + data, control := os.Getenv("OBJECT_GATEWAY_DATA_TOKEN"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN") + if len(data) < 32 || len(control) < 32 || data == control { + return errors.New("gateway_distinct_tokens_required") + } + pub, e := base64.StdEncoding.DecodeString(os.Getenv("OBJECT_GATEWAY_AUDIT_PUBLIC_KEY")) + if e != nil || len(pub) != ed25519.PublicKeySize { + return errors.New("gateway_auditor_key_required") + } + provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET") + scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket)) + pool, e := pgxpool.New(ctx, os.Getenv("OBJECT_GATEWAY_DATABASE_URL")) + if e != nil { + return objectgateway.ErrPersistence + } + defer pool.Close() + backend, e := v071.NewGatewayBackend(v071.S3ObjectStoreConfig{Provider: provider, Endpoint: endpoint, Bucket: bucket, Region: os.Getenv("OBJECT_GATEWAY_REGION"), AccessKeyID: os.Getenv("OBJECT_GATEWAY_ACCESS_KEY"), SecretAccessKey: os.Getenv("OBJECT_GATEWAY_SECRET_KEY"), MaxObjectBytes: 256 * 1024 * 1024}) + if e != nil { + return errors.New("gateway_backend_configuration_invalid") + } + g := &objectgateway.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 256 * 1024 * 1024, Isolation: objectgateway.AuditVerifier{Path: os.Getenv("OBJECT_GATEWAY_AUDIT_FILE"), PublicKey: pub, Scope: scope, GatewayID: os.Getenv("OBJECT_GATEWAY_ID")}} + if governanceURL := os.Getenv("OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL"); governanceURL != "" { + cfg, e := pgxpool.ParseConfig(governanceURL) + if e != nil { + return errors.New("gateway_control_authority_unavailable") + } + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + cfg.MaxConns = 4 + governancePool, e := pgxpool.NewWithConfig(ctx, cfg) + if e != nil { + return errors.New("gateway_control_authority_unavailable") + } + defer governancePool.Close() + authority := &v071.GatewayGovernanceAuthority{Pool: governancePool, ArchiveRoot: os.Getenv("OBJECT_GATEWAY_ARCHIVE_ROOT"), TargetID: os.Getenv("OBJECT_GATEWAY_TARGET_ID"), Provider: provider, Bucket: bucket} + if e = authority.Ready(ctx); e != nil { + return e + } + g.Authority = authority + } + probe, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if e = g.Ready(probe); e != nil { + return e + } + server := &http.Server{Addr: addr, Handler: objectgateway.HTTPServer{Gateway: g, DataToken: data, ControlToken: control}, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 3 * time.Minute, WriteTimeout: 3 * time.Minute, IdleTimeout: 30 * time.Second} + if server.ListenAndServe() != nil { + return errors.New("gateway_listener_stopped") + } + return nil +} diff --git a/cmd/release-worker/legacy_gate_test.go b/cmd/release-worker/legacy_gate_test.go new file mode 100644 index 0000000..8cd89c8 --- /dev/null +++ b/cmd/release-worker/legacy_gate_test.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + v071 "knowledge-governance/internal/releaseadapter/v071" + "testing" +) + +func TestLegacyReleaseRuntimeRefusesBeforeCredentials(t *testing.T) { + // Empty credential paths would fail differently if route assembly were reached. + runtime, e := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{}) + if runtime != nil || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() { + t.Fatal("legacy runtime did not fail at the no-write gate", e) + } + route, pools, e := buildReleaseRoute(context.Background(), "", releaseTargetSnapshotConfig{}, releaseRouteConfig{}) + if route != nil || len(pools) != 0 || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() { + t.Fatal("direct route bypassed the no-write gate", e) + } +} diff --git a/docs/object-write-gateway.md b/docs/object-write-gateway.md new file mode 100644 index 0000000..0c5eb41 --- /dev/null +++ b/docs/object-write-gateway.md @@ -0,0 +1,93 @@ +# 对象写入网关(开发候选,尚不可部署业务) + +## 协议与持久化 + +`cmd/object-gateway` 提供 `kg-object-gateway-v1`,只监听 loopback;外部接入需要另行审核的 TLS 终结与网络隔离。治理 PG 保存不可变计划、grant、intent、逐版本 receipt 和 attempt→grant delivery。部署迁移身份显式应用 `internal/objectgateway/schema.sql`;启动不自动迁移。运行身份只授必要 SELECT/INSERT 和 scope 行 UPDATE,不授 DDL/TRUNCATE/证据修改或业务库写权限。 + +scope 是 provider、原 endpoint、bucket 的 NUL 分隔 SHA256。同一物理桶只能有一个受审核 scope 和一套共享 journal,不能用 endpoint 别名或另一日志库建立平行写域。 + +数据认证端点:`POST /v1/acquire`、`/v1/resolve`、`/v1/execute`、`/v1/reconcile`、`/v1/observe`,以及 `GET /v1/capabilities`。所有请求核验协议和 scope。`capabilities.automatic_grants` 必须为真,正式 stack 才接受自动授权能力;未接控制源、只读角色权限异常、协议不符或不可达均拒绝,无直写回退。 + +PG 行锁负责原子准入,不是远端存储 fence。intent 必须持久提交后才发送一次请求;连接死亡、租约过期不清除未知 intent。任何未终态 intent 均阻止 scope 换 epoch。重复 operation 只读回执或精确对账,不重发 PUT;暂时查不到版本仍 held。删除必须绑定已确认 create 的精确 VersionID,不删 latest、不生成删除标记。 + +不可变证据表拒绝 UPDATE/DELETE/TRUNCATE。Ready 核 fsync、synchronous_commit、full_page_writes、主库可写及必要表/触发器;管理员与迁移身份仍是可信边界,触发器不能抵御 DBA 的 DDL。 + +## 正式控制交付路径 + +生产代码路径已接入,并已在专属本机隔离栈运行正式 worker 的两次 SIGKILL 接管;这不是生产部署验收: + +`cmd/worker.runOwnedLibraryAttempt → attempt context → librarystack → GatewayObjectStore(Automatic) → ProductionLibraryTarget.Apply → EnsureObjectGrant → /v1/acquire → GatewayGovernanceAuthority → 原持久 WriteSet → journal delivery → 数据 execute`。 + +冻结/采用计划时只保存自校验后的 WriteSet,不提前索取 grant;observer 先把摘要锚定到原预备份事件,首笔业务 PG Apply 前才请求授权。worker 只提交 job/operation/attempt epoch/owner/plan digest,不提交可替换的 key/body allowlist,也不能选择网关 epoch。 + +可信控制域使用独立 SELECT-only 治理连接,核对目标身份、job/operation、当前 attempt、owner、有效租约、running 状态和原预备份事件中的摘要、job、backup、目标快照;再从归档加载并验 hash、范围、provider/bucket。客户端自述不能替代这些持久事实。控制读角色不得是 superuser、CREATEDB、CREATEROLE,不能写/截断/建触发器到这些治理表。每次加载和能力查询重验角色。 + +网关在 scope 锁内再次核对授权事实,原子写入 grant、当前 epoch、不可变 delivery。同一 attempt 重试返回原 grant;owner/operation/plan 替换拒绝;同一 job 的后续 attempt 必须沿用原 operation 和 plan。旧发送未终态时,新 attempt 可先只读对账,但无充分证据仍 held,不能交接。 + +控制服务配置了治理 authority 后,旧 `/v1/plans`、`/v1/advance` 手动控制端点关闭,控制 token 也不能绕过原计划绑定。旧 `cmd/object-gateway-control` 仅保留离线/协议兼容路径,不是正式 worker 的交付方案。 + +## 独立恢复请求 + +`library.recover` 的队列ID由原窗口job确定;它具有独立恢复operation与request digest,绑定scope、queue、原window/operation、target、source、backup和原WriteSet。原计划的job、operation和摘要不改写。Claim中的`recovery_job_id`与`recovery_operation_id`仅为独立授权上下文,原`job_id`/`operation_id`仍是发送域。网关重新读取持久队列payload、running状态、当前owner/attempt/有效租约与恢复operation,不信任worker自行声明。可选显式来源/目标/backup/plan字段只能收紧原绑定,不能覆盖它。 + +SELECT-only治理角色需要读取`library_jobs`、`library_operations`、`library_attempts`、`library_versions`、`jobs`,均不能INSERT/UPDATE/DELETE/TRUNCATE/TRIGGER。已有原窗口恢复请求时不自动重启原发布;过期running恢复队列只在相同target、未释放维护窗口、原业务租约也过期时重新投递,业务attempt/epoch仍重新申请。 + +## 索引unknown恢复边界 + +原冻结before-image非空时,恢复先持久化`restore-mark-intent`,再以原CAS(或确认缺失的create)写入包含原计划/index/id归属标记的完整before-image。丢响应后仅匹配完整标记正文及有效CAS才能登记`restore-claimed`并CAS去标记;已知回执的代次不一致拒绝。恢复完成的`restored`回执仍须匹配正文和CAS,不能接受同正文的新代次。 + +去标记前持久化`restore-finalize-intent`。发布clean-finalization或恢复clean-finalization已提交但响应丢失时,当前干净正文并非充分归属证据;继续held,不盲删、不重发、不把相同外观当回执。旧unknown缺少写前标记intent亦不自动升级认领。本候选没有服务端可验证的clean提交关联凭证,因此这部分自动收敛仍是明确缺口,不是可部署完整恢复保证。 + +## 跨 epoch 的不可变发送身份 + +`Request.Grant` 继续代表首次发送的不可变身份,后端 claim、intent 和 receipt 都不改。新 `ExecuteRequest.authorization` 仅代表本次准入权限。`/v1/resolve` 根据相同 operation/key/action/hash/length/owner/plan 取回旧请求;只允许 epoch 差异,不能换内容。 + +新 attempt 获得新 grant 后,已发送 key 沿用旧 request/claim,未发送 key 使用新授权 epoch 建立首次 intent。当前授权必须匹配 journal 的当前 epoch;旧 epoch 的新写拒绝。新 epoch 补偿删除仍使用原 create 回执和精确版本,删除本身有独立不可变发送身份。 + +## 配置与凭据归属 + +网关进程环境: + +- `OBJECT_GATEWAY_LISTEN`:仅 `127.0.0.1:` 或 `[::1]:`。 +- `OBJECT_GATEWAY_DATABASE_URL`:独立 journal 连接,迁移与运行角色分离。 +- `OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL`:仅可信网关持有的 SELECT-only 治理读连接;独立于 journal 运行身份和普通 worker 身份。 +- `OBJECT_GATEWAY_ARCHIVE_ROOT`:原持久计划所在归档绝对路径,部署只读挂载。 +- `OBJECT_GATEWAY_TARGET_ID`:与正式 worker `LibraryTargetIdentity()` 完全相同的逻辑目标身份;不能用另一个环境的值。 +- `OBJECT_GATEWAY_PROVIDER`、`OBJECT_GATEWAY_ENDPOINT`、`OBJECT_GATEWAY_BUCKET`、`OBJECT_GATEWAY_REGION`:精确存储身份。 +- `OBJECT_GATEWAY_ACCESS_KEY`、`OBJECT_GATEWAY_SECRET_KEY`:仅网关持有的对象写凭据。 +- `OBJECT_GATEWAY_DATA_TOKEN`、`OBJECT_GATEWAY_CONTROL_TOKEN`:不同且至少 32 字节;普通 worker 仅持有 data token。 +- `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`:独立审核者的 Ed25519 签名隔离证据,绑定协议、scope、gateway ID、时间窗及 writer/policy/拒写/排空摘要。审核私钥不能交给 worker 或网关。 + +秘密只经服务环境/私有文件注入,不进 argv、URL 或日志。所有调用端错误都使用固定原因码。 + +正式 worker 使用 `LIBRARY_OBJECT_GATEWAY_URL` 和 `LIBRARY_OBJECT_GATEWAY_TOKEN`;不再依赖预先生成的 grant 文件。`LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY` 保留旧离线兼容用途,不是自动模式的授权来源。发布 stack 对所有对象 provider 强制网关;S3 构造仍用于只读捕获/预备份,部署必须使用独立只读凭据。 + +## 已关闭的本仓旁路 + +- `ObjectImporter.Stage/Promote` 无条件拒绝,不留配置开关,不执行 PUT/COPY。 +- legacy `release-worker` runtime/route 在读凭据及连接目标前拒绝。 +- `release-diag`、`release-channel-e2e` 旧写型入口启动即非零退出,避免 probe PUT 或文档级旁路。 +- `librarystack` 不再为缺网关的 MinIO/S3 返回直写 writer。 +- `libraryctl` capture 的 S3 构造与读取保留;未把读取当作写入口禁用。 + +这不等于已经撤销运行环境旧进程的凭据;源码变更尚未部署。`librarydrill` 等无 owned-attempt 的旧写型工具不能靠静态 grant 绕过自动模式,需补正式执行身份后才可用于写入。低层 S3 adapter 仍用于网关后端、读取及显式隔离夹具;部署权限才负责防止外部旧进程直接调用存储。 + +## 实测与未完成边界 + +已实跑:真实治理 PG(独立 SELECT-only 登录角色)、原 archive、数据 HTTP、业务 PG 与 MinIO 的自动授权/跨 epoch 链;未锚定计划在第一笔 PG 前拒绝,接管保持原发送身份且不增加 PUT。另有真实网关二进制的两次中断验证。隔离审计 authority/故障注入仍是测试设施,不是生产权限证明。 + +正式 worker 隔离进程证据:真实 Docker 入口闸门/队列、只读 worker 对象身份、独立治理读取角色、网关与业务 PG/ES/MinIO;首次 PG 后和对象提交后两次 SIGKILL,同一原 job/operation/plan 的三代 attempt 完成发布,原对象发送 epoch 与回执不变。隔离构建标签 `kg_isolated_drill` 仅把真实租约缩短至4秒,并拒绝非 loopback 一次性 r8 库;正常构建仍为10分钟。维护上游是无业务凭据的独立 HTTP 夹具,不是运行中的 WeKnora;签名证据绑定本次真实拒写/排空材料,不代表生产独占审核。 + +已补独立 `library.recover` 正式worker链、恢复中再次SIGKILL、完整before-image/CAS/精确版本读回,以及来源/目标/原job/计划/operation/任意队列身份拒绝。ES故障代理只转发真实ES并丢弃已提交响应,未伪造服务响应。 + +尚未完成:clean-finalization unknown的充分归属与自动收敛;独立运行环境只读对象凭据、ACL/网络和全部writer排空;网关真实OSS端到端;原D0/F11/F12与55项其余缺口。测试/隔离验收与真实部署独占分列,不把生产变更授权当成本机开发授权的前置。 + +维护隔离、独占审核与 grant 是独立前置。配置 true、健康 200、测试自签 authority 都不能代替已部署拒写/排空证据。目前未签发真实环境独占证明,未改生产权限或上游 WeKnora 源码/Schema。 + +## 部署与回退(未执行) + +未来测试主机 `120.76.217.51`、生产 `120.76.218.137` 各自独立批准。每批先冻结主机、桶、全部角色/STS/预签名写入口、端口/网络规则、影响、维护窗口与回滚;当前没有执行这些外部变更。 + +先关新写准入→排空并对账旧请求→撤旧直写权限/切只读身份并实测拒写→独立审核→批准的新限定对象验收→父复核,才可切业务客户端。r5 的已完成 OSS 测试授权不能重复使用。 + +回退先关准入并保留 journal;所有已发送请求有终态才可撤网关,unknown 继续 held。不得因网关故障自动恢复旧直写凭据。权限/网络回退须另批。当前整模块 INCOMPLETE,禁止部署部分候选。 diff --git a/internal/objectgateway/acquire.go b/internal/objectgateway/acquire.go new file mode 100644 index 0000000..8dc959c --- /dev/null +++ b/internal/objectgateway/acquire.go @@ -0,0 +1,159 @@ +package objectgateway + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "github.com/jackc/pgx/v5" + "math" +) + +// Claim names an already persisted application attempt and write-set anchor. +// It never carries keys, bodies, a replacement plan or a chosen gateway epoch. +type Claim struct { + JobID string `json:"job_id"` + OperationID string `json:"operation_id"` + AttemptEpoch int64 `json:"attempt_epoch"` + Owner string `json:"owner"` + PlanDigest string `json:"plan_digest"` + RecoveryJobID string `json:"recovery_job_id,omitempty"` + RecoveryOperationID string `json:"recovery_operation_id,omitempty"` +} +type PlanAuthority interface { + Load(context.Context, Claim) (Plan, error) +} + +func (g *Gateway) Acquire(ctx context.Context, c Claim) (Grant, error) { + if g.Authority == nil { + return Grant{}, ErrAuthority + } + if c.JobID == "" || c.OperationID == "" || c.Owner == "" || c.AttemptEpoch < 1 || !validHash(c.PlanDigest) { + return Grant{}, ErrEpoch + } + if e := g.Ready(ctx); e != nil { + return Grant{}, e + } + plan, e := g.Authority.Load(ctx, c) + if e != nil { + return Grant{}, e + } + if plan.Digest != c.PlanDigest || plan.Owner != Digest([]byte(c.JobID)) { + return Grant{}, ErrConflict + } + if e = g.InstallPlan(ctx, plan); e != nil { + return Grant{}, e + } + // Recover only sends in this exact frozen plan. No absent observation can + // settle an unknown send; the locked advancement below still checks all intents. + rows, e := g.Pool.Query(ctx, `SELECT i.request FROM kg_object_gateway_intents i + LEFT JOIN kg_object_gateway_receipts r ON r.scope=i.scope AND r.operation=i.operation + WHERE i.scope=$1 AND r.operation IS NULL`, g.Scope) + if e != nil { + return Grant{}, ErrPersistence + } + var pending []Request + for rows.Next() { + var raw []byte + var request Request + if rows.Scan(&raw) != nil || json.Unmarshal(raw, &request) != nil { + rows.Close() + return Grant{}, ErrPersistence + } + if request.Grant.Plan == plan.Digest { + pending = append(pending, request) + } + } + e = rows.Err() + rows.Close() + if e != nil { + return Grant{}, ErrPersistence + } + for _, request := range pending { + if request.Grant.Owner != plan.Owner { + return Grant{}, ErrConflict + } + if _, e = g.Reconcile(ctx, request); e != nil { + return Grant{}, e + } + } + tx, e := g.Pool.Begin(ctx) + if e != nil { + return Grant{}, ErrPersistence + } + defer tx.Rollback(ctx) + if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); e != nil { + return Grant{}, ErrPersistence + } + var epoch int64 + if e = tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch); e != nil { + return Grant{}, ErrPersistence + } + // Recheck after serialization. This is admission evidence, not an OSS fence: + // a persisted uncertain object intent still prevents every epoch transition. + verified, e := g.Authority.Load(ctx, c) + if e != nil { + return Grant{}, e + } + frozen, _ := json.Marshal(plan) + again, _ := json.Marshal(verified) + if !bytes.Equal(frozen, again) { + return Grant{}, ErrConflict + } + raw, _ := json.Marshal(c) + var oldClaim, oldGrant []byte + e = tx.QueryRow(ctx, "SELECT claim,grant_body FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch=$3", g.Scope, c.JobID, c.AttemptEpoch).Scan(&oldClaim, &oldGrant) + if e == nil { + var grant Grant + if !bytes.Equal(raw, oldClaim) || json.Unmarshal(oldGrant, &grant) != nil || !grant.valid() { + return Grant{}, ErrConflict + } + if grant.Epoch != epoch { + return Grant{}, ErrEpoch + } + return grant, nil + } + if !errors.Is(e, pgx.ErrNoRows) { + return Grant{}, ErrPersistence + } + var firstRaw []byte + firstErr := tx.QueryRow(ctx, "SELECT claim FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 ORDER BY attempt_epoch LIMIT 1", g.Scope, c.JobID).Scan(&firstRaw) + if firstErr == nil { + var first Claim + if json.Unmarshal(firstRaw, &first) != nil { + return Grant{}, ErrPersistence + } + if first.JobID != c.JobID || first.OperationID != c.OperationID || first.PlanDigest != c.PlanDigest { + return Grant{}, ErrConflict + } + } else if !errors.Is(firstErr, pgx.ErrNoRows) { + return Grant{}, ErrPersistence + } + var stale bool + if tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch>=$3)", g.Scope, c.JobID, c.AttemptEpoch).Scan(&stale) != nil { + return Grant{}, ErrPersistence + } + if stale || epoch == math.MaxInt64 { + return Grant{}, ErrEpoch + } + next := Grant{Epoch: epoch + 1, Owner: plan.Owner, Plan: plan.Digest} + if e = g.advanceLocked(ctx, tx, epoch, next); e != nil { + return Grant{}, e + } + grantRaw, _ := json.Marshal(next) + if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_deliveries(scope,job,attempt_epoch,claim,grant_epoch,grant_body) VALUES($1,$2,$3,$4,$5,$6)", g.Scope, c.JobID, c.AttemptEpoch, raw, next.Epoch, grantRaw); e != nil { + return Grant{}, ErrPersistence + } + if tx.Commit(ctx) != nil { + return Grant{}, ErrPersistence + } + return next, nil +} +func (c *Client) Acquire(ctx context.Context, claim Claim) (Grant, error) { + var out Grant + e := c.call(ctx, "/v1/acquire", claim, &out) + if e == nil && (!out.valid() || out.Plan != claim.PlanDigest || out.Owner != Digest([]byte(claim.JobID))) { + e = ErrConflict + } + return out, e +} diff --git a/internal/objectgateway/acquire_integration_test.go b/internal/objectgateway/acquire_integration_test.go new file mode 100644 index 0000000..4588448 --- /dev/null +++ b/internal/objectgateway/acquire_integration_test.go @@ -0,0 +1,122 @@ +package objectgateway_test + +import ( + "context" + "errors" + g "knowledge-governance/internal/objectgateway" + "testing" +) + +type currentPlanAuthority struct { + claim g.Claim + plan g.Plan +} + +func (a *currentPlanAuthority) Load(_ context.Context, c g.Claim) (g.Plan, error) { + if c != a.claim { + return g.Plan{}, g.ErrEpoch + } + return a.plan, nil +} +func TestGatewayAcquisitionPersistsAttemptDelivery(t *testing.T) { + ctx, pool, backend := fixture(t) + claim := g.Claim{JobID: "job-acquire", OperationID: "operation-acquire", AttemptEpoch: 1, Owner: "worker-1", PlanDigest: g.Digest([]byte(t.Name()))} + expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest} + req := request("acquired-object", expected) + authority := ¤tPlanAuthority{claim, planFor(expected, req)} + gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority} + grant, e := gateway.Acquire(ctx, claim) + if e != nil || grant != expected { + t.Fatal("anchored active attempt grant unavailable", e) + } + replay, e := gateway.Acquire(ctx, claim) + if e != nil || replay != grant { + t.Fatal("grant delivery replay changed epoch", e) + } + oldClaim := claim + claim.AttemptEpoch = 2 + claim.Owner = "worker-2" + authority.claim = claim + next, e := gateway.Acquire(ctx, claim) + if e != nil || next.Epoch != 2 { + t.Fatal("next active attempt", e) + } + if _, e = gateway.Acquire(ctx, oldClaim); !errors.Is(e, g.ErrEpoch) { + t.Fatal("stale attempt acquired grant", e) + } + replacement := claim + replacement.AttemptEpoch++ + replacement.PlanDigest = g.Digest([]byte("replacement-plan")) + authority.claim = replacement + authority.plan.Digest = replacement.PlanDigest + if _, e = gateway.Acquire(ctx, replacement); !errors.Is(e, g.ErrConflict) { + t.Fatal("new attempt replaced original plan", e) + } + authority.claim = claim + authority.plan.Digest = claim.PlanDigest + req.Grant = next + receipt, e := gateway.ExecuteAuthorized(ctx, next, req, []byte("gateway fixture acquired-object")) + if e != nil { + t.Fatal(e) + } + cleanup(t, ctx, gateway, receipt) + var count int + if e = pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 2 { + t.Fatal("immutable delivery count", count, e) + } + for _, statement := range []string{"UPDATE kg_object_gateway_deliveries SET job=job", "DELETE FROM kg_object_gateway_deliveries", "TRUNCATE kg_object_gateway_deliveries"} { + if _, e = pool.Exec(ctx, statement); e == nil { + t.Fatal("mutable grant delivery", statement) + } + } +} + +func TestGatewayAcquisitionCannotHandOverUnknownSend(t *testing.T) { + ctx, pool, real := fixture(t) + backend := &stoppedBeforeSend{Backend: real} + claim := g.Claim{JobID: "job-held", OperationID: "operation-held", AttemptEpoch: 1, Owner: "worker-first", PlanDigest: g.Digest([]byte(t.Name()))} + expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest} + req := request("acquire-held", expected) + body := []byte("gateway fixture acquire-held") + authority := ¤tPlanAuthority{claim, planFor(expected, req)} + gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority} + if _, e := gateway.Acquire(ctx, claim); e != nil { + t.Fatal(e) + } + if _, e := gateway.ExecuteAuthorized(ctx, expected, req, body); !errors.Is(e, g.ErrHeld) { + t.Fatal(e) + } + claim.AttemptEpoch++ + claim.Owner = "worker-next" + authority.claim = claim + for i := 0; i < 2; i++ { + if _, e := gateway.Acquire(ctx, claim); !errors.Is(e, g.ErrHeld) { + t.Fatal("absent send handed over", e) + } + } + if backend.calls != 1 { + t.Fatal("control acquisition re-sent PUT") + } + var count int + if e := pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 1 { + t.Fatal("unknown send advanced delivery", count, e) + } + // Deliver the previously uncertain request late, using its exact immutable claim. + if _, e := real.Put(ctx, req.Key, body, req.Claim()); e != nil { + t.Fatal(e) + } + next, e := gateway.Acquire(ctx, claim) + if e != nil || next.Epoch != 2 { + t.Fatal("committed send was not reconciled before handover", e) + } + receipt, e := gateway.ExecuteAuthorized(ctx, next, req, body) + if e != nil || receipt.Request != req { + t.Fatal(e) + } + if backend.calls != 1 { + t.Fatal("reconciled send was retried") + } + receipt.Request.Grant = next + cleanup(t, ctx, gateway, receipt) + t.Log("new controller attempt stayed held while absent; late committed original recovered without replay PUT") +} diff --git a/internal/objectgateway/audit.go b/internal/objectgateway/audit.go new file mode 100644 index 0000000..8470471 --- /dev/null +++ b/internal/objectgateway/audit.go @@ -0,0 +1,57 @@ +package objectgateway + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "os" + "time" +) + +// Attestation is issued by an independent deployment auditor, never by a worker. +// Its signature authenticates an audit, not the truth of self-reported config. +// Runtime cutover must invalidate/replace the audit whenever writer rights change. +type Attestation struct { + Protocol string `json:"protocol"` + Scope string `json:"scope"` + GatewayID string `json:"gateway_id"` + IssuedAt time.Time `json:"issued_at"` + ExpiresAt time.Time `json:"expires_at"` + WriterInventory string `json:"writer_inventory_sha256"` + PolicyEvidence string `json:"policy_evidence_sha256"` + DenialEvidence string `json:"denial_evidence_sha256"` + DrainEvidence string `json:"drain_evidence_sha256"` +} +type SignedAttestation struct { + Payload json.RawMessage `json:"payload"` + Signature []byte `json:"signature"` +} +type AuditVerifier struct { + Path string + PublicKey ed25519.PublicKey + Scope, GatewayID string +} + +func (v AuditVerifier) Verify(ctx context.Context) error { + if ctx.Err() != nil { + return ErrIsolation + } + st, e := os.Lstat(v.Path) + if e != nil || !st.Mode().IsRegular() || st.Size() > 16384 || st.Mode().Perm()&0022 != 0 { + return ErrIsolation + } + raw, e := os.ReadFile(v.Path) + if e != nil { + return ErrIsolation + } + var signed SignedAttestation + var a Attestation + if json.Unmarshal(raw, &signed) != nil || len(v.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(v.PublicKey, signed.Payload, signed.Signature) || json.Unmarshal(signed.Payload, &a) != nil { + return ErrIsolation + } + now := time.Now() + if a.Protocol != Protocol || a.Scope != v.Scope || a.GatewayID != v.GatewayID || a.GatewayID == "" || a.IssuedAt.After(now) || !a.ExpiresAt.After(now) || a.ExpiresAt.Sub(a.IssuedAt) > time.Hour || !validHash(a.WriterInventory) || !validHash(a.PolicyEvidence) || !validHash(a.DenialEvidence) || !validHash(a.DrainEvidence) { + return ErrIsolation + } + return nil +} diff --git a/internal/objectgateway/audit_test.go b/internal/objectgateway/audit_test.go new file mode 100644 index 0000000..60562ec --- /dev/null +++ b/internal/objectgateway/audit_test.go @@ -0,0 +1,59 @@ +package objectgateway_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + g "knowledge-governance/internal/objectgateway" + "os" + "path/filepath" + "testing" + "time" +) + +func TestGatewayAuditCannotBeConfigurationBoolean(t *testing.T) { + pub, priv, e := ed25519.GenerateKey(rand.Reader) + if e != nil { + t.Fatal(e) + } + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + verifier := g.AuditVerifier{Path: path, PublicKey: pub, Scope: "target", GatewayID: "gateway"} + if verifier.Verify(context.Background()) == nil { + t.Fatal("missing audit accepted") + } + if os.WriteFile(path, []byte(`{"isolated":true,"ready":true}`), 0600) != nil { + t.Fatal("fixture") + } + if verifier.Verify(context.Background()) == nil { + t.Fatal("boolean accepted as isolation proof") + } + a := g.Attestation{Protocol: g.Protocol, Scope: "target", GatewayID: "gateway", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("inventory")), PolicyEvidence: g.Digest([]byte("policy")), DenialEvidence: g.Digest([]byte("denials")), DrainEvidence: g.Digest([]byte("drain"))} + write := func(v g.Attestation) { + t.Helper() + raw, _ := json.Marshal(v) + signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)}) + if os.WriteFile(path, signed, 0600) != nil { + t.Fatal("fixture") + } + } + write(a) + if verifier.Verify(context.Background()) != nil { + t.Fatal("valid test auditor signature rejected") + } + for _, mutate := range []func(*g.Attestation){func(v *g.Attestation) { v.Scope = "other" }, func(v *g.Attestation) { v.GatewayID = "other" }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *g.Attestation) { v.DenialEvidence = "" }, func(v *g.Attestation) { v.IssuedAt = time.Now().Add(time.Hour) }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(24 * time.Hour) }} { + bad := a + mutate(&bad) + write(bad) + if verifier.Verify(context.Background()) == nil { + t.Fatal("invalid isolation audit accepted") + } + } + write(a) + other, _, _ := ed25519.GenerateKey(rand.Reader) + verifier.PublicKey = other + if verifier.Verify(context.Background()) == nil { + t.Fatal("untrusted signer accepted") + } +} diff --git a/internal/objectgateway/claim_context.go b/internal/objectgateway/claim_context.go new file mode 100644 index 0000000..efd1e01 --- /dev/null +++ b/internal/objectgateway/claim_context.go @@ -0,0 +1,15 @@ +package objectgateway + +import "context" + +type claimContextKey struct{} + +// WithClaim is set by the owned-attempt runner, never from an HTTP caller's +// proposed plan. The controller independently checks every field against PG. +func WithClaim(ctx context.Context, c Claim) context.Context { + return context.WithValue(ctx, claimContextKey{}, c) +} +func ClaimFromContext(ctx context.Context) Claim { + c, _ := ctx.Value(claimContextKey{}).(Claim) + return c +} diff --git a/internal/objectgateway/gateway.go b/internal/objectgateway/gateway.go new file mode 100644 index 0000000..5e43cf3 --- /dev/null +++ b/internal/objectgateway/gateway.go @@ -0,0 +1,326 @@ +package objectgateway + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Schema is applied explicitly by the migration identity, never on server startup. +// +//go:embed schema.sql +var Schema string + +type Gateway struct { + Authority PlanAuthority + Pool *pgxpool.Pool + Scope string + Backend Backend + Isolation IsolationVerifier + MaxBytes int64 +} + +func (g *Gateway) Ready(ctx context.Context) error { + if g.Pool == nil || g.Scope == "" || g.Backend == nil || g.MaxBytes < 1 { + return ErrPersistence + } + if g.Isolation == nil || g.Isolation.Verify(ctx) != nil { + return ErrIsolation + } + var sync, fsync, full string + if g.Pool.QueryRow(ctx, "SELECT current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes')").Scan(&sync, &fsync, &full) != nil || sync != "on" || fsync != "on" || full != "on" { + return ErrPersistence + } + var journal bool + if g.Pool.QueryRow(ctx, `SELECT NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off' + AND to_regclass('kg_object_gateway_plans') IS NOT NULL + AND to_regclass('kg_object_gateway_scopes') IS NOT NULL + AND to_regclass('kg_object_gateway_grants') IS NOT NULL + AND to_regclass('kg_object_gateway_intents') IS NOT NULL + AND to_regclass('kg_object_gateway_receipts') IS NOT NULL + AND to_regclass('kg_object_gateway_deliveries') IS NOT NULL + AND (SELECT count(*)=10 FROM pg_trigger WHERE tgname IN ('kg_object_gateway_deliveries_immutable','kg_object_gateway_deliveries_no_truncate','kg_object_gateway_plans_no_truncate','kg_object_gateway_intents_no_truncate','kg_object_gateway_receipts_no_truncate','kg_object_gateway_grants_no_truncate','kg_object_gateway_plans_immutable','kg_object_gateway_intents_immutable','kg_object_gateway_receipts_immutable','kg_object_gateway_grants_immutable') AND tgenabled='O')`).Scan(&journal) != nil || !journal { + return ErrPersistence + } + if g.Backend.Ready(ctx) != nil { + return errors.New("gateway_backend_not_ready") + } + return nil +} +func (g *Gateway) Advance(ctx context.Context, expected int64, next Grant) error { + if !next.valid() || next.Epoch != expected+1 { + return ErrEpoch + } + if err := g.Ready(ctx); err != nil { + return err + } + var frozen []byte + var plan Plan + if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, next.Plan).Scan(&frozen) != nil { + return ErrConflict + } + if json.Unmarshal(frozen, &plan) != nil || !plan.valid() || plan.Owner != next.Owner { + return ErrConflict + } + tx, err := g.Pool.Begin(ctx) + if err != nil { + return ErrPersistence + } + defer tx.Rollback(ctx) + if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); err != nil { + return ErrPersistence + } + var epoch int64 + if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil { + return ErrPersistence + } + if err = g.advanceLocked(ctx, tx, epoch, next); err != nil { + return err + } + if tx.Commit(ctx) != nil { + return ErrPersistence + } + return nil +} + +func (g *Gateway) advanceLocked(ctx context.Context, tx pgx.Tx, epoch int64, next Grant) error { + var err error + if epoch == next.Epoch { + var owner, plan string + if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil { + return ErrPersistence + } + if owner != next.Owner || plan != next.Plan { + return ErrEpoch + } + return nil + } + if epoch != next.Epoch-1 { + return ErrEpoch + } + var pending bool + if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts r WHERE r.scope=i.scope AND r.operation=i.operation))`, g.Scope).Scan(&pending) != nil { + return ErrPersistence + } + if pending { + return ErrHeld + } + if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_grants(scope,epoch,owner,plan) VALUES($1,$2,$3,$4)", g.Scope, next.Epoch, next.Owner, next.Plan); err != nil { + return ErrPersistence + } + if _, err = tx.Exec(ctx, "UPDATE kg_object_gateway_scopes SET epoch=$2 WHERE scope=$1", g.Scope, next.Epoch); err != nil { + return ErrPersistence + } + return nil +} + +// reserve commits an immutable may-send intent. Even a commit timeout cannot be +// followed by a send. Another process can reconcile, but cannot retry this PUT. +func (g *Gateway) reserve(ctx context.Context, r Request, authorization *Grant) (bool, error) { + tx, err := g.Pool.Begin(ctx) + if err != nil { + return false, ErrPersistence + } + defer tx.Rollback(ctx) + var epoch int64 + if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil { + return false, ErrEpoch + } + if authorization != nil { + var owner, plan string + if epoch != authorization.Epoch { + return false, ErrEpoch + } + if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil { + return false, ErrPersistence + } + if owner != authorization.Owner || plan != authorization.Plan { + return false, ErrEpoch + } + } + raw, _ := json.Marshal(r) + var previous []byte + err = tx.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&previous) + if err == nil { + if string(previous) != string(raw) { + return false, ErrConflict + } + return false, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return false, ErrPersistence + } + var owner, plan string + if epoch != r.Grant.Epoch { + return false, ErrEpoch + } + if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil { + return false, ErrPersistence + } + if owner != r.Grant.Owner || plan != r.Grant.Plan { + return false, ErrEpoch + } + var pending bool + if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND i.key=$2 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts x WHERE x.scope=i.scope AND x.operation=i.operation))`, g.Scope, r.Key).Scan(&pending) != nil { + return false, ErrPersistence + } + if pending { + return false, ErrHeld + } + var frozen []byte + var allowed Plan + if tx.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, r.Grant.Plan).Scan(&frozen) != nil { + return false, ErrPersistence + } + if json.Unmarshal(frozen, &allowed) != nil || !allowed.valid() || !allowed.allows(r) { + return false, ErrConflict + } + if r.Action == "delete" { + previous, err := g.receipt(ctx, r.CreatedBy) + if err != nil || previous.State != "committed" || previous.Request.Key != r.Key || previous.Version != r.Version { + return false, ErrConflict + } + } + if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_intents(scope,operation,key,request) VALUES($1,$2,$3,$4)", g.Scope, r.Operation, r.Key, raw); err != nil { + return false, ErrPersistence + } + if tx.Commit(ctx) != nil { + return false, ErrPersistence + } + return true, nil +} +func (g *Gateway) receipt(ctx context.Context, id string) (Receipt, error) { + var raw []byte + var r Receipt + err := g.Pool.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE scope=$1 AND operation=$2", g.Scope, id).Scan(&raw) + if err != nil { + return r, err + } + if json.Unmarshal(raw, &r) != nil { + return r, ErrPersistence + } + return r, nil +} +func (g *Gateway) finish(ctx context.Context, r Request, state string, v Version) (Receipt, error) { + receipt := Receipt{r, state, v} + raw, _ := json.Marshal(receipt) + _, err := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_receipts(scope,operation,receipt) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, r.Operation, raw) + if err != nil { + return Receipt{}, ErrPersistence + } + got, err := g.receipt(ctx, r.Operation) + if err != nil { + return Receipt{}, ErrPersistence + } + if got != receipt { + return Receipt{}, ErrConflict + } + return got, nil +} +func (g *Gateway) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) { + return g.execute(ctx, r, body, nil) +} +func (g *Gateway) execute(ctx context.Context, r Request, body []byte, authorization *Grant) (Receipt, error) { + if err := r.validate(); err != nil { + return Receipt{}, err + } + if r.Bytes > g.MaxBytes || r.Action == "create" && (int64(len(body)) != r.Bytes || Digest(body) != r.SHA256) || r.Action == "delete" && len(body) != 0 { + return Receipt{}, ErrConflict + } + if err := g.Ready(ctx); err != nil { + return Receipt{}, err + } + fresh, err := g.reserve(ctx, r, authorization) + if err != nil { + return Receipt{}, err + } + if !fresh { + return g.Reconcile(ctx, r) + } + if r.Action == "create" { + _, exists, err := g.Backend.Observe(ctx, r.Key, "") + if err != nil { + return Receipt{}, ErrHeld + } + if exists { + return g.finish(ctx, r, "rejected", Version{}) + } + v, err := g.Backend.Put(ctx, r.Key, body, r.token()) + if err != nil { + return Receipt{}, ErrHeld + } + exact, exists, err := g.Backend.Observe(ctx, r.Key, v.ID) + if err != nil || !exists || !r.matches(exact) || v != exact { + return Receipt{}, ErrHeld + } + return g.finish(ctx, r, "committed", v) + } + v, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID) + if err != nil { + return Receipt{}, ErrHeld + } + if exists && v != r.Version { + return Receipt{}, ErrConflict + } + if exists { + if g.Backend.Delete(ctx, r.Key, r.Version) != nil { + return Receipt{}, ErrHeld + } + } + return g.Reconcile(ctx, r) +} +func (g *Gateway) Reconcile(ctx context.Context, r Request) (Receipt, error) { + if err := r.validate(); err != nil { + return Receipt{}, err + } + if err := g.Ready(ctx); err != nil { + return Receipt{}, err + } + var raw []byte + if g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&raw) != nil { + return Receipt{}, ErrConflict + } + want, _ := json.Marshal(r) + if string(raw) != string(want) { + return Receipt{}, ErrConflict + } + got, err := g.receipt(ctx, r.Operation) + if err == nil { + return got, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return Receipt{}, ErrPersistence + } + if r.Action == "delete" { + _, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID) + if err != nil || exists { + return Receipt{}, ErrHeld + } + return g.finish(ctx, r, "removed", r.Version) + } + versions, err := g.Backend.List(ctx, r.Key) + if err != nil { + return Receipt{}, ErrHeld + } + var found []Version + for _, v := range versions { + if v.Owner == r.token() { + if !r.matches(v) { + return Receipt{}, ErrHeld + } + found = append(found, v) + } + } + if len(found) != 1 { + return Receipt{}, ErrHeld + } + exact, exists, err := g.Backend.Observe(ctx, r.Key, found[0].ID) + if err != nil || !exists || exact != found[0] { + return Receipt{}, ErrHeld + } + return g.finish(ctx, r, "committed", exact) +} diff --git a/internal/objectgateway/gateway_integration_test.go b/internal/objectgateway/gateway_integration_test.go new file mode 100644 index 0000000..26e13e6 --- /dev/null +++ b/internal/objectgateway/gateway_integration_test.go @@ -0,0 +1,399 @@ +package objectgateway_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/jackc/pgx/v5/pgxpool" + g "knowledge-governance/internal/objectgateway" +) + +type verifiedFixture struct{} + +func (verifiedFixture) Verify(context.Context) error { return nil } +func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) { + t.Helper() + dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("isolated gateway PG/MinIO opt-in required") + } + u, e := url.Parse(dsn) + if e != nil || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "kg_library_test_gateway_") || u.Hostname() != "127.0.0.1" { + t.Fatal("unsafe gateway test database") + } + ctx := context.Background() + pool, e := pgxpool.New(ctx, dsn) + if e != nil { + t.Fatal("PG unavailable") + } + t.Cleanup(pool.Close) + if _, e = pool.Exec(ctx, g.Schema); e != nil { + t.Fatal(e) + } + endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT") + if endpoint != "http://127.0.0.1:19100" { + t.Fatal("unsafe test MinIO endpoint") + } + bucket := "kg-library-test-gateway-" + g.Digest([]byte(t.Name() + time.Now().String()))[:16] + client := s3.New(s3.Options{Region: "us-east-1", BaseEndpoint: aws.String(endpoint), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "")}) + if _, e = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); e != nil { + t.Fatal("create isolated bucket failed") + } + if _, e = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}}); e != nil { + t.Fatal(e) + } + t.Cleanup(func() { + list, e := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)}) + if e != nil { + t.Error("residue list unavailable") + return + } + if len(list.Versions) != 0 || len(list.DeleteMarkers) != 0 { + t.Errorf("preserving residue bucket %s: versions=%d markers=%d", bucket, len(list.Versions), len(list.DeleteMarkers)) + return + } + if _, e = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); e != nil { + t.Error("delete empty test bucket failed") + } + t.Logf("exact sandbox %s: versions=0 markers=0; empty bucket deleted", bucket) + }) + return ctx, pool, &g.S3Backend{Client: client, Bucket: bucket, MaxBytes: 1 << 20} +} +func planFor(grant g.Grant, requests ...g.Request) g.Plan { + p := g.Plan{Digest: grant.Plan, Owner: grant.Owner} + for _, r := range requests { + p.Objects = append(p.Objects, g.ObjectBinding{Operation: r.Operation, Key: r.Key, SHA256: r.SHA256, Bytes: r.Bytes}) + } + return p +} +func request(label string, grant g.Grant) g.Request { + body := []byte("gateway fixture " + label) + return g.Request{Grant: grant, Operation: g.Digest([]byte(label)), Key: "unchanged/" + label, Action: "create", SHA256: g.Digest(body), Bytes: int64(len(body))} +} +func cleanup(t *testing.T, ctx context.Context, gw *g.Gateway, receipt g.Receipt) { + t.Helper() + r := receipt.Request + r.Operation = g.Digest([]byte(r.Operation + "delete")) + r.Action = "delete" + r.CreatedBy = receipt.Request.Operation + r.Version = receipt.Version + if got, e := gw.Execute(ctx, r, nil); e != nil || got.State != "removed" { + t.Fatalf("exact cleanup: %v %s", e, got.State) + } +} +func TestGatewayReadinessRequiresJournal(t *testing.T) { + ctx, pool, backend := fixture(t) + // An isolated search_path lacks the journal while PG settings remain healthy. + conn, e := pool.Acquire(ctx) + if e != nil { + t.Fatal(e) + } + conn.Release() + cfg := pool.Config() + cfg.ConnConfig.RuntimeParams["search_path"] = "pg_catalog" + other, e := pgxpool.NewWithConfig(ctx, cfg) + if e != nil { + t.Fatal(e) + } + defer other.Close() + gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) { + t.Fatalf("missing durable journal accepted: %v", e) + } +} +func TestGatewayRealReplayAndImmutableLedger(t *testing.T) { + ctx, pool, backend := fixture(t) + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))} + gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists { + t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists) + } + if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil { + t.Fatal(e) + } + if e := gw.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + req := request(t.Name(), grant) + body := []byte("gateway fixture " + t.Name()) + one, e := gw.Execute(ctx, req, body) + if e != nil || one.State != "committed" { + t.Fatalf("create %v %s", e, one.State) + } + two, e := gw.Execute(ctx, req, body) + if e != nil || two != one { + t.Fatal("replay changed receipt", e) + } + bad := req + bad.Key += "-different" + if _, e = gw.Execute(ctx, bad, body); !errors.Is(e, g.ErrConflict) { + t.Fatal("operation rebound", e) + } + for _, table := range []string{"intents", "receipts", "grants", "plans"} { + if _, e = pool.Exec(ctx, "DELETE FROM kg_object_gateway_"+table+" WHERE scope=$1", gw.Scope); e == nil { + t.Fatal("immutable evidence deleted", table) + } + } + for _, table := range []string{"plans", "intents", "receipts", "grants"} { + tx, e := pool.Begin(ctx) + if e != nil { + t.Fatal(e) + } + _, e = tx.Exec(ctx, "TRUNCATE kg_object_gateway_"+table+" CASCADE") + _ = tx.Rollback(ctx) + if e == nil { + cleanup(t, ctx, gw, one) + t.Fatal("immutable evidence truncation accepted", table) + } + } + cleanup(t, ctx, gw, one) + next := grant + next.Epoch = 2 + if e = gw.Advance(ctx, 1, next); e != nil { + t.Fatal(e) + } + late := request("late", grant) + if _, e = gw.Execute(ctx, late, []byte("gateway fixture late")); !errors.Is(e, g.ErrEpoch) { + t.Fatal("old epoch sent", e) + } +} + +func signedAudit(t *testing.T, scope string) (string, string) { + t.Helper() + pub, priv, e := ed25519.GenerateKey(rand.Reader) + if e != nil { + t.Fatal(e) + } + // This is a test authority. It exercises verification, NOT production exclusivity. + a := g.Attestation{Protocol: g.Protocol, Scope: scope, GatewayID: "isolated-fixture", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("fixture inventory")), PolicyEvidence: g.Digest([]byte("fixture policy")), DenialEvidence: g.Digest([]byte("fixture denial")), DrainEvidence: g.Digest([]byte("fixture drain"))} + raw, _ := json.Marshal(a) + signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)}) + path := filepath.Join(t.TempDir(), "audit.json") + if os.WriteFile(path, signed, 0600) != nil { + t.Fatal("audit write failed") + } + return path, base64.StdEncoding.EncodeToString(pub) +} +func startGateway(t *testing.T, env []string, client *g.Client) *exec.Cmd { + t.Helper() + binary := os.Getenv("KG_GATEWAY_TEST_BINARY") + if binary == "" { + t.Fatal("actual gateway binary is required") + } + cmd := exec.Command(binary) + cmd.Env = env + log, e := os.Create(filepath.Join(t.TempDir(), "service.log")) + if e != nil { + t.Fatal(e) + } + cmd.Stdout = log + cmd.Stderr = log + if e = cmd.Start(); e != nil { + t.Fatal(e) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); _ = log.Close() }) + for i := 0; i < 100; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + e = client.Ready(ctx) + cancel() + if e == nil { + return cmd + } + time.Sleep(30 * time.Millisecond) + } + raw, _ := os.ReadFile(log.Name()) + t.Fatalf("actual gateway not ready: %s", raw) + return nil +} +func TestGatewayActualServiceTwoInterruptions(t *testing.T) { + ctx, pool, backend := fixture(t) + upstream, _ := url.Parse(os.Getenv("KG_GATEWAY_TEST_ENDPOINT")) + proxy := httputil.NewSingleHostReverseProxy(upstream) + beforeSend := make(chan struct{}, 2) + allowSend := make(chan struct{}, 2) + committed := make(chan struct{}, 2) + release := make(chan struct{}, 2) + var puts atomic.Int32 + proxy.ModifyResponse = func(resp *http.Response) error { + if resp.Request.Method == "PUT" && !strings.Contains(resp.Request.URL.RawQuery, "versioning") { + puts.Add(1) + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + committed <- struct{}{} + select { + case <-release: + case <-resp.Request.Context().Done(): + } + return errors.New("injected lost response after actual MinIO commit") + } + } + return nil + } + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) { http.Error(w, "injected_transport_loss", 502) } + p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" && !strings.Contains(r.URL.RawQuery, "versioning") { + beforeSend <- struct{}{} + select { + case <-allowSend: + case <-r.Context().Done(): + return + } + } + proxy.ServeHTTP(w, r) + })) + defer p.Close() + scope := g.Digest([]byte("minio\x00" + p.URL + "\x00" + backend.Bucket)) + audit, pub := signedAudit(t, scope) + listener, e := net.Listen("tcp", "127.0.0.1:0") + if e != nil { + t.Fatal(e) + } + addr := listener.Addr().String() + listener.Close() + data := g.Digest([]byte("data " + scope)) + control := g.Digest([]byte("control " + scope)) + client, e := g.NewClient("http://"+addr, data, scope) + if e != nil { + t.Fatal(e) + } + admin, _ := g.NewClient("http://"+addr, control, scope) + env := append(os.Environ(), "OBJECT_GATEWAY_LISTEN="+addr, "OBJECT_GATEWAY_DATABASE_URL="+os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"), "OBJECT_GATEWAY_PROVIDER=minio", "OBJECT_GATEWAY_ENDPOINT="+p.URL, "OBJECT_GATEWAY_BUCKET="+backend.Bucket, "OBJECT_GATEWAY_ACCESS_KEY="+os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), "OBJECT_GATEWAY_SECRET_KEY="+os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY="+pub, "OBJECT_GATEWAY_AUDIT_FILE="+audit, "OBJECT_GATEWAY_ID=isolated-fixture", "OBJECT_GATEWAY_DATA_TOKEN="+data, "OBJECT_GATEWAY_CONTROL_TOKEN="+control) + cmd := startGateway(t, env, client) + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("actual-worker")), Plan: g.Digest([]byte("frozen-actual-plan"))} + if e = client.InstallPlan(ctx, planFor(grant, request("actual-0", grant))); e == nil { + t.Fatal("data client installed plan") + } + if e = admin.InstallPlan(ctx, planFor(grant, request("actual-0", grant), request("actual-1", grant))); e != nil { + t.Fatal(e) + } + if e = admin.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + if e = client.Advance(ctx, 0, grant); e == nil { + t.Fatal("data client acquired control authority") + } + for iteration := 0; iteration < 2; iteration++ { + label := fmt.Sprintf("actual-%d", iteration) + r := request(label, grant) + body := []byte("gateway fixture " + label) + done := make(chan error, 1) + go func() { _, e := client.Execute(ctx, r, body); done <- e }() + select { + case <-beforeSend: + case <-time.After(15 * time.Second): + t.Fatal("gateway did not send to proxy") + } + if _, e = client.Reconcile(ctx, r); !errors.Is(e, g.ErrHeld) { + t.Fatal("not-yet-visible PUT treated as terminal", e) + } + blockedNext := grant + blockedNext.Epoch++ + if e = admin.Advance(ctx, grant.Epoch, blockedNext); !errors.Is(e, g.ErrHeld) { + t.Fatal("epoch crossed delayed precommit request", e) + } + allowSend <- struct{}{} + select { + case <-committed: + case <-time.After(15 * time.Second): + t.Fatal("actual MinIO PUT did not commit") + } + next := grant + next.Epoch++ + if e = admin.Advance(ctx, grant.Epoch, next); !errors.Is(e, g.ErrHeld) { + t.Fatal("epoch crossed in-flight write", e) + } + sameKey := r + sameKey.Operation = g.Digest([]byte(label + "other")) + if _, e = client.Execute(ctx, sameKey, body); !errors.Is(e, g.ErrHeld) { + t.Fatal("same-key competitor admitted", e) + } + if e = cmd.Process.Kill(); e != nil { + t.Fatal(e) + } + _ = cmd.Wait() + release <- struct{}{} + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("dead gateway call did not end") + } + t.Logf("actual gateway SIGKILL %d after MinIO commit before receipt", iteration+1) + cmd = startGateway(t, env, client) + receipt, e := client.Reconcile(ctx, r) + if e != nil || receipt.State != "committed" { + t.Fatalf("restart reconcile %v %s", e, receipt.State) + } + repeated, e := client.Execute(ctx, r, body) + if e != nil || receipt != repeated { + t.Fatal("restart duplicate not idempotent", e) + } + del := r + del.Action = "delete" + del.CreatedBy = r.Operation + del.Operation = g.Digest([]byte(label + "cleanup")) + del.Version = receipt.Version + cleaned, e := client.Execute(ctx, del, nil) + if e != nil || cleaned.State != "removed" { + t.Fatal("actual exact cleanup failed", e) + } + if e = admin.Advance(ctx, grant.Epoch, next); e != nil { + t.Fatal(e) + } + late := request(label+"late", grant) + if _, e = client.Execute(ctx, late, []byte("gateway fixture "+label+"late")); !errors.Is(e, g.ErrEpoch) { + t.Fatal("late stale epoch admitted", e) + } + grant = next + } + if puts.Load() != 2 { + t.Fatalf("blind PUT replay count=%d", puts.Load()) + } + var n int + if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", scope).Scan(&n) != nil || n != 4 { + t.Fatalf("durable receipts=%d", n) + } + _ = cmd.Process.Kill() + _ = cmd.Wait() + if e = client.Ready(ctx); e == nil { + t.Fatal("unavailable gateway accepted") + } + t.Log("actual binary + HTTP + durable PG + MinIO: two kills/restarts, 2 PUTs, 4 receipts, exact cleanup") +} + +func TestGatewayClientRejectsBadProtocolAndLeaks(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("secret") != "" { + t.Error("secret in URL") + } + _, _ = io.WriteString(w, `{"protocol":"other","scope":"scope","ready":true}`) + })) + defer server.Close() + c, e := g.NewClient(server.URL, strings.Repeat("x", 32), "scope") + if e != nil { + t.Fatal(e) + } + if c.Ready(context.Background()) == nil { + t.Fatal("wrong protocol accepted") + } +} diff --git a/internal/objectgateway/http.go b/internal/objectgateway/http.go new file mode 100644 index 0000000..e2eedac --- /dev/null +++ b/internal/objectgateway/http.go @@ -0,0 +1,316 @@ +package objectgateway + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type ExecuteRequest struct { + Authorization *Grant `json:"authorization,omitempty"` + Request Request `json:"request"` + Body []byte `json:"body,omitempty"` +} +type AdvanceRequest struct { + Expected int64 `json:"expected"` + Grant Grant `json:"grant"` +} +type Capabilities struct { + AutomaticGrants bool `json:"automatic_grants"` + Protocol string `json:"protocol"` + Scope string `json:"scope"` + Ready bool `json:"ready"` +} +type HTTPServer struct { + Gateway *Gateway + DataToken, ControlToken string +} + +func (s HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + token := s.DataToken + if r.URL.Path == "/v1/advance" || r.URL.Path == "/v1/plans" { + token = s.ControlToken + } + if len(token) < 32 || s.DataToken == s.ControlToken || subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+token)) != 1 { + http.Error(w, "gateway_unauthorized", 401) + return + } + if r.Header.Get("X-KG-Gateway-Protocol") != Protocol || r.Header.Get("X-KG-Gateway-Scope") != s.Gateway.Scope { + http.Error(w, "gateway_protocol_or_target_mismatch", 409) + return + } + fail := func(e error) { + code := 503 + if errors.Is(e, ErrEpoch) || errors.Is(e, ErrConflict) || errors.Is(e, ErrHeld) { + code = 409 + } + http.Error(w, e.Error(), code) + } + decode := func(v any) bool { + r.Body = http.MaxBytesReader(w, r.Body, s.Gateway.MaxBytes*2+16384) + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if d.Decode(v) != nil || d.Decode(&struct{}{}) != io.EOF { + http.Error(w, "gateway_bad_request", 400) + return false + } + return true + } + if r.Method == "GET" && r.URL.Path == "/v1/capabilities" { + if e := s.Gateway.Ready(r.Context()); e != nil { + fail(e) + return + } + if authority, ok := s.Gateway.Authority.(interface{ Ready(context.Context) error }); ok { + if e := authority.Ready(r.Context()); e != nil { + fail(e) + return + } + } + _ = json.NewEncoder(w).Encode(Capabilities{Protocol: Protocol, Scope: s.Gateway.Scope, Ready: true, AutomaticGrants: s.Gateway.Authority != nil}) + return + } + if r.Method != "POST" { + http.Error(w, "gateway_route_not_found", 404) + return + } + switch r.URL.Path { + case "/v1/acquire": + var in Claim + if !decode(&in) { + return + } + out, e := s.Gateway.Acquire(r.Context(), in) + if e != nil { + fail(e) + return + } + _ = json.NewEncoder(w).Encode(out) + case "/v1/plans": + if s.Gateway.Authority != nil { + fail(ErrConflict) + return + } + var in Plan + if !decode(&in) { + return + } + if e := s.Gateway.InstallPlan(r.Context(), in); e != nil { + fail(e) + return + } + _ = json.NewEncoder(w).Encode(in) + case "/v1/advance": + if s.Gateway.Authority != nil { + fail(ErrConflict) + return + } + var in AdvanceRequest + if !decode(&in) { + return + } + if e := s.Gateway.Advance(r.Context(), in.Expected, in.Grant); e != nil { + fail(e) + return + } + _ = json.NewEncoder(w).Encode(in) + case "/v1/execute", "/v1/reconcile": + var in ExecuteRequest + if !decode(&in) { + return + } + var out Receipt + var e error + if r.URL.Path == "/v1/execute" { + if in.Authorization != nil { + out, e = s.Gateway.ExecuteAuthorized(r.Context(), *in.Authorization, in.Request, in.Body) + } else { + out, e = s.Gateway.Execute(r.Context(), in.Request, in.Body) + } + } else { + out, e = s.Gateway.Reconcile(r.Context(), in.Request) + } + if e != nil { + fail(e) + return + } + _ = json.NewEncoder(w).Encode(out) + case "/v1/resolve": + var in Request + if !decode(&in) { + return + } + out, e := s.Gateway.Resolve(r.Context(), in) + if e != nil { + fail(e) + return + } + _ = json.NewEncoder(w).Encode(out) + case "/v1/observe": + var in struct { + Key string `json:"key"` + Version string `json:"version"` + } + if !decode(&in) { + return + } + if in.Key == "" { + http.Error(w, "gateway_bad_key", 400) + return + } + if e := s.Gateway.Ready(r.Context()); e != nil { + fail(e) + return + } + v, exists, e := s.Gateway.Backend.Observe(r.Context(), in.Key, in.Version) + if e != nil { + http.Error(w, "gateway_backend_read_failed", 503) + return + } + _ = json.NewEncoder(w).Encode(struct { + Version Version `json:"version"` + Exists bool `json:"exists"` + }{v, exists}) + default: + http.Error(w, "gateway_route_not_found", 404) + } +} + +type Client struct { + Endpoint, Token, Scope string + HTTP *http.Client +} + +func NewClient(endpoint, token, scope string) (*Client, error) { + u, e := url.Parse(endpoint) + if e != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" || len(token) < 32 || scope == "" { + return nil, errors.New("gateway_configuration_invalid") + } + if u.Scheme != "https" && (u.Scheme != "http" || (u.Hostname() != "127.0.0.1" && u.Hostname() != "::1")) { + return nil, errors.New("gateway_transport_requires_tls") + } + return &Client{endpoint, token, scope, &http.Client{Timeout: 2 * time.Minute, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("gateway_redirect_refused") }}}, nil +} +func (c *Client) call(ctx context.Context, path string, in, out any) error { + method := "GET" + var body io.Reader + if in != nil { + raw, e := json.Marshal(in) + if e != nil { + return ErrConflict + } + method = "POST" + body = struct{ io.Reader }{bytes.NewReader(raw)} + } + req, e := http.NewRequestWithContext(ctx, method, c.Endpoint+path, body) + if e != nil { + return errors.New("gateway_request_invalid") + } + req.Header.Set("Authorization", "Bearer "+c.Token) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-KG-Gateway-Protocol", Protocol) + req.Header.Set("X-KG-Gateway-Scope", c.Scope) + resp, e := c.HTTP.Do(req) + if e != nil { + return errors.New("gateway_unreachable_or_outcome_unknown") + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 128)) + reason := strings.TrimSpace(string(raw)) + switch reason { + case ErrAuthority.Error(): + return ErrAuthority + case ErrAuthorityPrivileges.Error(): + return ErrAuthorityPrivileges + case ErrEpoch.Error(): + return ErrEpoch + case ErrConflict.Error(): + return ErrConflict + case ErrHeld.Error(): + return ErrHeld + case ErrPersistence.Error(): + return ErrPersistence + case ErrIsolation.Error(): + return ErrIsolation + } + return errors.New("gateway_request_refused") + } + if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out) != nil { + return errors.New("gateway_response_invalid") + } + return nil +} +func (c *Client) Ready(ctx context.Context) error { return c.checkReady(ctx, false) } +func (c *Client) ReadyForAcquisition(ctx context.Context) error { return c.checkReady(ctx, true) } +func (c *Client) checkReady(ctx context.Context, automatic bool) error { + var caps Capabilities + if e := c.call(ctx, "/v1/capabilities", nil, &caps); e != nil { + return e + } + if automatic && !caps.AutomaticGrants { + return ErrAuthority + } + if caps.Protocol != Protocol || caps.Scope != c.Scope || !caps.Ready { + return errors.New("gateway_protocol_or_target_mismatch") + } + return nil +} +func (c *Client) InstallPlan(ctx context.Context, p Plan) error { + var out Plan + if e := c.call(ctx, "/v1/plans", p, &out); e != nil { + return e + } + want, _ := json.Marshal(p) + got, _ := json.Marshal(out) + if string(want) != string(got) { + return ErrConflict + } + return nil +} +func (c *Client) Advance(ctx context.Context, expected int64, g Grant) error { + var out AdvanceRequest + in := AdvanceRequest{expected, g} + if e := c.call(ctx, "/v1/advance", in, &out); e != nil { + return e + } + if in != out { + return ErrConflict + } + return nil +} +func (c *Client) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) { + var out Receipt + e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body}, &out) + if e == nil && (out.Request != r || !out.valid()) { + e = ErrConflict + } + return out, e +} +func (c *Client) Reconcile(ctx context.Context, r Request) (Receipt, error) { + var out Receipt + e := c.call(ctx, "/v1/reconcile", ExecuteRequest{Request: r}, &out) + if e == nil && (out.Request != r || !out.valid()) { + e = ErrConflict + } + return out, e +} +func (c *Client) Observe(ctx context.Context, key, id string) (Version, bool, error) { + var out struct { + Version Version `json:"version"` + Exists bool `json:"exists"` + } + e := c.call(ctx, "/v1/observe", struct { + Key string `json:"key"` + Version string `json:"version"` + }{key, id}, &out) + return out.Version, out.Exists, e +} diff --git a/internal/objectgateway/identity_integration_test.go b/internal/objectgateway/identity_integration_test.go new file mode 100644 index 0000000..7d7ba15 --- /dev/null +++ b/internal/objectgateway/identity_integration_test.go @@ -0,0 +1,50 @@ +package objectgateway_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + g "knowledge-governance/internal/objectgateway" +) + +func TestGatewayTargetIdentityOnEveryCall(t *testing.T) { + ctx, pool, backend := fixture(t) + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))} + gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil { + t.Fatal(e) + } + if e := gw.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + token := strings.Repeat("a", 32) + server := httptest.NewServer(g.HTTPServer{Gateway: gw, DataToken: token, ControlToken: strings.Repeat("b", 32)}) + defer server.Close() + wrong, _ := g.NewClient(server.URL, token, "wrong-target") + req := request(t.Name(), grant) + receipt, e := wrong.Execute(ctx, req, []byte("gateway fixture "+t.Name())) + if e == nil { + cleanup(t, ctx, gw, receipt) + t.Fatal("mutation accepted with wrong target identity") + } + var n int + if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gw.Scope).Scan(&n) != nil || n != 0 { + t.Fatal("wrong target wrote intent") + } +} +func TestGatewayClientRejectsIncompleteReceipt(t *testing.T) { + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("owner")), Plan: g.Digest([]byte("plan"))} + req := request("receipt", grant) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(g.Receipt{Request: req, State: "committed"}) + })) + defer server.Close() + c, _ := g.NewClient(server.URL, strings.Repeat("a", 32), "scope") + if _, e := c.Execute(context.Background(), req, []byte("gateway fixture receipt")); e == nil { + t.Fatal("committed receipt with missing exact version accepted") + } +} diff --git a/internal/objectgateway/persistence_integration_test.go b/internal/objectgateway/persistence_integration_test.go new file mode 100644 index 0000000..0ed69c9 --- /dev/null +++ b/internal/objectgateway/persistence_integration_test.go @@ -0,0 +1,44 @@ +package objectgateway_test + +import ( + "errors" + g "knowledge-governance/internal/objectgateway" + "testing" +) + +func TestGatewayIntentCommitFailureNeverSends(t *testing.T) { + ctx, pool, real := fixture(t) + backend := &stoppedBeforeSend{Backend: real} + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))} + gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil { + t.Fatal(e) + } + if e := gateway.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + // Deferred failure fires on COMMIT, not on INSERT. This exercises the uncertain + // persistence boundary while confirming no provider request is sent. + _, e := pool.Exec(ctx, `CREATE FUNCTION kg_gateway_fixture_commit_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected durable commit failure'; END $$; + CREATE CONSTRAINT TRIGGER kg_gateway_fixture_commit_failure AFTER INSERT ON kg_object_gateway_intents DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION kg_gateway_fixture_commit_failure()`) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { + _, e := pool.Exec(ctx, `DROP TRIGGER kg_gateway_fixture_commit_failure ON kg_object_gateway_intents; DROP FUNCTION kg_gateway_fixture_commit_failure()`) + if e != nil { + t.Error(e) + } + }) + req := request(t.Name(), grant) + if _, e = gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrPersistence) { + t.Fatal("commit error not propagated", e) + } + if backend.calls != 0 { + t.Fatal("object request after failed journal commit") + } + var n int + if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 { + t.Fatal("failed transaction left intent") + } +} diff --git a/internal/objectgateway/plan.go b/internal/objectgateway/plan.go new file mode 100644 index 0000000..56bbdda --- /dev/null +++ b/internal/objectgateway/plan.go @@ -0,0 +1,78 @@ +package objectgateway + +import ( + "context" + "encoding/json" +) + +// Plan is frozen by the authenticated controller, not by the data client. Digest +// is the upstream write-set digest; the controller must verify that write set. +// This allowlist prevents a valid epoch from authorizing arbitrary extra keys. +type Plan struct { + Digest string `json:"digest"` + Owner string `json:"owner"` + Objects []ObjectBinding `json:"objects"` +} +type ObjectBinding struct { + Operation string `json:"operation"` + Key string `json:"key"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` +} + +func (p Plan) valid() bool { + if !validHash(p.Digest) || !validHash(p.Owner) { + return false + } + keys, ops := map[string]bool{}, map[string]bool{} + for _, o := range p.Objects { + r := Request{Grant: Grant{1, p.Owner, p.Digest}, Operation: o.Operation, Key: o.Key, Action: "create", SHA256: o.SHA256, Bytes: o.Bytes} + if r.validate() != nil || keys[o.Key] || ops[o.Operation] { + return false + } + keys[o.Key] = true + ops[o.Operation] = true + } + return true +} +func (p Plan) allows(r Request) bool { + op := r.Operation + if r.Action == "delete" { + op = r.CreatedBy + } + if p.Owner != r.Grant.Owner || p.Digest != r.Grant.Plan { + return false + } + for _, o := range p.Objects { + if o.Operation == op && o.Key == r.Key && o.SHA256 == r.SHA256 && o.Bytes == r.Bytes { + return true + } + } + return false +} +func (g *Gateway) InstallPlan(ctx context.Context, p Plan) error { + if !p.valid() { + return ErrConflict + } + for _, o := range p.Objects { + if o.Bytes > g.MaxBytes { + return ErrConflict + } + } + if e := g.Ready(ctx); e != nil { + return e + } + raw, _ := json.Marshal(p) + _, e := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_plans(scope,plan,body) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, p.Digest, raw) + if e != nil { + return ErrPersistence + } + var found []byte + if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, p.Digest).Scan(&found) != nil { + return ErrPersistence + } + if string(found) != string(raw) { + return ErrConflict + } + return nil +} diff --git a/internal/objectgateway/plan_integration_test.go b/internal/objectgateway/plan_integration_test.go new file mode 100644 index 0000000..bb7661a --- /dev/null +++ b/internal/objectgateway/plan_integration_test.go @@ -0,0 +1,37 @@ +package objectgateway_test + +import ( + "errors" + g "knowledge-governance/internal/objectgateway" + "testing" +) + +func TestGatewayGrantCannotAuthorizeUnplannedKey(t *testing.T) { + ctx, pool, backend := fixture(t) + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))} + gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e := gateway.Advance(ctx, 0, grant); !errors.Is(e, g.ErrConflict) { + t.Fatal("unregistered plan grant", e) + } + allowed := request("allowed-plan-key", grant) + plan := planFor(grant, allowed) + if e := gateway.InstallPlan(ctx, plan); e != nil { + t.Fatal(e) + } + changed := planFor(grant, request("extra-plan-key", grant)) + if e := gateway.InstallPlan(ctx, changed); !errors.Is(e, g.ErrConflict) { + t.Fatal("immutable plan rebound", e) + } + if e := gateway.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + req := request(t.Name(), grant) + if _, e := gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrConflict) { + t.Fatal("unplanned key admitted", e) + } + receipt, e := gateway.Execute(ctx, allowed, []byte("gateway fixture allowed-plan-key")) + if e != nil || receipt.State != "committed" { + t.Fatal("valid planned key refused", e) + } + cleanup(t, ctx, gateway, receipt) +} diff --git a/internal/objectgateway/protocol.go b/internal/objectgateway/protocol.go new file mode 100644 index 0000000..b120fc2 --- /dev/null +++ b/internal/objectgateway/protocol.go @@ -0,0 +1,109 @@ +// Package objectgateway implements non-preemptive object write ownership. +package objectgateway + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" +) + +const Protocol = "kg-object-gateway-v1" + +var ( + ErrHeld = errors.New("gateway_held_unresolved_intent") + ErrEpoch = errors.New("gateway_epoch_or_plan_mismatch") + ErrConflict = errors.New("gateway_identity_conflict") + ErrPersistence = errors.New("gateway_persistence_unavailable") + ErrAuthority = errors.New("gateway_control_authority_unavailable") + ErrAuthorityPrivileges = errors.New("gateway_control_reader_privileges_invalid") + ErrIsolation = errors.New("gateway_exclusive_writer_evidence_unverified") + ErrRejected = errors.New("gateway_create_rejected_existing_object") +) + +type Grant struct { + Epoch int64 `json:"epoch"` + Owner string `json:"owner"` + Plan string `json:"plan"` +} +type Version struct { + ID string `json:"version_id"` + Owner string `json:"owner"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + ETag string `json:"etag"` +} +type Request struct { + Grant Grant `json:"grant"` + Operation string `json:"operation"` + Key string `json:"key"` + Action string `json:"action"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + CreatedBy string `json:"created_by,omitempty"` + Version Version `json:"version,omitempty"` +} +type Receipt struct { + Request Request `json:"request"` + State string `json:"state"` + Version Version `json:"version"` +} + +// Backend never retries mutations. List is complete and scoped to exactly key. +// Not finding a PUT is never evidence that its request cannot still arrive. +type Backend interface { + Ready(context.Context) error + Observe(context.Context, string, string) (Version, bool, error) + List(context.Context, string) ([]Version, error) + Put(context.Context, string, []byte, string) (Version, error) + Delete(context.Context, string, Version) error +} +type IsolationVerifier interface{ Verify(context.Context) error } + +func Digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } +func validHash(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == 32 && strings.ToLower(s) == s +} +func (g Grant) valid() bool { return g.Epoch > 0 && validHash(g.Owner) && validHash(g.Plan) } +func (r Request) validate() error { + if !r.Grant.valid() || !validHash(r.Operation) || r.Key == "" || len(r.Key) > 2048 || strings.ContainsAny(r.Key, "\x00\r\n") || strings.HasPrefix(r.Key, "/") || !validHash(r.SHA256) || r.Bytes < 0 { + return ErrConflict + } + if r.Action != "create" && r.Action != "delete" { + return ErrConflict + } + if r.Action == "create" && (r.CreatedBy != "" || r.Version != (Version{})) { + return ErrConflict + } + if r.Action == "delete" && (!validHash(r.CreatedBy) || !validVersion(r.Version) || r.Version.SHA256 != r.SHA256 || r.Version.Bytes != r.Bytes) { + return ErrConflict + } + return nil +} +func validVersion(v Version) bool { + return v.ID != "" && v.ID != "null" && validHash(v.Owner) && validHash(v.SHA256) && v.Bytes >= 0 && v.ETag != "" +} + +// Claim binds backend ownership to the complete immutable gateway request. +func (r Request) Claim() string { return r.token() } +func (r Request) token() string { raw, _ := json.Marshal(r); return Digest(raw) } +func (r Receipt) valid() bool { + if r.Request.validate() != nil { + return false + } + switch r.State { + case "committed": + return r.Request.Action == "create" && r.Request.matches(r.Version) + case "rejected": + return r.Request.Action == "create" && r.Version == (Version{}) + case "removed": + return r.Request.Action == "delete" && r.Version == r.Request.Version + } + return false +} +func (r Request) matches(v Version) bool { + return validVersion(v) && v.Owner == r.token() && v.Bytes == r.Bytes && v.SHA256 == r.SHA256 +} diff --git a/internal/objectgateway/s3.go b/internal/objectgateway/s3.go new file mode 100644 index 0000000..f871c81 --- /dev/null +++ b/internal/objectgateway/s3.go @@ -0,0 +1,102 @@ +package objectgateway + +import ( + "bytes" + "context" + "errors" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +// S3Backend is for independently versioned S3/MinIO stores. No SDK mutation retries. +type S3Backend struct { + Client *s3.Client + Bucket string + MaxBytes int64 +} + +func (s *S3Backend) Ready(ctx context.Context) error { + r, e := s.Client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{Bucket: aws.String(s.Bucket)}) + if e != nil || string(r.Status) != "Enabled" { + return errors.New("gateway_backend_versioning_unverified") + } + return nil +} +func (s *S3Backend) Observe(ctx context.Context, key, id string) (Version, bool, error) { + req := &s3.GetObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key)} + if id != "" { + req.VersionId = aws.String(id) + } + got, e := s.Client.GetObject(ctx, req) + if e != nil { + var api smithy.APIError + if errors.As(e, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NoSuchVersion") { + return Version{}, false, nil + } + return Version{}, false, errors.New("gateway_backend_read_failed") + } + defer got.Body.Close() + if got.ContentLength == nil || *got.ContentLength < 0 || *got.ContentLength > s.MaxBytes { + return Version{}, false, errors.New("gateway_backend_size_invalid") + } + body, e := io.ReadAll(io.LimitReader(got.Body, s.MaxBytes+1)) + if e != nil || int64(len(body)) != *got.ContentLength { + return Version{}, false, errors.New("gateway_backend_body_incomplete") + } + v := Version{aws.ToString(got.VersionId), got.Metadata["kg-release-owner"], Digest(body), int64(len(body)), aws.ToString(got.ETag)} + // Existing unowned objects are observable but cannot be claimed/deleted. + if v.ID == "" || v.ID == "null" || v.ETag == "" || id != "" && v.ID != id { + return Version{}, false, errors.New("gateway_backend_version_invalid") + } + return v, true, nil +} +func (s *S3Backend) List(ctx context.Context, key string) ([]Version, error) { + got, e := s.Client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(s.Bucket), Prefix: aws.String(key), MaxKeys: aws.Int32(1000)}) + if e != nil || aws.ToBool(got.IsTruncated) { + return nil, errors.New("gateway_backend_listing_incomplete") + } + var versions []Version + for _, m := range got.DeleteMarkers { + if aws.ToString(m.Key) == key { + return nil, errors.New("gateway_backend_delete_marker_conflict") + } + } + for _, v := range got.Versions { + if aws.ToString(v.Key) == key { + one, exists, e := s.Observe(ctx, key, aws.ToString(v.VersionId)) + if e != nil || !exists { + return nil, errors.New("gateway_backend_listing_changed") + } + versions = append(versions, one) + } + } + return versions, nil +} +func (s *S3Backend) Put(ctx context.Context, key string, body []byte, owner string) (Version, error) { + // SDK hashing requires a seekable body. RetryMaxAttempts=1 prevents SDK + // retries; PUT carries no HTTP idempotency header or replay authorization. + reader := bytes.NewReader(body) + got, e := s.Client.PutObject(v4.SetPayloadHash(ctx, Digest(body)), &s3.PutObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(int64(len(body))), Metadata: map[string]string{"kg-release-owner": owner}}, func(o *s3.Options) { o.RetryMaxAttempts = 1 }) + if e != nil || aws.ToString(got.VersionId) == "" { + return Version{}, ErrHeld + } + v, exists, e := s.Observe(ctx, key, aws.ToString(got.VersionId)) + if e != nil || !exists { + return Version{}, ErrHeld + } + return v, nil +} +func (s *S3Backend) Delete(ctx context.Context, key string, v Version) error { + if !validVersion(v) { + return ErrConflict + } + _, e := s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), VersionId: aws.String(v.ID)}, func(o *s3.Options) { o.RetryMaxAttempts = 1 }) + if e != nil { + return ErrHeld + } + return nil +} diff --git a/internal/objectgateway/schema.sql b/internal/objectgateway/schema.sql new file mode 100644 index 0000000..da3da28 --- /dev/null +++ b/internal/objectgateway/schema.sql @@ -0,0 +1,74 @@ +-- Independent governance journal; never install in a WeKnora business schema. +CREATE TABLE IF NOT EXISTS kg_object_gateway_plans ( + scope text NOT NULL, + plan text NOT NULL, + body bytea NOT NULL, + PRIMARY KEY(scope,plan) +); +CREATE TABLE IF NOT EXISTS kg_object_gateway_scopes ( + scope text PRIMARY KEY, + epoch bigint NOT NULL DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS kg_object_gateway_grants ( + scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope), + epoch bigint NOT NULL CHECK (epoch > 0), + owner text NOT NULL, + plan text NOT NULL, + PRIMARY KEY(scope,epoch) +); +CREATE TABLE IF NOT EXISTS kg_object_gateway_intents ( + scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope), + operation text NOT NULL, + key text NOT NULL, + request bytea NOT NULL, + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(scope,operation) +); +CREATE INDEX IF NOT EXISTS kg_object_gateway_key ON kg_object_gateway_intents(scope,key); +CREATE TABLE IF NOT EXISTS kg_object_gateway_receipts ( + scope text NOT NULL, + operation text NOT NULL, + receipt bytea NOT NULL, + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(scope,operation), + FOREIGN KEY(scope,operation) REFERENCES kg_object_gateway_intents(scope,operation) +); +CREATE TABLE IF NOT EXISTS kg_object_gateway_deliveries ( + scope text NOT NULL, + job text NOT NULL, + attempt_epoch bigint NOT NULL CHECK(attempt_epoch>0), + claim bytea NOT NULL, + grant_epoch bigint NOT NULL, + grant_body bytea NOT NULL, + PRIMARY KEY(scope,job,attempt_epoch), + FOREIGN KEY(scope,grant_epoch) REFERENCES kg_object_gateway_grants(scope,epoch) +); +CREATE OR REPLACE FUNCTION kg_object_gateway_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'object gateway evidence is immutable'; +END; +$$; +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_deliveries_immutable') THEN + CREATE TRIGGER kg_object_gateway_deliveries_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_deliveries FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_plans_immutable') THEN + CREATE TRIGGER kg_object_gateway_plans_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_plans FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_intents_immutable') THEN + CREATE TRIGGER kg_object_gateway_intents_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_intents FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_receipts_immutable') THEN + CREATE TRIGGER kg_object_gateway_receipts_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_receipts FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_grants_immutable') THEN + CREATE TRIGGER kg_object_gateway_grants_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_grants FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable(); + END IF; +END $$; +DO $$ DECLARE t text; BEGIN + FOREACH t IN ARRAY ARRAY['plans','grants','intents','receipts','deliveries'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_'||t||'_no_truncate') THEN + EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION kg_object_gateway_immutable()', 'kg_object_gateway_'||t||'_no_truncate', 'kg_object_gateway_'||t); + END IF; + END LOOP; +END $$; diff --git a/internal/objectgateway/takeover.go b/internal/objectgateway/takeover.go new file mode 100644 index 0000000..35f86e0 --- /dev/null +++ b/internal/objectgateway/takeover.go @@ -0,0 +1,60 @@ +package objectgateway + +import ( + "context" + "encoding/json" + "errors" + "github.com/jackc/pgx/v5" +) + +// Resolve preserves the immutable send epoch and backend claim of an operation. +// An absent intent is not a claim about an earlier send; Execute still reserves +// atomically and refuses stale authorizations before any new side effect. +func (g *Gateway) Resolve(ctx context.Context, desired Request) (Request, error) { + if e := desired.validate(); e != nil { + return Request{}, e + } + if e := g.Ready(ctx); e != nil { + return Request{}, e + } + var raw []byte + e := g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, desired.Operation).Scan(&raw) + if errors.Is(e, pgx.ErrNoRows) { + return desired, nil + } + if e != nil { + return Request{}, ErrPersistence + } + var original Request + if json.Unmarshal(raw, &original) != nil || original.validate() != nil { + return Request{}, ErrPersistence + } + if !sameOperation(original, desired) || original.Grant.Epoch > desired.Grant.Epoch { + return Request{}, ErrConflict + } + return original, nil +} +func sameOperation(a, b Request) bool { a.Grant.Epoch = b.Grant.Epoch; return a == b } + +func (g *Gateway) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) { + if !authorization.valid() || authorization.Owner != r.Grant.Owner || authorization.Plan != r.Grant.Plan || authorization.Epoch < r.Grant.Epoch { + return Receipt{}, ErrEpoch + } + return g.execute(ctx, r, body, &authorization) +} +func (c *Client) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) { + var out Receipt + e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body, Authorization: &authorization}, &out) + if e == nil && (out.Request != r || !out.valid()) { + e = ErrConflict + } + return out, e +} +func (c *Client) Resolve(ctx context.Context, desired Request) (Request, error) { + var out Request + e := c.call(ctx, "/v1/resolve", desired, &out) + if e == nil && (out.validate() != nil || !sameOperation(out, desired) || out.Grant.Epoch > desired.Grant.Epoch) { + e = ErrConflict + } + return out, e +} diff --git a/internal/objectgateway/takeover_integration_test.go b/internal/objectgateway/takeover_integration_test.go new file mode 100644 index 0000000..1f90d3f --- /dev/null +++ b/internal/objectgateway/takeover_integration_test.go @@ -0,0 +1,111 @@ +package objectgateway_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + g "knowledge-governance/internal/objectgateway" +) + +func TestGatewayNewEpochPreservesOriginalSendIdentity(t *testing.T) { + ctx, pool, backend := fixture(t) + gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + first := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("takeover-plan"))} + sent, unsent := request("takeover-sent", first), request("takeover-unsent", first) + if e := gateway.InstallPlan(ctx, planFor(first, sent, unsent)); e != nil { + t.Fatal(e) + } + if e := gateway.Advance(ctx, 0, first); e != nil { + t.Fatal(e) + } + old, e := gateway.Execute(ctx, sent, []byte("gateway fixture takeover-sent")) + if e != nil { + t.Fatal(e) + } + // Always clean only this test's exact original version, including on RED. + cleanupGrant := first + defer func() { copy := old; copy.Request.Grant = cleanupGrant; cleanup(t, ctx, gateway, copy) }() + var original []byte + if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&original); e != nil { + t.Fatal(e) + } + next := first + next.Epoch++ + if e = gateway.Advance(ctx, 1, next); e != nil { + t.Fatal(e) + } + cleanupGrant = next + data, control := g.Digest([]byte("takeover-data")), g.Digest([]byte("takeover-control")) + server := httptest.NewServer(g.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: control}) + defer server.Close() + post := func(path string, in any) (int, []byte) { + raw, _ := json.Marshal(in) + req, _ := http.NewRequestWithContext(ctx, "POST", server.URL+path, bytes.NewReader(raw)) + req.Header.Set("Authorization", "Bearer "+data) + req.Header.Set("X-KG-Gateway-Protocol", g.Protocol) + req.Header.Set("X-KG-Gateway-Scope", gateway.Scope) + resp, e := http.DefaultClient.Do(req) + if e != nil { + t.Fatal(e) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return resp.StatusCode, body + } + code, body := post("/v1/execute", map[string]any{"request": sent, "authorization": next, "body": []byte("gateway fixture takeover-sent")}) + if code != 200 { + t.Fatalf("new authorization cannot replay immutable original request: status=%d", code) + } + var replay g.Receipt + if json.Unmarshal(body, &replay) != nil || replay != old { + t.Fatal("replay rewrote original receipt") + } + desired := sent + desired.Grant = next + code, body = post("/v1/resolve", desired) + var resolved g.Request + if code != 200 || json.Unmarshal(body, &resolved) != nil || resolved != sent { + t.Fatal("original send identity not resolved", code) + } + changed := sent + changed.SHA256 = g.Digest([]byte("different")) + if code, _ = post("/v1/resolve", changed); code == 200 { + t.Fatal("operation rebound body") + } + if code, _ = post("/v1/execute", map[string]any{"request": unsent, "authorization": first, "body": []byte("gateway fixture takeover-unsent")}); code == 200 { + t.Fatal("stale epoch created unsent key") + } + unsent.Grant = next + code, body = post("/v1/execute", map[string]any{"request": unsent, "authorization": next, "body": []byte("gateway fixture takeover-unsent")}) + var created g.Receipt + if code != 200 || json.Unmarshal(body, &created) != nil || created.State != "committed" { + t.Fatal("new epoch did not continue unsent key", code) + } + cleanup(t, ctx, gateway, created) + var after []byte + if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&after); e != nil || !bytes.Equal(after, original) { + t.Fatal("immutable intent changed", e) + } + versions, e := backend.List(ctx, sent.Key) + if e != nil || len(versions) != 1 || versions[0] != old.Version { + t.Fatal("replay created extra version", e) + } + // Deletion is a new send in the current epoch, not a rewritten create intent. + deletion := sent + deletion.Grant = next + deletion.Operation = g.Digest([]byte(sent.Operation + "delete")) + deletion.Action = "delete" + deletion.CreatedBy = sent.Operation + deletion.Version = old.Version + code, _ = post("/v1/execute", map[string]any{"request": deletion, "authorization": next}) + if code != 200 { + t.Fatal("new epoch exact cleanup refused", code) + } + // deferred cleanup replays the same already settled delete request below. + old.Request.Grant = next + t.Log("new epoch replay retained original request/receipt/claim; unsent key committed once; stale request rejected") +} diff --git a/internal/objectgateway/unknown_integration_test.go b/internal/objectgateway/unknown_integration_test.go new file mode 100644 index 0000000..b5232f0 --- /dev/null +++ b/internal/objectgateway/unknown_integration_test.go @@ -0,0 +1,69 @@ +package objectgateway_test + +import ( + "context" + "errors" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + g "knowledge-governance/internal/objectgateway" +) + +type stoppedBeforeSend struct { + g.Backend + calls int +} + +func (b *stoppedBeforeSend) Put(context.Context, string, []byte, string) (g.Version, error) { + b.calls++ + return g.Version{}, errors.New("injected uncertain send with no visible version") +} +func TestGatewayAbsenceNeverSettlesUnknown(t *testing.T) { + ctx, pool, real := fixture(t) + backend := &stoppedBeforeSend{Backend: real} + grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))} + gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil { + t.Fatal(e) + } + if e := gateway.Advance(ctx, 0, grant); e != nil { + t.Fatal(e) + } + req := request(t.Name(), grant) + body := []byte("gateway fixture " + t.Name()) + if _, e := gateway.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) { + t.Fatal(e) + } + restarted := *gateway + for i := 0; i < 2; i++ { + if _, e := restarted.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) { + t.Fatal("absent version treated as terminal", e) + } + } + if backend.calls != 1 { + t.Fatal("PUT replayed", backend.calls) + } + next := grant + next.Epoch++ + if e := restarted.Advance(ctx, 1, next); !errors.Is(e, g.ErrHeld) { + t.Fatal("unknown operation allowed epoch handover", e) + } + var n int + if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 { + t.Fatal("absence wrote false terminal receipt") + } +} +func TestGatewayReadOnlyPersistenceBlocksSend(t *testing.T) { + ctx, pool, backend := fixture(t) + cfg := pool.Config() + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + other, e := pgxpool.NewWithConfig(ctx, cfg) + if e != nil { + t.Fatal(e) + } + defer other.Close() + gateway := &g.Gateway{Pool: other, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20} + if e = gateway.Ready(ctx); !errors.Is(e, g.ErrPersistence) { + t.Fatal("read-only persistence accepted", e) + } +} diff --git a/internal/releaseadapter/v071/gateway_authority.go b/internal/releaseadapter/v071/gateway_authority.go new file mode 100644 index 0000000..4175765 --- /dev/null +++ b/internal/releaseadapter/v071/gateway_authority.go @@ -0,0 +1,131 @@ +package v071 + +import ( + "context" + "errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" + "path/filepath" +) + +// GatewayGovernanceAuthority runs only in the trusted gateway control domain. +// Its governance connection is SELECT-only; the ordinary worker never receives +// the gateway control credential or backend write credential. Archive contents +// are trusted only after their immutable digest is found in the original +// persisted prebackup event belonging to the currently leased attempt. +type GatewayGovernanceAuthority struct { + Pool *pgxpool.Pool + ArchiveRoot string + TargetID string + Provider string + Bucket string +} + +func (a *GatewayGovernanceAuthority) Ready(ctx context.Context) error { + if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" { + return objectgateway.ErrAuthority + } + var safe bool + e := a.Pool.QueryRow(ctx, `SELECT NOT rolsuper AND NOT rolcreatedb AND NOT rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&safe) + if e != nil || !safe { + return objectgateway.ErrAuthorityPrivileges + } + for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} { + e = a.Pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT') AND NOT has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&safe) + if e != nil || !safe { + return objectgateway.ErrAuthorityPrivileges + } + } + return nil +} +func (a *GatewayGovernanceAuthority) Load(ctx context.Context, c objectgateway.Claim) (objectgateway.Plan, error) { + if e := a.Ready(ctx); e != nil { + return objectgateway.Plan{}, e + } + if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" { + return objectgateway.Plan{}, objectgateway.ErrAuthority + } + recovery, err := a.verifyRecoveryClaim(ctx, c) + if err != nil { + return objectgateway.Plan{}, err + } + var scope, backup, snapshot string + var matches int + e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',count(*) OVER() + FROM library_jobs j + JOIN library_operations o ON o.id=j.operation_id AND o.target_id=j.target_id + JOIN library_attempts a ON a.operation_id=j.operation_id AND a.epoch=j.attempt_epoch AND a.job_id=j.id + JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id AND v.scope=j.scope AND v.kind='production_prebackup' AND v.target_id=j.target_id + WHERE j.id=$1 AND j.operation_id=$2 AND j.attempt_epoch=$3 AND j.lease_owner=$4 AND j.target_id=$6 + AND j.lease_until>clock_timestamp() AND (j.state='running' OR $7) + AND a.owner=$4 AND a.state IN ('claimed','running') AND a.lease_until>clock_timestamp() + AND v.evidence->>'writeset_digest'=$5 + AND v.evidence->>'job'=j.id AND v.evidence->>'backup_snapshot'=v.snapshot_id + AND (o.target_snapshot_id='' OR o.target_snapshot_id=v.evidence->>'writeset_target_snapshot') + AND NOT EXISTS(SELECT 1 FROM library_attempts newer WHERE newer.operation_id=a.operation_id AND newer.epoch>a.epoch)`, c.JobID, c.OperationID, c.AttemptEpoch, c.Owner, c.PlanDigest, a.TargetID, recovery).Scan(&scope, &backup, &snapshot, &matches) + if errors.Is(e, pgx.ErrNoRows) { + return objectgateway.Plan{}, objectgateway.ErrEpoch + } + if e != nil { + return objectgateway.Plan{}, objectgateway.ErrPersistence + } + if matches != 1 { + return objectgateway.Plan{}, objectgateway.ErrConflict + } + set, e := LoadWriteSet(a.ArchiveRoot, c.PlanDigest) + if e != nil || set == nil { + return objectgateway.Plan{}, objectgateway.ErrConflict + } + if set.Version != libraryrelease.MigrationWriteSetVersion || set.JobID != c.JobID || set.Target != scope || set.BackupSnapshotID != backup || set.TargetSnapshotID != snapshot { + return objectgateway.Plan{}, objectgateway.ErrConflict + } + for _, obj := range set.Objects { + if obj.Provider != a.Provider || obj.Bucket != a.Bucket { + return objectgateway.Plan{}, objectgateway.ErrConflict + } + } + return ObjectGatewayPlan(*set) +} + +// Recovery authorization does not replace c.JobID/c.OperationID (the original +// send domain). Its separate queue and operation must both be durable, leased, +// and bound to the same before-image, source, target and frozen plan. +func (a *GatewayGovernanceAuthority) verifyRecoveryClaim(ctx context.Context, c objectgateway.Claim) (bool, error) { + if c.RecoveryJobID == "" && c.RecoveryOperationID == "" { + return false, nil + } + if c.RecoveryJobID != governance.LibraryRecoveryQueueID(c.JobID) { + return false, objectgateway.ErrConflict + } + var scope, backup, source, recorded string + e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',o.request_digest + FROM library_jobs j JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id + AND v.target_id=j.target_id AND v.scope=j.scope AND v.kind='production_prebackup' + JOIN jobs q ON q.id=$3 AND q.type='library.recover' AND q.state='running' + JOIN library_operations o ON o.id=$4 AND o.kind='recover' AND o.scope=j.scope AND o.target_id=j.target_id + WHERE j.id=$1 AND j.operation_id=$2 AND j.target_id=$5 + AND q.payload->>'scope'=j.scope AND q.payload->>'window_job_id'=j.id + AND COALESCE(NULLIF(q.payload->>'source_snapshot_id',''),v.evidence->>'writeset_target_snapshot')=v.evidence->>'writeset_target_snapshot' + AND COALESCE(NULLIF(q.payload->>'target_id',''),j.target_id)=j.target_id + AND COALESCE(NULLIF(q.payload->>'original_operation_id',''),j.operation_id)=j.operation_id + AND COALESCE(NULLIF(q.payload->>'backup_snapshot_id',''),v.snapshot_id)=v.snapshot_id + AND COALESCE(NULLIF(q.payload->>'plan_digest',''),$7)=$7 + AND q.idempotency_key='library.recover:'||j.id AND q.lease_until>clock_timestamp() + AND q.lease_owner||'#'||q.id||'#'||q.attempt::text=$6 + AND o.operation_key=q.id AND o.target_snapshot_id=v.snapshot_id + AND v.evidence->>'writeset_digest'=$7`, c.JobID, c.OperationID, c.RecoveryJobID, c.RecoveryOperationID, a.TargetID, c.Owner, c.PlanDigest).Scan(&scope, &backup, &source, &recorded) + if errors.Is(e, pgx.ErrNoRows) { + return false, objectgateway.ErrConflict + } + if e != nil { + return false, objectgateway.ErrPersistence + } + digest, e := governance.LibraryRecoveryDigest(scope, c.RecoveryJobID, c.JobID, c.OperationID, a.TargetID, source, backup, c.PlanDigest) + if e != nil || recorded != digest || c.RecoveryOperationID != governance.LibraryOperationID(scope, governance.LibraryJobRecover, c.RecoveryJobID) { + return false, objectgateway.ErrConflict + } + return true, nil +} diff --git a/internal/releaseadapter/v071/gateway_authority_integration_test.go b/internal/releaseadapter/v071/gateway_authority_integration_test.go new file mode 100644 index 0000000..f135849 --- /dev/null +++ b/internal/releaseadapter/v071/gateway_authority_integration_test.go @@ -0,0 +1,236 @@ +package v071 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/platform/postgres/pgtest" + "net/http/httptest" + "net/url" + "os" + "sync/atomic" + "testing" + "time" +) + +type gatewayCountBackend struct { + objectgateway.Backend + puts atomic.Int64 +} + +func (b *gatewayCountBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) { + b.puts.Add(1) + return b.Backend.Put(ctx, key, body, owner) +} + +func gatewayReadOnlyGovernance(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) *pgxpool.Pool { + t.Helper() + random := make([]byte, 24) + if _, e := rand.Read(random); e != nil { + t.Fatal(e) + } + suffix := hex.EncodeToString(random) + role := "kg_gateway_reader_" + suffix[:12] + // This role is created only in the explicitly disposable fixture cluster. + if _, e := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); e != nil { + t.Fatal("create isolated reader role failed") + } + var schema string + if admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema) != nil { + t.Fatal("fixture schema") + } + t.Cleanup(func() { + cleanup := context.Background() + if _, e := admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize()); e != nil { + t.Errorf("reader role privilege cleanup failed") + } + if _, e := admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize()); e != nil { + t.Errorf("reader role cleanup failed") + } + }) + if _, e := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); e != nil { + t.Fatal("reader schema grant") + } + if _, e := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); e != nil { + t.Fatal("reader table grant") + } + parsed, e := url.Parse(dsn) + if e != nil { + t.Fatal("fixture DSN") + } + parsed.User = url.UserPassword(role, suffix) + config, e := pgxpool.ParseConfig(parsed.String()) + if e != nil { + t.Fatal("reader config") + } + config.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + pool, e := pgxpool.NewWithConfig(ctx, config) + if e != nil { + t.Fatal("reader connect") + } + t.Cleanup(pool.Close) + t.Log("isolated SELECT-only governance role created; exact role cleanup registered:", role) + return pool +} + +// This is real authority/PG/HTTP/storage integration, not the formal worker +// process test: no grant decorator or control credential is given to the target. +func TestContractGatewayAuthorityFrozenPlanIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + ctx := fx.ctx + dsn := pgtest.IsolatedDatabaseURL(t, os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")) + governancePool, e := pgxpool.New(ctx, dsn) + if e != nil { + t.Fatal("governance pool") + } + t.Cleanup(governancePool.Close) + if e = postgresstore.ApplyMigrations(ctx, governancePool); e != nil { + t.Fatal(e) + } + if _, e = governancePool.Exec(ctx, objectgateway.Schema); e != nil { + t.Fatal(e) + } + reader := gatewayReadOnlyGovernance(t, ctx, governancePool, dsn) + const jobID = "gateway-authority-job" + const targetID = "gateway-authority-target" + store := postgresstore.NewStore(governancePool).WithLibraryTarget(targetID) + snapshot := sealSourceSnapshot(t, fx, nil) + now := time.Now().UTC() + op := governance.LibraryOperation{ID: "gateway-authority-operation", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: snapshot, RequestDigest: "gateway-authority-params", OperationKey: "gateway-authority-key", CreatedAt: now} + if _, _, e = store.EnsureLibraryOperation(ctx, op); e != nil { + t.Fatal(e) + } + if _, _, e = store.CreateLibraryJob(ctx, governance.LibraryJob{ID: jobID, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: op.ID, IdempotencyKey: jobID, RequestDigest: op.RequestDigest, TargetSnapshotID: snapshot}, governance.SyncStages(), now); e != nil { + t.Fatal(e) + } + attempt, e := store.BeginLibraryAttempt(ctx, jobID, "worker-first", time.Hour, now) + if e != nil { + t.Fatal(e) + } + if _, e = governancePool.Exec(ctx, "UPDATE library_jobs SET state='running' WHERE id=$1", jobID); e != nil { + t.Fatal(e) + } + authority := &GatewayGovernanceAuthority{Pool: reader, ArchiveRoot: fx.archiveRoot, TargetID: targetID, Provider: "minio", Bucket: fx.bucket} + if e = authority.Ready(ctx); e != nil { + t.Fatal(e) + } + privileged := *authority + privileged.Pool = governancePool + if privileged.Ready(ctx) == nil { + t.Fatal("privileged controller reader accepted") + } + backend := &gatewayCountBackend{Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}} + gateway := &objectgateway.Gateway{Pool: governancePool, Scope: fx.bucket, Backend: backend, Authority: authority, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20} + data := objectgateway.Digest([]byte("authority-data")) + server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: objectgateway.Digest([]byte("authority-control"))}) + defer server.Close() + client, e := objectgateway.NewClient(server.URL, data, fx.bucket) + if e != nil { + t.Fatal(e) + } + claim := objectgateway.Claim{JobID: jobID, OperationID: op.ID, AttemptEpoch: int64(attempt.Epoch), Owner: attempt.Owner} + objects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: claim} + target := fx.target(t) + target.config.Objects = objects + current, e := target.Current(ctx) + if e != nil { + t.Fatal(e) + } + backup, e := fx.backups(t, target).Prepare(ctx, snapshot, current) + if e != nil { + t.Fatal(e) + } + set, e := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: jobID, Target: "internal", BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshot, Universe: backup.Universe}) + if e != nil { + t.Fatal(e) + } + claim.PlanDigest = set.Digest + before := universeDump(t, fx.pool) + if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e == nil { + t.Fatal("unanchored plan permitted first PG write") + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatal("unanchored apply mutated PG", diff) + } + if e = store.RecordLibraryVersion(ctx, governance.LibraryVersion{ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, "internal", backup.SnapshotID, op.ID), Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: jobID, OperationID: op.ID, AttemptEpoch: attempt.Epoch, AttemptOwner: attempt.Owner, CreatedAt: now, Evidence: map[string]any{"job": jobID, "backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, "writeset_digest": set.Digest, "writeset_target_snapshot": snapshot}}); e != nil { + t.Fatal(e) + } + for _, bad := range []objectgateway.Claim{{JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: "other-worker", PlanDigest: set.Digest}, {JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: claim.Owner, PlanDigest: objectgateway.Digest([]byte("replacement"))}} { + if _, e = client.Acquire(ctx, bad); e == nil { + t.Fatal("untrusted claim admitted") + } + } + if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e != nil { + t.Fatal(e) + } + if e = target.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil { + t.Fatal(e) + } + count := backend.puts.Load() + if count == 0 { + t.Fatal("no actual object send") + } + firstRequests := objects.requests + if e = store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptFailed, "isolated takeover", time.Now().UTC()); e != nil { + t.Fatal(e) + } + next, e := store.BeginLibraryAttempt(ctx, jobID, "worker-second", time.Hour, time.Now().UTC()) + if e != nil { + t.Fatal(e) + } + nextClaim := claim + nextClaim.AttemptEpoch = int64(next.Epoch) + nextClaim.Owner = next.Owner + nextObjects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: nextClaim} + nextTarget := fx.target(t) + nextTarget.config.Objects = nextObjects + if e = nextTarget.AdoptWriteSet(set); e != nil { + t.Fatal(e) + } + if e = nextObjects.EnsureObjectGrant(ctx); e != nil { + t.Fatal(e) + } + for key, original := range firstRequests { + if nextObjects.requests[key] != original { + t.Fatal("send identity replaced during new epoch", key) + } + } + if nextObjects.authorization.Epoch <= objects.authorization.Epoch { + t.Fatal("authorization did not advance") + } + if _, e = client.Acquire(ctx, claim); !errors.Is(e, objectgateway.ErrEpoch) { + t.Fatal("old active owner admitted", e) + } + if e = nextTarget.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil { + t.Fatal(e) + } + if backend.puts.Load() != count { + t.Fatal("resumed object send duplicated") + } + for _, obj := range set.Objects { + if obj.Existing { + continue + } + locator := obj.Locator + observation, e := nextObjects.ObserveObject(ctx, obj.Provider, locator) + if e != nil { + t.Fatal(e) + } + if e = nextObjects.RemoveObject(ctx, obj.Provider, locator, observation); e != nil { + t.Fatal(e) + } + versions, e := backend.List(ctx, obj.Key) + if e != nil || len(versions) != 0 { + t.Fatal("created object version residue", e) + } + } + t.Logf("real readonly governance authority -> data-only HTTP acquisition -> PG/MinIO; first-PG anchor denial, new epoch unchanged send identity, PUTs=%d, original version exact cleanup", count) +} diff --git a/internal/releaseadapter/v071/gateway_backend.go b/internal/releaseadapter/v071/gateway_backend.go new file mode 100644 index 0000000..0f0d661 --- /dev/null +++ b/internal/releaseadapter/v071/gateway_backend.go @@ -0,0 +1,66 @@ +package v071 + +import ( + "context" + "errors" + + "knowledge-governance/internal/objectgateway" +) + +// NewGatewayBackend is server-side only. Clients never receive this credentialed object. +func NewGatewayBackend(c S3ObjectStoreConfig) (objectgateway.Backend, error) { + if c.Provider == "oss" { + s, e := newOSSVersionStore(c) + if e != nil { + return nil, e + } + return ossGatewayBackend{s}, nil + } + if c.Provider != "s3" && c.Provider != "minio" { + return nil, errors.New("gateway_backend_provider_unsupported") + } + s, e := NewS3ObjectStore(c) + if e != nil { + return nil, e + } + return &objectgateway.S3Backend{Client: s.client, Bucket: s.bucket, MaxBytes: s.maxObjectBytes}, nil +} + +type ossGatewayBackend struct{ s *ossVersionStore } + +func toGateway(v objectObservation) objectgateway.Version { + return objectgateway.Version{ID: v.VersionID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag} +} +func fromGateway(v objectgateway.Version) objectObservation { + return objectObservation{Exists: true, VersionID: v.ID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag} +} +func (b ossGatewayBackend) Ready(ctx context.Context) error { return b.s.versioning(ctx) } +func (b ossGatewayBackend) Observe(ctx context.Context, key, id string) (objectgateway.Version, bool, error) { + v, e := b.s.observeVersion(ctx, key, id) + return toGateway(v), v.Exists, e +} +func (b ossGatewayBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) { + v, e := b.s.appendVersion(ctx, key, body, owner) + return toGateway(v), e +} +func (b ossGatewayBackend) Delete(ctx context.Context, key string, v objectgateway.Version) error { + return b.s.removeVersion(ctx, key, fromGateway(v)) +} +func (b ossGatewayBackend) List(ctx context.Context, key string) ([]objectgateway.Version, error) { + listing, e := b.s.listExactVersions(ctx, key) + if e != nil { + return nil, e + } + if len(listing.DeleteMarkers) != 0 { + return nil, errors.New("gateway_backend_delete_marker_conflict") + } + var out []objectgateway.Version + for _, v := range listing.Versions { + o, e := b.s.observeVersion(ctx, key, v.VersionID) + if e != nil || !o.Exists { + return nil, errors.New("gateway_backend_listing_changed") + } + out = append(out, toGateway(o)) + } + return out, nil +} diff --git a/internal/releaseadapter/v071/gateway_grant.go b/internal/releaseadapter/v071/gateway_grant.go new file mode 100644 index 0000000..4aad64f --- /dev/null +++ b/internal/releaseadapter/v071/gateway_grant.go @@ -0,0 +1,49 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" +) + +// PrepareObjectGatewayGrant runs only in the trusted control plane. The data +// worker receives the immutable grant file, never the control bearer token. +// Replay uses the original expected epoch; it does not guess the latest epoch. +func PrepareObjectGatewayGrant(ctx context.Context, client *objectgateway.Client, set libraryrelease.WriteSet, expected int64, provider, bucket, directory string) error { + if client == nil || !filepath.IsAbs(directory) || expected < 0 { + return errors.New("gateway grant controller configuration invalid") + } + plan, e := ObjectGatewayPlan(set) + if e != nil { + return e + } + for _, obj := range set.Objects { + if obj.Provider != provider || obj.Bucket != bucket { + return errors.New("gateway grant target does not match frozen object plan") + } + } + grant := objectgateway.Grant{Epoch: expected + 1, Owner: plan.Owner, Plan: plan.Digest} + if grant.Epoch < 1 { + return errors.New("gateway epoch exhausted") + } + raw, e := json.Marshal(grant) + if e != nil { + return e + } + if e = persistImmutableJSON(filepath.Join(directory, set.Digest+".intent.json"), raw); e != nil { + return e + } + // Network outcomes may be unknown. Both endpoints compare immutable identity + // on a repeat, so no caller invents another epoch or another operation ID. + if e = client.InstallPlan(ctx, plan); e != nil { + return e + } + if e = client.Advance(ctx, expected, grant); e != nil { + return e + } + return persistImmutableJSON(filepath.Join(directory, set.Digest+".json"), raw) +} diff --git a/internal/releaseadapter/v071/gateway_object_store.go b/internal/releaseadapter/v071/gateway_object_store.go new file mode 100644 index 0000000..b41ddc6 --- /dev/null +++ b/internal/releaseadapter/v071/gateway_object_store.go @@ -0,0 +1,250 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" +) + +// GatewayObjectStore has no write credentials or direct-write fallback. The +// separate reader must be deployed with read-only rights (audit prerequisite). +type GatewayObjectStore struct { + Automatic bool + Claim objectgateway.Claim + frozen *libraryrelease.WriteSet + authorization objectgateway.Grant + Client *objectgateway.Client + Provider, Bucket, GrantDirectory string + requests map[string]objectgateway.Request + owners map[string]string +} + +func (s *GatewayObjectStore) ConditionalCapability(ctx context.Context) error { + if s.Client == nil || s.Provider == "" || s.Bucket == "" || (!s.Automatic && !filepath.IsAbs(s.GrantDirectory)) { + return errors.New("object gateway configuration/grant directory missing") + } + if s.Automatic { + return s.Client.ReadyForAcquisition(ctx) + } + return s.Client.Ready(ctx) +} + +// ObjectGatewayPlan derives the controller allowlist from the verified original +// write set. Existing shared objects remain read-only, never write-authorized. +func ObjectGatewayPlan(set libraryrelease.WriteSet) (objectgateway.Plan, error) { + if err := set.Verify(); err != nil { + return objectgateway.Plan{}, err + } + p := objectgateway.Plan{Digest: set.Digest, Owner: objectgateway.Digest([]byte(set.JobID))} + for _, obj := range set.Objects { + if !obj.Existing { + p.Objects = append(p.Objects, objectgateway.ObjectBinding{Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, SHA256: obj.SHA256, Bytes: obj.Bytes}) + } + } + return p, nil +} +func gatewayCreateOperation(plan, owner string) string { + return objectgateway.Digest([]byte(plan + "\x00" + owner + "\x00create")) +} +func (s *GatewayObjectStore) BindObjectPlan(set libraryrelease.WriteSet) error { + if !validLowerSHA256(set.Digest) || set.JobID == "" { + return errors.New("gateway frozen plan identity missing") + } + if _, e := ObjectGatewayPlan(set); e != nil { + return e + } + if s.frozen != nil && s.frozen.Digest != set.Digest { + return errors.New("gateway store cannot replace frozen identity") + } + if s.Automatic { + if s.frozen == nil { + copy := set + s.frozen = © + } + return nil + } + path := filepath.Join(s.GrantDirectory, set.Digest+".json") + info, e := os.Lstat(path) + if e != nil || !info.Mode().IsRegular() || info.Size() > 4096 || info.Mode().Perm()&0022 != 0 { + return errors.New("gateway controller grant unavailable") + } + raw, e := os.ReadFile(path) + if e != nil { + return errors.New("gateway controller grant unavailable") + } + var grant objectgateway.Grant + if json.Unmarshal(raw, &grant) != nil || grant.Epoch < 1 || grant.Plan != set.Digest || grant.Owner != objectgateway.Digest([]byte(set.JobID)) { + return errors.New("gateway controller grant differs from frozen job/plan") + } + return s.bindGrant(context.Background(), set, grant) +} +func (s *GatewayObjectStore) EnsureObjectGrant(ctx context.Context) error { + if !s.Automatic || s.requests != nil { + return nil + } + if s.frozen == nil || s.Claim.JobID != s.frozen.JobID { + return errors.New("gateway active attempt/frozen plan missing") + } + claim := s.Claim + claim.PlanDigest = s.frozen.Digest + grant, e := s.Client.Acquire(ctx, claim) + if e != nil { + return e + } + return s.bindGrant(ctx, *s.frozen, grant) +} +func (s *GatewayObjectStore) bindGrant(ctx context.Context, set libraryrelease.WriteSet, grant objectgateway.Grant) error { + requests := map[string]objectgateway.Request{} + owners := map[string]string{} + for _, obj := range set.Objects { + if obj.Provider != s.Provider || obj.Bucket != s.Bucket { + return errors.New("gateway plan target mismatch") + } + if obj.Existing { + continue + } + req := objectgateway.Request{Grant: grant, Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, Action: "create", SHA256: obj.SHA256, Bytes: obj.Bytes} + original, e := s.Client.Resolve(ctx, req) + if e != nil { + return e + } + requests[obj.Key] = original + owners[obj.Key] = obj.OwnerToken + } + s.authorization = grant + s.requests = requests + s.owners = owners + return nil +} +func (s *GatewayObjectStore) key(provider, locator string) (string, error) { + if provider != s.Provider || !validObjectKey(locator) { + return "", errors.New("gateway provider/locator mismatch") + } + parts := strings.SplitN(strings.TrimPrefix(locator, "storage://"), "/", 2) + if !strings.HasPrefix(locator, "storage://") || len(parts) != 2 || parts[0] == "" { + return "", errors.New("gateway explicit locator required") + } + prefix := s.Provider + "://" + s.Bucket + "/" + if !strings.HasPrefix(parts[1], prefix) { + return "", errors.New("gateway bucket mismatch") + } + key := strings.TrimPrefix(parts[1], prefix) + if key == "" { + return "", errors.New("gateway key missing") + } + return key, nil +} +func (s *GatewayObjectStore) ObserveObject(ctx context.Context, provider, locator string) (objectObservation, error) { + if s.frozen != nil { + if e := s.EnsureObjectGrant(ctx); e != nil { + return objectObservation{}, e + } + } + key, e := s.key(provider, locator) + if e != nil { + return objectObservation{}, e + } + v, exists, e := s.Client.Observe(ctx, key, "") + if e != nil { + return objectObservation{}, e + } + if !exists { + return objectObservation{}, nil + } + o := fromGateway(v) + if req, ok := s.requests[key]; ok && v.Owner == req.Claim() { + o.Owner = s.owners[key] + } + return o, nil +} +func (s *GatewayObjectStore) CreateObject(ctx context.Context, provider, locator string, body io.Reader, size int64, owner string) error { + if e := s.EnsureObjectGrant(ctx); e != nil { + return e + } + key, e := s.key(provider, locator) + if e != nil { + return e + } + r, ok := s.requests[key] + if !ok || owner != s.owners[key] || size != r.Bytes { + return errObjectWriteRejected + } + raw, e := io.ReadAll(io.LimitReader(body, size+1)) + if e != nil || int64(len(raw)) != size || objectgateway.Digest(raw) != r.SHA256 { + return errObjectWriteRejected + } + receipt, e := s.Client.ExecuteAuthorized(ctx, s.authorization, r, raw) + if e != nil { + return e + } + if receipt.State == "rejected" { + return errObjectWriteRejected + } + if receipt.State != "committed" { + return errors.New("gateway create not committed") + } + return nil +} +func (s *GatewayObjectStore) RemoveObject(ctx context.Context, provider, locator string, want objectObservation) error { + if e := s.EnsureObjectGrant(ctx); e != nil { + return e + } + key, e := s.key(provider, locator) + if e != nil { + return e + } + create, ok := s.requests[key] + if !ok || want.Owner != s.owners[key] { + return errors.New("gateway cleanup lacks frozen ownership") + } + receipt, e := s.Client.Reconcile(ctx, create) + if e != nil { + return e + } + raw := want + raw.Owner = create.Claim() + if receipt.State != "committed" || receipt.Version != toGateway(raw) { + return errors.New("gateway cleanup receipt mismatch") + } + req := create + req.Grant = s.authorization + req.Operation = objectgateway.Digest([]byte(create.Operation + "\x00delete")) + req.Action = "delete" + req.CreatedBy = create.Operation + req.Version = receipt.Version + original, e := s.Client.Resolve(ctx, req) + if e != nil { + return e + } + deleted, e := s.Client.ExecuteAuthorized(ctx, s.authorization, original, nil) + if e != nil { + return e + } + if deleted.State != "removed" { + return errors.New("gateway deletion not confirmed") + } + return nil +} +func (s *GatewayObjectStore) Put(context.Context, string, string, io.Reader, int64) error { + return errors.New("gateway refuses unplanned Put; frozen object plan required") +} +func (s *GatewayObjectStore) Copy(context.Context, string, string, string) error { + return errors.New("gateway refuses unplanned Copy; frozen object plan required") +} +func (s *GatewayObjectStore) Stat(ctx context.Context, provider, locator string) (int64, string, error) { + o, e := s.ObserveObject(ctx, provider, locator) + if e != nil { + return 0, "", e + } + if !o.Exists { + return 0, "", errors.New("gateway object absent") + } + return o.Bytes, o.SHA256, nil +} diff --git a/internal/releaseadapter/v071/gateway_object_store_test.go b/internal/releaseadapter/v071/gateway_object_store_test.go new file mode 100644 index 0000000..ed0799c --- /dev/null +++ b/internal/releaseadapter/v071/gateway_object_store_test.go @@ -0,0 +1,86 @@ +package v071 + +import ( + "context" + "net/http/httptest" + "testing" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" +) + +type gatewayFixtureIsolation struct{} + +func (gatewayFixtureIsolation) Verify(context.Context) error { return nil } + +// Test controller issues the grant only after the saga has frozen its actual +// plan. Production controller delivery is a separate integration prerequisite. +type gatewayFixtureController struct { + *GatewayObjectStore + control *objectgateway.Client +} + +func (s *gatewayFixtureController) BindObjectPlan(set libraryrelease.WriteSet) error { + if e := PrepareObjectGatewayGrant(context.Background(), s.control, set, 0, s.Provider, s.Bucket, s.GrantDirectory); e != nil { + return e + } + return s.GatewayObjectStore.BindObjectPlan(set) +} +func TestContractGatewayActualSagaRollbackIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + if _, e := fx.pool.Exec(fx.ctx, objectgateway.Schema); e != nil { + t.Fatal(e) + } + before := universeDump(t, fx.pool) + snapshot := sealSourceSnapshot(t, fx, nil) + target := fx.target(t) + gateway := &objectgateway.Gateway{Pool: fx.pool, Scope: fx.bucket, Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20} + token := objectgateway.Digest([]byte("data-fixture")) + server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: token, ControlToken: objectgateway.Digest([]byte("control-fixture"))}) + defer server.Close() + client, e := objectgateway.NewClient(server.URL, token, fx.bucket) + if e != nil { + t.Fatal(e) + } + store := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, GrantDirectory: t.TempDir()} + control, e := objectgateway.NewClient(server.URL, objectgateway.Digest([]byte("control-fixture")), fx.bucket) + if e != nil { + t.Fatal(e) + } + target.config.Objects = &gatewayFixtureController{store, control} + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, fx.backups(t, target), nil) + if outcome.State != "rolled_back" { + t.Fatalf("actual gateway saga: %+v", outcome) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatal(diff) + } + key := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-src") + observed, e := store.ObserveObject(fx.ctx, "minio", key) + if e != nil || observed.Exists { + t.Fatal("gateway object residue", e) + } + physical, e := store.key("minio", key) + if e != nil { + t.Fatal(e) + } + versions, e := gateway.Backend.List(fx.ctx, physical) + if e != nil || len(versions) != 0 { + t.Fatal("exact created key version residue", e) + } + var n int + if fx.pool.QueryRow(fx.ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", fx.bucket).Scan(&n) != nil || n != 2 { + t.Fatalf("gateway saga receipts=%d", n) + } + t.Log("actual release.Run -> plan adoption -> gateway HTTP -> PG/MinIO -> late Wiki failure -> exact-version rollback; all original PG rows retained") +} +func TestGatewayUnplannedWriteNeverFallsBack(t *testing.T) { + store := &GatewayObjectStore{} + if store.Put(context.Background(), "oss", "key", nil, 0) == nil || store.Copy(context.Background(), "oss", "from", "to") == nil { + t.Fatal("unplanned direct-write fallback") + } + if store.BindObjectPlan(libraryrelease.WriteSet{}) == nil { + t.Fatal("unbound plan accepted") + } +} diff --git a/internal/releaseadapter/v071/legacy_object_gate_test.go b/internal/releaseadapter/v071/legacy_object_gate_test.go new file mode 100644 index 0000000..60d68f4 --- /dev/null +++ b/internal/releaseadapter/v071/legacy_object_gate_test.go @@ -0,0 +1,27 @@ +package v071 + +import ( + "context" + "testing" +) + +func TestLegacyObjectImporterCannotStageOrPromote(t *testing.T) { + data := []byte("hello") + snapshot := ObjectSnapshot{Objects: []AssetObject{{ResourceID: "res-1", Handle: "ABCDEFGHIJKLMNOPQRSTUV", Provider: "minio", PhysicalPath: "storage://source-backend/10000/knowledge/source.txt", Size: 5, SHA256: shaBytes(data), Data: data}}, TotalBytes: 5} + snapshot.SHA256 = objectSnapshotDigest(snapshot.Objects) + store := &fakeObjectStore{objects: map[string][]byte{}} + importer, e := NewObjectImporter(store, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "minio"}) + if e != nil { + t.Fatal(e) + } + if _, e = importer.Stage(context.Background(), "legacy-bypass", snapshot); e == nil { + t.Fatal("legacy importer emitted unplanned stage PUT") + } + stage := ObjectStage{ReleaseRevision: "legacy-bypass", Provider: "minio", Objects: []StagedObject{{Handle: snapshot.Objects[0].Handle, StagingPath: "stage/object", FinalPath: "target/object", Size: 5, SHA256: shaBytes(data)}}} + if e = importer.Promote(context.Background(), stage); e == nil { + t.Fatal("legacy importer emitted unplanned promotion COPY") + } + if len(store.objects) != 0 { + t.Fatal("legacy importer mutated object store") + } +}