feat(object-gateway): 统一对象写网关与不可变治理日志

方案A:所有对象写入经 loopback 网关,持久计划/grant/intent/逐版本 receipt 与
attempt→grant delivery 落在治理 PG;scope 由 provider+endpoint+bucket 决定,
同一物理桶只允许一个写域。版本 append 不等于 CAS,未锚定计划在首笔业务写前拒绝。

本提交只含网关与其适配层接线、协议文档;生产独占、凭据切换与桶设置未改动。
This commit is contained in:
姚章浩
2026-11-01 11:35:05 +08:00
parent d0c6d57283
commit 5eef13b615
29 changed files with 3287 additions and 0 deletions
+61
View File
@@ -0,0 +1,61 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"time"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// This command belongs to the separately authenticated controller. It must not
// be installed with control credentials in an ordinary data worker container.
func main() {
if e := run(); e != nil {
fmt.Fprintln(os.Stderr, e)
os.Exit(1)
}
}
func run() error {
path := flag.String("plan", "", "exact frozen write-set JSON")
directory := flag.String("grant-directory", "", "private durable grant output directory")
expected := flag.Int64("expected-epoch", -1, "previous epoch from the frozen controller intent; never guessed")
flag.Parse()
if *path == "" || *directory == "" || *expected < 0 || flag.NArg() != 0 {
return errors.New("exact plan, grant directory and expected epoch required")
}
f, e := os.Open(*path)
if e != nil {
return errors.New("frozen gateway plan unavailable")
}
defer f.Close()
var set libraryrelease.WriteSet
d := json.NewDecoder(io.LimitReader(f, 16<<20))
d.DisallowUnknownFields()
if d.Decode(&set) != nil || d.Decode(&struct{}{}) != io.EOF {
return errors.New("frozen gateway plan invalid")
}
provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET")
if provider == "" || endpoint == "" || bucket == "" {
return errors.New("exact gateway target configuration required")
}
scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket))
client, e := objectgateway.NewClient(os.Getenv("OBJECT_GATEWAY_URL"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN"), scope)
if e != nil {
return e
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if e = v071.PrepareObjectGatewayGrant(ctx, client, set, *expected, provider, bucket, *directory); e != nil {
return e
}
fmt.Printf("gateway grant persisted: plan=%s epoch=%d\n", set.Digest, *expected+1)
return nil
}
+82
View File
@@ -0,0 +1,82 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"errors"
"log"
"net"
"net/http"
"os"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/objectgateway"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
func main() {
if err := run(); err != nil {
log.Print(err)
os.Exit(1)
}
}
func run() error {
ctx := context.Background()
addr := os.Getenv("OBJECT_GATEWAY_LISTEN")
host, _, e := net.SplitHostPort(addr)
// A reviewed TLS terminator may expose this loopback-only listener. Never bind
// authenticated plaintext to a LAN address by an accidental config default.
if e != nil || (host != "127.0.0.1" && host != "::1") {
return errors.New("gateway_requires_loopback_listener")
}
data, control := os.Getenv("OBJECT_GATEWAY_DATA_TOKEN"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN")
if len(data) < 32 || len(control) < 32 || data == control {
return errors.New("gateway_distinct_tokens_required")
}
pub, e := base64.StdEncoding.DecodeString(os.Getenv("OBJECT_GATEWAY_AUDIT_PUBLIC_KEY"))
if e != nil || len(pub) != ed25519.PublicKeySize {
return errors.New("gateway_auditor_key_required")
}
provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET")
scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket))
pool, e := pgxpool.New(ctx, os.Getenv("OBJECT_GATEWAY_DATABASE_URL"))
if e != nil {
return objectgateway.ErrPersistence
}
defer pool.Close()
backend, e := v071.NewGatewayBackend(v071.S3ObjectStoreConfig{Provider: provider, Endpoint: endpoint, Bucket: bucket, Region: os.Getenv("OBJECT_GATEWAY_REGION"), AccessKeyID: os.Getenv("OBJECT_GATEWAY_ACCESS_KEY"), SecretAccessKey: os.Getenv("OBJECT_GATEWAY_SECRET_KEY"), MaxObjectBytes: 256 * 1024 * 1024})
if e != nil {
return errors.New("gateway_backend_configuration_invalid")
}
g := &objectgateway.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 256 * 1024 * 1024, Isolation: objectgateway.AuditVerifier{Path: os.Getenv("OBJECT_GATEWAY_AUDIT_FILE"), PublicKey: pub, Scope: scope, GatewayID: os.Getenv("OBJECT_GATEWAY_ID")}}
if governanceURL := os.Getenv("OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL"); governanceURL != "" {
cfg, e := pgxpool.ParseConfig(governanceURL)
if e != nil {
return errors.New("gateway_control_authority_unavailable")
}
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
cfg.MaxConns = 4
governancePool, e := pgxpool.NewWithConfig(ctx, cfg)
if e != nil {
return errors.New("gateway_control_authority_unavailable")
}
defer governancePool.Close()
authority := &v071.GatewayGovernanceAuthority{Pool: governancePool, ArchiveRoot: os.Getenv("OBJECT_GATEWAY_ARCHIVE_ROOT"), TargetID: os.Getenv("OBJECT_GATEWAY_TARGET_ID"), Provider: provider, Bucket: bucket}
if e = authority.Ready(ctx); e != nil {
return e
}
g.Authority = authority
}
probe, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if e = g.Ready(probe); e != nil {
return e
}
server := &http.Server{Addr: addr, Handler: objectgateway.HTTPServer{Gateway: g, DataToken: data, ControlToken: control}, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 3 * time.Minute, WriteTimeout: 3 * time.Minute, IdleTimeout: 30 * time.Second}
if server.ListenAndServe() != nil {
return errors.New("gateway_listener_stopped")
}
return nil
}
+19
View File
@@ -0,0 +1,19 @@
package main
import (
"context"
v071 "knowledge-governance/internal/releaseadapter/v071"
"testing"
)
func TestLegacyReleaseRuntimeRefusesBeforeCredentials(t *testing.T) {
// Empty credential paths would fail differently if route assembly were reached.
runtime, e := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{})
if runtime != nil || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() {
t.Fatal("legacy runtime did not fail at the no-write gate", e)
}
route, pools, e := buildReleaseRoute(context.Background(), "", releaseTargetSnapshotConfig{}, releaseRouteConfig{})
if route != nil || len(pools) != 0 || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() {
t.Fatal("direct route bypassed the no-write gate", e)
}
}
+93
View File
@@ -0,0 +1,93 @@
# 对象写入网关(开发候选,尚不可部署业务)
## 协议与持久化
`cmd/object-gateway` 提供 `kg-object-gateway-v1`,只监听 loopback;外部接入需要另行审核的 TLS 终结与网络隔离。治理 PG 保存不可变计划、grant、intent、逐版本 receipt 和 attempt→grant delivery。部署迁移身份显式应用 `internal/objectgateway/schema.sql`;启动不自动迁移。运行身份只授必要 SELECT/INSERT 和 scope 行 UPDATE,不授 DDL/TRUNCATE/证据修改或业务库写权限。
scope 是 provider、原 endpoint、bucket 的 NUL 分隔 SHA256。同一物理桶只能有一个受审核 scope 和一套共享 journal,不能用 endpoint 别名或另一日志库建立平行写域。
数据认证端点:`POST /v1/acquire`、`/v1/resolve`、`/v1/execute`、`/v1/reconcile`、`/v1/observe`,以及 `GET /v1/capabilities`。所有请求核验协议和 scope。`capabilities.automatic_grants` 必须为真,正式 stack 才接受自动授权能力;未接控制源、只读角色权限异常、协议不符或不可达均拒绝,无直写回退。
PG 行锁负责原子准入,不是远端存储 fence。intent 必须持久提交后才发送一次请求;连接死亡、租约过期不清除未知 intent。任何未终态 intent 均阻止 scope 换 epoch。重复 operation 只读回执或精确对账,不重发 PUT;暂时查不到版本仍 held。删除必须绑定已确认 create 的精确 VersionID,不删 latest、不生成删除标记。
不可变证据表拒绝 UPDATE/DELETE/TRUNCATE。Ready 核 fsync、synchronous_commit、full_page_writes、主库可写及必要表/触发器;管理员与迁移身份仍是可信边界,触发器不能抵御 DBA 的 DDL。
## 正式控制交付路径
生产代码路径已接入,并已在专属本机隔离栈运行正式 worker 的两次 SIGKILL 接管;这不是生产部署验收:
`cmd/worker.runOwnedLibraryAttempt → attempt context → librarystack → GatewayObjectStore(Automatic) → ProductionLibraryTarget.Apply → EnsureObjectGrant → /v1/acquire → GatewayGovernanceAuthority → 原持久 WriteSet → journal delivery → 数据 execute`。
冻结/采用计划时只保存自校验后的 WriteSet,不提前索取 grant;observer 先把摘要锚定到原预备份事件,首笔业务 PG Apply 前才请求授权。worker 只提交 job/operation/attempt epoch/owner/plan digest,不提交可替换的 key/body allowlist,也不能选择网关 epoch。
可信控制域使用独立 SELECT-only 治理连接,核对目标身份、job/operation、当前 attempt、owner、有效租约、running 状态和原预备份事件中的摘要、job、backup、目标快照;再从归档加载并验 hash、范围、provider/bucket。客户端自述不能替代这些持久事实。控制读角色不得是 superuser、CREATEDB、CREATEROLE,不能写/截断/建触发器到这些治理表。每次加载和能力查询重验角色。
网关在 scope 锁内再次核对授权事实,原子写入 grant、当前 epoch、不可变 delivery。同一 attempt 重试返回原 grant;owner/operation/plan 替换拒绝;同一 job 的后续 attempt 必须沿用原 operation 和 plan。旧发送未终态时,新 attempt 可先只读对账,但无充分证据仍 held,不能交接。
控制服务配置了治理 authority 后,旧 `/v1/plans`、`/v1/advance` 手动控制端点关闭,控制 token 也不能绕过原计划绑定。旧 `cmd/object-gateway-control` 仅保留离线/协议兼容路径,不是正式 worker 的交付方案。
## 独立恢复请求
`library.recover` 的队列ID由原窗口job确定;它具有独立恢复operation与request digest,绑定scope、queue、原window/operation、target、source、backup和原WriteSet。原计划的job、operation和摘要不改写。Claim中的`recovery_job_id`与`recovery_operation_id`仅为独立授权上下文,原`job_id`/`operation_id`仍是发送域。网关重新读取持久队列payload、running状态、当前owner/attempt/有效租约与恢复operation,不信任worker自行声明。可选显式来源/目标/backup/plan字段只能收紧原绑定,不能覆盖它。
SELECT-only治理角色需要读取`library_jobs`、`library_operations`、`library_attempts`、`library_versions`、`jobs`,均不能INSERT/UPDATE/DELETE/TRUNCATE/TRIGGER。已有原窗口恢复请求时不自动重启原发布;过期running恢复队列只在相同target、未释放维护窗口、原业务租约也过期时重新投递,业务attempt/epoch仍重新申请。
## 索引unknown恢复边界
原冻结before-image非空时,恢复先持久化`restore-mark-intent`,再以原CAS(或确认缺失的create)写入包含原计划/index/id归属标记的完整before-image。丢响应后仅匹配完整标记正文及有效CAS才能登记`restore-claimed`并CAS去标记;已知回执的代次不一致拒绝。恢复完成的`restored`回执仍须匹配正文和CAS,不能接受同正文的新代次。
去标记前持久化`restore-finalize-intent`。发布clean-finalization或恢复clean-finalization已提交但响应丢失时,当前干净正文并非充分归属证据;继续held,不盲删、不重发、不把相同外观当回执。旧unknown缺少写前标记intent亦不自动升级认领。本候选没有服务端可验证的clean提交关联凭证,因此这部分自动收敛仍是明确缺口,不是可部署完整恢复保证。
## 跨 epoch 的不可变发送身份
`Request.Grant` 继续代表首次发送的不可变身份,后端 claim、intent 和 receipt 都不改。新 `ExecuteRequest.authorization` 仅代表本次准入权限。`/v1/resolve` 根据相同 operation/key/action/hash/length/owner/plan 取回旧请求;只允许 epoch 差异,不能换内容。
新 attempt 获得新 grant 后,已发送 key 沿用旧 request/claim,未发送 key 使用新授权 epoch 建立首次 intent。当前授权必须匹配 journal 的当前 epoch;旧 epoch 的新写拒绝。新 epoch 补偿删除仍使用原 create 回执和精确版本,删除本身有独立不可变发送身份。
## 配置与凭据归属
网关进程环境:
- `OBJECT_GATEWAY_LISTEN`:仅 `127.0.0.1:<port>` 或 `[::1]:<port>`。
- `OBJECT_GATEWAY_DATABASE_URL`:独立 journal 连接,迁移与运行角色分离。
- `OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL`:仅可信网关持有的 SELECT-only 治理读连接;独立于 journal 运行身份和普通 worker 身份。
- `OBJECT_GATEWAY_ARCHIVE_ROOT`:原持久计划所在归档绝对路径,部署只读挂载。
- `OBJECT_GATEWAY_TARGET_ID`:与正式 worker `LibraryTargetIdentity()` 完全相同的逻辑目标身份;不能用另一个环境的值。
- `OBJECT_GATEWAY_PROVIDER`、`OBJECT_GATEWAY_ENDPOINT`、`OBJECT_GATEWAY_BUCKET`、`OBJECT_GATEWAY_REGION`:精确存储身份。
- `OBJECT_GATEWAY_ACCESS_KEY`、`OBJECT_GATEWAY_SECRET_KEY`:仅网关持有的对象写凭据。
- `OBJECT_GATEWAY_DATA_TOKEN`、`OBJECT_GATEWAY_CONTROL_TOKEN`:不同且至少 32 字节;普通 worker 仅持有 data token。
- `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`:独立审核者的 Ed25519 签名隔离证据,绑定协议、scope、gateway ID、时间窗及 writer/policy/拒写/排空摘要。审核私钥不能交给 worker 或网关。
秘密只经服务环境/私有文件注入,不进 argv、URL 或日志。所有调用端错误都使用固定原因码。
正式 worker 使用 `LIBRARY_OBJECT_GATEWAY_URL` 和 `LIBRARY_OBJECT_GATEWAY_TOKEN`;不再依赖预先生成的 grant 文件。`LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY` 保留旧离线兼容用途,不是自动模式的授权来源。发布 stack 对所有对象 provider 强制网关;S3 构造仍用于只读捕获/预备份,部署必须使用独立只读凭据。
## 已关闭的本仓旁路
- `ObjectImporter.Stage/Promote` 无条件拒绝,不留配置开关,不执行 PUT/COPY。
- legacy `release-worker` runtime/route 在读凭据及连接目标前拒绝。
- `release-diag`、`release-channel-e2e` 旧写型入口启动即非零退出,避免 probe PUT 或文档级旁路。
- `librarystack` 不再为缺网关的 MinIO/S3 返回直写 writer。
- `libraryctl` capture 的 S3 构造与读取保留;未把读取当作写入口禁用。
这不等于已经撤销运行环境旧进程的凭据;源码变更尚未部署。`librarydrill` 等无 owned-attempt 的旧写型工具不能靠静态 grant 绕过自动模式,需补正式执行身份后才可用于写入。低层 S3 adapter 仍用于网关后端、读取及显式隔离夹具;部署权限才负责防止外部旧进程直接调用存储。
## 实测与未完成边界
已实跑:真实治理 PG(独立 SELECT-only 登录角色)、原 archive、数据 HTTP、业务 PG 与 MinIO 的自动授权/跨 epoch 链;未锚定计划在第一笔 PG 前拒绝,接管保持原发送身份且不增加 PUT。另有真实网关二进制的两次中断验证。隔离审计 authority/故障注入仍是测试设施,不是生产权限证明。
正式 worker 隔离进程证据:真实 Docker 入口闸门/队列、只读 worker 对象身份、独立治理读取角色、网关与业务 PG/ES/MinIO;首次 PG 后和对象提交后两次 SIGKILL,同一原 job/operation/plan 的三代 attempt 完成发布,原对象发送 epoch 与回执不变。隔离构建标签 `kg_isolated_drill` 仅把真实租约缩短至4秒,并拒绝非 loopback 一次性 r8 库;正常构建仍为10分钟。维护上游是无业务凭据的独立 HTTP 夹具,不是运行中的 WeKnora;签名证据绑定本次真实拒写/排空材料,不代表生产独占审核。
已补独立 `library.recover` 正式worker链、恢复中再次SIGKILL、完整before-image/CAS/精确版本读回,以及来源/目标/原job/计划/operation/任意队列身份拒绝。ES故障代理只转发真实ES并丢弃已提交响应,未伪造服务响应。
尚未完成:clean-finalization unknown的充分归属与自动收敛;独立运行环境只读对象凭据、ACL/网络和全部writer排空;网关真实OSS端到端;原D0/F11/F12与55项其余缺口。测试/隔离验收与真实部署独占分列,不把生产变更授权当成本机开发授权的前置。
维护隔离、独占审核与 grant 是独立前置。配置 true、健康 200、测试自签 authority 都不能代替已部署拒写/排空证据。目前未签发真实环境独占证明,未改生产权限或上游 WeKnora 源码/Schema。
## 部署与回退(未执行)
未来测试主机 `120.76.217.51`、生产 `120.76.218.137` 各自独立批准。每批先冻结主机、桶、全部角色/STS/预签名写入口、端口/网络规则、影响、维护窗口与回滚;当前没有执行这些外部变更。
先关新写准入→排空并对账旧请求→撤旧直写权限/切只读身份并实测拒写→独立审核→批准的新限定对象验收→父复核,才可切业务客户端。r5 的已完成 OSS 测试授权不能重复使用。
回退先关准入并保留 journal;所有已发送请求有终态才可撤网关,unknown 继续 held。不得因网关故障自动恢复旧直写凭据。权限/网络回退须另批。当前整模块 INCOMPLETE,禁止部署部分候选。
+159
View File
@@ -0,0 +1,159 @@
package objectgateway
import (
"bytes"
"context"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5"
"math"
)
// Claim names an already persisted application attempt and write-set anchor.
// It never carries keys, bodies, a replacement plan or a chosen gateway epoch.
type Claim struct {
JobID string `json:"job_id"`
OperationID string `json:"operation_id"`
AttemptEpoch int64 `json:"attempt_epoch"`
Owner string `json:"owner"`
PlanDigest string `json:"plan_digest"`
RecoveryJobID string `json:"recovery_job_id,omitempty"`
RecoveryOperationID string `json:"recovery_operation_id,omitempty"`
}
type PlanAuthority interface {
Load(context.Context, Claim) (Plan, error)
}
func (g *Gateway) Acquire(ctx context.Context, c Claim) (Grant, error) {
if g.Authority == nil {
return Grant{}, ErrAuthority
}
if c.JobID == "" || c.OperationID == "" || c.Owner == "" || c.AttemptEpoch < 1 || !validHash(c.PlanDigest) {
return Grant{}, ErrEpoch
}
if e := g.Ready(ctx); e != nil {
return Grant{}, e
}
plan, e := g.Authority.Load(ctx, c)
if e != nil {
return Grant{}, e
}
if plan.Digest != c.PlanDigest || plan.Owner != Digest([]byte(c.JobID)) {
return Grant{}, ErrConflict
}
if e = g.InstallPlan(ctx, plan); e != nil {
return Grant{}, e
}
// Recover only sends in this exact frozen plan. No absent observation can
// settle an unknown send; the locked advancement below still checks all intents.
rows, e := g.Pool.Query(ctx, `SELECT i.request FROM kg_object_gateway_intents i
LEFT JOIN kg_object_gateway_receipts r ON r.scope=i.scope AND r.operation=i.operation
WHERE i.scope=$1 AND r.operation IS NULL`, g.Scope)
if e != nil {
return Grant{}, ErrPersistence
}
var pending []Request
for rows.Next() {
var raw []byte
var request Request
if rows.Scan(&raw) != nil || json.Unmarshal(raw, &request) != nil {
rows.Close()
return Grant{}, ErrPersistence
}
if request.Grant.Plan == plan.Digest {
pending = append(pending, request)
}
}
e = rows.Err()
rows.Close()
if e != nil {
return Grant{}, ErrPersistence
}
for _, request := range pending {
if request.Grant.Owner != plan.Owner {
return Grant{}, ErrConflict
}
if _, e = g.Reconcile(ctx, request); e != nil {
return Grant{}, e
}
}
tx, e := g.Pool.Begin(ctx)
if e != nil {
return Grant{}, ErrPersistence
}
defer tx.Rollback(ctx)
if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); e != nil {
return Grant{}, ErrPersistence
}
var epoch int64
if e = tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch); e != nil {
return Grant{}, ErrPersistence
}
// Recheck after serialization. This is admission evidence, not an OSS fence:
// a persisted uncertain object intent still prevents every epoch transition.
verified, e := g.Authority.Load(ctx, c)
if e != nil {
return Grant{}, e
}
frozen, _ := json.Marshal(plan)
again, _ := json.Marshal(verified)
if !bytes.Equal(frozen, again) {
return Grant{}, ErrConflict
}
raw, _ := json.Marshal(c)
var oldClaim, oldGrant []byte
e = tx.QueryRow(ctx, "SELECT claim,grant_body FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch=$3", g.Scope, c.JobID, c.AttemptEpoch).Scan(&oldClaim, &oldGrant)
if e == nil {
var grant Grant
if !bytes.Equal(raw, oldClaim) || json.Unmarshal(oldGrant, &grant) != nil || !grant.valid() {
return Grant{}, ErrConflict
}
if grant.Epoch != epoch {
return Grant{}, ErrEpoch
}
return grant, nil
}
if !errors.Is(e, pgx.ErrNoRows) {
return Grant{}, ErrPersistence
}
var firstRaw []byte
firstErr := tx.QueryRow(ctx, "SELECT claim FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 ORDER BY attempt_epoch LIMIT 1", g.Scope, c.JobID).Scan(&firstRaw)
if firstErr == nil {
var first Claim
if json.Unmarshal(firstRaw, &first) != nil {
return Grant{}, ErrPersistence
}
if first.JobID != c.JobID || first.OperationID != c.OperationID || first.PlanDigest != c.PlanDigest {
return Grant{}, ErrConflict
}
} else if !errors.Is(firstErr, pgx.ErrNoRows) {
return Grant{}, ErrPersistence
}
var stale bool
if tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch>=$3)", g.Scope, c.JobID, c.AttemptEpoch).Scan(&stale) != nil {
return Grant{}, ErrPersistence
}
if stale || epoch == math.MaxInt64 {
return Grant{}, ErrEpoch
}
next := Grant{Epoch: epoch + 1, Owner: plan.Owner, Plan: plan.Digest}
if e = g.advanceLocked(ctx, tx, epoch, next); e != nil {
return Grant{}, e
}
grantRaw, _ := json.Marshal(next)
if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_deliveries(scope,job,attempt_epoch,claim,grant_epoch,grant_body) VALUES($1,$2,$3,$4,$5,$6)", g.Scope, c.JobID, c.AttemptEpoch, raw, next.Epoch, grantRaw); e != nil {
return Grant{}, ErrPersistence
}
if tx.Commit(ctx) != nil {
return Grant{}, ErrPersistence
}
return next, nil
}
func (c *Client) Acquire(ctx context.Context, claim Claim) (Grant, error) {
var out Grant
e := c.call(ctx, "/v1/acquire", claim, &out)
if e == nil && (!out.valid() || out.Plan != claim.PlanDigest || out.Owner != Digest([]byte(claim.JobID))) {
e = ErrConflict
}
return out, e
}
@@ -0,0 +1,122 @@
package objectgateway_test
import (
"context"
"errors"
g "knowledge-governance/internal/objectgateway"
"testing"
)
type currentPlanAuthority struct {
claim g.Claim
plan g.Plan
}
func (a *currentPlanAuthority) Load(_ context.Context, c g.Claim) (g.Plan, error) {
if c != a.claim {
return g.Plan{}, g.ErrEpoch
}
return a.plan, nil
}
func TestGatewayAcquisitionPersistsAttemptDelivery(t *testing.T) {
ctx, pool, backend := fixture(t)
claim := g.Claim{JobID: "job-acquire", OperationID: "operation-acquire", AttemptEpoch: 1, Owner: "worker-1", PlanDigest: g.Digest([]byte(t.Name()))}
expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest}
req := request("acquired-object", expected)
authority := &currentPlanAuthority{claim, planFor(expected, req)}
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
grant, e := gateway.Acquire(ctx, claim)
if e != nil || grant != expected {
t.Fatal("anchored active attempt grant unavailable", e)
}
replay, e := gateway.Acquire(ctx, claim)
if e != nil || replay != grant {
t.Fatal("grant delivery replay changed epoch", e)
}
oldClaim := claim
claim.AttemptEpoch = 2
claim.Owner = "worker-2"
authority.claim = claim
next, e := gateway.Acquire(ctx, claim)
if e != nil || next.Epoch != 2 {
t.Fatal("next active attempt", e)
}
if _, e = gateway.Acquire(ctx, oldClaim); !errors.Is(e, g.ErrEpoch) {
t.Fatal("stale attempt acquired grant", e)
}
replacement := claim
replacement.AttemptEpoch++
replacement.PlanDigest = g.Digest([]byte("replacement-plan"))
authority.claim = replacement
authority.plan.Digest = replacement.PlanDigest
if _, e = gateway.Acquire(ctx, replacement); !errors.Is(e, g.ErrConflict) {
t.Fatal("new attempt replaced original plan", e)
}
authority.claim = claim
authority.plan.Digest = claim.PlanDigest
req.Grant = next
receipt, e := gateway.ExecuteAuthorized(ctx, next, req, []byte("gateway fixture acquired-object"))
if e != nil {
t.Fatal(e)
}
cleanup(t, ctx, gateway, receipt)
var count int
if e = pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 2 {
t.Fatal("immutable delivery count", count, e)
}
for _, statement := range []string{"UPDATE kg_object_gateway_deliveries SET job=job", "DELETE FROM kg_object_gateway_deliveries", "TRUNCATE kg_object_gateway_deliveries"} {
if _, e = pool.Exec(ctx, statement); e == nil {
t.Fatal("mutable grant delivery", statement)
}
}
}
func TestGatewayAcquisitionCannotHandOverUnknownSend(t *testing.T) {
ctx, pool, real := fixture(t)
backend := &stoppedBeforeSend{Backend: real}
claim := g.Claim{JobID: "job-held", OperationID: "operation-held", AttemptEpoch: 1, Owner: "worker-first", PlanDigest: g.Digest([]byte(t.Name()))}
expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest}
req := request("acquire-held", expected)
body := []byte("gateway fixture acquire-held")
authority := &currentPlanAuthority{claim, planFor(expected, req)}
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
if _, e := gateway.Acquire(ctx, claim); e != nil {
t.Fatal(e)
}
if _, e := gateway.ExecuteAuthorized(ctx, expected, req, body); !errors.Is(e, g.ErrHeld) {
t.Fatal(e)
}
claim.AttemptEpoch++
claim.Owner = "worker-next"
authority.claim = claim
for i := 0; i < 2; i++ {
if _, e := gateway.Acquire(ctx, claim); !errors.Is(e, g.ErrHeld) {
t.Fatal("absent send handed over", e)
}
}
if backend.calls != 1 {
t.Fatal("control acquisition re-sent PUT")
}
var count int
if e := pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 1 {
t.Fatal("unknown send advanced delivery", count, e)
}
// Deliver the previously uncertain request late, using its exact immutable claim.
if _, e := real.Put(ctx, req.Key, body, req.Claim()); e != nil {
t.Fatal(e)
}
next, e := gateway.Acquire(ctx, claim)
if e != nil || next.Epoch != 2 {
t.Fatal("committed send was not reconciled before handover", e)
}
receipt, e := gateway.ExecuteAuthorized(ctx, next, req, body)
if e != nil || receipt.Request != req {
t.Fatal(e)
}
if backend.calls != 1 {
t.Fatal("reconciled send was retried")
}
receipt.Request.Grant = next
cleanup(t, ctx, gateway, receipt)
t.Log("new controller attempt stayed held while absent; late committed original recovered without replay PUT")
}
+57
View File
@@ -0,0 +1,57 @@
package objectgateway
import (
"context"
"crypto/ed25519"
"encoding/json"
"os"
"time"
)
// Attestation is issued by an independent deployment auditor, never by a worker.
// Its signature authenticates an audit, not the truth of self-reported config.
// Runtime cutover must invalidate/replace the audit whenever writer rights change.
type Attestation struct {
Protocol string `json:"protocol"`
Scope string `json:"scope"`
GatewayID string `json:"gateway_id"`
IssuedAt time.Time `json:"issued_at"`
ExpiresAt time.Time `json:"expires_at"`
WriterInventory string `json:"writer_inventory_sha256"`
PolicyEvidence string `json:"policy_evidence_sha256"`
DenialEvidence string `json:"denial_evidence_sha256"`
DrainEvidence string `json:"drain_evidence_sha256"`
}
type SignedAttestation struct {
Payload json.RawMessage `json:"payload"`
Signature []byte `json:"signature"`
}
type AuditVerifier struct {
Path string
PublicKey ed25519.PublicKey
Scope, GatewayID string
}
func (v AuditVerifier) Verify(ctx context.Context) error {
if ctx.Err() != nil {
return ErrIsolation
}
st, e := os.Lstat(v.Path)
if e != nil || !st.Mode().IsRegular() || st.Size() > 16384 || st.Mode().Perm()&0022 != 0 {
return ErrIsolation
}
raw, e := os.ReadFile(v.Path)
if e != nil {
return ErrIsolation
}
var signed SignedAttestation
var a Attestation
if json.Unmarshal(raw, &signed) != nil || len(v.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(v.PublicKey, signed.Payload, signed.Signature) || json.Unmarshal(signed.Payload, &a) != nil {
return ErrIsolation
}
now := time.Now()
if a.Protocol != Protocol || a.Scope != v.Scope || a.GatewayID != v.GatewayID || a.GatewayID == "" || a.IssuedAt.After(now) || !a.ExpiresAt.After(now) || a.ExpiresAt.Sub(a.IssuedAt) > time.Hour || !validHash(a.WriterInventory) || !validHash(a.PolicyEvidence) || !validHash(a.DenialEvidence) || !validHash(a.DrainEvidence) {
return ErrIsolation
}
return nil
}
+59
View File
@@ -0,0 +1,59 @@
package objectgateway_test
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/json"
g "knowledge-governance/internal/objectgateway"
"os"
"path/filepath"
"testing"
"time"
)
func TestGatewayAuditCannotBeConfigurationBoolean(t *testing.T) {
pub, priv, e := ed25519.GenerateKey(rand.Reader)
if e != nil {
t.Fatal(e)
}
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
verifier := g.AuditVerifier{Path: path, PublicKey: pub, Scope: "target", GatewayID: "gateway"}
if verifier.Verify(context.Background()) == nil {
t.Fatal("missing audit accepted")
}
if os.WriteFile(path, []byte(`{"isolated":true,"ready":true}`), 0600) != nil {
t.Fatal("fixture")
}
if verifier.Verify(context.Background()) == nil {
t.Fatal("boolean accepted as isolation proof")
}
a := g.Attestation{Protocol: g.Protocol, Scope: "target", GatewayID: "gateway", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("inventory")), PolicyEvidence: g.Digest([]byte("policy")), DenialEvidence: g.Digest([]byte("denials")), DrainEvidence: g.Digest([]byte("drain"))}
write := func(v g.Attestation) {
t.Helper()
raw, _ := json.Marshal(v)
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
if os.WriteFile(path, signed, 0600) != nil {
t.Fatal("fixture")
}
}
write(a)
if verifier.Verify(context.Background()) != nil {
t.Fatal("valid test auditor signature rejected")
}
for _, mutate := range []func(*g.Attestation){func(v *g.Attestation) { v.Scope = "other" }, func(v *g.Attestation) { v.GatewayID = "other" }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *g.Attestation) { v.DenialEvidence = "" }, func(v *g.Attestation) { v.IssuedAt = time.Now().Add(time.Hour) }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(24 * time.Hour) }} {
bad := a
mutate(&bad)
write(bad)
if verifier.Verify(context.Background()) == nil {
t.Fatal("invalid isolation audit accepted")
}
}
write(a)
other, _, _ := ed25519.GenerateKey(rand.Reader)
verifier.PublicKey = other
if verifier.Verify(context.Background()) == nil {
t.Fatal("untrusted signer accepted")
}
}
+15
View File
@@ -0,0 +1,15 @@
package objectgateway
import "context"
type claimContextKey struct{}
// WithClaim is set by the owned-attempt runner, never from an HTTP caller's
// proposed plan. The controller independently checks every field against PG.
func WithClaim(ctx context.Context, c Claim) context.Context {
return context.WithValue(ctx, claimContextKey{}, c)
}
func ClaimFromContext(ctx context.Context) Claim {
c, _ := ctx.Value(claimContextKey{}).(Claim)
return c
}
+326
View File
@@ -0,0 +1,326 @@
package objectgateway
import (
"context"
_ "embed"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
// Schema is applied explicitly by the migration identity, never on server startup.
//
//go:embed schema.sql
var Schema string
type Gateway struct {
Authority PlanAuthority
Pool *pgxpool.Pool
Scope string
Backend Backend
Isolation IsolationVerifier
MaxBytes int64
}
func (g *Gateway) Ready(ctx context.Context) error {
if g.Pool == nil || g.Scope == "" || g.Backend == nil || g.MaxBytes < 1 {
return ErrPersistence
}
if g.Isolation == nil || g.Isolation.Verify(ctx) != nil {
return ErrIsolation
}
var sync, fsync, full string
if g.Pool.QueryRow(ctx, "SELECT current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes')").Scan(&sync, &fsync, &full) != nil || sync != "on" || fsync != "on" || full != "on" {
return ErrPersistence
}
var journal bool
if g.Pool.QueryRow(ctx, `SELECT NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off'
AND to_regclass('kg_object_gateway_plans') IS NOT NULL
AND to_regclass('kg_object_gateway_scopes') IS NOT NULL
AND to_regclass('kg_object_gateway_grants') IS NOT NULL
AND to_regclass('kg_object_gateway_intents') IS NOT NULL
AND to_regclass('kg_object_gateway_receipts') IS NOT NULL
AND to_regclass('kg_object_gateway_deliveries') IS NOT NULL
AND (SELECT count(*)=10 FROM pg_trigger WHERE tgname IN ('kg_object_gateway_deliveries_immutable','kg_object_gateway_deliveries_no_truncate','kg_object_gateway_plans_no_truncate','kg_object_gateway_intents_no_truncate','kg_object_gateway_receipts_no_truncate','kg_object_gateway_grants_no_truncate','kg_object_gateway_plans_immutable','kg_object_gateway_intents_immutable','kg_object_gateway_receipts_immutable','kg_object_gateway_grants_immutable') AND tgenabled='O')`).Scan(&journal) != nil || !journal {
return ErrPersistence
}
if g.Backend.Ready(ctx) != nil {
return errors.New("gateway_backend_not_ready")
}
return nil
}
func (g *Gateway) Advance(ctx context.Context, expected int64, next Grant) error {
if !next.valid() || next.Epoch != expected+1 {
return ErrEpoch
}
if err := g.Ready(ctx); err != nil {
return err
}
var frozen []byte
var plan Plan
if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, next.Plan).Scan(&frozen) != nil {
return ErrConflict
}
if json.Unmarshal(frozen, &plan) != nil || !plan.valid() || plan.Owner != next.Owner {
return ErrConflict
}
tx, err := g.Pool.Begin(ctx)
if err != nil {
return ErrPersistence
}
defer tx.Rollback(ctx)
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); err != nil {
return ErrPersistence
}
var epoch int64
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
return ErrPersistence
}
if err = g.advanceLocked(ctx, tx, epoch, next); err != nil {
return err
}
if tx.Commit(ctx) != nil {
return ErrPersistence
}
return nil
}
func (g *Gateway) advanceLocked(ctx context.Context, tx pgx.Tx, epoch int64, next Grant) error {
var err error
if epoch == next.Epoch {
var owner, plan string
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
return ErrPersistence
}
if owner != next.Owner || plan != next.Plan {
return ErrEpoch
}
return nil
}
if epoch != next.Epoch-1 {
return ErrEpoch
}
var pending bool
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts r WHERE r.scope=i.scope AND r.operation=i.operation))`, g.Scope).Scan(&pending) != nil {
return ErrPersistence
}
if pending {
return ErrHeld
}
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_grants(scope,epoch,owner,plan) VALUES($1,$2,$3,$4)", g.Scope, next.Epoch, next.Owner, next.Plan); err != nil {
return ErrPersistence
}
if _, err = tx.Exec(ctx, "UPDATE kg_object_gateway_scopes SET epoch=$2 WHERE scope=$1", g.Scope, next.Epoch); err != nil {
return ErrPersistence
}
return nil
}
// reserve commits an immutable may-send intent. Even a commit timeout cannot be
// followed by a send. Another process can reconcile, but cannot retry this PUT.
func (g *Gateway) reserve(ctx context.Context, r Request, authorization *Grant) (bool, error) {
tx, err := g.Pool.Begin(ctx)
if err != nil {
return false, ErrPersistence
}
defer tx.Rollback(ctx)
var epoch int64
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
return false, ErrEpoch
}
if authorization != nil {
var owner, plan string
if epoch != authorization.Epoch {
return false, ErrEpoch
}
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
return false, ErrPersistence
}
if owner != authorization.Owner || plan != authorization.Plan {
return false, ErrEpoch
}
}
raw, _ := json.Marshal(r)
var previous []byte
err = tx.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&previous)
if err == nil {
if string(previous) != string(raw) {
return false, ErrConflict
}
return false, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return false, ErrPersistence
}
var owner, plan string
if epoch != r.Grant.Epoch {
return false, ErrEpoch
}
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
return false, ErrPersistence
}
if owner != r.Grant.Owner || plan != r.Grant.Plan {
return false, ErrEpoch
}
var pending bool
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND i.key=$2 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts x WHERE x.scope=i.scope AND x.operation=i.operation))`, g.Scope, r.Key).Scan(&pending) != nil {
return false, ErrPersistence
}
if pending {
return false, ErrHeld
}
var frozen []byte
var allowed Plan
if tx.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, r.Grant.Plan).Scan(&frozen) != nil {
return false, ErrPersistence
}
if json.Unmarshal(frozen, &allowed) != nil || !allowed.valid() || !allowed.allows(r) {
return false, ErrConflict
}
if r.Action == "delete" {
previous, err := g.receipt(ctx, r.CreatedBy)
if err != nil || previous.State != "committed" || previous.Request.Key != r.Key || previous.Version != r.Version {
return false, ErrConflict
}
}
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_intents(scope,operation,key,request) VALUES($1,$2,$3,$4)", g.Scope, r.Operation, r.Key, raw); err != nil {
return false, ErrPersistence
}
if tx.Commit(ctx) != nil {
return false, ErrPersistence
}
return true, nil
}
func (g *Gateway) receipt(ctx context.Context, id string) (Receipt, error) {
var raw []byte
var r Receipt
err := g.Pool.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE scope=$1 AND operation=$2", g.Scope, id).Scan(&raw)
if err != nil {
return r, err
}
if json.Unmarshal(raw, &r) != nil {
return r, ErrPersistence
}
return r, nil
}
func (g *Gateway) finish(ctx context.Context, r Request, state string, v Version) (Receipt, error) {
receipt := Receipt{r, state, v}
raw, _ := json.Marshal(receipt)
_, err := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_receipts(scope,operation,receipt) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, r.Operation, raw)
if err != nil {
return Receipt{}, ErrPersistence
}
got, err := g.receipt(ctx, r.Operation)
if err != nil {
return Receipt{}, ErrPersistence
}
if got != receipt {
return Receipt{}, ErrConflict
}
return got, nil
}
func (g *Gateway) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) {
return g.execute(ctx, r, body, nil)
}
func (g *Gateway) execute(ctx context.Context, r Request, body []byte, authorization *Grant) (Receipt, error) {
if err := r.validate(); err != nil {
return Receipt{}, err
}
if r.Bytes > g.MaxBytes || r.Action == "create" && (int64(len(body)) != r.Bytes || Digest(body) != r.SHA256) || r.Action == "delete" && len(body) != 0 {
return Receipt{}, ErrConflict
}
if err := g.Ready(ctx); err != nil {
return Receipt{}, err
}
fresh, err := g.reserve(ctx, r, authorization)
if err != nil {
return Receipt{}, err
}
if !fresh {
return g.Reconcile(ctx, r)
}
if r.Action == "create" {
_, exists, err := g.Backend.Observe(ctx, r.Key, "")
if err != nil {
return Receipt{}, ErrHeld
}
if exists {
return g.finish(ctx, r, "rejected", Version{})
}
v, err := g.Backend.Put(ctx, r.Key, body, r.token())
if err != nil {
return Receipt{}, ErrHeld
}
exact, exists, err := g.Backend.Observe(ctx, r.Key, v.ID)
if err != nil || !exists || !r.matches(exact) || v != exact {
return Receipt{}, ErrHeld
}
return g.finish(ctx, r, "committed", v)
}
v, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
if err != nil {
return Receipt{}, ErrHeld
}
if exists && v != r.Version {
return Receipt{}, ErrConflict
}
if exists {
if g.Backend.Delete(ctx, r.Key, r.Version) != nil {
return Receipt{}, ErrHeld
}
}
return g.Reconcile(ctx, r)
}
func (g *Gateway) Reconcile(ctx context.Context, r Request) (Receipt, error) {
if err := r.validate(); err != nil {
return Receipt{}, err
}
if err := g.Ready(ctx); err != nil {
return Receipt{}, err
}
var raw []byte
if g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&raw) != nil {
return Receipt{}, ErrConflict
}
want, _ := json.Marshal(r)
if string(raw) != string(want) {
return Receipt{}, ErrConflict
}
got, err := g.receipt(ctx, r.Operation)
if err == nil {
return got, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return Receipt{}, ErrPersistence
}
if r.Action == "delete" {
_, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
if err != nil || exists {
return Receipt{}, ErrHeld
}
return g.finish(ctx, r, "removed", r.Version)
}
versions, err := g.Backend.List(ctx, r.Key)
if err != nil {
return Receipt{}, ErrHeld
}
var found []Version
for _, v := range versions {
if v.Owner == r.token() {
if !r.matches(v) {
return Receipt{}, ErrHeld
}
found = append(found, v)
}
}
if len(found) != 1 {
return Receipt{}, ErrHeld
}
exact, exists, err := g.Backend.Observe(ctx, r.Key, found[0].ID)
if err != nil || !exists || exact != found[0] {
return Receipt{}, ErrHeld
}
return g.finish(ctx, r, "committed", exact)
}
@@ -0,0 +1,399 @@
package objectgateway_test
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/jackc/pgx/v5/pgxpool"
g "knowledge-governance/internal/objectgateway"
)
type verifiedFixture struct{}
func (verifiedFixture) Verify(context.Context) error { return nil }
func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) {
t.Helper()
dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("isolated gateway PG/MinIO opt-in required")
}
u, e := url.Parse(dsn)
if e != nil || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "kg_library_test_gateway_") || u.Hostname() != "127.0.0.1" {
t.Fatal("unsafe gateway test database")
}
ctx := context.Background()
pool, e := pgxpool.New(ctx, dsn)
if e != nil {
t.Fatal("PG unavailable")
}
t.Cleanup(pool.Close)
if _, e = pool.Exec(ctx, g.Schema); e != nil {
t.Fatal(e)
}
endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT")
if endpoint != "http://127.0.0.1:19100" {
t.Fatal("unsafe test MinIO endpoint")
}
bucket := "kg-library-test-gateway-" + g.Digest([]byte(t.Name() + time.Now().String()))[:16]
client := s3.New(s3.Options{Region: "us-east-1", BaseEndpoint: aws.String(endpoint), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "")})
if _, e = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); e != nil {
t.Fatal("create isolated bucket failed")
}
if _, e = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}}); e != nil {
t.Fatal(e)
}
t.Cleanup(func() {
list, e := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
if e != nil {
t.Error("residue list unavailable")
return
}
if len(list.Versions) != 0 || len(list.DeleteMarkers) != 0 {
t.Errorf("preserving residue bucket %s: versions=%d markers=%d", bucket, len(list.Versions), len(list.DeleteMarkers))
return
}
if _, e = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); e != nil {
t.Error("delete empty test bucket failed")
}
t.Logf("exact sandbox %s: versions=0 markers=0; empty bucket deleted", bucket)
})
return ctx, pool, &g.S3Backend{Client: client, Bucket: bucket, MaxBytes: 1 << 20}
}
func planFor(grant g.Grant, requests ...g.Request) g.Plan {
p := g.Plan{Digest: grant.Plan, Owner: grant.Owner}
for _, r := range requests {
p.Objects = append(p.Objects, g.ObjectBinding{Operation: r.Operation, Key: r.Key, SHA256: r.SHA256, Bytes: r.Bytes})
}
return p
}
func request(label string, grant g.Grant) g.Request {
body := []byte("gateway fixture " + label)
return g.Request{Grant: grant, Operation: g.Digest([]byte(label)), Key: "unchanged/" + label, Action: "create", SHA256: g.Digest(body), Bytes: int64(len(body))}
}
func cleanup(t *testing.T, ctx context.Context, gw *g.Gateway, receipt g.Receipt) {
t.Helper()
r := receipt.Request
r.Operation = g.Digest([]byte(r.Operation + "delete"))
r.Action = "delete"
r.CreatedBy = receipt.Request.Operation
r.Version = receipt.Version
if got, e := gw.Execute(ctx, r, nil); e != nil || got.State != "removed" {
t.Fatalf("exact cleanup: %v %s", e, got.State)
}
}
func TestGatewayReadinessRequiresJournal(t *testing.T) {
ctx, pool, backend := fixture(t)
// An isolated search_path lacks the journal while PG settings remain healthy.
conn, e := pool.Acquire(ctx)
if e != nil {
t.Fatal(e)
}
conn.Release()
cfg := pool.Config()
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_catalog"
other, e := pgxpool.NewWithConfig(ctx, cfg)
if e != nil {
t.Fatal(e)
}
defer other.Close()
gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
t.Fatalf("missing durable journal accepted: %v", e)
}
}
func TestGatewayRealReplayAndImmutableLedger(t *testing.T) {
ctx, pool, backend := fixture(t)
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists {
t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists)
}
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
t.Fatal(e)
}
if e := gw.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
req := request(t.Name(), grant)
body := []byte("gateway fixture " + t.Name())
one, e := gw.Execute(ctx, req, body)
if e != nil || one.State != "committed" {
t.Fatalf("create %v %s", e, one.State)
}
two, e := gw.Execute(ctx, req, body)
if e != nil || two != one {
t.Fatal("replay changed receipt", e)
}
bad := req
bad.Key += "-different"
if _, e = gw.Execute(ctx, bad, body); !errors.Is(e, g.ErrConflict) {
t.Fatal("operation rebound", e)
}
for _, table := range []string{"intents", "receipts", "grants", "plans"} {
if _, e = pool.Exec(ctx, "DELETE FROM kg_object_gateway_"+table+" WHERE scope=$1", gw.Scope); e == nil {
t.Fatal("immutable evidence deleted", table)
}
}
for _, table := range []string{"plans", "intents", "receipts", "grants"} {
tx, e := pool.Begin(ctx)
if e != nil {
t.Fatal(e)
}
_, e = tx.Exec(ctx, "TRUNCATE kg_object_gateway_"+table+" CASCADE")
_ = tx.Rollback(ctx)
if e == nil {
cleanup(t, ctx, gw, one)
t.Fatal("immutable evidence truncation accepted", table)
}
}
cleanup(t, ctx, gw, one)
next := grant
next.Epoch = 2
if e = gw.Advance(ctx, 1, next); e != nil {
t.Fatal(e)
}
late := request("late", grant)
if _, e = gw.Execute(ctx, late, []byte("gateway fixture late")); !errors.Is(e, g.ErrEpoch) {
t.Fatal("old epoch sent", e)
}
}
func signedAudit(t *testing.T, scope string) (string, string) {
t.Helper()
pub, priv, e := ed25519.GenerateKey(rand.Reader)
if e != nil {
t.Fatal(e)
}
// This is a test authority. It exercises verification, NOT production exclusivity.
a := g.Attestation{Protocol: g.Protocol, Scope: scope, GatewayID: "isolated-fixture", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("fixture inventory")), PolicyEvidence: g.Digest([]byte("fixture policy")), DenialEvidence: g.Digest([]byte("fixture denial")), DrainEvidence: g.Digest([]byte("fixture drain"))}
raw, _ := json.Marshal(a)
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
path := filepath.Join(t.TempDir(), "audit.json")
if os.WriteFile(path, signed, 0600) != nil {
t.Fatal("audit write failed")
}
return path, base64.StdEncoding.EncodeToString(pub)
}
func startGateway(t *testing.T, env []string, client *g.Client) *exec.Cmd {
t.Helper()
binary := os.Getenv("KG_GATEWAY_TEST_BINARY")
if binary == "" {
t.Fatal("actual gateway binary is required")
}
cmd := exec.Command(binary)
cmd.Env = env
log, e := os.Create(filepath.Join(t.TempDir(), "service.log"))
if e != nil {
t.Fatal(e)
}
cmd.Stdout = log
cmd.Stderr = log
if e = cmd.Start(); e != nil {
t.Fatal(e)
}
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); _ = log.Close() })
for i := 0; i < 100; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
e = client.Ready(ctx)
cancel()
if e == nil {
return cmd
}
time.Sleep(30 * time.Millisecond)
}
raw, _ := os.ReadFile(log.Name())
t.Fatalf("actual gateway not ready: %s", raw)
return nil
}
func TestGatewayActualServiceTwoInterruptions(t *testing.T) {
ctx, pool, backend := fixture(t)
upstream, _ := url.Parse(os.Getenv("KG_GATEWAY_TEST_ENDPOINT"))
proxy := httputil.NewSingleHostReverseProxy(upstream)
beforeSend := make(chan struct{}, 2)
allowSend := make(chan struct{}, 2)
committed := make(chan struct{}, 2)
release := make(chan struct{}, 2)
var puts atomic.Int32
proxy.ModifyResponse = func(resp *http.Response) error {
if resp.Request.Method == "PUT" && !strings.Contains(resp.Request.URL.RawQuery, "versioning") {
puts.Add(1)
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
committed <- struct{}{}
select {
case <-release:
case <-resp.Request.Context().Done():
}
return errors.New("injected lost response after actual MinIO commit")
}
}
return nil
}
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) { http.Error(w, "injected_transport_loss", 502) }
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "PUT" && !strings.Contains(r.URL.RawQuery, "versioning") {
beforeSend <- struct{}{}
select {
case <-allowSend:
case <-r.Context().Done():
return
}
}
proxy.ServeHTTP(w, r)
}))
defer p.Close()
scope := g.Digest([]byte("minio\x00" + p.URL + "\x00" + backend.Bucket))
audit, pub := signedAudit(t, scope)
listener, e := net.Listen("tcp", "127.0.0.1:0")
if e != nil {
t.Fatal(e)
}
addr := listener.Addr().String()
listener.Close()
data := g.Digest([]byte("data " + scope))
control := g.Digest([]byte("control " + scope))
client, e := g.NewClient("http://"+addr, data, scope)
if e != nil {
t.Fatal(e)
}
admin, _ := g.NewClient("http://"+addr, control, scope)
env := append(os.Environ(), "OBJECT_GATEWAY_LISTEN="+addr, "OBJECT_GATEWAY_DATABASE_URL="+os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"), "OBJECT_GATEWAY_PROVIDER=minio", "OBJECT_GATEWAY_ENDPOINT="+p.URL, "OBJECT_GATEWAY_BUCKET="+backend.Bucket, "OBJECT_GATEWAY_ACCESS_KEY="+os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), "OBJECT_GATEWAY_SECRET_KEY="+os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY="+pub, "OBJECT_GATEWAY_AUDIT_FILE="+audit, "OBJECT_GATEWAY_ID=isolated-fixture", "OBJECT_GATEWAY_DATA_TOKEN="+data, "OBJECT_GATEWAY_CONTROL_TOKEN="+control)
cmd := startGateway(t, env, client)
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("actual-worker")), Plan: g.Digest([]byte("frozen-actual-plan"))}
if e = client.InstallPlan(ctx, planFor(grant, request("actual-0", grant))); e == nil {
t.Fatal("data client installed plan")
}
if e = admin.InstallPlan(ctx, planFor(grant, request("actual-0", grant), request("actual-1", grant))); e != nil {
t.Fatal(e)
}
if e = admin.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
if e = client.Advance(ctx, 0, grant); e == nil {
t.Fatal("data client acquired control authority")
}
for iteration := 0; iteration < 2; iteration++ {
label := fmt.Sprintf("actual-%d", iteration)
r := request(label, grant)
body := []byte("gateway fixture " + label)
done := make(chan error, 1)
go func() { _, e := client.Execute(ctx, r, body); done <- e }()
select {
case <-beforeSend:
case <-time.After(15 * time.Second):
t.Fatal("gateway did not send to proxy")
}
if _, e = client.Reconcile(ctx, r); !errors.Is(e, g.ErrHeld) {
t.Fatal("not-yet-visible PUT treated as terminal", e)
}
blockedNext := grant
blockedNext.Epoch++
if e = admin.Advance(ctx, grant.Epoch, blockedNext); !errors.Is(e, g.ErrHeld) {
t.Fatal("epoch crossed delayed precommit request", e)
}
allowSend <- struct{}{}
select {
case <-committed:
case <-time.After(15 * time.Second):
t.Fatal("actual MinIO PUT did not commit")
}
next := grant
next.Epoch++
if e = admin.Advance(ctx, grant.Epoch, next); !errors.Is(e, g.ErrHeld) {
t.Fatal("epoch crossed in-flight write", e)
}
sameKey := r
sameKey.Operation = g.Digest([]byte(label + "other"))
if _, e = client.Execute(ctx, sameKey, body); !errors.Is(e, g.ErrHeld) {
t.Fatal("same-key competitor admitted", e)
}
if e = cmd.Process.Kill(); e != nil {
t.Fatal(e)
}
_ = cmd.Wait()
release <- struct{}{}
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("dead gateway call did not end")
}
t.Logf("actual gateway SIGKILL %d after MinIO commit before receipt", iteration+1)
cmd = startGateway(t, env, client)
receipt, e := client.Reconcile(ctx, r)
if e != nil || receipt.State != "committed" {
t.Fatalf("restart reconcile %v %s", e, receipt.State)
}
repeated, e := client.Execute(ctx, r, body)
if e != nil || receipt != repeated {
t.Fatal("restart duplicate not idempotent", e)
}
del := r
del.Action = "delete"
del.CreatedBy = r.Operation
del.Operation = g.Digest([]byte(label + "cleanup"))
del.Version = receipt.Version
cleaned, e := client.Execute(ctx, del, nil)
if e != nil || cleaned.State != "removed" {
t.Fatal("actual exact cleanup failed", e)
}
if e = admin.Advance(ctx, grant.Epoch, next); e != nil {
t.Fatal(e)
}
late := request(label+"late", grant)
if _, e = client.Execute(ctx, late, []byte("gateway fixture "+label+"late")); !errors.Is(e, g.ErrEpoch) {
t.Fatal("late stale epoch admitted", e)
}
grant = next
}
if puts.Load() != 2 {
t.Fatalf("blind PUT replay count=%d", puts.Load())
}
var n int
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", scope).Scan(&n) != nil || n != 4 {
t.Fatalf("durable receipts=%d", n)
}
_ = cmd.Process.Kill()
_ = cmd.Wait()
if e = client.Ready(ctx); e == nil {
t.Fatal("unavailable gateway accepted")
}
t.Log("actual binary + HTTP + durable PG + MinIO: two kills/restarts, 2 PUTs, 4 receipts, exact cleanup")
}
func TestGatewayClientRejectsBadProtocolAndLeaks(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("secret") != "" {
t.Error("secret in URL")
}
_, _ = io.WriteString(w, `{"protocol":"other","scope":"scope","ready":true}`)
}))
defer server.Close()
c, e := g.NewClient(server.URL, strings.Repeat("x", 32), "scope")
if e != nil {
t.Fatal(e)
}
if c.Ready(context.Background()) == nil {
t.Fatal("wrong protocol accepted")
}
}
+316
View File
@@ -0,0 +1,316 @@
package objectgateway
import (
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"strings"
"time"
)
type ExecuteRequest struct {
Authorization *Grant `json:"authorization,omitempty"`
Request Request `json:"request"`
Body []byte `json:"body,omitempty"`
}
type AdvanceRequest struct {
Expected int64 `json:"expected"`
Grant Grant `json:"grant"`
}
type Capabilities struct {
AutomaticGrants bool `json:"automatic_grants"`
Protocol string `json:"protocol"`
Scope string `json:"scope"`
Ready bool `json:"ready"`
}
type HTTPServer struct {
Gateway *Gateway
DataToken, ControlToken string
}
func (s HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
token := s.DataToken
if r.URL.Path == "/v1/advance" || r.URL.Path == "/v1/plans" {
token = s.ControlToken
}
if len(token) < 32 || s.DataToken == s.ControlToken || subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+token)) != 1 {
http.Error(w, "gateway_unauthorized", 401)
return
}
if r.Header.Get("X-KG-Gateway-Protocol") != Protocol || r.Header.Get("X-KG-Gateway-Scope") != s.Gateway.Scope {
http.Error(w, "gateway_protocol_or_target_mismatch", 409)
return
}
fail := func(e error) {
code := 503
if errors.Is(e, ErrEpoch) || errors.Is(e, ErrConflict) || errors.Is(e, ErrHeld) {
code = 409
}
http.Error(w, e.Error(), code)
}
decode := func(v any) bool {
r.Body = http.MaxBytesReader(w, r.Body, s.Gateway.MaxBytes*2+16384)
d := json.NewDecoder(r.Body)
d.DisallowUnknownFields()
if d.Decode(v) != nil || d.Decode(&struct{}{}) != io.EOF {
http.Error(w, "gateway_bad_request", 400)
return false
}
return true
}
if r.Method == "GET" && r.URL.Path == "/v1/capabilities" {
if e := s.Gateway.Ready(r.Context()); e != nil {
fail(e)
return
}
if authority, ok := s.Gateway.Authority.(interface{ Ready(context.Context) error }); ok {
if e := authority.Ready(r.Context()); e != nil {
fail(e)
return
}
}
_ = json.NewEncoder(w).Encode(Capabilities{Protocol: Protocol, Scope: s.Gateway.Scope, Ready: true, AutomaticGrants: s.Gateway.Authority != nil})
return
}
if r.Method != "POST" {
http.Error(w, "gateway_route_not_found", 404)
return
}
switch r.URL.Path {
case "/v1/acquire":
var in Claim
if !decode(&in) {
return
}
out, e := s.Gateway.Acquire(r.Context(), in)
if e != nil {
fail(e)
return
}
_ = json.NewEncoder(w).Encode(out)
case "/v1/plans":
if s.Gateway.Authority != nil {
fail(ErrConflict)
return
}
var in Plan
if !decode(&in) {
return
}
if e := s.Gateway.InstallPlan(r.Context(), in); e != nil {
fail(e)
return
}
_ = json.NewEncoder(w).Encode(in)
case "/v1/advance":
if s.Gateway.Authority != nil {
fail(ErrConflict)
return
}
var in AdvanceRequest
if !decode(&in) {
return
}
if e := s.Gateway.Advance(r.Context(), in.Expected, in.Grant); e != nil {
fail(e)
return
}
_ = json.NewEncoder(w).Encode(in)
case "/v1/execute", "/v1/reconcile":
var in ExecuteRequest
if !decode(&in) {
return
}
var out Receipt
var e error
if r.URL.Path == "/v1/execute" {
if in.Authorization != nil {
out, e = s.Gateway.ExecuteAuthorized(r.Context(), *in.Authorization, in.Request, in.Body)
} else {
out, e = s.Gateway.Execute(r.Context(), in.Request, in.Body)
}
} else {
out, e = s.Gateway.Reconcile(r.Context(), in.Request)
}
if e != nil {
fail(e)
return
}
_ = json.NewEncoder(w).Encode(out)
case "/v1/resolve":
var in Request
if !decode(&in) {
return
}
out, e := s.Gateway.Resolve(r.Context(), in)
if e != nil {
fail(e)
return
}
_ = json.NewEncoder(w).Encode(out)
case "/v1/observe":
var in struct {
Key string `json:"key"`
Version string `json:"version"`
}
if !decode(&in) {
return
}
if in.Key == "" {
http.Error(w, "gateway_bad_key", 400)
return
}
if e := s.Gateway.Ready(r.Context()); e != nil {
fail(e)
return
}
v, exists, e := s.Gateway.Backend.Observe(r.Context(), in.Key, in.Version)
if e != nil {
http.Error(w, "gateway_backend_read_failed", 503)
return
}
_ = json.NewEncoder(w).Encode(struct {
Version Version `json:"version"`
Exists bool `json:"exists"`
}{v, exists})
default:
http.Error(w, "gateway_route_not_found", 404)
}
}
type Client struct {
Endpoint, Token, Scope string
HTTP *http.Client
}
func NewClient(endpoint, token, scope string) (*Client, error) {
u, e := url.Parse(endpoint)
if e != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" || len(token) < 32 || scope == "" {
return nil, errors.New("gateway_configuration_invalid")
}
if u.Scheme != "https" && (u.Scheme != "http" || (u.Hostname() != "127.0.0.1" && u.Hostname() != "::1")) {
return nil, errors.New("gateway_transport_requires_tls")
}
return &Client{endpoint, token, scope, &http.Client{Timeout: 2 * time.Minute, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("gateway_redirect_refused") }}}, nil
}
func (c *Client) call(ctx context.Context, path string, in, out any) error {
method := "GET"
var body io.Reader
if in != nil {
raw, e := json.Marshal(in)
if e != nil {
return ErrConflict
}
method = "POST"
body = struct{ io.Reader }{bytes.NewReader(raw)}
}
req, e := http.NewRequestWithContext(ctx, method, c.Endpoint+path, body)
if e != nil {
return errors.New("gateway_request_invalid")
}
req.Header.Set("Authorization", "Bearer "+c.Token)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-KG-Gateway-Protocol", Protocol)
req.Header.Set("X-KG-Gateway-Scope", c.Scope)
resp, e := c.HTTP.Do(req)
if e != nil {
return errors.New("gateway_unreachable_or_outcome_unknown")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 128))
reason := strings.TrimSpace(string(raw))
switch reason {
case ErrAuthority.Error():
return ErrAuthority
case ErrAuthorityPrivileges.Error():
return ErrAuthorityPrivileges
case ErrEpoch.Error():
return ErrEpoch
case ErrConflict.Error():
return ErrConflict
case ErrHeld.Error():
return ErrHeld
case ErrPersistence.Error():
return ErrPersistence
case ErrIsolation.Error():
return ErrIsolation
}
return errors.New("gateway_request_refused")
}
if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out) != nil {
return errors.New("gateway_response_invalid")
}
return nil
}
func (c *Client) Ready(ctx context.Context) error { return c.checkReady(ctx, false) }
func (c *Client) ReadyForAcquisition(ctx context.Context) error { return c.checkReady(ctx, true) }
func (c *Client) checkReady(ctx context.Context, automatic bool) error {
var caps Capabilities
if e := c.call(ctx, "/v1/capabilities", nil, &caps); e != nil {
return e
}
if automatic && !caps.AutomaticGrants {
return ErrAuthority
}
if caps.Protocol != Protocol || caps.Scope != c.Scope || !caps.Ready {
return errors.New("gateway_protocol_or_target_mismatch")
}
return nil
}
func (c *Client) InstallPlan(ctx context.Context, p Plan) error {
var out Plan
if e := c.call(ctx, "/v1/plans", p, &out); e != nil {
return e
}
want, _ := json.Marshal(p)
got, _ := json.Marshal(out)
if string(want) != string(got) {
return ErrConflict
}
return nil
}
func (c *Client) Advance(ctx context.Context, expected int64, g Grant) error {
var out AdvanceRequest
in := AdvanceRequest{expected, g}
if e := c.call(ctx, "/v1/advance", in, &out); e != nil {
return e
}
if in != out {
return ErrConflict
}
return nil
}
func (c *Client) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) {
var out Receipt
e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body}, &out)
if e == nil && (out.Request != r || !out.valid()) {
e = ErrConflict
}
return out, e
}
func (c *Client) Reconcile(ctx context.Context, r Request) (Receipt, error) {
var out Receipt
e := c.call(ctx, "/v1/reconcile", ExecuteRequest{Request: r}, &out)
if e == nil && (out.Request != r || !out.valid()) {
e = ErrConflict
}
return out, e
}
func (c *Client) Observe(ctx context.Context, key, id string) (Version, bool, error) {
var out struct {
Version Version `json:"version"`
Exists bool `json:"exists"`
}
e := c.call(ctx, "/v1/observe", struct {
Key string `json:"key"`
Version string `json:"version"`
}{key, id}, &out)
return out.Version, out.Exists, e
}
@@ -0,0 +1,50 @@
package objectgateway_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
g "knowledge-governance/internal/objectgateway"
)
func TestGatewayTargetIdentityOnEveryCall(t *testing.T) {
ctx, pool, backend := fixture(t)
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
t.Fatal(e)
}
if e := gw.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
token := strings.Repeat("a", 32)
server := httptest.NewServer(g.HTTPServer{Gateway: gw, DataToken: token, ControlToken: strings.Repeat("b", 32)})
defer server.Close()
wrong, _ := g.NewClient(server.URL, token, "wrong-target")
req := request(t.Name(), grant)
receipt, e := wrong.Execute(ctx, req, []byte("gateway fixture "+t.Name()))
if e == nil {
cleanup(t, ctx, gw, receipt)
t.Fatal("mutation accepted with wrong target identity")
}
var n int
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gw.Scope).Scan(&n) != nil || n != 0 {
t.Fatal("wrong target wrote intent")
}
}
func TestGatewayClientRejectsIncompleteReceipt(t *testing.T) {
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("owner")), Plan: g.Digest([]byte("plan"))}
req := request("receipt", grant)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(g.Receipt{Request: req, State: "committed"})
}))
defer server.Close()
c, _ := g.NewClient(server.URL, strings.Repeat("a", 32), "scope")
if _, e := c.Execute(context.Background(), req, []byte("gateway fixture receipt")); e == nil {
t.Fatal("committed receipt with missing exact version accepted")
}
}
@@ -0,0 +1,44 @@
package objectgateway_test
import (
"errors"
g "knowledge-governance/internal/objectgateway"
"testing"
)
func TestGatewayIntentCommitFailureNeverSends(t *testing.T) {
ctx, pool, real := fixture(t)
backend := &stoppedBeforeSend{Backend: real}
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
t.Fatal(e)
}
if e := gateway.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
// Deferred failure fires on COMMIT, not on INSERT. This exercises the uncertain
// persistence boundary while confirming no provider request is sent.
_, e := pool.Exec(ctx, `CREATE FUNCTION kg_gateway_fixture_commit_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected durable commit failure'; END $$;
CREATE CONSTRAINT TRIGGER kg_gateway_fixture_commit_failure AFTER INSERT ON kg_object_gateway_intents DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION kg_gateway_fixture_commit_failure()`)
if e != nil {
t.Fatal(e)
}
t.Cleanup(func() {
_, e := pool.Exec(ctx, `DROP TRIGGER kg_gateway_fixture_commit_failure ON kg_object_gateway_intents; DROP FUNCTION kg_gateway_fixture_commit_failure()`)
if e != nil {
t.Error(e)
}
})
req := request(t.Name(), grant)
if _, e = gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrPersistence) {
t.Fatal("commit error not propagated", e)
}
if backend.calls != 0 {
t.Fatal("object request after failed journal commit")
}
var n int
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 {
t.Fatal("failed transaction left intent")
}
}
+78
View File
@@ -0,0 +1,78 @@
package objectgateway
import (
"context"
"encoding/json"
)
// Plan is frozen by the authenticated controller, not by the data client. Digest
// is the upstream write-set digest; the controller must verify that write set.
// This allowlist prevents a valid epoch from authorizing arbitrary extra keys.
type Plan struct {
Digest string `json:"digest"`
Owner string `json:"owner"`
Objects []ObjectBinding `json:"objects"`
}
type ObjectBinding struct {
Operation string `json:"operation"`
Key string `json:"key"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
}
func (p Plan) valid() bool {
if !validHash(p.Digest) || !validHash(p.Owner) {
return false
}
keys, ops := map[string]bool{}, map[string]bool{}
for _, o := range p.Objects {
r := Request{Grant: Grant{1, p.Owner, p.Digest}, Operation: o.Operation, Key: o.Key, Action: "create", SHA256: o.SHA256, Bytes: o.Bytes}
if r.validate() != nil || keys[o.Key] || ops[o.Operation] {
return false
}
keys[o.Key] = true
ops[o.Operation] = true
}
return true
}
func (p Plan) allows(r Request) bool {
op := r.Operation
if r.Action == "delete" {
op = r.CreatedBy
}
if p.Owner != r.Grant.Owner || p.Digest != r.Grant.Plan {
return false
}
for _, o := range p.Objects {
if o.Operation == op && o.Key == r.Key && o.SHA256 == r.SHA256 && o.Bytes == r.Bytes {
return true
}
}
return false
}
func (g *Gateway) InstallPlan(ctx context.Context, p Plan) error {
if !p.valid() {
return ErrConflict
}
for _, o := range p.Objects {
if o.Bytes > g.MaxBytes {
return ErrConflict
}
}
if e := g.Ready(ctx); e != nil {
return e
}
raw, _ := json.Marshal(p)
_, e := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_plans(scope,plan,body) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, p.Digest, raw)
if e != nil {
return ErrPersistence
}
var found []byte
if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, p.Digest).Scan(&found) != nil {
return ErrPersistence
}
if string(found) != string(raw) {
return ErrConflict
}
return nil
}
@@ -0,0 +1,37 @@
package objectgateway_test
import (
"errors"
g "knowledge-governance/internal/objectgateway"
"testing"
)
func TestGatewayGrantCannotAuthorizeUnplannedKey(t *testing.T) {
ctx, pool, backend := fixture(t)
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e := gateway.Advance(ctx, 0, grant); !errors.Is(e, g.ErrConflict) {
t.Fatal("unregistered plan grant", e)
}
allowed := request("allowed-plan-key", grant)
plan := planFor(grant, allowed)
if e := gateway.InstallPlan(ctx, plan); e != nil {
t.Fatal(e)
}
changed := planFor(grant, request("extra-plan-key", grant))
if e := gateway.InstallPlan(ctx, changed); !errors.Is(e, g.ErrConflict) {
t.Fatal("immutable plan rebound", e)
}
if e := gateway.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
req := request(t.Name(), grant)
if _, e := gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrConflict) {
t.Fatal("unplanned key admitted", e)
}
receipt, e := gateway.Execute(ctx, allowed, []byte("gateway fixture allowed-plan-key"))
if e != nil || receipt.State != "committed" {
t.Fatal("valid planned key refused", e)
}
cleanup(t, ctx, gateway, receipt)
}
+109
View File
@@ -0,0 +1,109 @@
// Package objectgateway implements non-preemptive object write ownership.
package objectgateway
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
)
const Protocol = "kg-object-gateway-v1"
var (
ErrHeld = errors.New("gateway_held_unresolved_intent")
ErrEpoch = errors.New("gateway_epoch_or_plan_mismatch")
ErrConflict = errors.New("gateway_identity_conflict")
ErrPersistence = errors.New("gateway_persistence_unavailable")
ErrAuthority = errors.New("gateway_control_authority_unavailable")
ErrAuthorityPrivileges = errors.New("gateway_control_reader_privileges_invalid")
ErrIsolation = errors.New("gateway_exclusive_writer_evidence_unverified")
ErrRejected = errors.New("gateway_create_rejected_existing_object")
)
type Grant struct {
Epoch int64 `json:"epoch"`
Owner string `json:"owner"`
Plan string `json:"plan"`
}
type Version struct {
ID string `json:"version_id"`
Owner string `json:"owner"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
ETag string `json:"etag"`
}
type Request struct {
Grant Grant `json:"grant"`
Operation string `json:"operation"`
Key string `json:"key"`
Action string `json:"action"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
CreatedBy string `json:"created_by,omitempty"`
Version Version `json:"version,omitempty"`
}
type Receipt struct {
Request Request `json:"request"`
State string `json:"state"`
Version Version `json:"version"`
}
// Backend never retries mutations. List is complete and scoped to exactly key.
// Not finding a PUT is never evidence that its request cannot still arrive.
type Backend interface {
Ready(context.Context) error
Observe(context.Context, string, string) (Version, bool, error)
List(context.Context, string) ([]Version, error)
Put(context.Context, string, []byte, string) (Version, error)
Delete(context.Context, string, Version) error
}
type IsolationVerifier interface{ Verify(context.Context) error }
func Digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
func validHash(s string) bool {
b, e := hex.DecodeString(s)
return e == nil && len(b) == 32 && strings.ToLower(s) == s
}
func (g Grant) valid() bool { return g.Epoch > 0 && validHash(g.Owner) && validHash(g.Plan) }
func (r Request) validate() error {
if !r.Grant.valid() || !validHash(r.Operation) || r.Key == "" || len(r.Key) > 2048 || strings.ContainsAny(r.Key, "\x00\r\n") || strings.HasPrefix(r.Key, "/") || !validHash(r.SHA256) || r.Bytes < 0 {
return ErrConflict
}
if r.Action != "create" && r.Action != "delete" {
return ErrConflict
}
if r.Action == "create" && (r.CreatedBy != "" || r.Version != (Version{})) {
return ErrConflict
}
if r.Action == "delete" && (!validHash(r.CreatedBy) || !validVersion(r.Version) || r.Version.SHA256 != r.SHA256 || r.Version.Bytes != r.Bytes) {
return ErrConflict
}
return nil
}
func validVersion(v Version) bool {
return v.ID != "" && v.ID != "null" && validHash(v.Owner) && validHash(v.SHA256) && v.Bytes >= 0 && v.ETag != ""
}
// Claim binds backend ownership to the complete immutable gateway request.
func (r Request) Claim() string { return r.token() }
func (r Request) token() string { raw, _ := json.Marshal(r); return Digest(raw) }
func (r Receipt) valid() bool {
if r.Request.validate() != nil {
return false
}
switch r.State {
case "committed":
return r.Request.Action == "create" && r.Request.matches(r.Version)
case "rejected":
return r.Request.Action == "create" && r.Version == (Version{})
case "removed":
return r.Request.Action == "delete" && r.Version == r.Request.Version
}
return false
}
func (r Request) matches(v Version) bool {
return validVersion(v) && v.Owner == r.token() && v.Bytes == r.Bytes && v.SHA256 == r.SHA256
}
+102
View File
@@ -0,0 +1,102 @@
package objectgateway
import (
"bytes"
"context"
"errors"
"io"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
)
// S3Backend is for independently versioned S3/MinIO stores. No SDK mutation retries.
type S3Backend struct {
Client *s3.Client
Bucket string
MaxBytes int64
}
func (s *S3Backend) Ready(ctx context.Context) error {
r, e := s.Client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{Bucket: aws.String(s.Bucket)})
if e != nil || string(r.Status) != "Enabled" {
return errors.New("gateway_backend_versioning_unverified")
}
return nil
}
func (s *S3Backend) Observe(ctx context.Context, key, id string) (Version, bool, error) {
req := &s3.GetObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key)}
if id != "" {
req.VersionId = aws.String(id)
}
got, e := s.Client.GetObject(ctx, req)
if e != nil {
var api smithy.APIError
if errors.As(e, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NoSuchVersion") {
return Version{}, false, nil
}
return Version{}, false, errors.New("gateway_backend_read_failed")
}
defer got.Body.Close()
if got.ContentLength == nil || *got.ContentLength < 0 || *got.ContentLength > s.MaxBytes {
return Version{}, false, errors.New("gateway_backend_size_invalid")
}
body, e := io.ReadAll(io.LimitReader(got.Body, s.MaxBytes+1))
if e != nil || int64(len(body)) != *got.ContentLength {
return Version{}, false, errors.New("gateway_backend_body_incomplete")
}
v := Version{aws.ToString(got.VersionId), got.Metadata["kg-release-owner"], Digest(body), int64(len(body)), aws.ToString(got.ETag)}
// Existing unowned objects are observable but cannot be claimed/deleted.
if v.ID == "" || v.ID == "null" || v.ETag == "" || id != "" && v.ID != id {
return Version{}, false, errors.New("gateway_backend_version_invalid")
}
return v, true, nil
}
func (s *S3Backend) List(ctx context.Context, key string) ([]Version, error) {
got, e := s.Client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(s.Bucket), Prefix: aws.String(key), MaxKeys: aws.Int32(1000)})
if e != nil || aws.ToBool(got.IsTruncated) {
return nil, errors.New("gateway_backend_listing_incomplete")
}
var versions []Version
for _, m := range got.DeleteMarkers {
if aws.ToString(m.Key) == key {
return nil, errors.New("gateway_backend_delete_marker_conflict")
}
}
for _, v := range got.Versions {
if aws.ToString(v.Key) == key {
one, exists, e := s.Observe(ctx, key, aws.ToString(v.VersionId))
if e != nil || !exists {
return nil, errors.New("gateway_backend_listing_changed")
}
versions = append(versions, one)
}
}
return versions, nil
}
func (s *S3Backend) Put(ctx context.Context, key string, body []byte, owner string) (Version, error) {
// SDK hashing requires a seekable body. RetryMaxAttempts=1 prevents SDK
// retries; PUT carries no HTTP idempotency header or replay authorization.
reader := bytes.NewReader(body)
got, e := s.Client.PutObject(v4.SetPayloadHash(ctx, Digest(body)), &s3.PutObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(int64(len(body))), Metadata: map[string]string{"kg-release-owner": owner}}, func(o *s3.Options) { o.RetryMaxAttempts = 1 })
if e != nil || aws.ToString(got.VersionId) == "" {
return Version{}, ErrHeld
}
v, exists, e := s.Observe(ctx, key, aws.ToString(got.VersionId))
if e != nil || !exists {
return Version{}, ErrHeld
}
return v, nil
}
func (s *S3Backend) Delete(ctx context.Context, key string, v Version) error {
if !validVersion(v) {
return ErrConflict
}
_, e := s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), VersionId: aws.String(v.ID)}, func(o *s3.Options) { o.RetryMaxAttempts = 1 })
if e != nil {
return ErrHeld
}
return nil
}
+74
View File
@@ -0,0 +1,74 @@
-- Independent governance journal; never install in a WeKnora business schema.
CREATE TABLE IF NOT EXISTS kg_object_gateway_plans (
scope text NOT NULL,
plan text NOT NULL,
body bytea NOT NULL,
PRIMARY KEY(scope,plan)
);
CREATE TABLE IF NOT EXISTS kg_object_gateway_scopes (
scope text PRIMARY KEY,
epoch bigint NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS kg_object_gateway_grants (
scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope),
epoch bigint NOT NULL CHECK (epoch > 0),
owner text NOT NULL,
plan text NOT NULL,
PRIMARY KEY(scope,epoch)
);
CREATE TABLE IF NOT EXISTS kg_object_gateway_intents (
scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope),
operation text NOT NULL,
key text NOT NULL,
request bytea NOT NULL,
recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(scope,operation)
);
CREATE INDEX IF NOT EXISTS kg_object_gateway_key ON kg_object_gateway_intents(scope,key);
CREATE TABLE IF NOT EXISTS kg_object_gateway_receipts (
scope text NOT NULL,
operation text NOT NULL,
receipt bytea NOT NULL,
recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(scope,operation),
FOREIGN KEY(scope,operation) REFERENCES kg_object_gateway_intents(scope,operation)
);
CREATE TABLE IF NOT EXISTS kg_object_gateway_deliveries (
scope text NOT NULL,
job text NOT NULL,
attempt_epoch bigint NOT NULL CHECK(attempt_epoch>0),
claim bytea NOT NULL,
grant_epoch bigint NOT NULL,
grant_body bytea NOT NULL,
PRIMARY KEY(scope,job,attempt_epoch),
FOREIGN KEY(scope,grant_epoch) REFERENCES kg_object_gateway_grants(scope,epoch)
);
CREATE OR REPLACE FUNCTION kg_object_gateway_immutable() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
RAISE EXCEPTION 'object gateway evidence is immutable';
END;
$$;
DO $$ BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_deliveries_immutable') THEN
CREATE TRIGGER kg_object_gateway_deliveries_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_deliveries FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_plans_immutable') THEN
CREATE TRIGGER kg_object_gateway_plans_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_plans FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_intents_immutable') THEN
CREATE TRIGGER kg_object_gateway_intents_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_intents FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_receipts_immutable') THEN
CREATE TRIGGER kg_object_gateway_receipts_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_receipts FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_grants_immutable') THEN
CREATE TRIGGER kg_object_gateway_grants_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_grants FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
END IF;
END $$;
DO $$ DECLARE t text; BEGIN
FOREACH t IN ARRAY ARRAY['plans','grants','intents','receipts','deliveries'] LOOP
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_'||t||'_no_truncate') THEN
EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION kg_object_gateway_immutable()', 'kg_object_gateway_'||t||'_no_truncate', 'kg_object_gateway_'||t);
END IF;
END LOOP;
END $$;
+60
View File
@@ -0,0 +1,60 @@
package objectgateway
import (
"context"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5"
)
// Resolve preserves the immutable send epoch and backend claim of an operation.
// An absent intent is not a claim about an earlier send; Execute still reserves
// atomically and refuses stale authorizations before any new side effect.
func (g *Gateway) Resolve(ctx context.Context, desired Request) (Request, error) {
if e := desired.validate(); e != nil {
return Request{}, e
}
if e := g.Ready(ctx); e != nil {
return Request{}, e
}
var raw []byte
e := g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, desired.Operation).Scan(&raw)
if errors.Is(e, pgx.ErrNoRows) {
return desired, nil
}
if e != nil {
return Request{}, ErrPersistence
}
var original Request
if json.Unmarshal(raw, &original) != nil || original.validate() != nil {
return Request{}, ErrPersistence
}
if !sameOperation(original, desired) || original.Grant.Epoch > desired.Grant.Epoch {
return Request{}, ErrConflict
}
return original, nil
}
func sameOperation(a, b Request) bool { a.Grant.Epoch = b.Grant.Epoch; return a == b }
func (g *Gateway) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) {
if !authorization.valid() || authorization.Owner != r.Grant.Owner || authorization.Plan != r.Grant.Plan || authorization.Epoch < r.Grant.Epoch {
return Receipt{}, ErrEpoch
}
return g.execute(ctx, r, body, &authorization)
}
func (c *Client) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) {
var out Receipt
e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body, Authorization: &authorization}, &out)
if e == nil && (out.Request != r || !out.valid()) {
e = ErrConflict
}
return out, e
}
func (c *Client) Resolve(ctx context.Context, desired Request) (Request, error) {
var out Request
e := c.call(ctx, "/v1/resolve", desired, &out)
if e == nil && (out.validate() != nil || !sameOperation(out, desired) || out.Grant.Epoch > desired.Grant.Epoch) {
e = ErrConflict
}
return out, e
}
@@ -0,0 +1,111 @@
package objectgateway_test
import (
"bytes"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
g "knowledge-governance/internal/objectgateway"
)
func TestGatewayNewEpochPreservesOriginalSendIdentity(t *testing.T) {
ctx, pool, backend := fixture(t)
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
first := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("takeover-plan"))}
sent, unsent := request("takeover-sent", first), request("takeover-unsent", first)
if e := gateway.InstallPlan(ctx, planFor(first, sent, unsent)); e != nil {
t.Fatal(e)
}
if e := gateway.Advance(ctx, 0, first); e != nil {
t.Fatal(e)
}
old, e := gateway.Execute(ctx, sent, []byte("gateway fixture takeover-sent"))
if e != nil {
t.Fatal(e)
}
// Always clean only this test's exact original version, including on RED.
cleanupGrant := first
defer func() { copy := old; copy.Request.Grant = cleanupGrant; cleanup(t, ctx, gateway, copy) }()
var original []byte
if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&original); e != nil {
t.Fatal(e)
}
next := first
next.Epoch++
if e = gateway.Advance(ctx, 1, next); e != nil {
t.Fatal(e)
}
cleanupGrant = next
data, control := g.Digest([]byte("takeover-data")), g.Digest([]byte("takeover-control"))
server := httptest.NewServer(g.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: control})
defer server.Close()
post := func(path string, in any) (int, []byte) {
raw, _ := json.Marshal(in)
req, _ := http.NewRequestWithContext(ctx, "POST", server.URL+path, bytes.NewReader(raw))
req.Header.Set("Authorization", "Bearer "+data)
req.Header.Set("X-KG-Gateway-Protocol", g.Protocol)
req.Header.Set("X-KG-Gateway-Scope", gateway.Scope)
resp, e := http.DefaultClient.Do(req)
if e != nil {
t.Fatal(e)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, body
}
code, body := post("/v1/execute", map[string]any{"request": sent, "authorization": next, "body": []byte("gateway fixture takeover-sent")})
if code != 200 {
t.Fatalf("new authorization cannot replay immutable original request: status=%d", code)
}
var replay g.Receipt
if json.Unmarshal(body, &replay) != nil || replay != old {
t.Fatal("replay rewrote original receipt")
}
desired := sent
desired.Grant = next
code, body = post("/v1/resolve", desired)
var resolved g.Request
if code != 200 || json.Unmarshal(body, &resolved) != nil || resolved != sent {
t.Fatal("original send identity not resolved", code)
}
changed := sent
changed.SHA256 = g.Digest([]byte("different"))
if code, _ = post("/v1/resolve", changed); code == 200 {
t.Fatal("operation rebound body")
}
if code, _ = post("/v1/execute", map[string]any{"request": unsent, "authorization": first, "body": []byte("gateway fixture takeover-unsent")}); code == 200 {
t.Fatal("stale epoch created unsent key")
}
unsent.Grant = next
code, body = post("/v1/execute", map[string]any{"request": unsent, "authorization": next, "body": []byte("gateway fixture takeover-unsent")})
var created g.Receipt
if code != 200 || json.Unmarshal(body, &created) != nil || created.State != "committed" {
t.Fatal("new epoch did not continue unsent key", code)
}
cleanup(t, ctx, gateway, created)
var after []byte
if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&after); e != nil || !bytes.Equal(after, original) {
t.Fatal("immutable intent changed", e)
}
versions, e := backend.List(ctx, sent.Key)
if e != nil || len(versions) != 1 || versions[0] != old.Version {
t.Fatal("replay created extra version", e)
}
// Deletion is a new send in the current epoch, not a rewritten create intent.
deletion := sent
deletion.Grant = next
deletion.Operation = g.Digest([]byte(sent.Operation + "delete"))
deletion.Action = "delete"
deletion.CreatedBy = sent.Operation
deletion.Version = old.Version
code, _ = post("/v1/execute", map[string]any{"request": deletion, "authorization": next})
if code != 200 {
t.Fatal("new epoch exact cleanup refused", code)
}
// deferred cleanup replays the same already settled delete request below.
old.Request.Grant = next
t.Log("new epoch replay retained original request/receipt/claim; unsent key committed once; stale request rejected")
}
@@ -0,0 +1,69 @@
package objectgateway_test
import (
"context"
"errors"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
g "knowledge-governance/internal/objectgateway"
)
type stoppedBeforeSend struct {
g.Backend
calls int
}
func (b *stoppedBeforeSend) Put(context.Context, string, []byte, string) (g.Version, error) {
b.calls++
return g.Version{}, errors.New("injected uncertain send with no visible version")
}
func TestGatewayAbsenceNeverSettlesUnknown(t *testing.T) {
ctx, pool, real := fixture(t)
backend := &stoppedBeforeSend{Backend: real}
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
t.Fatal(e)
}
if e := gateway.Advance(ctx, 0, grant); e != nil {
t.Fatal(e)
}
req := request(t.Name(), grant)
body := []byte("gateway fixture " + t.Name())
if _, e := gateway.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) {
t.Fatal(e)
}
restarted := *gateway
for i := 0; i < 2; i++ {
if _, e := restarted.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) {
t.Fatal("absent version treated as terminal", e)
}
}
if backend.calls != 1 {
t.Fatal("PUT replayed", backend.calls)
}
next := grant
next.Epoch++
if e := restarted.Advance(ctx, 1, next); !errors.Is(e, g.ErrHeld) {
t.Fatal("unknown operation allowed epoch handover", e)
}
var n int
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 {
t.Fatal("absence wrote false terminal receipt")
}
}
func TestGatewayReadOnlyPersistenceBlocksSend(t *testing.T) {
ctx, pool, backend := fixture(t)
cfg := pool.Config()
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
other, e := pgxpool.NewWithConfig(ctx, cfg)
if e != nil {
t.Fatal(e)
}
defer other.Close()
gateway := &g.Gateway{Pool: other, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
if e = gateway.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
t.Fatal("read-only persistence accepted", e)
}
}
@@ -0,0 +1,131 @@
package v071
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
"path/filepath"
)
// GatewayGovernanceAuthority runs only in the trusted gateway control domain.
// Its governance connection is SELECT-only; the ordinary worker never receives
// the gateway control credential or backend write credential. Archive contents
// are trusted only after their immutable digest is found in the original
// persisted prebackup event belonging to the currently leased attempt.
type GatewayGovernanceAuthority struct {
Pool *pgxpool.Pool
ArchiveRoot string
TargetID string
Provider string
Bucket string
}
func (a *GatewayGovernanceAuthority) Ready(ctx context.Context) error {
if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" {
return objectgateway.ErrAuthority
}
var safe bool
e := a.Pool.QueryRow(ctx, `SELECT NOT rolsuper AND NOT rolcreatedb AND NOT rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&safe)
if e != nil || !safe {
return objectgateway.ErrAuthorityPrivileges
}
for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
e = a.Pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT') AND NOT has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&safe)
if e != nil || !safe {
return objectgateway.ErrAuthorityPrivileges
}
}
return nil
}
func (a *GatewayGovernanceAuthority) Load(ctx context.Context, c objectgateway.Claim) (objectgateway.Plan, error) {
if e := a.Ready(ctx); e != nil {
return objectgateway.Plan{}, e
}
if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" {
return objectgateway.Plan{}, objectgateway.ErrAuthority
}
recovery, err := a.verifyRecoveryClaim(ctx, c)
if err != nil {
return objectgateway.Plan{}, err
}
var scope, backup, snapshot string
var matches int
e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',count(*) OVER()
FROM library_jobs j
JOIN library_operations o ON o.id=j.operation_id AND o.target_id=j.target_id
JOIN library_attempts a ON a.operation_id=j.operation_id AND a.epoch=j.attempt_epoch AND a.job_id=j.id
JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id AND v.scope=j.scope AND v.kind='production_prebackup' AND v.target_id=j.target_id
WHERE j.id=$1 AND j.operation_id=$2 AND j.attempt_epoch=$3 AND j.lease_owner=$4 AND j.target_id=$6
AND j.lease_until>clock_timestamp() AND (j.state='running' OR $7)
AND a.owner=$4 AND a.state IN ('claimed','running') AND a.lease_until>clock_timestamp()
AND v.evidence->>'writeset_digest'=$5
AND v.evidence->>'job'=j.id AND v.evidence->>'backup_snapshot'=v.snapshot_id
AND (o.target_snapshot_id='' OR o.target_snapshot_id=v.evidence->>'writeset_target_snapshot')
AND NOT EXISTS(SELECT 1 FROM library_attempts newer WHERE newer.operation_id=a.operation_id AND newer.epoch>a.epoch)`, c.JobID, c.OperationID, c.AttemptEpoch, c.Owner, c.PlanDigest, a.TargetID, recovery).Scan(&scope, &backup, &snapshot, &matches)
if errors.Is(e, pgx.ErrNoRows) {
return objectgateway.Plan{}, objectgateway.ErrEpoch
}
if e != nil {
return objectgateway.Plan{}, objectgateway.ErrPersistence
}
if matches != 1 {
return objectgateway.Plan{}, objectgateway.ErrConflict
}
set, e := LoadWriteSet(a.ArchiveRoot, c.PlanDigest)
if e != nil || set == nil {
return objectgateway.Plan{}, objectgateway.ErrConflict
}
if set.Version != libraryrelease.MigrationWriteSetVersion || set.JobID != c.JobID || set.Target != scope || set.BackupSnapshotID != backup || set.TargetSnapshotID != snapshot {
return objectgateway.Plan{}, objectgateway.ErrConflict
}
for _, obj := range set.Objects {
if obj.Provider != a.Provider || obj.Bucket != a.Bucket {
return objectgateway.Plan{}, objectgateway.ErrConflict
}
}
return ObjectGatewayPlan(*set)
}
// Recovery authorization does not replace c.JobID/c.OperationID (the original
// send domain). Its separate queue and operation must both be durable, leased,
// and bound to the same before-image, source, target and frozen plan.
func (a *GatewayGovernanceAuthority) verifyRecoveryClaim(ctx context.Context, c objectgateway.Claim) (bool, error) {
if c.RecoveryJobID == "" && c.RecoveryOperationID == "" {
return false, nil
}
if c.RecoveryJobID != governance.LibraryRecoveryQueueID(c.JobID) {
return false, objectgateway.ErrConflict
}
var scope, backup, source, recorded string
e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',o.request_digest
FROM library_jobs j JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id
AND v.target_id=j.target_id AND v.scope=j.scope AND v.kind='production_prebackup'
JOIN jobs q ON q.id=$3 AND q.type='library.recover' AND q.state='running'
JOIN library_operations o ON o.id=$4 AND o.kind='recover' AND o.scope=j.scope AND o.target_id=j.target_id
WHERE j.id=$1 AND j.operation_id=$2 AND j.target_id=$5
AND q.payload->>'scope'=j.scope AND q.payload->>'window_job_id'=j.id
AND COALESCE(NULLIF(q.payload->>'source_snapshot_id',''),v.evidence->>'writeset_target_snapshot')=v.evidence->>'writeset_target_snapshot'
AND COALESCE(NULLIF(q.payload->>'target_id',''),j.target_id)=j.target_id
AND COALESCE(NULLIF(q.payload->>'original_operation_id',''),j.operation_id)=j.operation_id
AND COALESCE(NULLIF(q.payload->>'backup_snapshot_id',''),v.snapshot_id)=v.snapshot_id
AND COALESCE(NULLIF(q.payload->>'plan_digest',''),$7)=$7
AND q.idempotency_key='library.recover:'||j.id AND q.lease_until>clock_timestamp()
AND q.lease_owner||'#'||q.id||'#'||q.attempt::text=$6
AND o.operation_key=q.id AND o.target_snapshot_id=v.snapshot_id
AND v.evidence->>'writeset_digest'=$7`, c.JobID, c.OperationID, c.RecoveryJobID, c.RecoveryOperationID, a.TargetID, c.Owner, c.PlanDigest).Scan(&scope, &backup, &source, &recorded)
if errors.Is(e, pgx.ErrNoRows) {
return false, objectgateway.ErrConflict
}
if e != nil {
return false, objectgateway.ErrPersistence
}
digest, e := governance.LibraryRecoveryDigest(scope, c.RecoveryJobID, c.JobID, c.OperationID, a.TargetID, source, backup, c.PlanDigest)
if e != nil || recorded != digest || c.RecoveryOperationID != governance.LibraryOperationID(scope, governance.LibraryJobRecover, c.RecoveryJobID) {
return false, objectgateway.ErrConflict
}
return true, nil
}
@@ -0,0 +1,236 @@
package v071
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/postgres/pgtest"
"net/http/httptest"
"net/url"
"os"
"sync/atomic"
"testing"
"time"
)
type gatewayCountBackend struct {
objectgateway.Backend
puts atomic.Int64
}
func (b *gatewayCountBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) {
b.puts.Add(1)
return b.Backend.Put(ctx, key, body, owner)
}
func gatewayReadOnlyGovernance(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) *pgxpool.Pool {
t.Helper()
random := make([]byte, 24)
if _, e := rand.Read(random); e != nil {
t.Fatal(e)
}
suffix := hex.EncodeToString(random)
role := "kg_gateway_reader_" + suffix[:12]
// This role is created only in the explicitly disposable fixture cluster.
if _, e := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); e != nil {
t.Fatal("create isolated reader role failed")
}
var schema string
if admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema) != nil {
t.Fatal("fixture schema")
}
t.Cleanup(func() {
cleanup := context.Background()
if _, e := admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize()); e != nil {
t.Errorf("reader role privilege cleanup failed")
}
if _, e := admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize()); e != nil {
t.Errorf("reader role cleanup failed")
}
})
if _, e := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); e != nil {
t.Fatal("reader schema grant")
}
if _, e := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); e != nil {
t.Fatal("reader table grant")
}
parsed, e := url.Parse(dsn)
if e != nil {
t.Fatal("fixture DSN")
}
parsed.User = url.UserPassword(role, suffix)
config, e := pgxpool.ParseConfig(parsed.String())
if e != nil {
t.Fatal("reader config")
}
config.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
pool, e := pgxpool.NewWithConfig(ctx, config)
if e != nil {
t.Fatal("reader connect")
}
t.Cleanup(pool.Close)
t.Log("isolated SELECT-only governance role created; exact role cleanup registered:", role)
return pool
}
// This is real authority/PG/HTTP/storage integration, not the formal worker
// process test: no grant decorator or control credential is given to the target.
func TestContractGatewayAuthorityFrozenPlanIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
ctx := fx.ctx
dsn := pgtest.IsolatedDatabaseURL(t, os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"))
governancePool, e := pgxpool.New(ctx, dsn)
if e != nil {
t.Fatal("governance pool")
}
t.Cleanup(governancePool.Close)
if e = postgresstore.ApplyMigrations(ctx, governancePool); e != nil {
t.Fatal(e)
}
if _, e = governancePool.Exec(ctx, objectgateway.Schema); e != nil {
t.Fatal(e)
}
reader := gatewayReadOnlyGovernance(t, ctx, governancePool, dsn)
const jobID = "gateway-authority-job"
const targetID = "gateway-authority-target"
store := postgresstore.NewStore(governancePool).WithLibraryTarget(targetID)
snapshot := sealSourceSnapshot(t, fx, nil)
now := time.Now().UTC()
op := governance.LibraryOperation{ID: "gateway-authority-operation", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: snapshot, RequestDigest: "gateway-authority-params", OperationKey: "gateway-authority-key", CreatedAt: now}
if _, _, e = store.EnsureLibraryOperation(ctx, op); e != nil {
t.Fatal(e)
}
if _, _, e = store.CreateLibraryJob(ctx, governance.LibraryJob{ID: jobID, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: op.ID, IdempotencyKey: jobID, RequestDigest: op.RequestDigest, TargetSnapshotID: snapshot}, governance.SyncStages(), now); e != nil {
t.Fatal(e)
}
attempt, e := store.BeginLibraryAttempt(ctx, jobID, "worker-first", time.Hour, now)
if e != nil {
t.Fatal(e)
}
if _, e = governancePool.Exec(ctx, "UPDATE library_jobs SET state='running' WHERE id=$1", jobID); e != nil {
t.Fatal(e)
}
authority := &GatewayGovernanceAuthority{Pool: reader, ArchiveRoot: fx.archiveRoot, TargetID: targetID, Provider: "minio", Bucket: fx.bucket}
if e = authority.Ready(ctx); e != nil {
t.Fatal(e)
}
privileged := *authority
privileged.Pool = governancePool
if privileged.Ready(ctx) == nil {
t.Fatal("privileged controller reader accepted")
}
backend := &gatewayCountBackend{Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}}
gateway := &objectgateway.Gateway{Pool: governancePool, Scope: fx.bucket, Backend: backend, Authority: authority, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20}
data := objectgateway.Digest([]byte("authority-data"))
server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: objectgateway.Digest([]byte("authority-control"))})
defer server.Close()
client, e := objectgateway.NewClient(server.URL, data, fx.bucket)
if e != nil {
t.Fatal(e)
}
claim := objectgateway.Claim{JobID: jobID, OperationID: op.ID, AttemptEpoch: int64(attempt.Epoch), Owner: attempt.Owner}
objects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: claim}
target := fx.target(t)
target.config.Objects = objects
current, e := target.Current(ctx)
if e != nil {
t.Fatal(e)
}
backup, e := fx.backups(t, target).Prepare(ctx, snapshot, current)
if e != nil {
t.Fatal(e)
}
set, e := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: jobID, Target: "internal", BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshot, Universe: backup.Universe})
if e != nil {
t.Fatal(e)
}
claim.PlanDigest = set.Digest
before := universeDump(t, fx.pool)
if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e == nil {
t.Fatal("unanchored plan permitted first PG write")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatal("unanchored apply mutated PG", diff)
}
if e = store.RecordLibraryVersion(ctx, governance.LibraryVersion{ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, "internal", backup.SnapshotID, op.ID), Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: jobID, OperationID: op.ID, AttemptEpoch: attempt.Epoch, AttemptOwner: attempt.Owner, CreatedAt: now, Evidence: map[string]any{"job": jobID, "backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, "writeset_digest": set.Digest, "writeset_target_snapshot": snapshot}}); e != nil {
t.Fatal(e)
}
for _, bad := range []objectgateway.Claim{{JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: "other-worker", PlanDigest: set.Digest}, {JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: claim.Owner, PlanDigest: objectgateway.Digest([]byte("replacement"))}} {
if _, e = client.Acquire(ctx, bad); e == nil {
t.Fatal("untrusted claim admitted")
}
}
if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e != nil {
t.Fatal(e)
}
if e = target.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil {
t.Fatal(e)
}
count := backend.puts.Load()
if count == 0 {
t.Fatal("no actual object send")
}
firstRequests := objects.requests
if e = store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptFailed, "isolated takeover", time.Now().UTC()); e != nil {
t.Fatal(e)
}
next, e := store.BeginLibraryAttempt(ctx, jobID, "worker-second", time.Hour, time.Now().UTC())
if e != nil {
t.Fatal(e)
}
nextClaim := claim
nextClaim.AttemptEpoch = int64(next.Epoch)
nextClaim.Owner = next.Owner
nextObjects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: nextClaim}
nextTarget := fx.target(t)
nextTarget.config.Objects = nextObjects
if e = nextTarget.AdoptWriteSet(set); e != nil {
t.Fatal(e)
}
if e = nextObjects.EnsureObjectGrant(ctx); e != nil {
t.Fatal(e)
}
for key, original := range firstRequests {
if nextObjects.requests[key] != original {
t.Fatal("send identity replaced during new epoch", key)
}
}
if nextObjects.authorization.Epoch <= objects.authorization.Epoch {
t.Fatal("authorization did not advance")
}
if _, e = client.Acquire(ctx, claim); !errors.Is(e, objectgateway.ErrEpoch) {
t.Fatal("old active owner admitted", e)
}
if e = nextTarget.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil {
t.Fatal(e)
}
if backend.puts.Load() != count {
t.Fatal("resumed object send duplicated")
}
for _, obj := range set.Objects {
if obj.Existing {
continue
}
locator := obj.Locator
observation, e := nextObjects.ObserveObject(ctx, obj.Provider, locator)
if e != nil {
t.Fatal(e)
}
if e = nextObjects.RemoveObject(ctx, obj.Provider, locator, observation); e != nil {
t.Fatal(e)
}
versions, e := backend.List(ctx, obj.Key)
if e != nil || len(versions) != 0 {
t.Fatal("created object version residue", e)
}
}
t.Logf("real readonly governance authority -> data-only HTTP acquisition -> PG/MinIO; first-PG anchor denial, new epoch unchanged send identity, PUTs=%d, original version exact cleanup", count)
}
@@ -0,0 +1,66 @@
package v071
import (
"context"
"errors"
"knowledge-governance/internal/objectgateway"
)
// NewGatewayBackend is server-side only. Clients never receive this credentialed object.
func NewGatewayBackend(c S3ObjectStoreConfig) (objectgateway.Backend, error) {
if c.Provider == "oss" {
s, e := newOSSVersionStore(c)
if e != nil {
return nil, e
}
return ossGatewayBackend{s}, nil
}
if c.Provider != "s3" && c.Provider != "minio" {
return nil, errors.New("gateway_backend_provider_unsupported")
}
s, e := NewS3ObjectStore(c)
if e != nil {
return nil, e
}
return &objectgateway.S3Backend{Client: s.client, Bucket: s.bucket, MaxBytes: s.maxObjectBytes}, nil
}
type ossGatewayBackend struct{ s *ossVersionStore }
func toGateway(v objectObservation) objectgateway.Version {
return objectgateway.Version{ID: v.VersionID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag}
}
func fromGateway(v objectgateway.Version) objectObservation {
return objectObservation{Exists: true, VersionID: v.ID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag}
}
func (b ossGatewayBackend) Ready(ctx context.Context) error { return b.s.versioning(ctx) }
func (b ossGatewayBackend) Observe(ctx context.Context, key, id string) (objectgateway.Version, bool, error) {
v, e := b.s.observeVersion(ctx, key, id)
return toGateway(v), v.Exists, e
}
func (b ossGatewayBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) {
v, e := b.s.appendVersion(ctx, key, body, owner)
return toGateway(v), e
}
func (b ossGatewayBackend) Delete(ctx context.Context, key string, v objectgateway.Version) error {
return b.s.removeVersion(ctx, key, fromGateway(v))
}
func (b ossGatewayBackend) List(ctx context.Context, key string) ([]objectgateway.Version, error) {
listing, e := b.s.listExactVersions(ctx, key)
if e != nil {
return nil, e
}
if len(listing.DeleteMarkers) != 0 {
return nil, errors.New("gateway_backend_delete_marker_conflict")
}
var out []objectgateway.Version
for _, v := range listing.Versions {
o, e := b.s.observeVersion(ctx, key, v.VersionID)
if e != nil || !o.Exists {
return nil, errors.New("gateway_backend_listing_changed")
}
out = append(out, toGateway(o))
}
return out, nil
}
@@ -0,0 +1,49 @@
package v071
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
)
// PrepareObjectGatewayGrant runs only in the trusted control plane. The data
// worker receives the immutable grant file, never the control bearer token.
// Replay uses the original expected epoch; it does not guess the latest epoch.
func PrepareObjectGatewayGrant(ctx context.Context, client *objectgateway.Client, set libraryrelease.WriteSet, expected int64, provider, bucket, directory string) error {
if client == nil || !filepath.IsAbs(directory) || expected < 0 {
return errors.New("gateway grant controller configuration invalid")
}
plan, e := ObjectGatewayPlan(set)
if e != nil {
return e
}
for _, obj := range set.Objects {
if obj.Provider != provider || obj.Bucket != bucket {
return errors.New("gateway grant target does not match frozen object plan")
}
}
grant := objectgateway.Grant{Epoch: expected + 1, Owner: plan.Owner, Plan: plan.Digest}
if grant.Epoch < 1 {
return errors.New("gateway epoch exhausted")
}
raw, e := json.Marshal(grant)
if e != nil {
return e
}
if e = persistImmutableJSON(filepath.Join(directory, set.Digest+".intent.json"), raw); e != nil {
return e
}
// Network outcomes may be unknown. Both endpoints compare immutable identity
// on a repeat, so no caller invents another epoch or another operation ID.
if e = client.InstallPlan(ctx, plan); e != nil {
return e
}
if e = client.Advance(ctx, expected, grant); e != nil {
return e
}
return persistImmutableJSON(filepath.Join(directory, set.Digest+".json"), raw)
}
@@ -0,0 +1,250 @@
package v071
import (
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"strings"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
)
// GatewayObjectStore has no write credentials or direct-write fallback. The
// separate reader must be deployed with read-only rights (audit prerequisite).
type GatewayObjectStore struct {
Automatic bool
Claim objectgateway.Claim
frozen *libraryrelease.WriteSet
authorization objectgateway.Grant
Client *objectgateway.Client
Provider, Bucket, GrantDirectory string
requests map[string]objectgateway.Request
owners map[string]string
}
func (s *GatewayObjectStore) ConditionalCapability(ctx context.Context) error {
if s.Client == nil || s.Provider == "" || s.Bucket == "" || (!s.Automatic && !filepath.IsAbs(s.GrantDirectory)) {
return errors.New("object gateway configuration/grant directory missing")
}
if s.Automatic {
return s.Client.ReadyForAcquisition(ctx)
}
return s.Client.Ready(ctx)
}
// ObjectGatewayPlan derives the controller allowlist from the verified original
// write set. Existing shared objects remain read-only, never write-authorized.
func ObjectGatewayPlan(set libraryrelease.WriteSet) (objectgateway.Plan, error) {
if err := set.Verify(); err != nil {
return objectgateway.Plan{}, err
}
p := objectgateway.Plan{Digest: set.Digest, Owner: objectgateway.Digest([]byte(set.JobID))}
for _, obj := range set.Objects {
if !obj.Existing {
p.Objects = append(p.Objects, objectgateway.ObjectBinding{Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, SHA256: obj.SHA256, Bytes: obj.Bytes})
}
}
return p, nil
}
func gatewayCreateOperation(plan, owner string) string {
return objectgateway.Digest([]byte(plan + "\x00" + owner + "\x00create"))
}
func (s *GatewayObjectStore) BindObjectPlan(set libraryrelease.WriteSet) error {
if !validLowerSHA256(set.Digest) || set.JobID == "" {
return errors.New("gateway frozen plan identity missing")
}
if _, e := ObjectGatewayPlan(set); e != nil {
return e
}
if s.frozen != nil && s.frozen.Digest != set.Digest {
return errors.New("gateway store cannot replace frozen identity")
}
if s.Automatic {
if s.frozen == nil {
copy := set
s.frozen = &copy
}
return nil
}
path := filepath.Join(s.GrantDirectory, set.Digest+".json")
info, e := os.Lstat(path)
if e != nil || !info.Mode().IsRegular() || info.Size() > 4096 || info.Mode().Perm()&0022 != 0 {
return errors.New("gateway controller grant unavailable")
}
raw, e := os.ReadFile(path)
if e != nil {
return errors.New("gateway controller grant unavailable")
}
var grant objectgateway.Grant
if json.Unmarshal(raw, &grant) != nil || grant.Epoch < 1 || grant.Plan != set.Digest || grant.Owner != objectgateway.Digest([]byte(set.JobID)) {
return errors.New("gateway controller grant differs from frozen job/plan")
}
return s.bindGrant(context.Background(), set, grant)
}
func (s *GatewayObjectStore) EnsureObjectGrant(ctx context.Context) error {
if !s.Automatic || s.requests != nil {
return nil
}
if s.frozen == nil || s.Claim.JobID != s.frozen.JobID {
return errors.New("gateway active attempt/frozen plan missing")
}
claim := s.Claim
claim.PlanDigest = s.frozen.Digest
grant, e := s.Client.Acquire(ctx, claim)
if e != nil {
return e
}
return s.bindGrant(ctx, *s.frozen, grant)
}
func (s *GatewayObjectStore) bindGrant(ctx context.Context, set libraryrelease.WriteSet, grant objectgateway.Grant) error {
requests := map[string]objectgateway.Request{}
owners := map[string]string{}
for _, obj := range set.Objects {
if obj.Provider != s.Provider || obj.Bucket != s.Bucket {
return errors.New("gateway plan target mismatch")
}
if obj.Existing {
continue
}
req := objectgateway.Request{Grant: grant, Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, Action: "create", SHA256: obj.SHA256, Bytes: obj.Bytes}
original, e := s.Client.Resolve(ctx, req)
if e != nil {
return e
}
requests[obj.Key] = original
owners[obj.Key] = obj.OwnerToken
}
s.authorization = grant
s.requests = requests
s.owners = owners
return nil
}
func (s *GatewayObjectStore) key(provider, locator string) (string, error) {
if provider != s.Provider || !validObjectKey(locator) {
return "", errors.New("gateway provider/locator mismatch")
}
parts := strings.SplitN(strings.TrimPrefix(locator, "storage://"), "/", 2)
if !strings.HasPrefix(locator, "storage://") || len(parts) != 2 || parts[0] == "" {
return "", errors.New("gateway explicit locator required")
}
prefix := s.Provider + "://" + s.Bucket + "/"
if !strings.HasPrefix(parts[1], prefix) {
return "", errors.New("gateway bucket mismatch")
}
key := strings.TrimPrefix(parts[1], prefix)
if key == "" {
return "", errors.New("gateway key missing")
}
return key, nil
}
func (s *GatewayObjectStore) ObserveObject(ctx context.Context, provider, locator string) (objectObservation, error) {
if s.frozen != nil {
if e := s.EnsureObjectGrant(ctx); e != nil {
return objectObservation{}, e
}
}
key, e := s.key(provider, locator)
if e != nil {
return objectObservation{}, e
}
v, exists, e := s.Client.Observe(ctx, key, "")
if e != nil {
return objectObservation{}, e
}
if !exists {
return objectObservation{}, nil
}
o := fromGateway(v)
if req, ok := s.requests[key]; ok && v.Owner == req.Claim() {
o.Owner = s.owners[key]
}
return o, nil
}
func (s *GatewayObjectStore) CreateObject(ctx context.Context, provider, locator string, body io.Reader, size int64, owner string) error {
if e := s.EnsureObjectGrant(ctx); e != nil {
return e
}
key, e := s.key(provider, locator)
if e != nil {
return e
}
r, ok := s.requests[key]
if !ok || owner != s.owners[key] || size != r.Bytes {
return errObjectWriteRejected
}
raw, e := io.ReadAll(io.LimitReader(body, size+1))
if e != nil || int64(len(raw)) != size || objectgateway.Digest(raw) != r.SHA256 {
return errObjectWriteRejected
}
receipt, e := s.Client.ExecuteAuthorized(ctx, s.authorization, r, raw)
if e != nil {
return e
}
if receipt.State == "rejected" {
return errObjectWriteRejected
}
if receipt.State != "committed" {
return errors.New("gateway create not committed")
}
return nil
}
func (s *GatewayObjectStore) RemoveObject(ctx context.Context, provider, locator string, want objectObservation) error {
if e := s.EnsureObjectGrant(ctx); e != nil {
return e
}
key, e := s.key(provider, locator)
if e != nil {
return e
}
create, ok := s.requests[key]
if !ok || want.Owner != s.owners[key] {
return errors.New("gateway cleanup lacks frozen ownership")
}
receipt, e := s.Client.Reconcile(ctx, create)
if e != nil {
return e
}
raw := want
raw.Owner = create.Claim()
if receipt.State != "committed" || receipt.Version != toGateway(raw) {
return errors.New("gateway cleanup receipt mismatch")
}
req := create
req.Grant = s.authorization
req.Operation = objectgateway.Digest([]byte(create.Operation + "\x00delete"))
req.Action = "delete"
req.CreatedBy = create.Operation
req.Version = receipt.Version
original, e := s.Client.Resolve(ctx, req)
if e != nil {
return e
}
deleted, e := s.Client.ExecuteAuthorized(ctx, s.authorization, original, nil)
if e != nil {
return e
}
if deleted.State != "removed" {
return errors.New("gateway deletion not confirmed")
}
return nil
}
func (s *GatewayObjectStore) Put(context.Context, string, string, io.Reader, int64) error {
return errors.New("gateway refuses unplanned Put; frozen object plan required")
}
func (s *GatewayObjectStore) Copy(context.Context, string, string, string) error {
return errors.New("gateway refuses unplanned Copy; frozen object plan required")
}
func (s *GatewayObjectStore) Stat(ctx context.Context, provider, locator string) (int64, string, error) {
o, e := s.ObserveObject(ctx, provider, locator)
if e != nil {
return 0, "", e
}
if !o.Exists {
return 0, "", errors.New("gateway object absent")
}
return o.Bytes, o.SHA256, nil
}
@@ -0,0 +1,86 @@
package v071
import (
"context"
"net/http/httptest"
"testing"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/objectgateway"
)
type gatewayFixtureIsolation struct{}
func (gatewayFixtureIsolation) Verify(context.Context) error { return nil }
// Test controller issues the grant only after the saga has frozen its actual
// plan. Production controller delivery is a separate integration prerequisite.
type gatewayFixtureController struct {
*GatewayObjectStore
control *objectgateway.Client
}
func (s *gatewayFixtureController) BindObjectPlan(set libraryrelease.WriteSet) error {
if e := PrepareObjectGatewayGrant(context.Background(), s.control, set, 0, s.Provider, s.Bucket, s.GrantDirectory); e != nil {
return e
}
return s.GatewayObjectStore.BindObjectPlan(set)
}
func TestContractGatewayActualSagaRollbackIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
if _, e := fx.pool.Exec(fx.ctx, objectgateway.Schema); e != nil {
t.Fatal(e)
}
before := universeDump(t, fx.pool)
snapshot := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
gateway := &objectgateway.Gateway{Pool: fx.pool, Scope: fx.bucket, Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20}
token := objectgateway.Digest([]byte("data-fixture"))
server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: token, ControlToken: objectgateway.Digest([]byte("control-fixture"))})
defer server.Close()
client, e := objectgateway.NewClient(server.URL, token, fx.bucket)
if e != nil {
t.Fatal(e)
}
store := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, GrantDirectory: t.TempDir()}
control, e := objectgateway.NewClient(server.URL, objectgateway.Digest([]byte("control-fixture")), fx.bucket)
if e != nil {
t.Fatal(e)
}
target.config.Objects = &gatewayFixtureController{store, control}
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, fx.backups(t, target), nil)
if outcome.State != "rolled_back" {
t.Fatalf("actual gateway saga: %+v", outcome)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatal(diff)
}
key := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-src")
observed, e := store.ObserveObject(fx.ctx, "minio", key)
if e != nil || observed.Exists {
t.Fatal("gateway object residue", e)
}
physical, e := store.key("minio", key)
if e != nil {
t.Fatal(e)
}
versions, e := gateway.Backend.List(fx.ctx, physical)
if e != nil || len(versions) != 0 {
t.Fatal("exact created key version residue", e)
}
var n int
if fx.pool.QueryRow(fx.ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", fx.bucket).Scan(&n) != nil || n != 2 {
t.Fatalf("gateway saga receipts=%d", n)
}
t.Log("actual release.Run -> plan adoption -> gateway HTTP -> PG/MinIO -> late Wiki failure -> exact-version rollback; all original PG rows retained")
}
func TestGatewayUnplannedWriteNeverFallsBack(t *testing.T) {
store := &GatewayObjectStore{}
if store.Put(context.Background(), "oss", "key", nil, 0) == nil || store.Copy(context.Background(), "oss", "from", "to") == nil {
t.Fatal("unplanned direct-write fallback")
}
if store.BindObjectPlan(libraryrelease.WriteSet{}) == nil {
t.Fatal("unbound plan accepted")
}
}
@@ -0,0 +1,27 @@
package v071
import (
"context"
"testing"
)
func TestLegacyObjectImporterCannotStageOrPromote(t *testing.T) {
data := []byte("hello")
snapshot := ObjectSnapshot{Objects: []AssetObject{{ResourceID: "res-1", Handle: "ABCDEFGHIJKLMNOPQRSTUV", Provider: "minio", PhysicalPath: "storage://source-backend/10000/knowledge/source.txt", Size: 5, SHA256: shaBytes(data), Data: data}}, TotalBytes: 5}
snapshot.SHA256 = objectSnapshotDigest(snapshot.Objects)
store := &fakeObjectStore{objects: map[string][]byte{}}
importer, e := NewObjectImporter(store, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "minio"})
if e != nil {
t.Fatal(e)
}
if _, e = importer.Stage(context.Background(), "legacy-bypass", snapshot); e == nil {
t.Fatal("legacy importer emitted unplanned stage PUT")
}
stage := ObjectStage{ReleaseRevision: "legacy-bypass", Provider: "minio", Objects: []StagedObject{{Handle: snapshot.Objects[0].Handle, StagingPath: "stage/object", FinalPath: "target/object", Size: 5, SHA256: shaBytes(data)}}}
if e = importer.Promote(context.Background(), stage); e == nil {
t.Fatal("legacy importer emitted unplanned promotion COPY")
}
if len(store.objects) != 0 {
t.Fatal("legacy importer mutated object store")
}
}