feat(object-gateway): 统一对象写网关与不可变治理日志
方案A:所有对象写入经 loopback 网关,持久计划/grant/intent/逐版本 receipt 与 attempt→grant delivery 落在治理 PG;scope 由 provider+endpoint+bucket 决定, 同一物理桶只允许一个写域。版本 append 不等于 CAS,未锚定计划在首笔业务写前拒绝。 本提交只含网关与其适配层接线、协议文档;生产独占、凭据切换与桶设置未改动。
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
// This command belongs to the separately authenticated controller. It must not
|
||||
// be installed with control credentials in an ordinary data worker container.
|
||||
func main() {
|
||||
if e := run(); e != nil {
|
||||
fmt.Fprintln(os.Stderr, e)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
path := flag.String("plan", "", "exact frozen write-set JSON")
|
||||
directory := flag.String("grant-directory", "", "private durable grant output directory")
|
||||
expected := flag.Int64("expected-epoch", -1, "previous epoch from the frozen controller intent; never guessed")
|
||||
flag.Parse()
|
||||
if *path == "" || *directory == "" || *expected < 0 || flag.NArg() != 0 {
|
||||
return errors.New("exact plan, grant directory and expected epoch required")
|
||||
}
|
||||
f, e := os.Open(*path)
|
||||
if e != nil {
|
||||
return errors.New("frozen gateway plan unavailable")
|
||||
}
|
||||
defer f.Close()
|
||||
var set libraryrelease.WriteSet
|
||||
d := json.NewDecoder(io.LimitReader(f, 16<<20))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&set) != nil || d.Decode(&struct{}{}) != io.EOF {
|
||||
return errors.New("frozen gateway plan invalid")
|
||||
}
|
||||
provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET")
|
||||
if provider == "" || endpoint == "" || bucket == "" {
|
||||
return errors.New("exact gateway target configuration required")
|
||||
}
|
||||
scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket))
|
||||
client, e := objectgateway.NewClient(os.Getenv("OBJECT_GATEWAY_URL"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN"), scope)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
if e = v071.PrepareObjectGatewayGrant(ctx, client, set, *expected, provider, bucket, *directory); e != nil {
|
||||
return e
|
||||
}
|
||||
fmt.Printf("gateway grant persisted: plan=%s epoch=%d\n", set.Digest, *expected+1)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
log.Print(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
ctx := context.Background()
|
||||
addr := os.Getenv("OBJECT_GATEWAY_LISTEN")
|
||||
host, _, e := net.SplitHostPort(addr)
|
||||
// A reviewed TLS terminator may expose this loopback-only listener. Never bind
|
||||
// authenticated plaintext to a LAN address by an accidental config default.
|
||||
if e != nil || (host != "127.0.0.1" && host != "::1") {
|
||||
return errors.New("gateway_requires_loopback_listener")
|
||||
}
|
||||
data, control := os.Getenv("OBJECT_GATEWAY_DATA_TOKEN"), os.Getenv("OBJECT_GATEWAY_CONTROL_TOKEN")
|
||||
if len(data) < 32 || len(control) < 32 || data == control {
|
||||
return errors.New("gateway_distinct_tokens_required")
|
||||
}
|
||||
pub, e := base64.StdEncoding.DecodeString(os.Getenv("OBJECT_GATEWAY_AUDIT_PUBLIC_KEY"))
|
||||
if e != nil || len(pub) != ed25519.PublicKeySize {
|
||||
return errors.New("gateway_auditor_key_required")
|
||||
}
|
||||
provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET")
|
||||
scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket))
|
||||
pool, e := pgxpool.New(ctx, os.Getenv("OBJECT_GATEWAY_DATABASE_URL"))
|
||||
if e != nil {
|
||||
return objectgateway.ErrPersistence
|
||||
}
|
||||
defer pool.Close()
|
||||
backend, e := v071.NewGatewayBackend(v071.S3ObjectStoreConfig{Provider: provider, Endpoint: endpoint, Bucket: bucket, Region: os.Getenv("OBJECT_GATEWAY_REGION"), AccessKeyID: os.Getenv("OBJECT_GATEWAY_ACCESS_KEY"), SecretAccessKey: os.Getenv("OBJECT_GATEWAY_SECRET_KEY"), MaxObjectBytes: 256 * 1024 * 1024})
|
||||
if e != nil {
|
||||
return errors.New("gateway_backend_configuration_invalid")
|
||||
}
|
||||
g := &objectgateway.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 256 * 1024 * 1024, Isolation: objectgateway.AuditVerifier{Path: os.Getenv("OBJECT_GATEWAY_AUDIT_FILE"), PublicKey: pub, Scope: scope, GatewayID: os.Getenv("OBJECT_GATEWAY_ID")}}
|
||||
if governanceURL := os.Getenv("OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL"); governanceURL != "" {
|
||||
cfg, e := pgxpool.ParseConfig(governanceURL)
|
||||
if e != nil {
|
||||
return errors.New("gateway_control_authority_unavailable")
|
||||
}
|
||||
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
cfg.MaxConns = 4
|
||||
governancePool, e := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if e != nil {
|
||||
return errors.New("gateway_control_authority_unavailable")
|
||||
}
|
||||
defer governancePool.Close()
|
||||
authority := &v071.GatewayGovernanceAuthority{Pool: governancePool, ArchiveRoot: os.Getenv("OBJECT_GATEWAY_ARCHIVE_ROOT"), TargetID: os.Getenv("OBJECT_GATEWAY_TARGET_ID"), Provider: provider, Bucket: bucket}
|
||||
if e = authority.Ready(ctx); e != nil {
|
||||
return e
|
||||
}
|
||||
g.Authority = authority
|
||||
}
|
||||
probe, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if e = g.Ready(probe); e != nil {
|
||||
return e
|
||||
}
|
||||
server := &http.Server{Addr: addr, Handler: objectgateway.HTTPServer{Gateway: g, DataToken: data, ControlToken: control}, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 3 * time.Minute, WriteTimeout: 3 * time.Minute, IdleTimeout: 30 * time.Second}
|
||||
if server.ListenAndServe() != nil {
|
||||
return errors.New("gateway_listener_stopped")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLegacyReleaseRuntimeRefusesBeforeCredentials(t *testing.T) {
|
||||
// Empty credential paths would fail differently if route assembly were reached.
|
||||
runtime, e := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{})
|
||||
if runtime != nil || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() {
|
||||
t.Fatal("legacy runtime did not fail at the no-write gate", e)
|
||||
}
|
||||
route, pools, e := buildReleaseRoute(context.Background(), "", releaseTargetSnapshotConfig{}, releaseRouteConfig{})
|
||||
if route != nil || len(pools) != 0 || e == nil || e.Error() != v071.CheckLegacyObjectWrites().Error() {
|
||||
t.Fatal("direct route bypassed the no-write gate", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
# 对象写入网关(开发候选,尚不可部署业务)
|
||||
|
||||
## 协议与持久化
|
||||
|
||||
`cmd/object-gateway` 提供 `kg-object-gateway-v1`,只监听 loopback;外部接入需要另行审核的 TLS 终结与网络隔离。治理 PG 保存不可变计划、grant、intent、逐版本 receipt 和 attempt→grant delivery。部署迁移身份显式应用 `internal/objectgateway/schema.sql`;启动不自动迁移。运行身份只授必要 SELECT/INSERT 和 scope 行 UPDATE,不授 DDL/TRUNCATE/证据修改或业务库写权限。
|
||||
|
||||
scope 是 provider、原 endpoint、bucket 的 NUL 分隔 SHA256。同一物理桶只能有一个受审核 scope 和一套共享 journal,不能用 endpoint 别名或另一日志库建立平行写域。
|
||||
|
||||
数据认证端点:`POST /v1/acquire`、`/v1/resolve`、`/v1/execute`、`/v1/reconcile`、`/v1/observe`,以及 `GET /v1/capabilities`。所有请求核验协议和 scope。`capabilities.automatic_grants` 必须为真,正式 stack 才接受自动授权能力;未接控制源、只读角色权限异常、协议不符或不可达均拒绝,无直写回退。
|
||||
|
||||
PG 行锁负责原子准入,不是远端存储 fence。intent 必须持久提交后才发送一次请求;连接死亡、租约过期不清除未知 intent。任何未终态 intent 均阻止 scope 换 epoch。重复 operation 只读回执或精确对账,不重发 PUT;暂时查不到版本仍 held。删除必须绑定已确认 create 的精确 VersionID,不删 latest、不生成删除标记。
|
||||
|
||||
不可变证据表拒绝 UPDATE/DELETE/TRUNCATE。Ready 核 fsync、synchronous_commit、full_page_writes、主库可写及必要表/触发器;管理员与迁移身份仍是可信边界,触发器不能抵御 DBA 的 DDL。
|
||||
|
||||
## 正式控制交付路径
|
||||
|
||||
生产代码路径已接入,并已在专属本机隔离栈运行正式 worker 的两次 SIGKILL 接管;这不是生产部署验收:
|
||||
|
||||
`cmd/worker.runOwnedLibraryAttempt → attempt context → librarystack → GatewayObjectStore(Automatic) → ProductionLibraryTarget.Apply → EnsureObjectGrant → /v1/acquire → GatewayGovernanceAuthority → 原持久 WriteSet → journal delivery → 数据 execute`。
|
||||
|
||||
冻结/采用计划时只保存自校验后的 WriteSet,不提前索取 grant;observer 先把摘要锚定到原预备份事件,首笔业务 PG Apply 前才请求授权。worker 只提交 job/operation/attempt epoch/owner/plan digest,不提交可替换的 key/body allowlist,也不能选择网关 epoch。
|
||||
|
||||
可信控制域使用独立 SELECT-only 治理连接,核对目标身份、job/operation、当前 attempt、owner、有效租约、running 状态和原预备份事件中的摘要、job、backup、目标快照;再从归档加载并验 hash、范围、provider/bucket。客户端自述不能替代这些持久事实。控制读角色不得是 superuser、CREATEDB、CREATEROLE,不能写/截断/建触发器到这些治理表。每次加载和能力查询重验角色。
|
||||
|
||||
网关在 scope 锁内再次核对授权事实,原子写入 grant、当前 epoch、不可变 delivery。同一 attempt 重试返回原 grant;owner/operation/plan 替换拒绝;同一 job 的后续 attempt 必须沿用原 operation 和 plan。旧发送未终态时,新 attempt 可先只读对账,但无充分证据仍 held,不能交接。
|
||||
|
||||
控制服务配置了治理 authority 后,旧 `/v1/plans`、`/v1/advance` 手动控制端点关闭,控制 token 也不能绕过原计划绑定。旧 `cmd/object-gateway-control` 仅保留离线/协议兼容路径,不是正式 worker 的交付方案。
|
||||
|
||||
## 独立恢复请求
|
||||
|
||||
`library.recover` 的队列ID由原窗口job确定;它具有独立恢复operation与request digest,绑定scope、queue、原window/operation、target、source、backup和原WriteSet。原计划的job、operation和摘要不改写。Claim中的`recovery_job_id`与`recovery_operation_id`仅为独立授权上下文,原`job_id`/`operation_id`仍是发送域。网关重新读取持久队列payload、running状态、当前owner/attempt/有效租约与恢复operation,不信任worker自行声明。可选显式来源/目标/backup/plan字段只能收紧原绑定,不能覆盖它。
|
||||
|
||||
SELECT-only治理角色需要读取`library_jobs`、`library_operations`、`library_attempts`、`library_versions`、`jobs`,均不能INSERT/UPDATE/DELETE/TRUNCATE/TRIGGER。已有原窗口恢复请求时不自动重启原发布;过期running恢复队列只在相同target、未释放维护窗口、原业务租约也过期时重新投递,业务attempt/epoch仍重新申请。
|
||||
|
||||
## 索引unknown恢复边界
|
||||
|
||||
原冻结before-image非空时,恢复先持久化`restore-mark-intent`,再以原CAS(或确认缺失的create)写入包含原计划/index/id归属标记的完整before-image。丢响应后仅匹配完整标记正文及有效CAS才能登记`restore-claimed`并CAS去标记;已知回执的代次不一致拒绝。恢复完成的`restored`回执仍须匹配正文和CAS,不能接受同正文的新代次。
|
||||
|
||||
去标记前持久化`restore-finalize-intent`。发布clean-finalization或恢复clean-finalization已提交但响应丢失时,当前干净正文并非充分归属证据;继续held,不盲删、不重发、不把相同外观当回执。旧unknown缺少写前标记intent亦不自动升级认领。本候选没有服务端可验证的clean提交关联凭证,因此这部分自动收敛仍是明确缺口,不是可部署完整恢复保证。
|
||||
|
||||
## 跨 epoch 的不可变发送身份
|
||||
|
||||
`Request.Grant` 继续代表首次发送的不可变身份,后端 claim、intent 和 receipt 都不改。新 `ExecuteRequest.authorization` 仅代表本次准入权限。`/v1/resolve` 根据相同 operation/key/action/hash/length/owner/plan 取回旧请求;只允许 epoch 差异,不能换内容。
|
||||
|
||||
新 attempt 获得新 grant 后,已发送 key 沿用旧 request/claim,未发送 key 使用新授权 epoch 建立首次 intent。当前授权必须匹配 journal 的当前 epoch;旧 epoch 的新写拒绝。新 epoch 补偿删除仍使用原 create 回执和精确版本,删除本身有独立不可变发送身份。
|
||||
|
||||
## 配置与凭据归属
|
||||
|
||||
网关进程环境:
|
||||
|
||||
- `OBJECT_GATEWAY_LISTEN`:仅 `127.0.0.1:<port>` 或 `[::1]:<port>`。
|
||||
- `OBJECT_GATEWAY_DATABASE_URL`:独立 journal 连接,迁移与运行角色分离。
|
||||
- `OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL`:仅可信网关持有的 SELECT-only 治理读连接;独立于 journal 运行身份和普通 worker 身份。
|
||||
- `OBJECT_GATEWAY_ARCHIVE_ROOT`:原持久计划所在归档绝对路径,部署只读挂载。
|
||||
- `OBJECT_GATEWAY_TARGET_ID`:与正式 worker `LibraryTargetIdentity()` 完全相同的逻辑目标身份;不能用另一个环境的值。
|
||||
- `OBJECT_GATEWAY_PROVIDER`、`OBJECT_GATEWAY_ENDPOINT`、`OBJECT_GATEWAY_BUCKET`、`OBJECT_GATEWAY_REGION`:精确存储身份。
|
||||
- `OBJECT_GATEWAY_ACCESS_KEY`、`OBJECT_GATEWAY_SECRET_KEY`:仅网关持有的对象写凭据。
|
||||
- `OBJECT_GATEWAY_DATA_TOKEN`、`OBJECT_GATEWAY_CONTROL_TOKEN`:不同且至少 32 字节;普通 worker 仅持有 data token。
|
||||
- `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`:独立审核者的 Ed25519 签名隔离证据,绑定协议、scope、gateway ID、时间窗及 writer/policy/拒写/排空摘要。审核私钥不能交给 worker 或网关。
|
||||
|
||||
秘密只经服务环境/私有文件注入,不进 argv、URL 或日志。所有调用端错误都使用固定原因码。
|
||||
|
||||
正式 worker 使用 `LIBRARY_OBJECT_GATEWAY_URL` 和 `LIBRARY_OBJECT_GATEWAY_TOKEN`;不再依赖预先生成的 grant 文件。`LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY` 保留旧离线兼容用途,不是自动模式的授权来源。发布 stack 对所有对象 provider 强制网关;S3 构造仍用于只读捕获/预备份,部署必须使用独立只读凭据。
|
||||
|
||||
## 已关闭的本仓旁路
|
||||
|
||||
- `ObjectImporter.Stage/Promote` 无条件拒绝,不留配置开关,不执行 PUT/COPY。
|
||||
- legacy `release-worker` runtime/route 在读凭据及连接目标前拒绝。
|
||||
- `release-diag`、`release-channel-e2e` 旧写型入口启动即非零退出,避免 probe PUT 或文档级旁路。
|
||||
- `librarystack` 不再为缺网关的 MinIO/S3 返回直写 writer。
|
||||
- `libraryctl` capture 的 S3 构造与读取保留;未把读取当作写入口禁用。
|
||||
|
||||
这不等于已经撤销运行环境旧进程的凭据;源码变更尚未部署。`librarydrill` 等无 owned-attempt 的旧写型工具不能靠静态 grant 绕过自动模式,需补正式执行身份后才可用于写入。低层 S3 adapter 仍用于网关后端、读取及显式隔离夹具;部署权限才负责防止外部旧进程直接调用存储。
|
||||
|
||||
## 实测与未完成边界
|
||||
|
||||
已实跑:真实治理 PG(独立 SELECT-only 登录角色)、原 archive、数据 HTTP、业务 PG 与 MinIO 的自动授权/跨 epoch 链;未锚定计划在第一笔 PG 前拒绝,接管保持原发送身份且不增加 PUT。另有真实网关二进制的两次中断验证。隔离审计 authority/故障注入仍是测试设施,不是生产权限证明。
|
||||
|
||||
正式 worker 隔离进程证据:真实 Docker 入口闸门/队列、只读 worker 对象身份、独立治理读取角色、网关与业务 PG/ES/MinIO;首次 PG 后和对象提交后两次 SIGKILL,同一原 job/operation/plan 的三代 attempt 完成发布,原对象发送 epoch 与回执不变。隔离构建标签 `kg_isolated_drill` 仅把真实租约缩短至4秒,并拒绝非 loopback 一次性 r8 库;正常构建仍为10分钟。维护上游是无业务凭据的独立 HTTP 夹具,不是运行中的 WeKnora;签名证据绑定本次真实拒写/排空材料,不代表生产独占审核。
|
||||
|
||||
已补独立 `library.recover` 正式worker链、恢复中再次SIGKILL、完整before-image/CAS/精确版本读回,以及来源/目标/原job/计划/operation/任意队列身份拒绝。ES故障代理只转发真实ES并丢弃已提交响应,未伪造服务响应。
|
||||
|
||||
尚未完成:clean-finalization unknown的充分归属与自动收敛;独立运行环境只读对象凭据、ACL/网络和全部writer排空;网关真实OSS端到端;原D0/F11/F12与55项其余缺口。测试/隔离验收与真实部署独占分列,不把生产变更授权当成本机开发授权的前置。
|
||||
|
||||
维护隔离、独占审核与 grant 是独立前置。配置 true、健康 200、测试自签 authority 都不能代替已部署拒写/排空证据。目前未签发真实环境独占证明,未改生产权限或上游 WeKnora 源码/Schema。
|
||||
|
||||
## 部署与回退(未执行)
|
||||
|
||||
未来测试主机 `120.76.217.51`、生产 `120.76.218.137` 各自独立批准。每批先冻结主机、桶、全部角色/STS/预签名写入口、端口/网络规则、影响、维护窗口与回滚;当前没有执行这些外部变更。
|
||||
|
||||
先关新写准入→排空并对账旧请求→撤旧直写权限/切只读身份并实测拒写→独立审核→批准的新限定对象验收→父复核,才可切业务客户端。r5 的已完成 OSS 测试授权不能重复使用。
|
||||
|
||||
回退先关准入并保留 journal;所有已发送请求有终态才可撤网关,unknown 继续 held。不得因网关故障自动恢复旧直写凭据。权限/网络回退须另批。当前整模块 INCOMPLETE,禁止部署部分候选。
|
||||
@@ -0,0 +1,159 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"math"
|
||||
)
|
||||
|
||||
// Claim names an already persisted application attempt and write-set anchor.
|
||||
// It never carries keys, bodies, a replacement plan or a chosen gateway epoch.
|
||||
type Claim struct {
|
||||
JobID string `json:"job_id"`
|
||||
OperationID string `json:"operation_id"`
|
||||
AttemptEpoch int64 `json:"attempt_epoch"`
|
||||
Owner string `json:"owner"`
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
RecoveryJobID string `json:"recovery_job_id,omitempty"`
|
||||
RecoveryOperationID string `json:"recovery_operation_id,omitempty"`
|
||||
}
|
||||
type PlanAuthority interface {
|
||||
Load(context.Context, Claim) (Plan, error)
|
||||
}
|
||||
|
||||
func (g *Gateway) Acquire(ctx context.Context, c Claim) (Grant, error) {
|
||||
if g.Authority == nil {
|
||||
return Grant{}, ErrAuthority
|
||||
}
|
||||
if c.JobID == "" || c.OperationID == "" || c.Owner == "" || c.AttemptEpoch < 1 || !validHash(c.PlanDigest) {
|
||||
return Grant{}, ErrEpoch
|
||||
}
|
||||
if e := g.Ready(ctx); e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
plan, e := g.Authority.Load(ctx, c)
|
||||
if e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
if plan.Digest != c.PlanDigest || plan.Owner != Digest([]byte(c.JobID)) {
|
||||
return Grant{}, ErrConflict
|
||||
}
|
||||
if e = g.InstallPlan(ctx, plan); e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
// Recover only sends in this exact frozen plan. No absent observation can
|
||||
// settle an unknown send; the locked advancement below still checks all intents.
|
||||
rows, e := g.Pool.Query(ctx, `SELECT i.request FROM kg_object_gateway_intents i
|
||||
LEFT JOIN kg_object_gateway_receipts r ON r.scope=i.scope AND r.operation=i.operation
|
||||
WHERE i.scope=$1 AND r.operation IS NULL`, g.Scope)
|
||||
if e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
var pending []Request
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
var request Request
|
||||
if rows.Scan(&raw) != nil || json.Unmarshal(raw, &request) != nil {
|
||||
rows.Close()
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
if request.Grant.Plan == plan.Digest {
|
||||
pending = append(pending, request)
|
||||
}
|
||||
}
|
||||
e = rows.Err()
|
||||
rows.Close()
|
||||
if e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
for _, request := range pending {
|
||||
if request.Grant.Owner != plan.Owner {
|
||||
return Grant{}, ErrConflict
|
||||
}
|
||||
if _, e = g.Reconcile(ctx, request); e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
}
|
||||
tx, e := g.Pool.Begin(ctx)
|
||||
if e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
var epoch int64
|
||||
if e = tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch); e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
// Recheck after serialization. This is admission evidence, not an OSS fence:
|
||||
// a persisted uncertain object intent still prevents every epoch transition.
|
||||
verified, e := g.Authority.Load(ctx, c)
|
||||
if e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
frozen, _ := json.Marshal(plan)
|
||||
again, _ := json.Marshal(verified)
|
||||
if !bytes.Equal(frozen, again) {
|
||||
return Grant{}, ErrConflict
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
var oldClaim, oldGrant []byte
|
||||
e = tx.QueryRow(ctx, "SELECT claim,grant_body FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch=$3", g.Scope, c.JobID, c.AttemptEpoch).Scan(&oldClaim, &oldGrant)
|
||||
if e == nil {
|
||||
var grant Grant
|
||||
if !bytes.Equal(raw, oldClaim) || json.Unmarshal(oldGrant, &grant) != nil || !grant.valid() {
|
||||
return Grant{}, ErrConflict
|
||||
}
|
||||
if grant.Epoch != epoch {
|
||||
return Grant{}, ErrEpoch
|
||||
}
|
||||
return grant, nil
|
||||
}
|
||||
if !errors.Is(e, pgx.ErrNoRows) {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
var firstRaw []byte
|
||||
firstErr := tx.QueryRow(ctx, "SELECT claim FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 ORDER BY attempt_epoch LIMIT 1", g.Scope, c.JobID).Scan(&firstRaw)
|
||||
if firstErr == nil {
|
||||
var first Claim
|
||||
if json.Unmarshal(firstRaw, &first) != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
if first.JobID != c.JobID || first.OperationID != c.OperationID || first.PlanDigest != c.PlanDigest {
|
||||
return Grant{}, ErrConflict
|
||||
}
|
||||
} else if !errors.Is(firstErr, pgx.ErrNoRows) {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
var stale bool
|
||||
if tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM kg_object_gateway_deliveries WHERE scope=$1 AND job=$2 AND attempt_epoch>=$3)", g.Scope, c.JobID, c.AttemptEpoch).Scan(&stale) != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
if stale || epoch == math.MaxInt64 {
|
||||
return Grant{}, ErrEpoch
|
||||
}
|
||||
next := Grant{Epoch: epoch + 1, Owner: plan.Owner, Plan: plan.Digest}
|
||||
if e = g.advanceLocked(ctx, tx, epoch, next); e != nil {
|
||||
return Grant{}, e
|
||||
}
|
||||
grantRaw, _ := json.Marshal(next)
|
||||
if _, e = tx.Exec(ctx, "INSERT INTO kg_object_gateway_deliveries(scope,job,attempt_epoch,claim,grant_epoch,grant_body) VALUES($1,$2,$3,$4,$5,$6)", g.Scope, c.JobID, c.AttemptEpoch, raw, next.Epoch, grantRaw); e != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
if tx.Commit(ctx) != nil {
|
||||
return Grant{}, ErrPersistence
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
func (c *Client) Acquire(ctx context.Context, claim Claim) (Grant, error) {
|
||||
var out Grant
|
||||
e := c.call(ctx, "/v1/acquire", claim, &out)
|
||||
if e == nil && (!out.valid() || out.Plan != claim.PlanDigest || out.Owner != Digest([]byte(claim.JobID))) {
|
||||
e = ErrConflict
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type currentPlanAuthority struct {
|
||||
claim g.Claim
|
||||
plan g.Plan
|
||||
}
|
||||
|
||||
func (a *currentPlanAuthority) Load(_ context.Context, c g.Claim) (g.Plan, error) {
|
||||
if c != a.claim {
|
||||
return g.Plan{}, g.ErrEpoch
|
||||
}
|
||||
return a.plan, nil
|
||||
}
|
||||
func TestGatewayAcquisitionPersistsAttemptDelivery(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
claim := g.Claim{JobID: "job-acquire", OperationID: "operation-acquire", AttemptEpoch: 1, Owner: "worker-1", PlanDigest: g.Digest([]byte(t.Name()))}
|
||||
expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest}
|
||||
req := request("acquired-object", expected)
|
||||
authority := ¤tPlanAuthority{claim, planFor(expected, req)}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
|
||||
grant, e := gateway.Acquire(ctx, claim)
|
||||
if e != nil || grant != expected {
|
||||
t.Fatal("anchored active attempt grant unavailable", e)
|
||||
}
|
||||
replay, e := gateway.Acquire(ctx, claim)
|
||||
if e != nil || replay != grant {
|
||||
t.Fatal("grant delivery replay changed epoch", e)
|
||||
}
|
||||
oldClaim := claim
|
||||
claim.AttemptEpoch = 2
|
||||
claim.Owner = "worker-2"
|
||||
authority.claim = claim
|
||||
next, e := gateway.Acquire(ctx, claim)
|
||||
if e != nil || next.Epoch != 2 {
|
||||
t.Fatal("next active attempt", e)
|
||||
}
|
||||
if _, e = gateway.Acquire(ctx, oldClaim); !errors.Is(e, g.ErrEpoch) {
|
||||
t.Fatal("stale attempt acquired grant", e)
|
||||
}
|
||||
replacement := claim
|
||||
replacement.AttemptEpoch++
|
||||
replacement.PlanDigest = g.Digest([]byte("replacement-plan"))
|
||||
authority.claim = replacement
|
||||
authority.plan.Digest = replacement.PlanDigest
|
||||
if _, e = gateway.Acquire(ctx, replacement); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("new attempt replaced original plan", e)
|
||||
}
|
||||
authority.claim = claim
|
||||
authority.plan.Digest = claim.PlanDigest
|
||||
req.Grant = next
|
||||
receipt, e := gateway.ExecuteAuthorized(ctx, next, req, []byte("gateway fixture acquired-object"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cleanup(t, ctx, gateway, receipt)
|
||||
var count int
|
||||
if e = pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 2 {
|
||||
t.Fatal("immutable delivery count", count, e)
|
||||
}
|
||||
for _, statement := range []string{"UPDATE kg_object_gateway_deliveries SET job=job", "DELETE FROM kg_object_gateway_deliveries", "TRUNCATE kg_object_gateway_deliveries"} {
|
||||
if _, e = pool.Exec(ctx, statement); e == nil {
|
||||
t.Fatal("mutable grant delivery", statement)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayAcquisitionCannotHandOverUnknownSend(t *testing.T) {
|
||||
ctx, pool, real := fixture(t)
|
||||
backend := &stoppedBeforeSend{Backend: real}
|
||||
claim := g.Claim{JobID: "job-held", OperationID: "operation-held", AttemptEpoch: 1, Owner: "worker-first", PlanDigest: g.Digest([]byte(t.Name()))}
|
||||
expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest}
|
||||
req := request("acquire-held", expected)
|
||||
body := []byte("gateway fixture acquire-held")
|
||||
authority := ¤tPlanAuthority{claim, planFor(expected, req)}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
|
||||
if _, e := gateway.Acquire(ctx, claim); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := gateway.ExecuteAuthorized(ctx, expected, req, body); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
claim.AttemptEpoch++
|
||||
claim.Owner = "worker-next"
|
||||
authority.claim = claim
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, e := gateway.Acquire(ctx, claim); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("absent send handed over", e)
|
||||
}
|
||||
}
|
||||
if backend.calls != 1 {
|
||||
t.Fatal("control acquisition re-sent PUT")
|
||||
}
|
||||
var count int
|
||||
if e := pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_deliveries WHERE scope=$1", gateway.Scope).Scan(&count); e != nil || count != 1 {
|
||||
t.Fatal("unknown send advanced delivery", count, e)
|
||||
}
|
||||
// Deliver the previously uncertain request late, using its exact immutable claim.
|
||||
if _, e := real.Put(ctx, req.Key, body, req.Claim()); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
next, e := gateway.Acquire(ctx, claim)
|
||||
if e != nil || next.Epoch != 2 {
|
||||
t.Fatal("committed send was not reconciled before handover", e)
|
||||
}
|
||||
receipt, e := gateway.ExecuteAuthorized(ctx, next, req, body)
|
||||
if e != nil || receipt.Request != req {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if backend.calls != 1 {
|
||||
t.Fatal("reconciled send was retried")
|
||||
}
|
||||
receipt.Request.Grant = next
|
||||
cleanup(t, ctx, gateway, receipt)
|
||||
t.Log("new controller attempt stayed held while absent; late committed original recovered without replay PUT")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attestation is issued by an independent deployment auditor, never by a worker.
|
||||
// Its signature authenticates an audit, not the truth of self-reported config.
|
||||
// Runtime cutover must invalidate/replace the audit whenever writer rights change.
|
||||
type Attestation struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Scope string `json:"scope"`
|
||||
GatewayID string `json:"gateway_id"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
WriterInventory string `json:"writer_inventory_sha256"`
|
||||
PolicyEvidence string `json:"policy_evidence_sha256"`
|
||||
DenialEvidence string `json:"denial_evidence_sha256"`
|
||||
DrainEvidence string `json:"drain_evidence_sha256"`
|
||||
}
|
||||
type SignedAttestation struct {
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
Signature []byte `json:"signature"`
|
||||
}
|
||||
type AuditVerifier struct {
|
||||
Path string
|
||||
PublicKey ed25519.PublicKey
|
||||
Scope, GatewayID string
|
||||
}
|
||||
|
||||
func (v AuditVerifier) Verify(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
st, e := os.Lstat(v.Path)
|
||||
if e != nil || !st.Mode().IsRegular() || st.Size() > 16384 || st.Mode().Perm()&0022 != 0 {
|
||||
return ErrIsolation
|
||||
}
|
||||
raw, e := os.ReadFile(v.Path)
|
||||
if e != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
var signed SignedAttestation
|
||||
var a Attestation
|
||||
if json.Unmarshal(raw, &signed) != nil || len(v.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(v.PublicKey, signed.Payload, signed.Signature) || json.Unmarshal(signed.Payload, &a) != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
now := time.Now()
|
||||
if a.Protocol != Protocol || a.Scope != v.Scope || a.GatewayID != v.GatewayID || a.GatewayID == "" || a.IssuedAt.After(now) || !a.ExpiresAt.After(now) || a.ExpiresAt.Sub(a.IssuedAt) > time.Hour || !validHash(a.WriterInventory) || !validHash(a.PolicyEvidence) || !validHash(a.DenialEvidence) || !validHash(a.DrainEvidence) {
|
||||
return ErrIsolation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGatewayAuditCannotBeConfigurationBoolean(t *testing.T) {
|
||||
pub, priv, e := ed25519.GenerateKey(rand.Reader)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
verifier := g.AuditVerifier{Path: path, PublicKey: pub, Scope: "target", GatewayID: "gateway"}
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("missing audit accepted")
|
||||
}
|
||||
if os.WriteFile(path, []byte(`{"isolated":true,"ready":true}`), 0600) != nil {
|
||||
t.Fatal("fixture")
|
||||
}
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("boolean accepted as isolation proof")
|
||||
}
|
||||
a := g.Attestation{Protocol: g.Protocol, Scope: "target", GatewayID: "gateway", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("inventory")), PolicyEvidence: g.Digest([]byte("policy")), DenialEvidence: g.Digest([]byte("denials")), DrainEvidence: g.Digest([]byte("drain"))}
|
||||
write := func(v g.Attestation) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(v)
|
||||
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
|
||||
if os.WriteFile(path, signed, 0600) != nil {
|
||||
t.Fatal("fixture")
|
||||
}
|
||||
}
|
||||
write(a)
|
||||
if verifier.Verify(context.Background()) != nil {
|
||||
t.Fatal("valid test auditor signature rejected")
|
||||
}
|
||||
for _, mutate := range []func(*g.Attestation){func(v *g.Attestation) { v.Scope = "other" }, func(v *g.Attestation) { v.GatewayID = "other" }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *g.Attestation) { v.DenialEvidence = "" }, func(v *g.Attestation) { v.IssuedAt = time.Now().Add(time.Hour) }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(24 * time.Hour) }} {
|
||||
bad := a
|
||||
mutate(&bad)
|
||||
write(bad)
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("invalid isolation audit accepted")
|
||||
}
|
||||
}
|
||||
write(a)
|
||||
other, _, _ := ed25519.GenerateKey(rand.Reader)
|
||||
verifier.PublicKey = other
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("untrusted signer accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package objectgateway
|
||||
|
||||
import "context"
|
||||
|
||||
type claimContextKey struct{}
|
||||
|
||||
// WithClaim is set by the owned-attempt runner, never from an HTTP caller's
|
||||
// proposed plan. The controller independently checks every field against PG.
|
||||
func WithClaim(ctx context.Context, c Claim) context.Context {
|
||||
return context.WithValue(ctx, claimContextKey{}, c)
|
||||
}
|
||||
func ClaimFromContext(ctx context.Context) Claim {
|
||||
c, _ := ctx.Value(claimContextKey{}).(Claim)
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Schema is applied explicitly by the migration identity, never on server startup.
|
||||
//
|
||||
//go:embed schema.sql
|
||||
var Schema string
|
||||
|
||||
type Gateway struct {
|
||||
Authority PlanAuthority
|
||||
Pool *pgxpool.Pool
|
||||
Scope string
|
||||
Backend Backend
|
||||
Isolation IsolationVerifier
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
func (g *Gateway) Ready(ctx context.Context) error {
|
||||
if g.Pool == nil || g.Scope == "" || g.Backend == nil || g.MaxBytes < 1 {
|
||||
return ErrPersistence
|
||||
}
|
||||
if g.Isolation == nil || g.Isolation.Verify(ctx) != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
var sync, fsync, full string
|
||||
if g.Pool.QueryRow(ctx, "SELECT current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes')").Scan(&sync, &fsync, &full) != nil || sync != "on" || fsync != "on" || full != "on" {
|
||||
return ErrPersistence
|
||||
}
|
||||
var journal bool
|
||||
if g.Pool.QueryRow(ctx, `SELECT NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off'
|
||||
AND to_regclass('kg_object_gateway_plans') IS NOT NULL
|
||||
AND to_regclass('kg_object_gateway_scopes') IS NOT NULL
|
||||
AND to_regclass('kg_object_gateway_grants') IS NOT NULL
|
||||
AND to_regclass('kg_object_gateway_intents') IS NOT NULL
|
||||
AND to_regclass('kg_object_gateway_receipts') IS NOT NULL
|
||||
AND to_regclass('kg_object_gateway_deliveries') IS NOT NULL
|
||||
AND (SELECT count(*)=10 FROM pg_trigger WHERE tgname IN ('kg_object_gateway_deliveries_immutable','kg_object_gateway_deliveries_no_truncate','kg_object_gateway_plans_no_truncate','kg_object_gateway_intents_no_truncate','kg_object_gateway_receipts_no_truncate','kg_object_gateway_grants_no_truncate','kg_object_gateway_plans_immutable','kg_object_gateway_intents_immutable','kg_object_gateway_receipts_immutable','kg_object_gateway_grants_immutable') AND tgenabled='O')`).Scan(&journal) != nil || !journal {
|
||||
return ErrPersistence
|
||||
}
|
||||
if g.Backend.Ready(ctx) != nil {
|
||||
return errors.New("gateway_backend_not_ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (g *Gateway) Advance(ctx context.Context, expected int64, next Grant) error {
|
||||
if !next.valid() || next.Epoch != expected+1 {
|
||||
return ErrEpoch
|
||||
}
|
||||
if err := g.Ready(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var frozen []byte
|
||||
var plan Plan
|
||||
if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, next.Plan).Scan(&frozen) != nil {
|
||||
return ErrConflict
|
||||
}
|
||||
if json.Unmarshal(frozen, &plan) != nil || !plan.valid() || plan.Owner != next.Owner {
|
||||
return ErrConflict
|
||||
}
|
||||
tx, err := g.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_scopes(scope) VALUES($1) ON CONFLICT DO NOTHING", g.Scope); err != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
var epoch int64
|
||||
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
if err = g.advanceLocked(ctx, tx, epoch, next); err != nil {
|
||||
return err
|
||||
}
|
||||
if tx.Commit(ctx) != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gateway) advanceLocked(ctx context.Context, tx pgx.Tx, epoch int64, next Grant) error {
|
||||
var err error
|
||||
if epoch == next.Epoch {
|
||||
var owner, plan string
|
||||
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
if owner != next.Owner || plan != next.Plan {
|
||||
return ErrEpoch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if epoch != next.Epoch-1 {
|
||||
return ErrEpoch
|
||||
}
|
||||
var pending bool
|
||||
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts r WHERE r.scope=i.scope AND r.operation=i.operation))`, g.Scope).Scan(&pending) != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
if pending {
|
||||
return ErrHeld
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_grants(scope,epoch,owner,plan) VALUES($1,$2,$3,$4)", g.Scope, next.Epoch, next.Owner, next.Plan); err != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "UPDATE kg_object_gateway_scopes SET epoch=$2 WHERE scope=$1", g.Scope, next.Epoch); err != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reserve commits an immutable may-send intent. Even a commit timeout cannot be
|
||||
// followed by a send. Another process can reconcile, but cannot retry this PUT.
|
||||
func (g *Gateway) reserve(ctx context.Context, r Request, authorization *Grant) (bool, error) {
|
||||
tx, err := g.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var epoch int64
|
||||
if tx.QueryRow(ctx, "SELECT epoch FROM kg_object_gateway_scopes WHERE scope=$1 FOR UPDATE", g.Scope).Scan(&epoch) != nil {
|
||||
return false, ErrEpoch
|
||||
}
|
||||
if authorization != nil {
|
||||
var owner, plan string
|
||||
if epoch != authorization.Epoch {
|
||||
return false, ErrEpoch
|
||||
}
|
||||
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
if owner != authorization.Owner || plan != authorization.Plan {
|
||||
return false, ErrEpoch
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(r)
|
||||
var previous []byte
|
||||
err = tx.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&previous)
|
||||
if err == nil {
|
||||
if string(previous) != string(raw) {
|
||||
return false, ErrConflict
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
var owner, plan string
|
||||
if epoch != r.Grant.Epoch {
|
||||
return false, ErrEpoch
|
||||
}
|
||||
if tx.QueryRow(ctx, "SELECT owner,plan FROM kg_object_gateway_grants WHERE scope=$1 AND epoch=$2", g.Scope, epoch).Scan(&owner, &plan) != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
if owner != r.Grant.Owner || plan != r.Grant.Plan {
|
||||
return false, ErrEpoch
|
||||
}
|
||||
var pending bool
|
||||
if tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM kg_object_gateway_intents i WHERE i.scope=$1 AND i.key=$2 AND NOT EXISTS(SELECT 1 FROM kg_object_gateway_receipts x WHERE x.scope=i.scope AND x.operation=i.operation))`, g.Scope, r.Key).Scan(&pending) != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
if pending {
|
||||
return false, ErrHeld
|
||||
}
|
||||
var frozen []byte
|
||||
var allowed Plan
|
||||
if tx.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, r.Grant.Plan).Scan(&frozen) != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
if json.Unmarshal(frozen, &allowed) != nil || !allowed.valid() || !allowed.allows(r) {
|
||||
return false, ErrConflict
|
||||
}
|
||||
if r.Action == "delete" {
|
||||
previous, err := g.receipt(ctx, r.CreatedBy)
|
||||
if err != nil || previous.State != "committed" || previous.Request.Key != r.Key || previous.Version != r.Version {
|
||||
return false, ErrConflict
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO kg_object_gateway_intents(scope,operation,key,request) VALUES($1,$2,$3,$4)", g.Scope, r.Operation, r.Key, raw); err != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
if tx.Commit(ctx) != nil {
|
||||
return false, ErrPersistence
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
func (g *Gateway) receipt(ctx context.Context, id string) (Receipt, error) {
|
||||
var raw []byte
|
||||
var r Receipt
|
||||
err := g.Pool.QueryRow(ctx, "SELECT receipt FROM kg_object_gateway_receipts WHERE scope=$1 AND operation=$2", g.Scope, id).Scan(&raw)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
if json.Unmarshal(raw, &r) != nil {
|
||||
return r, ErrPersistence
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
func (g *Gateway) finish(ctx context.Context, r Request, state string, v Version) (Receipt, error) {
|
||||
receipt := Receipt{r, state, v}
|
||||
raw, _ := json.Marshal(receipt)
|
||||
_, err := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_receipts(scope,operation,receipt) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, r.Operation, raw)
|
||||
if err != nil {
|
||||
return Receipt{}, ErrPersistence
|
||||
}
|
||||
got, err := g.receipt(ctx, r.Operation)
|
||||
if err != nil {
|
||||
return Receipt{}, ErrPersistence
|
||||
}
|
||||
if got != receipt {
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
func (g *Gateway) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) {
|
||||
return g.execute(ctx, r, body, nil)
|
||||
}
|
||||
func (g *Gateway) execute(ctx context.Context, r Request, body []byte, authorization *Grant) (Receipt, error) {
|
||||
if err := r.validate(); err != nil {
|
||||
return Receipt{}, err
|
||||
}
|
||||
if r.Bytes > g.MaxBytes || r.Action == "create" && (int64(len(body)) != r.Bytes || Digest(body) != r.SHA256) || r.Action == "delete" && len(body) != 0 {
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
if err := g.Ready(ctx); err != nil {
|
||||
return Receipt{}, err
|
||||
}
|
||||
fresh, err := g.reserve(ctx, r, authorization)
|
||||
if err != nil {
|
||||
return Receipt{}, err
|
||||
}
|
||||
if !fresh {
|
||||
return g.Reconcile(ctx, r)
|
||||
}
|
||||
if r.Action == "create" {
|
||||
_, exists, err := g.Backend.Observe(ctx, r.Key, "")
|
||||
if err != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
if exists {
|
||||
return g.finish(ctx, r, "rejected", Version{})
|
||||
}
|
||||
v, err := g.Backend.Put(ctx, r.Key, body, r.token())
|
||||
if err != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
exact, exists, err := g.Backend.Observe(ctx, r.Key, v.ID)
|
||||
if err != nil || !exists || !r.matches(exact) || v != exact {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
return g.finish(ctx, r, "committed", v)
|
||||
}
|
||||
v, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
|
||||
if err != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
if exists && v != r.Version {
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
if exists {
|
||||
if g.Backend.Delete(ctx, r.Key, r.Version) != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
}
|
||||
return g.Reconcile(ctx, r)
|
||||
}
|
||||
func (g *Gateway) Reconcile(ctx context.Context, r Request) (Receipt, error) {
|
||||
if err := r.validate(); err != nil {
|
||||
return Receipt{}, err
|
||||
}
|
||||
if err := g.Ready(ctx); err != nil {
|
||||
return Receipt{}, err
|
||||
}
|
||||
var raw []byte
|
||||
if g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, r.Operation).Scan(&raw) != nil {
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
want, _ := json.Marshal(r)
|
||||
if string(raw) != string(want) {
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
got, err := g.receipt(ctx, r.Operation)
|
||||
if err == nil {
|
||||
return got, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Receipt{}, ErrPersistence
|
||||
}
|
||||
if r.Action == "delete" {
|
||||
_, exists, err := g.Backend.Observe(ctx, r.Key, r.Version.ID)
|
||||
if err != nil || exists {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
return g.finish(ctx, r, "removed", r.Version)
|
||||
}
|
||||
versions, err := g.Backend.List(ctx, r.Key)
|
||||
if err != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
var found []Version
|
||||
for _, v := range versions {
|
||||
if v.Owner == r.token() {
|
||||
if !r.matches(v) {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
found = append(found, v)
|
||||
}
|
||||
}
|
||||
if len(found) != 1 {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
exact, exists, err := g.Backend.Observe(ctx, r.Key, found[0].ID)
|
||||
if err != nil || !exists || exact != found[0] {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
return g.finish(ctx, r, "committed", exact)
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
type verifiedFixture struct{}
|
||||
|
||||
func (verifiedFixture) Verify(context.Context) error { return nil }
|
||||
func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("isolated gateway PG/MinIO opt-in required")
|
||||
}
|
||||
u, e := url.Parse(dsn)
|
||||
if e != nil || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "kg_library_test_gateway_") || u.Hostname() != "127.0.0.1" {
|
||||
t.Fatal("unsafe gateway test database")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, e := pgxpool.New(ctx, dsn)
|
||||
if e != nil {
|
||||
t.Fatal("PG unavailable")
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if _, e = pool.Exec(ctx, g.Schema); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT")
|
||||
if endpoint != "http://127.0.0.1:19100" {
|
||||
t.Fatal("unsafe test MinIO endpoint")
|
||||
}
|
||||
bucket := "kg-library-test-gateway-" + g.Digest([]byte(t.Name() + time.Now().String()))[:16]
|
||||
client := s3.New(s3.Options{Region: "us-east-1", BaseEndpoint: aws.String(endpoint), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "")})
|
||||
if _, e = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); e != nil {
|
||||
t.Fatal("create isolated bucket failed")
|
||||
}
|
||||
if _, e = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
list, e := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
|
||||
if e != nil {
|
||||
t.Error("residue list unavailable")
|
||||
return
|
||||
}
|
||||
if len(list.Versions) != 0 || len(list.DeleteMarkers) != 0 {
|
||||
t.Errorf("preserving residue bucket %s: versions=%d markers=%d", bucket, len(list.Versions), len(list.DeleteMarkers))
|
||||
return
|
||||
}
|
||||
if _, e = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); e != nil {
|
||||
t.Error("delete empty test bucket failed")
|
||||
}
|
||||
t.Logf("exact sandbox %s: versions=0 markers=0; empty bucket deleted", bucket)
|
||||
})
|
||||
return ctx, pool, &g.S3Backend{Client: client, Bucket: bucket, MaxBytes: 1 << 20}
|
||||
}
|
||||
func planFor(grant g.Grant, requests ...g.Request) g.Plan {
|
||||
p := g.Plan{Digest: grant.Plan, Owner: grant.Owner}
|
||||
for _, r := range requests {
|
||||
p.Objects = append(p.Objects, g.ObjectBinding{Operation: r.Operation, Key: r.Key, SHA256: r.SHA256, Bytes: r.Bytes})
|
||||
}
|
||||
return p
|
||||
}
|
||||
func request(label string, grant g.Grant) g.Request {
|
||||
body := []byte("gateway fixture " + label)
|
||||
return g.Request{Grant: grant, Operation: g.Digest([]byte(label)), Key: "unchanged/" + label, Action: "create", SHA256: g.Digest(body), Bytes: int64(len(body))}
|
||||
}
|
||||
func cleanup(t *testing.T, ctx context.Context, gw *g.Gateway, receipt g.Receipt) {
|
||||
t.Helper()
|
||||
r := receipt.Request
|
||||
r.Operation = g.Digest([]byte(r.Operation + "delete"))
|
||||
r.Action = "delete"
|
||||
r.CreatedBy = receipt.Request.Operation
|
||||
r.Version = receipt.Version
|
||||
if got, e := gw.Execute(ctx, r, nil); e != nil || got.State != "removed" {
|
||||
t.Fatalf("exact cleanup: %v %s", e, got.State)
|
||||
}
|
||||
}
|
||||
func TestGatewayReadinessRequiresJournal(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
// An isolated search_path lacks the journal while PG settings remain healthy.
|
||||
conn, e := pool.Acquire(ctx)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
conn.Release()
|
||||
cfg := pool.Config()
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_catalog"
|
||||
other, e := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer other.Close()
|
||||
gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
|
||||
t.Fatalf("missing durable journal accepted: %v", e)
|
||||
}
|
||||
}
|
||||
func TestGatewayRealReplayAndImmutableLedger(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists {
|
||||
t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists)
|
||||
}
|
||||
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := gw.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
req := request(t.Name(), grant)
|
||||
body := []byte("gateway fixture " + t.Name())
|
||||
one, e := gw.Execute(ctx, req, body)
|
||||
if e != nil || one.State != "committed" {
|
||||
t.Fatalf("create %v %s", e, one.State)
|
||||
}
|
||||
two, e := gw.Execute(ctx, req, body)
|
||||
if e != nil || two != one {
|
||||
t.Fatal("replay changed receipt", e)
|
||||
}
|
||||
bad := req
|
||||
bad.Key += "-different"
|
||||
if _, e = gw.Execute(ctx, bad, body); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("operation rebound", e)
|
||||
}
|
||||
for _, table := range []string{"intents", "receipts", "grants", "plans"} {
|
||||
if _, e = pool.Exec(ctx, "DELETE FROM kg_object_gateway_"+table+" WHERE scope=$1", gw.Scope); e == nil {
|
||||
t.Fatal("immutable evidence deleted", table)
|
||||
}
|
||||
}
|
||||
for _, table := range []string{"plans", "intents", "receipts", "grants"} {
|
||||
tx, e := pool.Begin(ctx)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_, e = tx.Exec(ctx, "TRUNCATE kg_object_gateway_"+table+" CASCADE")
|
||||
_ = tx.Rollback(ctx)
|
||||
if e == nil {
|
||||
cleanup(t, ctx, gw, one)
|
||||
t.Fatal("immutable evidence truncation accepted", table)
|
||||
}
|
||||
}
|
||||
cleanup(t, ctx, gw, one)
|
||||
next := grant
|
||||
next.Epoch = 2
|
||||
if e = gw.Advance(ctx, 1, next); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
late := request("late", grant)
|
||||
if _, e = gw.Execute(ctx, late, []byte("gateway fixture late")); !errors.Is(e, g.ErrEpoch) {
|
||||
t.Fatal("old epoch sent", e)
|
||||
}
|
||||
}
|
||||
|
||||
func signedAudit(t *testing.T, scope string) (string, string) {
|
||||
t.Helper()
|
||||
pub, priv, e := ed25519.GenerateKey(rand.Reader)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// This is a test authority. It exercises verification, NOT production exclusivity.
|
||||
a := g.Attestation{Protocol: g.Protocol, Scope: scope, GatewayID: "isolated-fixture", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("fixture inventory")), PolicyEvidence: g.Digest([]byte("fixture policy")), DenialEvidence: g.Digest([]byte("fixture denial")), DrainEvidence: g.Digest([]byte("fixture drain"))}
|
||||
raw, _ := json.Marshal(a)
|
||||
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
|
||||
path := filepath.Join(t.TempDir(), "audit.json")
|
||||
if os.WriteFile(path, signed, 0600) != nil {
|
||||
t.Fatal("audit write failed")
|
||||
}
|
||||
return path, base64.StdEncoding.EncodeToString(pub)
|
||||
}
|
||||
func startGateway(t *testing.T, env []string, client *g.Client) *exec.Cmd {
|
||||
t.Helper()
|
||||
binary := os.Getenv("KG_GATEWAY_TEST_BINARY")
|
||||
if binary == "" {
|
||||
t.Fatal("actual gateway binary is required")
|
||||
}
|
||||
cmd := exec.Command(binary)
|
||||
cmd.Env = env
|
||||
log, e := os.Create(filepath.Join(t.TempDir(), "service.log"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cmd.Stdout = log
|
||||
cmd.Stderr = log
|
||||
if e = cmd.Start(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); _ = log.Close() })
|
||||
for i := 0; i < 100; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
e = client.Ready(ctx)
|
||||
cancel()
|
||||
if e == nil {
|
||||
return cmd
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
raw, _ := os.ReadFile(log.Name())
|
||||
t.Fatalf("actual gateway not ready: %s", raw)
|
||||
return nil
|
||||
}
|
||||
func TestGatewayActualServiceTwoInterruptions(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
upstream, _ := url.Parse(os.Getenv("KG_GATEWAY_TEST_ENDPOINT"))
|
||||
proxy := httputil.NewSingleHostReverseProxy(upstream)
|
||||
beforeSend := make(chan struct{}, 2)
|
||||
allowSend := make(chan struct{}, 2)
|
||||
committed := make(chan struct{}, 2)
|
||||
release := make(chan struct{}, 2)
|
||||
var puts atomic.Int32
|
||||
proxy.ModifyResponse = func(resp *http.Response) error {
|
||||
if resp.Request.Method == "PUT" && !strings.Contains(resp.Request.URL.RawQuery, "versioning") {
|
||||
puts.Add(1)
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
committed <- struct{}{}
|
||||
select {
|
||||
case <-release:
|
||||
case <-resp.Request.Context().Done():
|
||||
}
|
||||
return errors.New("injected lost response after actual MinIO commit")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) { http.Error(w, "injected_transport_loss", 502) }
|
||||
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "PUT" && !strings.Contains(r.URL.RawQuery, "versioning") {
|
||||
beforeSend <- struct{}{}
|
||||
select {
|
||||
case <-allowSend:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
}))
|
||||
defer p.Close()
|
||||
scope := g.Digest([]byte("minio\x00" + p.URL + "\x00" + backend.Bucket))
|
||||
audit, pub := signedAudit(t, scope)
|
||||
listener, e := net.Listen("tcp", "127.0.0.1:0")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
addr := listener.Addr().String()
|
||||
listener.Close()
|
||||
data := g.Digest([]byte("data " + scope))
|
||||
control := g.Digest([]byte("control " + scope))
|
||||
client, e := g.NewClient("http://"+addr, data, scope)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
admin, _ := g.NewClient("http://"+addr, control, scope)
|
||||
env := append(os.Environ(), "OBJECT_GATEWAY_LISTEN="+addr, "OBJECT_GATEWAY_DATABASE_URL="+os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"), "OBJECT_GATEWAY_PROVIDER=minio", "OBJECT_GATEWAY_ENDPOINT="+p.URL, "OBJECT_GATEWAY_BUCKET="+backend.Bucket, "OBJECT_GATEWAY_ACCESS_KEY="+os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"), "OBJECT_GATEWAY_SECRET_KEY="+os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"), "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY="+pub, "OBJECT_GATEWAY_AUDIT_FILE="+audit, "OBJECT_GATEWAY_ID=isolated-fixture", "OBJECT_GATEWAY_DATA_TOKEN="+data, "OBJECT_GATEWAY_CONTROL_TOKEN="+control)
|
||||
cmd := startGateway(t, env, client)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("actual-worker")), Plan: g.Digest([]byte("frozen-actual-plan"))}
|
||||
if e = client.InstallPlan(ctx, planFor(grant, request("actual-0", grant))); e == nil {
|
||||
t.Fatal("data client installed plan")
|
||||
}
|
||||
if e = admin.InstallPlan(ctx, planFor(grant, request("actual-0", grant), request("actual-1", grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = admin.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = client.Advance(ctx, 0, grant); e == nil {
|
||||
t.Fatal("data client acquired control authority")
|
||||
}
|
||||
for iteration := 0; iteration < 2; iteration++ {
|
||||
label := fmt.Sprintf("actual-%d", iteration)
|
||||
r := request(label, grant)
|
||||
body := []byte("gateway fixture " + label)
|
||||
done := make(chan error, 1)
|
||||
go func() { _, e := client.Execute(ctx, r, body); done <- e }()
|
||||
select {
|
||||
case <-beforeSend:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("gateway did not send to proxy")
|
||||
}
|
||||
if _, e = client.Reconcile(ctx, r); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("not-yet-visible PUT treated as terminal", e)
|
||||
}
|
||||
blockedNext := grant
|
||||
blockedNext.Epoch++
|
||||
if e = admin.Advance(ctx, grant.Epoch, blockedNext); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("epoch crossed delayed precommit request", e)
|
||||
}
|
||||
allowSend <- struct{}{}
|
||||
select {
|
||||
case <-committed:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("actual MinIO PUT did not commit")
|
||||
}
|
||||
next := grant
|
||||
next.Epoch++
|
||||
if e = admin.Advance(ctx, grant.Epoch, next); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("epoch crossed in-flight write", e)
|
||||
}
|
||||
sameKey := r
|
||||
sameKey.Operation = g.Digest([]byte(label + "other"))
|
||||
if _, e = client.Execute(ctx, sameKey, body); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("same-key competitor admitted", e)
|
||||
}
|
||||
if e = cmd.Process.Kill(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_ = cmd.Wait()
|
||||
release <- struct{}{}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("dead gateway call did not end")
|
||||
}
|
||||
t.Logf("actual gateway SIGKILL %d after MinIO commit before receipt", iteration+1)
|
||||
cmd = startGateway(t, env, client)
|
||||
receipt, e := client.Reconcile(ctx, r)
|
||||
if e != nil || receipt.State != "committed" {
|
||||
t.Fatalf("restart reconcile %v %s", e, receipt.State)
|
||||
}
|
||||
repeated, e := client.Execute(ctx, r, body)
|
||||
if e != nil || receipt != repeated {
|
||||
t.Fatal("restart duplicate not idempotent", e)
|
||||
}
|
||||
del := r
|
||||
del.Action = "delete"
|
||||
del.CreatedBy = r.Operation
|
||||
del.Operation = g.Digest([]byte(label + "cleanup"))
|
||||
del.Version = receipt.Version
|
||||
cleaned, e := client.Execute(ctx, del, nil)
|
||||
if e != nil || cleaned.State != "removed" {
|
||||
t.Fatal("actual exact cleanup failed", e)
|
||||
}
|
||||
if e = admin.Advance(ctx, grant.Epoch, next); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
late := request(label+"late", grant)
|
||||
if _, e = client.Execute(ctx, late, []byte("gateway fixture "+label+"late")); !errors.Is(e, g.ErrEpoch) {
|
||||
t.Fatal("late stale epoch admitted", e)
|
||||
}
|
||||
grant = next
|
||||
}
|
||||
if puts.Load() != 2 {
|
||||
t.Fatalf("blind PUT replay count=%d", puts.Load())
|
||||
}
|
||||
var n int
|
||||
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", scope).Scan(&n) != nil || n != 4 {
|
||||
t.Fatalf("durable receipts=%d", n)
|
||||
}
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
if e = client.Ready(ctx); e == nil {
|
||||
t.Fatal("unavailable gateway accepted")
|
||||
}
|
||||
t.Log("actual binary + HTTP + durable PG + MinIO: two kills/restarts, 2 PUTs, 4 receipts, exact cleanup")
|
||||
}
|
||||
|
||||
func TestGatewayClientRejectsBadProtocolAndLeaks(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("secret") != "" {
|
||||
t.Error("secret in URL")
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"protocol":"other","scope":"scope","ready":true}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
c, e := g.NewClient(server.URL, strings.Repeat("x", 32), "scope")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if c.Ready(context.Background()) == nil {
|
||||
t.Fatal("wrong protocol accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ExecuteRequest struct {
|
||||
Authorization *Grant `json:"authorization,omitempty"`
|
||||
Request Request `json:"request"`
|
||||
Body []byte `json:"body,omitempty"`
|
||||
}
|
||||
type AdvanceRequest struct {
|
||||
Expected int64 `json:"expected"`
|
||||
Grant Grant `json:"grant"`
|
||||
}
|
||||
type Capabilities struct {
|
||||
AutomaticGrants bool `json:"automatic_grants"`
|
||||
Protocol string `json:"protocol"`
|
||||
Scope string `json:"scope"`
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
type HTTPServer struct {
|
||||
Gateway *Gateway
|
||||
DataToken, ControlToken string
|
||||
}
|
||||
|
||||
func (s HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
token := s.DataToken
|
||||
if r.URL.Path == "/v1/advance" || r.URL.Path == "/v1/plans" {
|
||||
token = s.ControlToken
|
||||
}
|
||||
if len(token) < 32 || s.DataToken == s.ControlToken || subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+token)) != 1 {
|
||||
http.Error(w, "gateway_unauthorized", 401)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("X-KG-Gateway-Protocol") != Protocol || r.Header.Get("X-KG-Gateway-Scope") != s.Gateway.Scope {
|
||||
http.Error(w, "gateway_protocol_or_target_mismatch", 409)
|
||||
return
|
||||
}
|
||||
fail := func(e error) {
|
||||
code := 503
|
||||
if errors.Is(e, ErrEpoch) || errors.Is(e, ErrConflict) || errors.Is(e, ErrHeld) {
|
||||
code = 409
|
||||
}
|
||||
http.Error(w, e.Error(), code)
|
||||
}
|
||||
decode := func(v any) bool {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.Gateway.MaxBytes*2+16384)
|
||||
d := json.NewDecoder(r.Body)
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(v) != nil || d.Decode(&struct{}{}) != io.EOF {
|
||||
http.Error(w, "gateway_bad_request", 400)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
if r.Method == "GET" && r.URL.Path == "/v1/capabilities" {
|
||||
if e := s.Gateway.Ready(r.Context()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
if authority, ok := s.Gateway.Authority.(interface{ Ready(context.Context) error }); ok {
|
||||
if e := authority.Ready(r.Context()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(Capabilities{Protocol: Protocol, Scope: s.Gateway.Scope, Ready: true, AutomaticGrants: s.Gateway.Authority != nil})
|
||||
return
|
||||
}
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "gateway_route_not_found", 404)
|
||||
return
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/v1/acquire":
|
||||
var in Claim
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
out, e := s.Gateway.Acquire(r.Context(), in)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case "/v1/plans":
|
||||
if s.Gateway.Authority != nil {
|
||||
fail(ErrConflict)
|
||||
return
|
||||
}
|
||||
var in Plan
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
if e := s.Gateway.InstallPlan(r.Context(), in); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(in)
|
||||
case "/v1/advance":
|
||||
if s.Gateway.Authority != nil {
|
||||
fail(ErrConflict)
|
||||
return
|
||||
}
|
||||
var in AdvanceRequest
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
if e := s.Gateway.Advance(r.Context(), in.Expected, in.Grant); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(in)
|
||||
case "/v1/execute", "/v1/reconcile":
|
||||
var in ExecuteRequest
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
var out Receipt
|
||||
var e error
|
||||
if r.URL.Path == "/v1/execute" {
|
||||
if in.Authorization != nil {
|
||||
out, e = s.Gateway.ExecuteAuthorized(r.Context(), *in.Authorization, in.Request, in.Body)
|
||||
} else {
|
||||
out, e = s.Gateway.Execute(r.Context(), in.Request, in.Body)
|
||||
}
|
||||
} else {
|
||||
out, e = s.Gateway.Reconcile(r.Context(), in.Request)
|
||||
}
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case "/v1/resolve":
|
||||
var in Request
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
out, e := s.Gateway.Resolve(r.Context(), in)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case "/v1/observe":
|
||||
var in struct {
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if !decode(&in) {
|
||||
return
|
||||
}
|
||||
if in.Key == "" {
|
||||
http.Error(w, "gateway_bad_key", 400)
|
||||
return
|
||||
}
|
||||
if e := s.Gateway.Ready(r.Context()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
v, exists, e := s.Gateway.Backend.Observe(r.Context(), in.Key, in.Version)
|
||||
if e != nil {
|
||||
http.Error(w, "gateway_backend_read_failed", 503)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(struct {
|
||||
Version Version `json:"version"`
|
||||
Exists bool `json:"exists"`
|
||||
}{v, exists})
|
||||
default:
|
||||
http.Error(w, "gateway_route_not_found", 404)
|
||||
}
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
Endpoint, Token, Scope string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
func NewClient(endpoint, token, scope string) (*Client, error) {
|
||||
u, e := url.Parse(endpoint)
|
||||
if e != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" || len(token) < 32 || scope == "" {
|
||||
return nil, errors.New("gateway_configuration_invalid")
|
||||
}
|
||||
if u.Scheme != "https" && (u.Scheme != "http" || (u.Hostname() != "127.0.0.1" && u.Hostname() != "::1")) {
|
||||
return nil, errors.New("gateway_transport_requires_tls")
|
||||
}
|
||||
return &Client{endpoint, token, scope, &http.Client{Timeout: 2 * time.Minute, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("gateway_redirect_refused") }}}, nil
|
||||
}
|
||||
func (c *Client) call(ctx context.Context, path string, in, out any) error {
|
||||
method := "GET"
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
raw, e := json.Marshal(in)
|
||||
if e != nil {
|
||||
return ErrConflict
|
||||
}
|
||||
method = "POST"
|
||||
body = struct{ io.Reader }{bytes.NewReader(raw)}
|
||||
}
|
||||
req, e := http.NewRequestWithContext(ctx, method, c.Endpoint+path, body)
|
||||
if e != nil {
|
||||
return errors.New("gateway_request_invalid")
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.Token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-KG-Gateway-Protocol", Protocol)
|
||||
req.Header.Set("X-KG-Gateway-Scope", c.Scope)
|
||||
resp, e := c.HTTP.Do(req)
|
||||
if e != nil {
|
||||
return errors.New("gateway_unreachable_or_outcome_unknown")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 128))
|
||||
reason := strings.TrimSpace(string(raw))
|
||||
switch reason {
|
||||
case ErrAuthority.Error():
|
||||
return ErrAuthority
|
||||
case ErrAuthorityPrivileges.Error():
|
||||
return ErrAuthorityPrivileges
|
||||
case ErrEpoch.Error():
|
||||
return ErrEpoch
|
||||
case ErrConflict.Error():
|
||||
return ErrConflict
|
||||
case ErrHeld.Error():
|
||||
return ErrHeld
|
||||
case ErrPersistence.Error():
|
||||
return ErrPersistence
|
||||
case ErrIsolation.Error():
|
||||
return ErrIsolation
|
||||
}
|
||||
return errors.New("gateway_request_refused")
|
||||
}
|
||||
if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out) != nil {
|
||||
return errors.New("gateway_response_invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *Client) Ready(ctx context.Context) error { return c.checkReady(ctx, false) }
|
||||
func (c *Client) ReadyForAcquisition(ctx context.Context) error { return c.checkReady(ctx, true) }
|
||||
func (c *Client) checkReady(ctx context.Context, automatic bool) error {
|
||||
var caps Capabilities
|
||||
if e := c.call(ctx, "/v1/capabilities", nil, &caps); e != nil {
|
||||
return e
|
||||
}
|
||||
if automatic && !caps.AutomaticGrants {
|
||||
return ErrAuthority
|
||||
}
|
||||
if caps.Protocol != Protocol || caps.Scope != c.Scope || !caps.Ready {
|
||||
return errors.New("gateway_protocol_or_target_mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *Client) InstallPlan(ctx context.Context, p Plan) error {
|
||||
var out Plan
|
||||
if e := c.call(ctx, "/v1/plans", p, &out); e != nil {
|
||||
return e
|
||||
}
|
||||
want, _ := json.Marshal(p)
|
||||
got, _ := json.Marshal(out)
|
||||
if string(want) != string(got) {
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *Client) Advance(ctx context.Context, expected int64, g Grant) error {
|
||||
var out AdvanceRequest
|
||||
in := AdvanceRequest{expected, g}
|
||||
if e := c.call(ctx, "/v1/advance", in, &out); e != nil {
|
||||
return e
|
||||
}
|
||||
if in != out {
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *Client) Execute(ctx context.Context, r Request, body []byte) (Receipt, error) {
|
||||
var out Receipt
|
||||
e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body}, &out)
|
||||
if e == nil && (out.Request != r || !out.valid()) {
|
||||
e = ErrConflict
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
func (c *Client) Reconcile(ctx context.Context, r Request) (Receipt, error) {
|
||||
var out Receipt
|
||||
e := c.call(ctx, "/v1/reconcile", ExecuteRequest{Request: r}, &out)
|
||||
if e == nil && (out.Request != r || !out.valid()) {
|
||||
e = ErrConflict
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
func (c *Client) Observe(ctx context.Context, key, id string) (Version, bool, error) {
|
||||
var out struct {
|
||||
Version Version `json:"version"`
|
||||
Exists bool `json:"exists"`
|
||||
}
|
||||
e := c.call(ctx, "/v1/observe", struct {
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
}{key, id}, &out)
|
||||
return out.Version, out.Exists, e
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
func TestGatewayTargetIdentityOnEveryCall(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := gw.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
token := strings.Repeat("a", 32)
|
||||
server := httptest.NewServer(g.HTTPServer{Gateway: gw, DataToken: token, ControlToken: strings.Repeat("b", 32)})
|
||||
defer server.Close()
|
||||
wrong, _ := g.NewClient(server.URL, token, "wrong-target")
|
||||
req := request(t.Name(), grant)
|
||||
receipt, e := wrong.Execute(ctx, req, []byte("gateway fixture "+t.Name()))
|
||||
if e == nil {
|
||||
cleanup(t, ctx, gw, receipt)
|
||||
t.Fatal("mutation accepted with wrong target identity")
|
||||
}
|
||||
var n int
|
||||
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gw.Scope).Scan(&n) != nil || n != 0 {
|
||||
t.Fatal("wrong target wrote intent")
|
||||
}
|
||||
}
|
||||
func TestGatewayClientRejectsIncompleteReceipt(t *testing.T) {
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("owner")), Plan: g.Digest([]byte("plan"))}
|
||||
req := request("receipt", grant)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(g.Receipt{Request: req, State: "committed"})
|
||||
}))
|
||||
defer server.Close()
|
||||
c, _ := g.NewClient(server.URL, strings.Repeat("a", 32), "scope")
|
||||
if _, e := c.Execute(context.Background(), req, []byte("gateway fixture receipt")); e == nil {
|
||||
t.Fatal("committed receipt with missing exact version accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGatewayIntentCommitFailureNeverSends(t *testing.T) {
|
||||
ctx, pool, real := fixture(t)
|
||||
backend := &stoppedBeforeSend{Backend: real}
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := gateway.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Deferred failure fires on COMMIT, not on INSERT. This exercises the uncertain
|
||||
// persistence boundary while confirming no provider request is sent.
|
||||
_, e := pool.Exec(ctx, `CREATE FUNCTION kg_gateway_fixture_commit_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected durable commit failure'; END $$;
|
||||
CREATE CONSTRAINT TRIGGER kg_gateway_fixture_commit_failure AFTER INSERT ON kg_object_gateway_intents DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION kg_gateway_fixture_commit_failure()`)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, e := pool.Exec(ctx, `DROP TRIGGER kg_gateway_fixture_commit_failure ON kg_object_gateway_intents; DROP FUNCTION kg_gateway_fixture_commit_failure()`)
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
})
|
||||
req := request(t.Name(), grant)
|
||||
if _, e = gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrPersistence) {
|
||||
t.Fatal("commit error not propagated", e)
|
||||
}
|
||||
if backend.calls != 0 {
|
||||
t.Fatal("object request after failed journal commit")
|
||||
}
|
||||
var n int
|
||||
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_intents WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 {
|
||||
t.Fatal("failed transaction left intent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// Plan is frozen by the authenticated controller, not by the data client. Digest
|
||||
// is the upstream write-set digest; the controller must verify that write set.
|
||||
// This allowlist prevents a valid epoch from authorizing arbitrary extra keys.
|
||||
type Plan struct {
|
||||
Digest string `json:"digest"`
|
||||
Owner string `json:"owner"`
|
||||
Objects []ObjectBinding `json:"objects"`
|
||||
}
|
||||
type ObjectBinding struct {
|
||||
Operation string `json:"operation"`
|
||||
Key string `json:"key"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
func (p Plan) valid() bool {
|
||||
if !validHash(p.Digest) || !validHash(p.Owner) {
|
||||
return false
|
||||
}
|
||||
keys, ops := map[string]bool{}, map[string]bool{}
|
||||
for _, o := range p.Objects {
|
||||
r := Request{Grant: Grant{1, p.Owner, p.Digest}, Operation: o.Operation, Key: o.Key, Action: "create", SHA256: o.SHA256, Bytes: o.Bytes}
|
||||
if r.validate() != nil || keys[o.Key] || ops[o.Operation] {
|
||||
return false
|
||||
}
|
||||
keys[o.Key] = true
|
||||
ops[o.Operation] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (p Plan) allows(r Request) bool {
|
||||
op := r.Operation
|
||||
if r.Action == "delete" {
|
||||
op = r.CreatedBy
|
||||
}
|
||||
if p.Owner != r.Grant.Owner || p.Digest != r.Grant.Plan {
|
||||
return false
|
||||
}
|
||||
for _, o := range p.Objects {
|
||||
if o.Operation == op && o.Key == r.Key && o.SHA256 == r.SHA256 && o.Bytes == r.Bytes {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func (g *Gateway) InstallPlan(ctx context.Context, p Plan) error {
|
||||
if !p.valid() {
|
||||
return ErrConflict
|
||||
}
|
||||
for _, o := range p.Objects {
|
||||
if o.Bytes > g.MaxBytes {
|
||||
return ErrConflict
|
||||
}
|
||||
}
|
||||
if e := g.Ready(ctx); e != nil {
|
||||
return e
|
||||
}
|
||||
raw, _ := json.Marshal(p)
|
||||
_, e := g.Pool.Exec(ctx, "INSERT INTO kg_object_gateway_plans(scope,plan,body) VALUES($1,$2,$3) ON CONFLICT DO NOTHING", g.Scope, p.Digest, raw)
|
||||
if e != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
var found []byte
|
||||
if g.Pool.QueryRow(ctx, "SELECT body FROM kg_object_gateway_plans WHERE scope=$1 AND plan=$2", g.Scope, p.Digest).Scan(&found) != nil {
|
||||
return ErrPersistence
|
||||
}
|
||||
if string(found) != string(raw) {
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGatewayGrantCannotAuthorizeUnplannedKey(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e := gateway.Advance(ctx, 0, grant); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("unregistered plan grant", e)
|
||||
}
|
||||
allowed := request("allowed-plan-key", grant)
|
||||
plan := planFor(grant, allowed)
|
||||
if e := gateway.InstallPlan(ctx, plan); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
changed := planFor(grant, request("extra-plan-key", grant))
|
||||
if e := gateway.InstallPlan(ctx, changed); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("immutable plan rebound", e)
|
||||
}
|
||||
if e := gateway.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
req := request(t.Name(), grant)
|
||||
if _, e := gateway.Execute(ctx, req, []byte("gateway fixture "+t.Name())); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("unplanned key admitted", e)
|
||||
}
|
||||
receipt, e := gateway.Execute(ctx, allowed, []byte("gateway fixture allowed-plan-key"))
|
||||
if e != nil || receipt.State != "committed" {
|
||||
t.Fatal("valid planned key refused", e)
|
||||
}
|
||||
cleanup(t, ctx, gateway, receipt)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Package objectgateway implements non-preemptive object write ownership.
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Protocol = "kg-object-gateway-v1"
|
||||
|
||||
var (
|
||||
ErrHeld = errors.New("gateway_held_unresolved_intent")
|
||||
ErrEpoch = errors.New("gateway_epoch_or_plan_mismatch")
|
||||
ErrConflict = errors.New("gateway_identity_conflict")
|
||||
ErrPersistence = errors.New("gateway_persistence_unavailable")
|
||||
ErrAuthority = errors.New("gateway_control_authority_unavailable")
|
||||
ErrAuthorityPrivileges = errors.New("gateway_control_reader_privileges_invalid")
|
||||
ErrIsolation = errors.New("gateway_exclusive_writer_evidence_unverified")
|
||||
ErrRejected = errors.New("gateway_create_rejected_existing_object")
|
||||
)
|
||||
|
||||
type Grant struct {
|
||||
Epoch int64 `json:"epoch"`
|
||||
Owner string `json:"owner"`
|
||||
Plan string `json:"plan"`
|
||||
}
|
||||
type Version struct {
|
||||
ID string `json:"version_id"`
|
||||
Owner string `json:"owner"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
ETag string `json:"etag"`
|
||||
}
|
||||
type Request struct {
|
||||
Grant Grant `json:"grant"`
|
||||
Operation string `json:"operation"`
|
||||
Key string `json:"key"`
|
||||
Action string `json:"action"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
Version Version `json:"version,omitempty"`
|
||||
}
|
||||
type Receipt struct {
|
||||
Request Request `json:"request"`
|
||||
State string `json:"state"`
|
||||
Version Version `json:"version"`
|
||||
}
|
||||
|
||||
// Backend never retries mutations. List is complete and scoped to exactly key.
|
||||
// Not finding a PUT is never evidence that its request cannot still arrive.
|
||||
type Backend interface {
|
||||
Ready(context.Context) error
|
||||
Observe(context.Context, string, string) (Version, bool, error)
|
||||
List(context.Context, string) ([]Version, error)
|
||||
Put(context.Context, string, []byte, string) (Version, error)
|
||||
Delete(context.Context, string, Version) error
|
||||
}
|
||||
type IsolationVerifier interface{ Verify(context.Context) error }
|
||||
|
||||
func Digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
func validHash(s string) bool {
|
||||
b, e := hex.DecodeString(s)
|
||||
return e == nil && len(b) == 32 && strings.ToLower(s) == s
|
||||
}
|
||||
func (g Grant) valid() bool { return g.Epoch > 0 && validHash(g.Owner) && validHash(g.Plan) }
|
||||
func (r Request) validate() error {
|
||||
if !r.Grant.valid() || !validHash(r.Operation) || r.Key == "" || len(r.Key) > 2048 || strings.ContainsAny(r.Key, "\x00\r\n") || strings.HasPrefix(r.Key, "/") || !validHash(r.SHA256) || r.Bytes < 0 {
|
||||
return ErrConflict
|
||||
}
|
||||
if r.Action != "create" && r.Action != "delete" {
|
||||
return ErrConflict
|
||||
}
|
||||
if r.Action == "create" && (r.CreatedBy != "" || r.Version != (Version{})) {
|
||||
return ErrConflict
|
||||
}
|
||||
if r.Action == "delete" && (!validHash(r.CreatedBy) || !validVersion(r.Version) || r.Version.SHA256 != r.SHA256 || r.Version.Bytes != r.Bytes) {
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func validVersion(v Version) bool {
|
||||
return v.ID != "" && v.ID != "null" && validHash(v.Owner) && validHash(v.SHA256) && v.Bytes >= 0 && v.ETag != ""
|
||||
}
|
||||
|
||||
// Claim binds backend ownership to the complete immutable gateway request.
|
||||
func (r Request) Claim() string { return r.token() }
|
||||
func (r Request) token() string { raw, _ := json.Marshal(r); return Digest(raw) }
|
||||
func (r Receipt) valid() bool {
|
||||
if r.Request.validate() != nil {
|
||||
return false
|
||||
}
|
||||
switch r.State {
|
||||
case "committed":
|
||||
return r.Request.Action == "create" && r.Request.matches(r.Version)
|
||||
case "rejected":
|
||||
return r.Request.Action == "create" && r.Version == (Version{})
|
||||
case "removed":
|
||||
return r.Request.Action == "delete" && r.Version == r.Request.Version
|
||||
}
|
||||
return false
|
||||
}
|
||||
func (r Request) matches(v Version) bool {
|
||||
return validVersion(v) && v.Owner == r.token() && v.Bytes == r.Bytes && v.SHA256 == r.SHA256
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
)
|
||||
|
||||
// S3Backend is for independently versioned S3/MinIO stores. No SDK mutation retries.
|
||||
type S3Backend struct {
|
||||
Client *s3.Client
|
||||
Bucket string
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
func (s *S3Backend) Ready(ctx context.Context) error {
|
||||
r, e := s.Client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{Bucket: aws.String(s.Bucket)})
|
||||
if e != nil || string(r.Status) != "Enabled" {
|
||||
return errors.New("gateway_backend_versioning_unverified")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *S3Backend) Observe(ctx context.Context, key, id string) (Version, bool, error) {
|
||||
req := &s3.GetObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key)}
|
||||
if id != "" {
|
||||
req.VersionId = aws.String(id)
|
||||
}
|
||||
got, e := s.Client.GetObject(ctx, req)
|
||||
if e != nil {
|
||||
var api smithy.APIError
|
||||
if errors.As(e, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NoSuchVersion") {
|
||||
return Version{}, false, nil
|
||||
}
|
||||
return Version{}, false, errors.New("gateway_backend_read_failed")
|
||||
}
|
||||
defer got.Body.Close()
|
||||
if got.ContentLength == nil || *got.ContentLength < 0 || *got.ContentLength > s.MaxBytes {
|
||||
return Version{}, false, errors.New("gateway_backend_size_invalid")
|
||||
}
|
||||
body, e := io.ReadAll(io.LimitReader(got.Body, s.MaxBytes+1))
|
||||
if e != nil || int64(len(body)) != *got.ContentLength {
|
||||
return Version{}, false, errors.New("gateway_backend_body_incomplete")
|
||||
}
|
||||
v := Version{aws.ToString(got.VersionId), got.Metadata["kg-release-owner"], Digest(body), int64(len(body)), aws.ToString(got.ETag)}
|
||||
// Existing unowned objects are observable but cannot be claimed/deleted.
|
||||
if v.ID == "" || v.ID == "null" || v.ETag == "" || id != "" && v.ID != id {
|
||||
return Version{}, false, errors.New("gateway_backend_version_invalid")
|
||||
}
|
||||
return v, true, nil
|
||||
}
|
||||
func (s *S3Backend) List(ctx context.Context, key string) ([]Version, error) {
|
||||
got, e := s.Client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(s.Bucket), Prefix: aws.String(key), MaxKeys: aws.Int32(1000)})
|
||||
if e != nil || aws.ToBool(got.IsTruncated) {
|
||||
return nil, errors.New("gateway_backend_listing_incomplete")
|
||||
}
|
||||
var versions []Version
|
||||
for _, m := range got.DeleteMarkers {
|
||||
if aws.ToString(m.Key) == key {
|
||||
return nil, errors.New("gateway_backend_delete_marker_conflict")
|
||||
}
|
||||
}
|
||||
for _, v := range got.Versions {
|
||||
if aws.ToString(v.Key) == key {
|
||||
one, exists, e := s.Observe(ctx, key, aws.ToString(v.VersionId))
|
||||
if e != nil || !exists {
|
||||
return nil, errors.New("gateway_backend_listing_changed")
|
||||
}
|
||||
versions = append(versions, one)
|
||||
}
|
||||
}
|
||||
return versions, nil
|
||||
}
|
||||
func (s *S3Backend) Put(ctx context.Context, key string, body []byte, owner string) (Version, error) {
|
||||
// SDK hashing requires a seekable body. RetryMaxAttempts=1 prevents SDK
|
||||
// retries; PUT carries no HTTP idempotency header or replay authorization.
|
||||
reader := bytes.NewReader(body)
|
||||
got, e := s.Client.PutObject(v4.SetPayloadHash(ctx, Digest(body)), &s3.PutObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(int64(len(body))), Metadata: map[string]string{"kg-release-owner": owner}}, func(o *s3.Options) { o.RetryMaxAttempts = 1 })
|
||||
if e != nil || aws.ToString(got.VersionId) == "" {
|
||||
return Version{}, ErrHeld
|
||||
}
|
||||
v, exists, e := s.Observe(ctx, key, aws.ToString(got.VersionId))
|
||||
if e != nil || !exists {
|
||||
return Version{}, ErrHeld
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
func (s *S3Backend) Delete(ctx context.Context, key string, v Version) error {
|
||||
if !validVersion(v) {
|
||||
return ErrConflict
|
||||
}
|
||||
_, e := s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.Bucket), Key: aws.String(key), VersionId: aws.String(v.ID)}, func(o *s3.Options) { o.RetryMaxAttempts = 1 })
|
||||
if e != nil {
|
||||
return ErrHeld
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
-- Independent governance journal; never install in a WeKnora business schema.
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_plans (
|
||||
scope text NOT NULL,
|
||||
plan text NOT NULL,
|
||||
body bytea NOT NULL,
|
||||
PRIMARY KEY(scope,plan)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_scopes (
|
||||
scope text PRIMARY KEY,
|
||||
epoch bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_grants (
|
||||
scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope),
|
||||
epoch bigint NOT NULL CHECK (epoch > 0),
|
||||
owner text NOT NULL,
|
||||
plan text NOT NULL,
|
||||
PRIMARY KEY(scope,epoch)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_intents (
|
||||
scope text NOT NULL REFERENCES kg_object_gateway_scopes(scope),
|
||||
operation text NOT NULL,
|
||||
key text NOT NULL,
|
||||
request bytea NOT NULL,
|
||||
recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY(scope,operation)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS kg_object_gateway_key ON kg_object_gateway_intents(scope,key);
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_receipts (
|
||||
scope text NOT NULL,
|
||||
operation text NOT NULL,
|
||||
receipt bytea NOT NULL,
|
||||
recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY(scope,operation),
|
||||
FOREIGN KEY(scope,operation) REFERENCES kg_object_gateway_intents(scope,operation)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS kg_object_gateway_deliveries (
|
||||
scope text NOT NULL,
|
||||
job text NOT NULL,
|
||||
attempt_epoch bigint NOT NULL CHECK(attempt_epoch>0),
|
||||
claim bytea NOT NULL,
|
||||
grant_epoch bigint NOT NULL,
|
||||
grant_body bytea NOT NULL,
|
||||
PRIMARY KEY(scope,job,attempt_epoch),
|
||||
FOREIGN KEY(scope,grant_epoch) REFERENCES kg_object_gateway_grants(scope,epoch)
|
||||
);
|
||||
CREATE OR REPLACE FUNCTION kg_object_gateway_immutable() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'object gateway evidence is immutable';
|
||||
END;
|
||||
$$;
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_deliveries_immutable') THEN
|
||||
CREATE TRIGGER kg_object_gateway_deliveries_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_deliveries FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_plans_immutable') THEN
|
||||
CREATE TRIGGER kg_object_gateway_plans_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_plans FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_intents_immutable') THEN
|
||||
CREATE TRIGGER kg_object_gateway_intents_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_intents FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_receipts_immutable') THEN
|
||||
CREATE TRIGGER kg_object_gateway_receipts_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_receipts FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_grants_immutable') THEN
|
||||
CREATE TRIGGER kg_object_gateway_grants_immutable BEFORE UPDATE OR DELETE ON kg_object_gateway_grants FOR EACH ROW EXECUTE FUNCTION kg_object_gateway_immutable();
|
||||
END IF;
|
||||
END $$;
|
||||
DO $$ DECLARE t text; BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['plans','grants','intents','receipts','deliveries'] LOOP
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname='kg_object_gateway_'||t||'_no_truncate') THEN
|
||||
EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION kg_object_gateway_immutable()', 'kg_object_gateway_'||t||'_no_truncate', 'kg_object_gateway_'||t);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END $$;
|
||||
@@ -0,0 +1,60 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Resolve preserves the immutable send epoch and backend claim of an operation.
|
||||
// An absent intent is not a claim about an earlier send; Execute still reserves
|
||||
// atomically and refuses stale authorizations before any new side effect.
|
||||
func (g *Gateway) Resolve(ctx context.Context, desired Request) (Request, error) {
|
||||
if e := desired.validate(); e != nil {
|
||||
return Request{}, e
|
||||
}
|
||||
if e := g.Ready(ctx); e != nil {
|
||||
return Request{}, e
|
||||
}
|
||||
var raw []byte
|
||||
e := g.Pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", g.Scope, desired.Operation).Scan(&raw)
|
||||
if errors.Is(e, pgx.ErrNoRows) {
|
||||
return desired, nil
|
||||
}
|
||||
if e != nil {
|
||||
return Request{}, ErrPersistence
|
||||
}
|
||||
var original Request
|
||||
if json.Unmarshal(raw, &original) != nil || original.validate() != nil {
|
||||
return Request{}, ErrPersistence
|
||||
}
|
||||
if !sameOperation(original, desired) || original.Grant.Epoch > desired.Grant.Epoch {
|
||||
return Request{}, ErrConflict
|
||||
}
|
||||
return original, nil
|
||||
}
|
||||
func sameOperation(a, b Request) bool { a.Grant.Epoch = b.Grant.Epoch; return a == b }
|
||||
|
||||
func (g *Gateway) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) {
|
||||
if !authorization.valid() || authorization.Owner != r.Grant.Owner || authorization.Plan != r.Grant.Plan || authorization.Epoch < r.Grant.Epoch {
|
||||
return Receipt{}, ErrEpoch
|
||||
}
|
||||
return g.execute(ctx, r, body, &authorization)
|
||||
}
|
||||
func (c *Client) ExecuteAuthorized(ctx context.Context, authorization Grant, r Request, body []byte) (Receipt, error) {
|
||||
var out Receipt
|
||||
e := c.call(ctx, "/v1/execute", ExecuteRequest{Request: r, Body: body, Authorization: &authorization}, &out)
|
||||
if e == nil && (out.Request != r || !out.valid()) {
|
||||
e = ErrConflict
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
func (c *Client) Resolve(ctx context.Context, desired Request) (Request, error) {
|
||||
var out Request
|
||||
e := c.call(ctx, "/v1/resolve", desired, &out)
|
||||
if e == nil && (out.validate() != nil || !sameOperation(out, desired) || out.Grant.Epoch > desired.Grant.Epoch) {
|
||||
e = ErrConflict
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
func TestGatewayNewEpochPreservesOriginalSendIdentity(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
first := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("takeover-plan"))}
|
||||
sent, unsent := request("takeover-sent", first), request("takeover-unsent", first)
|
||||
if e := gateway.InstallPlan(ctx, planFor(first, sent, unsent)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := gateway.Advance(ctx, 0, first); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
old, e := gateway.Execute(ctx, sent, []byte("gateway fixture takeover-sent"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Always clean only this test's exact original version, including on RED.
|
||||
cleanupGrant := first
|
||||
defer func() { copy := old; copy.Request.Grant = cleanupGrant; cleanup(t, ctx, gateway, copy) }()
|
||||
var original []byte
|
||||
if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&original); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
next := first
|
||||
next.Epoch++
|
||||
if e = gateway.Advance(ctx, 1, next); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cleanupGrant = next
|
||||
data, control := g.Digest([]byte("takeover-data")), g.Digest([]byte("takeover-control"))
|
||||
server := httptest.NewServer(g.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: control})
|
||||
defer server.Close()
|
||||
post := func(path string, in any) (int, []byte) {
|
||||
raw, _ := json.Marshal(in)
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", server.URL+path, bytes.NewReader(raw))
|
||||
req.Header.Set("Authorization", "Bearer "+data)
|
||||
req.Header.Set("X-KG-Gateway-Protocol", g.Protocol)
|
||||
req.Header.Set("X-KG-Gateway-Scope", gateway.Scope)
|
||||
resp, e := http.DefaultClient.Do(req)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
code, body := post("/v1/execute", map[string]any{"request": sent, "authorization": next, "body": []byte("gateway fixture takeover-sent")})
|
||||
if code != 200 {
|
||||
t.Fatalf("new authorization cannot replay immutable original request: status=%d", code)
|
||||
}
|
||||
var replay g.Receipt
|
||||
if json.Unmarshal(body, &replay) != nil || replay != old {
|
||||
t.Fatal("replay rewrote original receipt")
|
||||
}
|
||||
desired := sent
|
||||
desired.Grant = next
|
||||
code, body = post("/v1/resolve", desired)
|
||||
var resolved g.Request
|
||||
if code != 200 || json.Unmarshal(body, &resolved) != nil || resolved != sent {
|
||||
t.Fatal("original send identity not resolved", code)
|
||||
}
|
||||
changed := sent
|
||||
changed.SHA256 = g.Digest([]byte("different"))
|
||||
if code, _ = post("/v1/resolve", changed); code == 200 {
|
||||
t.Fatal("operation rebound body")
|
||||
}
|
||||
if code, _ = post("/v1/execute", map[string]any{"request": unsent, "authorization": first, "body": []byte("gateway fixture takeover-unsent")}); code == 200 {
|
||||
t.Fatal("stale epoch created unsent key")
|
||||
}
|
||||
unsent.Grant = next
|
||||
code, body = post("/v1/execute", map[string]any{"request": unsent, "authorization": next, "body": []byte("gateway fixture takeover-unsent")})
|
||||
var created g.Receipt
|
||||
if code != 200 || json.Unmarshal(body, &created) != nil || created.State != "committed" {
|
||||
t.Fatal("new epoch did not continue unsent key", code)
|
||||
}
|
||||
cleanup(t, ctx, gateway, created)
|
||||
var after []byte
|
||||
if e = pool.QueryRow(ctx, "SELECT request FROM kg_object_gateway_intents WHERE scope=$1 AND operation=$2", gateway.Scope, sent.Operation).Scan(&after); e != nil || !bytes.Equal(after, original) {
|
||||
t.Fatal("immutable intent changed", e)
|
||||
}
|
||||
versions, e := backend.List(ctx, sent.Key)
|
||||
if e != nil || len(versions) != 1 || versions[0] != old.Version {
|
||||
t.Fatal("replay created extra version", e)
|
||||
}
|
||||
// Deletion is a new send in the current epoch, not a rewritten create intent.
|
||||
deletion := sent
|
||||
deletion.Grant = next
|
||||
deletion.Operation = g.Digest([]byte(sent.Operation + "delete"))
|
||||
deletion.Action = "delete"
|
||||
deletion.CreatedBy = sent.Operation
|
||||
deletion.Version = old.Version
|
||||
code, _ = post("/v1/execute", map[string]any{"request": deletion, "authorization": next})
|
||||
if code != 200 {
|
||||
t.Fatal("new epoch exact cleanup refused", code)
|
||||
}
|
||||
// deferred cleanup replays the same already settled delete request below.
|
||||
old.Request.Grant = next
|
||||
t.Log("new epoch replay retained original request/receipt/claim; unsent key committed once; stale request rejected")
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
type stoppedBeforeSend struct {
|
||||
g.Backend
|
||||
calls int
|
||||
}
|
||||
|
||||
func (b *stoppedBeforeSend) Put(context.Context, string, []byte, string) (g.Version, error) {
|
||||
b.calls++
|
||||
return g.Version{}, errors.New("injected uncertain send with no visible version")
|
||||
}
|
||||
func TestGatewayAbsenceNeverSettlesUnknown(t *testing.T) {
|
||||
ctx, pool, real := fixture(t)
|
||||
backend := &stoppedBeforeSend{Backend: real}
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := gateway.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
req := request(t.Name(), grant)
|
||||
body := []byte("gateway fixture " + t.Name())
|
||||
if _, e := gateway.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
restarted := *gateway
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, e := restarted.Execute(ctx, req, body); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("absent version treated as terminal", e)
|
||||
}
|
||||
}
|
||||
if backend.calls != 1 {
|
||||
t.Fatal("PUT replayed", backend.calls)
|
||||
}
|
||||
next := grant
|
||||
next.Epoch++
|
||||
if e := restarted.Advance(ctx, 1, next); !errors.Is(e, g.ErrHeld) {
|
||||
t.Fatal("unknown operation allowed epoch handover", e)
|
||||
}
|
||||
var n int
|
||||
if pool.QueryRow(ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", gateway.Scope).Scan(&n) != nil || n != 0 {
|
||||
t.Fatal("absence wrote false terminal receipt")
|
||||
}
|
||||
}
|
||||
func TestGatewayReadOnlyPersistenceBlocksSend(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
cfg := pool.Config()
|
||||
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
other, e := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer other.Close()
|
||||
gateway := &g.Gateway{Pool: other, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
if e = gateway.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
|
||||
t.Fatal("read-only persistence accepted", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// GatewayGovernanceAuthority runs only in the trusted gateway control domain.
|
||||
// Its governance connection is SELECT-only; the ordinary worker never receives
|
||||
// the gateway control credential or backend write credential. Archive contents
|
||||
// are trusted only after their immutable digest is found in the original
|
||||
// persisted prebackup event belonging to the currently leased attempt.
|
||||
type GatewayGovernanceAuthority struct {
|
||||
Pool *pgxpool.Pool
|
||||
ArchiveRoot string
|
||||
TargetID string
|
||||
Provider string
|
||||
Bucket string
|
||||
}
|
||||
|
||||
func (a *GatewayGovernanceAuthority) Ready(ctx context.Context) error {
|
||||
if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" {
|
||||
return objectgateway.ErrAuthority
|
||||
}
|
||||
var safe bool
|
||||
e := a.Pool.QueryRow(ctx, `SELECT NOT rolsuper AND NOT rolcreatedb AND NOT rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&safe)
|
||||
if e != nil || !safe {
|
||||
return objectgateway.ErrAuthorityPrivileges
|
||||
}
|
||||
for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
|
||||
e = a.Pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT') AND NOT has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&safe)
|
||||
if e != nil || !safe {
|
||||
return objectgateway.ErrAuthorityPrivileges
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (a *GatewayGovernanceAuthority) Load(ctx context.Context, c objectgateway.Claim) (objectgateway.Plan, error) {
|
||||
if e := a.Ready(ctx); e != nil {
|
||||
return objectgateway.Plan{}, e
|
||||
}
|
||||
if a == nil || a.Pool == nil || !filepath.IsAbs(a.ArchiveRoot) || a.TargetID == "" || a.Provider == "" || a.Bucket == "" {
|
||||
return objectgateway.Plan{}, objectgateway.ErrAuthority
|
||||
}
|
||||
recovery, err := a.verifyRecoveryClaim(ctx, c)
|
||||
if err != nil {
|
||||
return objectgateway.Plan{}, err
|
||||
}
|
||||
var scope, backup, snapshot string
|
||||
var matches int
|
||||
e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',count(*) OVER()
|
||||
FROM library_jobs j
|
||||
JOIN library_operations o ON o.id=j.operation_id AND o.target_id=j.target_id
|
||||
JOIN library_attempts a ON a.operation_id=j.operation_id AND a.epoch=j.attempt_epoch AND a.job_id=j.id
|
||||
JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id AND v.scope=j.scope AND v.kind='production_prebackup' AND v.target_id=j.target_id
|
||||
WHERE j.id=$1 AND j.operation_id=$2 AND j.attempt_epoch=$3 AND j.lease_owner=$4 AND j.target_id=$6
|
||||
AND j.lease_until>clock_timestamp() AND (j.state='running' OR $7)
|
||||
AND a.owner=$4 AND a.state IN ('claimed','running') AND a.lease_until>clock_timestamp()
|
||||
AND v.evidence->>'writeset_digest'=$5
|
||||
AND v.evidence->>'job'=j.id AND v.evidence->>'backup_snapshot'=v.snapshot_id
|
||||
AND (o.target_snapshot_id='' OR o.target_snapshot_id=v.evidence->>'writeset_target_snapshot')
|
||||
AND NOT EXISTS(SELECT 1 FROM library_attempts newer WHERE newer.operation_id=a.operation_id AND newer.epoch>a.epoch)`, c.JobID, c.OperationID, c.AttemptEpoch, c.Owner, c.PlanDigest, a.TargetID, recovery).Scan(&scope, &backup, &snapshot, &matches)
|
||||
if errors.Is(e, pgx.ErrNoRows) {
|
||||
return objectgateway.Plan{}, objectgateway.ErrEpoch
|
||||
}
|
||||
if e != nil {
|
||||
return objectgateway.Plan{}, objectgateway.ErrPersistence
|
||||
}
|
||||
if matches != 1 {
|
||||
return objectgateway.Plan{}, objectgateway.ErrConflict
|
||||
}
|
||||
set, e := LoadWriteSet(a.ArchiveRoot, c.PlanDigest)
|
||||
if e != nil || set == nil {
|
||||
return objectgateway.Plan{}, objectgateway.ErrConflict
|
||||
}
|
||||
if set.Version != libraryrelease.MigrationWriteSetVersion || set.JobID != c.JobID || set.Target != scope || set.BackupSnapshotID != backup || set.TargetSnapshotID != snapshot {
|
||||
return objectgateway.Plan{}, objectgateway.ErrConflict
|
||||
}
|
||||
for _, obj := range set.Objects {
|
||||
if obj.Provider != a.Provider || obj.Bucket != a.Bucket {
|
||||
return objectgateway.Plan{}, objectgateway.ErrConflict
|
||||
}
|
||||
}
|
||||
return ObjectGatewayPlan(*set)
|
||||
}
|
||||
|
||||
// Recovery authorization does not replace c.JobID/c.OperationID (the original
|
||||
// send domain). Its separate queue and operation must both be durable, leased,
|
||||
// and bound to the same before-image, source, target and frozen plan.
|
||||
func (a *GatewayGovernanceAuthority) verifyRecoveryClaim(ctx context.Context, c objectgateway.Claim) (bool, error) {
|
||||
if c.RecoveryJobID == "" && c.RecoveryOperationID == "" {
|
||||
return false, nil
|
||||
}
|
||||
if c.RecoveryJobID != governance.LibraryRecoveryQueueID(c.JobID) {
|
||||
return false, objectgateway.ErrConflict
|
||||
}
|
||||
var scope, backup, source, recorded string
|
||||
e := a.Pool.QueryRow(ctx, `SELECT j.scope,v.snapshot_id,v.evidence->>'writeset_target_snapshot',o.request_digest
|
||||
FROM library_jobs j JOIN library_versions v ON v.source_job_id=j.id AND v.operation_id=j.operation_id
|
||||
AND v.target_id=j.target_id AND v.scope=j.scope AND v.kind='production_prebackup'
|
||||
JOIN jobs q ON q.id=$3 AND q.type='library.recover' AND q.state='running'
|
||||
JOIN library_operations o ON o.id=$4 AND o.kind='recover' AND o.scope=j.scope AND o.target_id=j.target_id
|
||||
WHERE j.id=$1 AND j.operation_id=$2 AND j.target_id=$5
|
||||
AND q.payload->>'scope'=j.scope AND q.payload->>'window_job_id'=j.id
|
||||
AND COALESCE(NULLIF(q.payload->>'source_snapshot_id',''),v.evidence->>'writeset_target_snapshot')=v.evidence->>'writeset_target_snapshot'
|
||||
AND COALESCE(NULLIF(q.payload->>'target_id',''),j.target_id)=j.target_id
|
||||
AND COALESCE(NULLIF(q.payload->>'original_operation_id',''),j.operation_id)=j.operation_id
|
||||
AND COALESCE(NULLIF(q.payload->>'backup_snapshot_id',''),v.snapshot_id)=v.snapshot_id
|
||||
AND COALESCE(NULLIF(q.payload->>'plan_digest',''),$7)=$7
|
||||
AND q.idempotency_key='library.recover:'||j.id AND q.lease_until>clock_timestamp()
|
||||
AND q.lease_owner||'#'||q.id||'#'||q.attempt::text=$6
|
||||
AND o.operation_key=q.id AND o.target_snapshot_id=v.snapshot_id
|
||||
AND v.evidence->>'writeset_digest'=$7`, c.JobID, c.OperationID, c.RecoveryJobID, c.RecoveryOperationID, a.TargetID, c.Owner, c.PlanDigest).Scan(&scope, &backup, &source, &recorded)
|
||||
if errors.Is(e, pgx.ErrNoRows) {
|
||||
return false, objectgateway.ErrConflict
|
||||
}
|
||||
if e != nil {
|
||||
return false, objectgateway.ErrPersistence
|
||||
}
|
||||
digest, e := governance.LibraryRecoveryDigest(scope, c.RecoveryJobID, c.JobID, c.OperationID, a.TargetID, source, backup, c.PlanDigest)
|
||||
if e != nil || recorded != digest || c.RecoveryOperationID != governance.LibraryOperationID(scope, governance.LibraryJobRecover, c.RecoveryJobID) {
|
||||
return false, objectgateway.ErrConflict
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/postgres/pgtest"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type gatewayCountBackend struct {
|
||||
objectgateway.Backend
|
||||
puts atomic.Int64
|
||||
}
|
||||
|
||||
func (b *gatewayCountBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) {
|
||||
b.puts.Add(1)
|
||||
return b.Backend.Put(ctx, key, body, owner)
|
||||
}
|
||||
|
||||
func gatewayReadOnlyGovernance(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
random := make([]byte, 24)
|
||||
if _, e := rand.Read(random); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
suffix := hex.EncodeToString(random)
|
||||
role := "kg_gateway_reader_" + suffix[:12]
|
||||
// This role is created only in the explicitly disposable fixture cluster.
|
||||
if _, e := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); e != nil {
|
||||
t.Fatal("create isolated reader role failed")
|
||||
}
|
||||
var schema string
|
||||
if admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema) != nil {
|
||||
t.Fatal("fixture schema")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup := context.Background()
|
||||
if _, e := admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize()); e != nil {
|
||||
t.Errorf("reader role privilege cleanup failed")
|
||||
}
|
||||
if _, e := admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize()); e != nil {
|
||||
t.Errorf("reader role cleanup failed")
|
||||
}
|
||||
})
|
||||
if _, e := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); e != nil {
|
||||
t.Fatal("reader schema grant")
|
||||
}
|
||||
if _, e := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); e != nil {
|
||||
t.Fatal("reader table grant")
|
||||
}
|
||||
parsed, e := url.Parse(dsn)
|
||||
if e != nil {
|
||||
t.Fatal("fixture DSN")
|
||||
}
|
||||
parsed.User = url.UserPassword(role, suffix)
|
||||
config, e := pgxpool.ParseConfig(parsed.String())
|
||||
if e != nil {
|
||||
t.Fatal("reader config")
|
||||
}
|
||||
config.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
pool, e := pgxpool.NewWithConfig(ctx, config)
|
||||
if e != nil {
|
||||
t.Fatal("reader connect")
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
t.Log("isolated SELECT-only governance role created; exact role cleanup registered:", role)
|
||||
return pool
|
||||
}
|
||||
|
||||
// This is real authority/PG/HTTP/storage integration, not the formal worker
|
||||
// process test: no grant decorator or control credential is given to the target.
|
||||
func TestContractGatewayAuthorityFrozenPlanIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
ctx := fx.ctx
|
||||
dsn := pgtest.IsolatedDatabaseURL(t, os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"))
|
||||
governancePool, e := pgxpool.New(ctx, dsn)
|
||||
if e != nil {
|
||||
t.Fatal("governance pool")
|
||||
}
|
||||
t.Cleanup(governancePool.Close)
|
||||
if e = postgresstore.ApplyMigrations(ctx, governancePool); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = governancePool.Exec(ctx, objectgateway.Schema); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
reader := gatewayReadOnlyGovernance(t, ctx, governancePool, dsn)
|
||||
const jobID = "gateway-authority-job"
|
||||
const targetID = "gateway-authority-target"
|
||||
store := postgresstore.NewStore(governancePool).WithLibraryTarget(targetID)
|
||||
snapshot := sealSourceSnapshot(t, fx, nil)
|
||||
now := time.Now().UTC()
|
||||
op := governance.LibraryOperation{ID: "gateway-authority-operation", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: snapshot, RequestDigest: "gateway-authority-params", OperationKey: "gateway-authority-key", CreatedAt: now}
|
||||
if _, _, e = store.EnsureLibraryOperation(ctx, op); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, _, e = store.CreateLibraryJob(ctx, governance.LibraryJob{ID: jobID, Scope: "internal", Kind: governance.LibraryJobSync, OperationID: op.ID, IdempotencyKey: jobID, RequestDigest: op.RequestDigest, TargetSnapshotID: snapshot}, governance.SyncStages(), now); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
attempt, e := store.BeginLibraryAttempt(ctx, jobID, "worker-first", time.Hour, now)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = governancePool.Exec(ctx, "UPDATE library_jobs SET state='running' WHERE id=$1", jobID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
authority := &GatewayGovernanceAuthority{Pool: reader, ArchiveRoot: fx.archiveRoot, TargetID: targetID, Provider: "minio", Bucket: fx.bucket}
|
||||
if e = authority.Ready(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
privileged := *authority
|
||||
privileged.Pool = governancePool
|
||||
if privileged.Ready(ctx) == nil {
|
||||
t.Fatal("privileged controller reader accepted")
|
||||
}
|
||||
backend := &gatewayCountBackend{Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}}
|
||||
gateway := &objectgateway.Gateway{Pool: governancePool, Scope: fx.bucket, Backend: backend, Authority: authority, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20}
|
||||
data := objectgateway.Digest([]byte("authority-data"))
|
||||
server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: data, ControlToken: objectgateway.Digest([]byte("authority-control"))})
|
||||
defer server.Close()
|
||||
client, e := objectgateway.NewClient(server.URL, data, fx.bucket)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
claim := objectgateway.Claim{JobID: jobID, OperationID: op.ID, AttemptEpoch: int64(attempt.Epoch), Owner: attempt.Owner}
|
||||
objects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: claim}
|
||||
target := fx.target(t)
|
||||
target.config.Objects = objects
|
||||
current, e := target.Current(ctx)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
backup, e := fx.backups(t, target).Prepare(ctx, snapshot, current)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
set, e := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: jobID, Target: "internal", BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshot, Universe: backup.Universe})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
claim.PlanDigest = set.Digest
|
||||
before := universeDump(t, fx.pool)
|
||||
if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e == nil {
|
||||
t.Fatal("unanchored plan permitted first PG write")
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatal("unanchored apply mutated PG", diff)
|
||||
}
|
||||
if e = store.RecordLibraryVersion(ctx, governance.LibraryVersion{ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, "internal", backup.SnapshotID, op.ID), Scope: "internal", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: jobID, OperationID: op.ID, AttemptEpoch: attempt.Epoch, AttemptOwner: attempt.Owner, CreatedAt: now, Evidence: map[string]any{"job": jobID, "backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, "writeset_digest": set.Digest, "writeset_target_snapshot": snapshot}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, bad := range []objectgateway.Claim{{JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: "other-worker", PlanDigest: set.Digest}, {JobID: jobID, OperationID: op.ID, AttemptEpoch: claim.AttemptEpoch, Owner: claim.Owner, PlanDigest: objectgateway.Digest([]byte("replacement"))}} {
|
||||
if _, e = client.Acquire(ctx, bad); e == nil {
|
||||
t.Fatal("untrusted claim admitted")
|
||||
}
|
||||
}
|
||||
if e = target.Apply(ctx, libraryrelease.StageDatabase, snapshot); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = target.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
count := backend.puts.Load()
|
||||
if count == 0 {
|
||||
t.Fatal("no actual object send")
|
||||
}
|
||||
firstRequests := objects.requests
|
||||
if e = store.FinishLibraryAttempt(ctx, attempt, governance.LibraryAttemptFailed, "isolated takeover", time.Now().UTC()); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
next, e := store.BeginLibraryAttempt(ctx, jobID, "worker-second", time.Hour, time.Now().UTC())
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
nextClaim := claim
|
||||
nextClaim.AttemptEpoch = int64(next.Epoch)
|
||||
nextClaim.Owner = next.Owner
|
||||
nextObjects := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, Automatic: true, Claim: nextClaim}
|
||||
nextTarget := fx.target(t)
|
||||
nextTarget.config.Objects = nextObjects
|
||||
if e = nextTarget.AdoptWriteSet(set); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = nextObjects.EnsureObjectGrant(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for key, original := range firstRequests {
|
||||
if nextObjects.requests[key] != original {
|
||||
t.Fatal("send identity replaced during new epoch", key)
|
||||
}
|
||||
}
|
||||
if nextObjects.authorization.Epoch <= objects.authorization.Epoch {
|
||||
t.Fatal("authorization did not advance")
|
||||
}
|
||||
if _, e = client.Acquire(ctx, claim); !errors.Is(e, objectgateway.ErrEpoch) {
|
||||
t.Fatal("old active owner admitted", e)
|
||||
}
|
||||
if e = nextTarget.Apply(ctx, libraryrelease.StageResources, snapshot); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if backend.puts.Load() != count {
|
||||
t.Fatal("resumed object send duplicated")
|
||||
}
|
||||
for _, obj := range set.Objects {
|
||||
if obj.Existing {
|
||||
continue
|
||||
}
|
||||
locator := obj.Locator
|
||||
observation, e := nextObjects.ObserveObject(ctx, obj.Provider, locator)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = nextObjects.RemoveObject(ctx, obj.Provider, locator, observation); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
versions, e := backend.List(ctx, obj.Key)
|
||||
if e != nil || len(versions) != 0 {
|
||||
t.Fatal("created object version residue", e)
|
||||
}
|
||||
}
|
||||
t.Logf("real readonly governance authority -> data-only HTTP acquisition -> PG/MinIO; first-PG anchor denial, new epoch unchanged send identity, PUTs=%d, original version exact cleanup", count)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
// NewGatewayBackend is server-side only. Clients never receive this credentialed object.
|
||||
func NewGatewayBackend(c S3ObjectStoreConfig) (objectgateway.Backend, error) {
|
||||
if c.Provider == "oss" {
|
||||
s, e := newOSSVersionStore(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return ossGatewayBackend{s}, nil
|
||||
}
|
||||
if c.Provider != "s3" && c.Provider != "minio" {
|
||||
return nil, errors.New("gateway_backend_provider_unsupported")
|
||||
}
|
||||
s, e := NewS3ObjectStore(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return &objectgateway.S3Backend{Client: s.client, Bucket: s.bucket, MaxBytes: s.maxObjectBytes}, nil
|
||||
}
|
||||
|
||||
type ossGatewayBackend struct{ s *ossVersionStore }
|
||||
|
||||
func toGateway(v objectObservation) objectgateway.Version {
|
||||
return objectgateway.Version{ID: v.VersionID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag}
|
||||
}
|
||||
func fromGateway(v objectgateway.Version) objectObservation {
|
||||
return objectObservation{Exists: true, VersionID: v.ID, Owner: v.Owner, SHA256: v.SHA256, Bytes: v.Bytes, ETag: v.ETag}
|
||||
}
|
||||
func (b ossGatewayBackend) Ready(ctx context.Context) error { return b.s.versioning(ctx) }
|
||||
func (b ossGatewayBackend) Observe(ctx context.Context, key, id string) (objectgateway.Version, bool, error) {
|
||||
v, e := b.s.observeVersion(ctx, key, id)
|
||||
return toGateway(v), v.Exists, e
|
||||
}
|
||||
func (b ossGatewayBackend) Put(ctx context.Context, key string, body []byte, owner string) (objectgateway.Version, error) {
|
||||
v, e := b.s.appendVersion(ctx, key, body, owner)
|
||||
return toGateway(v), e
|
||||
}
|
||||
func (b ossGatewayBackend) Delete(ctx context.Context, key string, v objectgateway.Version) error {
|
||||
return b.s.removeVersion(ctx, key, fromGateway(v))
|
||||
}
|
||||
func (b ossGatewayBackend) List(ctx context.Context, key string) ([]objectgateway.Version, error) {
|
||||
listing, e := b.s.listExactVersions(ctx, key)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(listing.DeleteMarkers) != 0 {
|
||||
return nil, errors.New("gateway_backend_delete_marker_conflict")
|
||||
}
|
||||
var out []objectgateway.Version
|
||||
for _, v := range listing.Versions {
|
||||
o, e := b.s.observeVersion(ctx, key, v.VersionID)
|
||||
if e != nil || !o.Exists {
|
||||
return nil, errors.New("gateway_backend_listing_changed")
|
||||
}
|
||||
out = append(out, toGateway(o))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
// PrepareObjectGatewayGrant runs only in the trusted control plane. The data
|
||||
// worker receives the immutable grant file, never the control bearer token.
|
||||
// Replay uses the original expected epoch; it does not guess the latest epoch.
|
||||
func PrepareObjectGatewayGrant(ctx context.Context, client *objectgateway.Client, set libraryrelease.WriteSet, expected int64, provider, bucket, directory string) error {
|
||||
if client == nil || !filepath.IsAbs(directory) || expected < 0 {
|
||||
return errors.New("gateway grant controller configuration invalid")
|
||||
}
|
||||
plan, e := ObjectGatewayPlan(set)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, obj := range set.Objects {
|
||||
if obj.Provider != provider || obj.Bucket != bucket {
|
||||
return errors.New("gateway grant target does not match frozen object plan")
|
||||
}
|
||||
}
|
||||
grant := objectgateway.Grant{Epoch: expected + 1, Owner: plan.Owner, Plan: plan.Digest}
|
||||
if grant.Epoch < 1 {
|
||||
return errors.New("gateway epoch exhausted")
|
||||
}
|
||||
raw, e := json.Marshal(grant)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = persistImmutableJSON(filepath.Join(directory, set.Digest+".intent.json"), raw); e != nil {
|
||||
return e
|
||||
}
|
||||
// Network outcomes may be unknown. Both endpoints compare immutable identity
|
||||
// on a repeat, so no caller invents another epoch or another operation ID.
|
||||
if e = client.InstallPlan(ctx, plan); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = client.Advance(ctx, expected, grant); e != nil {
|
||||
return e
|
||||
}
|
||||
return persistImmutableJSON(filepath.Join(directory, set.Digest+".json"), raw)
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
// GatewayObjectStore has no write credentials or direct-write fallback. The
|
||||
// separate reader must be deployed with read-only rights (audit prerequisite).
|
||||
type GatewayObjectStore struct {
|
||||
Automatic bool
|
||||
Claim objectgateway.Claim
|
||||
frozen *libraryrelease.WriteSet
|
||||
authorization objectgateway.Grant
|
||||
Client *objectgateway.Client
|
||||
Provider, Bucket, GrantDirectory string
|
||||
requests map[string]objectgateway.Request
|
||||
owners map[string]string
|
||||
}
|
||||
|
||||
func (s *GatewayObjectStore) ConditionalCapability(ctx context.Context) error {
|
||||
if s.Client == nil || s.Provider == "" || s.Bucket == "" || (!s.Automatic && !filepath.IsAbs(s.GrantDirectory)) {
|
||||
return errors.New("object gateway configuration/grant directory missing")
|
||||
}
|
||||
if s.Automatic {
|
||||
return s.Client.ReadyForAcquisition(ctx)
|
||||
}
|
||||
return s.Client.Ready(ctx)
|
||||
}
|
||||
|
||||
// ObjectGatewayPlan derives the controller allowlist from the verified original
|
||||
// write set. Existing shared objects remain read-only, never write-authorized.
|
||||
func ObjectGatewayPlan(set libraryrelease.WriteSet) (objectgateway.Plan, error) {
|
||||
if err := set.Verify(); err != nil {
|
||||
return objectgateway.Plan{}, err
|
||||
}
|
||||
p := objectgateway.Plan{Digest: set.Digest, Owner: objectgateway.Digest([]byte(set.JobID))}
|
||||
for _, obj := range set.Objects {
|
||||
if !obj.Existing {
|
||||
p.Objects = append(p.Objects, objectgateway.ObjectBinding{Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, SHA256: obj.SHA256, Bytes: obj.Bytes})
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
func gatewayCreateOperation(plan, owner string) string {
|
||||
return objectgateway.Digest([]byte(plan + "\x00" + owner + "\x00create"))
|
||||
}
|
||||
func (s *GatewayObjectStore) BindObjectPlan(set libraryrelease.WriteSet) error {
|
||||
if !validLowerSHA256(set.Digest) || set.JobID == "" {
|
||||
return errors.New("gateway frozen plan identity missing")
|
||||
}
|
||||
if _, e := ObjectGatewayPlan(set); e != nil {
|
||||
return e
|
||||
}
|
||||
if s.frozen != nil && s.frozen.Digest != set.Digest {
|
||||
return errors.New("gateway store cannot replace frozen identity")
|
||||
}
|
||||
if s.Automatic {
|
||||
if s.frozen == nil {
|
||||
copy := set
|
||||
s.frozen = ©
|
||||
}
|
||||
return nil
|
||||
}
|
||||
path := filepath.Join(s.GrantDirectory, set.Digest+".json")
|
||||
info, e := os.Lstat(path)
|
||||
if e != nil || !info.Mode().IsRegular() || info.Size() > 4096 || info.Mode().Perm()&0022 != 0 {
|
||||
return errors.New("gateway controller grant unavailable")
|
||||
}
|
||||
raw, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return errors.New("gateway controller grant unavailable")
|
||||
}
|
||||
var grant objectgateway.Grant
|
||||
if json.Unmarshal(raw, &grant) != nil || grant.Epoch < 1 || grant.Plan != set.Digest || grant.Owner != objectgateway.Digest([]byte(set.JobID)) {
|
||||
return errors.New("gateway controller grant differs from frozen job/plan")
|
||||
}
|
||||
return s.bindGrant(context.Background(), set, grant)
|
||||
}
|
||||
func (s *GatewayObjectStore) EnsureObjectGrant(ctx context.Context) error {
|
||||
if !s.Automatic || s.requests != nil {
|
||||
return nil
|
||||
}
|
||||
if s.frozen == nil || s.Claim.JobID != s.frozen.JobID {
|
||||
return errors.New("gateway active attempt/frozen plan missing")
|
||||
}
|
||||
claim := s.Claim
|
||||
claim.PlanDigest = s.frozen.Digest
|
||||
grant, e := s.Client.Acquire(ctx, claim)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return s.bindGrant(ctx, *s.frozen, grant)
|
||||
}
|
||||
func (s *GatewayObjectStore) bindGrant(ctx context.Context, set libraryrelease.WriteSet, grant objectgateway.Grant) error {
|
||||
requests := map[string]objectgateway.Request{}
|
||||
owners := map[string]string{}
|
||||
for _, obj := range set.Objects {
|
||||
if obj.Provider != s.Provider || obj.Bucket != s.Bucket {
|
||||
return errors.New("gateway plan target mismatch")
|
||||
}
|
||||
if obj.Existing {
|
||||
continue
|
||||
}
|
||||
req := objectgateway.Request{Grant: grant, Operation: gatewayCreateOperation(set.Digest, obj.OwnerToken), Key: obj.Key, Action: "create", SHA256: obj.SHA256, Bytes: obj.Bytes}
|
||||
original, e := s.Client.Resolve(ctx, req)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
requests[obj.Key] = original
|
||||
owners[obj.Key] = obj.OwnerToken
|
||||
}
|
||||
s.authorization = grant
|
||||
s.requests = requests
|
||||
s.owners = owners
|
||||
return nil
|
||||
}
|
||||
func (s *GatewayObjectStore) key(provider, locator string) (string, error) {
|
||||
if provider != s.Provider || !validObjectKey(locator) {
|
||||
return "", errors.New("gateway provider/locator mismatch")
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimPrefix(locator, "storage://"), "/", 2)
|
||||
if !strings.HasPrefix(locator, "storage://") || len(parts) != 2 || parts[0] == "" {
|
||||
return "", errors.New("gateway explicit locator required")
|
||||
}
|
||||
prefix := s.Provider + "://" + s.Bucket + "/"
|
||||
if !strings.HasPrefix(parts[1], prefix) {
|
||||
return "", errors.New("gateway bucket mismatch")
|
||||
}
|
||||
key := strings.TrimPrefix(parts[1], prefix)
|
||||
if key == "" {
|
||||
return "", errors.New("gateway key missing")
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
func (s *GatewayObjectStore) ObserveObject(ctx context.Context, provider, locator string) (objectObservation, error) {
|
||||
if s.frozen != nil {
|
||||
if e := s.EnsureObjectGrant(ctx); e != nil {
|
||||
return objectObservation{}, e
|
||||
}
|
||||
}
|
||||
key, e := s.key(provider, locator)
|
||||
if e != nil {
|
||||
return objectObservation{}, e
|
||||
}
|
||||
v, exists, e := s.Client.Observe(ctx, key, "")
|
||||
if e != nil {
|
||||
return objectObservation{}, e
|
||||
}
|
||||
if !exists {
|
||||
return objectObservation{}, nil
|
||||
}
|
||||
o := fromGateway(v)
|
||||
if req, ok := s.requests[key]; ok && v.Owner == req.Claim() {
|
||||
o.Owner = s.owners[key]
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
func (s *GatewayObjectStore) CreateObject(ctx context.Context, provider, locator string, body io.Reader, size int64, owner string) error {
|
||||
if e := s.EnsureObjectGrant(ctx); e != nil {
|
||||
return e
|
||||
}
|
||||
key, e := s.key(provider, locator)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
r, ok := s.requests[key]
|
||||
if !ok || owner != s.owners[key] || size != r.Bytes {
|
||||
return errObjectWriteRejected
|
||||
}
|
||||
raw, e := io.ReadAll(io.LimitReader(body, size+1))
|
||||
if e != nil || int64(len(raw)) != size || objectgateway.Digest(raw) != r.SHA256 {
|
||||
return errObjectWriteRejected
|
||||
}
|
||||
receipt, e := s.Client.ExecuteAuthorized(ctx, s.authorization, r, raw)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if receipt.State == "rejected" {
|
||||
return errObjectWriteRejected
|
||||
}
|
||||
if receipt.State != "committed" {
|
||||
return errors.New("gateway create not committed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *GatewayObjectStore) RemoveObject(ctx context.Context, provider, locator string, want objectObservation) error {
|
||||
if e := s.EnsureObjectGrant(ctx); e != nil {
|
||||
return e
|
||||
}
|
||||
key, e := s.key(provider, locator)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
create, ok := s.requests[key]
|
||||
if !ok || want.Owner != s.owners[key] {
|
||||
return errors.New("gateway cleanup lacks frozen ownership")
|
||||
}
|
||||
receipt, e := s.Client.Reconcile(ctx, create)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
raw := want
|
||||
raw.Owner = create.Claim()
|
||||
if receipt.State != "committed" || receipt.Version != toGateway(raw) {
|
||||
return errors.New("gateway cleanup receipt mismatch")
|
||||
}
|
||||
req := create
|
||||
req.Grant = s.authorization
|
||||
req.Operation = objectgateway.Digest([]byte(create.Operation + "\x00delete"))
|
||||
req.Action = "delete"
|
||||
req.CreatedBy = create.Operation
|
||||
req.Version = receipt.Version
|
||||
original, e := s.Client.Resolve(ctx, req)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
deleted, e := s.Client.ExecuteAuthorized(ctx, s.authorization, original, nil)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if deleted.State != "removed" {
|
||||
return errors.New("gateway deletion not confirmed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *GatewayObjectStore) Put(context.Context, string, string, io.Reader, int64) error {
|
||||
return errors.New("gateway refuses unplanned Put; frozen object plan required")
|
||||
}
|
||||
func (s *GatewayObjectStore) Copy(context.Context, string, string, string) error {
|
||||
return errors.New("gateway refuses unplanned Copy; frozen object plan required")
|
||||
}
|
||||
func (s *GatewayObjectStore) Stat(ctx context.Context, provider, locator string) (int64, string, error) {
|
||||
o, e := s.ObserveObject(ctx, provider, locator)
|
||||
if e != nil {
|
||||
return 0, "", e
|
||||
}
|
||||
if !o.Exists {
|
||||
return 0, "", errors.New("gateway object absent")
|
||||
}
|
||||
return o.Bytes, o.SHA256, nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
type gatewayFixtureIsolation struct{}
|
||||
|
||||
func (gatewayFixtureIsolation) Verify(context.Context) error { return nil }
|
||||
|
||||
// Test controller issues the grant only after the saga has frozen its actual
|
||||
// plan. Production controller delivery is a separate integration prerequisite.
|
||||
type gatewayFixtureController struct {
|
||||
*GatewayObjectStore
|
||||
control *objectgateway.Client
|
||||
}
|
||||
|
||||
func (s *gatewayFixtureController) BindObjectPlan(set libraryrelease.WriteSet) error {
|
||||
if e := PrepareObjectGatewayGrant(context.Background(), s.control, set, 0, s.Provider, s.Bucket, s.GrantDirectory); e != nil {
|
||||
return e
|
||||
}
|
||||
return s.GatewayObjectStore.BindObjectPlan(set)
|
||||
}
|
||||
func TestContractGatewayActualSagaRollbackIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
if _, e := fx.pool.Exec(fx.ctx, objectgateway.Schema); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
before := universeDump(t, fx.pool)
|
||||
snapshot := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
gateway := &objectgateway.Gateway{Pool: fx.pool, Scope: fx.bucket, Backend: &objectgateway.S3Backend{Client: fx.objects.s3.client, Bucket: fx.bucket, MaxBytes: 1 << 20}, Isolation: gatewayFixtureIsolation{}, MaxBytes: 1 << 20}
|
||||
token := objectgateway.Digest([]byte("data-fixture"))
|
||||
server := httptest.NewServer(objectgateway.HTTPServer{Gateway: gateway, DataToken: token, ControlToken: objectgateway.Digest([]byte("control-fixture"))})
|
||||
defer server.Close()
|
||||
client, e := objectgateway.NewClient(server.URL, token, fx.bucket)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
store := &GatewayObjectStore{Client: client, Provider: "minio", Bucket: fx.bucket, GrantDirectory: t.TempDir()}
|
||||
control, e := objectgateway.NewClient(server.URL, objectgateway.Digest([]byte("control-fixture")), fx.bucket)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
target.config.Objects = &gatewayFixtureController{store, control}
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, fx.backups(t, target), nil)
|
||||
if outcome.State != "rolled_back" {
|
||||
t.Fatalf("actual gateway saga: %+v", outcome)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatal(diff)
|
||||
}
|
||||
key := wsLocator("backend-target", "minio", fx.bucket, wsTargetTenant, "res-src")
|
||||
observed, e := store.ObserveObject(fx.ctx, "minio", key)
|
||||
if e != nil || observed.Exists {
|
||||
t.Fatal("gateway object residue", e)
|
||||
}
|
||||
physical, e := store.key("minio", key)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
versions, e := gateway.Backend.List(fx.ctx, physical)
|
||||
if e != nil || len(versions) != 0 {
|
||||
t.Fatal("exact created key version residue", e)
|
||||
}
|
||||
var n int
|
||||
if fx.pool.QueryRow(fx.ctx, "SELECT count(*) FROM kg_object_gateway_receipts WHERE scope=$1", fx.bucket).Scan(&n) != nil || n != 2 {
|
||||
t.Fatalf("gateway saga receipts=%d", n)
|
||||
}
|
||||
t.Log("actual release.Run -> plan adoption -> gateway HTTP -> PG/MinIO -> late Wiki failure -> exact-version rollback; all original PG rows retained")
|
||||
}
|
||||
func TestGatewayUnplannedWriteNeverFallsBack(t *testing.T) {
|
||||
store := &GatewayObjectStore{}
|
||||
if store.Put(context.Background(), "oss", "key", nil, 0) == nil || store.Copy(context.Background(), "oss", "from", "to") == nil {
|
||||
t.Fatal("unplanned direct-write fallback")
|
||||
}
|
||||
if store.BindObjectPlan(libraryrelease.WriteSet{}) == nil {
|
||||
t.Fatal("unbound plan accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLegacyObjectImporterCannotStageOrPromote(t *testing.T) {
|
||||
data := []byte("hello")
|
||||
snapshot := ObjectSnapshot{Objects: []AssetObject{{ResourceID: "res-1", Handle: "ABCDEFGHIJKLMNOPQRSTUV", Provider: "minio", PhysicalPath: "storage://source-backend/10000/knowledge/source.txt", Size: 5, SHA256: shaBytes(data), Data: data}}, TotalBytes: 5}
|
||||
snapshot.SHA256 = objectSnapshotDigest(snapshot.Objects)
|
||||
store := &fakeObjectStore{objects: map[string][]byte{}}
|
||||
importer, e := NewObjectImporter(store, TargetObjectConfig{StorageBackendID: "target-backend", Provider: "minio"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = importer.Stage(context.Background(), "legacy-bypass", snapshot); e == nil {
|
||||
t.Fatal("legacy importer emitted unplanned stage PUT")
|
||||
}
|
||||
stage := ObjectStage{ReleaseRevision: "legacy-bypass", Provider: "minio", Objects: []StagedObject{{Handle: snapshot.Objects[0].Handle, StagingPath: "stage/object", FinalPath: "target/object", Size: 5, SHA256: shaBytes(data)}}}
|
||||
if e = importer.Promote(context.Background(), stage); e == nil {
|
||||
t.Fatal("legacy importer emitted unplanned promotion COPY")
|
||||
}
|
||||
if len(store.objects) != 0 {
|
||||
t.Fatal("legacy importer mutated object store")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user