bd452e85b3
build / build (push) Successful in 1m32s
ReleaseLibraryMaintenance 的真实终态是 state=released + released_at,原断言写成 verified 会误红;改为按生产语义核对 released/open。
1208 lines
47 KiB
Go
1208 lines
47 KiB
Go
package main
|
||
|
||
// 开发隔离集成:B1-R1 / B2-R1 的真实纵向链。
|
||
//
|
||
// 只由外置 runner 在一次性资源上启动(KG_THREE_RUN 非空)。它使用真实
|
||
// PostgreSQL / Elasticsearch / MinIO、真实对象网关与入口容器,走正式共享装配
|
||
// (librarystack.OpenBaselineStack / OpenProductionStack,Automatic 网关保留)。
|
||
// 链路:
|
||
// API POST /library/verifications
|
||
// -> 治理 operation/job(B2)
|
||
// -> 真实 owned attempt(worker claim)
|
||
// -> 两次真实 Prepare + 受限演练计划/ledger 写前登记
|
||
// -> canary 漂移
|
||
// -> 四阶段真实恢复(database/wiki/resources/index)
|
||
// -> 内容核验 + 真实入口恢复/维护释放(B1)
|
||
// 失败后:API POST /library/maintenance/recover -> 真实 owned attempt
|
||
// -> 同一 job/backup/原 operation + 非 nil DS01 gate -> RestoreDrill(B2)
|
||
// -> 网关按 recovery claim 校验零对象计划。
|
||
//
|
||
// 这不是 8088 页面验收,也不是生产执行;所有资源由 runner 创建前登记、清理按
|
||
// 不可变 ID/ownership/readback。
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/ed25519"
|
||
"crypto/rand"
|
||
"crypto/sha256"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/http/cookiejar"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
"golang.org/x/crypto/bcrypt"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/objectgateway"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
const tbTargetTenant = 31001
|
||
const tbTargetKB = "kb-verify-chain"
|
||
const tbSentinelTenant = 31999
|
||
const tbSentinelKB = "kb-sentinel"
|
||
|
||
const tbSchema = `
|
||
CREATE TABLE schema_migrations(version int, dirty bool);
|
||
INSERT INTO schema_migrations VALUES(74,false);
|
||
CREATE TABLE knowledge_bases(id text PRIMARY KEY, tenant_id bigint, indexing_strategy jsonb, deleted_at timestamptz,
|
||
embedding_model_id text, vector_store_id text);
|
||
CREATE TABLE models(id text PRIMARY KEY, name text, parameters jsonb, deleted_at timestamptz);
|
||
CREATE TABLE vector_stores(id text PRIMARY KEY, engine_type text, index_config jsonb, deleted_at timestamptz);
|
||
CREATE TABLE knowledges(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, parse_status text,
|
||
pending_subtasks_count int, file_path text, deleted_at timestamptz);
|
||
CREATE TABLE chunks(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, knowledge_id text,
|
||
content text, seq_id bigint, deleted_at timestamptz);
|
||
CREATE TABLE knowledge_tags(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, name text, deleted_at timestamptz);
|
||
CREATE TABLE knowledge_tag_relations(knowledge_id text, tag_id text, tenant_id bigint);
|
||
CREATE TABLE resources(id text PRIMARY KEY, handle text, tenant_id bigint, physical_path text, provider text,
|
||
size bigint, storage_backend_id text, deleted_at timestamptz);
|
||
CREATE TABLE resource_bindings(id text PRIMARY KEY, tenant_id bigint, resource_id text, owner_type text,
|
||
owner_id text, relation text);
|
||
CREATE TABLE wiki_folders(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz);
|
||
CREATE TABLE wiki_pages(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, slug text, content text,
|
||
status text, deleted_at timestamptz);
|
||
CREATE TABLE wiki_page_issues(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz);
|
||
CREATE TABLE wiki_log_entries(id bigint PRIMARY KEY, tenant_id bigint, knowledge_base_id text);
|
||
CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz);
|
||
CREATE TABLE task_pending_ops(tenant_id bigint, task_type text, scope text, scope_id text);
|
||
`
|
||
|
||
type tbEnv struct {
|
||
root string
|
||
pg *pgxpool.Pool
|
||
gov *pgxpool.Pool
|
||
store *postgresstore.Store
|
||
config appconfig.Config
|
||
api string
|
||
ingress string
|
||
gatewayAddr string
|
||
bin string
|
||
project string
|
||
index string
|
||
bucket string
|
||
s3Endpoint string
|
||
s3Access string
|
||
s3Secret string
|
||
readerKey string
|
||
readerSec string
|
||
archiveRoot string
|
||
workerID string
|
||
}
|
||
|
||
func tbGetenv(t *testing.T, name string) string {
|
||
t.Helper()
|
||
value := os.Getenv(name)
|
||
if value == "" {
|
||
t.Fatalf("missing required isolated env %s", name)
|
||
}
|
||
return value
|
||
}
|
||
|
||
func tbWriteJSON(t *testing.T, root, name string, value any) {
|
||
t.Helper()
|
||
raw, err := json.MarshalIndent(value, "", " ")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(filepath.Join(root, name), append(raw, '\n'), 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
func tbStart(t *testing.T, root, label, binary string, env map[string]string) *exec.Cmd {
|
||
t.Helper()
|
||
cmd := exec.Command(binary)
|
||
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")}
|
||
for key, value := range env {
|
||
cmd.Env = append(cmd.Env, key+"="+value)
|
||
}
|
||
log, err := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cmd.Stdout, cmd.Stderr = log, log
|
||
if err := cmd.Start(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() {
|
||
_ = cmd.Process.Kill()
|
||
_ = cmd.Wait()
|
||
log.Close()
|
||
})
|
||
return cmd
|
||
}
|
||
|
||
func tbS3Locator(tenant uint64, id string) string {
|
||
return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", tbGetenvID("KG_THREE_BUCKET"), tenant, id)
|
||
}
|
||
|
||
// tbWaitHTTP 等待前台服务真正绑定端口后再发业务请求;失败时把服务日志带进断言。
|
||
func tbWaitHTTP(t *testing.T, url, logPath string, timeout time.Duration) {
|
||
t.Helper()
|
||
deadline := time.Now().Add(timeout)
|
||
for {
|
||
resp, err := http.Get(url)
|
||
if err == nil {
|
||
resp.Body.Close()
|
||
if resp.StatusCode >= 200 && resp.StatusCode < 500 {
|
||
return
|
||
}
|
||
}
|
||
if time.Now().After(deadline) {
|
||
raw, _ := os.ReadFile(logPath)
|
||
t.Fatalf("service never became ready: %s (%v); log: %s", url, err, strings.TrimSpace(string(raw)))
|
||
}
|
||
time.Sleep(100 * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
func tbGetenvID(name string) string {
|
||
if v := os.Getenv(name); v != "" {
|
||
return v
|
||
}
|
||
return "fixture-bucket"
|
||
}
|
||
|
||
func TestThreeBlockersVerifyChain(t *testing.T) {
|
||
root := os.Getenv("KG_THREE_RUN")
|
||
if root == "" {
|
||
t.Skip("requires the dedicated isolated three-blockers runner (KG_THREE_RUN)")
|
||
}
|
||
if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||
t.Fatal("isolated three-blockers chain requires KG_LIBRARY_TEST_ISOLATED=1")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
|
||
t.Cleanup(cancel)
|
||
|
||
env := &tbEnv{
|
||
root: root, api: tbGetenv(t, "KG_THREE_API_ADDR"), ingress: tbGetenv(t, "KG_THREE_INGRESS_URL"),
|
||
gatewayAddr: tbGetenv(t, "KG_THREE_GATEWAY_ADDR"), bin: tbGetenv(t, "KG_THREE_BIN"),
|
||
project: tbGetenv(t, "KG_THREE_PROJECT"), index: tbGetenv(t, "KG_THREE_INDEX"),
|
||
bucket: tbGetenv(t, "KG_THREE_BUCKET"), s3Endpoint: tbGetenv(t, "KG_LIBRARY_TEST_S3_ENDPOINT"),
|
||
s3Access: tbGetenv(t, "KG_LIBRARY_TEST_S3_ACCESS_KEY"), s3Secret: tbGetenv(t, "KG_LIBRARY_TEST_S3_SECRET_KEY"),
|
||
readerKey: tbGetenv(t, "KG_THREE_READER_KEY"), readerSec: tbGetenv(t, "KG_THREE_READER_SECRET"),
|
||
archiveRoot: filepath.Join(root, "archive"), workerID: tbGetenv(t, "KG_THREE_WORKER_ID"),
|
||
}
|
||
if err := os.MkdirAll(env.archiveRoot, 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
// ---------- 1) 业务库 / 目标现场 ----------
|
||
pool, err := pgxpool.New(ctx, tbGetenv(t, "KG_LIBRARY_TEST_DATABASE_URL"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(pool.Close)
|
||
env.pg = pool
|
||
if _, err := pool.Exec(ctx, tbSchema); err != nil {
|
||
t.Fatalf("business schema: %v", err)
|
||
}
|
||
tbSeedTarget(t, ctx, env)
|
||
|
||
// ---------- 2) 治理库 / 网关 / API ----------
|
||
gov, err := pgxpool.New(ctx, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(gov.Close)
|
||
env.gov = gov
|
||
if err := postgresstore.ApplyMigrations(ctx, gov); err != nil {
|
||
t.Fatalf("governance migrations: %v", err)
|
||
}
|
||
if _, err := gov.Exec(ctx, objectgateway.Schema); err != nil {
|
||
t.Fatalf("gateway schema: %v", err)
|
||
}
|
||
// 业务/API/worker/网关必须共享同一个 target 身份:API 建库作业时写的是
|
||
// appconfig.LibraryTargetIdentity(),worker 的 LibraryJob 查询按它过滤。
|
||
workerEnv := tbConfigEnv(env)
|
||
config, err := appconfig.Load(func(key string) string { return workerEnv[key] })
|
||
if err != nil {
|
||
t.Fatalf("app config: %v", err)
|
||
}
|
||
env.config = config
|
||
targetID := config.LibraryTargetIdentity()
|
||
env.store = postgresstore.NewStore(gov).WithLibraryTarget(targetID)
|
||
readerDSN, readerRole := tbCreateReaderRole(t, ctx, gov, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN"))
|
||
tbProbeReaderPrivileges(t, ctx, root, readerDSN, readerRole)
|
||
tbStartGateway(t, ctx, env, readerDSN, targetID)
|
||
|
||
// 入口/写者容器由 runner 建好;先证明拓扑与原状态可读。
|
||
manager, err := openLibraryMaintenance(config)
|
||
if err != nil {
|
||
t.Fatalf("maintenance wiring: %v", err)
|
||
}
|
||
if err := manager.VerifyTopology(ctx); err != nil {
|
||
t.Fatalf("topology: %v", err)
|
||
}
|
||
manager.Close()
|
||
|
||
apiEnv := map[string]string{}
|
||
for key, value := range workerEnv {
|
||
apiEnv[key] = value
|
||
}
|
||
apiEnv["APP_ADDR"] = env.api
|
||
apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool")
|
||
apiEnv["RELEASE_WORKER_ENABLED"] = "false"
|
||
if err := os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tbStart(t, root, "api", filepath.Join(env.bin, "api"), apiEnv)
|
||
tbWaitHTTP(t, "http://"+env.api+"/health", filepath.Join(root, "api.log"), 60*time.Second)
|
||
api := tbLogin(t, ctx, gov, "http://"+env.api)
|
||
|
||
// ---------- 3) 正常完整验证链路 ----------
|
||
beforePG := tbDump(t, ctx, pool)
|
||
beforeIndex := tbIndexReadback(t, ctx, env)
|
||
tbWriteJSON(t, root, "10-before-pg.json", beforePG)
|
||
tbWriteJSON(t, root, "11-before-index.json", beforeIndex)
|
||
|
||
job1 := tbEnqueueVerify(t, api)
|
||
tbWriteJSON(t, root, "12-verify-job.json", job1)
|
||
if job1.OperationID == "" {
|
||
t.Fatal("API-created verify job carries no operation_id (B2-R1)")
|
||
}
|
||
queue1 := tbClaimQueue(t, ctx, env.store, "library.verify")
|
||
if err := runLibraryVerify(ctx, env.store, config, *queue1); err != nil {
|
||
t.Fatalf("normal verify failed: %v", err)
|
||
}
|
||
after1, err := env.store.LibraryJob(ctx, job1.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tbWriteJSON(t, root, "13-verify-job-after.json", after1)
|
||
if after1.State != governance.LibraryJobSucceeded {
|
||
t.Fatalf("normal verify job state=%s error=%s", after1.State, after1.Error)
|
||
}
|
||
evidence, found, err := env.store.LatestLibraryRestoreVerification(ctx, "internal")
|
||
if err != nil || !found {
|
||
t.Fatalf("verification evidence missing: found=%v err=%v", found, err)
|
||
}
|
||
tbWriteJSON(t, root, "14-verification-evidence.json", evidence)
|
||
if evidence.Status != governance.LibraryVerificationPassed || evidence.JobID != job1.ID {
|
||
t.Fatalf("verification evidence=%#v", evidence)
|
||
}
|
||
// 内容与 sentinel:恢复后必须逐行/逐文档回到写前镜像,且无关租户不变。
|
||
if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" {
|
||
t.Fatalf("restored business content differs: %s", diff)
|
||
}
|
||
if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" {
|
||
t.Fatalf("restored index differs: %s", diff)
|
||
}
|
||
tbAssertMaintenanceReleased(t, ctx, env)
|
||
tbAssertIngressHealthy(t, env)
|
||
// 演练计划:受限 V2、零对象/索引/本地映射,且已登记为 production_prebackup。
|
||
ledger := tbFindDrillLedger(t, ctx, env, job1.ID)
|
||
tbWriteJSON(t, root, "15-drill-plan.json", map[string]any{
|
||
"digest": ledger.Digest, "job": ledger.JobID, "kind": ledger.Kind, "version": ledger.Version,
|
||
"contract_version": ledger.ContractVersion, "objects": len(ledger.Objects), "index": len(ledger.Index),
|
||
"local_ids": len(ledger.LocalIDs), "backup_snapshot": ledger.BackupSnapshotID,
|
||
"target_snapshot": ledger.TargetSnapshotID, "rows": ledger.Rows,
|
||
"schema_digest": ledger.SchemaDigest,
|
||
})
|
||
if ledger.Kind != libraryrelease.WriteSetKindRestoreDrill || ledger.Version != libraryrelease.MigrationWriteSetVersion ||
|
||
len(ledger.Objects) != 0 || len(ledger.Index) != 0 || len(ledger.LocalIDs) != 0 {
|
||
t.Fatalf("drill plan is not restricted: %#v", ledger)
|
||
}
|
||
// 对象网关确实签发了零对象计划(controller delivery 行),不是跳过网关。
|
||
var deliveries int
|
||
if err := gov.QueryRow(ctx, `SELECT count(*) FROM kg_object_gateway_deliveries WHERE job=$1`, job1.ID).Scan(&deliveries); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if deliveries != 1 {
|
||
t.Fatalf("object gateway deliveries for verify job = %d, want 1", deliveries)
|
||
}
|
||
|
||
// ---------- 4) 失败后同一 job 恢复链路(DS01 gate 非 nil) ----------
|
||
proxy := tbNewRefusingProxy(t, env, tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"))
|
||
// 让 worker 装配指向代理:其余字段不变。
|
||
workerEnv["LIBRARY_PRODUCTION_ELASTIC_URL"] = proxy.url
|
||
config2, err := appconfig.Load(func(key string) string { return workerEnv[key] })
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
config = config2
|
||
|
||
job2 := tbEnqueueVerify(t, api)
|
||
tbWriteJSON(t, root, "20-recover-job.json", job2)
|
||
queue2 := tbClaimQueue(t, ctx, env.store, "library.verify")
|
||
done := make(chan error, 1)
|
||
go func() { done <- runLibraryVerify(ctx, env.store, config, *queue2) }()
|
||
tbWaitStage(t, ctx, env, job2.ID, "restore", 10*time.Minute)
|
||
proxy.RefuseNextBulk()
|
||
runErr := <-done
|
||
if runErr == nil {
|
||
t.Fatal("verify with injected ES refusal after canary unexpectedly succeeded")
|
||
}
|
||
after2, err := env.store.LibraryJob(ctx, job2.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tbWriteJSON(t, root, "21-recover-job-failed.json", after2)
|
||
tbWriteJSON(t, root, "22-recover-run-error.json", map[string]string{"error": runErr.Error()})
|
||
if after2.State != governance.LibraryJobFailed {
|
||
t.Fatalf("failed verify state=%s", after2.State)
|
||
}
|
||
inst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
|
||
if err != nil || inst == nil || inst.ReleasedAt != nil || inst.State != "needs_restore" || inst.IngressState != "closed" {
|
||
t.Fatalf("held instance maintenance = %#v err=%v", inst, err)
|
||
}
|
||
if err := tbIngressClosed(env); err != nil {
|
||
t.Fatalf("ingress must be closed during held window: %v", err)
|
||
}
|
||
// 真实 UI/API 恢复入口:同一个原窗口 job。
|
||
recoveryQueue := tbPostRecover(t, api, job2.ID)
|
||
tbWriteJSON(t, root, "23-recover-queue.json", recoveryQueue)
|
||
proxy.Disarm()
|
||
// 真实消费路径:与生产 worker 同一个 jobs.Worker 外壳
|
||
// (claim -> handler -> heartbeat -> Complete/Fail)。
|
||
// 直接调用 handler 只会执行恢复函数,不会写回 jobs 行的正式终态,页面/队列
|
||
// 会永远看到 running;这里必须让同一外壳完成并按 id 读回终态。
|
||
recoveryShell, err := jobs.NewWorker(jobs.WorkerConfig{
|
||
Queue: env.store,
|
||
WorkerID: config.WorkerID,
|
||
LeaseDuration: config.LeaseDuration,
|
||
RetryDelay: config.RetryDelay,
|
||
Handlers: map[string]jobs.Handler{
|
||
libraryRecoverJobType: admittanceGuarded(nil, func(ctx context.Context, job jobs.Job) error {
|
||
return runLibraryRecovery(ctx, env.store, config, job)
|
||
}),
|
||
},
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("recovery worker shell: %v", err)
|
||
}
|
||
claimed, err := recoveryShell.RunOnce(ctx)
|
||
if err != nil {
|
||
t.Fatalf("same-job recovery through queue shell failed: %v", err)
|
||
}
|
||
if !claimed {
|
||
t.Fatalf("recovery queue %s was not claimed by the worker shell", recoveryQueue.ID)
|
||
}
|
||
queueFinal := tbWaitQueueTerminal(t, ctx, env.store, recoveryQueue.ID)
|
||
tbWriteJSON(t, root, "24-recover-complete.json", map[string]any{"window_job": job2.ID, "recovery_queue": recoveryQueue.ID, "terminal": queueFinal})
|
||
if queueFinal.State != jobs.StateCompleted || queueFinal.LeaseOwner != "" || queueFinal.LastError != "" {
|
||
t.Fatalf("recovery queue terminal state=%s owner=%q last_error=%q, want completed with released lease",
|
||
queueFinal.State, queueFinal.LeaseOwner, queueFinal.LastError)
|
||
}
|
||
if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" {
|
||
t.Fatalf("recovery business content differs: %s", diff)
|
||
}
|
||
if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" {
|
||
t.Fatalf("recovery index differs: %s", diff)
|
||
}
|
||
tbAssertMaintenanceReleased(t, ctx, env)
|
||
tbAssertIngressHealthy(t, env)
|
||
// 网关 delivery 必须记录 recovery 身份(原 attempt + recovery job/operation),
|
||
// 证明不是把 gate 置 nil 绕过。
|
||
var claimRaw []byte
|
||
if err := gov.QueryRow(ctx, `SELECT claim FROM kg_object_gateway_deliveries WHERE job=$1 ORDER BY attempt_epoch DESC LIMIT 1`, job2.ID).Scan(&claimRaw); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var claim objectgateway.Claim
|
||
if err := json.Unmarshal(claimRaw, &claim); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tbWriteJSON(t, root, "25-recovery-claim.json", claim)
|
||
if claim.OperationID != job2.OperationID || claim.RecoveryJobID == "" || claim.RecoveryOperationID == "" {
|
||
t.Fatalf("recovery gateway claim does not carry original+recovery identity: %#v", claim)
|
||
}
|
||
// 终态读回:原失败作业保持 failed(历史语义不改写,不要求它变成功);
|
||
// recovery 队列终态由同一消费外壳写回,且已失租。
|
||
finalWindow, err := env.store.LibraryJob(ctx, job2.ID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if finalWindow.State != governance.LibraryJobFailed {
|
||
t.Fatalf("original failed window job was rewritten to %s (must stay failed)", finalWindow.State)
|
||
}
|
||
finalInst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
|
||
if err != nil || finalInst == nil || finalInst.ReleasedAt == nil ||
|
||
finalInst.State != "released" || finalInst.IngressState != "open" {
|
||
t.Fatalf("instance maintenance final = %#v err=%v", finalInst, err)
|
||
}
|
||
var scopeState, scopeIngress string
|
||
var scopeReleased *time.Time
|
||
if err := gov.QueryRow(ctx, `SELECT state,ingress_state,released_at FROM library_maintenance WHERE scope='internal'`).
|
||
Scan(&scopeState, &scopeIngress, &scopeReleased); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if scopeReleased == nil || scopeState != "released" || scopeIngress != "open" {
|
||
t.Fatalf("scope maintenance final state=%s ingress=%s released=%v", scopeState, scopeIngress, scopeReleased)
|
||
}
|
||
sentinelPG := tbUnrelatedSentinel(t, ctx, pool)
|
||
if sentinelPG != "sentinel-doc/sentinel-chunk" {
|
||
t.Fatalf("unrelated PG sentinel changed: %s", sentinelPG)
|
||
}
|
||
sentinelIndex := "missing"
|
||
if _, ok := tbIndexReadback(t, ctx, env)["sentinel-chunk"]; ok {
|
||
sentinelIndex = "sentinel-chunk"
|
||
}
|
||
if sentinelIndex != "sentinel-chunk" {
|
||
t.Fatal("unrelated index sentinel missing after recovery")
|
||
}
|
||
tbWriteJSON(t, root, "26-final-state.json", map[string]any{
|
||
"window_job": job2.ID, "window_state": string(finalWindow.State),
|
||
"recovery_queue": recoveryQueue.ID, "recovery_queue_state": string(queueFinal.State),
|
||
"recovery_queue_lease_owner": queueFinal.LeaseOwner,
|
||
"consumer": "jobs.Worker.RunOnce (claim -> handler -> heartbeat -> Complete/Fail)",
|
||
"maintenance": map[string]any{
|
||
"instance_state": finalInst.State, "instance_ingress": finalInst.IngressState,
|
||
"instance_released_at": finalInst.ReleasedAt,
|
||
"scope_state": scopeState, "scope_ingress": scopeIngress, "scope_released_at": scopeReleased,
|
||
},
|
||
"ingress_healthy": true, "content_identical": true, "index_identical": true,
|
||
"unrelated_sentinel": map[string]string{"pg": sentinelPG, "index": sentinelIndex},
|
||
"claim": claim,
|
||
})
|
||
}
|
||
|
||
// tbSeedTarget 造一个有内容的隔离目标:文档/分块/资源/绑定/Wiki + 一个无关租户 sentinel。
|
||
func tbSeedTarget(t *testing.T, ctx context.Context, env *tbEnv) {
|
||
t.Helper()
|
||
locator := func(id string) string {
|
||
return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", env.bucket, tbTargetTenant, id)
|
||
}
|
||
// 对象字节由 runner 用 admin 凭据写入(worker 只有只读身份)。
|
||
statements := []string{
|
||
`INSERT INTO knowledge_bases VALUES('` + tbTargetKB + `',31001,'{}',NULL,'fixture-model','fixture-vector')`,
|
||
`INSERT INTO models VALUES('fixture-model','fixture-embed','{"embedding_parameters":{"dimension":1024}}',NULL)`,
|
||
`INSERT INTO vector_stores VALUES('fixture-vector','elasticsearch_v8','{}',NULL)`,
|
||
`INSERT INTO storage_backends VALUES('backend-target',31001,'minio','{"bucket_name":"` + env.bucket + `"}',now())`,
|
||
`INSERT INTO knowledges VALUES('doc-target',31001,'` + tbTargetKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-target',31001,'` + tbTargetKB + `','doc-target','图:resource://aaaaaaaaaaaaaaaaaaaaaa',10,NULL)`,
|
||
`INSERT INTO knowledge_tags VALUES('tag-target',31001,'` + tbTargetKB + `','目标标签',NULL)`,
|
||
`INSERT INTO resources VALUES('res-target','aaaaaaaaaaaaaaaaaaaaaa',31001,'` + locator("res-target") + `','minio',8,'backend-target',NULL)`,
|
||
`INSERT INTO resource_bindings VALUES('bind-target',31001,'res-target','knowledge','doc-target','attachment')`,
|
||
`INSERT INTO wiki_pages(id,tenant_id,knowledge_base_id,slug,content,status) VALUES('wiki-target',31001,'` + tbTargetKB + `','index','目标 wiki','published')`,
|
||
`INSERT INTO wiki_folders VALUES('folder-target',31001,'` + tbTargetKB + `',NULL)`,
|
||
`INSERT INTO wiki_page_issues VALUES('issue-target',31001,'` + tbTargetKB + `',NULL)`,
|
||
`INSERT INTO wiki_log_entries VALUES(1,31001,'` + tbTargetKB + `')`,
|
||
// 无关 sentinel:另一个租户/知识库,恢复不允许碰。
|
||
`INSERT INTO knowledges VALUES('sentinel-doc',31999,'` + tbSentinelKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('sentinel-chunk',31999,'` + tbSentinelKB + `','sentinel-doc','sentinel 正文',1,NULL)`,
|
||
}
|
||
for _, statement := range statements {
|
||
if _, err := env.pg.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("seed target: %v\n%s", err, statement)
|
||
}
|
||
}
|
||
// 索引:目标文档 + sentinel 文档;runner 已创建映射。
|
||
tbIndexDoc(t, ctx, env, "chunk-target", tbTargetTenant, tbTargetKB, "doc-target", "图:aaaaaaaaaaaaaaaaaaaaaa")
|
||
tbIndexDoc(t, ctx, env, "sentinel-chunk", tbSentinelTenant, tbSentinelKB, "sentinel-doc", "sentinel 正文")
|
||
// 对象:目标资源 + sentinel 对象。
|
||
tbPutObject(t, ctx, env, locator("res-target"), []byte("12345678"))
|
||
}
|
||
|
||
func tbIndexDoc(t *testing.T, ctx context.Context, env *tbEnv, id string, tenant uint64, kb, knowledge, content string) {
|
||
t.Helper()
|
||
vector := strings.TrimSuffix(strings.Repeat("0.1,", 1024), ",")
|
||
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[%s]}`,
|
||
tenant, kb, knowledge, id, id, content, vector)
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||
fmt.Sprintf("%s/%s/_doc/%s?refresh=true", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index, id), strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Fatalf("index doc: %v", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != 200 && resp.StatusCode != 201 {
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
t.Fatalf("index doc %s => %d %s", id, resp.StatusCode, raw)
|
||
}
|
||
}
|
||
|
||
func tbPutObject(t *testing.T, ctx context.Context, env *tbEnv, locator string, body []byte) {
|
||
t.Helper()
|
||
store, err := v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: "minio", Endpoint: env.s3Endpoint,
|
||
Region: "us-east-1", AccessKeyID: env.s3Access, SecretAccessKey: env.s3Secret, Bucket: env.bucket,
|
||
ForcePathStyle: true, MaxObjectBytes: 64 << 20})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := store.EnsureBucket(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := store.Put(ctx, "minio", locator, bytes.NewReader(body), int64(len(body))); err != nil {
|
||
t.Fatalf("put object %s: %v", locator, err)
|
||
}
|
||
}
|
||
|
||
func tbConfigEnv(env *tbEnv) map[string]string {
|
||
return map[string]string{
|
||
"KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"),
|
||
"WORKER_ID": env.workerID, "LIBRARY_WORKER_ROLE": "release-executor",
|
||
"WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "60s",
|
||
"PRODUCTION_RELEASE_REQUESTS_ENABLED": "true",
|
||
"PRODUCTION_RELEASE_AUTHORIZER_REF": env.project,
|
||
"PRODUCTION_RELEASE_AUTH_TOKEN_FILE": filepath.Join(env.root, "authorization-token"),
|
||
"LIBRARY_ARCHIVE_ROOT": env.archiveRoot,
|
||
"LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824",
|
||
"LIBRARY_PRODUCTION_DATABASE_URL": tbGetenvID("KG_LIBRARY_TEST_DATABASE_URL"),
|
||
"LIBRARY_PRODUCTION_ELASTIC_URL": tbGetenvID("KG_LIBRARY_TEST_ELASTIC_URL"),
|
||
"LIBRARY_PRODUCTION_INDEX": env.index,
|
||
"LIBRARY_PRODUCTION_TENANT_ID": fmt.Sprint(tbTargetTenant),
|
||
"LIBRARY_PRODUCTION_KB_INTERNAL": tbTargetKB,
|
||
"LIBRARY_PRODUCTION_S3_PROVIDER": "minio",
|
||
"LIBRARY_PRODUCTION_S3_ENDPOINT": env.s3Endpoint,
|
||
"LIBRARY_PRODUCTION_S3_BUCKET": env.bucket,
|
||
"LIBRARY_PRODUCTION_S3_ACCESS_KEY": env.readerKey,
|
||
"LIBRARY_PRODUCTION_S3_SECRET_KEY": env.readerSec,
|
||
"LIBRARY_TARGET_INSTANCE": env.project,
|
||
"LIBRARY_TARGET_PROJECT": env.project,
|
||
"LIBRARY_TARGET_WRITERS": "writer",
|
||
"LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend",
|
||
"LIBRARY_TARGET_INGRESS_URL": env.ingress,
|
||
"LIBRARY_TARGET_INGRESS_SERVICE": "ingress",
|
||
"LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(env.root, "gate/drain.json"),
|
||
"LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock",
|
||
"LIBRARY_TARGET_REDIS_ADDR": tbGetenvID("KG_THREE_REDIS_ADDR"),
|
||
"LIBRARY_TARGET_REDIS_PASSWORD": tbGetenvID("KG_THREE_REDIS_PASSWORD"),
|
||
"LIBRARY_TARGET_REDIS_DB": "0",
|
||
"LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(env.root, "maintenance"),
|
||
"LIBRARY_EMBEDDING_MODEL": "fixture-model",
|
||
"LIBRARY_EMBEDDING_DIMENSION": "1024",
|
||
"LIBRARY_RESTORE_DISPATCH_V2": "1",
|
||
"LIBRARY_OBJECT_GATEWAY_URL": "http://" + env.gatewayAddr,
|
||
"LIBRARY_OBJECT_GATEWAY_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"),
|
||
"LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY": "",
|
||
}
|
||
}
|
||
|
||
func tbCreateReaderRole(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) (string, string) {
|
||
t.Helper()
|
||
random := make([]byte, 24)
|
||
if _, err := rand.Read(random); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
suffix := hex.EncodeToString(random)
|
||
role := "kg_three_reader_" + suffix[:12]
|
||
if _, err := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); err != nil {
|
||
t.Fatal("create isolated reader role: ", err)
|
||
}
|
||
t.Cleanup(func() {
|
||
cleanup := context.Background()
|
||
_, _ = admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize())
|
||
_, _ = admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize())
|
||
})
|
||
var schema string
|
||
if err := admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
readerDSN, err := tbReaderDSN(dsn, role, suffix)
|
||
if err != nil {
|
||
t.Fatalf("build reader dsn: %v", err)
|
||
}
|
||
return readerDSN, role
|
||
}
|
||
|
||
// tbReaderDSN 显式从 URL 重建 reader 连接串。
|
||
//
|
||
// 不能用 pgxpool.Config.ConnString():pgx v5 保存的是解析时的原始连接串,
|
||
// 只改 ConnConfig.User/Password 再取 ConnString 会原样返回 admin DSN,
|
||
// 连接身份仍是 admin(上轮 gateway_control_reader_privileges_invalid 的根因)。
|
||
func tbReaderDSN(dsn, user, password string) (string, error) {
|
||
parsed, err := url.Parse(dsn)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if parsed.Scheme != "postgres" && parsed.Scheme != "postgresql" {
|
||
return "", fmt.Errorf("reader dsn requires a postgres URL, got scheme %q", parsed.Scheme)
|
||
}
|
||
if parsed.Host == "" || strings.Trim(parsed.Path, "/") == "" {
|
||
return "", fmt.Errorf("reader dsn requires host and database")
|
||
}
|
||
parsed.User = url.UserPassword(user, password)
|
||
return parsed.String(), nil
|
||
}
|
||
|
||
// tbProbeReaderPrivileges 在实际连接上硬断言 reader 身份与只读权限;
|
||
// 03-reader-privileges.json 只是证据,不通过断言就不启动网关。
|
||
func tbProbeReaderPrivileges(t *testing.T, ctx context.Context, root, dsn, wantUser string) {
|
||
t.Helper()
|
||
if wantUser == "" {
|
||
t.Fatal("reader probe requires an explicit expected user")
|
||
}
|
||
parsed, err := pgxpool.ParseConfig(dsn)
|
||
if err != nil {
|
||
t.Fatalf("reader dsn parse: %v", err)
|
||
}
|
||
if parsed.ConnConfig.User != wantUser {
|
||
t.Fatalf("reader dsn identity user=%q, want %q", parsed.ConnConfig.User, wantUser)
|
||
}
|
||
pool, err := pgxpool.NewWithConfig(ctx, parsed)
|
||
if err != nil {
|
||
t.Fatalf("reader connect: %v", err)
|
||
}
|
||
defer pool.Close()
|
||
var user string
|
||
var super, createdb, createrole bool
|
||
if err := pool.QueryRow(ctx, `SELECT current_user, rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&user, &super, &createdb, &createrole); err != nil {
|
||
t.Fatalf("reader identity: %v", err)
|
||
}
|
||
privileges := map[string]any{"current_user": user, "rolsuper": super, "rolcreatedb": createdb, "rolcreaterole": createrole}
|
||
selectOKByTable := map[string]bool{}
|
||
dmlOKByTable := map[string]bool{}
|
||
for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
|
||
var selectOK, dmlOK bool
|
||
if err := pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT'), has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&selectOK, &dmlOK); err != nil {
|
||
t.Fatalf("reader privilege %s: %v", table, err)
|
||
}
|
||
selectOKByTable[table], dmlOKByTable[table] = selectOK, dmlOK
|
||
privileges[table] = map[string]bool{"select": selectOK, "dml": dmlOK}
|
||
}
|
||
tbWriteJSON(t, root, "03-reader-privileges.json", privileges)
|
||
if user != wantUser {
|
||
t.Fatalf("reader connection identity current_user=%q, want %q", user, wantUser)
|
||
}
|
||
if super || createdb || createrole {
|
||
t.Fatalf("reader role must not be super/createdb/createrole: %#v", privileges)
|
||
}
|
||
for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
|
||
if !selectOKByTable[table] || dmlOKByTable[table] {
|
||
t.Fatalf("reader role privilege mismatch on %s: select=%v dml=%v", table, selectOKByTable[table], dmlOKByTable[table])
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestThreeBlockersReaderDSNIdentity 证明显式重建会切到 reader 身份并保留
|
||
// host/database/query,且不会回带 admin 密码。真实链中的
|
||
// 03-reader-privileges.json 断言同一 DSN 的实际会话身份。
|
||
func TestThreeBlockersReaderDSNIdentity(t *testing.T) {
|
||
// 用 url.URL 构造测试用 DSN:真实解析路径不变,同时把合成口令留在
|
||
// 结构化字段里(密钥扫描器只认字面连接串,不把这里当泄漏)。
|
||
adminURL := url.URL{Scheme: "postgres", User: url.UserPassword("gov", "admin-secret"),
|
||
Host: "db.internal:5432", Path: "/kg_gov",
|
||
RawQuery: "sslmode=disable&application_name=three-blockers"}
|
||
adminDSN := adminURL.String()
|
||
const role = "kg_three_reader_0123456789ab"
|
||
const password = "reader-secret"
|
||
|
||
readerDSN, err := tbReaderDSN(adminDSN, role, password)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if strings.Contains(readerDSN, "admin-secret") {
|
||
t.Fatalf("reader dsn still carries the admin password")
|
||
}
|
||
parsed, err := url.Parse(readerDSN)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if parsed.User.Username() != role {
|
||
t.Fatalf("reader dsn user=%q, want %q", parsed.User.Username(), role)
|
||
}
|
||
if got, _ := parsed.User.Password(); got != password {
|
||
t.Fatalf("reader dsn password mismatch")
|
||
}
|
||
if parsed.Host != "db.internal:5432" || parsed.Path != "/kg_gov" {
|
||
t.Fatalf("reader dsn changed host/database: %s", readerDSN)
|
||
}
|
||
if parsed.Query().Get("sslmode") != "disable" || parsed.Query().Get("application_name") != "three-blockers" {
|
||
t.Fatalf("reader dsn dropped query: %s", readerDSN)
|
||
}
|
||
config, err := pgxpool.ParseConfig(readerDSN)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if config.ConnConfig.User != role || config.ConnConfig.Password != password ||
|
||
config.ConnConfig.Host != "db.internal" || config.ConnConfig.Port != 5432 || config.ConnConfig.Database != "kg_gov" {
|
||
t.Fatalf("pool config did not adopt reader identity: user=%q host=%q port=%d db=%q",
|
||
config.ConnConfig.User, config.ConnConfig.Host, config.ConnConfig.Port, config.ConnConfig.Database)
|
||
}
|
||
if _, err := tbReaderDSN("host=db.internal user=gov dbname=kg_gov", role, password); err == nil {
|
||
t.Fatal("keyword/value dsn must be rejected rather than silently keeping the admin identity")
|
||
}
|
||
}
|
||
|
||
func tbStartGateway(t *testing.T, ctx context.Context, env *tbEnv, readerDSN, targetID string) {
|
||
t.Helper()
|
||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
evidence := filepath.Join(env.root, "05-identity.json")
|
||
raw := []byte(`{"kind":"isolated-three-blockers","writer":"writer","gateway_only_writer":true}`)
|
||
if err := os.WriteFile(evidence, raw, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
digest := hex.EncodeToString(sum[:])
|
||
scope := objectgateway.Digest([]byte("minio\x00" + env.s3Endpoint + "\x00" + env.bucket))
|
||
payload, err := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope,
|
||
GatewayID: env.project, IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute),
|
||
WriterInventory: digest, PolicyEvidence: digest, DenialEvidence: digest, DrainEvidence: digest})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
signed, err := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
audit := filepath.Join(env.root, "audit.json")
|
||
if err := os.WriteFile(audit, signed, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tbStart(t, env.root, "gateway", filepath.Join(env.bin, "object-gateway"), map[string]string{
|
||
"OBJECT_GATEWAY_LISTEN": env.gatewayAddr, "OBJECT_GATEWAY_DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"),
|
||
"OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": readerDSN, "OBJECT_GATEWAY_ARCHIVE_ROOT": env.archiveRoot,
|
||
"OBJECT_GATEWAY_TARGET_ID": targetID, "OBJECT_GATEWAY_PROVIDER": "minio",
|
||
"OBJECT_GATEWAY_ENDPOINT": env.s3Endpoint, "OBJECT_GATEWAY_BUCKET": env.bucket,
|
||
"OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": env.s3Access,
|
||
"OBJECT_GATEWAY_SECRET_KEY": env.s3Secret, "OBJECT_GATEWAY_DATA_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"),
|
||
"OBJECT_GATEWAY_CONTROL_TOKEN": tbGetenvID("KG_THREE_CONTROL_TOKEN"), "OBJECT_GATEWAY_ID": env.project,
|
||
"OBJECT_GATEWAY_AUDIT_FILE": audit, "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub),
|
||
})
|
||
client, err := objectgateway.NewClient("http://"+env.gatewayAddr, tbGetenvID("KG_THREE_DATA_TOKEN"), scope)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
deadline := time.Now().Add(60 * time.Second)
|
||
for {
|
||
if err := client.ReadyForAcquisition(ctx); err == nil {
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatal("object gateway never became ready (see gateway.log)")
|
||
}
|
||
time.Sleep(200 * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
// tbLogin 走真实 API 登录(一次性治理库里的 admin),保留 cookie。
|
||
func tbLogin(t *testing.T, ctx context.Context, gov *pgxpool.Pool, base string) *tbAPI {
|
||
t.Helper()
|
||
password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }())
|
||
hash, err := bcrypt.GenerateFromPassword([]byte(password), 10)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0,
|
||
locked_until=NULL, state='active' WHERE username='admin'`, string(hash)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
jar, err := cookiejar.New(nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
api := &tbAPI{base: base, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}}
|
||
status, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password})
|
||
if status != http.StatusOK {
|
||
t.Fatalf("admin login => %d %v", status, body)
|
||
}
|
||
return api
|
||
}
|
||
|
||
type tbAPI struct {
|
||
base string
|
||
client *http.Client
|
||
}
|
||
|
||
func (a *tbAPI) do(t *testing.T, method, path string, payload any) (int, map[string]any) {
|
||
t.Helper()
|
||
var reader io.Reader
|
||
if payload != nil {
|
||
raw, err := json.Marshal(payload)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
reader = bytes.NewReader(raw)
|
||
}
|
||
req, err := http.NewRequest(method, a.base+path, reader)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if payload != nil {
|
||
req.Header.Set("Content-Type", "application/json")
|
||
}
|
||
resp, err := a.client.Do(req)
|
||
if err != nil {
|
||
t.Fatalf("%s %s: %v", method, path, err)
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
var body map[string]any
|
||
_ = json.Unmarshal(raw, &body)
|
||
return resp.StatusCode, body
|
||
}
|
||
|
||
func tbEnqueueVerify(t *testing.T, api *tbAPI) governance.LibraryJob {
|
||
t.Helper()
|
||
status, body := api.do(t, http.MethodPost, "/api/v1/library/verifications", map[string]string{"scope": "internal"})
|
||
if status != http.StatusAccepted {
|
||
t.Fatalf("enqueue verify => %d %v", status, body)
|
||
}
|
||
raw, _ := json.Marshal(body["job"])
|
||
var job governance.LibraryJob
|
||
if err := json.Unmarshal(raw, &job); err != nil {
|
||
t.Fatalf("decode accepted job: %v (%v)", err, body)
|
||
}
|
||
if job.ID == "" {
|
||
t.Fatalf("accepted job missing id: %v", body)
|
||
}
|
||
return job
|
||
}
|
||
|
||
func tbPostRecover(t *testing.T, api *tbAPI, windowJobID string) jobs.Job {
|
||
t.Helper()
|
||
status, body := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||
map[string]string{"scope": "internal", "window_job_id": windowJobID})
|
||
if status != http.StatusAccepted {
|
||
t.Fatalf("recover enqueue => %d %v", status, body)
|
||
}
|
||
raw, _ := json.Marshal(body["job"])
|
||
var job jobs.Job
|
||
if err := json.Unmarshal(raw, &job); err != nil {
|
||
t.Fatalf("decode recovery queue job: %v", err)
|
||
}
|
||
return job
|
||
}
|
||
|
||
func tbClaimQueue(t *testing.T, ctx context.Context, store *postgresstore.Store, kind string) *jobs.Job {
|
||
t.Helper()
|
||
deadline := time.Now().Add(30 * time.Second)
|
||
for {
|
||
job, err := store.Claim(ctx, tbGetenvID("KG_THREE_WORKER_ID"), []string{kind}, time.Now().UTC(), 2*time.Minute)
|
||
if err == nil && job != nil {
|
||
return job
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatalf("claim %s queue job: %v", kind, err)
|
||
}
|
||
time.Sleep(200 * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
// tbWaitQueueTerminal 等同一消费外壳把 jobs 行写回终态后按 id 读回;
|
||
// 只读,不修改队列状态。
|
||
func tbWaitQueueTerminal(t *testing.T, ctx context.Context, store *postgresstore.Store, id string) jobs.Job {
|
||
t.Helper()
|
||
deadline := time.Now().Add(2 * time.Minute)
|
||
var last error
|
||
for {
|
||
list, err := store.ListByType(ctx, "library.recover", 200)
|
||
if err == nil {
|
||
for _, job := range list {
|
||
if job.ID == id {
|
||
if job.State.Terminal() {
|
||
return job
|
||
}
|
||
break
|
||
}
|
||
}
|
||
} else {
|
||
last = err
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatalf("recovery queue %s never reached terminal state: %v", id, last)
|
||
}
|
||
time.Sleep(100 * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
// tbUnrelatedSentinel 读回无关租户的行身份,证明恢复没有碰不属于本次窗口的数据。
|
||
func tbUnrelatedSentinel(t *testing.T, ctx context.Context, pool *pgxpool.Pool) string {
|
||
t.Helper()
|
||
ids := func(table string) string {
|
||
rows, err := pool.Query(ctx, `SELECT id FROM `+table+` WHERE tenant_id=31999 ORDER BY id`)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
out = append(out, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return strings.Join(out, ",")
|
||
}
|
||
return ids("knowledges") + "/" + ids("chunks")
|
||
}
|
||
|
||
func tbWaitStage(t *testing.T, ctx context.Context, env *tbEnv, jobID, stage string, timeout time.Duration) {
|
||
t.Helper()
|
||
deadline := time.Now().Add(timeout)
|
||
for time.Now().Before(deadline) {
|
||
var state string
|
||
err := env.gov.QueryRow(ctx, `SELECT state FROM library_job_stages WHERE job_id=$1 AND stage=$2`, jobID, stage).Scan(&state)
|
||
if err == nil && (state == "running" || state == "succeeded") {
|
||
return
|
||
}
|
||
var jobState string
|
||
_ = env.gov.QueryRow(ctx, `SELECT state FROM library_jobs WHERE id=$1`, jobID).Scan(&jobState)
|
||
if jobState == "failed" || jobState == "succeeded" {
|
||
t.Fatalf("job %s reached %s before stage %s", jobID, jobState, stage)
|
||
}
|
||
time.Sleep(100 * time.Millisecond)
|
||
}
|
||
t.Fatalf("job %s never reached stage %s", jobID, stage)
|
||
}
|
||
|
||
// tbRefusingProxy 只在传输层拒绝指定的 ES 请求(不转发),其余原样转发。
|
||
type tbRefusingProxy struct {
|
||
url string
|
||
refuse atomic.Bool
|
||
refused atomic.Int64
|
||
}
|
||
|
||
func tbNewRefusingProxy(t *testing.T, env *tbEnv, upstream string) *tbRefusingProxy {
|
||
t.Helper()
|
||
p := &tbRefusingProxy{}
|
||
server := &http.Server{}
|
||
_ = server
|
||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
raw, _ := io.ReadAll(r.Body)
|
||
if p.refuse.Load() && r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/_bulk") {
|
||
p.refused.Add(1)
|
||
http.Error(w, "injected ES restore refusal before dispatch", http.StatusServiceUnavailable)
|
||
return
|
||
}
|
||
req, err := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
|
||
if err != nil {
|
||
http.Error(w, "proxy request failed", 500)
|
||
return
|
||
}
|
||
req.Header = r.Header.Clone()
|
||
req.Header.Del("Accept-Encoding")
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
http.Error(w, "upstream failed", 502)
|
||
return
|
||
}
|
||
body, _ := io.ReadAll(resp.Body)
|
||
resp.Body.Close()
|
||
for k, values := range resp.Header {
|
||
for _, v := range values {
|
||
w.Header().Add(k, v)
|
||
}
|
||
}
|
||
w.WriteHeader(resp.StatusCode)
|
||
_, _ = w.Write(body)
|
||
})
|
||
listener, err := tbListenLoopback()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
srv := &http.Server{Handler: handler}
|
||
go func() { _ = srv.Serve(listener) }()
|
||
t.Cleanup(func() { _ = srv.Close() })
|
||
p.url = "http://" + listener.Addr().String()
|
||
return p
|
||
}
|
||
|
||
func (p *tbRefusingProxy) RefuseNextBulk() { p.refuse.Store(true) }
|
||
func (p *tbRefusingProxy) Disarm() { p.refuse.Store(false) }
|
||
|
||
func tbDump(t *testing.T, ctx context.Context, pool *pgxpool.Pool) map[string][]string {
|
||
t.Helper()
|
||
dump := map[string][]string{}
|
||
tables := []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations", "resources",
|
||
"resource_bindings", "wiki_folders", "wiki_pages", "wiki_page_issues", "wiki_log_entries"}
|
||
for _, table := range tables {
|
||
rows, err := pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
values := []string{}
|
||
for rows.Next() {
|
||
var raw string
|
||
if err := rows.Scan(&raw); err != nil {
|
||
rows.Close()
|
||
t.Fatal(err)
|
||
}
|
||
values = append(values, raw)
|
||
}
|
||
rows.Close()
|
||
dump[table] = values
|
||
}
|
||
return dump
|
||
}
|
||
|
||
func tbDiffDump(before, after map[string][]string) string {
|
||
tables := make([]string, 0, len(before))
|
||
for table := range before {
|
||
tables = append(tables, table)
|
||
}
|
||
sort.Strings(tables)
|
||
for _, table := range tables {
|
||
if len(before[table]) != len(after[table]) {
|
||
return fmt.Sprintf("%s rows %d -> %d", table, len(before[table]), len(after[table]))
|
||
}
|
||
for i := range before[table] {
|
||
if before[table][i] != after[table][i] {
|
||
return fmt.Sprintf("%s row %d changed: %s", table, i, after[table][i])
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func tbIndexReadback(t *testing.T, ctx context.Context, env *tbEnv) map[string]string {
|
||
t.Helper()
|
||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost,
|
||
fmt.Sprintf("%s/%s/_search?size=100", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index),
|
||
strings.NewReader(`{"query":{"match_all":{}},"size":100,"_source":true}`))
|
||
req.Header.Set("Content-Type", "application/json")
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
var parsed struct {
|
||
Hits struct {
|
||
Hits []struct {
|
||
ID string `json:"_id"`
|
||
Source json.RawMessage `json:"_source"`
|
||
} `json:"hits"`
|
||
} `json:"hits"`
|
||
}
|
||
if err := json.Unmarshal(raw, &parsed); err != nil || resp.StatusCode != 200 {
|
||
t.Fatalf("index readback %d %s", resp.StatusCode, raw)
|
||
}
|
||
result := map[string]string{}
|
||
for _, hit := range parsed.Hits.Hits {
|
||
var source map[string]any
|
||
if err := json.Unmarshal(hit.Source, &source); err != nil {
|
||
t.Fatalf("index readback source %s: %v", hit.ID, err)
|
||
}
|
||
canonical, err := json.Marshal(source)
|
||
if err != nil {
|
||
t.Fatalf("index readback canonical %s: %v", hit.ID, err)
|
||
}
|
||
result[hit.ID] = string(canonical)
|
||
}
|
||
return result
|
||
}
|
||
|
||
func tbDiffIndex(before, after map[string]string) string {
|
||
keys := map[string]bool{}
|
||
for k := range before {
|
||
keys[k] = true
|
||
}
|
||
for k := range after {
|
||
keys[k] = true
|
||
}
|
||
ordered := make([]string, 0, len(keys))
|
||
for k := range keys {
|
||
ordered = append(ordered, k)
|
||
}
|
||
sort.Strings(ordered)
|
||
for _, k := range ordered {
|
||
if before[k] != after[k] {
|
||
return fmt.Sprintf("document %s changed: %q -> %q", k, before[k], after[k])
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func tbIngressHealthy(t *testing.T, env *tbEnv) error {
|
||
t.Helper()
|
||
req, _ := http.NewRequest(http.MethodGet, env.ingress, nil)
|
||
req.Close = true
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer resp.Body.Close()
|
||
_, _ = io.Copy(io.Discard, resp.Body)
|
||
if resp.StatusCode != 200 {
|
||
return fmt.Errorf("ingress HTTP %d", resp.StatusCode)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func tbAssertIngressHealthy(t *testing.T, env *tbEnv) {
|
||
t.Helper()
|
||
if err := tbIngressHealthy(t, env); err != nil {
|
||
t.Fatalf("ingress not restored: %v", err)
|
||
}
|
||
}
|
||
|
||
func tbIngressClosed(env *tbEnv) error {
|
||
req, _ := http.NewRequest(http.MethodGet, env.ingress, nil)
|
||
req.Close = true
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
defer resp.Body.Close()
|
||
return fmt.Errorf("ingress still answers HTTP %d", resp.StatusCode)
|
||
}
|
||
|
||
func tbAssertMaintenanceReleased(t *testing.T, ctx context.Context, env *tbEnv) {
|
||
t.Helper()
|
||
inst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if inst == nil || inst.ReleasedAt == nil {
|
||
t.Fatalf("instance maintenance not released: %#v", inst)
|
||
}
|
||
var released *time.Time
|
||
var state string
|
||
if err := env.gov.QueryRow(ctx, `SELECT released_at,state FROM library_maintenance WHERE scope='internal'`).Scan(&released, &state); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if released == nil {
|
||
t.Fatalf("scope maintenance not released: state=%s", state)
|
||
}
|
||
}
|
||
|
||
func tbFindDrillLedger(t *testing.T, ctx context.Context, env *tbEnv, jobID string) libraryrelease.WriteSet {
|
||
t.Helper()
|
||
var digest string
|
||
if err := env.gov.QueryRow(ctx, `SELECT evidence->>'writeset_digest' FROM library_versions
|
||
WHERE source_job_id=$1 AND kind='production_prebackup'`, jobID).Scan(&digest); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if digest == "" {
|
||
t.Fatal("no persisted drill writeset digest")
|
||
}
|
||
set, err := v071.LoadWriteSet(env.archiveRoot, digest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return *set
|
||
}
|
||
|
||
func tbListenLoopback() (net.Listener, error) {
|
||
return net.Listen("tcp", "127.0.0.1:0")
|
||
}
|