Files
knowledge-governance/internal/releaseadapter/v071/candidate_ui_fixture_test.go
T
姚章浩 6827fa922f
build / build (push) Successful in 1m55s
fix(candidate): 夹具用产品探针证空并复用真实索引 mapping
首轮试跑暴露三处夹具缺陷(都在 test-only 文件里,产品代码未动):
- 目标骨架只核 knowledge_bases:最小夹具 schema 没有 tenants 表,改为按 KB 行数与
  产品 ScopeContentEmpty 的十表计数证空;
- 运行期索引曾把 GET _mapping 的结果再包一层 mappings,ES 400;
  现在直接回放原 mapping 文档(properties 层级与封存时完全一致);
- 目标指纹改在夹具分配桶名之后用同一份 env 计算,运行器再把桶名写进 API/worker env。

试跑两次均按 cleanup_empty_target_ui.py 精确清理并读回(resources 全不存在)。
2026-11-02 11:11:08 +08:00

367 lines
15 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
// 候选页面(空目标首次发布)的夹具准备:只在显式 opt-in 时运行,产物留在运行器登记的
// 隔离资源里,供父在真实浏览器里点"签发授权 → 启动同步"。
//
// 与已审 r39 publish-http 的区别(必须写清,避免把两件事混成一件):
// - r39 证明的是"正式 publish HTTP + 同请求幂等",它的目标是非空并先建了基线;
// - 这里只准备一个**空**目标(KB 行存在、十表内容为 0)与一份**独立 fullshape 测试来源**,
// 来源快照由本夹具装配并封存,明确标 fixture,不冒称原 C01 两份封存的云发布;
// - 不做任何发布:写入必须由父在页面上触发,交付前目标保持为空。
import (
"bytes"
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"testing"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/objectgateway"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func TestCandidateEmptyTargetFixture(t *testing.T) {
if os.Getenv("KG_UI_CANDIDATE_PREP") != "1" {
t.Skip("requires explicit candidate fixture opt-in (KG_UI_CANDIDATE_PREP=1)")
}
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
defer cancel()
out := os.Getenv("KG_UI_PREP_OUT")
if out == "" || !filepath.IsAbs(out) {
t.Fatal("KG_UI_PREP_OUT must be an absolute path")
}
sourceArchive := os.Getenv("KG_UI_SOURCE_ARCHIVE")
govDSN := os.Getenv("KG_UI_GOVERNANCE_DSN")
runtimeIndex := os.Getenv("KG_UI_RUNTIME_INDEX")
if sourceArchive == "" || govDSN == "" || runtimeIndex == "" {
t.Fatal("KG_UI_SOURCE_ARCHIVE / KG_UI_GOVERNANCE_DSN / KG_UI_RUNTIME_INDEX are required")
}
// 目标指纹必须与 API/release worker 用同一份 env 推导(运行器给三者同一份 env 文件)。
// 它只绑数据身份(库/租户/知识库/索引/桶),换端点不改指纹。
// 桶名由夹具分配(reset 时会生成唯一桶),所以指纹必须用夹具真实桶名计算,
// 运行器随后把它写进 API/worker 的 env —— 两边因此得到同一个指纹。
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, archiveRoot: sourceArchive, embeddingDimension: 1024})
if err := os.Setenv("LIBRARY_PRODUCTION_S3_BUCKET", fx.bucket); err != nil {
t.Fatal(err)
}
cfg, err := appconfig.Load(os.Getenv)
if err != nil {
t.Fatalf("load product config from candidate env: %v", err)
}
fingerprint := cfg.LibraryTargetIdentity()
if fingerprint == "" {
t.Fatal("candidate target fingerprint is empty: product config is incomplete")
}
// 适配层解析 embedding 模型/向量库需要这两列与两张表(与 r39 fullshape 同一装配)。
if _, err := fx.pool.Exec(ctx, `ALTER TABLE knowledge_bases ADD COLUMN embedding_model_id text, ADD COLUMN vector_store_id text;
CREATE TABLE models(id text PRIMARY KEY,name text,parameters jsonb,deleted_at timestamptz);
CREATE TABLE vector_stores(id text PRIMARY KEY,engine_type text,index_config jsonb,deleted_at timestamptz);
INSERT INTO models VALUES('candidate-model','text-embedding-v4','{"embedding_parameters":{"dimension":1024}}',NULL);
INSERT INTO vector_stores VALUES('candidate-vector','elasticsearch_v8','{}',NULL);
UPDATE knowledge_bases SET embedding_model_id='candidate-model',vector_store_id='candidate-vector';`); err != nil {
t.Fatalf("seed embedding model/vector store: %v", err)
}
// 空目标的第一条证据:产品自己的范围探针必须说"空",且逐表计数为 0。
empty, counts, err := ScopeContentEmpty(ctx, fx.pool, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
if err != nil {
t.Fatalf("product scope probe on empty target: %v", err)
}
if !empty {
t.Fatalf("candidate target scope is not empty: %v", counts)
}
for table, n := range counts {
if n != 0 {
t.Fatalf("candidate target %s has %d rows in scope; the target must be empty before delivery", table, n)
}
}
var kbRows int64
if err := fx.pool.QueryRow(ctx, `SELECT count(*) FROM knowledge_bases WHERE id=$1 AND tenant_id=$2`, wsTargetKB, wsTargetTenant).Scan(&kbRows); err != nil {
t.Fatal(err)
}
if kbRows != 1 {
t.Fatalf("empty target still needs its KB skeleton: knowledge_bases rows=%d", kbRows)
}
mapping, err := fx.target(t).Search().MappingSHA256(ctx)
if err != nil {
t.Fatalf("read fixture index mapping digest: %v", err)
}
// 独立 fullshape 测试来源:声明行 → 真实 PG schema 落库并读回 → 用读回封存。
spec := r38LegalSourceSpec(t, fx)
readback, sourceStoreEvidence := r39SeedSourceStore(t, fx, spec)
source := r38SealFullShape(t, fx, ds02Spec{rows: readback, vectors: ds02Vectors, objects: ds02Objects}, mapping)
parts := r38SourcePartCounts(t, fx, source)
if len(parts) != len(ds02TargetTables) {
t.Fatalf("sealed source has %d table parts, want %d", len(parts), len(ds02TargetTables))
}
// 运行期索引:夹具索引会在测试结束时被删;用同一份 mapping 另建索引,
// 让封存里的 index_mapping_sha256 与运行期目标索引仍逐字节一致。
mappingBody, err := candidateIndexMapping(ctx, fx.searchBase, fx.index)
if err != nil {
t.Fatalf("read fixture index mapping: %v", err)
}
if err := candidateCreateIndex(ctx, fx.searchBase, runtimeIndex, mappingBody); err != nil {
t.Fatalf("create candidate runtime index: %v", err)
}
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: fx.searchBase, Index: runtimeIndex})
if err != nil {
t.Fatal(err)
}
runtimeMapping, err := exporter.MappingSHA256(ctx)
if err != nil {
t.Fatalf("read runtime index mapping digest: %v", err)
}
if runtimeMapping != mapping {
t.Fatalf("runtime index mapping %s != sealed %s", runtimeMapping, mapping)
}
// 一次性治理库:迁移 + 网关 schema,再登记这份 test_sealed 版本。
gov, err := pgxpool.New(ctx, govDSN)
if err != nil {
t.Fatal(err)
}
defer gov.Close()
if err := postgresstore.ApplyMigrations(ctx, gov); err != nil {
t.Fatalf("apply governance migrations: %v", err)
}
if _, err := gov.Exec(ctx, objectgateway.Schema); err != nil {
t.Fatalf("apply object gateway schema: %v", err)
}
store := postgresstore.NewStore(gov).WithLibraryTarget(fingerprint)
sourceIdentity := governance.LibrarySourceIdentity{Project: os.Getenv("LIBRARY_SOURCE_PROJECT"),
App: os.Getenv("LIBRARY_SOURCE_APP"), Scope: "internal",
TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB}
sourceID, err := sourceIdentity.ID()
if err != nil {
t.Fatal(err)
}
versionID := governance.LibraryVersionID(governance.LibraryVersionTestSealed, "internal", source)
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ID: versionID, Scope: "internal",
Kind: governance.LibraryVersionTestSealed, SnapshotID: source,
Evidence: map[string]any{
"source_id": sourceID, "source_identity": sourceIdentity,
"capture_job": "candidate-empty-target-fixture",
"assembly": map[string]any{"kind": "fixture-assembled-legal-fullshape",
"source_store": "r39_source schema in the isolated candidate business DB",
"note": "候选页面的独立 fullshape 测试夹具;不是 WeKnora agent 捕获,也不是原 C01 两份封存,不声称 Capture/Export HTTP 已验收",
"manifest_sha256": source, "source_parts_row_counts": parts},
},
CreatedBy: "admin", CreatedAt: time.Now().UTC()}); err != nil {
t.Fatalf("register candidate test_sealed version: %v", err)
}
candidate, found, err := store.LibrarySourceCandidate(ctx, "internal", sourceID)
if err != nil {
t.Fatal(err)
}
if !found || candidate.SnapshotID != source {
t.Fatalf("registered candidate is not readable back by source_id: found=%v snapshot=%v", found, candidate.SnapshotID)
}
// 网关的只读权威角色:角色名/口令由运行器生成并登记所有权,这里只负责创建与授权。
// 授权范围与网关真正读取的表一致(jobs / library_*),不给写权限。
readerRole, readerPassword := os.Getenv("KG_UI_READER_ROLE"), os.Getenv("KG_UI_READER_PASSWORD")
readerDSNFile := os.Getenv("KG_UI_READER_DSN_FILE")
if readerRole == "" || readerPassword == "" || !filepath.IsAbs(readerDSNFile) {
t.Fatal("KG_UI_READER_ROLE / KG_UI_READER_PASSWORD / KG_UI_READER_DSN_FILE are required")
}
quotedRole := pgx.Identifier{readerRole}.Sanitize()
if _, err := gov.Exec(ctx, "CREATE ROLE "+quotedRole+" LOGIN PASSWORD '"+readerPassword+"'"); err != nil {
t.Fatalf("create gateway reader role: %v", err)
}
if _, err := gov.Exec(ctx, "GRANT USAGE ON SCHEMA public TO "+quotedRole); err != nil {
t.Fatalf("grant reader schema usage: %v", err)
}
if _, err := gov.Exec(ctx, "GRANT SELECT ON jobs,library_jobs,library_operations,library_attempts,library_versions TO "+quotedRole); err != nil {
t.Fatalf("grant reader table select: %v", err)
}
readerURL, err := url.Parse(govDSN)
if err != nil {
t.Fatal(err)
}
readerURL.User = url.UserPassword(readerRole, readerPassword)
if err := os.WriteFile(readerDSNFile, []byte(readerURL.String()), 0o600); err != nil {
t.Fatalf("write reader DSN file: %v", err)
}
report := map[string]any{
"generated_at": time.Now().UTC().Format(time.RFC3339Nano),
"kind": "candidate-empty-target-fixture",
"target_fingerprint": fingerprint,
"target": map[string]any{"tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB,
"database": os.Getenv("KG_LIBRARY_TEST_DATABASE_URL") != "", "empty_probe_counts": counts,
"knowledge_bases_rows": kbRows,
"note": "KB 骨架存在、十表内容为 0:这是首次发布的空目标,不是'没有目标'"},
"runtime_index": runtimeIndex,
"mapping_sha256": mapping,
"runtime_mapping_sha256": runtimeMapping,
"bucket": fx.bucket,
"s3_endpoint": os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT"),
"source": map[string]any{"snapshot": source, "source_id": sourceID,
"project": sourceIdentity.Project, "app": sourceIdentity.App,
"tenant_id": sourceIdentity.TenantID, "knowledge_base_id": sourceIdentity.KnowledgeBaseID,
"parts": parts, "assembly": "fixture-assembled-legal-fullshape",
"source_store": sourceStoreEvidence},
"version": map[string]any{"id": versionID, "kind": string(governance.LibraryVersionTestSealed),
"snapshot": source, "created_by": "admin"},
}
raw, err := json.MarshalIndent(report, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(out), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(out, raw, 0o600); err != nil {
t.Fatal(err)
}
t.Logf("candidate fixture ready: snapshot=%s source_id=%s version=%s", source, sourceID, versionID)
}
func TestCandidateAttestation(t *testing.T) {
if os.Getenv("KG_UI_ATTESTATION") != "1" {
t.Skip("requires explicit candidate attestation opt-in (KG_UI_ATTESTATION=1)")
}
// 网关每次请求都重读审计文件,所以刷新只需原子替换该文件:不用重启网关,
// 也不会与正在进行的发布互相干扰。有效期上限是 1 小时(objectgateway.AuditVerifier)。
seedRaw, err := hex.DecodeString(os.Getenv("KG_UI_ATTESTATION_SEED"))
if err != nil || len(seedRaw) != ed25519.SeedSize {
t.Fatal("KG_UI_ATTESTATION_SEED must be a 32-byte hex seed")
}
path := os.Getenv("KG_UI_AUDIT_FILE")
pubPath := os.Getenv("KG_UI_ATTESTATION_PUB_FILE")
if !filepath.IsAbs(path) || !filepath.IsAbs(pubPath) {
t.Fatal("KG_UI_AUDIT_FILE and KG_UI_ATTESTATION_PUB_FILE must be absolute paths")
}
evidence := map[string]string{}
for key, env := range map[string]string{
"writer_inventory_sha256": "KG_UI_EVIDENCE_WRITER",
"policy_evidence_sha256": "KG_UI_EVIDENCE_POLICY",
"denial_evidence_sha256": "KG_UI_EVIDENCE_DENIAL",
"drain_evidence_sha256": "KG_UI_EVIDENCE_DRAIN",
} {
value := os.Getenv(env)
if len(value) != 64 {
t.Fatalf("%s must be a 64-hex digest of a real evidence file", env)
}
evidence[key] = value
}
scope := objectgateway.Digest([]byte(os.Getenv("OBJECT_GATEWAY_PROVIDER") + "\x00" +
os.Getenv("OBJECT_GATEWAY_ENDPOINT") + "\x00" + os.Getenv("OBJECT_GATEWAY_BUCKET")))
now := time.Now().UTC()
key := ed25519.NewKeyFromSeed(seedRaw)
payload, err := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope,
GatewayID: os.Getenv("OBJECT_GATEWAY_ID"), IssuedAt: now.Add(-2 * time.Second), ExpiresAt: now.Add(45 * time.Minute),
WriterInventory: evidence["writer_inventory_sha256"], PolicyEvidence: evidence["policy_evidence_sha256"],
DenialEvidence: evidence["denial_evidence_sha256"], DrainEvidence: evidence["drain_evidence_sha256"]})
if err != nil {
t.Fatal(err)
}
signed, err := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(key, payload)})
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
temp := path + ".new"
if err := os.WriteFile(temp, signed, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chmod(temp, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Rename(temp, path); err != nil {
t.Fatal(err)
}
public := base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey))
if err := os.WriteFile(pubPath, []byte(public), 0o600); err != nil {
t.Fatalf("write gateway audit public key: %v", err)
}
t.Logf("candidate attestation refreshed: scope=%s gateway=%s expires=%s", scope, os.Getenv("OBJECT_GATEWAY_ID"), now.Add(45*time.Minute).Format(time.RFC3339))
}
func candidateIndexMapping(ctx context.Context, base, index string) (map[string]any, error) {
response, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/"+index+"/_mapping", nil)
if err != nil {
return nil, err
}
client := &http.Client{Timeout: 20 * time.Second}
res, err := client.Do(response)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("read mapping %s: HTTP %d", index, res.StatusCode)
}
raw, err := io.ReadAll(io.LimitReader(res.Body, 4<<20))
if err != nil {
return nil, err
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(raw, &payload); err != nil {
return nil, err
}
mappings, ok := payload[index]
if !ok {
return nil, fmt.Errorf("mapping response does not contain index %s", index)
}
var value map[string]any
if err := json.Unmarshal(mappings, &value); err != nil {
return nil, err
}
return value, nil
}
func candidateCreateIndex(ctx context.Context, base, index string, body map[string]any) error {
client := &http.Client{Timeout: 20 * time.Second}
check, err := http.NewRequestWithContext(ctx, http.MethodHead, base+"/"+index, nil)
if err != nil {
return err
}
if res, err := client.Do(check); err == nil {
res.Body.Close()
if res.StatusCode == http.StatusOK {
return fmt.Errorf("runtime index %s already exists: refusing to reuse an unknown index", index)
}
}
payload, err := json.Marshal(body)
if err != nil {
return err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPut, base+"/"+index, bytes.NewReader(payload))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
res, err := client.Do(request)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
detail, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
return fmt.Errorf("create runtime index %s: HTTP %d %s", index, res.StatusCode, detail)
}
return nil
}