6827fa922f
build / build (push) Successful in 1m55s
首轮试跑暴露三处夹具缺陷(都在 test-only 文件里,产品代码未动): - 目标骨架只核 knowledge_bases:最小夹具 schema 没有 tenants 表,改为按 KB 行数与 产品 ScopeContentEmpty 的十表计数证空; - 运行期索引曾把 GET _mapping 的结果再包一层 mappings,ES 400; 现在直接回放原 mapping 文档(properties 层级与封存时完全一致); - 目标指纹改在夹具分配桶名之后用同一份 env 计算,运行器再把桶名写进 API/worker env。 试跑两次均按 cleanup_empty_target_ui.py 精确清理并读回(resources 全不存在)。
367 lines
15 KiB
Go
367 lines
15 KiB
Go
package v071
|
||
|
||
// 候选页面(空目标首次发布)的夹具准备:只在显式 opt-in 时运行,产物留在运行器登记的
|
||
// 隔离资源里,供父在真实浏览器里点"签发授权 → 启动同步"。
|
||
//
|
||
// 与已审 r39 publish-http 的区别(必须写清,避免把两件事混成一件):
|
||
// - r39 证明的是"正式 publish HTTP + 同请求幂等",它的目标是非空并先建了基线;
|
||
// - 这里只准备一个**空**目标(KB 行存在、十表内容为 0)与一份**独立 fullshape 测试来源**,
|
||
// 来源快照由本夹具装配并封存,明确标 fixture,不冒称原 C01 两份封存的云发布;
|
||
// - 不做任何发布:写入必须由父在页面上触发,交付前目标保持为空。
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/ed25519"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/objectgateway"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
func TestCandidateEmptyTargetFixture(t *testing.T) {
|
||
if os.Getenv("KG_UI_CANDIDATE_PREP") != "1" {
|
||
t.Skip("requires explicit candidate fixture opt-in (KG_UI_CANDIDATE_PREP=1)")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||
defer cancel()
|
||
|
||
out := os.Getenv("KG_UI_PREP_OUT")
|
||
if out == "" || !filepath.IsAbs(out) {
|
||
t.Fatal("KG_UI_PREP_OUT must be an absolute path")
|
||
}
|
||
sourceArchive := os.Getenv("KG_UI_SOURCE_ARCHIVE")
|
||
govDSN := os.Getenv("KG_UI_GOVERNANCE_DSN")
|
||
runtimeIndex := os.Getenv("KG_UI_RUNTIME_INDEX")
|
||
if sourceArchive == "" || govDSN == "" || runtimeIndex == "" {
|
||
t.Fatal("KG_UI_SOURCE_ARCHIVE / KG_UI_GOVERNANCE_DSN / KG_UI_RUNTIME_INDEX are required")
|
||
}
|
||
|
||
// 目标指纹必须与 API/release worker 用同一份 env 推导(运行器给三者同一份 env 文件)。
|
||
// 它只绑数据身份(库/租户/知识库/索引/桶),换端点不改指纹。
|
||
// 桶名由夹具分配(reset 时会生成唯一桶),所以指纹必须用夹具真实桶名计算,
|
||
// 运行器随后把它写进 API/worker 的 env —— 两边因此得到同一个指纹。
|
||
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, archiveRoot: sourceArchive, embeddingDimension: 1024})
|
||
if err := os.Setenv("LIBRARY_PRODUCTION_S3_BUCKET", fx.bucket); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg, err := appconfig.Load(os.Getenv)
|
||
if err != nil {
|
||
t.Fatalf("load product config from candidate env: %v", err)
|
||
}
|
||
fingerprint := cfg.LibraryTargetIdentity()
|
||
if fingerprint == "" {
|
||
t.Fatal("candidate target fingerprint is empty: product config is incomplete")
|
||
}
|
||
// 适配层解析 embedding 模型/向量库需要这两列与两张表(与 r39 fullshape 同一装配)。
|
||
if _, err := fx.pool.Exec(ctx, `ALTER TABLE knowledge_bases ADD COLUMN embedding_model_id text, ADD COLUMN vector_store_id text;
|
||
CREATE TABLE models(id text PRIMARY KEY,name text,parameters jsonb,deleted_at timestamptz);
|
||
CREATE TABLE vector_stores(id text PRIMARY KEY,engine_type text,index_config jsonb,deleted_at timestamptz);
|
||
INSERT INTO models VALUES('candidate-model','text-embedding-v4','{"embedding_parameters":{"dimension":1024}}',NULL);
|
||
INSERT INTO vector_stores VALUES('candidate-vector','elasticsearch_v8','{}',NULL);
|
||
UPDATE knowledge_bases SET embedding_model_id='candidate-model',vector_store_id='candidate-vector';`); err != nil {
|
||
t.Fatalf("seed embedding model/vector store: %v", err)
|
||
}
|
||
|
||
// 空目标的第一条证据:产品自己的范围探针必须说"空",且逐表计数为 0。
|
||
empty, counts, err := ScopeContentEmpty(ctx, fx.pool, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
|
||
if err != nil {
|
||
t.Fatalf("product scope probe on empty target: %v", err)
|
||
}
|
||
if !empty {
|
||
t.Fatalf("candidate target scope is not empty: %v", counts)
|
||
}
|
||
for table, n := range counts {
|
||
if n != 0 {
|
||
t.Fatalf("candidate target %s has %d rows in scope; the target must be empty before delivery", table, n)
|
||
}
|
||
}
|
||
var kbRows int64
|
||
if err := fx.pool.QueryRow(ctx, `SELECT count(*) FROM knowledge_bases WHERE id=$1 AND tenant_id=$2`, wsTargetKB, wsTargetTenant).Scan(&kbRows); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if kbRows != 1 {
|
||
t.Fatalf("empty target still needs its KB skeleton: knowledge_bases rows=%d", kbRows)
|
||
}
|
||
|
||
mapping, err := fx.target(t).Search().MappingSHA256(ctx)
|
||
if err != nil {
|
||
t.Fatalf("read fixture index mapping digest: %v", err)
|
||
}
|
||
|
||
// 独立 fullshape 测试来源:声明行 → 真实 PG schema 落库并读回 → 用读回封存。
|
||
spec := r38LegalSourceSpec(t, fx)
|
||
readback, sourceStoreEvidence := r39SeedSourceStore(t, fx, spec)
|
||
source := r38SealFullShape(t, fx, ds02Spec{rows: readback, vectors: ds02Vectors, objects: ds02Objects}, mapping)
|
||
parts := r38SourcePartCounts(t, fx, source)
|
||
if len(parts) != len(ds02TargetTables) {
|
||
t.Fatalf("sealed source has %d table parts, want %d", len(parts), len(ds02TargetTables))
|
||
}
|
||
|
||
// 运行期索引:夹具索引会在测试结束时被删;用同一份 mapping 另建索引,
|
||
// 让封存里的 index_mapping_sha256 与运行期目标索引仍逐字节一致。
|
||
mappingBody, err := candidateIndexMapping(ctx, fx.searchBase, fx.index)
|
||
if err != nil {
|
||
t.Fatalf("read fixture index mapping: %v", err)
|
||
}
|
||
if err := candidateCreateIndex(ctx, fx.searchBase, runtimeIndex, mappingBody); err != nil {
|
||
t.Fatalf("create candidate runtime index: %v", err)
|
||
}
|
||
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: fx.searchBase, Index: runtimeIndex})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
runtimeMapping, err := exporter.MappingSHA256(ctx)
|
||
if err != nil {
|
||
t.Fatalf("read runtime index mapping digest: %v", err)
|
||
}
|
||
if runtimeMapping != mapping {
|
||
t.Fatalf("runtime index mapping %s != sealed %s", runtimeMapping, mapping)
|
||
}
|
||
|
||
// 一次性治理库:迁移 + 网关 schema,再登记这份 test_sealed 版本。
|
||
gov, err := pgxpool.New(ctx, govDSN)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer gov.Close()
|
||
if err := postgresstore.ApplyMigrations(ctx, gov); err != nil {
|
||
t.Fatalf("apply governance migrations: %v", err)
|
||
}
|
||
if _, err := gov.Exec(ctx, objectgateway.Schema); err != nil {
|
||
t.Fatalf("apply object gateway schema: %v", err)
|
||
}
|
||
store := postgresstore.NewStore(gov).WithLibraryTarget(fingerprint)
|
||
sourceIdentity := governance.LibrarySourceIdentity{Project: os.Getenv("LIBRARY_SOURCE_PROJECT"),
|
||
App: os.Getenv("LIBRARY_SOURCE_APP"), Scope: "internal",
|
||
TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB}
|
||
sourceID, err := sourceIdentity.ID()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
versionID := governance.LibraryVersionID(governance.LibraryVersionTestSealed, "internal", source)
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ID: versionID, Scope: "internal",
|
||
Kind: governance.LibraryVersionTestSealed, SnapshotID: source,
|
||
Evidence: map[string]any{
|
||
"source_id": sourceID, "source_identity": sourceIdentity,
|
||
"capture_job": "candidate-empty-target-fixture",
|
||
"assembly": map[string]any{"kind": "fixture-assembled-legal-fullshape",
|
||
"source_store": "r39_source schema in the isolated candidate business DB",
|
||
"note": "候选页面的独立 fullshape 测试夹具;不是 WeKnora agent 捕获,也不是原 C01 两份封存,不声称 Capture/Export HTTP 已验收",
|
||
"manifest_sha256": source, "source_parts_row_counts": parts},
|
||
},
|
||
CreatedBy: "admin", CreatedAt: time.Now().UTC()}); err != nil {
|
||
t.Fatalf("register candidate test_sealed version: %v", err)
|
||
}
|
||
candidate, found, err := store.LibrarySourceCandidate(ctx, "internal", sourceID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !found || candidate.SnapshotID != source {
|
||
t.Fatalf("registered candidate is not readable back by source_id: found=%v snapshot=%v", found, candidate.SnapshotID)
|
||
}
|
||
|
||
// 网关的只读权威角色:角色名/口令由运行器生成并登记所有权,这里只负责创建与授权。
|
||
// 授权范围与网关真正读取的表一致(jobs / library_*),不给写权限。
|
||
readerRole, readerPassword := os.Getenv("KG_UI_READER_ROLE"), os.Getenv("KG_UI_READER_PASSWORD")
|
||
readerDSNFile := os.Getenv("KG_UI_READER_DSN_FILE")
|
||
if readerRole == "" || readerPassword == "" || !filepath.IsAbs(readerDSNFile) {
|
||
t.Fatal("KG_UI_READER_ROLE / KG_UI_READER_PASSWORD / KG_UI_READER_DSN_FILE are required")
|
||
}
|
||
quotedRole := pgx.Identifier{readerRole}.Sanitize()
|
||
if _, err := gov.Exec(ctx, "CREATE ROLE "+quotedRole+" LOGIN PASSWORD '"+readerPassword+"'"); err != nil {
|
||
t.Fatalf("create gateway reader role: %v", err)
|
||
}
|
||
if _, err := gov.Exec(ctx, "GRANT USAGE ON SCHEMA public TO "+quotedRole); err != nil {
|
||
t.Fatalf("grant reader schema usage: %v", err)
|
||
}
|
||
if _, err := gov.Exec(ctx, "GRANT SELECT ON jobs,library_jobs,library_operations,library_attempts,library_versions TO "+quotedRole); err != nil {
|
||
t.Fatalf("grant reader table select: %v", err)
|
||
}
|
||
readerURL, err := url.Parse(govDSN)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
readerURL.User = url.UserPassword(readerRole, readerPassword)
|
||
if err := os.WriteFile(readerDSNFile, []byte(readerURL.String()), 0o600); err != nil {
|
||
t.Fatalf("write reader DSN file: %v", err)
|
||
}
|
||
|
||
report := map[string]any{
|
||
"generated_at": time.Now().UTC().Format(time.RFC3339Nano),
|
||
"kind": "candidate-empty-target-fixture",
|
||
"target_fingerprint": fingerprint,
|
||
"target": map[string]any{"tenant_id": wsTargetTenant, "knowledge_base_id": wsTargetKB,
|
||
"database": os.Getenv("KG_LIBRARY_TEST_DATABASE_URL") != "", "empty_probe_counts": counts,
|
||
"knowledge_bases_rows": kbRows,
|
||
"note": "KB 骨架存在、十表内容为 0:这是首次发布的空目标,不是'没有目标'"},
|
||
"runtime_index": runtimeIndex,
|
||
"mapping_sha256": mapping,
|
||
"runtime_mapping_sha256": runtimeMapping,
|
||
"bucket": fx.bucket,
|
||
"s3_endpoint": os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT"),
|
||
"source": map[string]any{"snapshot": source, "source_id": sourceID,
|
||
"project": sourceIdentity.Project, "app": sourceIdentity.App,
|
||
"tenant_id": sourceIdentity.TenantID, "knowledge_base_id": sourceIdentity.KnowledgeBaseID,
|
||
"parts": parts, "assembly": "fixture-assembled-legal-fullshape",
|
||
"source_store": sourceStoreEvidence},
|
||
"version": map[string]any{"id": versionID, "kind": string(governance.LibraryVersionTestSealed),
|
||
"snapshot": source, "created_by": "admin"},
|
||
}
|
||
raw, err := json.MarshalIndent(report, "", " ")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.MkdirAll(filepath.Dir(out), 0o700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(out, raw, 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Logf("candidate fixture ready: snapshot=%s source_id=%s version=%s", source, sourceID, versionID)
|
||
}
|
||
|
||
func TestCandidateAttestation(t *testing.T) {
|
||
if os.Getenv("KG_UI_ATTESTATION") != "1" {
|
||
t.Skip("requires explicit candidate attestation opt-in (KG_UI_ATTESTATION=1)")
|
||
}
|
||
// 网关每次请求都重读审计文件,所以刷新只需原子替换该文件:不用重启网关,
|
||
// 也不会与正在进行的发布互相干扰。有效期上限是 1 小时(objectgateway.AuditVerifier)。
|
||
seedRaw, err := hex.DecodeString(os.Getenv("KG_UI_ATTESTATION_SEED"))
|
||
if err != nil || len(seedRaw) != ed25519.SeedSize {
|
||
t.Fatal("KG_UI_ATTESTATION_SEED must be a 32-byte hex seed")
|
||
}
|
||
path := os.Getenv("KG_UI_AUDIT_FILE")
|
||
pubPath := os.Getenv("KG_UI_ATTESTATION_PUB_FILE")
|
||
if !filepath.IsAbs(path) || !filepath.IsAbs(pubPath) {
|
||
t.Fatal("KG_UI_AUDIT_FILE and KG_UI_ATTESTATION_PUB_FILE must be absolute paths")
|
||
}
|
||
evidence := map[string]string{}
|
||
for key, env := range map[string]string{
|
||
"writer_inventory_sha256": "KG_UI_EVIDENCE_WRITER",
|
||
"policy_evidence_sha256": "KG_UI_EVIDENCE_POLICY",
|
||
"denial_evidence_sha256": "KG_UI_EVIDENCE_DENIAL",
|
||
"drain_evidence_sha256": "KG_UI_EVIDENCE_DRAIN",
|
||
} {
|
||
value := os.Getenv(env)
|
||
if len(value) != 64 {
|
||
t.Fatalf("%s must be a 64-hex digest of a real evidence file", env)
|
||
}
|
||
evidence[key] = value
|
||
}
|
||
scope := objectgateway.Digest([]byte(os.Getenv("OBJECT_GATEWAY_PROVIDER") + "\x00" +
|
||
os.Getenv("OBJECT_GATEWAY_ENDPOINT") + "\x00" + os.Getenv("OBJECT_GATEWAY_BUCKET")))
|
||
now := time.Now().UTC()
|
||
key := ed25519.NewKeyFromSeed(seedRaw)
|
||
payload, err := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope,
|
||
GatewayID: os.Getenv("OBJECT_GATEWAY_ID"), IssuedAt: now.Add(-2 * time.Second), ExpiresAt: now.Add(45 * time.Minute),
|
||
WriterInventory: evidence["writer_inventory_sha256"], PolicyEvidence: evidence["policy_evidence_sha256"],
|
||
DenialEvidence: evidence["denial_evidence_sha256"], DrainEvidence: evidence["drain_evidence_sha256"]})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
signed, err := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(key, payload)})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
temp := path + ".new"
|
||
if err := os.WriteFile(temp, signed, 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.Chmod(temp, 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.Rename(temp, path); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
public := base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey))
|
||
if err := os.WriteFile(pubPath, []byte(public), 0o600); err != nil {
|
||
t.Fatalf("write gateway audit public key: %v", err)
|
||
}
|
||
t.Logf("candidate attestation refreshed: scope=%s gateway=%s expires=%s", scope, os.Getenv("OBJECT_GATEWAY_ID"), now.Add(45*time.Minute).Format(time.RFC3339))
|
||
}
|
||
|
||
func candidateIndexMapping(ctx context.Context, base, index string) (map[string]any, error) {
|
||
response, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/"+index+"/_mapping", nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
client := &http.Client{Timeout: 20 * time.Second}
|
||
res, err := client.Do(response)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer res.Body.Close()
|
||
if res.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("read mapping %s: HTTP %d", index, res.StatusCode)
|
||
}
|
||
raw, err := io.ReadAll(io.LimitReader(res.Body, 4<<20))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var payload map[string]json.RawMessage
|
||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||
return nil, err
|
||
}
|
||
mappings, ok := payload[index]
|
||
if !ok {
|
||
return nil, fmt.Errorf("mapping response does not contain index %s", index)
|
||
}
|
||
var value map[string]any
|
||
if err := json.Unmarshal(mappings, &value); err != nil {
|
||
return nil, err
|
||
}
|
||
return value, nil
|
||
}
|
||
|
||
func candidateCreateIndex(ctx context.Context, base, index string, body map[string]any) error {
|
||
client := &http.Client{Timeout: 20 * time.Second}
|
||
check, err := http.NewRequestWithContext(ctx, http.MethodHead, base+"/"+index, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if res, err := client.Do(check); err == nil {
|
||
res.Body.Close()
|
||
if res.StatusCode == http.StatusOK {
|
||
return fmt.Errorf("runtime index %s already exists: refusing to reuse an unknown index", index)
|
||
}
|
||
}
|
||
payload, err := json.Marshal(body)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, base+"/"+index, bytes.NewReader(payload))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
res, err := client.Do(request)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer res.Body.Close()
|
||
if res.StatusCode != http.StatusOK {
|
||
detail, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||
return fmt.Errorf("create runtime index %s: HTTP %d %s", index, res.StatusCode, detail)
|
||
}
|
||
return nil
|
||
}
|