Files
knowledge-governance/internal/releaseadapter/v071/library_writeset_integration_test.go
T
姚章浩 943b617b1f
build / build (push) Successful in 1m28s
test(isolated): 支持保留隔离演练现场供本机 UI/API 验收
KG_R8_KEEP_SCENE=1 时不删索引与对象桶、落网关审计公钥与派生令牌(0600),
默认行为不变(断言后精确清理)。仅测试侧改动,产品代码与数据契约不动。
2026-11-01 18:26:07 +08:00

779 lines
32 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
//
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
//
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
type writeSetFixture struct {
embeddingDimension int
ctx context.Context
pool *pgxpool.Pool
client *http.Client
searchBase string
index string
archiveRoot string
objects *flakyStore
bucket string
}
const (
wsTargetTenant = 10001
wsTargetKB = "kb-target"
wsOtherKB = "kb-other"
wsSourceTenant = 10002
wsSourceKB = "kb-source"
wsForeignTnt = 20000
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
wsHandleOther = "cccccccccccccccccccccc"
wsHandleOrphan = "dddddddddddddddddddddd"
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
wsHandleForeign = "ffffffffffffffffffffff"
)
// A per-run bucket isolates object writes even when the S3 service is shared.
var wsBucket = func() string {
if bucket := os.Getenv("KG_LIBRARY_TEST_S3_BUCKET"); strings.HasPrefix(bucket, "kg-library-test-") {
return bucket
}
return "kg-wire-test-bucket"
}()
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
}
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
// Open/Stat/Copy 全部透传。
type flakyStore struct {
s3 *S3ObjectStore
failPut atomic.Bool
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
failPath string
puts atomic.Int64
}
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
return errors.New("注入的对象写入失败")
}
s.puts.Add(1)
return s.s3.Put(ctx, provider, path, reader, size)
}
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
return s.s3.Stat(ctx, provider, path)
}
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
return s.s3.Copy(ctx, provider, source, target)
}
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
return s.s3.Open(ctx, provider, path)
}
func openWriteSetFixture(t *testing.T) *writeSetFixture {
t.Helper()
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
}
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
type writeSetFixtureOptions struct {
embeddingDimension int
reset bool
archiveRoot string
index string
}
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
if options.embeddingDimension == 0 {
options.embeddingDimension = 2
}
t.Helper()
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Skip("requires isolated PG, ES and object storage opt-in")
}
cfg, err := pgxpool.ParseConfig(dsn)
if options.reset {
previousBucket := wsBucket
var namespace [8]byte
if _, err := rand.Read(namespace[:]); err != nil {
t.Fatal(err)
}
wsBucket = "kg-library-test-" + hex.EncodeToString(namespace[:])
t.Cleanup(func() { wsBucket = previousBucket })
t.Setenv("KG_LIBRARY_TEST_S3_BUCKET", wsBucket)
t.Logf("isolated object namespace=%s", wsBucket)
}
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
t.Fatal("isolated database required")
}
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
t.Cleanup(cancel)
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatalf("open isolated PG: %v", err)
}
t.Cleanup(pool.Close)
if options.reset {
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
t.Fatalf("reset isolated schema: %v", err)
}
}
schemaStatements := []string{
libraryTestSchema,
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
// 免得"目标非空"被别的用例的行弄成不确定。
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
}
if options.reset {
for _, statement := range schemaStatements {
if _, err := pool.Exec(ctx, statement); err != nil {
t.Fatalf("fixture schema: %v", err)
}
}
}
token := make([]byte, 8)
if _, err := rand.Read(token); err != nil {
t.Fatal(err)
}
index := "kg_library_test_" + hex.EncodeToString(token)
if options.index != "" {
index = options.index
}
// Register each disposable namespace before its first ES/object mutation.
r10Save(t, "namespace", map[string]string{"bucket": wsBucket, "index": index})
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
call := func(method, path, body string, want int) {
t.Helper()
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := client.Do(request)
if err != nil {
t.Fatalf("fixture HTTP: %v", err)
}
defer response.Body.Close()
if response.StatusCode != want {
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
}
}
if options.reset {
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":`+strconv.Itoa(options.embeddingDimension)+`}}}}`, 200)
}
if options.reset {
t.Cleanup(func() {
// 保留 UI 验收现场时不删索引:父要能对着真实目标点恢复。
if os.Getenv("KG_R8_KEEP_SCENE") == "1" {
return
}
call("DELETE", "/"+index, "", 200)
call("HEAD", "/"+index, "", 404)
})
}
root := options.archiveRoot
if root == "" {
root = t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
}
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
if err != nil {
t.Fatalf("object store: %v", err)
}
if err := store.EnsureBucket(ctx); err != nil {
t.Fatalf("ensure bucket: %v", err)
}
if options.reset {
enableIsolatedObjectVersions(t, ctx, store)
}
return &writeSetFixture{embeddingDimension: options.embeddingDimension, ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
}
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
func seedTargetData(t *testing.T, fx *writeSetFixture) {
t.Helper()
ctx := fx.ctx
locator := func(backend string, tenant uint64, id string) string {
return wsLocator(backend, "minio", wsBucket, tenant, id)
}
for _, object := range []struct {
tenant uint64
id string
}{
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
{10001, "res-orphan"}, {20000, "res-foreign"},
} {
path := locator("backend-target", object.tenant, object.id)
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
t.Fatalf("seed object %s: %v", object.id, err)
}
}
statements := []string{
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
// 同租户另一个知识库:数据与资源都必须原样保留。
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
// 共享资源:同时被目标知识库与另一个知识库绑定。
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
// 另一个租户的行。
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
}
for _, statement := range statements {
if _, err := fx.pool.Exec(ctx, statement); err != nil {
t.Fatalf("seed target: %v\n%s", err, statement)
}
}
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
}
func (fx *writeSetFixture) embeddingJSON() []byte {
n := fx.embeddingDimension
if n == 0 {
n = 2
}
v := make([]float64, n)
v[0] = 0.1
v[1] = 0.2
b, _ := json.Marshal(v)
return b
}
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
t.Helper()
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":%s}`,
tenant, kb, knowledge, id, id, content, fx.embeddingJSON())
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := fx.client.Do(request)
if err != nil {
t.Fatalf("index doc: %v", err)
}
defer response.Body.Close()
if response.StatusCode != 201 && response.StatusCode != 200 {
t.Fatalf("index doc %s => %d", id, response.StatusCode)
}
}
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
t.Helper()
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
}
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
t.Helper()
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
encode := func(value any) []byte {
encoded, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
return encoded
}
parts := map[string][]map[string]any{
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
"knowledge_tags": {},
"knowledge_tag_relations": {},
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
"storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
"size": int64(8)}},
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
"wiki_folders": {},
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
"wiki_page_issues": {},
"wiki_log_entries": {},
}
if tweak != nil {
tweak(parts)
}
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
manifestParts := make([]librarysnapshot.Part, 0, 14)
names := make([]string, 0, len(parts))
for name := range parts {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
manifestParts = append(manifestParts, part)
}
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
if err != nil {
t.Fatal(err)
}
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
manifestParts = append(manifestParts, resourcePart)
extraNames := make([]string, 0, len(extraObjects))
for name := range extraObjects {
extraNames = append(extraNames, name)
}
sort.Strings(extraNames)
for _, name := range extraNames {
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = "resource", name, 1
manifestParts = append(manifestParts, part)
}
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
// 校验会把"写入正确"判成索引缺文档。
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
for _, chunk := range parts["chunks"] {
id, _ := chunk["id"].(string)
knowledge, _ := chunk["knowledge_id"].(string)
content, _ := chunk["content"].(string)
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
"chunk_id": id, "source_id": id, "content": content, "embedding": json.RawMessage(fx.embeddingJSON())}})
}
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
if err != nil {
t.Fatal(err)
}
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
manifestParts = append(manifestParts, indexPart)
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
CreatedAt: time.Now().UTC(), Parts: manifestParts,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
id, err := archive.Seal(manifest)
if err != nil {
t.Fatalf("seal source snapshot: %v", err)
}
return id
}
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
t.Helper()
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
if err != nil {
t.Fatalf("target: %v", err)
}
t.Cleanup(target.Close)
return target
}
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
t.Helper()
mapping, err := target.Search().MappingSHA256(fx.ctx)
if err != nil {
t.Fatal(err)
}
backups, err := NewProductionBackupStore(ProductionBackupConfig{
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
})
if err != nil {
t.Fatalf("backup store: %v", err)
}
return backups
}
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
t.Helper()
dump := map[string][]string{}
for _, table := range productionLibraryTables {
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
if err != nil {
t.Fatalf("dump %s: %v", table, err)
}
values := make([]string, 0, 32)
for rows.Next() {
var raw string
if err := rows.Scan(&raw); err != nil {
rows.Close()
t.Fatalf("dump scan %s: %v", table, err)
}
values = append(values, raw)
}
rows.Close()
if err := rows.Err(); err != nil {
t.Fatalf("dump %s: %v", table, err)
}
dump[table] = values
}
return dump
}
func diffUniverse(before, after map[string][]string) string {
tables := make([]string, 0, len(before))
for table := range before {
tables = append(tables, table)
}
sort.Strings(tables)
for _, table := range tables {
if len(before[table]) != len(after[table]) {
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
}
for index := range before[table] {
if before[table][index] != after[table][index] {
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
}
}
}
return ""
}
func requestFor(snapshotID string) libraryrelease.Request {
now := time.Now().UTC()
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
ParamsDigest: "digest-writeset", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
}
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageResources {
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
}
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
outcome.Backup.WriteSet.Digest)
after := universeDump(t, fx.pool)
if diff := diffUniverse(before, after); diff != "" {
t.Fatalf("回滚后目标数据被改动:%s", diff)
}
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
var orphan string
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
}
var sourceScope int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
t.Fatal(err)
}
if sourceScope != 0 {
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
}
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatalf("本次同步新增的资源行未被清理")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
createdResources := ledger.CreatedIDs("resources")
if len(createdResources) != 1 || createdResources[0] != "res-src" {
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
}
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
created := map[string]bool{}
for _, id := range ledger.CreatedIDs("resources") {
created[id] = true
}
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
if created[id] {
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
}
}
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
}
if ledger.UniverseDigest() == "" {
t.Fatal("写集合必须带整实例内容证据")
}
}
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
// 更不能把校验降级成"跳过整实例比对"。
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
before := universeDump(t, fx.pool)
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
}
}
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
// 不能 ON CONFLICT 改写别人的行。
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
// 另一个租户里已经有一条同 ID 的资源行。
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
t.Fatal(err)
}
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
}
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
}
t.Logf("conflict rejected before write: %s", outcome.Error)
}
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
// 本次新增的资源也清掉,目标原有数据逐行不变。
func TestLibraryDuplicateSlugRejectedBeforeWriteIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
// 重复 slug 保留为写前拒绝负例;晚期故障由独立用例注入。
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
parts["wiki_pages"] = []map[string]any{
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
}
})
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "failed" || !strings.Contains(outcome.Error, "source duplicates natural key") {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
}
if fx.objects.puts.Load() != 0 {
t.Fatal("duplicate slug reached object write")
}
}
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), contractFailWikiTarget{target}, fx.backups(t, target), nil)
if outcome.State != "rolled_back" || outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("late failure: %+v", outcome)
}
if fx.objects.puts.Load() == 0 {
t.Fatal("late failure never reached real object write")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatal(diff)
}
observed, err := fx.objects.ObserveObject(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src"))
if err != nil || observed.Exists {
t.Fatalf("created object residue: %+v %v", observed, err)
}
}
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
t.Fatalf("outcome = %+v", outcome)
}
digest := outcome.Backup.WriteSet.Digest
path := writeSetPath(fx.archiveRoot, digest)
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
original, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
if bytes.Equal(tampered, original) {
t.Fatalf("篡改夹具没生效:%s", original)
}
if err := os.Chmod(path, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, tampered, 0600); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合被篡改后取回必须失败")
}
// 内存里带着被篡改的记录去恢复,同样必须失败。
bad := *outcome.Backup.WriteSet
bad.TenantID = 10002
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
t.Fatal("写集合内容与摘要不符时恢复必须失败")
}
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
if err := os.WriteFile(path, original, 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合记录被删除后取回必须失败")
}
t.Logf("tampered/removed ledger refused, digest=%s", digest)
}