Files
knowledge-governance/internal/releaseadapter/v071/library_writeset_integration_test.go
T
姚章浩 0252492e53 fix(library): 恢复保留下来的资源凭可审计证据复用
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时,
resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次
同步在写第一笔之前被拒。

“目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足
它。改成唯一放行路径——可审计复用(v071/library_reuse.go):
1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份,
   且同一资源行在来源里内容一致;
2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致;
3. 该资源的对象字节与本次快照归档摘要一致。

三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里
记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进
写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续
拒绝,且拒绝发生在写入之前。

为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的
资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。

验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、
无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建);
v071 整包与全仓 go test 通过;make verify 15 项全绿。
2026-10-30 01:59:32 +08:00

720 lines
31 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
//
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
//
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
type writeSetFixture struct {
ctx context.Context
pool *pgxpool.Pool
client *http.Client
searchBase string
index string
archiveRoot string
objects *flakyStore
bucket string
}
const (
wsTargetTenant = 10001
wsTargetKB = "kb-target"
wsOtherKB = "kb-other"
wsSourceTenant = 10002
wsSourceKB = "kb-source"
wsForeignTnt = 20000
wsBucket = "kg-wire-test-bucket"
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
wsHandleOther = "cccccccccccccccccccccc"
wsHandleOrphan = "dddddddddddddddddddddd"
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
wsHandleForeign = "ffffffffffffffffffffff"
)
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
}
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
// Open/Stat/Copy 全部透传。
type flakyStore struct {
s3 *S3ObjectStore
failPut atomic.Bool
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
failPath string
puts atomic.Int64
}
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
return errors.New("注入的对象写入失败")
}
s.puts.Add(1)
return s.s3.Put(ctx, provider, path, reader, size)
}
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
return s.s3.Stat(ctx, provider, path)
}
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
return s.s3.Copy(ctx, provider, source, target)
}
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
return s.s3.Open(ctx, provider, path)
}
func openWriteSetFixture(t *testing.T) *writeSetFixture {
t.Helper()
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
}
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
type writeSetFixtureOptions struct {
reset bool
archiveRoot string
index string
}
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
t.Helper()
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Skip("requires isolated PG, ES and object storage opt-in")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
t.Fatal("isolated database required")
}
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
t.Cleanup(cancel)
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatalf("open isolated PG: %v", err)
}
t.Cleanup(pool.Close)
if options.reset {
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
t.Fatalf("reset isolated schema: %v", err)
}
}
schemaStatements := []string{
libraryTestSchema,
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
// 免得"目标非空"被别的用例的行弄成不确定。
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
}
if options.reset {
for _, statement := range schemaStatements {
if _, err := pool.Exec(ctx, statement); err != nil {
t.Fatalf("fixture schema: %v", err)
}
}
}
token := make([]byte, 8)
if _, err := rand.Read(token); err != nil {
t.Fatal(err)
}
index := "kg_library_test_" + hex.EncodeToString(token)
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
call := func(method, path, body string, want int) {
t.Helper()
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := client.Do(request)
if err != nil {
t.Fatalf("fixture HTTP: %v", err)
}
defer response.Body.Close()
if response.StatusCode != want {
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
}
}
if options.index != "" {
index = options.index
}
if options.reset {
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200)
}
if options.reset {
t.Cleanup(func() {
call("DELETE", "/"+index, "", 200)
call("HEAD", "/"+index, "", 404)
})
}
root := options.archiveRoot
if root == "" {
root = t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
}
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
if err != nil {
t.Fatalf("object store: %v", err)
}
if err := store.EnsureBucket(ctx); err != nil {
t.Fatalf("ensure bucket: %v", err)
}
return &writeSetFixture{ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
}
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
func seedTargetData(t *testing.T, fx *writeSetFixture) {
t.Helper()
ctx := fx.ctx
locator := func(backend string, tenant uint64, id string) string {
return wsLocator(backend, "minio", wsBucket, tenant, id)
}
for _, object := range []struct {
tenant uint64
id string
}{
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
{10001, "res-orphan"}, {20000, "res-foreign"},
} {
path := locator("backend-target", object.tenant, object.id)
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
t.Fatalf("seed object %s: %v", object.id, err)
}
}
statements := []string{
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
// 同租户另一个知识库:数据与资源都必须原样保留。
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
// 共享资源:同时被目标知识库与另一个知识库绑定。
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
// 另一个租户的行。
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
}
for _, statement := range statements {
if _, err := fx.pool.Exec(ctx, statement); err != nil {
t.Fatalf("seed target: %v\n%s", err, statement)
}
}
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
}
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
t.Helper()
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[0.1,0.2]}`,
tenant, kb, knowledge, id, id, content)
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := fx.client.Do(request)
if err != nil {
t.Fatalf("index doc: %v", err)
}
defer response.Body.Close()
if response.StatusCode != 201 && response.StatusCode != 200 {
t.Fatalf("index doc %s => %d", id, response.StatusCode)
}
}
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
t.Helper()
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
}
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
t.Helper()
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
encode := func(value any) []byte {
encoded, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
return encoded
}
parts := map[string][]map[string]any{
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
"knowledge_tags": {},
"knowledge_tag_relations": {},
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
"storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
"size": int64(8)}},
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
"wiki_folders": {},
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
"wiki_page_issues": {},
"wiki_log_entries": {},
}
if tweak != nil {
tweak(parts)
}
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
manifestParts := make([]librarysnapshot.Part, 0, 14)
names := make([]string, 0, len(parts))
for name := range parts {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
manifestParts = append(manifestParts, part)
}
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
if err != nil {
t.Fatal(err)
}
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
manifestParts = append(manifestParts, resourcePart)
extraNames := make([]string, 0, len(extraObjects))
for name := range extraObjects {
extraNames = append(extraNames, name)
}
sort.Strings(extraNames)
for _, name := range extraNames {
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = "resource", name, 1
manifestParts = append(manifestParts, part)
}
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
// 校验会把"写入正确"判成索引缺文档。
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
for _, chunk := range parts["chunks"] {
id, _ := chunk["id"].(string)
knowledge, _ := chunk["knowledge_id"].(string)
content, _ := chunk["content"].(string)
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
"chunk_id": id, "source_id": id, "content": content, "embedding": []float64{0.1, 0.2}}})
}
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
if err != nil {
t.Fatal(err)
}
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
manifestParts = append(manifestParts, indexPart)
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
CreatedAt: time.Now().UTC(), Parts: manifestParts,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
id, err := archive.Seal(manifest)
if err != nil {
t.Fatalf("seal source snapshot: %v", err)
}
return id
}
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
t.Helper()
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
if err != nil {
t.Fatalf("target: %v", err)
}
t.Cleanup(target.Close)
return target
}
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
t.Helper()
mapping, err := target.Search().MappingSHA256(fx.ctx)
if err != nil {
t.Fatal(err)
}
backups, err := NewProductionBackupStore(ProductionBackupConfig{
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
})
if err != nil {
t.Fatalf("backup store: %v", err)
}
return backups
}
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
t.Helper()
dump := map[string][]string{}
for _, table := range productionLibraryTables {
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
if err != nil {
t.Fatalf("dump %s: %v", table, err)
}
values := make([]string, 0, 32)
for rows.Next() {
var raw string
if err := rows.Scan(&raw); err != nil {
rows.Close()
t.Fatalf("dump scan %s: %v", table, err)
}
values = append(values, raw)
}
rows.Close()
if err := rows.Err(); err != nil {
t.Fatalf("dump %s: %v", table, err)
}
dump[table] = values
}
return dump
}
func diffUniverse(before, after map[string][]string) string {
tables := make([]string, 0, len(before))
for table := range before {
tables = append(tables, table)
}
sort.Strings(tables)
for _, table := range tables {
if len(before[table]) != len(after[table]) {
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
}
for index := range before[table] {
if before[table][index] != after[table][index] {
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
}
}
}
return ""
}
func requestFor(snapshotID string) libraryrelease.Request {
now := time.Now().UTC()
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
ParamsDigest: "digest-writeset", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
}
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageResources {
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
}
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
outcome.Backup.WriteSet.Digest)
after := universeDump(t, fx.pool)
if diff := diffUniverse(before, after); diff != "" {
t.Fatalf("回滚后目标数据被改动:%s", diff)
}
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
var orphan string
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
}
var sourceScope int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
t.Fatal(err)
}
if sourceScope != 0 {
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
}
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatalf("本次同步新增的资源行未被清理")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
createdResources := ledger.CreatedIDs("resources")
if len(createdResources) != 1 || createdResources[0] != "res-src" {
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
}
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
created := map[string]bool{}
for _, id := range ledger.CreatedIDs("resources") {
created[id] = true
}
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
if created[id] {
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
}
}
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
}
if ledger.UniverseDigest() == "" {
t.Fatal("写集合必须带整实例内容证据")
}
}
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
// 更不能把校验降级成"跳过整实例比对"。
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
before := universeDump(t, fx.pool)
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
}
}
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
// 不能 ON CONFLICT 改写别人的行。
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
// 另一个租户里已经有一条同 ID 的资源行。
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
t.Fatal(err)
}
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
}
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
}
t.Logf("conflict rejected before write: %s", outcome.Error)
}
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
// 本次新增的资源也清掉,目标原有数据逐行不变。
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
// 同一个作用域里两页同 slug:真实 schema 的唯一索引会让 wiki 阶段写失败,
// 而 database/resources 阶段已经成功提交(半写现场)。
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
parts["wiki_pages"] = []map[string]any{
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
}
})
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
}
// 回滚只回滚行,不删对象:这个定位符必须仍然**读得到**,而且 Stat 要吃真实
// 定位符(storage://<backend>/<provider>://<bucket>/<key>),不是裸 key——
// 曾经这里写成"读不到也当通过",把 Stat 与 Open/Put 的定位符不一致盖住了。
if size, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err != nil {
t.Fatalf("回滚后对象字节必须仍在桶里(只回滚行):%v", err)
} else if size <= 0 {
t.Fatalf("回滚后对象字节大小异常:%d", size)
}
t.Logf("wiki half-write rolled back: %s", outcome.Rollback)
}
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
t.Fatalf("outcome = %+v", outcome)
}
digest := outcome.Backup.WriteSet.Digest
path := writeSetPath(fx.archiveRoot, digest)
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
original, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
if bytes.Equal(tampered, original) {
t.Fatalf("篡改夹具没生效:%s", original)
}
if err := os.Chmod(path, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, tampered, 0600); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合被篡改后取回必须失败")
}
// 内存里带着被篡改的记录去恢复,同样必须失败。
bad := *outcome.Backup.WriteSet
bad.TenantID = 10002
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
t.Fatal("写集合内容与摘要不符时恢复必须失败")
}
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
if err := os.WriteFile(path, original, 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合记录被删除后取回必须失败")
}
t.Logf("tampered/removed ledger refused, digest=%s", digest)
}