0252492e53
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时, resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次 同步在写第一笔之前被拒。 “目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足 它。改成唯一放行路径——可审计复用(v071/library_reuse.go): 1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份, 且同一资源行在来源里内容一致; 2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致; 3. 该资源的对象字节与本次快照归档摘要一致。 三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里 记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进 写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续 拒绝,且拒绝发生在写入之前。 为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的 资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。 验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、 无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建); v071 整包与全仓 go test 通过;make verify 15 项全绿。
720 lines
31 KiB
Go
720 lines
31 KiB
Go
package v071
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"sort"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
|
||
//
|
||
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
|
||
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
|
||
//
|
||
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
|
||
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
|
||
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
|
||
|
||
type writeSetFixture struct {
|
||
ctx context.Context
|
||
pool *pgxpool.Pool
|
||
client *http.Client
|
||
searchBase string
|
||
index string
|
||
archiveRoot string
|
||
objects *flakyStore
|
||
bucket string
|
||
}
|
||
|
||
const (
|
||
wsTargetTenant = 10001
|
||
wsTargetKB = "kb-target"
|
||
wsOtherKB = "kb-other"
|
||
wsSourceTenant = 10002
|
||
wsSourceKB = "kb-source"
|
||
wsForeignTnt = 20000
|
||
wsBucket = "kg-wire-test-bucket"
|
||
|
||
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
|
||
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
|
||
wsHandleOther = "cccccccccccccccccccccc"
|
||
wsHandleOrphan = "dddddddddddddddddddddd"
|
||
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
|
||
wsHandleForeign = "ffffffffffffffffffffff"
|
||
)
|
||
|
||
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
|
||
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
|
||
}
|
||
|
||
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
|
||
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
|
||
// Open/Stat/Copy 全部透传。
|
||
type flakyStore struct {
|
||
s3 *S3ObjectStore
|
||
failPut atomic.Bool
|
||
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
|
||
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
|
||
failPath string
|
||
puts atomic.Int64
|
||
}
|
||
|
||
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
|
||
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
|
||
return errors.New("注入的对象写入失败")
|
||
}
|
||
s.puts.Add(1)
|
||
return s.s3.Put(ctx, provider, path, reader, size)
|
||
}
|
||
|
||
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
|
||
return s.s3.Stat(ctx, provider, path)
|
||
}
|
||
|
||
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
|
||
return s.s3.Copy(ctx, provider, source, target)
|
||
}
|
||
|
||
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
|
||
return s.s3.Open(ctx, provider, path)
|
||
}
|
||
|
||
func openWriteSetFixture(t *testing.T) *writeSetFixture {
|
||
t.Helper()
|
||
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
|
||
}
|
||
|
||
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
|
||
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
|
||
type writeSetFixtureOptions struct {
|
||
reset bool
|
||
archiveRoot string
|
||
index string
|
||
}
|
||
|
||
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
|
||
t.Helper()
|
||
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
||
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
|
||
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
|
||
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
|
||
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
|
||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||
t.Skip("requires isolated PG, ES and object storage opt-in")
|
||
}
|
||
cfg, err := pgxpool.ParseConfig(dsn)
|
||
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
|
||
t.Fatal("isolated database required")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||
t.Cleanup(cancel)
|
||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||
if err != nil {
|
||
t.Fatalf("open isolated PG: %v", err)
|
||
}
|
||
t.Cleanup(pool.Close)
|
||
if options.reset {
|
||
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
||
t.Fatalf("reset isolated schema: %v", err)
|
||
}
|
||
}
|
||
schemaStatements := []string{
|
||
libraryTestSchema,
|
||
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
|
||
// 免得"目标非空"被别的用例的行弄成不确定。
|
||
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
|
||
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
|
||
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
|
||
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
|
||
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
|
||
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
|
||
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
|
||
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
|
||
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
|
||
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
|
||
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
|
||
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
|
||
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
|
||
}
|
||
if options.reset {
|
||
for _, statement := range schemaStatements {
|
||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("fixture schema: %v", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
token := make([]byte, 8)
|
||
if _, err := rand.Read(token); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
index := "kg_library_test_" + hex.EncodeToString(token)
|
||
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||
return http.ErrUseLastResponse
|
||
}}
|
||
call := func(method, path, body string, want int) {
|
||
t.Helper()
|
||
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := client.Do(request)
|
||
if err != nil {
|
||
t.Fatalf("fixture HTTP: %v", err)
|
||
}
|
||
defer response.Body.Close()
|
||
if response.StatusCode != want {
|
||
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
|
||
}
|
||
}
|
||
if options.index != "" {
|
||
index = options.index
|
||
}
|
||
if options.reset {
|
||
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200)
|
||
}
|
||
if options.reset {
|
||
t.Cleanup(func() {
|
||
call("DELETE", "/"+index, "", 200)
|
||
call("HEAD", "/"+index, "", 404)
|
||
})
|
||
}
|
||
|
||
root := options.archiveRoot
|
||
if root == "" {
|
||
root = t.TempDir()
|
||
if err := os.Chmod(root, 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
|
||
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
|
||
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
|
||
if err != nil {
|
||
t.Fatalf("object store: %v", err)
|
||
}
|
||
if err := store.EnsureBucket(ctx); err != nil {
|
||
t.Fatalf("ensure bucket: %v", err)
|
||
}
|
||
return &writeSetFixture{ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
|
||
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
|
||
}
|
||
|
||
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
|
||
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
|
||
func seedTargetData(t *testing.T, fx *writeSetFixture) {
|
||
t.Helper()
|
||
ctx := fx.ctx
|
||
locator := func(backend string, tenant uint64, id string) string {
|
||
return wsLocator(backend, "minio", wsBucket, tenant, id)
|
||
}
|
||
for _, object := range []struct {
|
||
tenant uint64
|
||
id string
|
||
}{
|
||
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
|
||
{10001, "res-orphan"}, {20000, "res-foreign"},
|
||
} {
|
||
path := locator("backend-target", object.tenant, object.id)
|
||
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
|
||
t.Fatalf("seed object %s: %v", object.id, err)
|
||
}
|
||
}
|
||
statements := []string{
|
||
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
|
||
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
|
||
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
|
||
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
|
||
// 同租户另一个知识库:数据与资源都必须原样保留。
|
||
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
|
||
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
|
||
// 共享资源:同时被目标知识库与另一个知识库绑定。
|
||
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
|
||
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
|
||
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
|
||
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
|
||
// 另一个租户的行。
|
||
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
|
||
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
|
||
}
|
||
for _, statement := range statements {
|
||
if _, err := fx.pool.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("seed target: %v\n%s", err, statement)
|
||
}
|
||
}
|
||
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
|
||
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
|
||
}
|
||
|
||
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
|
||
t.Helper()
|
||
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[0.1,0.2]}`,
|
||
tenant, kb, knowledge, id, id, content)
|
||
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
|
||
strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := fx.client.Do(request)
|
||
if err != nil {
|
||
t.Fatalf("index doc: %v", err)
|
||
}
|
||
defer response.Body.Close()
|
||
if response.StatusCode != 201 && response.StatusCode != 200 {
|
||
t.Fatalf("index doc %s => %d", id, response.StatusCode)
|
||
}
|
||
}
|
||
|
||
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
|
||
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
|
||
t.Helper()
|
||
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
|
||
}
|
||
|
||
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
|
||
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
|
||
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
|
||
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
|
||
t.Helper()
|
||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer archive.Close()
|
||
encode := func(value any) []byte {
|
||
encoded, err := json.Marshal(value)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return encoded
|
||
}
|
||
parts := map[string][]map[string]any{
|
||
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
|
||
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
|
||
"knowledge_tags": {},
|
||
"knowledge_tag_relations": {},
|
||
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
|
||
"storage_backend_id": "backend-source", "provider": "oss",
|
||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
|
||
"size": int64(8)}},
|
||
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
|
||
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
|
||
"wiki_folders": {},
|
||
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
|
||
"wiki_page_issues": {},
|
||
"wiki_log_entries": {},
|
||
}
|
||
if tweak != nil {
|
||
tweak(parts)
|
||
}
|
||
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
||
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
||
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
||
manifestParts := make([]librarysnapshot.Part, 0, 14)
|
||
names := make([]string, 0, len(parts))
|
||
for name := range parts {
|
||
names = append(names, name)
|
||
}
|
||
sort.Strings(names)
|
||
for _, name := range names {
|
||
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
|
||
manifestParts = append(manifestParts, part)
|
||
}
|
||
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
|
||
manifestParts = append(manifestParts, resourcePart)
|
||
extraNames := make([]string, 0, len(extraObjects))
|
||
for name := range extraObjects {
|
||
extraNames = append(extraNames, name)
|
||
}
|
||
sort.Strings(extraNames)
|
||
for _, name := range extraNames {
|
||
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = "resource", name, 1
|
||
manifestParts = append(manifestParts, part)
|
||
}
|
||
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
|
||
// 校验会把"写入正确"判成索引缺文档。
|
||
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
|
||
for _, chunk := range parts["chunks"] {
|
||
id, _ := chunk["id"].(string)
|
||
knowledge, _ := chunk["knowledge_id"].(string)
|
||
content, _ := chunk["content"].(string)
|
||
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
||
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
|
||
"chunk_id": id, "source_id": id, "content": content, "embedding": []float64{0.1, 0.2}}})
|
||
}
|
||
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
||
manifestParts = append(manifestParts, indexPart)
|
||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
|
||
CreatedAt: time.Now().UTC(), Parts: manifestParts,
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
||
id, err := archive.Seal(manifest)
|
||
if err != nil {
|
||
t.Fatalf("seal source snapshot: %v", err)
|
||
}
|
||
return id
|
||
}
|
||
|
||
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
|
||
t.Helper()
|
||
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
|
||
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
|
||
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
|
||
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
|
||
if err != nil {
|
||
t.Fatalf("target: %v", err)
|
||
}
|
||
t.Cleanup(target.Close)
|
||
return target
|
||
}
|
||
|
||
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
|
||
t.Helper()
|
||
mapping, err := target.Search().MappingSHA256(fx.ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
backups, err := NewProductionBackupStore(ProductionBackupConfig{
|
||
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("backup store: %v", err)
|
||
}
|
||
return backups
|
||
}
|
||
|
||
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
|
||
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
|
||
t.Helper()
|
||
dump := map[string][]string{}
|
||
for _, table := range productionLibraryTables {
|
||
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
|
||
if err != nil {
|
||
t.Fatalf("dump %s: %v", table, err)
|
||
}
|
||
values := make([]string, 0, 32)
|
||
for rows.Next() {
|
||
var raw string
|
||
if err := rows.Scan(&raw); err != nil {
|
||
rows.Close()
|
||
t.Fatalf("dump scan %s: %v", table, err)
|
||
}
|
||
values = append(values, raw)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
t.Fatalf("dump %s: %v", table, err)
|
||
}
|
||
dump[table] = values
|
||
}
|
||
return dump
|
||
}
|
||
|
||
func diffUniverse(before, after map[string][]string) string {
|
||
tables := make([]string, 0, len(before))
|
||
for table := range before {
|
||
tables = append(tables, table)
|
||
}
|
||
sort.Strings(tables)
|
||
for _, table := range tables {
|
||
if len(before[table]) != len(after[table]) {
|
||
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
|
||
}
|
||
for index := range before[table] {
|
||
if before[table][index] != after[table][index] {
|
||
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func requestFor(snapshotID string) libraryrelease.Request {
|
||
now := time.Now().UTC()
|
||
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
|
||
ParamsDigest: "digest-writeset", Now: now,
|
||
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
|
||
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
|
||
}
|
||
|
||
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
|
||
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
|
||
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
|
||
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
before := universeDump(t, fx.pool)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != libraryrelease.StageResources {
|
||
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
|
||
}
|
||
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
|
||
outcome.Backup.WriteSet.Digest)
|
||
after := universeDump(t, fx.pool)
|
||
if diff := diffUniverse(before, after); diff != "" {
|
||
t.Fatalf("回滚后目标数据被改动:%s", diff)
|
||
}
|
||
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
|
||
var orphan string
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
|
||
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
|
||
}
|
||
var sourceScope int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if sourceScope != 0 {
|
||
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
|
||
}
|
||
var residue int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if residue != 0 {
|
||
t.Fatalf("本次同步新增的资源行未被清理")
|
||
}
|
||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||
t.Fatalf("回滚必须与预备份一致:%v", err)
|
||
}
|
||
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
|
||
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
|
||
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
|
||
}
|
||
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
|
||
if err != nil {
|
||
t.Fatalf("写集合记录取不回来:%v", err)
|
||
}
|
||
createdResources := ledger.CreatedIDs("resources")
|
||
if len(createdResources) != 1 || createdResources[0] != "res-src" {
|
||
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
|
||
}
|
||
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
|
||
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
|
||
created := map[string]bool{}
|
||
for _, id := range ledger.CreatedIDs("resources") {
|
||
created[id] = true
|
||
}
|
||
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
|
||
if created[id] {
|
||
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
|
||
}
|
||
}
|
||
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
|
||
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
|
||
}
|
||
if ledger.UniverseDigest() == "" {
|
||
t.Fatal("写集合必须带整实例内容证据")
|
||
}
|
||
}
|
||
|
||
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
|
||
// 更不能把校验降级成"跳过整实例比对"。
|
||
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
before := universeDump(t, fx.pool)
|
||
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
|
||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
|
||
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
|
||
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
|
||
}
|
||
}
|
||
|
||
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
|
||
// 不能 ON CONFLICT 改写别人的行。
|
||
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
// 另一个租户里已经有一条同 ID 的资源行。
|
||
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
|
||
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
before := universeDump(t, fx.pool)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
if outcome.State != "failed" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != "" {
|
||
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
|
||
}
|
||
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
|
||
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
|
||
}
|
||
t.Logf("conflict rejected before write: %s", outcome.Error)
|
||
}
|
||
|
||
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
|
||
// 本次新增的资源也清掉,目标原有数据逐行不变。
|
||
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
before := universeDump(t, fx.pool)
|
||
// 同一个作用域里两页同 slug:真实 schema 的唯一索引会让 wiki 阶段写失败,
|
||
// 而 database/resources 阶段已经成功提交(半写现场)。
|
||
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
|
||
parts["wiki_pages"] = []map[string]any{
|
||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
|
||
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
|
||
}
|
||
})
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != libraryrelease.StageWiki {
|
||
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
|
||
}
|
||
// 回滚只回滚行,不删对象:这个定位符必须仍然**读得到**,而且 Stat 要吃真实
|
||
// 定位符(storage://<backend>/<provider>://<bucket>/<key>),不是裸 key——
|
||
// 曾经这里写成"读不到也当通过",把 Stat 与 Open/Put 的定位符不一致盖住了。
|
||
if size, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err != nil {
|
||
t.Fatalf("回滚后对象字节必须仍在桶里(只回滚行):%v", err)
|
||
} else if size <= 0 {
|
||
t.Fatalf("回滚后对象字节大小异常:%d", size)
|
||
}
|
||
t.Logf("wiki half-write rolled back: %s", outcome.Rollback)
|
||
}
|
||
|
||
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
|
||
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
digest := outcome.Backup.WriteSet.Digest
|
||
path := writeSetPath(fx.archiveRoot, digest)
|
||
|
||
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
|
||
original, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
|
||
if bytes.Equal(tampered, original) {
|
||
t.Fatalf("篡改夹具没生效:%s", original)
|
||
}
|
||
if err := os.Chmod(path, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, tampered, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||
t.Fatal("写集合被篡改后取回必须失败")
|
||
}
|
||
// 内存里带着被篡改的记录去恢复,同样必须失败。
|
||
bad := *outcome.Backup.WriteSet
|
||
bad.TenantID = 10002
|
||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
|
||
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
|
||
t.Fatal("写集合内容与摘要不符时恢复必须失败")
|
||
}
|
||
|
||
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
|
||
if err := os.WriteFile(path, original, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.Remove(path); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||
t.Fatal("写集合记录被删除后取回必须失败")
|
||
}
|
||
t.Logf("tampered/removed ledger refused, digest=%s", digest)
|
||
}
|