e760d85427
build / build (push) Successful in 1m25s
为什么:B29.2 的 PG 子断言缺一条真实命中 PG 恢复步骤的失败闭环。故障不改产品 返回值、不停共享 PG、不扩 ACL、不改 schema:恢复最后一段 PG 写入(wiki 阶段) 会把 before-image 的日志行按原 id 插回目标范围;先用一行"外部范围、同主键"的 日志占住这个 id,那条真实 INSERT 就得到 23505。用例记录服务器 SQLSTATE、同一 语句的探针读回(没有多出行)、失败阶段事务回滚,以及真实的部分恢复现场: database 阶段已提交、wiki 表仍是发布写入、共享 resources 的新增行按设计留到 四阶段全部成功后再清。解除只删这一行,随后走同一 window_job_id 的正式 recover。 同时把夹具的索引登记接通运行器(KG_R38_INDEX 被真实消费),并落实际 namespace, 使 wrapper 在夹具未执行 t.Cleanup 时也只清登记的那一个索引。 验证:make verify 16/16 success(local 档案,identity=skipped 属本地常态); PG 闭环的真实运行在提交后按同一 SHA 单独执行并留 raw。
794 lines
33 KiB
Go
794 lines
33 KiB
Go
package v071
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
|
||
//
|
||
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
|
||
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
|
||
//
|
||
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
|
||
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
|
||
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
|
||
|
||
type writeSetFixture struct {
|
||
embeddingDimension int
|
||
ctx context.Context
|
||
pool *pgxpool.Pool
|
||
client *http.Client
|
||
searchBase string
|
||
index string
|
||
archiveRoot string
|
||
objects *flakyStore
|
||
bucket string
|
||
}
|
||
|
||
const (
|
||
wsTargetTenant = 10001
|
||
wsTargetKB = "kb-target"
|
||
wsOtherKB = "kb-other"
|
||
wsSourceTenant = 10002
|
||
wsSourceKB = "kb-source"
|
||
wsForeignTnt = 20000
|
||
|
||
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
|
||
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
|
||
wsHandleOther = "cccccccccccccccccccccc"
|
||
wsHandleOrphan = "dddddddddddddddddddddd"
|
||
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
|
||
wsHandleForeign = "ffffffffffffffffffffff"
|
||
)
|
||
|
||
// A per-run bucket isolates object writes even when the S3 service is shared.
|
||
var wsBucket = func() string {
|
||
if bucket := os.Getenv("KG_LIBRARY_TEST_S3_BUCKET"); strings.HasPrefix(bucket, "kg-library-test-") {
|
||
return bucket
|
||
}
|
||
return "kg-wire-test-bucket"
|
||
}()
|
||
|
||
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
|
||
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
|
||
}
|
||
|
||
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
|
||
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
|
||
// Open/Stat/Copy 全部透传。
|
||
type flakyStore struct {
|
||
s3 *S3ObjectStore
|
||
failPut atomic.Bool
|
||
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
|
||
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
|
||
failPath string
|
||
puts atomic.Int64
|
||
}
|
||
|
||
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
|
||
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
|
||
return errors.New("注入的对象写入失败")
|
||
}
|
||
s.puts.Add(1)
|
||
return s.s3.Put(ctx, provider, path, reader, size)
|
||
}
|
||
|
||
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
|
||
return s.s3.Stat(ctx, provider, path)
|
||
}
|
||
|
||
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
|
||
return s.s3.Copy(ctx, provider, source, target)
|
||
}
|
||
|
||
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
|
||
return s.s3.Open(ctx, provider, path)
|
||
}
|
||
|
||
func openWriteSetFixture(t *testing.T) *writeSetFixture {
|
||
t.Helper()
|
||
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
|
||
}
|
||
|
||
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
|
||
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
|
||
type writeSetFixtureOptions struct {
|
||
embeddingDimension int
|
||
reset bool
|
||
archiveRoot string
|
||
index string
|
||
// ddl 非空时替换最小夹具 schema:DS-02 冻结 bundle 用同一份 DDL 初始化两个隔离目标。
|
||
ddl []string
|
||
}
|
||
|
||
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
|
||
if options.embeddingDimension == 0 {
|
||
options.embeddingDimension = 2
|
||
}
|
||
t.Helper()
|
||
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
||
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
|
||
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
|
||
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
|
||
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
|
||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||
t.Skip("requires isolated PG, ES and object storage opt-in")
|
||
}
|
||
cfg, err := pgxpool.ParseConfig(dsn)
|
||
if options.reset {
|
||
previousBucket := wsBucket
|
||
var namespace [8]byte
|
||
if _, err := rand.Read(namespace[:]); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wsBucket = "kg-library-test-" + hex.EncodeToString(namespace[:])
|
||
t.Cleanup(func() { wsBucket = previousBucket })
|
||
t.Setenv("KG_LIBRARY_TEST_S3_BUCKET", wsBucket)
|
||
t.Logf("isolated object namespace=%s", wsBucket)
|
||
}
|
||
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
|
||
t.Fatal("isolated database required")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||
t.Cleanup(cancel)
|
||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||
if err != nil {
|
||
t.Fatalf("open isolated PG: %v", err)
|
||
}
|
||
t.Cleanup(pool.Close)
|
||
if options.reset {
|
||
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
||
t.Fatalf("reset isolated schema: %v", err)
|
||
}
|
||
}
|
||
// 冻结 bundle 自带完整 DDL 时,不叠加最小夹具(否则两个目标的 schema 又不同)。
|
||
if len(options.ddl) > 0 {
|
||
for _, statement := range options.ddl {
|
||
statement = strings.ReplaceAll(statement, "%BUCKET%", wsBucket)
|
||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("bundle schema: %v\n%s", err, statement)
|
||
}
|
||
}
|
||
}
|
||
schemaStatements := []string{
|
||
libraryTestSchema,
|
||
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
|
||
// 免得"目标非空"被别的用例的行弄成不确定。
|
||
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
|
||
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
|
||
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
|
||
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
|
||
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
|
||
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
|
||
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
|
||
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
|
||
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
|
||
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
|
||
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
|
||
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
|
||
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
|
||
}
|
||
if options.reset && len(options.ddl) == 0 {
|
||
for _, statement := range schemaStatements {
|
||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("fixture schema: %v", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
token := make([]byte, 8)
|
||
if _, err := rand.Read(token); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
index := "kg_library_test_" + hex.EncodeToString(token)
|
||
if options.index != "" {
|
||
index = options.index
|
||
} else if registered := os.Getenv("KG_R38_INDEX"); registered != "" {
|
||
// 隔离运行器在创建前登记了唯一索引;夹具必须消费它,否则 wrapper 清理的是
|
||
// 一个从未被使用的名字(真索引只能靠 t.Cleanup,进程提前退出就漏掉)。
|
||
index = registered
|
||
}
|
||
// Register each disposable namespace before its first ES/object mutation.
|
||
r10Save(t, "namespace", map[string]string{"bucket": wsBucket, "index": index})
|
||
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||
return http.ErrUseLastResponse
|
||
}}
|
||
call := func(method, path, body string, want int) {
|
||
t.Helper()
|
||
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := client.Do(request)
|
||
if err != nil {
|
||
t.Fatalf("fixture HTTP: %v", err)
|
||
}
|
||
defer response.Body.Close()
|
||
if response.StatusCode != want {
|
||
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
|
||
}
|
||
}
|
||
if options.reset {
|
||
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":`+strconv.Itoa(options.embeddingDimension)+`}}}}`, 200)
|
||
}
|
||
if options.reset {
|
||
t.Cleanup(func() {
|
||
// 保留 UI 验收现场时不删索引:父要能对着真实目标点恢复。
|
||
if os.Getenv("KG_R8_KEEP_SCENE") == "1" {
|
||
return
|
||
}
|
||
call("DELETE", "/"+index, "", 200)
|
||
call("HEAD", "/"+index, "", 404)
|
||
})
|
||
}
|
||
|
||
root := options.archiveRoot
|
||
if root == "" {
|
||
root = t.TempDir()
|
||
if err := os.Chmod(root, 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
|
||
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
|
||
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
|
||
if err != nil {
|
||
t.Fatalf("object store: %v", err)
|
||
}
|
||
if err := store.EnsureBucket(ctx); err != nil {
|
||
t.Fatalf("ensure bucket: %v", err)
|
||
}
|
||
if options.reset {
|
||
enableIsolatedObjectVersions(t, ctx, store)
|
||
}
|
||
return &writeSetFixture{embeddingDimension: options.embeddingDimension, ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
|
||
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
|
||
}
|
||
|
||
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
|
||
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
|
||
func seedTargetData(t *testing.T, fx *writeSetFixture) {
|
||
t.Helper()
|
||
ctx := fx.ctx
|
||
locator := func(backend string, tenant uint64, id string) string {
|
||
return wsLocator(backend, "minio", wsBucket, tenant, id)
|
||
}
|
||
for _, object := range []struct {
|
||
tenant uint64
|
||
id string
|
||
}{
|
||
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
|
||
{10001, "res-orphan"}, {20000, "res-foreign"},
|
||
} {
|
||
path := locator("backend-target", object.tenant, object.id)
|
||
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
|
||
t.Fatalf("seed object %s: %v", object.id, err)
|
||
}
|
||
}
|
||
statements := []string{
|
||
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
|
||
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
|
||
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
|
||
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
|
||
// 同租户另一个知识库:数据与资源都必须原样保留。
|
||
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
|
||
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
|
||
// 共享资源:同时被目标知识库与另一个知识库绑定。
|
||
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
|
||
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
|
||
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
|
||
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
|
||
// 另一个租户的行。
|
||
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
|
||
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
|
||
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
|
||
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
|
||
}
|
||
for _, statement := range statements {
|
||
if _, err := fx.pool.Exec(ctx, statement); err != nil {
|
||
t.Fatalf("seed target: %v\n%s", err, statement)
|
||
}
|
||
}
|
||
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
|
||
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
|
||
}
|
||
|
||
func (fx *writeSetFixture) embeddingJSON() []byte {
|
||
n := fx.embeddingDimension
|
||
if n == 0 {
|
||
n = 2
|
||
}
|
||
v := make([]float64, n)
|
||
v[0] = 0.1
|
||
v[1] = 0.2
|
||
b, _ := json.Marshal(v)
|
||
return b
|
||
}
|
||
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
|
||
t.Helper()
|
||
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":%s}`,
|
||
tenant, kb, knowledge, id, id, content, fx.embeddingJSON())
|
||
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
|
||
strings.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := fx.client.Do(request)
|
||
if err != nil {
|
||
t.Fatalf("index doc: %v", err)
|
||
}
|
||
defer response.Body.Close()
|
||
if response.StatusCode != 201 && response.StatusCode != 200 {
|
||
t.Fatalf("index doc %s => %d", id, response.StatusCode)
|
||
}
|
||
}
|
||
|
||
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
|
||
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
|
||
t.Helper()
|
||
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
|
||
}
|
||
|
||
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
|
||
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
|
||
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
|
||
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
|
||
t.Helper()
|
||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer archive.Close()
|
||
encode := func(value any) []byte {
|
||
encoded, err := json.Marshal(value)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return encoded
|
||
}
|
||
parts := map[string][]map[string]any{
|
||
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
|
||
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
|
||
"knowledge_tags": {},
|
||
"knowledge_tag_relations": {},
|
||
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
|
||
"storage_backend_id": "backend-source", "provider": "oss",
|
||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
|
||
"size": int64(8)}},
|
||
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
|
||
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
|
||
"wiki_folders": {},
|
||
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
|
||
"wiki_page_issues": {},
|
||
"wiki_log_entries": {},
|
||
}
|
||
if tweak != nil {
|
||
tweak(parts)
|
||
}
|
||
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
||
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
||
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
||
manifestParts := make([]librarysnapshot.Part, 0, 14)
|
||
names := make([]string, 0, len(parts))
|
||
for name := range parts {
|
||
names = append(names, name)
|
||
}
|
||
sort.Strings(names)
|
||
for _, name := range names {
|
||
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
|
||
manifestParts = append(manifestParts, part)
|
||
}
|
||
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
|
||
manifestParts = append(manifestParts, resourcePart)
|
||
extraNames := make([]string, 0, len(extraObjects))
|
||
for name := range extraObjects {
|
||
extraNames = append(extraNames, name)
|
||
}
|
||
sort.Strings(extraNames)
|
||
for _, name := range extraNames {
|
||
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
part.Kind, part.Name, part.Count = "resource", name, 1
|
||
manifestParts = append(manifestParts, part)
|
||
}
|
||
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
|
||
// 校验会把"写入正确"判成索引缺文档。
|
||
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
|
||
for _, chunk := range parts["chunks"] {
|
||
id, _ := chunk["id"].(string)
|
||
knowledge, _ := chunk["knowledge_id"].(string)
|
||
content, _ := chunk["content"].(string)
|
||
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
||
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
|
||
"chunk_id": id, "source_id": id, "content": content, "embedding": json.RawMessage(fx.embeddingJSON())}})
|
||
}
|
||
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
||
manifestParts = append(manifestParts, indexPart)
|
||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
|
||
CreatedAt: time.Now().UTC(), Parts: manifestParts,
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
||
id, err := archive.Seal(manifest)
|
||
if err != nil {
|
||
t.Fatalf("seal source snapshot: %v", err)
|
||
}
|
||
return id
|
||
}
|
||
|
||
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
|
||
t.Helper()
|
||
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
|
||
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
|
||
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
|
||
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
|
||
if err != nil {
|
||
t.Fatalf("target: %v", err)
|
||
}
|
||
t.Cleanup(target.Close)
|
||
return target
|
||
}
|
||
|
||
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
|
||
t.Helper()
|
||
mapping, err := target.Search().MappingSHA256(fx.ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
backups, err := NewProductionBackupStore(ProductionBackupConfig{
|
||
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("backup store: %v", err)
|
||
}
|
||
return backups
|
||
}
|
||
|
||
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
|
||
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
|
||
t.Helper()
|
||
dump := map[string][]string{}
|
||
for _, table := range productionLibraryTables {
|
||
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
|
||
if err != nil {
|
||
t.Fatalf("dump %s: %v", table, err)
|
||
}
|
||
values := make([]string, 0, 32)
|
||
for rows.Next() {
|
||
var raw string
|
||
if err := rows.Scan(&raw); err != nil {
|
||
rows.Close()
|
||
t.Fatalf("dump scan %s: %v", table, err)
|
||
}
|
||
values = append(values, raw)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
t.Fatalf("dump %s: %v", table, err)
|
||
}
|
||
dump[table] = values
|
||
}
|
||
return dump
|
||
}
|
||
|
||
func diffUniverse(before, after map[string][]string) string {
|
||
tables := make([]string, 0, len(before))
|
||
for table := range before {
|
||
tables = append(tables, table)
|
||
}
|
||
sort.Strings(tables)
|
||
for _, table := range tables {
|
||
if len(before[table]) != len(after[table]) {
|
||
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
|
||
}
|
||
for index := range before[table] {
|
||
if before[table][index] != after[table][index] {
|
||
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func requestFor(snapshotID string) libraryrelease.Request {
|
||
now := time.Now().UTC()
|
||
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
|
||
ParamsDigest: "digest-writeset", Now: now,
|
||
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
|
||
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
|
||
}
|
||
|
||
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
|
||
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
|
||
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
|
||
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
before := universeDump(t, fx.pool)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != libraryrelease.StageResources {
|
||
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
|
||
}
|
||
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
|
||
outcome.Backup.WriteSet.Digest)
|
||
after := universeDump(t, fx.pool)
|
||
if diff := diffUniverse(before, after); diff != "" {
|
||
t.Fatalf("回滚后目标数据被改动:%s", diff)
|
||
}
|
||
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
|
||
var orphan string
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
|
||
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
|
||
}
|
||
var sourceScope int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if sourceScope != 0 {
|
||
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
|
||
}
|
||
var residue int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if residue != 0 {
|
||
t.Fatalf("本次同步新增的资源行未被清理")
|
||
}
|
||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||
t.Fatalf("回滚必须与预备份一致:%v", err)
|
||
}
|
||
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
|
||
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
|
||
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
|
||
}
|
||
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
|
||
if err != nil {
|
||
t.Fatalf("写集合记录取不回来:%v", err)
|
||
}
|
||
createdResources := ledger.CreatedIDs("resources")
|
||
if len(createdResources) != 1 || createdResources[0] != "res-src" {
|
||
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
|
||
}
|
||
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
|
||
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
|
||
created := map[string]bool{}
|
||
for _, id := range ledger.CreatedIDs("resources") {
|
||
created[id] = true
|
||
}
|
||
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
|
||
if created[id] {
|
||
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
|
||
}
|
||
}
|
||
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
|
||
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
|
||
}
|
||
if ledger.UniverseDigest() == "" {
|
||
t.Fatal("写集合必须带整实例内容证据")
|
||
}
|
||
}
|
||
|
||
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
|
||
// 更不能把校验降级成"跳过整实例比对"。
|
||
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
before := universeDump(t, fx.pool)
|
||
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
|
||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
|
||
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
|
||
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
|
||
}
|
||
}
|
||
|
||
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
|
||
// 不能 ON CONFLICT 改写别人的行。
|
||
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
// 另一个租户里已经有一条同 ID 的资源行。
|
||
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
|
||
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
before := universeDump(t, fx.pool)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
if outcome.State != "failed" {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != "" {
|
||
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
|
||
}
|
||
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
|
||
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
|
||
}
|
||
t.Logf("conflict rejected before write: %s", outcome.Error)
|
||
}
|
||
|
||
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
|
||
// 本次新增的资源也清掉,目标原有数据逐行不变。
|
||
func TestLibraryDuplicateSlugRejectedBeforeWriteIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
before := universeDump(t, fx.pool)
|
||
// 重复 slug 保留为写前拒绝负例;晚期故障由独立用例注入。
|
||
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
|
||
parts["wiki_pages"] = []map[string]any{
|
||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
|
||
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
|
||
}
|
||
})
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
if outcome.State != "failed" || !strings.Contains(outcome.Error, "source duplicates natural key") {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
if outcome.FailedAt != "" {
|
||
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
|
||
}
|
||
if fx.objects.puts.Load() != 0 {
|
||
t.Fatal("duplicate slug reached object write")
|
||
}
|
||
}
|
||
|
||
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
before := universeDump(t, fx.pool)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), contractFailWikiTarget{target}, fx.backups(t, target), nil)
|
||
if outcome.State != "rolled_back" || outcome.FailedAt != libraryrelease.StageWiki {
|
||
t.Fatalf("late failure: %+v", outcome)
|
||
}
|
||
if fx.objects.puts.Load() == 0 {
|
||
t.Fatal("late failure never reached real object write")
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatal(diff)
|
||
}
|
||
observed, err := fx.objects.ObserveObject(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src"))
|
||
if err != nil || observed.Exists {
|
||
t.Fatalf("created object residue: %+v %v", observed, err)
|
||
}
|
||
}
|
||
|
||
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
|
||
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
fx.objects.failPath = "/res-src/"
|
||
fx.objects.failPut.Store(true)
|
||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||
fx.objects.failPut.Store(false)
|
||
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
|
||
t.Fatalf("outcome = %+v", outcome)
|
||
}
|
||
digest := outcome.Backup.WriteSet.Digest
|
||
path := writeSetPath(fx.archiveRoot, digest)
|
||
|
||
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
|
||
original, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
|
||
if bytes.Equal(tampered, original) {
|
||
t.Fatalf("篡改夹具没生效:%s", original)
|
||
}
|
||
if err := os.Chmod(path, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, tampered, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||
t.Fatal("写集合被篡改后取回必须失败")
|
||
}
|
||
// 内存里带着被篡改的记录去恢复,同样必须失败。
|
||
bad := *outcome.Backup.WriteSet
|
||
bad.TenantID = 10002
|
||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
|
||
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
|
||
t.Fatal("写集合内容与摘要不符时恢复必须失败")
|
||
}
|
||
|
||
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
|
||
if err := os.WriteFile(path, original, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.Remove(path); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||
t.Fatal("写集合记录被删除后取回必须失败")
|
||
}
|
||
t.Logf("tampered/removed ledger refused, digest=%s", digest)
|
||
}
|