diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d73122..fdb516e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 开发快照(跨库分块序列分配) + +### Fixed + +- 发布分块时由目标数据库分配本地 `seq_id`,不再复制源库序列而撞上其它知识库的唯一索引;文档和分块 UUID 不变。 +- 失败恢复仍重放预备份中的原始序列,并保留整实例逐行核验;发布内容校验仅排除分块的本地序列,不放宽正文和其它字段校验。 + ## 开发快照(云执行器授权配置与启动核验) ### Fixed diff --git a/internal/releaseadapter/v071/library_chunk_sequence_integration_test.go b/internal/releaseadapter/v071/library_chunk_sequence_integration_test.go new file mode 100644 index 0000000..145bd35 --- /dev/null +++ b/internal/releaseadapter/v071/library_chunk_sequence_integration_test.go @@ -0,0 +1,43 @@ +package v071 + +import ( + "knowledge-governance/internal/libraryrelease" + "testing" +) + +// Cross-database snapshots may reuse a local serial value already owned by an +// unrelated KB. UUIDs remain stable; the destination allocates its own serial. +func TestLibraryChunkSequenceIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + if _, err := fx.pool.Exec(fx.ctx, `ALTER TABLE chunks ADD COLUMN seq_id bigserial UNIQUE`); err != nil { + t.Fatal(err) + } + var occupied int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT seq_id FROM chunks WHERE knowledge_base_id=$1 LIMIT 1`, wsOtherKB).Scan(&occupied); err != nil { + t.Fatal(err) + } + before := universeDump(t, fx.pool) + snapshot := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) { parts["chunks"][0]["seq_id"] = occupied }) + target := fx.target(t) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, fx.backups(t, target), nil) + if outcome.State != "succeeded" { + t.Fatalf("cross-database publish must allocate local sequence: %+v", outcome) + } + var allocated int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT seq_id FROM chunks WHERE id='chunk-src'`).Scan(&allocated); err != nil { + t.Fatal(err) + } + if allocated == occupied || allocated <= 0 { + t.Fatalf("invalid destination serial %d (occupied %d)", allocated, occupied) + } + if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: snapshot}); err != nil { + t.Fatalf("content verification: %v", err) + } + if err := target.Restore(fx.ctx, outcome.Backup); err != nil { + t.Fatalf("restore must preserve original serial values: %v", err) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("restored universe differs: %s", diff) + } +} diff --git a/internal/releaseadapter/v071/library_chunk_sequence_test.go b/internal/releaseadapter/v071/library_chunk_sequence_test.go new file mode 100644 index 0000000..8bb9b9a --- /dev/null +++ b/internal/releaseadapter/v071/library_chunk_sequence_test.go @@ -0,0 +1,35 @@ +package v071 + +import ( + "encoding/json" + "testing" +) + +func TestChunkSequenceProjectionKeepsContent(t *testing.T) { + for _, restore := range []bool{false, true} { + target := &ProductionLibraryTarget{scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}, snapshotTenant: 10002, snapshotKB: "kb-source", preserveChunkSequence: restore} + row := map[string]json.RawMessage{"id": json.RawMessage(`"chunk-src"`), "tenant_id": json.RawMessage(`10002`), "knowledge_base_id": json.RawMessage(`"kb-source"`), "seq_id": json.RawMessage(`7`), "content": json.RawMessage(`"exact content"`)} + if err := target.rowTransform("chunks")(row); err != nil { + t.Fatal(err) + } + _, present := row["seq_id"] + if present != restore { + t.Fatalf("restore=%v seq_id present=%v", restore, present) + } + if string(row["content"]) != `"exact content"` || string(row["id"]) != `"chunk-src"` { + t.Fatal("content identity was changed") + } + before, err := canonicalRowDigest(row) + if err != nil { + t.Fatal(err) + } + row["content"] = json.RawMessage(`"corrupted"`) + after, err := canonicalRowDigest(row) + if err != nil { + t.Fatal(err) + } + if before == after { + t.Fatal("content corruption was hidden") + } + } +} diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index 8ca974f..f640c14 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -97,6 +97,8 @@ type ProductionLibraryTarget struct { // reusedResources 是"写前已存在、且已证明可复用"的资源身份:写入阶段跳过它们, // 既不重写行也不重传对象。PlanWriteSet 与 Apply 在同一个作业里串行执行。 reusedResources map[string]bool + // 回滚必须重放目标原始序列;普通发布由目标数据库分配本地序列。 + preserveChunkSequence bool } // resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写, @@ -399,6 +401,9 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if err := t.retargetRow(table, row); err != nil { return err } + if table == "chunks" && !t.preserveChunkSequence { + delete(row, "seq_id") + } if columns == nil { columns = make([]string, 0, len(row)) for key := range row { @@ -411,6 +416,14 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if table == "resources" { statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb) ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table) + } else if table == "chunks" && !t.preserveChunkSequence { + // Omitting the column (not inserting NULL) invokes the target default. + quoted := make([]string, len(columns)) + for i, column := range columns { + quoted[i] = pgx.Identifier{column}.Sanitize() + } + projection := strings.Join(quoted, ", ") + statement = fmt.Sprintf(`INSERT INTO chunks (%s) SELECT %s FROM jsonb_populate_record(NULL::chunks, $1::jsonb)`, projection, projection) } else { statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)`, table, table) @@ -460,7 +473,17 @@ type resourceLocation struct { // rowTransform 是"写入时对行做过的变换"的统一入口:校验要用同一份,否则摘要比的是 // 变换前的行、目标里是变换后的行,永远不一致。范围映射同样必须两边一致。 func (t *ProductionLibraryTarget) rowTransform(table string) func(map[string]json.RawMessage) error { - return func(row map[string]json.RawMessage) error { return t.retargetRow(table, row) } + return func(row map[string]json.RawMessage) error { + if err := t.retargetRow(table, row); err != nil { + return err + } + // seq_id is destination-local, not content identity. Full-universe + // rollback verification still compares the original value exactly. + if table == "chunks" && !t.preserveChunkSequence { + delete(row, "seq_id") + } + return nil + } } func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { @@ -839,6 +862,9 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致", backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) } + previous := t.preserveChunkSequence + t.preserveChunkSequence = true + defer func() { t.preserveChunkSequence = previous }() for _, stage := range libraryrelease.Stages() { if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil { return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err) diff --git a/scripts/test-scope.json b/scripts/test-scope.json index 341e284..2cc870b 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -193,6 +193,11 @@ "test": "TestScheduledSendRequiresConfirmationAndSendsExactlyOnce", "reason": "需要一个一次性 PostgreSQL 与测试主密钥(KG_WEEKLY_MAIL_TEST_DSN/KG_WEEKLY_MAIL_TEST_KEY),CI 不提供;本批已在隔离库真实跑通并通过" }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryChunkSequenceIsolated", + "reason": "需要隔离 kg_library_test_* 数据库、随机 ES 索引与真实 S3;本机已实跑跨库序列碰撞、目标分配和原始序列逐行恢复,CI 不提供这些服务" + }, { "package": "knowledge-governance/internal/releaseadapter/v071", "test": "TestLibraryRollbackKeepsPreExistingTargetDataIsolated",