fix(release): 分配目标分块序列并保持原值恢复
build / build (push) Successful in 1m26s

This commit is contained in:
姚章浩
2026-10-31 06:08:05 +08:00
parent a50cd73423
commit ee9d078794
5 changed files with 117 additions and 1 deletions
+7
View File
@@ -1,5 +1,12 @@
# Changelog
## 开发快照(跨库分块序列分配)
### Fixed
- 发布分块时由目标数据库分配本地 `seq_id`,不再复制源库序列而撞上其它知识库的唯一索引;文档和分块 UUID 不变。
- 失败恢复仍重放预备份中的原始序列,并保留整实例逐行核验;发布内容校验仅排除分块的本地序列,不放宽正文和其它字段校验。
## 开发快照(云执行器授权配置与启动核验)
### Fixed
@@ -0,0 +1,43 @@
package v071
import (
"knowledge-governance/internal/libraryrelease"
"testing"
)
// Cross-database snapshots may reuse a local serial value already owned by an
// unrelated KB. UUIDs remain stable; the destination allocates its own serial.
func TestLibraryChunkSequenceIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
if _, err := fx.pool.Exec(fx.ctx, `ALTER TABLE chunks ADD COLUMN seq_id bigserial UNIQUE`); err != nil {
t.Fatal(err)
}
var occupied int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT seq_id FROM chunks WHERE knowledge_base_id=$1 LIMIT 1`, wsOtherKB).Scan(&occupied); err != nil {
t.Fatal(err)
}
before := universeDump(t, fx.pool)
snapshot := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) { parts["chunks"][0]["seq_id"] = occupied })
target := fx.target(t)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), target, fx.backups(t, target), nil)
if outcome.State != "succeeded" {
t.Fatalf("cross-database publish must allocate local sequence: %+v", outcome)
}
var allocated int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT seq_id FROM chunks WHERE id='chunk-src'`).Scan(&allocated); err != nil {
t.Fatal(err)
}
if allocated == occupied || allocated <= 0 {
t.Fatalf("invalid destination serial %d (occupied %d)", allocated, occupied)
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: snapshot}); err != nil {
t.Fatalf("content verification: %v", err)
}
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
t.Fatalf("restore must preserve original serial values: %v", err)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("restored universe differs: %s", diff)
}
}
@@ -0,0 +1,35 @@
package v071
import (
"encoding/json"
"testing"
)
func TestChunkSequenceProjectionKeepsContent(t *testing.T) {
for _, restore := range []bool{false, true} {
target := &ProductionLibraryTarget{scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}, snapshotTenant: 10002, snapshotKB: "kb-source", preserveChunkSequence: restore}
row := map[string]json.RawMessage{"id": json.RawMessage(`"chunk-src"`), "tenant_id": json.RawMessage(`10002`), "knowledge_base_id": json.RawMessage(`"kb-source"`), "seq_id": json.RawMessage(`7`), "content": json.RawMessage(`"exact content"`)}
if err := target.rowTransform("chunks")(row); err != nil {
t.Fatal(err)
}
_, present := row["seq_id"]
if present != restore {
t.Fatalf("restore=%v seq_id present=%v", restore, present)
}
if string(row["content"]) != `"exact content"` || string(row["id"]) != `"chunk-src"` {
t.Fatal("content identity was changed")
}
before, err := canonicalRowDigest(row)
if err != nil {
t.Fatal(err)
}
row["content"] = json.RawMessage(`"corrupted"`)
after, err := canonicalRowDigest(row)
if err != nil {
t.Fatal(err)
}
if before == after {
t.Fatal("content corruption was hidden")
}
}
}
@@ -97,6 +97,8 @@ type ProductionLibraryTarget struct {
// reusedResources 是"写前已存在、且已证明可复用"的资源身份:写入阶段跳过它们,
// 既不重写行也不重传对象。PlanWriteSet 与 Apply 在同一个作业里串行执行。
reusedResources map[string]bool
// 回滚必须重放目标原始序列;普通发布由目标数据库分配本地序列。
preserveChunkSequence bool
}
// resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写,
@@ -399,6 +401,9 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
if err := t.retargetRow(table, row); err != nil {
return err
}
if table == "chunks" && !t.preserveChunkSequence {
delete(row, "seq_id")
}
if columns == nil {
columns = make([]string, 0, len(row))
for key := range row {
@@ -411,6 +416,14 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
if table == "resources" {
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)
ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table)
} else if table == "chunks" && !t.preserveChunkSequence {
// Omitting the column (not inserting NULL) invokes the target default.
quoted := make([]string, len(columns))
for i, column := range columns {
quoted[i] = pgx.Identifier{column}.Sanitize()
}
projection := strings.Join(quoted, ", ")
statement = fmt.Sprintf(`INSERT INTO chunks (%s) SELECT %s FROM jsonb_populate_record(NULL::chunks, $1::jsonb)`, projection, projection)
} else {
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)`,
table, table)
@@ -460,7 +473,17 @@ type resourceLocation struct {
// rowTransform 是"写入时对行做过的变换"的统一入口:校验要用同一份,否则摘要比的是
// 变换前的行、目标里是变换后的行,永远不一致。范围映射同样必须两边一致。
func (t *ProductionLibraryTarget) rowTransform(table string) func(map[string]json.RawMessage) error {
return func(row map[string]json.RawMessage) error { return t.retargetRow(table, row) }
return func(row map[string]json.RawMessage) error {
if err := t.retargetRow(table, row); err != nil {
return err
}
// seq_id is destination-local, not content identity. Full-universe
// rollback verification still compares the original value exactly.
if table == "chunks" && !t.preserveChunkSequence {
delete(row, "seq_id")
}
return nil
}
}
func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
@@ -839,6 +862,9 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致",
backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID)
}
previous := t.preserveChunkSequence
t.preserveChunkSequence = true
defer func() { t.preserveChunkSequence = previous }()
for _, stage := range libraryrelease.Stages() {
if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil {
return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err)
+5
View File
@@ -193,6 +193,11 @@
"test": "TestScheduledSendRequiresConfirmationAndSendsExactlyOnce",
"reason": "需要一个一次性 PostgreSQL 与测试主密钥(KG_WEEKLY_MAIL_TEST_DSN/KG_WEEKLY_MAIL_TEST_KEY),CI 不提供;本批已在隔离库真实跑通并通过"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryChunkSequenceIsolated",
"reason": "需要隔离 kg_library_test_* 数据库、随机 ES 索引与真实 S3;本机已实跑跨库序列碰撞、目标分配和原始序列逐行恢复,CI 不提供这些服务"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryRollbackKeepsPreExistingTargetDataIsolated",