Files
knowledge-governance/internal/releaseadapter/v071/library_target_ids.go
T
姚章浩 0d9091e464
build / build (push) Successful in 1m41s
fix(library): 目标隔离映射兼容旧计划与真实回归
空 mode 计划不映射;索引 source_id 按语义映射;relation 按字段表解析。
隔离回归按持久映射 ID 断言,DS02 故障注入真实命中。
2026-11-04 12:40:12 +08:00

497 lines
15 KiB
Go

package v071
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"knowledge-governance/internal/librarysnapshot"
)
// 目标隔离 ID 映射。
//
// 为什么必须映射:同一实例里可以存在多个知识库,源库的 UUID 主键被原样写进第二个
// 目标库时会与旧库的行撞主键(父证据:2 knowledges / 2 chunks / 23 wiki_pages /
// 17 wiki_folders 已存在于旧 KB),而写入前的"跨范围冲突"检查会(正确地)拒绝。
// 关掉检查等于让两个库共享身份,删除/恢复/校验都会互相误伤。
//
// 因此身份必须由"目标范围 + 表 + 源身份"确定性派生:
// - 同一目标重复同步/重试/跨版本 → 同一 ID(幂等、可恢复);
// - 不同目标 KB、不同表 → 不同 ID(隔离);
// - 源归档字节不变,映射只存在于目标侧派生视图。
//
// 资源表(resources)是多知识库共享的字节载体,ID 不重映射,沿用既有复用/归属证明;
// 绑定行的 owner_id 与自身 id 则按目标映射。
type targetIDMapper struct {
tenant uint64
kb string
ids map[string]string
reverse map[string]string
known map[string]map[string]bool
owner map[string]string
sequence int
// 不重映射的表:资源行是跨库共享对象,改了 ID 就等于复制一份字节,
// 现有复用契约要求"内容相同 + 来源可审计"才复用,不能靠换 ID 绕过。
skip map[string]bool
}
const targetIDNamespace = "knowledge-governance/target-identity/v1"
var errUnmappedTargetReference = errors.New("unmapped target reference")
func newTargetIDMapper(tenant uint64, kb string, _ any) *targetIDMapper {
return &targetIDMapper{
tenant: tenant, kb: kb,
ids: map[string]string{}, reverse: map[string]string{},
known: map[string]map[string]bool{}, owner: map[string]string{},
skip: map[string]bool{"resources": true},
}
}
func (m *targetIDMapper) mapID(table, source string) string {
if source == "" || m.skip[table] {
return source
}
key := table + "\x00" + source
if mapped, ok := m.ids[key]; ok {
return mapped
}
sum := sha256.Sum256([]byte(targetIDNamespace + "\x00" + strconv.FormatUint(m.tenant, 10) + "\x00" + m.kb + "\x00" + table + "\x00" + source))
var raw [16]byte
copy(raw[:], sum[:16])
raw[6] = raw[6]&0x0f | 0x50 // version 5 semantics: name-based, deterministic
raw[8] = raw[8]&0x3f | 0x80 // RFC 4122 variant
mapped := fmt.Sprintf("%x-%x-%x-%x-%x", raw[0:4], raw[4:6], raw[6:8], raw[8:10], raw[10:16])
if previous, exists := m.reverse[table+"\x00"+mapped]; exists && previous != source {
// 位移后的 SHA-256 也撞同一张表是密码学事件;宁可停住也不静默合并两个身份。
panic("target id collision for " + table)
}
m.ids[key] = mapped
m.reverse[table+"\x00"+mapped] = source
m.sequence++
return mapped
}
func (m *targetIDMapper) registerSourceID(table, id string) {
if id == "" || m.skip[table] {
return
}
// 本地自增主键(如 wiki_log_entries.id)不是跨库身份:既不能当引用匹配,
// 也不能参与"残留源 ID"判断,否则普通整数列会被误报。
if contract, ok := migrationIdentityContracts[table]; ok && (contract.Kind == "local-bigint" || contract.Kind == "tuple") {
// 本地自增/复合元组都不是可被别的表引用的实体身份:
// 关系表的两个分量必须由 knowledges/knowledge_tags 自己登记。
return
}
if m.known[table] == nil {
m.known[table] = map[string]bool{}
}
m.known[table][id] = true
if _, exists := m.owner[id]; !exists {
m.owner[id] = table
}
}
// remapRow 只改"身份与引用"字段;正文、向量、顺序、重复项一律原样保留。
func (m *targetIDMapper) remapRow(table string, row map[string]json.RawMessage) error {
if m.skip[table] {
return nil
}
switch table {
case "knowledges":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
case "chunks":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
if err := m.remapScalar(row, "knowledge_id", "knowledges"); err != nil {
return err
}
if err := m.remapScalar(row, "tag_id", "knowledge_tags"); err != nil {
return err
}
for _, column := range []string{"parent_chunk_id", "pre_chunk_id", "next_chunk_id"} {
if err := m.remapScalar(row, column, "chunks"); err != nil {
return err
}
}
for _, column := range []string{"relation_chunks", "indirect_relation_chunks"} {
// 这两列是"分块引用":目标表由字段语义决定(chunks),不由字符串
// 在别的表里也出现来决定;跨表同 ID 是合法数据,不是歧义。
if err := m.remapJSONExact(row, column, "chunks"); err != nil {
return err
}
}
case "knowledge_tags":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
case "knowledge_tag_relations":
if err := m.remapScalar(row, "knowledge_id", "knowledges"); err != nil {
return err
}
if err := m.remapScalar(row, "tag_id", "knowledge_tags"); err != nil {
return err
}
case "wiki_folders":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
if err := m.remapScalar(row, "parent_id", table); err != nil {
return err
}
case "wiki_pages":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
if err := m.remapScalar(row, "folder_id", "wiki_folders"); err != nil {
return err
}
if err := m.remapRefArray(row, "chunk_refs", "chunks"); err != nil {
return err
}
if err := m.remapSourceRefs(row); err != nil {
return err
}
case "wiki_page_issues":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
if err := m.remapRefArray(row, "suspected_knowledge_ids", "knowledges"); err != nil {
return err
}
for _, column := range []string{"page_id", "knowledge_id", "chunk_id"} {
if err := m.remapScalar(row, column, m.tableFor(column)); err != nil {
return err
}
}
case "wiki_log_entries":
// id 是目标本地的 bigint,由既有 LocalIDs 分配;历史日志可能引用已删除文档,
// 这类引用只做"已知才换",不能当作悬挂拒绝(否则整库同步被历史日志卡死)。
m.remapScalarIfKnown(row, "knowledge_id", "knowledges")
if err := m.remapScalar(row, "page_id", "wiki_pages"); err != nil {
return err
}
case "resource_bindings":
if err := m.remapScalar(row, "id", table); err != nil {
return err
}
var ownerType string
if raw, ok := row["owner_type"]; ok {
_ = json.Unmarshal(raw, &ownerType)
}
switch ownerType {
case "knowledge":
if err := m.remapScalar(row, "owner_id", "knowledges"); err != nil {
return err
}
case "chunk":
if err := m.remapScalar(row, "owner_id", "chunks"); err != nil {
return err
}
default:
return fmt.Errorf("resource_bindings: unknown owner_type %q", ownerType)
}
// resource_id 不映射:资源是共享字节,绑定只换归属。
}
return m.rejectUnmapped(table, row)
}
func (m *targetIDMapper) tableFor(column string) string {
switch column {
case "page_id":
return "wiki_pages"
case "knowledge_id":
return "knowledges"
case "chunk_id":
return "chunks"
}
return column
}
func (m *targetIDMapper) remapScalar(row map[string]json.RawMessage, column, table string) error {
raw, ok := row[column]
if !ok {
return nil
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
return fmt.Errorf("%s: unreadable %s", table, column)
}
if value == "" {
return nil
}
if !m.known[table][value] {
return fmt.Errorf("%w: %s.%s=%s", errUnmappedTargetReference, table, column, value)
}
encoded, _ := json.Marshal(m.mapID(table, value))
row[column] = encoded
return nil
}
func (m *targetIDMapper) remapScalarIfKnown(row map[string]json.RawMessage, column, table string) {
raw, ok := row[column]
if !ok {
return
}
var value string
if err := json.Unmarshal(raw, &value); err != nil || value == "" || !m.known[table][value] {
return
}
encoded, _ := json.Marshal(m.mapID(table, value))
row[column] = encoded
}
func (m *targetIDMapper) remapRefArray(row map[string]json.RawMessage, column, table string) error {
raw, ok := row[column]
if !ok {
return nil
}
var values []string
if err := json.Unmarshal(raw, &values); err != nil {
return nil
}
for i, value := range values {
if !m.known[table][value] {
return fmt.Errorf("%w: %s.%s=%s", errUnmappedTargetReference, table, column, value)
}
values[i] = m.mapID(table, value)
}
encoded, _ := json.Marshal(values)
row[column] = encoded
return nil
}
func (m *targetIDMapper) remapSourceRefs(row map[string]json.RawMessage) error {
raw, ok := row["source_refs"]
if !ok {
return nil
}
var refs []string
if err := json.Unmarshal(raw, &refs); err != nil {
return nil
}
for i, ref := range refs {
id, suffix := ref, ""
if cut := strings.IndexByte(ref, '|'); cut >= 0 {
id, suffix = ref[:cut], ref[cut:]
}
if !m.known["knowledges"][id] {
return fmt.Errorf("%w: wiki_pages.source_refs=%s", errUnmappedTargetReference, id)
}
refs[i] = m.mapID("knowledges", id) + suffix
}
encoded, _ := json.Marshal(refs)
row["source_refs"] = encoded
return nil
}
// remapJSONExact 只在指定 JSON 列里替换"整体等于已知身份"的字符串;
// 不做子串替换(正文里出现同形状文本不是引用)。
func (m *targetIDMapper) remapJSONExact(row map[string]json.RawMessage, column, table string) error {
raw, ok := row[column]
if !ok {
return nil
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber() // >2^53 的整数必须原样保留,不能经 float64
var value any
if err := decoder.Decode(&value); err != nil {
return fmt.Errorf("%s.%s 不可解析", table, column)
}
changed := false
var walk func(any) (any, error)
walk = func(v any) (any, error) {
switch typed := v.(type) {
case string:
if m.known[table][typed] {
changed = true
return m.mapID(table, typed), nil
}
for other, ids := range m.known {
if other == table || m.skip[other] {
continue
}
if ids[typed] {
return nil, fmt.Errorf("%w: %s.%s references %s entity %s", errUnmappedTargetReference, table, column, other, typed)
}
}
return typed, nil
case []any:
for i := range typed {
mapped, err := walk(typed[i])
if err != nil {
return nil, err
}
typed[i] = mapped
}
return typed, nil
case map[string]any:
for key := range typed {
mapped, err := walk(typed[key])
if err != nil {
return nil, err
}
typed[key] = mapped
}
return typed, nil
}
return v, nil
}
mapped, err := walk(value)
if err != nil {
return err
}
if !changed {
return nil
}
encoded, err := json.Marshal(mapped)
if err != nil {
return err
}
row[column] = encoded
return nil
}
// rejectUnmapped 保证"改写后不再残留任何已知源身份",并拒绝无法判定归属的悬挂引用。
func (m *targetIDMapper) rejectUnmapped(table string, row map[string]json.RawMessage) error {
body := map[string]bool{"content": true, "summary": true, "description": true, "physical_path": true,
"pages_affected": true, "slug": true, "metadata": true, "page_metadata": true, "title": true, "aliases": true, "in_links": true, "out_links": true}
for column, raw := range row {
if body[column] {
continue
}
if leftover := m.findKnown(string(raw)); leftover != "" {
return fmt.Errorf("%w: %s.%s still references source id %s", errUnmappedTargetReference, table, column, leftover)
}
}
return nil
}
func rawStrings(row map[string]json.RawMessage) []string {
out := make([]string, 0, len(row))
for _, raw := range row {
out = append(out, string(raw))
}
return out
}
func (m *targetIDMapper) findKnown(value string) string {
if table, ok := m.owner[value]; ok && !m.skip[table] {
return value
}
// JSON 字符串里嵌着的 id 也要查(只查完整 id 子串,不做模糊匹配)。
for id, table := range m.owner {
if m.skip[table] || len(id) < 16 || !strings.Contains(value, id) {
continue
}
// 允许 it 出现在正文类字段里的情况已由 rejectUnmapped 排除;这里只对
// 明确带引用的字段生效,因此命中即说明漏改。
return id
}
return ""
}
func (m *targetIDMapper) remapIndexDocument(id string, source json.RawMessage) (string, json.RawMessage, error) {
var fields map[string]json.RawMessage
if err := json.Unmarshal(source, &fields); err != nil {
return "", nil, errors.New("index document source is not an object")
}
for column, table := range map[string]string{
"chunk_id": "chunks", "knowledge_id": "knowledges", "tag_id": "knowledge_tags",
} {
raw, ok := fields[column]
if !ok {
continue
}
var value string
if err := json.Unmarshal(raw, &value); err != nil || value == "" {
continue
}
if !m.known[table][value] {
return "", nil, fmt.Errorf("%w: index %s=%s", errUnmappedTargetReference, column, value)
}
encoded, _ := json.Marshal(m.mapID(table, value))
fields[column] = encoded
}
// source_id 在不同索引器版本里既可能是 chunk id,也可能是外部文档 id:
// 只有能确定是分块/文档身份时才映射,opaque 外部 id 原样保留。
if raw, ok := fields["source_id"]; ok {
var value string
if json.Unmarshal(raw, &value) == nil && value != "" {
switch {
case m.known["chunks"][value]:
encoded, _ := json.Marshal(m.mapID("chunks", value))
fields["source_id"] = encoded
case m.known["knowledges"][value]:
encoded, _ := json.Marshal(m.mapID("knowledges", value))
fields["source_id"] = encoded
}
}
}
encoded, err := json.Marshal(fields)
if err != nil {
return "", nil, err
}
return m.mapID("index_chunks", id), encoded, nil
}
// targetIDMapperFromArchive 从目标快照里登记源身份,供引用映射与残留检查使用。
func targetIDMapperFromArchive(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, tenant uint64, kb string) (*targetIDMapper, error) {
mapper := newTargetIDMapper(tenant, kb, nil)
for _, part := range manifest.Parts {
switch part.Kind {
case "database", "wiki":
reader, err := archive.OpenPart(part)
if err != nil {
return nil, err
}
var rows []map[string]json.RawMessage
err = json.NewDecoder(reader).Decode(&rows)
reader.Close()
if err != nil {
return nil, err
}
for _, row := range rows {
if identity, err := parseRowIdentity(part.Name, row); err == nil {
for _, value := range identity.Values {
mapper.registerSourceID(part.Name, value)
}
}
// 引用值不登记为"实体存在":悬挂引用必须由主键表缺行来暴露。
}
case "index":
// 索引文档的 chunk_id/knowledge_id 同样只是引用,不能反过来证明实体存在。
reader, err := archive.OpenPart(part)
if err != nil {
return nil, err
}
documents, err := librarysnapshot.ReadIndexDocuments(reader)
reader.Close()
if err != nil {
return nil, err
}
_ = documents
}
}
return mapper, nil
}
// sortedSourceIDs 只用于稳定报告/调试,不参与映射。
func (m *targetIDMapper) sortedSourceIDs() []string {
out := make([]string, 0, len(m.ids))
for key := range m.ids {
out = append(out, strings.ReplaceAll(key, "\x00", "/"))
}
sort.Strings(out)
return out
}