0d9091e464
build / build (push) Successful in 1m41s
空 mode 计划不映射;索引 source_id 按语义映射;relation 按字段表解析。 隔离回归按持久映射 ID 断言,DS02 故障注入真实命中。
497 lines
15 KiB
Go
497 lines
15 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// 目标隔离 ID 映射。
|
|
//
|
|
// 为什么必须映射:同一实例里可以存在多个知识库,源库的 UUID 主键被原样写进第二个
|
|
// 目标库时会与旧库的行撞主键(父证据:2 knowledges / 2 chunks / 23 wiki_pages /
|
|
// 17 wiki_folders 已存在于旧 KB),而写入前的"跨范围冲突"检查会(正确地)拒绝。
|
|
// 关掉检查等于让两个库共享身份,删除/恢复/校验都会互相误伤。
|
|
//
|
|
// 因此身份必须由"目标范围 + 表 + 源身份"确定性派生:
|
|
// - 同一目标重复同步/重试/跨版本 → 同一 ID(幂等、可恢复);
|
|
// - 不同目标 KB、不同表 → 不同 ID(隔离);
|
|
// - 源归档字节不变,映射只存在于目标侧派生视图。
|
|
//
|
|
// 资源表(resources)是多知识库共享的字节载体,ID 不重映射,沿用既有复用/归属证明;
|
|
// 绑定行的 owner_id 与自身 id 则按目标映射。
|
|
type targetIDMapper struct {
|
|
tenant uint64
|
|
kb string
|
|
ids map[string]string
|
|
reverse map[string]string
|
|
known map[string]map[string]bool
|
|
owner map[string]string
|
|
sequence int
|
|
|
|
// 不重映射的表:资源行是跨库共享对象,改了 ID 就等于复制一份字节,
|
|
// 现有复用契约要求"内容相同 + 来源可审计"才复用,不能靠换 ID 绕过。
|
|
skip map[string]bool
|
|
}
|
|
|
|
const targetIDNamespace = "knowledge-governance/target-identity/v1"
|
|
|
|
var errUnmappedTargetReference = errors.New("unmapped target reference")
|
|
|
|
func newTargetIDMapper(tenant uint64, kb string, _ any) *targetIDMapper {
|
|
return &targetIDMapper{
|
|
tenant: tenant, kb: kb,
|
|
ids: map[string]string{}, reverse: map[string]string{},
|
|
known: map[string]map[string]bool{}, owner: map[string]string{},
|
|
skip: map[string]bool{"resources": true},
|
|
}
|
|
}
|
|
|
|
func (m *targetIDMapper) mapID(table, source string) string {
|
|
if source == "" || m.skip[table] {
|
|
return source
|
|
}
|
|
key := table + "\x00" + source
|
|
if mapped, ok := m.ids[key]; ok {
|
|
return mapped
|
|
}
|
|
sum := sha256.Sum256([]byte(targetIDNamespace + "\x00" + strconv.FormatUint(m.tenant, 10) + "\x00" + m.kb + "\x00" + table + "\x00" + source))
|
|
var raw [16]byte
|
|
copy(raw[:], sum[:16])
|
|
raw[6] = raw[6]&0x0f | 0x50 // version 5 semantics: name-based, deterministic
|
|
raw[8] = raw[8]&0x3f | 0x80 // RFC 4122 variant
|
|
mapped := fmt.Sprintf("%x-%x-%x-%x-%x", raw[0:4], raw[4:6], raw[6:8], raw[8:10], raw[10:16])
|
|
if previous, exists := m.reverse[table+"\x00"+mapped]; exists && previous != source {
|
|
// 位移后的 SHA-256 也撞同一张表是密码学事件;宁可停住也不静默合并两个身份。
|
|
panic("target id collision for " + table)
|
|
}
|
|
m.ids[key] = mapped
|
|
m.reverse[table+"\x00"+mapped] = source
|
|
m.sequence++
|
|
return mapped
|
|
}
|
|
|
|
func (m *targetIDMapper) registerSourceID(table, id string) {
|
|
if id == "" || m.skip[table] {
|
|
return
|
|
}
|
|
// 本地自增主键(如 wiki_log_entries.id)不是跨库身份:既不能当引用匹配,
|
|
// 也不能参与"残留源 ID"判断,否则普通整数列会被误报。
|
|
if contract, ok := migrationIdentityContracts[table]; ok && (contract.Kind == "local-bigint" || contract.Kind == "tuple") {
|
|
// 本地自增/复合元组都不是可被别的表引用的实体身份:
|
|
// 关系表的两个分量必须由 knowledges/knowledge_tags 自己登记。
|
|
return
|
|
}
|
|
if m.known[table] == nil {
|
|
m.known[table] = map[string]bool{}
|
|
}
|
|
m.known[table][id] = true
|
|
if _, exists := m.owner[id]; !exists {
|
|
m.owner[id] = table
|
|
}
|
|
}
|
|
|
|
// remapRow 只改"身份与引用"字段;正文、向量、顺序、重复项一律原样保留。
|
|
func (m *targetIDMapper) remapRow(table string, row map[string]json.RawMessage) error {
|
|
if m.skip[table] {
|
|
return nil
|
|
}
|
|
switch table {
|
|
case "knowledges":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
case "chunks":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapScalar(row, "knowledge_id", "knowledges"); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapScalar(row, "tag_id", "knowledge_tags"); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []string{"parent_chunk_id", "pre_chunk_id", "next_chunk_id"} {
|
|
if err := m.remapScalar(row, column, "chunks"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, column := range []string{"relation_chunks", "indirect_relation_chunks"} {
|
|
// 这两列是"分块引用":目标表由字段语义决定(chunks),不由字符串
|
|
// 在别的表里也出现来决定;跨表同 ID 是合法数据,不是歧义。
|
|
if err := m.remapJSONExact(row, column, "chunks"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
case "knowledge_tags":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
case "knowledge_tag_relations":
|
|
if err := m.remapScalar(row, "knowledge_id", "knowledges"); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapScalar(row, "tag_id", "knowledge_tags"); err != nil {
|
|
return err
|
|
}
|
|
case "wiki_folders":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapScalar(row, "parent_id", table); err != nil {
|
|
return err
|
|
}
|
|
case "wiki_pages":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapScalar(row, "folder_id", "wiki_folders"); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapRefArray(row, "chunk_refs", "chunks"); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapSourceRefs(row); err != nil {
|
|
return err
|
|
}
|
|
case "wiki_page_issues":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
if err := m.remapRefArray(row, "suspected_knowledge_ids", "knowledges"); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []string{"page_id", "knowledge_id", "chunk_id"} {
|
|
if err := m.remapScalar(row, column, m.tableFor(column)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
case "wiki_log_entries":
|
|
// id 是目标本地的 bigint,由既有 LocalIDs 分配;历史日志可能引用已删除文档,
|
|
// 这类引用只做"已知才换",不能当作悬挂拒绝(否则整库同步被历史日志卡死)。
|
|
m.remapScalarIfKnown(row, "knowledge_id", "knowledges")
|
|
if err := m.remapScalar(row, "page_id", "wiki_pages"); err != nil {
|
|
return err
|
|
}
|
|
case "resource_bindings":
|
|
if err := m.remapScalar(row, "id", table); err != nil {
|
|
return err
|
|
}
|
|
var ownerType string
|
|
if raw, ok := row["owner_type"]; ok {
|
|
_ = json.Unmarshal(raw, &ownerType)
|
|
}
|
|
switch ownerType {
|
|
case "knowledge":
|
|
if err := m.remapScalar(row, "owner_id", "knowledges"); err != nil {
|
|
return err
|
|
}
|
|
case "chunk":
|
|
if err := m.remapScalar(row, "owner_id", "chunks"); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
return fmt.Errorf("resource_bindings: unknown owner_type %q", ownerType)
|
|
}
|
|
// resource_id 不映射:资源是共享字节,绑定只换归属。
|
|
}
|
|
return m.rejectUnmapped(table, row)
|
|
}
|
|
|
|
func (m *targetIDMapper) tableFor(column string) string {
|
|
switch column {
|
|
case "page_id":
|
|
return "wiki_pages"
|
|
case "knowledge_id":
|
|
return "knowledges"
|
|
case "chunk_id":
|
|
return "chunks"
|
|
}
|
|
return column
|
|
}
|
|
|
|
func (m *targetIDMapper) remapScalar(row map[string]json.RawMessage, column, table string) error {
|
|
raw, ok := row[column]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var value string
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return fmt.Errorf("%s: unreadable %s", table, column)
|
|
}
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
if !m.known[table][value] {
|
|
return fmt.Errorf("%w: %s.%s=%s", errUnmappedTargetReference, table, column, value)
|
|
}
|
|
encoded, _ := json.Marshal(m.mapID(table, value))
|
|
row[column] = encoded
|
|
return nil
|
|
}
|
|
|
|
func (m *targetIDMapper) remapScalarIfKnown(row map[string]json.RawMessage, column, table string) {
|
|
raw, ok := row[column]
|
|
if !ok {
|
|
return
|
|
}
|
|
var value string
|
|
if err := json.Unmarshal(raw, &value); err != nil || value == "" || !m.known[table][value] {
|
|
return
|
|
}
|
|
encoded, _ := json.Marshal(m.mapID(table, value))
|
|
row[column] = encoded
|
|
}
|
|
|
|
func (m *targetIDMapper) remapRefArray(row map[string]json.RawMessage, column, table string) error {
|
|
raw, ok := row[column]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var values []string
|
|
if err := json.Unmarshal(raw, &values); err != nil {
|
|
return nil
|
|
}
|
|
for i, value := range values {
|
|
if !m.known[table][value] {
|
|
return fmt.Errorf("%w: %s.%s=%s", errUnmappedTargetReference, table, column, value)
|
|
}
|
|
values[i] = m.mapID(table, value)
|
|
}
|
|
encoded, _ := json.Marshal(values)
|
|
row[column] = encoded
|
|
return nil
|
|
}
|
|
|
|
func (m *targetIDMapper) remapSourceRefs(row map[string]json.RawMessage) error {
|
|
raw, ok := row["source_refs"]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var refs []string
|
|
if err := json.Unmarshal(raw, &refs); err != nil {
|
|
return nil
|
|
}
|
|
for i, ref := range refs {
|
|
id, suffix := ref, ""
|
|
if cut := strings.IndexByte(ref, '|'); cut >= 0 {
|
|
id, suffix = ref[:cut], ref[cut:]
|
|
}
|
|
if !m.known["knowledges"][id] {
|
|
return fmt.Errorf("%w: wiki_pages.source_refs=%s", errUnmappedTargetReference, id)
|
|
}
|
|
refs[i] = m.mapID("knowledges", id) + suffix
|
|
}
|
|
encoded, _ := json.Marshal(refs)
|
|
row["source_refs"] = encoded
|
|
return nil
|
|
}
|
|
|
|
// remapJSONExact 只在指定 JSON 列里替换"整体等于已知身份"的字符串;
|
|
// 不做子串替换(正文里出现同形状文本不是引用)。
|
|
func (m *targetIDMapper) remapJSONExact(row map[string]json.RawMessage, column, table string) error {
|
|
raw, ok := row[column]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
|
decoder.UseNumber() // >2^53 的整数必须原样保留,不能经 float64
|
|
var value any
|
|
if err := decoder.Decode(&value); err != nil {
|
|
return fmt.Errorf("%s.%s 不可解析", table, column)
|
|
}
|
|
changed := false
|
|
var walk func(any) (any, error)
|
|
walk = func(v any) (any, error) {
|
|
switch typed := v.(type) {
|
|
case string:
|
|
if m.known[table][typed] {
|
|
changed = true
|
|
return m.mapID(table, typed), nil
|
|
}
|
|
for other, ids := range m.known {
|
|
if other == table || m.skip[other] {
|
|
continue
|
|
}
|
|
if ids[typed] {
|
|
return nil, fmt.Errorf("%w: %s.%s references %s entity %s", errUnmappedTargetReference, table, column, other, typed)
|
|
}
|
|
}
|
|
return typed, nil
|
|
case []any:
|
|
for i := range typed {
|
|
mapped, err := walk(typed[i])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
typed[i] = mapped
|
|
}
|
|
return typed, nil
|
|
case map[string]any:
|
|
for key := range typed {
|
|
mapped, err := walk(typed[key])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
typed[key] = mapped
|
|
}
|
|
return typed, nil
|
|
}
|
|
return v, nil
|
|
}
|
|
mapped, err := walk(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !changed {
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(mapped)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
row[column] = encoded
|
|
return nil
|
|
}
|
|
|
|
// rejectUnmapped 保证"改写后不再残留任何已知源身份",并拒绝无法判定归属的悬挂引用。
|
|
func (m *targetIDMapper) rejectUnmapped(table string, row map[string]json.RawMessage) error {
|
|
body := map[string]bool{"content": true, "summary": true, "description": true, "physical_path": true,
|
|
"pages_affected": true, "slug": true, "metadata": true, "page_metadata": true, "title": true, "aliases": true, "in_links": true, "out_links": true}
|
|
for column, raw := range row {
|
|
if body[column] {
|
|
continue
|
|
}
|
|
if leftover := m.findKnown(string(raw)); leftover != "" {
|
|
return fmt.Errorf("%w: %s.%s still references source id %s", errUnmappedTargetReference, table, column, leftover)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func rawStrings(row map[string]json.RawMessage) []string {
|
|
out := make([]string, 0, len(row))
|
|
for _, raw := range row {
|
|
out = append(out, string(raw))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (m *targetIDMapper) findKnown(value string) string {
|
|
if table, ok := m.owner[value]; ok && !m.skip[table] {
|
|
return value
|
|
}
|
|
// JSON 字符串里嵌着的 id 也要查(只查完整 id 子串,不做模糊匹配)。
|
|
for id, table := range m.owner {
|
|
if m.skip[table] || len(id) < 16 || !strings.Contains(value, id) {
|
|
continue
|
|
}
|
|
// 允许 it 出现在正文类字段里的情况已由 rejectUnmapped 排除;这里只对
|
|
// 明确带引用的字段生效,因此命中即说明漏改。
|
|
return id
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (m *targetIDMapper) remapIndexDocument(id string, source json.RawMessage) (string, json.RawMessage, error) {
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(source, &fields); err != nil {
|
|
return "", nil, errors.New("index document source is not an object")
|
|
}
|
|
for column, table := range map[string]string{
|
|
"chunk_id": "chunks", "knowledge_id": "knowledges", "tag_id": "knowledge_tags",
|
|
} {
|
|
raw, ok := fields[column]
|
|
if !ok {
|
|
continue
|
|
}
|
|
var value string
|
|
if err := json.Unmarshal(raw, &value); err != nil || value == "" {
|
|
continue
|
|
}
|
|
if !m.known[table][value] {
|
|
return "", nil, fmt.Errorf("%w: index %s=%s", errUnmappedTargetReference, column, value)
|
|
}
|
|
encoded, _ := json.Marshal(m.mapID(table, value))
|
|
fields[column] = encoded
|
|
}
|
|
// source_id 在不同索引器版本里既可能是 chunk id,也可能是外部文档 id:
|
|
// 只有能确定是分块/文档身份时才映射,opaque 外部 id 原样保留。
|
|
if raw, ok := fields["source_id"]; ok {
|
|
var value string
|
|
if json.Unmarshal(raw, &value) == nil && value != "" {
|
|
switch {
|
|
case m.known["chunks"][value]:
|
|
encoded, _ := json.Marshal(m.mapID("chunks", value))
|
|
fields["source_id"] = encoded
|
|
case m.known["knowledges"][value]:
|
|
encoded, _ := json.Marshal(m.mapID("knowledges", value))
|
|
fields["source_id"] = encoded
|
|
}
|
|
}
|
|
}
|
|
encoded, err := json.Marshal(fields)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
return m.mapID("index_chunks", id), encoded, nil
|
|
}
|
|
|
|
// targetIDMapperFromArchive 从目标快照里登记源身份,供引用映射与残留检查使用。
|
|
func targetIDMapperFromArchive(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, tenant uint64, kb string) (*targetIDMapper, error) {
|
|
mapper := newTargetIDMapper(tenant, kb, nil)
|
|
for _, part := range manifest.Parts {
|
|
switch part.Kind {
|
|
case "database", "wiki":
|
|
reader, err := archive.OpenPart(part)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var rows []map[string]json.RawMessage
|
|
err = json.NewDecoder(reader).Decode(&rows)
|
|
reader.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, row := range rows {
|
|
if identity, err := parseRowIdentity(part.Name, row); err == nil {
|
|
for _, value := range identity.Values {
|
|
mapper.registerSourceID(part.Name, value)
|
|
}
|
|
}
|
|
// 引用值不登记为"实体存在":悬挂引用必须由主键表缺行来暴露。
|
|
}
|
|
case "index":
|
|
// 索引文档的 chunk_id/knowledge_id 同样只是引用,不能反过来证明实体存在。
|
|
reader, err := archive.OpenPart(part)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
documents, err := librarysnapshot.ReadIndexDocuments(reader)
|
|
reader.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_ = documents
|
|
}
|
|
}
|
|
return mapper, nil
|
|
}
|
|
|
|
// sortedSourceIDs 只用于稳定报告/调试,不参与映射。
|
|
func (m *targetIDMapper) sortedSourceIDs() []string {
|
|
out := make([]string, 0, len(m.ids))
|
|
for key := range m.ids {
|
|
out = append(out, strings.ReplaceAll(key, "\x00", "/"))
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|