Files
knowledge-governance/internal/releaseadapter/v071/library_archive.go
T
姚章浩 eaa388f76b
build / build (push) Successful in 1m42s
fix(release): 取消封存正文索引与父类型推断规则
按用户明确要求,治理封存不再替 WeKnora 判断正文块“应该”有独立索引:
删除 status/chunk_type/parent/mixed/flat 推导的 needOriginal、mixedDocs
及覆盖门槛;删除 parent 类型搭配限制,仅保留父引用存在与同 knowledge。
capture/preflight/archive/load/replay/writeset 共用校验同步更新,移除已无
用途的 worker 永久失败分类。

保留真实导出范围、PG/ES 原始正文/问题 metadata/vector、tenant/KB/
knowledge/chunk 身份、清单摘要、资源字节、Wiki/问题索引 source_id 及真实
悬空引用等完整性检查;不修改源端,不补数据/索引。旧测试改为原样捕获正例,
新增 status0/2 缺正文索引和父类型不匹配的接受回归,以及 archive/readback
逐行/逐记录比对。

验证:v071、cmd/worker、Go 全量、make verify 17 项。
2026-11-05 18:24:36 +08:00

335 lines
13 KiB
Go

package v071
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"knowledge-governance/internal/librarysnapshot"
)
// ArchiveLibraryCandidate persists independent recovery bytes. "Candidate" is
// intentional: matching IDs/counts does not prove shared Wiki semantic retraction
// or a common cross-system cut. No release worker consumes it as authorization.
func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archive, scope string, identity PostgresIdentity, contract librarysnapshot.Compatibility, pg PostgresSnapshot, es SearchSnapshot, objects ObjectReader, createdAt time.Time) (string, error) {
if archive == nil || objects == nil || identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" {
return "", errors.New("invalid library candidate configuration")
}
if err := validateLibraryCandidate(identity, pg, es); err != nil {
return "", err
}
return archiveLibraryRows(ctx, archive, scope, identity, contract, pg, es, objects, createdAt)
}
// Private: callers must distinguish a validated source candidate from raw
// target before-images. Never run source semantic filtering on recovery bytes.
func archiveLibraryRows(ctx context.Context, archive *librarysnapshot.Archive, scope string, identity PostgresIdentity, contract librarysnapshot.Compatibility, pg PostgresSnapshot, es SearchSnapshot, objects ObjectReader, createdAt time.Time) (string, error) {
// 空生产基线:索引不存在时没有映射可比,契约仍按目标快照记录;
// 只要这次导出的文档是零条,就不存在"拿错映射"的风险。
emptyIndex := len(es.Documents) == 0
if !emptyIndex && es.MappingSHA256 != contract.IndexMappingSHA256 {
return "", errors.New("library mapping contract mismatch")
}
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, Format: librarysnapshot.Format, Scope: scope, TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, CreatedAt: createdAt, Compatibility: contract}
put := func(kind, name string, count int64, data []byte) error {
if err := ctx.Err(); err != nil {
return err
}
p, err := archive.Put(bytes.NewReader(data))
if err != nil {
return err
}
p.Kind = kind
p.Name = name
p.Count = count
manifest.Parts = append(manifest.Parts, p)
return nil
}
if pg.Catalog != nil {
raw, err := json.Marshal(pg.Catalog)
if err != nil {
return "", err
}
if err = put("schema", "migration-catalog", 1, raw); err != nil {
return "", err
}
}
for _, table := range pg.Tables {
rows := table.Rows
if rows == nil {
rows = []json.RawMessage{}
}
data, err := json.Marshal(rows)
if err != nil {
return "", errors.New("encode library rows")
}
kind := "database"
if strings.HasPrefix(table.Name, "wiki_") {
kind = "wiki"
}
if err = put(kind, table.Name, int64(len(rows)), data); err != nil {
return "", err
}
}
documents := es.Documents
if documents == nil {
documents = []SearchDocument{}
}
data, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode library index")
}
if err = put("index", "chunks", int64(len(documents)), data); err != nil {
return "", err
}
for _, raw := range pg.Table("resources").Rows {
if err = ctx.Err(); err != nil {
return "", err
}
var resource struct {
ID string `json:"id"`
Provider string `json:"provider"`
PhysicalPath string `json:"physical_path"`
Size *int64 `json:"size"`
ContentHash string `json:"content_hash"`
}
if json.Unmarshal(raw, &resource) != nil || resource.Size == nil || *resource.Size < 0 || !validObjectProvider(resource.Provider) || !validObjectKey(resource.PhysicalPath) || resource.ContentHash != "" && !validLowerSHA256(resource.ContentHash) {
return "", errors.New("invalid library resource metadata")
}
reader, err := objects.Open(ctx, resource.Provider, resource.PhysicalPath)
if err != nil {
// 只写"read library resource"会让整条封存停在无信息的失败上:带上行 id
// 与底层原因(不含凭据),才能判断是对象缺失、桶错还是端点解析不到。
return "", fmt.Errorf("read library resource %s: %w", resource.ID, err)
}
part, readErr := archive.Put(reader)
closeErr := reader.Close()
if readErr != nil || closeErr != nil {
return "", errors.New("archive library resource")
}
if part.Bytes != *resource.Size || resource.ContentHash != "" && part.SHA256 != resource.ContentHash {
return "", errors.New("library resource integrity mismatch")
}
part.Kind = "resource"
part.Name = resource.ID
part.Count = 1
manifest.Parts = append(manifest.Parts, part)
}
if err = ctx.Err(); err != nil {
return "", err
}
return archive.Seal(manifest)
}
type libraryRow struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
ChunkID string `json:"chunk_id"`
TagID string `json:"tag_id"`
ResourceID string `json:"resource_id"`
Handle string `json:"handle"`
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ParentChunkID string `json:"parent_chunk_id"`
SourceID string `json:"source_id"`
Questions map[string]string `json:"-"`
}
func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es SearchSnapshot) error {
required := map[string]bool{"resources": true}
for _, entry := range libraryContentQueries {
required[entry.name] = true
}
tables := map[string]map[string]libraryRow{}
for _, table := range pg.Tables {
if !required[table.Name] || tables[table.Name] != nil || table.RowCount != len(table.Rows) || table.SHA256 != newTableSnapshot(table.Name, table.Rows).SHA256 {
return errors.New("invalid library table inventory")
}
rows := map[string]libraryRow{}
tables[table.Name] = rows
for _, raw := range table.Rows {
// Log identities are BIGSERIAL. All other owned rows use string IDs.
if table.Name == "wiki_log_entries" {
var row struct {
ID json.Number `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
}
if json.Unmarshal(raw, &row) != nil || row.ID == "" || row.TenantID != identity.TenantID || row.KnowledgeBaseID != identity.KnowledgeBaseID {
return errors.New("invalid Wiki log scope")
}
if _, ok := rows[string(row.ID)]; ok {
return errors.New("duplicate Wiki log identity")
}
rows[string(row.ID)] = libraryRow{}
continue
}
var row libraryRow
if json.Unmarshal(raw, &row) != nil {
return errors.New("invalid library content row")
}
// Decode chunk metadata only for its owning table; question metadata
// remains an integrity check, while index coverage is source-owned.
if table.Name == "chunks" {
var metadata struct {
Metadata struct {
GeneratedQuestions []struct {
ID string `json:"id"`
Question string `json:"question"`
} `json:"generated_questions"`
} `json:"metadata"`
}
if json.Unmarshal(raw, &metadata) != nil {
return errors.New("invalid library chunk status")
}
row.Questions = map[string]string{}
for _, question := range metadata.Metadata.GeneratedQuestions {
sourceID := row.ID + "-" + question.ID
if row.ChunkType != "text" || !validSearchIdentifier(question.ID) || strings.TrimSpace(question.Question) == "" || row.Questions[sourceID] != "" {
return errors.New("invalid library generated question metadata")
}
row.Questions[sourceID] = question.Question
}
}
if table.Name == "knowledge_tag_relations" {
if !validSearchIdentifier(row.KnowledgeID) || !validSearchIdentifier(row.TagID) {
return errors.New("invalid library tag relation")
}
row.ID = row.KnowledgeID + "/" + row.TagID
} else {
if !validSearchIdentifier(row.ID) || row.TenantID != identity.TenantID {
return errors.New("invalid library row identity")
}
if table.Name != "resources" && table.Name != "resource_bindings" && row.KnowledgeBaseID != identity.KnowledgeBaseID {
return errors.New("cross-scope library row")
}
}
if _, ok := rows[row.ID]; ok {
return errors.New("duplicate library row")
}
rows[row.ID] = row
}
}
for name := range required {
if tables[name] == nil {
return errors.New("missing library table")
}
}
if err := validateLibraryStructuredReferences(pg); err != nil {
return err
}
for _, chunk := range tables["chunks"] {
if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok {
return errors.New("orphan library chunk")
}
// A parent link must point at a real parent chunk of the same document;
// standalone text has an empty field and is never forced into a relation.
if chunk.ParentChunkID != "" {
parent, ok := tables["chunks"][chunk.ParentChunkID]
if !ok || parent.KnowledgeID != chunk.KnowledgeID {
return errors.New("invalid library parent chunk reference")
}
}
}
for _, relation := range tables["knowledge_tag_relations"] {
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
_, hasTag := tables["knowledge_tags"][relation.TagID]
if !hasKnowledge || !hasTag {
return errors.New("orphan library tag relation")
}
}
for _, binding := range tables["resource_bindings"] {
_, hasResource := tables["resources"][binding.ResourceID]
hasOwner := false
if binding.OwnerType == "knowledge" {
_, hasOwner = tables["knowledges"][binding.OwnerID]
} else if binding.OwnerType == "chunk" {
_, hasOwner = tables["chunks"][binding.OwnerID]
}
if !hasResource || !hasOwner {
return errors.New("orphan library resource binding")
}
}
handles := map[string]bool{}
for _, resource := range tables["resources"] {
if len(resource.Handle) != 22 || handles[resource.Handle] {
return errors.New("invalid resource handle")
}
handles[resource.Handle] = true
}
var bodies []json.RawMessage
for _, table := range pg.Tables {
bodies = append(bodies, table.Rows...)
}
references, err := libraryResourceHandles(bodies)
if err != nil {
return err
}
for _, handle := range references {
if !handles[handle] {
return errors.New("unresolved library resource")
}
}
sha, err := searchDocumentsDigest(es.Documents)
if err != nil || sha != es.DocumentsSHA256 {
return errors.New("invalid library index digest")
}
// 索引不存在(空生产基线)时没有映射可比:只允许"零文档 + 空映射"这种组合,
// 有文档却没有映射仍然是错误。
if !validLowerSHA256(es.MappingSHA256) && (es.MappingSHA256 != "" || len(es.Documents) != 0) {
return errors.New("invalid library index digest")
}
seen := map[string]bool{}
indexedQuestions := map[string]map[string]bool{}
for _, document := range es.Documents {
var row libraryRow
if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] {
return errors.New("invalid library search document")
}
seen[document.ID] = true
chunk, ok := tables["chunks"][row.ChunkID]
if !ok || row.KnowledgeID != chunk.KnowledgeID {
return errors.New("library index does not match database chunks")
}
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
return errors.New("cross-scope library index")
}
if row.SourceID != "" && row.SourceID != row.ChunkID {
question, exists := chunk.Questions[row.SourceID]
if !exists || question != row.Content {
return errors.New("library question index does not match persisted metadata")
}
if indexedQuestions[row.ChunkID] == nil {
indexedQuestions[row.ChunkID] = map[string]bool{}
}
indexedQuestions[row.ChunkID][row.SourceID] = true
} else {
// Preserve the captured artifact; transient generation context is not
// reconstructible from PG. This is not semantic acceptance or release approval.
if strings.TrimSpace(row.Content) == "" {
return errors.New("empty library index content")
}
}
}
// 原文索引是源端自己的选择:治理不根据 status/type/parent 推断某个
// chunk “必须有”基础正文记录。问题索引仍按 source_id 精确核对,防止
// 把问题记录冒充正文或漏掉问题;正文、向量按实际导出集合原样保存。
for id, chunk := range tables["chunks"] {
for sourceID := range chunk.Questions {
if !indexedQuestions[id][sourceID] {
return errors.New("missing library generated question index")
}
}
}
return nil
}