Files
knowledge-governance/internal/releaseadapter/v071/library_archive.go
T
姚章浩 c14dd01f08
build / build (push) Successful in 1m18s
fix(library): 封存读资源失败带上资源行与底层原因
真实封存卡在 `read library resource` 时没有任何可判断信息:对象缺失、桶写错、
宿主机解析不到端点都会走到同一个字符串。现在错误带资源行 id 与底层原因
(不含凭据),本轮就是用它在候选栈定位到"源对象被上一次 B02 演练删掉后没真正写回"。
2026-10-29 13:32:55 +08:00

318 lines
12 KiB
Go

package v071
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"knowledge-governance/internal/librarysnapshot"
)
// ArchiveLibraryCandidate persists independent recovery bytes. "Candidate" is
// intentional: matching IDs/counts does not prove shared Wiki semantic retraction
// or a common cross-system cut. No release worker consumes it as authorization.
func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archive, scope string, identity PostgresIdentity, contract librarysnapshot.Compatibility, pg PostgresSnapshot, es SearchSnapshot, objects ObjectReader, createdAt time.Time) (string, error) {
if archive == nil || objects == nil || identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" {
return "", errors.New("invalid library candidate configuration")
}
if err := validateLibraryCandidate(identity, pg, es); err != nil {
return "", err
}
// 空生产基线:索引不存在时没有映射可比,契约仍按目标快照记录;
// 只要这次导出的文档是零条,就不存在"拿错映射"的风险。
emptyIndex := len(es.Documents) == 0
if !emptyIndex && es.MappingSHA256 != contract.IndexMappingSHA256 {
return "", errors.New("library mapping contract mismatch")
}
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, Format: librarysnapshot.Format, Scope: scope, TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, CreatedAt: createdAt, Compatibility: contract}
put := func(kind, name string, count int64, data []byte) error {
if err := ctx.Err(); err != nil {
return err
}
p, err := archive.Put(bytes.NewReader(data))
if err != nil {
return err
}
p.Kind = kind
p.Name = name
p.Count = count
manifest.Parts = append(manifest.Parts, p)
return nil
}
for _, table := range pg.Tables {
rows := table.Rows
if rows == nil {
rows = []json.RawMessage{}
}
data, err := json.Marshal(rows)
if err != nil {
return "", errors.New("encode library rows")
}
kind := "database"
if strings.HasPrefix(table.Name, "wiki_") {
kind = "wiki"
}
if err = put(kind, table.Name, int64(len(rows)), data); err != nil {
return "", err
}
}
documents := es.Documents
if documents == nil {
documents = []SearchDocument{}
}
data, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode library index")
}
if err = put("index", "chunks", int64(len(documents)), data); err != nil {
return "", err
}
for _, raw := range pg.Table("resources").Rows {
if err = ctx.Err(); err != nil {
return "", err
}
var resource struct {
ID string `json:"id"`
Provider string `json:"provider"`
PhysicalPath string `json:"physical_path"`
Size *int64 `json:"size"`
ContentHash string `json:"content_hash"`
}
if json.Unmarshal(raw, &resource) != nil || resource.Size == nil || *resource.Size < 0 || !validObjectProvider(resource.Provider) || !validObjectKey(resource.PhysicalPath) || resource.ContentHash != "" && !validLowerSHA256(resource.ContentHash) {
return "", errors.New("invalid library resource metadata")
}
reader, err := objects.Open(ctx, resource.Provider, resource.PhysicalPath)
if err != nil {
// 只写"read library resource"会让整条封存停在无信息的失败上:带上行 id
// 与底层原因(不含凭据),才能判断是对象缺失、桶错还是端点解析不到。
return "", fmt.Errorf("read library resource %s: %w", resource.ID, err)
}
part, readErr := archive.Put(reader)
closeErr := reader.Close()
if readErr != nil || closeErr != nil {
return "", errors.New("archive library resource")
}
if part.Bytes != *resource.Size || resource.ContentHash != "" && part.SHA256 != resource.ContentHash {
return "", errors.New("library resource integrity mismatch")
}
part.Kind = "resource"
part.Name = resource.ID
part.Count = 1
manifest.Parts = append(manifest.Parts, part)
}
if err = ctx.Err(); err != nil {
return "", err
}
return archive.Seal(manifest)
}
type libraryRow struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
ChunkID string `json:"chunk_id"`
TagID string `json:"tag_id"`
ResourceID string `json:"resource_id"`
Handle string `json:"handle"`
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ChunkStatus int `json:"-"`
SourceID string `json:"source_id"`
Questions map[string]string `json:"-"`
}
func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es SearchSnapshot) error {
required := map[string]bool{"resources": true}
for _, entry := range libraryContentQueries {
required[entry.name] = true
}
tables := map[string]map[string]libraryRow{}
for _, table := range pg.Tables {
if !required[table.Name] || tables[table.Name] != nil || table.RowCount != len(table.Rows) || table.SHA256 != newTableSnapshot(table.Name, table.Rows).SHA256 {
return errors.New("invalid library table inventory")
}
rows := map[string]libraryRow{}
tables[table.Name] = rows
for _, raw := range table.Rows {
// Log identities are BIGSERIAL. All other owned rows use string IDs.
if table.Name == "wiki_log_entries" {
var row struct {
ID json.Number `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
}
if json.Unmarshal(raw, &row) != nil || row.ID == "" || row.TenantID != identity.TenantID || row.KnowledgeBaseID != identity.KnowledgeBaseID {
return errors.New("invalid Wiki log scope")
}
if _, ok := rows[string(row.ID)]; ok {
return errors.New("duplicate Wiki log identity")
}
rows[string(row.ID)] = libraryRow{}
continue
}
var row libraryRow
if json.Unmarshal(raw, &row) != nil {
return errors.New("invalid library content row")
}
// Status is table-specific: chunks use integers, Wiki pages/issues
// use strings. Decode chunk metadata only for its owning table.
if table.Name == "chunks" {
var metadata struct {
Status int `json:"status"`
Metadata struct {
GeneratedQuestions []struct {
ID string `json:"id"`
Question string `json:"question"`
} `json:"generated_questions"`
} `json:"metadata"`
}
if json.Unmarshal(raw, &metadata) != nil {
return errors.New("invalid library chunk status")
}
row.ChunkStatus = metadata.Status
row.Questions = map[string]string{}
for _, question := range metadata.Metadata.GeneratedQuestions {
sourceID := row.ID + "-" + question.ID
if row.ChunkType != "text" || !validSearchIdentifier(question.ID) || strings.TrimSpace(question.Question) == "" || row.Questions[sourceID] != "" {
return errors.New("invalid library generated question metadata")
}
row.Questions[sourceID] = question.Question
}
}
if table.Name == "knowledge_tag_relations" {
if !validSearchIdentifier(row.KnowledgeID) || !validSearchIdentifier(row.TagID) {
return errors.New("invalid library tag relation")
}
row.ID = row.KnowledgeID + "/" + row.TagID
} else {
if !validSearchIdentifier(row.ID) || row.TenantID != identity.TenantID {
return errors.New("invalid library row identity")
}
if table.Name != "resources" && table.Name != "resource_bindings" && row.KnowledgeBaseID != identity.KnowledgeBaseID {
return errors.New("cross-scope library row")
}
}
if _, ok := rows[row.ID]; ok {
return errors.New("duplicate library row")
}
rows[row.ID] = row
}
}
for name := range required {
if tables[name] == nil {
return errors.New("missing library table")
}
}
for _, chunk := range tables["chunks"] {
if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok {
return errors.New("orphan library chunk")
}
}
for _, relation := range tables["knowledge_tag_relations"] {
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
_, hasTag := tables["knowledge_tags"][relation.TagID]
if !hasKnowledge || !hasTag {
return errors.New("orphan library tag relation")
}
}
for _, binding := range tables["resource_bindings"] {
_, hasResource := tables["resources"][binding.ResourceID]
hasOwner := false
if binding.OwnerType == "knowledge" {
_, hasOwner = tables["knowledges"][binding.OwnerID]
} else if binding.OwnerType == "chunk" {
_, hasOwner = tables["chunks"][binding.OwnerID]
}
if !hasResource || !hasOwner {
return errors.New("orphan library resource binding")
}
}
handles := map[string]bool{}
for _, resource := range tables["resources"] {
if len(resource.Handle) != 22 || handles[resource.Handle] {
return errors.New("invalid resource handle")
}
handles[resource.Handle] = true
}
var bodies []json.RawMessage
for _, name := range []string{"knowledges", "chunks", "wiki_pages"} {
bodies = append(bodies, pg.Table(name).Rows...)
}
for _, handle := range referencedResourceHandles(bodies) {
if !handles[handle] {
return errors.New("unresolved library resource")
}
}
sha, err := searchDocumentsDigest(es.Documents)
if err != nil || sha != es.DocumentsSHA256 {
return errors.New("invalid library index digest")
}
// 索引不存在(空生产基线)时没有映射可比:只允许"零文档 + 空映射"这种组合,
// 有文档却没有映射仍然是错误。
if !validLowerSHA256(es.MappingSHA256) && (es.MappingSHA256 != "" || len(es.Documents) != 0) {
return errors.New("invalid library index digest")
}
seen := map[string]bool{}
indexedChunks := map[string]bool{}
indexedQuestions := map[string]map[string]bool{}
for _, document := range es.Documents {
var row libraryRow
if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] {
return errors.New("invalid library search document")
}
seen[document.ID] = true
chunk, ok := tables["chunks"][row.ChunkID]
if !ok || row.KnowledgeID != chunk.KnowledgeID {
return errors.New("library index does not match database chunks")
}
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
return errors.New("cross-scope library index")
}
if row.SourceID != "" && row.SourceID != row.ChunkID {
question, exists := chunk.Questions[row.SourceID]
if !exists || question != row.Content {
return errors.New("library question index does not match persisted metadata")
}
if indexedQuestions[row.ChunkID] == nil {
indexedQuestions[row.ChunkID] = map[string]bool{}
}
indexedQuestions[row.ChunkID][row.SourceID] = true
} else {
// Preserve the captured artifact; transient generation context is not
// reconstructible from PG. This is not semantic acceptance or release approval.
if strings.TrimSpace(row.Content) == "" {
return errors.New("empty library index content")
}
indexedChunks[row.ChunkID] = true
}
}
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
// records, so compare identities rather than counts. Stored-only parent
// chunks intentionally have no index and must not be rejected as missing.
// Text ingestion also leaves indexed chunks at the legacy default status
// zero. Without an archived no-index policy, absence is unverified, not
// proof that indexing was disabled. Parent chunks are context-only.
for id, chunk := range tables["chunks"] {
for sourceID := range chunk.Questions {
if !indexedQuestions[id][sourceID] {
return errors.New("missing library generated question index")
}
}
if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] {
return errors.New("unverified legacy library chunk index coverage")
}
if chunk.ChunkStatus == 2 && !indexedChunks[id] {
return errors.New("missing indexed library chunk")
}
}
return nil
}