Files
knowledge-governance/internal/releaseadapter/v071/library_search.go
T
姚章浩 ac3b395102
build / build (push) Successful in 1m21s
fix(library): 对齐真实 WeKnora Schema + 入口级真实服务验收
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设,
以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。

- knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。
- 写入顺序按真实外键;无代理键表用内容摘要核对。
- 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。
- 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。
- 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。
- finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。
- 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
2026-10-28 17:53:43 +08:00

150 lines
5.7 KiB
Go

package v071
import (
"context"
"encoding/json"
"errors"
"net/http"
"sort"
"time"
)
// ExportLibrary uses one Elasticsearch point-in-time rather than offset paging
// against a changing index. A PIT does not freeze PostgreSQL or object storage;
// the coordinator must still prove cross-lane consistency before publication.
func (e *SearchExporter) ExportLibrary(ctx context.Context, identity SearchIdentity, maxDocuments int) (snapshot SearchSnapshot, resultErr error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" || maxDocuments < 1 {
return SearchSnapshot{}, errors.New("invalid library search scope or limit")
}
inspection, err := e.inspectMapping(ctx)
if errors.Is(err, errIndexMissing) {
// 空生产基线:索引不存在 = 零文档、无映射可比。
// 调用方必须再确认数据库也为空,不能只看这一条。
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
if err != nil {
return SearchSnapshot{}, err
}
var opened struct {
ID string `json:"id"`
}
if err = e.requestJSON(ctx, http.MethodPost, "/"+e.index+"/_pit?keep_alive=2m&allow_partial_search_results=false", nil, &opened); err != nil {
return SearchSnapshot{}, err
}
if opened.ID == "" {
return SearchSnapshot{}, errors.New("missing Elasticsearch PIT")
}
pit := opened.ID
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
var result struct {
Succeeded bool `json:"succeeded"`
}
err := e.requestJSON(cleanup, http.MethodDelete, "/_pit", map[string]any{"id": pit}, &result)
if (err != nil || !result.Succeeded) && resultErr == nil {
snapshot = SearchSnapshot{}
resultErr = errors.New("close library snapshot PIT")
}
}()
filter := []any{map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}}
if inspection.fields.tenant != "" {
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.tenant: identity.TenantID}})
}
var cursor json.RawMessage
documents := make([]SearchDocument, 0)
seen := map[string]bool{}
var total *int64
for {
body := map[string]any{"pit": map[string]any{"id": pit, "keep_alive": "2m"}, "size": e.pageSize, "sort": []any{"_shard_doc"}, "track_total_hits": true, "query": map[string]any{"bool": map[string]any{"filter": filter}}}
if cursor != nil {
body["search_after"] = cursor
}
var page struct {
PITID string `json:"pit_id"`
TimedOut *bool `json:"timed_out"`
Shards struct {
Total int `json:"total"`
Successful int `json:"successful"`
Failed int `json:"failed"`
} `json:"_shards"`
Hits struct {
Total struct {
Value *int64 `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
Sort json.RawMessage `json:"sort"`
} `json:"hits"`
} `json:"hits"`
}
if err = e.requestJSON(ctx, http.MethodPost, "/_search?allow_partial_search_results=false", body, &page); err != nil {
return SearchSnapshot{}, err
}
if page.PITID != "" {
pit = page.PITID
}
if page.TimedOut == nil || *page.TimedOut || page.Shards.Total < 1 || page.Shards.Successful != page.Shards.Total || page.Shards.Failed != 0 || page.Hits.Total.Value == nil || page.Hits.Total.Relation != "eq" {
return SearchSnapshot{}, errors.New("incomplete Elasticsearch library snapshot")
}
if total == nil {
total = page.Hits.Total.Value
}
if *total < 0 || *total > int64(maxDocuments) || *total != *page.Hits.Total.Value {
return SearchSnapshot{}, errors.New("Elasticsearch library total changed or exceeds limit")
}
for _, hit := range page.Hits.Hits {
var scope struct {
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(hit.Source, &scope) != nil || !validSearchIdentifier(scope.KnowledgeID) || !validSearchIdentifier(hit.ID) || seen[hit.ID] {
return SearchSnapshot{}, errors.New("invalid or duplicate library search identity")
}
expected := identity
expected.KnowledgeID = scope.KnowledgeID
if !searchHitMatches(hit.Source, expected) {
return SearchSnapshot{}, errors.New("cross-scope library search document")
}
source, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("invalid library search data")
}
seen[hit.ID] = true
documents = append(documents, SearchDocument{ID: hit.ID, Source: source})
}
if int64(len(documents)) > *total {
return SearchSnapshot{}, errors.New("library search total mismatch")
}
if len(page.Hits.Hits) > 0 {
next := page.Hits.Hits[len(page.Hits.Hits)-1].Sort
var values []json.RawMessage
if json.Unmarshal(next, &values) != nil || len(values) == 0 || string(next) == string(cursor) {
return SearchSnapshot{}, errors.New("library search cursor did not advance")
}
cursor = next
}
if int64(len(documents)) == *total {
break
}
if len(page.Hits.Hits) == 0 {
return SearchSnapshot{}, errors.New("library search snapshot truncated")
}
}
finalMapping, err := e.inspectMapping(ctx)
if err != nil || finalMapping.digest != inspection.digest {
return SearchSnapshot{}, errors.New("library search mapping changed")
}
sort.Slice(documents, func(i, j int) bool { return documents[i].ID < documents[j].ID })
sha, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: sha}, nil
}