Files
knowledge-governance/internal/releaseadapter/v071/library_search.go
T
姚章浩 7d4ffdbfe5
build / build (push) Successful in 1m15s
fix(v071): 全新目标实例的空索引按空基线处理
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。

现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。

试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
2026-10-29 06:17:26 +08:00

166 lines
6.3 KiB
Go

package v071
import (
"context"
"encoding/json"
"errors"
"net/http"
"sort"
"time"
)
// ExportLibrary uses one Elasticsearch point-in-time rather than offset paging
// against a changing index. A PIT does not freeze PostgreSQL or object storage;
// the coordinator must still prove cross-lane consistency before publication.
func (e *SearchExporter) ExportLibrary(ctx context.Context, identity SearchIdentity, maxDocuments int) (snapshot SearchSnapshot, resultErr error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" || maxDocuments < 1 {
return SearchSnapshot{}, errors.New("invalid library search scope or limit")
}
inspection, err := e.inspectMapping(ctx)
if errors.Is(err, errIndexMissing) {
// 空生产基线:索引不存在 = 零文档、无映射可比。
// 调用方必须再确认数据库也为空,不能只看这一条。
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
if errors.Is(err, errScopeFieldsMissing) {
// 全新实例先用空映射建索引:没有范围字段、又确实 0 篇文档时就是空基线;
// 有文档却看不出归属必须报错(不能当成空库覆盖)。
empty, probeErr := e.emptyScopeIndex(ctx)
if probeErr != nil {
return SearchSnapshot{}, probeErr
}
if empty {
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
return SearchSnapshot{}, err
}
if err != nil {
return SearchSnapshot{}, err
}
var opened struct {
ID string `json:"id"`
}
if err = e.requestJSON(ctx, http.MethodPost, "/"+e.index+"/_pit?keep_alive=2m&allow_partial_search_results=false", nil, &opened); err != nil {
return SearchSnapshot{}, err
}
if opened.ID == "" {
return SearchSnapshot{}, errors.New("missing Elasticsearch PIT")
}
pit := opened.ID
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
var result struct {
Succeeded bool `json:"succeeded"`
}
err := e.requestJSON(cleanup, http.MethodDelete, "/_pit", map[string]any{"id": pit}, &result)
if (err != nil || !result.Succeeded) && resultErr == nil {
snapshot = SearchSnapshot{}
resultErr = errors.New("close library snapshot PIT")
}
}()
filter := []any{map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}}
if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
var cursor json.RawMessage
documents := make([]SearchDocument, 0)
seen := map[string]bool{}
var total *int64
for {
body := map[string]any{"pit": map[string]any{"id": pit, "keep_alive": "2m"}, "size": e.pageSize, "sort": []any{"_shard_doc"}, "track_total_hits": true, "query": map[string]any{"bool": map[string]any{"filter": filter}}}
if cursor != nil {
body["search_after"] = cursor
}
var page struct {
PITID string `json:"pit_id"`
TimedOut *bool `json:"timed_out"`
Shards struct {
Total int `json:"total"`
Successful int `json:"successful"`
Failed int `json:"failed"`
} `json:"_shards"`
Hits struct {
Total struct {
Value *int64 `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
Sort json.RawMessage `json:"sort"`
} `json:"hits"`
} `json:"hits"`
}
if err = e.requestJSON(ctx, http.MethodPost, "/_search?allow_partial_search_results=false", body, &page); err != nil {
return SearchSnapshot{}, err
}
if page.PITID != "" {
pit = page.PITID
}
if page.TimedOut == nil || *page.TimedOut || page.Shards.Total < 1 || page.Shards.Successful != page.Shards.Total || page.Shards.Failed != 0 || page.Hits.Total.Value == nil || page.Hits.Total.Relation != "eq" {
return SearchSnapshot{}, errors.New("incomplete Elasticsearch library snapshot")
}
if total == nil {
total = page.Hits.Total.Value
}
if *total < 0 || *total > int64(maxDocuments) || *total != *page.Hits.Total.Value {
return SearchSnapshot{}, errors.New("Elasticsearch library total changed or exceeds limit")
}
for _, hit := range page.Hits.Hits {
var scope struct {
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(hit.Source, &scope) != nil || !validSearchIdentifier(scope.KnowledgeID) || !validSearchIdentifier(hit.ID) || seen[hit.ID] {
return SearchSnapshot{}, errors.New("invalid or duplicate library search identity")
}
expected := identity
expected.KnowledgeID = scope.KnowledgeID
if !searchHitMatches(hit.Source, expected) {
return SearchSnapshot{}, errors.New("cross-scope library search document")
}
source, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("invalid library search data")
}
seen[hit.ID] = true
documents = append(documents, SearchDocument{ID: hit.ID, Source: source})
}
if int64(len(documents)) > *total {
return SearchSnapshot{}, errors.New("library search total mismatch")
}
if len(page.Hits.Hits) > 0 {
next := page.Hits.Hits[len(page.Hits.Hits)-1].Sort
var values []json.RawMessage
if json.Unmarshal(next, &values) != nil || len(values) == 0 || string(next) == string(cursor) {
return SearchSnapshot{}, errors.New("library search cursor did not advance")
}
cursor = next
}
if int64(len(documents)) == *total {
break
}
if len(page.Hits.Hits) == 0 {
return SearchSnapshot{}, errors.New("library search snapshot truncated")
}
}
finalMapping, err := e.inspectMapping(ctx)
if err != nil || finalMapping.digest != inspection.digest {
return SearchSnapshot{}, errors.New("library search mapping changed")
}
sort.Slice(documents, func(i, j int) bool { return documents[i].ID < documents[j].ID })
sha, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: sha}, nil
}