7d4ffdbfe5
build / build (push) Successful in 1m15s
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。
现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。
试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
166 lines
6.3 KiB
Go
166 lines
6.3 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
// ExportLibrary uses one Elasticsearch point-in-time rather than offset paging
|
|
// against a changing index. A PIT does not freeze PostgreSQL or object storage;
|
|
// the coordinator must still prove cross-lane consistency before publication.
|
|
func (e *SearchExporter) ExportLibrary(ctx context.Context, identity SearchIdentity, maxDocuments int) (snapshot SearchSnapshot, resultErr error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" || maxDocuments < 1 {
|
|
return SearchSnapshot{}, errors.New("invalid library search scope or limit")
|
|
}
|
|
inspection, err := e.inspectMapping(ctx)
|
|
if errors.Is(err, errIndexMissing) {
|
|
// 空生产基线:索引不存在 = 零文档、无映射可比。
|
|
// 调用方必须再确认数据库也为空,不能只看这一条。
|
|
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
|
|
if digestErr != nil {
|
|
return SearchSnapshot{}, digestErr
|
|
}
|
|
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
|
|
}
|
|
if errors.Is(err, errScopeFieldsMissing) {
|
|
// 全新实例先用空映射建索引:没有范围字段、又确实 0 篇文档时就是空基线;
|
|
// 有文档却看不出归属必须报错(不能当成空库覆盖)。
|
|
empty, probeErr := e.emptyScopeIndex(ctx)
|
|
if probeErr != nil {
|
|
return SearchSnapshot{}, probeErr
|
|
}
|
|
if empty {
|
|
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
|
|
if digestErr != nil {
|
|
return SearchSnapshot{}, digestErr
|
|
}
|
|
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
|
|
}
|
|
return SearchSnapshot{}, err
|
|
}
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
var opened struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err = e.requestJSON(ctx, http.MethodPost, "/"+e.index+"/_pit?keep_alive=2m&allow_partial_search_results=false", nil, &opened); err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
if opened.ID == "" {
|
|
return SearchSnapshot{}, errors.New("missing Elasticsearch PIT")
|
|
}
|
|
pit := opened.ID
|
|
defer func() {
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
var result struct {
|
|
Succeeded bool `json:"succeeded"`
|
|
}
|
|
err := e.requestJSON(cleanup, http.MethodDelete, "/_pit", map[string]any{"id": pit}, &result)
|
|
if (err != nil || !result.Succeeded) && resultErr == nil {
|
|
snapshot = SearchSnapshot{}
|
|
resultErr = errors.New("close library snapshot PIT")
|
|
}
|
|
}()
|
|
filter := []any{map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}}
|
|
if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil {
|
|
filter = append(filter, clause)
|
|
}
|
|
var cursor json.RawMessage
|
|
documents := make([]SearchDocument, 0)
|
|
seen := map[string]bool{}
|
|
var total *int64
|
|
for {
|
|
body := map[string]any{"pit": map[string]any{"id": pit, "keep_alive": "2m"}, "size": e.pageSize, "sort": []any{"_shard_doc"}, "track_total_hits": true, "query": map[string]any{"bool": map[string]any{"filter": filter}}}
|
|
if cursor != nil {
|
|
body["search_after"] = cursor
|
|
}
|
|
var page struct {
|
|
PITID string `json:"pit_id"`
|
|
TimedOut *bool `json:"timed_out"`
|
|
Shards struct {
|
|
Total int `json:"total"`
|
|
Successful int `json:"successful"`
|
|
Failed int `json:"failed"`
|
|
} `json:"_shards"`
|
|
Hits struct {
|
|
Total struct {
|
|
Value *int64 `json:"value"`
|
|
Relation string `json:"relation"`
|
|
} `json:"total"`
|
|
Hits []struct {
|
|
ID string `json:"_id"`
|
|
Source json.RawMessage `json:"_source"`
|
|
Sort json.RawMessage `json:"sort"`
|
|
} `json:"hits"`
|
|
} `json:"hits"`
|
|
}
|
|
if err = e.requestJSON(ctx, http.MethodPost, "/_search?allow_partial_search_results=false", body, &page); err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
if page.PITID != "" {
|
|
pit = page.PITID
|
|
}
|
|
if page.TimedOut == nil || *page.TimedOut || page.Shards.Total < 1 || page.Shards.Successful != page.Shards.Total || page.Shards.Failed != 0 || page.Hits.Total.Value == nil || page.Hits.Total.Relation != "eq" {
|
|
return SearchSnapshot{}, errors.New("incomplete Elasticsearch library snapshot")
|
|
}
|
|
if total == nil {
|
|
total = page.Hits.Total.Value
|
|
}
|
|
if *total < 0 || *total > int64(maxDocuments) || *total != *page.Hits.Total.Value {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch library total changed or exceeds limit")
|
|
}
|
|
for _, hit := range page.Hits.Hits {
|
|
var scope struct {
|
|
KnowledgeID string `json:"knowledge_id"`
|
|
}
|
|
if json.Unmarshal(hit.Source, &scope) != nil || !validSearchIdentifier(scope.KnowledgeID) || !validSearchIdentifier(hit.ID) || seen[hit.ID] {
|
|
return SearchSnapshot{}, errors.New("invalid or duplicate library search identity")
|
|
}
|
|
expected := identity
|
|
expected.KnowledgeID = scope.KnowledgeID
|
|
if !searchHitMatches(hit.Source, expected) {
|
|
return SearchSnapshot{}, errors.New("cross-scope library search document")
|
|
}
|
|
source, err := canonicalJSON(hit.Source)
|
|
if err != nil {
|
|
return SearchSnapshot{}, errors.New("invalid library search data")
|
|
}
|
|
seen[hit.ID] = true
|
|
documents = append(documents, SearchDocument{ID: hit.ID, Source: source})
|
|
}
|
|
if int64(len(documents)) > *total {
|
|
return SearchSnapshot{}, errors.New("library search total mismatch")
|
|
}
|
|
if len(page.Hits.Hits) > 0 {
|
|
next := page.Hits.Hits[len(page.Hits.Hits)-1].Sort
|
|
var values []json.RawMessage
|
|
if json.Unmarshal(next, &values) != nil || len(values) == 0 || string(next) == string(cursor) {
|
|
return SearchSnapshot{}, errors.New("library search cursor did not advance")
|
|
}
|
|
cursor = next
|
|
}
|
|
if int64(len(documents)) == *total {
|
|
break
|
|
}
|
|
if len(page.Hits.Hits) == 0 {
|
|
return SearchSnapshot{}, errors.New("library search snapshot truncated")
|
|
}
|
|
}
|
|
finalMapping, err := e.inspectMapping(ctx)
|
|
if err != nil || finalMapping.digest != inspection.digest {
|
|
return SearchSnapshot{}, errors.New("library search mapping changed")
|
|
}
|
|
sort.Slice(documents, func(i, j int) bool { return documents[i].ID < documents[j].ID })
|
|
sha, err := searchDocumentsDigest(documents)
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: sha}, nil
|
|
}
|