Files
knowledge-governance/internal/releaseadapter/v071/search_write.go
T
dev cbc6b981c6
build / build (push) Successful in 1m56s
fix(library): 0.9.0 索引写入/归属校验不补 tenant_id,冲突可定位
- SearchIdentity 增加 NoTenantField,按目标 profile 决定是否注入 tenant_id
  (写入、归属预检、恢复搬运三条路径)
- 归属冲突与写集合不匹配写明具体字段,排查不用再猜
- 网关侧 acquire 冲突打印 plan/claim/owner 三项
2026-11-17 11:59:45 +08:00

240 lines
8.0 KiB
Go

package v071
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"knowledge-governance/internal/librarysnapshot"
)
// Count 用与导出相同的 scope filter 数生产索引里的文档,供校验对比。
func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdentity) (int64, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return 0, errors.New("invalid search identity")
}
inspection, err := exporter.inspectMapping(ctx)
if errors.Is(err, errScopeFieldsMissing) {
// 新建索引还没有范围字段:只有确实为空时才等于"这个范围 0 条"。
empty, probeErr := exporter.emptyScopeIndex(ctx)
if probeErr != nil {
return 0, probeErr
}
if empty {
return 0, nil
}
return 0, err
}
if err != nil {
return 0, err
}
// 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件
// 会把过滤器变成"匹配空字符串",永远数不到东西。
filter := make([]any, 0, 2)
if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}})
payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}}
var response struct {
Count int64 `json:"count"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_count", payload, &response); err != nil {
return 0, err
}
if response.Count < 0 {
return 0, errors.New("Elasticsearch returned a negative count")
}
return response.Count, nil
}
// PruneScope 删除该知识库范围内、但不在这份快照里的索引文档:
// 撤销必须真的从生产检索里消失,不能只做"新增/覆盖"。
func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchIdentity, keepIDs []string) (int, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return 0, err
}
keep := make(map[string]bool, len(keepIDs))
for _, id := range keepIDs {
keep[id] = true
}
stale := make([]string, 0, 16)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from, "size": exporter.pageSize, "_source": false, "sort": []any{"_doc"},
"query": map[string]any{"bool": map[string]any{"filter": scopeFilterFor(inspection.fields, identity)}},
}
var response struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
} `json:"hits"`
} `json:"hits"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return 0, err
}
for _, hit := range response.Hits.Hits {
if !keep[hit.ID] {
stale = append(stale, hit.ID)
}
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return 0, errors.New("scope has too many documents to prune safely")
}
}
if len(stale) == 0 {
return 0, nil
}
for start := 0; start < len(stale); start += 500 {
end := start + 500
if end > len(stale) {
end = len(stale)
}
payload := map[string]any{
"query": map[string]any{"bool": map[string]any{"filter": append(scopeFilterFor(inspection.fields, identity), map[string]any{"ids": map[string]any{"values": stale[start:end]}})}},
}
var response struct {
Deleted int `json:"deleted"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_delete_by_query?refresh=true", payload, &response); err != nil {
return 0, err
}
}
return len(stale), nil
}
// scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。
func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
filter := make([]any, 0, 2)
if clause := tenantClause(fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}})
return filter
}
// BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。
// 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。
func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error {
return exporter.bulkIndex(ctx, documents, identity, nil)
}
func (exporter *SearchExporter) bulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity, guards map[string]indexWriteGuard) error {
if len(documents) == 0 {
return nil
}
var body bytes.Buffer
for _, document := range documents {
if document.ID == "" {
return errors.New("archived index document without id")
}
payload := document.Source
var source map[string]json.RawMessage
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("archived index document is not an object")
}
// 0.9.0 的索引文档没有 tenant_id:补上去会让恢复后的文档与目标契约不一致。
if !identity.NoTenantField {
source["tenant_id"], _ = json.Marshal(identity.TenantID)
}
source["knowledge_base_id"], _ = json.Marshal(identity.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
payload = encoded
operation := "index"
metadata := map[string]any{"_index": exporter.index, "_id": document.ID}
if guards != nil {
guard, ok := guards[document.ID]
if !ok {
return errors.New("missing index write guard")
}
if !guard.Found {
operation = "create"
} else {
metadata["if_seq_no"] = *guard.SeqNo
metadata["if_primary_term"] = *guard.PrimaryTerm
}
}
action, err := json.Marshal(map[string]any{operation: metadata})
if err != nil {
return err
}
body.Write(action)
body.WriteByte('\n')
body.Write(payload)
body.WriteByte('\n')
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost,
exporter.baseURL.String()+"/_bulk?refresh=true", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/x-ndjson")
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch bulk request failed")
}
defer response.Body.Close()
raw, err := io.ReadAll(io.LimitReader(response.Body, maxSearchResponseBytes))
if err != nil {
return errors.New("Elasticsearch bulk response unreadable")
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("Elasticsearch bulk status %d", response.StatusCode)
}
var parsed struct {
Errors bool `json:"errors"`
Items []map[string]struct {
Status int `json:"status"`
Error any `json:"error"`
ID string `json:"_id"`
} `json:"items"`
}
if err := json.Unmarshal(raw, &parsed); err != nil {
return errors.New("Elasticsearch bulk response invalid")
}
if len(parsed.Items) == len(documents) {
for i, item := range parsed.Items {
if len(item) != 1 {
return errors.New("invalid bulk item result")
}
for operation, result := range item {
if (operation != "index" && operation != "create") || result.ID != documents[i].ID || result.Status < 200 || result.Status >= 300 || result.Error != nil {
return fmt.Errorf("Elasticsearch item outcome not confirmed: %s status %d", documents[i].ID, result.Status)
}
}
}
}
if parsed.Errors {
for _, item := range parsed.Items {
for _, result := range item {
if result.Error != nil || result.Status >= 300 {
return fmt.Errorf("Elasticsearch rejected document %s (status %d)", result.ID, result.Status)
}
}
}
return errors.New("Elasticsearch bulk reported errors")
}
if len(parsed.Items) != len(documents) {
return fmt.Errorf("Elasticsearch applied %d of %d documents", len(parsed.Items), len(documents))
}
return nil
}
var _ = strings.TrimSpace