Files
knowledge-governance/internal/releaseadapter/v071/search_write.go
T
姚章浩 7d4ffdbfe5
build / build (push) Successful in 1m15s
fix(v071): 全新目标实例的空索引按空基线处理
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。

现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。

试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
2026-10-29 06:17:26 +08:00

209 lines
6.7 KiB
Go

package v071
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"knowledge-governance/internal/librarysnapshot"
)
// Count 用与导出相同的 scope filter 数生产索引里的文档,供校验对比。
func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdentity) (int64, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return 0, errors.New("invalid search identity")
}
inspection, err := exporter.inspectMapping(ctx)
if errors.Is(err, errScopeFieldsMissing) {
// 新建索引还没有范围字段:只有确实为空时才等于"这个范围 0 条"。
empty, probeErr := exporter.emptyScopeIndex(ctx)
if probeErr != nil {
return 0, probeErr
}
if empty {
return 0, nil
}
return 0, err
}
if err != nil {
return 0, err
}
// 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件
// 会把过滤器变成"匹配空字符串",永远数不到东西。
filter := make([]any, 0, 2)
if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}})
payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}}
var response struct {
Count int64 `json:"count"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_count", payload, &response); err != nil {
return 0, err
}
if response.Count < 0 {
return 0, errors.New("Elasticsearch returned a negative count")
}
return response.Count, nil
}
// PruneScope 删除该知识库范围内、但不在这份快照里的索引文档:
// 撤销必须真的从生产检索里消失,不能只做"新增/覆盖"。
func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchIdentity, keepIDs []string) (int, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return 0, err
}
keep := make(map[string]bool, len(keepIDs))
for _, id := range keepIDs {
keep[id] = true
}
stale := make([]string, 0, 16)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from, "size": exporter.pageSize, "_source": false, "sort": []any{"_doc"},
"query": map[string]any{"bool": map[string]any{"filter": scopeFilterFor(inspection.fields, identity)}},
}
var response struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
} `json:"hits"`
} `json:"hits"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return 0, err
}
for _, hit := range response.Hits.Hits {
if !keep[hit.ID] {
stale = append(stale, hit.ID)
}
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return 0, errors.New("scope has too many documents to prune safely")
}
}
if len(stale) == 0 {
return 0, nil
}
for start := 0; start < len(stale); start += 500 {
end := start + 500
if end > len(stale) {
end = len(stale)
}
payload := map[string]any{
"query": map[string]any{"ids": map[string]any{"values": stale[start:end]}},
}
var response struct {
Deleted int `json:"deleted"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_delete_by_query?refresh=true", payload, &response); err != nil {
return 0, err
}
}
return len(stale), nil
}
// scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。
func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
filter := make([]any, 0, 2)
if clause := tenantClause(fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}})
return filter
}
// BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。
// 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。
func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error {
if len(documents) == 0 {
return nil
}
var body bytes.Buffer
for _, document := range documents {
if document.ID == "" {
return errors.New("archived index document without id")
}
payload := document.Source
var source map[string]json.RawMessage
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("archived index document is not an object")
}
source["tenant_id"], _ = json.Marshal(identity.TenantID)
source["knowledge_base_id"], _ = json.Marshal(identity.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
payload = encoded
action, err := json.Marshal(map[string]any{"index": map[string]string{
"_index": exporter.index, "_id": document.ID,
}})
if err != nil {
return err
}
body.Write(action)
body.WriteByte('\n')
body.Write(payload)
body.WriteByte('\n')
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost,
exporter.baseURL.String()+"/_bulk?refresh=true", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/x-ndjson")
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch bulk request failed")
}
defer response.Body.Close()
raw, err := io.ReadAll(io.LimitReader(response.Body, maxSearchResponseBytes))
if err != nil {
return errors.New("Elasticsearch bulk response unreadable")
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("Elasticsearch bulk status %d", response.StatusCode)
}
var parsed struct {
Errors bool `json:"errors"`
Items []map[string]struct {
Status int `json:"status"`
Error any `json:"error"`
ID string `json:"_id"`
} `json:"items"`
}
if err := json.Unmarshal(raw, &parsed); err != nil {
return errors.New("Elasticsearch bulk response invalid")
}
if parsed.Errors {
for _, item := range parsed.Items {
for _, result := range item {
if result.Error != nil || result.Status >= 300 {
return fmt.Errorf("Elasticsearch rejected document %s (status %d)", result.ID, result.Status)
}
}
}
return errors.New("Elasticsearch bulk reported errors")
}
if len(parsed.Items) != len(documents) {
return fmt.Errorf("Elasticsearch applied %d of %d documents", len(parsed.Items), len(documents))
}
return nil
}
var _ = strings.TrimSpace