7d4ffdbfe5
build / build (push) Successful in 1m15s
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。
现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。
试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
209 lines
6.7 KiB
Go
209 lines
6.7 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// Count 用与导出相同的 scope filter 数生产索引里的文档,供校验对比。
|
|
func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdentity) (int64, error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
|
|
return 0, errors.New("invalid search identity")
|
|
}
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if errors.Is(err, errScopeFieldsMissing) {
|
|
// 新建索引还没有范围字段:只有确实为空时才等于"这个范围 0 条"。
|
|
empty, probeErr := exporter.emptyScopeIndex(ctx)
|
|
if probeErr != nil {
|
|
return 0, probeErr
|
|
}
|
|
if empty {
|
|
return 0, nil
|
|
}
|
|
return 0, err
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件
|
|
// 会把过滤器变成"匹配空字符串",永远数不到东西。
|
|
filter := make([]any, 0, 2)
|
|
if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil {
|
|
filter = append(filter, clause)
|
|
}
|
|
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}})
|
|
payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}}
|
|
var response struct {
|
|
Count int64 `json:"count"`
|
|
}
|
|
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_count", payload, &response); err != nil {
|
|
return 0, err
|
|
}
|
|
if response.Count < 0 {
|
|
return 0, errors.New("Elasticsearch returned a negative count")
|
|
}
|
|
return response.Count, nil
|
|
}
|
|
|
|
// PruneScope 删除该知识库范围内、但不在这份快照里的索引文档:
|
|
// 撤销必须真的从生产检索里消失,不能只做"新增/覆盖"。
|
|
func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchIdentity, keepIDs []string) (int, error) {
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
keep := make(map[string]bool, len(keepIDs))
|
|
for _, id := range keepIDs {
|
|
keep[id] = true
|
|
}
|
|
stale := make([]string, 0, 16)
|
|
for from := 0; ; from += exporter.pageSize {
|
|
payload := map[string]any{
|
|
"from": from, "size": exporter.pageSize, "_source": false, "sort": []any{"_doc"},
|
|
"query": map[string]any{"bool": map[string]any{"filter": scopeFilterFor(inspection.fields, identity)}},
|
|
}
|
|
var response struct {
|
|
Hits struct {
|
|
Hits []struct {
|
|
ID string `json:"_id"`
|
|
} `json:"hits"`
|
|
} `json:"hits"`
|
|
}
|
|
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
|
|
return 0, err
|
|
}
|
|
for _, hit := range response.Hits.Hits {
|
|
if !keep[hit.ID] {
|
|
stale = append(stale, hit.ID)
|
|
}
|
|
}
|
|
if len(response.Hits.Hits) < exporter.pageSize {
|
|
break
|
|
}
|
|
if from+exporter.pageSize >= 10000 {
|
|
return 0, errors.New("scope has too many documents to prune safely")
|
|
}
|
|
}
|
|
if len(stale) == 0 {
|
|
return 0, nil
|
|
}
|
|
for start := 0; start < len(stale); start += 500 {
|
|
end := start + 500
|
|
if end > len(stale) {
|
|
end = len(stale)
|
|
}
|
|
payload := map[string]any{
|
|
"query": map[string]any{"ids": map[string]any{"values": stale[start:end]}},
|
|
}
|
|
var response struct {
|
|
Deleted int `json:"deleted"`
|
|
}
|
|
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_delete_by_query?refresh=true", payload, &response); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return len(stale), nil
|
|
}
|
|
|
|
// scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。
|
|
func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
|
|
filter := make([]any, 0, 2)
|
|
if clause := tenantClause(fields, identity.TenantID); clause != nil {
|
|
filter = append(filter, clause)
|
|
}
|
|
filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}})
|
|
return filter
|
|
}
|
|
|
|
// BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。
|
|
// 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。
|
|
func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error {
|
|
if len(documents) == 0 {
|
|
return nil
|
|
}
|
|
var body bytes.Buffer
|
|
for _, document := range documents {
|
|
if document.ID == "" {
|
|
return errors.New("archived index document without id")
|
|
}
|
|
payload := document.Source
|
|
var source map[string]json.RawMessage
|
|
if err := json.Unmarshal(document.Source, &source); err != nil {
|
|
return errors.New("archived index document is not an object")
|
|
}
|
|
source["tenant_id"], _ = json.Marshal(identity.TenantID)
|
|
source["knowledge_base_id"], _ = json.Marshal(identity.KnowledgeBaseID)
|
|
encoded, err := json.Marshal(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
payload = encoded
|
|
action, err := json.Marshal(map[string]any{"index": map[string]string{
|
|
"_index": exporter.index, "_id": document.ID,
|
|
}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body.Write(action)
|
|
body.WriteByte('\n')
|
|
body.Write(payload)
|
|
body.WriteByte('\n')
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
|
exporter.baseURL.String()+"/_bulk?refresh=true", bytes.NewReader(body.Bytes()))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Content-Type", "application/x-ndjson")
|
|
if exporter.authorizationToken != "" {
|
|
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
|
|
}
|
|
response, err := exporter.client.Do(request)
|
|
if err != nil {
|
|
return errors.New("Elasticsearch bulk request failed")
|
|
}
|
|
defer response.Body.Close()
|
|
raw, err := io.ReadAll(io.LimitReader(response.Body, maxSearchResponseBytes))
|
|
if err != nil {
|
|
return errors.New("Elasticsearch bulk response unreadable")
|
|
}
|
|
if response.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("Elasticsearch bulk status %d", response.StatusCode)
|
|
}
|
|
var parsed struct {
|
|
Errors bool `json:"errors"`
|
|
Items []map[string]struct {
|
|
Status int `json:"status"`
|
|
Error any `json:"error"`
|
|
ID string `json:"_id"`
|
|
} `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(raw, &parsed); err != nil {
|
|
return errors.New("Elasticsearch bulk response invalid")
|
|
}
|
|
if parsed.Errors {
|
|
for _, item := range parsed.Items {
|
|
for _, result := range item {
|
|
if result.Error != nil || result.Status >= 300 {
|
|
return fmt.Errorf("Elasticsearch rejected document %s (status %d)", result.ID, result.Status)
|
|
}
|
|
}
|
|
}
|
|
return errors.New("Elasticsearch bulk reported errors")
|
|
}
|
|
if len(parsed.Items) != len(documents) {
|
|
return fmt.Errorf("Elasticsearch applied %d of %d documents", len(parsed.Items), len(documents))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var _ = strings.TrimSpace
|