package v071 import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "strings" "knowledge-governance/internal/librarysnapshot" ) // Count 用与导出相同的 scope filter 数生产索引里的文档,供校验对比。 func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdentity) (int64, error) { if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) { return 0, errors.New("invalid search identity") } inspection, err := exporter.inspectMapping(ctx) if errors.Is(err, errScopeFieldsMissing) { // 新建索引还没有范围字段:只有确实为空时才等于"这个范围 0 条"。 empty, probeErr := exporter.emptyScopeIndex(ctx) if probeErr != nil { return 0, probeErr } if empty { return 0, nil } return 0, err } if err != nil { return 0, err } // 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件 // 会把过滤器变成"匹配空字符串",永远数不到东西。 filter := make([]any, 0, 2) if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil { filter = append(filter, clause) } filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}) payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}} var response struct { Count int64 `json:"count"` } if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_count", payload, &response); err != nil { return 0, err } if response.Count < 0 { return 0, errors.New("Elasticsearch returned a negative count") } return response.Count, nil } // PruneScope 删除该知识库范围内、但不在这份快照里的索引文档: // 撤销必须真的从生产检索里消失,不能只做"新增/覆盖"。 func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchIdentity, keepIDs []string) (int, error) { inspection, err := exporter.inspectMapping(ctx) if err != nil { return 0, err } keep := make(map[string]bool, len(keepIDs)) for _, id := range keepIDs { keep[id] = true } stale := make([]string, 0, 16) for from := 0; ; from += exporter.pageSize { payload := map[string]any{ "from": from, "size": exporter.pageSize, "_source": false, "sort": []any{"_doc"}, "query": map[string]any{"bool": map[string]any{"filter": scopeFilterFor(inspection.fields, identity)}}, } var response struct { Hits struct { Hits []struct { ID string `json:"_id"` } `json:"hits"` } `json:"hits"` } if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil { return 0, err } for _, hit := range response.Hits.Hits { if !keep[hit.ID] { stale = append(stale, hit.ID) } } if len(response.Hits.Hits) < exporter.pageSize { break } if from+exporter.pageSize >= 10000 { return 0, errors.New("scope has too many documents to prune safely") } } if len(stale) == 0 { return 0, nil } for start := 0; start < len(stale); start += 500 { end := start + 500 if end > len(stale) { end = len(stale) } payload := map[string]any{ "query": map[string]any{"ids": map[string]any{"values": stale[start:end]}}, } var response struct { Deleted int `json:"deleted"` } if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_delete_by_query?refresh=true", payload, &response); err != nil { return 0, err } } return len(stale), nil } // scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。 func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any { filter := make([]any, 0, 2) if clause := tenantClause(fields, identity.TenantID); clause != nil { filter = append(filter, clause) } filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}}) return filter } // BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。 // 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。 func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error { if len(documents) == 0 { return nil } var body bytes.Buffer for _, document := range documents { if document.ID == "" { return errors.New("archived index document without id") } payload := document.Source var source map[string]json.RawMessage if err := json.Unmarshal(document.Source, &source); err != nil { return errors.New("archived index document is not an object") } source["tenant_id"], _ = json.Marshal(identity.TenantID) source["knowledge_base_id"], _ = json.Marshal(identity.KnowledgeBaseID) encoded, err := json.Marshal(source) if err != nil { return err } payload = encoded action, err := json.Marshal(map[string]any{"index": map[string]string{ "_index": exporter.index, "_id": document.ID, }}) if err != nil { return err } body.Write(action) body.WriteByte('\n') body.Write(payload) body.WriteByte('\n') } request, err := http.NewRequestWithContext(ctx, http.MethodPost, exporter.baseURL.String()+"/_bulk?refresh=true", bytes.NewReader(body.Bytes())) if err != nil { return err } request.Header.Set("Content-Type", "application/x-ndjson") if exporter.authorizationToken != "" { request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken) } response, err := exporter.client.Do(request) if err != nil { return errors.New("Elasticsearch bulk request failed") } defer response.Body.Close() raw, err := io.ReadAll(io.LimitReader(response.Body, maxSearchResponseBytes)) if err != nil { return errors.New("Elasticsearch bulk response unreadable") } if response.StatusCode != http.StatusOK { return fmt.Errorf("Elasticsearch bulk status %d", response.StatusCode) } var parsed struct { Errors bool `json:"errors"` Items []map[string]struct { Status int `json:"status"` Error any `json:"error"` ID string `json:"_id"` } `json:"items"` } if err := json.Unmarshal(raw, &parsed); err != nil { return errors.New("Elasticsearch bulk response invalid") } if parsed.Errors { for _, item := range parsed.Items { for _, result := range item { if result.Error != nil || result.Status >= 300 { return fmt.Errorf("Elasticsearch rejected document %s (status %d)", result.ID, result.Status) } } } return errors.New("Elasticsearch bulk reported errors") } if len(parsed.Items) != len(documents) { return fmt.Errorf("Elasticsearch applied %d of %d documents", len(parsed.Items), len(documents)) } return nil } var _ = strings.TrimSpace