Files
knowledge-governance/internal/releaseadapter/v071/search_write.go
T
姚章浩 281699a431
build / build (push) Successful in 1m21s
feat(library): 生产整库真实读写/校验/恢复与本机真实服务验收
为什么:生产目标此前是"无条件失败"的占位,编排再完整也不能落地;本轮把它做成真实实现,
并在本机 WeKnora 测试栈的真实 PG/ES 上跑通导入、撤销、预备份与故障恢复。

- ProductionLibraryTarget:Current 读行数+身份摘要;Apply 按阶段真实写入(database/wiki
  同事务先删本库再插、resources 传对象、index 搬运原始文档与向量);Verify 比对身份摘要;
  Restore 按预备份快照回放。
- 撤销真正生效:索引阶段写入后 PruneScope 删除快照里没有的文档。
- ProductionBackupStore:导出前独立完整预备份,导出前后行数不一致即失败。
- worker 用注入配置装配目标与预备份;缺项把变量名写进阶段并失败,不再有占位对象。
- capacity 安全加法防溢出;保留策略组合语义明确(各维度都超限才候选)。
- 新增本机真实服务集成测试(一次性 kg_library_test_* 库 + 随机 ES 索引),
  覆盖导入/校验/撤销/预备份/故障注入/恢复。
2026-10-28 15:54:01 +08:00

198 lines
6.5 KiB
Go

package v071
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"knowledge-governance/internal/librarysnapshot"
)
// Count 用与导出相同的 scope filter 数生产索引里的文档,供校验对比。
func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdentity) (int64, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return 0, errors.New("invalid search identity")
}
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return 0, err
}
// 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件
// 会把过滤器变成"匹配空字符串",永远数不到东西。
filter := make([]any, 0, 2)
if inspection.fields.tenant != "" {
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.tenant: identity.TenantID}})
}
filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}})
payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}}
var response struct {
Count int64 `json:"count"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_count", payload, &response); err != nil {
return 0, err
}
if response.Count < 0 {
return 0, errors.New("Elasticsearch returned a negative count")
}
return response.Count, nil
}
// PruneScope 删除该知识库范围内、但不在这份快照里的索引文档:
// 撤销必须真的从生产检索里消失,不能只做"新增/覆盖"。
func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchIdentity, keepIDs []string) (int, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return 0, err
}
keep := make(map[string]bool, len(keepIDs))
for _, id := range keepIDs {
keep[id] = true
}
stale := make([]string, 0, 16)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from, "size": exporter.pageSize, "_source": false, "sort": []any{"_doc"},
"query": map[string]any{"bool": map[string]any{"filter": scopeFilterFor(inspection.fields, identity)}},
}
var response struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
} `json:"hits"`
} `json:"hits"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return 0, err
}
for _, hit := range response.Hits.Hits {
if !keep[hit.ID] {
stale = append(stale, hit.ID)
}
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return 0, errors.New("scope has too many documents to prune safely")
}
}
if len(stale) == 0 {
return 0, nil
}
for start := 0; start < len(stale); start += 500 {
end := start + 500
if end > len(stale) {
end = len(stale)
}
payload := map[string]any{
"query": map[string]any{"ids": map[string]any{"values": stale[start:end]}},
}
var response struct {
Deleted int `json:"deleted"`
}
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_delete_by_query?refresh=true", payload, &response); err != nil {
return 0, err
}
}
return len(stale), nil
}
// scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。
func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
filter := make([]any, 0, 2)
if fields.tenant != "" {
filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}})
}
filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}})
return filter
}
// BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。
// 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。
func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error {
if len(documents) == 0 {
return nil
}
var body bytes.Buffer
for _, document := range documents {
if document.ID == "" {
return errors.New("archived index document without id")
}
payload := document.Source
var source map[string]json.RawMessage
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("archived index document is not an object")
}
source["tenant_id"], _ = json.Marshal(identity.TenantID)
source["knowledge_base_id"], _ = json.Marshal(identity.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
payload = encoded
action, err := json.Marshal(map[string]any{"index": map[string]string{
"_index": exporter.index, "_id": document.ID,
}})
if err != nil {
return err
}
body.Write(action)
body.WriteByte('\n')
body.Write(payload)
body.WriteByte('\n')
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost,
exporter.baseURL.String()+"/_bulk?refresh=true", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/x-ndjson")
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch bulk request failed")
}
defer response.Body.Close()
raw, err := io.ReadAll(io.LimitReader(response.Body, maxSearchResponseBytes))
if err != nil {
return errors.New("Elasticsearch bulk response unreadable")
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("Elasticsearch bulk status %d", response.StatusCode)
}
var parsed struct {
Errors bool `json:"errors"`
Items []map[string]struct {
Status int `json:"status"`
Error any `json:"error"`
ID string `json:"_id"`
} `json:"items"`
}
if err := json.Unmarshal(raw, &parsed); err != nil {
return errors.New("Elasticsearch bulk response invalid")
}
if parsed.Errors {
for _, item := range parsed.Items {
for _, result := range item {
if result.Error != nil || result.Status >= 300 {
return fmt.Errorf("Elasticsearch rejected document %s (status %d)", result.ID, result.Status)
}
}
}
return errors.New("Elasticsearch bulk reported errors")
}
if len(parsed.Items) != len(documents) {
return fmt.Errorf("Elasticsearch applied %d of %d documents", len(parsed.Items), len(documents))
}
return nil
}
var _ = strings.TrimSpace