Files
knowledge-governance/internal/releaseadapter/v071/search.go
T
姚章浩 7d4ffdbfe5
build / build (push) Successful in 1m15s
fix(v071): 全新目标实例的空索引按空基线处理
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。

现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。

试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
2026-10-29 06:17:26 +08:00

383 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
const maxSearchResponseBytes = 64 * 1024 * 1024
type SearchIdentity struct {
TenantID uint64
KnowledgeBaseID string
KnowledgeID string
}
type SearchDocument struct {
ID string
Source json.RawMessage
}
type SearchSnapshot struct {
Documents []SearchDocument
MappingSHA256 string
DocumentsSHA256 string
}
type SearchExporterConfig struct {
BaseURL string
Index string
AuthorizationToken string
HTTPClient *http.Client
PageSize int
}
type SearchExporter struct {
baseURL *url.URL
index string
authorizationToken string
client *http.Client
pageSize int
}
// searchFieldNames reflects the real v0.7.1 index shape:
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
// default) or as plain "keyword"; exact filters must target the right one.
// - Documents may carry no tenant_id field at all; tenancy is then implied
// by the knowledge_base_id/knowledge_id scope.
type searchFieldNames struct {
tenant string // "" when the index has no tenant_id field
knowledge string
base string
}
// mappingInspection is the canonical digest of the index mappings plus the
// logical scope field names derived from the real mapping.
// errIndexMissing 表示目标索引根本不存在(空生产基线)。
var errIndexMissing = errors.New("Elasticsearch index missing")
// errScopeFieldsMissing 表示索引存在但还没有发布范围字段:
// 全新的 WeKnora 实例会先用空映射建好索引(`{}`),字段要等第一次写入才动态生成。
// 这跟"索引被改坏了"必须分开:只有在索引里确实一篇文档都没有时才能当成空基线。
var errScopeFieldsMissing = errors.New("Elasticsearch index mapping is missing release scope fields")
// emptyScopeIndex 判断"没有范围字段"的索引是不是真的空。
// 有文档却没有范围字段 = 无法判断这批文档属于谁,必须报错,不能当成 0。
func (exporter *SearchExporter) emptyScopeIndex(ctx context.Context) (bool, error) {
var response struct {
Count int64 `json:"count"`
}
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_count", nil, &response); err != nil {
return false, err
}
return response.Count == 0, nil
}
type mappingInspection struct {
digest string
fields searchFieldNames
}
// inspectMapping fetches the index mapping once and derives both the contract
// digest and the exact-match field names from a single response.
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
var response map[string]json.RawMessage
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
if errors.Is(err, errIndexMissing) {
return mappingInspection{}, errIndexMissing
}
return mappingInspection{}, err
}
indexRaw, ok := response[exporter.index]
if !ok {
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
}
var index struct {
Mappings json.RawMessage `json:"mappings"`
}
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
canonical, err := canonicalJSON(index.Mappings)
if err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
digest := sha256.Sum256(canonical)
var mappings struct {
Properties map[string]struct {
Type string `json:"type"`
Fields map[string]struct {
Type string `json:"type"`
} `json:"fields"`
} `json:"properties"`
}
if err := json.Unmarshal(canonical, &mappings); err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
resolve := func(name string) string {
property, ok := mappings.Properties[name]
if !ok {
return ""
}
if property.Type == "keyword" {
return name
}
if _, ok := property.Fields["keyword"]; ok {
return name + ".keyword"
}
return name
}
fields := searchFieldNames{
tenant: resolve("tenant_id"),
knowledge: resolve("knowledge_id"),
base: resolve("knowledge_base_id"),
}
if fields.knowledge == "" || fields.base == "" {
return mappingInspection{}, errScopeFieldsMissing
}
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
}
// tenantClause 是"租户范围"这一条过滤条件的唯一写法。
//
// 为什么不是简单的 term:真实 v0.7.1 的 app 自己写索引时不带 tenant_id,而治理侧
// 同步写入时会补齐。于是 "mapping 里有 tenant_id" 不等于 "每篇文档都写了 tenant_id"。
// 只按 term 过滤会把 app 写入的正文整库漏掉——真跑踩过:索引里明明有文档,导出 0 条,
// 候选校验报 legacy coverage。这里改成:
//
// 显式写了本租户 → 收;完全没写 → 收(没有可比的租户声明);写了别的租户 → 排除。
//
// 知识库身份仍然是一条独立的 term,所以范围始终由知识库唯一 ID 收窄,不是放宽成"全库"。
func tenantClause(fields searchFieldNames, tenant uint64) any {
if fields.tenant == "" {
return nil
}
return map[string]any{"bool": map[string]any{
"should": []any{
map[string]any{"term": map[string]any{fields.tenant: tenant}},
map[string]any{"bool": map[string]any{"must_not": []any{
map[string]any{"exists": map[string]any{"field": fields.tenant}},
}}},
},
"minimum_should_match": 1,
}}
}
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
// 租户 + 知识库 + 单文档三层收窄:tenant 那条见 tenantClause(app 写入的文档不带租户字段)。
filter := make([]any, 0, 3)
if clause := tenantClause(fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter,
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
)
return filter
}
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
return nil, errors.New("invalid Elasticsearch base URL")
}
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
if !validSearchIdentifier(config.Index) {
return nil, errors.New("invalid Elasticsearch index")
}
pageSize := config.PageSize
if pageSize == 0 {
pageSize = 500
}
if pageSize < 1 || pageSize > 1000 {
return nil, errors.New("invalid Elasticsearch page size")
}
client := &http.Client{Timeout: 30 * time.Second}
if config.HTTPClient != nil {
clone := *config.HTTPClient
client = &clone
}
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
}
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return SearchSnapshot{}, errors.New("invalid search export identity")
}
inspection, err := exporter.inspectMapping(ctx)
if errors.Is(err, errIndexMissing) {
// 索引不存在 = 空基线:零文档、无映射可比。调用方必须再确认数据库也是空的。
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
if err != nil {
return SearchSnapshot{}, err
}
filter := scopedFilterFor(inspection.fields, identity)
documents := make([]SearchDocument, 0)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from,
"size": exporter.pageSize,
"query": map[string]any{"bool": map[string]any{"filter": filter}},
"sort": []any{"_doc"},
}
var response searchResponse
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return SearchSnapshot{}, err
}
for _, hit := range response.Hits.Hits {
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
}
canonical, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
}
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
}
}
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
digest, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
}
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return "", err
}
return inspection.digest, nil
}
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return errors.New("encode Elasticsearch request")
}
body = bytes.NewReader(encoded)
}
requestURL := *exporter.baseURL
relative, err := url.Parse(path)
if err != nil || relative.IsAbs() || relative.Host != "" {
return errors.New("invalid Elasticsearch request path")
}
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
requestURL.RawQuery = relative.RawQuery
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
if err != nil {
return errors.New("create Elasticsearch request")
}
request.Header.Set("Accept", "application/json")
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch request failed")
}
defer response.Body.Close()
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
return errors.New("Elasticsearch response origin mismatch")
}
if response.StatusCode == http.StatusNotFound {
// 索引不存在是"空生产"的正常状态,不是故障;调用方必须显式处理,
// 不能把 404 当成功,也不能把它当传输错误。
return fmt.Errorf("%w: HTTP 404", errIndexMissing)
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
}
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
if err := decoder.Decode(result); err != nil {
return errors.New("decode Elasticsearch response")
}
return nil
}
type searchResponse struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}
// searchHitMatches verifies the returned document belongs to the requested
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
// field is absent the KB/knowledge scope is authoritative, matching the
// export filter semantics. A present tenant must still match exactly.
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
var scoped struct {
TenantID *uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(source, &scoped) != nil {
return false
}
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
return false
}
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
}
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
payload, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode Elasticsearch document digest")
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
var value any
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
return json.Marshal(value)
}
func validSearchIdentifier(value string) bool {
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
return false
}
for _, char := range value {
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
return false
}
}
return true
}