Files
knowledge-governance/internal/releaseadapter/v071/search.go
T
姚章浩 37a8370f7b
build / build (push) Successful in 1m15s
fix(v071): 索引范围过滤接受未写 tenant_id 的文档
真实 v0.7.1 的 app 自己写索引时不带 tenant_id,治理侧同步写入时才补齐。
原来只按 term tenant_id 过滤,等于把 app 写入的正文整库漏掉:真跑时索引里有
2 条文档,导出 0 条,候选校验报 legacy coverage,封存被拒。

改为"显式本租户 或 完全没写租户字段"(写了别的租户仍然排除),知识库身份
仍是独立 term,范围没有放宽。Count/PruneScope/整库导出共用同一写法。

试过:make verify 15 项全绿;新增 3 个过滤条件用例;真实 preflight 由
candidate-invalid(0 条) 变为 candidate-valid(2 条)。
2026-10-29 06:00:30 +08:00

366 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
const maxSearchResponseBytes = 64 * 1024 * 1024
type SearchIdentity struct {
TenantID uint64
KnowledgeBaseID string
KnowledgeID string
}
type SearchDocument struct {
ID string
Source json.RawMessage
}
type SearchSnapshot struct {
Documents []SearchDocument
MappingSHA256 string
DocumentsSHA256 string
}
type SearchExporterConfig struct {
BaseURL string
Index string
AuthorizationToken string
HTTPClient *http.Client
PageSize int
}
type SearchExporter struct {
baseURL *url.URL
index string
authorizationToken string
client *http.Client
pageSize int
}
// searchFieldNames reflects the real v0.7.1 index shape:
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
// default) or as plain "keyword"; exact filters must target the right one.
// - Documents may carry no tenant_id field at all; tenancy is then implied
// by the knowledge_base_id/knowledge_id scope.
type searchFieldNames struct {
tenant string // "" when the index has no tenant_id field
knowledge string
base string
}
// mappingInspection is the canonical digest of the index mappings plus the
// logical scope field names derived from the real mapping.
// errIndexMissing 表示目标索引根本不存在(空生产基线)。
var errIndexMissing = errors.New("Elasticsearch index missing")
type mappingInspection struct {
digest string
fields searchFieldNames
}
// inspectMapping fetches the index mapping once and derives both the contract
// digest and the exact-match field names from a single response.
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
var response map[string]json.RawMessage
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
if errors.Is(err, errIndexMissing) {
return mappingInspection{}, errIndexMissing
}
return mappingInspection{}, err
}
indexRaw, ok := response[exporter.index]
if !ok {
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
}
var index struct {
Mappings json.RawMessage `json:"mappings"`
}
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
canonical, err := canonicalJSON(index.Mappings)
if err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
digest := sha256.Sum256(canonical)
var mappings struct {
Properties map[string]struct {
Type string `json:"type"`
Fields map[string]struct {
Type string `json:"type"`
} `json:"fields"`
} `json:"properties"`
}
if err := json.Unmarshal(canonical, &mappings); err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
resolve := func(name string) string {
property, ok := mappings.Properties[name]
if !ok {
return ""
}
if property.Type == "keyword" {
return name
}
if _, ok := property.Fields["keyword"]; ok {
return name + ".keyword"
}
return name
}
fields := searchFieldNames{
tenant: resolve("tenant_id"),
knowledge: resolve("knowledge_id"),
base: resolve("knowledge_base_id"),
}
if fields.knowledge == "" || fields.base == "" {
return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields")
}
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
}
// tenantClause 是"租户范围"这一条过滤条件的唯一写法。
//
// 为什么不是简单的 term:真实 v0.7.1 的 app 自己写索引时不带 tenant_id,而治理侧
// 同步写入时会补齐。于是 "mapping 里有 tenant_id" 不等于 "每篇文档都写了 tenant_id"。
// 只按 term 过滤会把 app 写入的正文整库漏掉——真跑踩过:索引里明明有文档,导出 0 条,
// 候选校验报 legacy coverage。这里改成:
//
// 显式写了本租户 → 收;完全没写 → 收(没有可比的租户声明);写了别的租户 → 排除。
//
// 知识库身份仍然是一条独立的 term,所以范围始终由知识库唯一 ID 收窄,不是放宽成"全库"。
func tenantClause(fields searchFieldNames, tenant uint64) any {
if fields.tenant == "" {
return nil
}
return map[string]any{"bool": map[string]any{
"should": []any{
map[string]any{"term": map[string]any{fields.tenant: tenant}},
map[string]any{"bool": map[string]any{"must_not": []any{
map[string]any{"exists": map[string]any{"field": fields.tenant}},
}}},
},
"minimum_should_match": 1,
}}
}
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
// 租户 + 知识库 + 单文档三层收窄:tenant 那条见 tenantClause(app 写入的文档不带租户字段)。
filter := make([]any, 0, 3)
if clause := tenantClause(fields, identity.TenantID); clause != nil {
filter = append(filter, clause)
}
filter = append(filter,
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
)
return filter
}
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
return nil, errors.New("invalid Elasticsearch base URL")
}
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
if !validSearchIdentifier(config.Index) {
return nil, errors.New("invalid Elasticsearch index")
}
pageSize := config.PageSize
if pageSize == 0 {
pageSize = 500
}
if pageSize < 1 || pageSize > 1000 {
return nil, errors.New("invalid Elasticsearch page size")
}
client := &http.Client{Timeout: 30 * time.Second}
if config.HTTPClient != nil {
clone := *config.HTTPClient
client = &clone
}
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
}
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return SearchSnapshot{}, errors.New("invalid search export identity")
}
inspection, err := exporter.inspectMapping(ctx)
if errors.Is(err, errIndexMissing) {
// 索引不存在 = 空基线:零文档、无映射可比。调用方必须再确认数据库也是空的。
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
if err != nil {
return SearchSnapshot{}, err
}
filter := scopedFilterFor(inspection.fields, identity)
documents := make([]SearchDocument, 0)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from,
"size": exporter.pageSize,
"query": map[string]any{"bool": map[string]any{"filter": filter}},
"sort": []any{"_doc"},
}
var response searchResponse
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return SearchSnapshot{}, err
}
for _, hit := range response.Hits.Hits {
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
}
canonical, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
}
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
}
}
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
digest, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
}
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return "", err
}
return inspection.digest, nil
}
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return errors.New("encode Elasticsearch request")
}
body = bytes.NewReader(encoded)
}
requestURL := *exporter.baseURL
relative, err := url.Parse(path)
if err != nil || relative.IsAbs() || relative.Host != "" {
return errors.New("invalid Elasticsearch request path")
}
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
requestURL.RawQuery = relative.RawQuery
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
if err != nil {
return errors.New("create Elasticsearch request")
}
request.Header.Set("Accept", "application/json")
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch request failed")
}
defer response.Body.Close()
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
return errors.New("Elasticsearch response origin mismatch")
}
if response.StatusCode == http.StatusNotFound {
// 索引不存在是"空生产"的正常状态,不是故障;调用方必须显式处理,
// 不能把 404 当成功,也不能把它当传输错误。
return fmt.Errorf("%w: HTTP 404", errIndexMissing)
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
}
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
if err := decoder.Decode(result); err != nil {
return errors.New("decode Elasticsearch response")
}
return nil
}
type searchResponse struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}
// searchHitMatches verifies the returned document belongs to the requested
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
// field is absent the KB/knowledge scope is authoritative, matching the
// export filter semantics. A present tenant must still match exactly.
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
var scoped struct {
TenantID *uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(source, &scoped) != nil {
return false
}
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
return false
}
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
}
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
payload, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode Elasticsearch document digest")
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
var value any
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
return json.Marshal(value)
}
func validSearchIdentifier(value string) bool {
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
return false
}
for _, char := range value {
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
return false
}
}
return true
}