Files
knowledge-governance/internal/releaseadapter/v071/search.go
T
姚章浩 ac3b395102
build / build (push) Successful in 1m21s
fix(library): 对齐真实 WeKnora Schema + 入口级真实服务验收
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设,
以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。

- knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。
- 写入顺序按真实外键;无代理键表用内容摘要核对。
- 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。
- 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。
- 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。
- finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。
- 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
2026-10-28 17:53:43 +08:00

342 lines
12 KiB
Go

package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
const maxSearchResponseBytes = 64 * 1024 * 1024
type SearchIdentity struct {
TenantID uint64
KnowledgeBaseID string
KnowledgeID string
}
type SearchDocument struct {
ID string
Source json.RawMessage
}
type SearchSnapshot struct {
Documents []SearchDocument
MappingSHA256 string
DocumentsSHA256 string
}
type SearchExporterConfig struct {
BaseURL string
Index string
AuthorizationToken string
HTTPClient *http.Client
PageSize int
}
type SearchExporter struct {
baseURL *url.URL
index string
authorizationToken string
client *http.Client
pageSize int
}
// searchFieldNames reflects the real v0.7.1 index shape:
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
// default) or as plain "keyword"; exact filters must target the right one.
// - Documents may carry no tenant_id field at all; tenancy is then implied
// by the knowledge_base_id/knowledge_id scope.
type searchFieldNames struct {
tenant string // "" when the index has no tenant_id field
knowledge string
base string
}
// mappingInspection is the canonical digest of the index mappings plus the
// logical scope field names derived from the real mapping.
// errIndexMissing 表示目标索引根本不存在(空生产基线)。
var errIndexMissing = errors.New("Elasticsearch index missing")
type mappingInspection struct {
digest string
fields searchFieldNames
}
// inspectMapping fetches the index mapping once and derives both the contract
// digest and the exact-match field names from a single response.
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
var response map[string]json.RawMessage
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
if errors.Is(err, errIndexMissing) {
return mappingInspection{}, errIndexMissing
}
return mappingInspection{}, err
}
indexRaw, ok := response[exporter.index]
if !ok {
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
}
var index struct {
Mappings json.RawMessage `json:"mappings"`
}
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
canonical, err := canonicalJSON(index.Mappings)
if err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
digest := sha256.Sum256(canonical)
var mappings struct {
Properties map[string]struct {
Type string `json:"type"`
Fields map[string]struct {
Type string `json:"type"`
} `json:"fields"`
} `json:"properties"`
}
if err := json.Unmarshal(canonical, &mappings); err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
resolve := func(name string) string {
property, ok := mappings.Properties[name]
if !ok {
return ""
}
if property.Type == "keyword" {
return name
}
if _, ok := property.Fields["keyword"]; ok {
return name + ".keyword"
}
return name
}
fields := searchFieldNames{
tenant: resolve("tenant_id"),
knowledge: resolve("knowledge_id"),
base: resolve("knowledge_base_id"),
}
if fields.knowledge == "" || fields.base == "" {
return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields")
}
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
}
// scopedFilter builds the tenant/KB/knowledge term filter adapted to the
// resolved index field names.
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
filter := make([]any, 0, 3)
if fields.tenant != "" {
filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}})
}
filter = append(filter,
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
)
return filter
}
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
return nil, errors.New("invalid Elasticsearch base URL")
}
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
if !validSearchIdentifier(config.Index) {
return nil, errors.New("invalid Elasticsearch index")
}
pageSize := config.PageSize
if pageSize == 0 {
pageSize = 500
}
if pageSize < 1 || pageSize > 1000 {
return nil, errors.New("invalid Elasticsearch page size")
}
client := &http.Client{Timeout: 30 * time.Second}
if config.HTTPClient != nil {
clone := *config.HTTPClient
client = &clone
}
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
}
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return SearchSnapshot{}, errors.New("invalid search export identity")
}
inspection, err := exporter.inspectMapping(ctx)
if errors.Is(err, errIndexMissing) {
// 索引不存在 = 空基线:零文档、无映射可比。调用方必须再确认数据库也是空的。
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
if digestErr != nil {
return SearchSnapshot{}, digestErr
}
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
}
if err != nil {
return SearchSnapshot{}, err
}
filter := scopedFilterFor(inspection.fields, identity)
documents := make([]SearchDocument, 0)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from,
"size": exporter.pageSize,
"query": map[string]any{"bool": map[string]any{"filter": filter}},
"sort": []any{"_doc"},
}
var response searchResponse
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return SearchSnapshot{}, err
}
for _, hit := range response.Hits.Hits {
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
}
canonical, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
}
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
}
}
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
digest, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
}
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return "", err
}
return inspection.digest, nil
}
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return errors.New("encode Elasticsearch request")
}
body = bytes.NewReader(encoded)
}
requestURL := *exporter.baseURL
relative, err := url.Parse(path)
if err != nil || relative.IsAbs() || relative.Host != "" {
return errors.New("invalid Elasticsearch request path")
}
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
requestURL.RawQuery = relative.RawQuery
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
if err != nil {
return errors.New("create Elasticsearch request")
}
request.Header.Set("Accept", "application/json")
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch request failed")
}
defer response.Body.Close()
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
return errors.New("Elasticsearch response origin mismatch")
}
if response.StatusCode == http.StatusNotFound {
// 索引不存在是"空生产"的正常状态,不是故障;调用方必须显式处理,
// 不能把 404 当成功,也不能把它当传输错误。
return fmt.Errorf("%w: HTTP 404", errIndexMissing)
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
}
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
if err := decoder.Decode(result); err != nil {
return errors.New("decode Elasticsearch response")
}
return nil
}
type searchResponse struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}
// searchHitMatches verifies the returned document belongs to the requested
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
// field is absent the KB/knowledge scope is authoritative, matching the
// export filter semantics. A present tenant must still match exactly.
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
var scoped struct {
TenantID *uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(source, &scoped) != nil {
return false
}
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
return false
}
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
}
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
payload, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode Elasticsearch document digest")
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
var value any
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
return json.Marshal(value)
}
func validSearchIdentifier(value string) bool {
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
return false
}
for _, char := range value {
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
return false
}
}
return true
}