Files
knowledge-governance/internal/releaseadapter/v071/search.go
T
zheromiku dcdc8d1ddd feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
2026-09-30 10:54:06 +08:00

323 lines
11 KiB
Go

package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
const maxSearchResponseBytes = 64 * 1024 * 1024
type SearchIdentity struct {
TenantID uint64
KnowledgeBaseID string
KnowledgeID string
}
type SearchDocument struct {
ID string
Source json.RawMessage
}
type SearchSnapshot struct {
Documents []SearchDocument
MappingSHA256 string
DocumentsSHA256 string
}
type SearchExporterConfig struct {
BaseURL string
Index string
AuthorizationToken string
HTTPClient *http.Client
PageSize int
}
type SearchExporter struct {
baseURL *url.URL
index string
authorizationToken string
client *http.Client
pageSize int
}
// searchFieldNames reflects the real v0.7.1 index shape:
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
// default) or as plain "keyword"; exact filters must target the right one.
// - Documents may carry no tenant_id field at all; tenancy is then implied
// by the knowledge_base_id/knowledge_id scope.
type searchFieldNames struct {
tenant string // "" when the index has no tenant_id field
knowledge string
base string
}
// mappingInspection is the canonical digest of the index mappings plus the
// logical scope field names derived from the real mapping.
type mappingInspection struct {
digest string
fields searchFieldNames
}
// inspectMapping fetches the index mapping once and derives both the contract
// digest and the exact-match field names from a single response.
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
var response map[string]json.RawMessage
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
return mappingInspection{}, err
}
indexRaw, ok := response[exporter.index]
if !ok {
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
}
var index struct {
Mappings json.RawMessage `json:"mappings"`
}
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
canonical, err := canonicalJSON(index.Mappings)
if err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
digest := sha256.Sum256(canonical)
var mappings struct {
Properties map[string]struct {
Type string `json:"type"`
Fields map[string]struct {
Type string `json:"type"`
} `json:"fields"`
} `json:"properties"`
}
if err := json.Unmarshal(canonical, &mappings); err != nil {
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
}
resolve := func(name string) string {
property, ok := mappings.Properties[name]
if !ok {
return ""
}
if property.Type == "keyword" {
return name
}
if _, ok := property.Fields["keyword"]; ok {
return name + ".keyword"
}
return name
}
fields := searchFieldNames{
tenant: resolve("tenant_id"),
knowledge: resolve("knowledge_id"),
base: resolve("knowledge_base_id"),
}
if fields.knowledge == "" || fields.base == "" {
return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields")
}
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
}
// scopedFilter builds the tenant/KB/knowledge term filter adapted to the
// resolved index field names.
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
filter := make([]any, 0, 3)
if fields.tenant != "" {
filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}})
}
filter = append(filter,
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
)
return filter
}
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
return nil, errors.New("invalid Elasticsearch base URL")
}
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
if !validSearchIdentifier(config.Index) {
return nil, errors.New("invalid Elasticsearch index")
}
pageSize := config.PageSize
if pageSize == 0 {
pageSize = 500
}
if pageSize < 1 || pageSize > 1000 {
return nil, errors.New("invalid Elasticsearch page size")
}
client := &http.Client{Timeout: 30 * time.Second}
if config.HTTPClient != nil {
clone := *config.HTTPClient
client = &clone
}
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
}
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
return SearchSnapshot{}, errors.New("invalid search export identity")
}
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return SearchSnapshot{}, err
}
filter := scopedFilterFor(inspection.fields, identity)
documents := make([]SearchDocument, 0)
for from := 0; ; from += exporter.pageSize {
payload := map[string]any{
"from": from,
"size": exporter.pageSize,
"query": map[string]any{"bool": map[string]any{"filter": filter}},
"sort": []any{"_doc"},
}
var response searchResponse
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
return SearchSnapshot{}, err
}
for _, hit := range response.Hits.Hits {
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
}
canonical, err := canonicalJSON(hit.Source)
if err != nil {
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
}
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
}
if len(response.Hits.Hits) < exporter.pageSize {
break
}
if from+exporter.pageSize >= 10000 {
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
}
}
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
digest, err := searchDocumentsDigest(documents)
if err != nil {
return SearchSnapshot{}, err
}
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
}
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
inspection, err := exporter.inspectMapping(ctx)
if err != nil {
return "", err
}
return inspection.digest, nil
}
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return errors.New("encode Elasticsearch request")
}
body = bytes.NewReader(encoded)
}
requestURL := *exporter.baseURL
relative, err := url.Parse(path)
if err != nil || relative.IsAbs() || relative.Host != "" {
return errors.New("invalid Elasticsearch request path")
}
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
requestURL.RawQuery = relative.RawQuery
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
if err != nil {
return errors.New("create Elasticsearch request")
}
request.Header.Set("Accept", "application/json")
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if exporter.authorizationToken != "" {
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
}
response, err := exporter.client.Do(request)
if err != nil {
return errors.New("Elasticsearch request failed")
}
defer response.Body.Close()
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
return errors.New("Elasticsearch response origin mismatch")
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
}
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
if err := decoder.Decode(result); err != nil {
return errors.New("decode Elasticsearch response")
}
return nil
}
type searchResponse struct {
Hits struct {
Hits []struct {
ID string `json:"_id"`
Source json.RawMessage `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}
// searchHitMatches verifies the returned document belongs to the requested
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
// field is absent the KB/knowledge scope is authoritative, matching the
// export filter semantics. A present tenant must still match exactly.
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
var scoped struct {
TenantID *uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(source, &scoped) != nil {
return false
}
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
return false
}
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
}
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
payload, err := json.Marshal(documents)
if err != nil {
return "", errors.New("encode Elasticsearch document digest")
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
var value any
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
return json.Marshal(value)
}
func validSearchIdentifier(value string) bool {
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
return false
}
for _, char := range value {
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
return false
}
}
return true
}