dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
323 lines
11 KiB
Go
323 lines
11 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const maxSearchResponseBytes = 64 * 1024 * 1024
|
|
|
|
type SearchIdentity struct {
|
|
TenantID uint64
|
|
KnowledgeBaseID string
|
|
KnowledgeID string
|
|
}
|
|
|
|
type SearchDocument struct {
|
|
ID string
|
|
Source json.RawMessage
|
|
}
|
|
|
|
type SearchSnapshot struct {
|
|
Documents []SearchDocument
|
|
MappingSHA256 string
|
|
DocumentsSHA256 string
|
|
}
|
|
|
|
type SearchExporterConfig struct {
|
|
BaseURL string
|
|
Index string
|
|
AuthorizationToken string
|
|
HTTPClient *http.Client
|
|
PageSize int
|
|
}
|
|
|
|
type SearchExporter struct {
|
|
baseURL *url.URL
|
|
index string
|
|
authorizationToken string
|
|
client *http.Client
|
|
pageSize int
|
|
}
|
|
|
|
// searchFieldNames reflects the real v0.7.1 index shape:
|
|
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
|
|
// default) or as plain "keyword"; exact filters must target the right one.
|
|
// - Documents may carry no tenant_id field at all; tenancy is then implied
|
|
// by the knowledge_base_id/knowledge_id scope.
|
|
type searchFieldNames struct {
|
|
tenant string // "" when the index has no tenant_id field
|
|
knowledge string
|
|
base string
|
|
}
|
|
|
|
// mappingInspection is the canonical digest of the index mappings plus the
|
|
// logical scope field names derived from the real mapping.
|
|
type mappingInspection struct {
|
|
digest string
|
|
fields searchFieldNames
|
|
}
|
|
|
|
// inspectMapping fetches the index mapping once and derives both the contract
|
|
// digest and the exact-match field names from a single response.
|
|
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
|
|
var response map[string]json.RawMessage
|
|
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
|
|
return mappingInspection{}, err
|
|
}
|
|
indexRaw, ok := response[exporter.index]
|
|
if !ok {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
|
|
}
|
|
var index struct {
|
|
Mappings json.RawMessage `json:"mappings"`
|
|
}
|
|
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
canonical, err := canonicalJSON(index.Mappings)
|
|
if err != nil {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
digest := sha256.Sum256(canonical)
|
|
var mappings struct {
|
|
Properties map[string]struct {
|
|
Type string `json:"type"`
|
|
Fields map[string]struct {
|
|
Type string `json:"type"`
|
|
} `json:"fields"`
|
|
} `json:"properties"`
|
|
}
|
|
if err := json.Unmarshal(canonical, &mappings); err != nil {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
resolve := func(name string) string {
|
|
property, ok := mappings.Properties[name]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
if property.Type == "keyword" {
|
|
return name
|
|
}
|
|
if _, ok := property.Fields["keyword"]; ok {
|
|
return name + ".keyword"
|
|
}
|
|
return name
|
|
}
|
|
fields := searchFieldNames{
|
|
tenant: resolve("tenant_id"),
|
|
knowledge: resolve("knowledge_id"),
|
|
base: resolve("knowledge_base_id"),
|
|
}
|
|
if fields.knowledge == "" || fields.base == "" {
|
|
return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields")
|
|
}
|
|
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
|
|
}
|
|
|
|
// scopedFilter builds the tenant/KB/knowledge term filter adapted to the
|
|
// resolved index field names.
|
|
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
|
|
filter := make([]any, 0, 3)
|
|
if fields.tenant != "" {
|
|
filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}})
|
|
}
|
|
filter = append(filter,
|
|
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
|
|
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
|
|
)
|
|
return filter
|
|
}
|
|
|
|
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
|
|
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
|
|
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
|
|
return nil, errors.New("invalid Elasticsearch base URL")
|
|
}
|
|
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
|
|
if !validSearchIdentifier(config.Index) {
|
|
return nil, errors.New("invalid Elasticsearch index")
|
|
}
|
|
pageSize := config.PageSize
|
|
if pageSize == 0 {
|
|
pageSize = 500
|
|
}
|
|
if pageSize < 1 || pageSize > 1000 {
|
|
return nil, errors.New("invalid Elasticsearch page size")
|
|
}
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
if config.HTTPClient != nil {
|
|
clone := *config.HTTPClient
|
|
client = &clone
|
|
}
|
|
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
|
|
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return SearchSnapshot{}, errors.New("invalid search export identity")
|
|
}
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
filter := scopedFilterFor(inspection.fields, identity)
|
|
documents := make([]SearchDocument, 0)
|
|
for from := 0; ; from += exporter.pageSize {
|
|
payload := map[string]any{
|
|
"from": from,
|
|
"size": exporter.pageSize,
|
|
"query": map[string]any{"bool": map[string]any{"filter": filter}},
|
|
"sort": []any{"_doc"},
|
|
}
|
|
var response searchResponse
|
|
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
for _, hit := range response.Hits.Hits {
|
|
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
|
|
}
|
|
canonical, err := canonicalJSON(hit.Source)
|
|
if err != nil {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
|
|
}
|
|
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
|
|
}
|
|
if len(response.Hits.Hits) < exporter.pageSize {
|
|
break
|
|
}
|
|
if from+exporter.pageSize >= 10000 {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
|
|
}
|
|
}
|
|
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
|
|
digest, err := searchDocumentsDigest(documents)
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return inspection.digest, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
|
|
var body io.Reader
|
|
if payload != nil {
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return errors.New("encode Elasticsearch request")
|
|
}
|
|
body = bytes.NewReader(encoded)
|
|
}
|
|
requestURL := *exporter.baseURL
|
|
relative, err := url.Parse(path)
|
|
if err != nil || relative.IsAbs() || relative.Host != "" {
|
|
return errors.New("invalid Elasticsearch request path")
|
|
}
|
|
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
|
|
requestURL.RawQuery = relative.RawQuery
|
|
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
|
|
if err != nil {
|
|
return errors.New("create Elasticsearch request")
|
|
}
|
|
request.Header.Set("Accept", "application/json")
|
|
if payload != nil {
|
|
request.Header.Set("Content-Type", "application/json")
|
|
}
|
|
if exporter.authorizationToken != "" {
|
|
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
|
|
}
|
|
response, err := exporter.client.Do(request)
|
|
if err != nil {
|
|
return errors.New("Elasticsearch request failed")
|
|
}
|
|
defer response.Body.Close()
|
|
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
|
|
return errors.New("Elasticsearch response origin mismatch")
|
|
}
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
|
|
}
|
|
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
|
|
if err := decoder.Decode(result); err != nil {
|
|
return errors.New("decode Elasticsearch response")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type searchResponse struct {
|
|
Hits struct {
|
|
Hits []struct {
|
|
ID string `json:"_id"`
|
|
Source json.RawMessage `json:"_source"`
|
|
} `json:"hits"`
|
|
} `json:"hits"`
|
|
}
|
|
|
|
// searchHitMatches verifies the returned document belongs to the requested
|
|
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
|
|
// field is absent the KB/knowledge scope is authoritative, matching the
|
|
// export filter semantics. A present tenant must still match exactly.
|
|
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
|
|
var scoped struct {
|
|
TenantID *uint64 `json:"tenant_id"`
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
KnowledgeID string `json:"knowledge_id"`
|
|
}
|
|
if json.Unmarshal(source, &scoped) != nil {
|
|
return false
|
|
}
|
|
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
|
|
return false
|
|
}
|
|
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
|
|
}
|
|
|
|
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
|
|
payload, err := json.Marshal(documents)
|
|
if err != nil {
|
|
return "", errors.New("encode Elasticsearch document digest")
|
|
}
|
|
digest := sha256.Sum256(payload)
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
|
|
var value any
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(value)
|
|
}
|
|
|
|
func validSearchIdentifier(value string) bool {
|
|
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
|
|
return false
|
|
}
|
|
for _, char := range value {
|
|
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|