dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
180 lines
7.2 KiB
Go
180 lines
7.2 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
)
|
|
|
|
type SearchImporter struct {
|
|
exporter *SearchExporter
|
|
}
|
|
|
|
func NewSearchImporter(config SearchExporterConfig) (*SearchImporter, error) {
|
|
exporter, err := NewSearchExporter(config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &SearchImporter{exporter: exporter}, nil
|
|
}
|
|
|
|
func (importer *SearchImporter) Replace(ctx context.Context, identity SearchIdentity, snapshot SearchSnapshot) (SearchSnapshot, error) {
|
|
if err := validateSearchSnapshot(snapshot, identity); err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
prior, err := importer.exporter.Export(ctx, identity)
|
|
if err != nil {
|
|
return SearchSnapshot{}, errors.New("read prior Elasticsearch document snapshot")
|
|
}
|
|
if prior.MappingSHA256 != snapshot.MappingSHA256 {
|
|
return SearchSnapshot{}, errors.New("target Elasticsearch mapping contract mismatch")
|
|
}
|
|
if err := importer.writeSnapshot(ctx, identity, snapshot); err != nil {
|
|
if restoreErr := importer.writeSnapshot(ctx, identity, prior); restoreErr != nil {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch replacement and compensation failed")
|
|
}
|
|
return SearchSnapshot{}, err
|
|
}
|
|
actual, err := importer.exporter.Export(ctx, identity)
|
|
if err != nil || actual.DocumentsSHA256 != snapshot.DocumentsSHA256 || len(actual.Documents) != len(snapshot.Documents) {
|
|
if restoreErr := importer.writeSnapshot(ctx, identity, prior); restoreErr != nil {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch verification and compensation failed")
|
|
}
|
|
return SearchSnapshot{}, errors.New("Elasticsearch replacement verification failed")
|
|
}
|
|
return prior, nil
|
|
}
|
|
|
|
func (importer *SearchImporter) Disable(ctx context.Context, identity SearchIdentity) error {
|
|
prior, err := importer.exporter.Export(ctx, identity)
|
|
if err != nil || len(prior.Documents) == 0 {
|
|
return errors.New("target Elasticsearch document not found")
|
|
}
|
|
inspection, err := importer.exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return errors.New("resolve target Elasticsearch scope fields")
|
|
}
|
|
filter := scopedFilterFor(inspection.fields, identity)
|
|
var result struct {
|
|
Failures []json.RawMessage `json:"failures"`
|
|
}
|
|
payload := map[string]any{
|
|
"script": map[string]any{"lang": "painless", "source": "ctx._source.is_enabled = false"},
|
|
"query": map[string]any{"bool": map[string]any{"filter": filter}},
|
|
}
|
|
if err := importer.exporter.requestJSON(ctx, http.MethodPost, "/"+importer.exporter.index+"/_update_by_query?refresh=true&conflicts=proceed", payload, &result); err != nil || len(result.Failures) > 0 {
|
|
return errors.New("disable target Elasticsearch documents")
|
|
}
|
|
actual, err := importer.exporter.Export(ctx, identity)
|
|
if err != nil || len(actual.Documents) != len(prior.Documents) {
|
|
return errors.New("verify disabled Elasticsearch documents")
|
|
}
|
|
for _, document := range actual.Documents {
|
|
var state struct {
|
|
IsEnabled *bool `json:"is_enabled"`
|
|
}
|
|
if json.Unmarshal(document.Source, &state) != nil || state.IsEnabled == nil || *state.IsEnabled {
|
|
return errors.New("verify disabled Elasticsearch documents")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateSearchSnapshot(snapshot SearchSnapshot, identity SearchIdentity) error {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) ||
|
|
!validLowerSHA256(snapshot.MappingSHA256) || !validLowerSHA256(snapshot.DocumentsSHA256) {
|
|
return errors.New("invalid Elasticsearch import snapshot")
|
|
}
|
|
digest, err := searchDocumentsDigest(snapshot.Documents)
|
|
if err != nil || digest != snapshot.DocumentsSHA256 {
|
|
return errors.New("invalid Elasticsearch import snapshot")
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, document := range snapshot.Documents {
|
|
if !validSearchIdentifier(document.ID) || seen[document.ID] || !searchHitMatches(document.Source, identity) {
|
|
return errors.New("cross-scope Elasticsearch import document")
|
|
}
|
|
seen[document.ID] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (importer *SearchImporter) writeSnapshot(ctx context.Context, identity SearchIdentity, snapshot SearchSnapshot) error {
|
|
if len(snapshot.Documents) > 0 {
|
|
var body bytes.Buffer
|
|
encoder := json.NewEncoder(&body)
|
|
for _, document := range snapshot.Documents {
|
|
if err := encoder.Encode(map[string]any{"index": map[string]string{"_index": importer.exporter.index, "_id": document.ID}}); err != nil {
|
|
return errors.New("encode Elasticsearch bulk release")
|
|
}
|
|
body.Write(document.Source)
|
|
body.WriteByte('\n')
|
|
}
|
|
var result struct {
|
|
Errors bool `json:"errors"`
|
|
}
|
|
if err := importer.requestRawJSON(ctx, http.MethodPost, "/_bulk?refresh=true", &body, "application/x-ndjson", &result); err != nil || result.Errors {
|
|
return errors.New("write Elasticsearch release documents")
|
|
}
|
|
}
|
|
inspection, err := importer.exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return errors.New("resolve target Elasticsearch scope fields")
|
|
}
|
|
filter := scopedFilterFor(inspection.fields, identity)
|
|
boolQuery := map[string]any{"filter": filter}
|
|
if len(snapshot.Documents) > 0 {
|
|
ids := make([]string, len(snapshot.Documents))
|
|
for index, document := range snapshot.Documents {
|
|
ids[index] = document.ID
|
|
}
|
|
boolQuery["must_not"] = []any{map[string]any{"ids": map[string]any{"values": ids}}}
|
|
}
|
|
var deleted struct {
|
|
Failures []json.RawMessage `json:"failures"`
|
|
}
|
|
if err := importer.exporter.requestJSON(ctx, http.MethodPost, "/"+importer.exporter.index+"/_delete_by_query?refresh=true&conflicts=proceed", map[string]any{"query": map[string]any{"bool": boolQuery}}, &deleted); err != nil || len(deleted.Failures) > 0 {
|
|
return errors.New("remove obsolete Elasticsearch release documents")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (importer *SearchImporter) requestRawJSON(ctx context.Context, method, requestPath string, body io.Reader, contentType string, result any) error {
|
|
relative, err := url.Parse(requestPath)
|
|
if err != nil || relative.IsAbs() || relative.Host != "" {
|
|
return errors.New("invalid Elasticsearch import path")
|
|
}
|
|
requestURL := *importer.exporter.baseURL
|
|
requestURL.Path = strings.TrimRight(importer.exporter.baseURL.Path, "/") + relative.Path
|
|
requestURL.RawQuery = relative.RawQuery
|
|
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
|
|
if err != nil {
|
|
return errors.New("create Elasticsearch import request")
|
|
}
|
|
request.Header.Set("Accept", "application/json")
|
|
request.Header.Set("Content-Type", contentType)
|
|
if importer.exporter.authorizationToken != "" {
|
|
request.Header.Set("Authorization", "Bearer "+importer.exporter.authorizationToken)
|
|
}
|
|
response, err := importer.exporter.client.Do(request)
|
|
if err != nil {
|
|
return errors.New("Elasticsearch import request failed")
|
|
}
|
|
defer response.Body.Close()
|
|
if response.Request == nil || response.Request.URL.Scheme != importer.exporter.baseURL.Scheme || response.Request.URL.Host != importer.exporter.baseURL.Host {
|
|
return errors.New("Elasticsearch import response origin mismatch")
|
|
}
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return errors.New("Elasticsearch import returned non-success status")
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1)).Decode(result); err != nil {
|
|
return errors.New("decode Elasticsearch import response")
|
|
}
|
|
return nil
|
|
}
|