ac3b395102
build / build (push) Successful in 1m21s
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设, 以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。 - knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。 - 写入顺序按真实外键;无代理键表用内容摘要核对。 - 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。 - 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。 - 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。 - finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。 - 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
342 lines
12 KiB
Go
342 lines
12 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const maxSearchResponseBytes = 64 * 1024 * 1024
|
|
|
|
type SearchIdentity struct {
|
|
TenantID uint64
|
|
KnowledgeBaseID string
|
|
KnowledgeID string
|
|
}
|
|
|
|
type SearchDocument struct {
|
|
ID string
|
|
Source json.RawMessage
|
|
}
|
|
|
|
type SearchSnapshot struct {
|
|
Documents []SearchDocument
|
|
MappingSHA256 string
|
|
DocumentsSHA256 string
|
|
}
|
|
|
|
type SearchExporterConfig struct {
|
|
BaseURL string
|
|
Index string
|
|
AuthorizationToken string
|
|
HTTPClient *http.Client
|
|
PageSize int
|
|
}
|
|
|
|
type SearchExporter struct {
|
|
baseURL *url.URL
|
|
index string
|
|
authorizationToken string
|
|
client *http.Client
|
|
pageSize int
|
|
}
|
|
|
|
// searchFieldNames reflects the real v0.7.1 index shape:
|
|
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
|
|
// default) or as plain "keyword"; exact filters must target the right one.
|
|
// - Documents may carry no tenant_id field at all; tenancy is then implied
|
|
// by the knowledge_base_id/knowledge_id scope.
|
|
type searchFieldNames struct {
|
|
tenant string // "" when the index has no tenant_id field
|
|
knowledge string
|
|
base string
|
|
}
|
|
|
|
// mappingInspection is the canonical digest of the index mappings plus the
|
|
// logical scope field names derived from the real mapping.
|
|
// errIndexMissing 表示目标索引根本不存在(空生产基线)。
|
|
var errIndexMissing = errors.New("Elasticsearch index missing")
|
|
|
|
type mappingInspection struct {
|
|
digest string
|
|
fields searchFieldNames
|
|
}
|
|
|
|
// inspectMapping fetches the index mapping once and derives both the contract
|
|
// digest and the exact-match field names from a single response.
|
|
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
|
|
var response map[string]json.RawMessage
|
|
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
|
|
if errors.Is(err, errIndexMissing) {
|
|
return mappingInspection{}, errIndexMissing
|
|
}
|
|
return mappingInspection{}, err
|
|
}
|
|
indexRaw, ok := response[exporter.index]
|
|
if !ok {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
|
|
}
|
|
var index struct {
|
|
Mappings json.RawMessage `json:"mappings"`
|
|
}
|
|
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
canonical, err := canonicalJSON(index.Mappings)
|
|
if err != nil {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
digest := sha256.Sum256(canonical)
|
|
var mappings struct {
|
|
Properties map[string]struct {
|
|
Type string `json:"type"`
|
|
Fields map[string]struct {
|
|
Type string `json:"type"`
|
|
} `json:"fields"`
|
|
} `json:"properties"`
|
|
}
|
|
if err := json.Unmarshal(canonical, &mappings); err != nil {
|
|
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
|
}
|
|
resolve := func(name string) string {
|
|
property, ok := mappings.Properties[name]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
if property.Type == "keyword" {
|
|
return name
|
|
}
|
|
if _, ok := property.Fields["keyword"]; ok {
|
|
return name + ".keyword"
|
|
}
|
|
return name
|
|
}
|
|
fields := searchFieldNames{
|
|
tenant: resolve("tenant_id"),
|
|
knowledge: resolve("knowledge_id"),
|
|
base: resolve("knowledge_base_id"),
|
|
}
|
|
if fields.knowledge == "" || fields.base == "" {
|
|
return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields")
|
|
}
|
|
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
|
|
}
|
|
|
|
// scopedFilter builds the tenant/KB/knowledge term filter adapted to the
|
|
// resolved index field names.
|
|
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
|
|
filter := make([]any, 0, 3)
|
|
if fields.tenant != "" {
|
|
filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}})
|
|
}
|
|
filter = append(filter,
|
|
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
|
|
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
|
|
)
|
|
return filter
|
|
}
|
|
|
|
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
|
|
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
|
|
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
|
|
return nil, errors.New("invalid Elasticsearch base URL")
|
|
}
|
|
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
|
|
if !validSearchIdentifier(config.Index) {
|
|
return nil, errors.New("invalid Elasticsearch index")
|
|
}
|
|
pageSize := config.PageSize
|
|
if pageSize == 0 {
|
|
pageSize = 500
|
|
}
|
|
if pageSize < 1 || pageSize > 1000 {
|
|
return nil, errors.New("invalid Elasticsearch page size")
|
|
}
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
if config.HTTPClient != nil {
|
|
clone := *config.HTTPClient
|
|
client = &clone
|
|
}
|
|
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
|
|
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
|
|
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
|
return SearchSnapshot{}, errors.New("invalid search export identity")
|
|
}
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if errors.Is(err, errIndexMissing) {
|
|
// 索引不存在 = 空基线:零文档、无映射可比。调用方必须再确认数据库也是空的。
|
|
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
|
|
if digestErr != nil {
|
|
return SearchSnapshot{}, digestErr
|
|
}
|
|
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
|
|
}
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
filter := scopedFilterFor(inspection.fields, identity)
|
|
documents := make([]SearchDocument, 0)
|
|
for from := 0; ; from += exporter.pageSize {
|
|
payload := map[string]any{
|
|
"from": from,
|
|
"size": exporter.pageSize,
|
|
"query": map[string]any{"bool": map[string]any{"filter": filter}},
|
|
"sort": []any{"_doc"},
|
|
}
|
|
var response searchResponse
|
|
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
for _, hit := range response.Hits.Hits {
|
|
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
|
|
}
|
|
canonical, err := canonicalJSON(hit.Source)
|
|
if err != nil {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
|
|
}
|
|
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
|
|
}
|
|
if len(response.Hits.Hits) < exporter.pageSize {
|
|
break
|
|
}
|
|
if from+exporter.pageSize >= 10000 {
|
|
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
|
|
}
|
|
}
|
|
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
|
|
digest, err := searchDocumentsDigest(documents)
|
|
if err != nil {
|
|
return SearchSnapshot{}, err
|
|
}
|
|
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
|
|
inspection, err := exporter.inspectMapping(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return inspection.digest, nil
|
|
}
|
|
|
|
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
|
|
var body io.Reader
|
|
if payload != nil {
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return errors.New("encode Elasticsearch request")
|
|
}
|
|
body = bytes.NewReader(encoded)
|
|
}
|
|
requestURL := *exporter.baseURL
|
|
relative, err := url.Parse(path)
|
|
if err != nil || relative.IsAbs() || relative.Host != "" {
|
|
return errors.New("invalid Elasticsearch request path")
|
|
}
|
|
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
|
|
requestURL.RawQuery = relative.RawQuery
|
|
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
|
|
if err != nil {
|
|
return errors.New("create Elasticsearch request")
|
|
}
|
|
request.Header.Set("Accept", "application/json")
|
|
if payload != nil {
|
|
request.Header.Set("Content-Type", "application/json")
|
|
}
|
|
if exporter.authorizationToken != "" {
|
|
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
|
|
}
|
|
response, err := exporter.client.Do(request)
|
|
if err != nil {
|
|
return errors.New("Elasticsearch request failed")
|
|
}
|
|
defer response.Body.Close()
|
|
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
|
|
return errors.New("Elasticsearch response origin mismatch")
|
|
}
|
|
if response.StatusCode == http.StatusNotFound {
|
|
// 索引不存在是"空生产"的正常状态,不是故障;调用方必须显式处理,
|
|
// 不能把 404 当成功,也不能把它当传输错误。
|
|
return fmt.Errorf("%w: HTTP 404", errIndexMissing)
|
|
}
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
|
|
}
|
|
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
|
|
if err := decoder.Decode(result); err != nil {
|
|
return errors.New("decode Elasticsearch response")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type searchResponse struct {
|
|
Hits struct {
|
|
Hits []struct {
|
|
ID string `json:"_id"`
|
|
Source json.RawMessage `json:"_source"`
|
|
} `json:"hits"`
|
|
} `json:"hits"`
|
|
}
|
|
|
|
// searchHitMatches verifies the returned document belongs to the requested
|
|
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
|
|
// field is absent the KB/knowledge scope is authoritative, matching the
|
|
// export filter semantics. A present tenant must still match exactly.
|
|
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
|
|
var scoped struct {
|
|
TenantID *uint64 `json:"tenant_id"`
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
KnowledgeID string `json:"knowledge_id"`
|
|
}
|
|
if json.Unmarshal(source, &scoped) != nil {
|
|
return false
|
|
}
|
|
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
|
|
return false
|
|
}
|
|
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
|
|
}
|
|
|
|
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
|
|
payload, err := json.Marshal(documents)
|
|
if err != nil {
|
|
return "", errors.New("encode Elasticsearch document digest")
|
|
}
|
|
digest := sha256.Sum256(payload)
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
|
|
var value any
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(value)
|
|
}
|
|
|
|
func validSearchIdentifier(value string) bool {
|
|
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
|
|
return false
|
|
}
|
|
for _, char := range value {
|
|
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|