7d4ffdbfe5
build / build (push) Successful in 1m15s
全新 WeKnora 会用空映射({})先建好索引,范围字段等第一次写入才动态生成。
首次同步在读目标当前状态时就报 "missing release scope fields" 并失败,等于
"往空的生产实例做第一次同步"这条路根本走不通。
现在:"没有范围字段 + 索引确实 0 篇文档" = 空基线;有文档却看不出归属仍然报错
(绝不把无法归属的文档当空库覆盖)。Count 与整库导出共用同一判断。
试过:make verify 15 项全绿;新增空/非空两种索引的用例;真实 sync 复现原错误。
383 lines
13 KiB
Go
383 lines
13 KiB
Go
package v071
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
const maxSearchResponseBytes = 64 * 1024 * 1024
|
||
|
||
type SearchIdentity struct {
|
||
TenantID uint64
|
||
KnowledgeBaseID string
|
||
KnowledgeID string
|
||
}
|
||
|
||
type SearchDocument struct {
|
||
ID string
|
||
Source json.RawMessage
|
||
}
|
||
|
||
type SearchSnapshot struct {
|
||
Documents []SearchDocument
|
||
MappingSHA256 string
|
||
DocumentsSHA256 string
|
||
}
|
||
|
||
type SearchExporterConfig struct {
|
||
BaseURL string
|
||
Index string
|
||
AuthorizationToken string
|
||
HTTPClient *http.Client
|
||
PageSize int
|
||
}
|
||
|
||
type SearchExporter struct {
|
||
baseURL *url.URL
|
||
index string
|
||
authorizationToken string
|
||
client *http.Client
|
||
pageSize int
|
||
}
|
||
|
||
// searchFieldNames reflects the real v0.7.1 index shape:
|
||
// - ID fields may be mapped as "text" with a ".keyword" subfield (community
|
||
// default) or as plain "keyword"; exact filters must target the right one.
|
||
// - Documents may carry no tenant_id field at all; tenancy is then implied
|
||
// by the knowledge_base_id/knowledge_id scope.
|
||
type searchFieldNames struct {
|
||
tenant string // "" when the index has no tenant_id field
|
||
knowledge string
|
||
base string
|
||
}
|
||
|
||
// mappingInspection is the canonical digest of the index mappings plus the
|
||
// logical scope field names derived from the real mapping.
|
||
// errIndexMissing 表示目标索引根本不存在(空生产基线)。
|
||
var errIndexMissing = errors.New("Elasticsearch index missing")
|
||
|
||
// errScopeFieldsMissing 表示索引存在但还没有发布范围字段:
|
||
// 全新的 WeKnora 实例会先用空映射建好索引(`{}`),字段要等第一次写入才动态生成。
|
||
// 这跟"索引被改坏了"必须分开:只有在索引里确实一篇文档都没有时才能当成空基线。
|
||
var errScopeFieldsMissing = errors.New("Elasticsearch index mapping is missing release scope fields")
|
||
|
||
// emptyScopeIndex 判断"没有范围字段"的索引是不是真的空。
|
||
// 有文档却没有范围字段 = 无法判断这批文档属于谁,必须报错,不能当成 0。
|
||
func (exporter *SearchExporter) emptyScopeIndex(ctx context.Context) (bool, error) {
|
||
var response struct {
|
||
Count int64 `json:"count"`
|
||
}
|
||
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_count", nil, &response); err != nil {
|
||
return false, err
|
||
}
|
||
return response.Count == 0, nil
|
||
}
|
||
|
||
type mappingInspection struct {
|
||
digest string
|
||
fields searchFieldNames
|
||
}
|
||
|
||
// inspectMapping fetches the index mapping once and derives both the contract
|
||
// digest and the exact-match field names from a single response.
|
||
func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInspection, error) {
|
||
var response map[string]json.RawMessage
|
||
if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_mapping", nil, &response); err != nil {
|
||
if errors.Is(err, errIndexMissing) {
|
||
return mappingInspection{}, errIndexMissing
|
||
}
|
||
return mappingInspection{}, err
|
||
}
|
||
indexRaw, ok := response[exporter.index]
|
||
if !ok {
|
||
return mappingInspection{}, errors.New("Elasticsearch mapping response missing configured index")
|
||
}
|
||
var index struct {
|
||
Mappings json.RawMessage `json:"mappings"`
|
||
}
|
||
if err := json.Unmarshal(indexRaw, &index); err != nil || len(index.Mappings) == 0 {
|
||
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
||
}
|
||
canonical, err := canonicalJSON(index.Mappings)
|
||
if err != nil {
|
||
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
||
}
|
||
digest := sha256.Sum256(canonical)
|
||
var mappings struct {
|
||
Properties map[string]struct {
|
||
Type string `json:"type"`
|
||
Fields map[string]struct {
|
||
Type string `json:"type"`
|
||
} `json:"fields"`
|
||
} `json:"properties"`
|
||
}
|
||
if err := json.Unmarshal(canonical, &mappings); err != nil {
|
||
return mappingInspection{}, errors.New("Elasticsearch mapping response is invalid")
|
||
}
|
||
resolve := func(name string) string {
|
||
property, ok := mappings.Properties[name]
|
||
if !ok {
|
||
return ""
|
||
}
|
||
if property.Type == "keyword" {
|
||
return name
|
||
}
|
||
if _, ok := property.Fields["keyword"]; ok {
|
||
return name + ".keyword"
|
||
}
|
||
return name
|
||
}
|
||
fields := searchFieldNames{
|
||
tenant: resolve("tenant_id"),
|
||
knowledge: resolve("knowledge_id"),
|
||
base: resolve("knowledge_base_id"),
|
||
}
|
||
if fields.knowledge == "" || fields.base == "" {
|
||
return mappingInspection{}, errScopeFieldsMissing
|
||
}
|
||
return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil
|
||
}
|
||
|
||
// tenantClause 是"租户范围"这一条过滤条件的唯一写法。
|
||
//
|
||
// 为什么不是简单的 term:真实 v0.7.1 的 app 自己写索引时不带 tenant_id,而治理侧
|
||
// 同步写入时会补齐。于是 "mapping 里有 tenant_id" 不等于 "每篇文档都写了 tenant_id"。
|
||
// 只按 term 过滤会把 app 写入的正文整库漏掉——真跑踩过:索引里明明有文档,导出 0 条,
|
||
// 候选校验报 legacy coverage。这里改成:
|
||
//
|
||
// 显式写了本租户 → 收;完全没写 → 收(没有可比的租户声明);写了别的租户 → 排除。
|
||
//
|
||
// 知识库身份仍然是一条独立的 term,所以范围始终由知识库唯一 ID 收窄,不是放宽成"全库"。
|
||
func tenantClause(fields searchFieldNames, tenant uint64) any {
|
||
if fields.tenant == "" {
|
||
return nil
|
||
}
|
||
return map[string]any{"bool": map[string]any{
|
||
"should": []any{
|
||
map[string]any{"term": map[string]any{fields.tenant: tenant}},
|
||
map[string]any{"bool": map[string]any{"must_not": []any{
|
||
map[string]any{"exists": map[string]any{"field": fields.tenant}},
|
||
}}},
|
||
},
|
||
"minimum_should_match": 1,
|
||
}}
|
||
}
|
||
|
||
func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any {
|
||
// 租户 + 知识库 + 单文档三层收窄:tenant 那条见 tenantClause(app 写入的文档不带租户字段)。
|
||
filter := make([]any, 0, 3)
|
||
if clause := tenantClause(fields, identity.TenantID); clause != nil {
|
||
filter = append(filter, clause)
|
||
}
|
||
filter = append(filter,
|
||
map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}},
|
||
map[string]any{"term": map[string]any{fields.knowledge: identity.KnowledgeID}},
|
||
)
|
||
return filter
|
||
}
|
||
|
||
func NewSearchExporter(config SearchExporterConfig) (*SearchExporter, error) {
|
||
baseURL, err := url.Parse(strings.TrimSpace(config.BaseURL))
|
||
if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" || baseURL.User != nil || baseURL.RawQuery != "" || baseURL.Fragment != "" {
|
||
return nil, errors.New("invalid Elasticsearch base URL")
|
||
}
|
||
baseURL.Path = strings.TrimRight(baseURL.Path, "/")
|
||
if !validSearchIdentifier(config.Index) {
|
||
return nil, errors.New("invalid Elasticsearch index")
|
||
}
|
||
pageSize := config.PageSize
|
||
if pageSize == 0 {
|
||
pageSize = 500
|
||
}
|
||
if pageSize < 1 || pageSize > 1000 {
|
||
return nil, errors.New("invalid Elasticsearch page size")
|
||
}
|
||
client := &http.Client{Timeout: 30 * time.Second}
|
||
if config.HTTPClient != nil {
|
||
clone := *config.HTTPClient
|
||
client = &clone
|
||
}
|
||
client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("Elasticsearch redirect rejected") }
|
||
return &SearchExporter{baseURL: baseURL, index: config.Index, authorizationToken: config.AuthorizationToken, client: client, pageSize: pageSize}, nil
|
||
}
|
||
|
||
func (exporter *SearchExporter) Export(ctx context.Context, identity SearchIdentity) (SearchSnapshot, error) {
|
||
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || !validSearchIdentifier(identity.KnowledgeID) {
|
||
return SearchSnapshot{}, errors.New("invalid search export identity")
|
||
}
|
||
inspection, err := exporter.inspectMapping(ctx)
|
||
if errors.Is(err, errIndexMissing) {
|
||
// 索引不存在 = 空基线:零文档、无映射可比。调用方必须再确认数据库也是空的。
|
||
emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{})
|
||
if digestErr != nil {
|
||
return SearchSnapshot{}, digestErr
|
||
}
|
||
return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil
|
||
}
|
||
if err != nil {
|
||
return SearchSnapshot{}, err
|
||
}
|
||
filter := scopedFilterFor(inspection.fields, identity)
|
||
documents := make([]SearchDocument, 0)
|
||
for from := 0; ; from += exporter.pageSize {
|
||
payload := map[string]any{
|
||
"from": from,
|
||
"size": exporter.pageSize,
|
||
"query": map[string]any{"bool": map[string]any{"filter": filter}},
|
||
"sort": []any{"_doc"},
|
||
}
|
||
var response searchResponse
|
||
if err := exporter.requestJSON(ctx, http.MethodPost, "/"+exporter.index+"/_search", payload, &response); err != nil {
|
||
return SearchSnapshot{}, err
|
||
}
|
||
for _, hit := range response.Hits.Hits {
|
||
if !validSearchIdentifier(hit.ID) || !searchHitMatches(hit.Source, identity) {
|
||
return SearchSnapshot{}, errors.New("Elasticsearch returned a cross-scope document")
|
||
}
|
||
canonical, err := canonicalJSON(hit.Source)
|
||
if err != nil {
|
||
return SearchSnapshot{}, errors.New("Elasticsearch returned an invalid document")
|
||
}
|
||
documents = append(documents, SearchDocument{ID: hit.ID, Source: canonical})
|
||
}
|
||
if len(response.Hits.Hits) < exporter.pageSize {
|
||
break
|
||
}
|
||
if from+exporter.pageSize >= 10000 {
|
||
return SearchSnapshot{}, errors.New("Elasticsearch document count exceeds safe export window")
|
||
}
|
||
}
|
||
sort.Slice(documents, func(left, right int) bool { return documents[left].ID < documents[right].ID })
|
||
digest, err := searchDocumentsDigest(documents)
|
||
if err != nil {
|
||
return SearchSnapshot{}, err
|
||
}
|
||
return SearchSnapshot{Documents: documents, MappingSHA256: inspection.digest, DocumentsSHA256: digest}, nil
|
||
}
|
||
|
||
func (exporter *SearchExporter) MappingSHA256(ctx context.Context) (string, error) {
|
||
inspection, err := exporter.inspectMapping(ctx)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return inspection.digest, nil
|
||
}
|
||
|
||
func (exporter *SearchExporter) requestJSON(ctx context.Context, method, path string, payload any, result any) error {
|
||
var body io.Reader
|
||
if payload != nil {
|
||
encoded, err := json.Marshal(payload)
|
||
if err != nil {
|
||
return errors.New("encode Elasticsearch request")
|
||
}
|
||
body = bytes.NewReader(encoded)
|
||
}
|
||
requestURL := *exporter.baseURL
|
||
relative, err := url.Parse(path)
|
||
if err != nil || relative.IsAbs() || relative.Host != "" {
|
||
return errors.New("invalid Elasticsearch request path")
|
||
}
|
||
requestURL.Path = strings.TrimRight(exporter.baseURL.Path, "/") + relative.Path
|
||
requestURL.RawQuery = relative.RawQuery
|
||
request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
|
||
if err != nil {
|
||
return errors.New("create Elasticsearch request")
|
||
}
|
||
request.Header.Set("Accept", "application/json")
|
||
if payload != nil {
|
||
request.Header.Set("Content-Type", "application/json")
|
||
}
|
||
if exporter.authorizationToken != "" {
|
||
request.Header.Set("Authorization", "Bearer "+exporter.authorizationToken)
|
||
}
|
||
response, err := exporter.client.Do(request)
|
||
if err != nil {
|
||
return errors.New("Elasticsearch request failed")
|
||
}
|
||
defer response.Body.Close()
|
||
if response.Request == nil || response.Request.URL.Scheme != exporter.baseURL.Scheme || response.Request.URL.Host != exporter.baseURL.Host {
|
||
return errors.New("Elasticsearch response origin mismatch")
|
||
}
|
||
if response.StatusCode == http.StatusNotFound {
|
||
// 索引不存在是"空生产"的正常状态,不是故障;调用方必须显式处理,
|
||
// 不能把 404 当成功,也不能把它当传输错误。
|
||
return fmt.Errorf("%w: HTTP 404", errIndexMissing)
|
||
}
|
||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||
return fmt.Errorf("Elasticsearch returned HTTP %d", response.StatusCode)
|
||
}
|
||
decoder := json.NewDecoder(io.LimitReader(response.Body, maxSearchResponseBytes+1))
|
||
if err := decoder.Decode(result); err != nil {
|
||
return errors.New("decode Elasticsearch response")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type searchResponse struct {
|
||
Hits struct {
|
||
Hits []struct {
|
||
ID string `json:"_id"`
|
||
Source json.RawMessage `json:"_source"`
|
||
} `json:"hits"`
|
||
} `json:"hits"`
|
||
}
|
||
|
||
// searchHitMatches verifies the returned document belongs to the requested
|
||
// scope. Real v0.7.1 chunk documents may omit tenant_id entirely; when the
|
||
// field is absent the KB/knowledge scope is authoritative, matching the
|
||
// export filter semantics. A present tenant must still match exactly.
|
||
func searchHitMatches(source json.RawMessage, identity SearchIdentity) bool {
|
||
var scoped struct {
|
||
TenantID *uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
}
|
||
if json.Unmarshal(source, &scoped) != nil {
|
||
return false
|
||
}
|
||
if scoped.TenantID != nil && *scoped.TenantID != identity.TenantID {
|
||
return false
|
||
}
|
||
return scoped.KnowledgeBaseID == identity.KnowledgeBaseID && scoped.KnowledgeID == identity.KnowledgeID
|
||
}
|
||
|
||
func searchDocumentsDigest(documents []SearchDocument) (string, error) {
|
||
payload, err := json.Marshal(documents)
|
||
if err != nil {
|
||
return "", errors.New("encode Elasticsearch document digest")
|
||
}
|
||
digest := sha256.Sum256(payload)
|
||
return hex.EncodeToString(digest[:]), nil
|
||
}
|
||
|
||
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||
var value any
|
||
if err := json.Unmarshal(raw, &value); err != nil {
|
||
return nil, err
|
||
}
|
||
return json.Marshal(value)
|
||
}
|
||
|
||
func validSearchIdentifier(value string) bool {
|
||
if value == "" || len(value) > 128 || value != strings.TrimSpace(value) {
|
||
return false
|
||
}
|
||
for _, char := range value {
|
||
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') && char != '-' && char != '_' && char != '.' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|