dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
221 lines
7.1 KiB
Go
221 lines
7.1 KiB
Go
package v071
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
func TestSearchImporterReplacesAndRestoresScopedDocuments(t *testing.T) {
|
|
state := newSearchState(map[string]json.RawMessage{"es-old": mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-old","embedding":[0.1]}`)[0], "other": mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"other-kb","knowledge_id":"other","chunk_id":"other","embedding":[0.9]}`)[0]})
|
|
server := httptest.NewServer(state)
|
|
defer server.Close()
|
|
config := SearchExporterConfig{BaseURL: server.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: server.Client(), PageSize: 100}
|
|
importer, err := NewSearchImporter(config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
identity := SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"}
|
|
exporter, _ := NewSearchExporter(config)
|
|
mapping, err := exporter.MappingSHA256(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
newDocs := []SearchDocument{{ID: "es-new", Source: mustCanonicalRows(t, `{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-new","embedding":[0.2]}`)[0]}}
|
|
newDigest, _ := searchDocumentsDigest(newDocs)
|
|
prior, err := importer.Replace(context.Background(), identity, SearchSnapshot{Documents: newDocs, MappingSHA256: mapping, DocumentsSHA256: newDigest})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(prior.Documents) != 1 || prior.Documents[0].ID != "es-old" {
|
|
t.Fatalf("prior=%#v", prior)
|
|
}
|
|
current, err := exporter.Export(context.Background(), identity)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(current.Documents) != 1 || current.Documents[0].ID != "es-new" || state.has("other") == false {
|
|
t.Fatalf("current=%#v other=%v", current, state.has("other"))
|
|
}
|
|
if _, err := importer.Replace(context.Background(), identity, prior); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
restored, err := exporter.Export(context.Background(), identity)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(restored.Documents) != 1 || restored.Documents[0].ID != "es-old" || state.has("es-new") {
|
|
t.Fatalf("restored=%#v", restored)
|
|
}
|
|
if err := importer.Disable(context.Background(), identity); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !state.disabled("es-old") || !state.has("es-old") || !state.has("other") {
|
|
t.Fatalf("disable state old=%v other=%v", state.disabled("es-old"), state.has("other"))
|
|
}
|
|
for _, requestPath := range state.paths {
|
|
if strings.Contains(requestPath, "knowledge-safe") || strings.Contains(requestPath, "kb-safe") {
|
|
t.Fatalf("identity leaked in URL: %s", requestPath)
|
|
}
|
|
}
|
|
joinedPaths := strings.Join(state.paths, "\n")
|
|
if !strings.Contains(joinedPaths, "/_bulk?refresh=true") || !strings.Contains(joinedPaths, "/chunks/_delete_by_query?refresh=true") {
|
|
t.Fatalf("refresh contract missing: %s", joinedPaths)
|
|
}
|
|
}
|
|
|
|
type searchState struct {
|
|
mu sync.Mutex
|
|
docs map[string]json.RawMessage
|
|
paths []string
|
|
}
|
|
|
|
func newSearchState(docs map[string]json.RawMessage) *searchState { return &searchState{docs: docs} }
|
|
func (s *searchState) has(id string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
_, ok := s.docs[id]
|
|
return ok
|
|
}
|
|
|
|
func (s *searchState) disabled(id string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var value struct {
|
|
IsEnabled bool `json:"is_enabled"`
|
|
}
|
|
return json.Unmarshal(s.docs[id], &value) == nil && !value.IsEnabled
|
|
}
|
|
func (s *searchState) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.paths = append(s.paths, r.URL.RequestURI())
|
|
if r.Header.Get("Authorization") != "Bearer safe-token" {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
switch r.URL.Path {
|
|
case "/chunks/_mapping":
|
|
_, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`))
|
|
case "/chunks/_search":
|
|
var q map[string]any
|
|
_ = json.NewDecoder(r.Body).Decode(&q)
|
|
identity := identityFromQuery(q)
|
|
hits := []map[string]any{}
|
|
for id, source := range s.docs {
|
|
if sourceMatches(source, identity) {
|
|
hits = append(hits, map[string]any{"_id": id, "_source": json.RawMessage(source)})
|
|
}
|
|
}
|
|
_, _ = w.Write(mustJSON(map[string]any{"hits": map[string]any{"hits": hits}}))
|
|
case "/_bulk":
|
|
scanner := bufio.NewScanner(r.Body)
|
|
for scanner.Scan() {
|
|
var action struct {
|
|
Index struct {
|
|
ID string `json:"_id"`
|
|
} `json:"index"`
|
|
}
|
|
_ = json.Unmarshal(scanner.Bytes(), &action)
|
|
if !scanner.Scan() {
|
|
break
|
|
}
|
|
s.docs[action.Index.ID] = append([]byte(nil), scanner.Bytes()...)
|
|
}
|
|
_, _ = w.Write([]byte(`{"errors":false,"items":[]}`))
|
|
case "/chunks/_delete_by_query":
|
|
var q map[string]any
|
|
_ = json.NewDecoder(r.Body).Decode(&q)
|
|
identity := identityFromQuery(q)
|
|
keep := idsFromQuery(q)
|
|
for id, source := range s.docs {
|
|
if sourceMatches(source, identity) && !keep[id] {
|
|
delete(s.docs, id)
|
|
}
|
|
}
|
|
_, _ = w.Write([]byte(`{"failures":[]}`))
|
|
case "/chunks/_update_by_query":
|
|
var q map[string]any
|
|
_ = json.NewDecoder(r.Body).Decode(&q)
|
|
identity := identityFromQuery(q)
|
|
for id, source := range s.docs {
|
|
if sourceMatches(source, identity) {
|
|
var value map[string]any
|
|
_ = json.Unmarshal(source, &value)
|
|
value["is_enabled"] = false
|
|
s.docs[id] = mustJSON(value)
|
|
}
|
|
}
|
|
_, _ = w.Write([]byte(`{"failures":[]}`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}
|
|
func identityFromQuery(q map[string]any) SearchIdentity {
|
|
raw, _ := json.Marshal(q)
|
|
var values map[string]any
|
|
_ = json.Unmarshal(raw, &values)
|
|
text := string(raw)
|
|
id := SearchIdentity{}
|
|
for _, part := range []struct {
|
|
key string
|
|
target *string
|
|
}{{"knowledge_base_id", &id.KnowledgeBaseID}, {"knowledge_id", &id.KnowledgeID}} {
|
|
found := false
|
|
for _, marker := range []string{"\"" + part.key + "\":\"", "\"" + part.key + ".keyword\":\""} {
|
|
if at := strings.Index(text, marker); at >= 0 {
|
|
rest := text[at+len(marker):]
|
|
if end := strings.Index(rest, "\""); end >= 0 {
|
|
*part.target = rest[:end]
|
|
}
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if found {
|
|
continue
|
|
}
|
|
}
|
|
for _, marker := range []string{`"tenant_id":10000`, `"tenant_id.keyword":10000`} {
|
|
if strings.Contains(text, marker) {
|
|
id.TenantID = 10000
|
|
break
|
|
}
|
|
}
|
|
return id
|
|
}
|
|
func idsFromQuery(q map[string]any) map[string]bool {
|
|
result := map[string]bool{}
|
|
var walk func(any)
|
|
walk = func(value any) {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
if rawIDs, ok := typed["ids"].(map[string]any); ok {
|
|
if values, ok := rawIDs["values"].([]any); ok {
|
|
for _, value := range values {
|
|
if id, ok := value.(string); ok {
|
|
result[id] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, child := range typed {
|
|
walk(child)
|
|
}
|
|
case []any:
|
|
for _, child := range typed {
|
|
walk(child)
|
|
}
|
|
}
|
|
}
|
|
walk(q)
|
|
return result
|
|
}
|
|
func sourceMatches(raw json.RawMessage, id SearchIdentity) bool { return searchHitMatches(raw, id) }
|
|
func mustJSON(value any) []byte { result, _ := json.Marshal(value); return result }
|