dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
160 lines
6.9 KiB
Go
160 lines
6.9 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestSearchExporterReadsOnlyScopedDocumentsAndMapping(t *testing.T) {
|
|
requests := []recordedSearchRequest{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body map[string]any
|
|
if r.Body != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
}
|
|
requests = append(requests, recordedSearchRequest{method: r.Method, path: r.URL.Path, body: body, authorization: r.Header.Get("Authorization")})
|
|
switch r.URL.Path {
|
|
case "/chunks/_mapping":
|
|
_, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"knowledge_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"tenant_id":{"type":"keyword"}}}}}`))
|
|
case "/chunks/_search":
|
|
if len(requests) == 2 {
|
|
_, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-1","embedding":[0.1,0.2]} ,"sort":["es-1"]}]}}`))
|
|
} else {
|
|
_, _ = w.Write([]byte(`{"hits":{"hits":[]}}`))
|
|
}
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", AuthorizationToken: "safe-token", HTTPClient: server.Client(), PageSize: 1})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
snapshot, err := exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(snapshot.Documents) != 1 || snapshot.Documents[0].ID != "es-1" || len(snapshot.MappingSHA256) != 64 || len(snapshot.DocumentsSHA256) != 64 {
|
|
t.Fatalf("snapshot=%#v", snapshot)
|
|
}
|
|
if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || requests[2].method != http.MethodPost {
|
|
t.Fatalf("requests=%#v", requests)
|
|
}
|
|
for _, request := range requests {
|
|
if request.authorization != "Bearer safe-token" {
|
|
t.Fatalf("authorization=%q", request.authorization)
|
|
}
|
|
if strings.Contains(strings.ToLower(request.path), "knowledge-safe") {
|
|
t.Fatalf("identity leaked into URL: %s", request.path)
|
|
}
|
|
}
|
|
encoded, _ := json.Marshal(requests[1].body)
|
|
for _, required := range []string{`"tenant_id":10000`, `"knowledge_base_id":"kb-safe"`, `"knowledge_id":"knowledge-safe"`} {
|
|
if !strings.Contains(string(encoded), required) {
|
|
t.Fatalf("query missing %s: %s", required, encoded)
|
|
}
|
|
}
|
|
second, _ := json.Marshal(requests[2].body)
|
|
if !strings.Contains(string(second), `"from":1`) {
|
|
t.Fatalf("pagination missing: %s", second)
|
|
}
|
|
}
|
|
|
|
func TestSearchExporterReadsMappingContractWithoutDocumentQuery(t *testing.T) {
|
|
requests := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests++
|
|
if r.URL.Path != "/chunks/_mapping" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"embedding":{"type":"dense_vector","dims":1024},"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`))
|
|
}))
|
|
defer server.Close()
|
|
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digest, err := exporter.MappingSHA256(context.Background())
|
|
if err != nil || len(digest) != 64 || requests != 1 {
|
|
t.Fatalf("digest=%q requests=%d err=%v", digest, requests, err)
|
|
}
|
|
}
|
|
|
|
func TestSearchExporterRejectsCrossScopedHit(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "_mapping") {
|
|
_, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"tenant_id":{"type":"keyword"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"}}}}}`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"tenant_id":10000,"knowledge_base_id":"wrong","knowledge_id":"knowledge-safe"},"sort":["es-1"]}]}}`))
|
|
}))
|
|
defer server.Close()
|
|
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"})
|
|
if err == nil || strings.Contains(err.Error(), "wrong") || strings.Contains(err.Error(), "knowledge-safe") {
|
|
t.Fatalf("error=%v", err)
|
|
}
|
|
}
|
|
|
|
type recordedSearchRequest struct {
|
|
method, path, authorization string
|
|
body map[string]any
|
|
}
|
|
|
|
// TestSearchExporterHandlesCommunityIndexShape locks the fix for the real
|
|
// WeKnora v0.7.1 default index shape discovered during cross-environment
|
|
// release verification: ID fields mapped as "text" with a ".keyword" subfield
|
|
// and documents without any tenant_id field. Export must exact-match through
|
|
// the subfields, omit the absent tenant filter, and accept tenant-less hits.
|
|
func TestSearchExporterHandlesCommunityIndexShape(t *testing.T) {
|
|
seenQueries := []map[string]any{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/chunks/_mapping":
|
|
_, _ = w.Write([]byte(`{"chunks":{"mappings":{"properties":{"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":1024},"knowledge_id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"knowledge_base_id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}}`))
|
|
case "/chunks/_search":
|
|
var body map[string]any
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
seenQueries = append(seenQueries, body)
|
|
_, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"es-1","_source":{"content":"text","knowledge_base_id":"kb-safe","knowledge_id":"knowledge-safe","chunk_id":"chunk-1","embedding":[0.1],"is_enabled":true}}]}}`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", HTTPClient: server.Client(), PageSize: 10})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
snapshot, err := exporter.Export(context.Background(), SearchIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(snapshot.Documents) != 1 || snapshot.Documents[0].ID != "es-1" {
|
|
t.Fatalf("snapshot=%#v", snapshot)
|
|
}
|
|
if len(seenQueries) == 0 {
|
|
t.Fatal("no search issued")
|
|
}
|
|
encoded, _ := json.Marshal(seenQueries[0])
|
|
text := string(encoded)
|
|
for _, required := range []string{`"knowledge_base_id.keyword":"kb-safe"`, `"knowledge_id.keyword":"knowledge-safe"`} {
|
|
if !strings.Contains(text, required) {
|
|
t.Fatalf("community-shape query missing %s: %s", required, text)
|
|
}
|
|
}
|
|
if strings.Contains(text, "tenant_id") {
|
|
t.Fatalf("query must omit absent tenant_id field: %s", text)
|
|
}
|
|
}
|