feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type reportStore interface {
|
||||
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
|
||||
ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error)
|
||||
}
|
||||
|
||||
type reportClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error)
|
||||
}
|
||||
|
||||
type e2eReport struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
RemoteStatus string `json:"remote_status"`
|
||||
ChunkTotal int64 `json:"chunk_total"`
|
||||
ChunkItemsRead int64 `json:"chunk_items_read"`
|
||||
AllChunksMatch bool `json:"all_chunks_match"`
|
||||
SearchResultCount int `json:"search_result_count"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
MarkerFound bool `json:"marker_found"`
|
||||
MaxScore float64 `json:"max_score"`
|
||||
StructuralPassed bool `json:"structural_passed"`
|
||||
SemanticState string `json:"semantic_state"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("test-weknora-e2e-report", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
publicationID := flags.String("publication-id", "", "governance publication ID")
|
||||
query := flags.String("query", "", "synthetic scoped retrieval query")
|
||||
expectedMarker := flags.String("expected-marker", "", "synthetic marker expected in a scoped result")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return errors.New("invalid report arguments")
|
||||
}
|
||||
if *publicationID == "" || *query == "" || *expectedMarker == "" {
|
||||
return errors.New("--publication-id, --query and --expected-marker are required")
|
||||
}
|
||||
databaseURL := getenv("DATABASE_URL")
|
||||
baseURL := getenv("TEST_WEKNORA_BASE_URL")
|
||||
keyFile := getenv("TEST_WEKNORA_API_KEY_FILE")
|
||||
if databaseURL == "" || baseURL == "" || keyFile == "" {
|
||||
return errors.New("database and test WeKnora configuration are required")
|
||||
}
|
||||
allowDockerHTTP := false
|
||||
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return errors.New("invalid Docker HTTP setting")
|
||||
}
|
||||
allowDockerHTTP = parsed
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(keyFile)
|
||||
if err != nil {
|
||||
return errors.New("read test WeKnora API key")
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: 60 * time.Second,
|
||||
AllowDockerHTTP: allowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("create test WeKnora client")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return errors.New("open governance database")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return errors.New("connect governance database")
|
||||
}
|
||||
report, err := verifyPublication(ctx, postgresstore.NewStore(pool), client, *publicationID, *query, *expectedMarker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(output).Encode(report)
|
||||
}
|
||||
|
||||
func verifyPublication(ctx context.Context, store reportStore, client reportClient, publicationID, query, expectedMarker string) (e2eReport, error) {
|
||||
publication, err := store.LoadPublicationForValidation(ctx, publicationID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("load completed publication")
|
||||
}
|
||||
knowledge, err := client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
if err != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
return e2eReport{}, errors.New("verify remote knowledge identity")
|
||||
}
|
||||
report := e2eReport{
|
||||
PublicationID: publication.ID,
|
||||
RemoteStatus: knowledge.ParseStatus,
|
||||
AllChunksMatch: true,
|
||||
}
|
||||
for page := 1; page <= 100; page++ {
|
||||
chunks, total, err := client.ListChunks(ctx, publication.KnowledgeID, page, 100, []string{"text", "image_ocr"})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read remote chunks")
|
||||
}
|
||||
report.ChunkTotal = total
|
||||
for _, chunk := range chunks {
|
||||
report.ChunkItemsRead++
|
||||
if chunk.KnowledgeID != publication.KnowledgeID {
|
||||
report.AllChunksMatch = false
|
||||
}
|
||||
}
|
||||
if report.ChunkItemsRead >= total || len(chunks) == 0 {
|
||||
break
|
||||
}
|
||||
if page == 100 {
|
||||
return e2eReport{}, errors.New("remote chunk pagination limit exceeded")
|
||||
}
|
||||
}
|
||||
results, err := client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{publication.KnowledgeID}})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("run scoped retrieval probe")
|
||||
}
|
||||
report.SearchResultCount = len(results)
|
||||
for _, result := range results {
|
||||
if result.KnowledgeID == publication.KnowledgeID {
|
||||
report.TargetSearchHits++
|
||||
if strings.Contains(result.Content, expectedMarker) {
|
||||
report.MarkerFound = true
|
||||
}
|
||||
}
|
||||
if result.Score > report.MaxScore {
|
||||
report.MaxScore = result.Score
|
||||
}
|
||||
}
|
||||
acceptance, err := store.ListAcceptanceResults(ctx, publication.ID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read governance acceptance results")
|
||||
}
|
||||
structural := map[string]bool{"knowledge_exists": false, "chunks_present": false}
|
||||
for _, record := range acceptance {
|
||||
if _, ok := structural[record.CheckType]; ok && record.State == string(governance.AcceptancePassed) {
|
||||
structural[record.CheckType] = true
|
||||
}
|
||||
if record.CheckType == "retrieval_hit" {
|
||||
report.SemanticState = record.State
|
||||
}
|
||||
}
|
||||
report.StructuralPassed = structural["knowledge_exists"] && structural["chunks_present"]
|
||||
if report.RemoteStatus != "completed" || report.ChunkTotal <= 0 || report.ChunkItemsRead != report.ChunkTotal || !report.AllChunksMatch || report.TargetSearchHits == 0 || !report.MarkerFound {
|
||||
return e2eReport{}, errors.New("end-to-end verification failed")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestVerifyPublicationReturnsSanitizedEvidence(t *testing.T) {
|
||||
store := &reportStoreStub{
|
||||
publication: postgresstore.PublicationForValidation{ID: "tp-1", TestKBID: "kb-1", KnowledgeID: "knowledge-secret-id"},
|
||||
acceptance: []governance.AcceptanceRecord{
|
||||
{CheckType: "knowledge_exists", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "chunks_present", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "retrieval_hit", State: string(governance.AcceptanceManualRequired)},
|
||||
},
|
||||
}
|
||||
client := &reportClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-secret-id", KnowledgeBaseID: "kb-1", ParseStatus: "completed"},
|
||||
chunks: []weknora.Chunk{
|
||||
{ID: "chunk-1", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
{ID: "chunk-2", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
},
|
||||
search: []weknora.SearchResult{{KnowledgeID: "knowledge-secret-id", Content: "answer contains MIKU-GOV-P3-5927", Score: 0.02}},
|
||||
}
|
||||
|
||||
report, err := verifyPublication(context.Background(), store, client, "tp-1", "unique marker", "MIKU-GOV-P3-5927")
|
||||
if err != nil {
|
||||
t.Fatalf("verifyPublication() error = %v", err)
|
||||
}
|
||||
if report.PublicationID != "tp-1" || report.RemoteStatus != "completed" || report.ChunkTotal != 2 || !report.AllChunksMatch || report.SearchResultCount != 1 || report.TargetSearchHits != 1 || !report.MarkerFound || !report.StructuralPassed || report.SemanticState != "manual_required" {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
encoded, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge-secret-id", "chunk-1", "unique marker", "MIKU-GOV-P3-5927"} {
|
||||
if string(encoded) == forbidden || contains(string(encoded), forbidden) {
|
||||
t.Fatalf("report leaked %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(value, fragment string) bool {
|
||||
for index := 0; index+len(fragment) <= len(value); index++ {
|
||||
if value[index:index+len(fragment)] == fragment {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type reportStoreStub struct {
|
||||
publication postgresstore.PublicationForValidation
|
||||
acceptance []governance.AcceptanceRecord
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) {
|
||||
return store.acceptance, nil
|
||||
}
|
||||
|
||||
type reportClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
chunks []weknora.Chunk
|
||||
search []weknora.SearchResult
|
||||
}
|
||||
|
||||
func (client *reportClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
return client.chunks, int64(len(client.chunks)), nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) {
|
||||
return client.search, nil
|
||||
}
|
||||
Reference in New Issue
Block a user