zheromiku
dcdc8d1ddd
全栈知识治理平台首次入库（P1-P10 + 发布链路模拟验证闭环）： - 治理核心：源目录/变更集/人工审核/发布批次/发布执行（fail-closed 闸口） - 测试通道：WeKnora 测试 KB 上传/轮询/验收；poll 完成时写入 parser_generation/index_generation，修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071（本地 source=测试角色 / 云端 target=生产角色 模拟验证通过）： * ES 导出/导入 mapping 自适应（.keyword 子字段、可选 tenant_id） * OSS locator 支持嵌套 scheme；chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换；verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id（strict 解码） - 三通道发布+回滚演练实测：publish 4/4 verified，检索命中， rollback 4/4 rolled_back，目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一，120 用例通过 - 门禁：go vet / go test 全包通过，gofmt 干净
225 lines
9.6 KiB
Go
225 lines
9.6 KiB
Go
package pp
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestFoldLedgerKeepsLatestTerminalRecordPerSourceURLHash(t *testing.T) {
|
|
sourceA := strings.Repeat("a", 64)
|
|
sourceB := strings.Repeat("b", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
ledger := strings.Join([]string{
|
|
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-old"}`,
|
|
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`,
|
|
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"parse_timeout","knowledge_id":"knowledge-1"}`,
|
|
`{"kb_id":"kb-1","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1","canonical_source_url_hash":"` + sourceA + `"}`,
|
|
}, "\n")
|
|
|
|
snapshot, err := FoldLedger(strings.NewReader(ledger))
|
|
if err != nil {
|
|
t.Fatalf("FoldLedger() error = %v", err)
|
|
}
|
|
if snapshot.EventCount != 4 || snapshot.KBID != "kb-1" || len(snapshot.Items) != 2 {
|
|
t.Fatalf("FoldLedger() snapshot = %#v", snapshot)
|
|
}
|
|
if snapshot.Items[0].SourceURLHash != sourceA || snapshot.Items[0].Status != StatusCompleted || snapshot.Items[0].KnowledgeID != "knowledge-1" {
|
|
t.Fatalf("completed item = %#v", snapshot.Items[0])
|
|
}
|
|
if snapshot.Items[1].SourceURLHash != sourceB || snapshot.Items[1].Status != StatusContentAlias || snapshot.Items[1].CanonicalSourceURLHash != sourceA {
|
|
t.Fatalf("alias item = %#v", snapshot.Items[1])
|
|
}
|
|
}
|
|
|
|
func TestFoldLedgerReportsUnresolvedSourceKeys(t *testing.T) {
|
|
source := strings.Repeat("a", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-1"}`
|
|
|
|
snapshot, err := FoldLedger(strings.NewReader(ledger))
|
|
if err != nil {
|
|
t.Fatalf("FoldLedger() error = %v", err)
|
|
}
|
|
if snapshot.UnresolvedCount != 1 {
|
|
t.Fatalf("UnresolvedCount = %d, want 1", snapshot.UnresolvedCount)
|
|
}
|
|
}
|
|
|
|
func TestValidateSnapshotRejectsUnresolvedSourceKeys(t *testing.T) {
|
|
err := ValidateSnapshot(Snapshot{KBID: "kb-1", UnresolvedCount: 1})
|
|
if err == nil || !strings.Contains(err.Error(), "unresolved") {
|
|
t.Fatalf("ValidateSnapshot() error = %v, want unresolved source error", err)
|
|
}
|
|
}
|
|
|
|
func TestFoldLedgerRejectsMixedKnowledgeBases(t *testing.T) {
|
|
sourceA := strings.Repeat("a", 64)
|
|
sourceB := strings.Repeat("b", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
ledger := strings.Join([]string{
|
|
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`,
|
|
`{"kb_id":"kb-2","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-2"}`,
|
|
}, "\n")
|
|
|
|
_, err := FoldLedger(strings.NewReader(ledger))
|
|
if err == nil || !strings.Contains(err.Error(), "multiple knowledge bases") {
|
|
t.Fatalf("FoldLedger() error = %v, want mixed knowledge base error", err)
|
|
}
|
|
}
|
|
|
|
func TestFoldLedgerRejectsInvalidTerminalSourceHash(t *testing.T) {
|
|
sha := strings.Repeat("c", 64)
|
|
ledger := `{"kb_id":"kb-1","source_url_hash":"not-a-hash","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
|
|
|
_, err := FoldLedger(strings.NewReader(ledger))
|
|
if err == nil || !strings.Contains(err.Error(), "source_url_hash") {
|
|
t.Fatalf("FoldLedger() error = %v, want source hash validation error", err)
|
|
}
|
|
}
|
|
|
|
func TestFoldLedgerRejectsIncompleteTerminalRecord(t *testing.T) {
|
|
source := strings.Repeat("a", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
tests := []struct {
|
|
name string
|
|
ledger string
|
|
}{
|
|
{"missing knowledge base", `{"source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
|
{"missing source IDs", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":[],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
|
{"negative bytes", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":-1,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
|
{"invalid optional SHA", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"bad","status":"completed","knowledge_id":"knowledge-1"}`},
|
|
{"missing knowledge ID", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed"}`},
|
|
{"missing alias canonical hash", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1"}`},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if _, err := FoldLedger(strings.NewReader(tt.ledger)); err == nil {
|
|
t.Fatal("FoldLedger() error = nil, want terminal record validation error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecoverMissingSHA256RequiresVerifiedManifestArtifact(t *testing.T) {
|
|
content := []byte("verified PP artifact")
|
|
contentDigest := sha256.Sum256(content)
|
|
contentHash := hex.EncodeToString(contentDigest[:])
|
|
url := "https://source.invalid/document"
|
|
urlDigest := sha256.Sum256([]byte(url))
|
|
sourceHash := hex.EncodeToString(urlDigest[:])
|
|
path := filepath.Join(t.TempDir(), "document.bin")
|
|
if err := os.WriteFile(path, content, 0o600); err != nil {
|
|
t.Fatalf("write fixture: %v", err)
|
|
}
|
|
manifest := `{"results":[{"url":"` + url + `","local_path":"` + path + `","bytes":20,"sha256":"` + contentHash + `"}]}`
|
|
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
|
KBID: "kb-1",
|
|
SourceURLHash: sourceHash,
|
|
SourceIDs: []string{"pp-1"},
|
|
Bytes: 20,
|
|
Status: StatusCompleted,
|
|
KnowledgeID: "knowledge-1",
|
|
}}}
|
|
|
|
recovered, report, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest))
|
|
if err != nil {
|
|
t.Fatalf("RecoverMissingSHA256() error = %v", err)
|
|
}
|
|
if report.Recovered != 1 || report.VerifiedBytes != int64(len(content)) {
|
|
t.Fatalf("RecoverMissingSHA256() report = %#v", report)
|
|
}
|
|
if recovered.Items[0].SHA256 != contentHash {
|
|
t.Fatalf("recovered SHA256 = %q, want %q", recovered.Items[0].SHA256, contentHash)
|
|
}
|
|
}
|
|
|
|
func TestRecoverMissingSHA256RejectsArtifactOverLegacyLimit(t *testing.T) {
|
|
url := "https://source.invalid/oversized"
|
|
urlDigest := sha256.Sum256([]byte(url))
|
|
sourceHash := hex.EncodeToString(urlDigest[:])
|
|
contentHash := strings.Repeat("c", 64)
|
|
oversized := int64(50*1024*1024 + 1)
|
|
manifest := `{"results":[{"url":"` + url + `","local_path":"/not-opened","bytes":` + "52428801" + `,"sha256":"` + contentHash + `"}]}`
|
|
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
|
KBID: "kb-1",
|
|
SourceURLHash: sourceHash,
|
|
SourceIDs: []string{"pp-1"},
|
|
Bytes: oversized,
|
|
Status: StatusCompleted,
|
|
KnowledgeID: "knowledge-1",
|
|
}}}
|
|
|
|
_, _, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest))
|
|
if err == nil || !strings.Contains(err.Error(), "50 MiB") {
|
|
t.Fatalf("RecoverMissingSHA256() error = %v, want legacy size limit error", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateSnapshotRejectsAliasWithoutCanonicalCompletedItem(t *testing.T) {
|
|
sourceAlias := strings.Repeat("b", 64)
|
|
missingCanonical := strings.Repeat("a", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
|
KBID: "kb-1",
|
|
SourceURLHash: sourceAlias,
|
|
SourceIDs: []string{"pp-2"},
|
|
Bytes: 10,
|
|
SHA256: sha,
|
|
Status: StatusContentAlias,
|
|
KnowledgeID: "knowledge-1",
|
|
CanonicalSourceURLHash: missingCanonical,
|
|
}}}
|
|
|
|
err := ValidateSnapshot(snapshot)
|
|
if err == nil || !strings.Contains(err.Error(), "canonical completed item") {
|
|
t.Fatalf("ValidateSnapshot() error = %v, want missing canonical error", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateSnapshotRejectsAliasMetadataMismatch(t *testing.T) {
|
|
sourceCanonical := strings.Repeat("a", 64)
|
|
sourceAlias := strings.Repeat("b", 64)
|
|
sha := strings.Repeat("c", 64)
|
|
canonical := Item{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sha, Status: StatusCompleted, KnowledgeID: "knowledge-1"}
|
|
alias := Item{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: sha, Status: StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*Item)
|
|
}{
|
|
{"SHA mismatch", func(item *Item) { item.SHA256 = strings.Repeat("d", 64) }},
|
|
{"knowledge mismatch", func(item *Item) { item.KnowledgeID = "knowledge-2" }},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
candidate := alias
|
|
tt.mutate(&candidate)
|
|
err := ValidateSnapshot(Snapshot{KBID: "kb-1", Items: []Item{canonical, candidate}})
|
|
if err == nil || !strings.Contains(err.Error(), "alias metadata") {
|
|
t.Fatalf("ValidateSnapshot() error = %v, want alias metadata error", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateSnapshotRequiresRecoveredSHA256(t *testing.T) {
|
|
source := strings.Repeat("a", 64)
|
|
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
|
KBID: "kb-1",
|
|
SourceURLHash: source,
|
|
SourceIDs: []string{"pp-1"},
|
|
Bytes: 10,
|
|
Status: StatusCompleted,
|
|
KnowledgeID: "knowledge-1",
|
|
}}}
|
|
|
|
err := ValidateSnapshot(snapshot)
|
|
if err == nil || !strings.Contains(err.Error(), "verified SHA-256") {
|
|
t.Fatalf("ValidateSnapshot() error = %v, want missing SHA error", err)
|
|
}
|
|
}
|