feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
package sourcescan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
func TestFilePPScannerReadsValidatedSnapshotUnderAllowedRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilePPScanner() error = %v", err)
|
||||
}
|
||||
snapshot, fingerprint, err := scanner.Scan(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Scan() error = %v", err)
|
||||
}
|
||||
if len(snapshot.Items) != 1 || snapshot.Items[0].SourceURLHash != source {
|
||||
t.Fatalf("snapshot = %#v", snapshot)
|
||||
}
|
||||
if !governance.ValidSHA256(fingerprint) {
|
||||
t.Fatalf("fingerprint = %q", fingerprint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePPScannerRejectsManifestItemWithoutLocalPath(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[{"local_path":""}]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := scanner.Scan(context.Background()); err == nil || !strings.Contains(err.Error(), "local_path") {
|
||||
t.Fatalf("Scan() error = %v, want missing local_path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePPScannerAllowsMissingArtifactForOverLimitRecord(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
manifest := `{"results":[{"url":"https://source.invalid/large","local_path":"` + filepath.Join(root, "not-downloaded.bin") + `","bytes":52428801,"status":"over_limit","source_ids":["pp-large"]}]}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, _, err := scanner.Scan(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Scan() error = %v", err)
|
||||
}
|
||||
if len(snapshot.Items) != 2 || snapshot.Items[1].Status != ppconnector.StatusOverLimit {
|
||||
t.Fatalf("snapshot = %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPPHandlerCompletesAggregateReadOnlyReconciliation(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC)
|
||||
sameSource := strings.Repeat("a", 64)
|
||||
updatedSource := strings.Repeat("b", 64)
|
||||
overLimitSource := strings.Repeat("c", 64)
|
||||
missingSource := strings.Repeat("d", 64)
|
||||
addedSource := strings.Repeat("e", 64)
|
||||
sameSHA := strings.Repeat("1", 64)
|
||||
newSHA := strings.Repeat("2", 64)
|
||||
oldSHA := strings.Repeat("3", 64)
|
||||
addedSHA := strings.Repeat("5", 64)
|
||||
|
||||
store := &fakePPScanStore{
|
||||
scan: governance.SourceScan{ID: "scan-1", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanPending},
|
||||
current: map[string]ppconnector.CurrentItem{
|
||||
sameSource: {Revision: sameSHA},
|
||||
updatedSource: {Revision: oldSHA},
|
||||
missingSource: {Revision: strings.Repeat("4", 64)},
|
||||
},
|
||||
}
|
||||
scanner := &fakePPScanner{snapshot: ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
|
||||
{KBID: "kb-1", SourceURLHash: sameSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sameSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
|
||||
{KBID: "kb-1", SourceURLHash: updatedSource, SourceIDs: []string{"pp-2"}, Bytes: 20, SHA256: newSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-2"},
|
||||
{KBID: "kb-1", SourceURLHash: overLimitSource, SourceIDs: []string{"pp-3"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
|
||||
{KBID: "kb-1", SourceURLHash: addedSource, SourceIDs: []string{"pp-4"}, Bytes: 30, SHA256: addedSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-4"},
|
||||
}}, snapshotSHA: strings.Repeat("f", 64)}
|
||||
|
||||
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPPHandler() error = %v", err)
|
||||
}
|
||||
job := jobs.Job{Type: "source.scan.pp", Payload: []byte(`{"scan_id":"scan-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
|
||||
want := governance.SourceScanResult{
|
||||
SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 4, UnchangedItems: 1, AddedItems: 1,
|
||||
UpdatedItems: 1, OverLimitItems: 1, MissingItems: 1, PlannedUploads: 1,
|
||||
Changes: []governance.SourceScanChange{
|
||||
{StableExternalID: updatedSource, ChangeType: governance.SourceScanUpdated, PreviousRevision: oldSHA, ObservedRevision: newSHA, ObservedSizeBytes: int64Pointer(20)},
|
||||
{StableExternalID: addedSource, ChangeType: governance.SourceScanAdded, ObservedRevision: addedSHA, ObservedSizeBytes: int64Pointer(30)},
|
||||
{StableExternalID: missingSource, ChangeType: governance.SourceScanUnavailable, PreviousRevision: strings.Repeat("4", 64)},
|
||||
},
|
||||
}
|
||||
if store.completedScanID != "scan-1" || !reflect.DeepEqual(store.completed, want) {
|
||||
t.Fatalf("completed scan = %q %#v, want %#v", store.completedScanID, store.completed, want)
|
||||
}
|
||||
if store.completedAt != now || store.failedCode != "" {
|
||||
t.Fatalf("completedAt=%v failedCode=%q", store.completedAt, store.failedCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPPHandlerNoOpsWhenReclaimedScanAlreadyCompleted(t *testing.T) {
|
||||
store := &fakePPScanStore{scan: governance.SourceScan{
|
||||
ID: "scan-completed", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanCompleted,
|
||||
}}
|
||||
scanner := &fakePPScanner{}
|
||||
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"scan_id":"scan-completed"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if scanner.calls != 0 || store.completedScanID != "" || store.failedCode != "" {
|
||||
t.Fatalf("scanner calls=%d completed=%q failed=%q", scanner.calls, store.completedScanID, store.failedCode)
|
||||
}
|
||||
}
|
||||
|
||||
func int64Pointer(value int64) *int64 { return &value }
|
||||
|
||||
type fakePPScanner struct {
|
||||
snapshot ppconnector.Snapshot
|
||||
snapshotSHA string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *fakePPScanner) Scan(context.Context) (ppconnector.Snapshot, string, error) {
|
||||
s.calls++
|
||||
return s.snapshot, s.snapshotSHA, s.err
|
||||
}
|
||||
|
||||
type fakePPScanStore struct {
|
||||
scan governance.SourceScan
|
||||
current map[string]ppconnector.CurrentItem
|
||||
completedScanID string
|
||||
completed governance.SourceScanResult
|
||||
completedAt time.Time
|
||||
failedCode string
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) {
|
||||
if s.scan.State == governance.SourceScanPending {
|
||||
s.scan.State = governance.SourceScanRunning
|
||||
}
|
||||
return s.scan, nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) {
|
||||
return s.current, nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) CompleteSourceScan(_ context.Context, scanID string, result governance.SourceScanResult, now time.Time) error {
|
||||
s.completedScanID, s.completed, s.completedAt = scanID, result, now
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) FailSourceScan(_ context.Context, _ string, code string, _ time.Time) error {
|
||||
s.failedCode = code
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user