feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
@@ -0,0 +1,194 @@
package sourcescan
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
ppconnector "knowledge-governance/internal/connectors/pp"
)
const maxPPMetadataBytes int64 = 64 * 1024 * 1024
type FilePPScannerConfig struct {
AllowedRoot string
LedgerPath string
ManifestPath string
}
type FilePPScanner struct {
root string
ledgerPath string
manifestPath string
}
func NewFilePPScanner(config FilePPScannerConfig) (*FilePPScanner, error) {
root, err := filepath.Abs(config.AllowedRoot)
if err != nil || config.AllowedRoot == "" {
return nil, errors.New("PP scan allowed root is required")
}
resolvedRoot, err := filepath.EvalSymlinks(root)
if err != nil || resolvedRoot != root {
return nil, errors.New("PP scan allowed root must be a canonical directory")
}
info, err := os.Lstat(root)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("PP scan allowed root must be a canonical directory")
}
if config.LedgerPath == "" || config.ManifestPath == "" {
return nil, errors.New("PP ledger and manifest paths are required")
}
return &FilePPScanner{root: root, ledgerPath: config.LedgerPath, manifestPath: config.ManifestPath}, nil
}
func (s *FilePPScanner) Scan(ctx context.Context) (ppconnector.Snapshot, string, error) {
if err := ctx.Err(); err != nil {
return ppconnector.Snapshot{}, "", err
}
ledger, err := s.readFile(s.ledgerPath, maxPPMetadataBytes)
if err != nil {
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP ledger: %w", err)
}
manifest, err := s.readFile(s.manifestPath, maxPPMetadataBytes)
if err != nil {
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP manifest: %w", err)
}
if err := s.validateManifestPaths(manifest); err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, err := ppconnector.FoldLedger(bytes.NewReader(ledger))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, _, err = ppconnector.RecoverMissingSHA256(snapshot, bytes.NewReader(manifest))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, _, err = ppconnector.MergeOverLimitManifest(snapshot, bytes.NewReader(manifest))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
if err := ppconnector.ValidateSnapshot(snapshot); err != nil {
return ppconnector.Snapshot{}, "", err
}
digest := sha256.New()
_, _ = digest.Write(ledger)
_, _ = digest.Write([]byte{0})
_, _ = digest.Write(manifest)
return snapshot, hex.EncodeToString(digest.Sum(nil)), nil
}
func (s *FilePPScanner) validateManifestPaths(body []byte) error {
var manifest struct {
Results []struct {
LocalPath string `json:"local_path"`
Status string `json:"status"`
} `json:"results"`
}
if err := json.Unmarshal(body, &manifest); err != nil {
return fmt.Errorf("decode PP manifest paths: %w", err)
}
for _, item := range manifest.Results {
if item.LocalPath == "" {
return errors.New("PP manifest artifact local_path is required")
}
if item.Status == string(ppconnector.StatusOverLimit) {
if err := s.validateDeclaredPath(item.LocalPath); err != nil {
return fmt.Errorf("validate PP over-limit artifact path: %w", err)
}
file, err := s.openRegular(item.LocalPath)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return fmt.Errorf("validate PP over-limit artifact: %w", err)
}
if err := file.Close(); err != nil {
return errors.New("close PP over-limit artifact")
}
continue
}
file, err := s.openRegular(item.LocalPath)
if err != nil {
return fmt.Errorf("validate PP manifest artifact: %w", err)
}
if err := file.Close(); err != nil {
return errors.New("close PP manifest artifact")
}
}
return nil
}
func (s *FilePPScanner) readFile(path string, maxBytes int64) ([]byte, error) {
file, err := s.openRegular(path)
if err != nil {
return nil, err
}
body, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1))
closeErr := file.Close()
if readErr != nil {
return nil, readErr
}
if closeErr != nil {
return nil, closeErr
}
if int64(len(body)) > maxBytes {
return nil, errors.New("PP metadata file exceeds size limit")
}
return body, nil
}
func (s *FilePPScanner) validateDeclaredPath(path string) error {
absolute, err := filepath.Abs(path)
if err != nil {
return errors.New("invalid PP source path")
}
relative, err := filepath.Rel(s.root, absolute)
if err != nil || relative == ".." || filepath.IsAbs(relative) || len(relative) >= 3 && relative[:3] == ".."+string(filepath.Separator) {
return errors.New("PP source path is outside allowed root")
}
parent := filepath.Dir(absolute)
resolvedParent, err := filepath.EvalSymlinks(parent)
if err != nil || resolvedParent != parent {
return errors.New("PP source path parent must not contain symlinks")
}
return nil
}
func (s *FilePPScanner) openRegular(path string) (*os.File, error) {
if err := s.validateDeclaredPath(path); err != nil {
return nil, err
}
absolute, _ := filepath.Abs(path)
resolved, err := filepath.EvalSymlinks(absolute)
if err != nil {
return nil, err
}
if resolved != absolute {
return nil, errors.New("PP source path must not contain symlinks")
}
before, err := os.Lstat(absolute)
if err != nil {
return nil, err
}
if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("PP source path must be a regular file")
}
file, err := os.Open(absolute)
if err != nil {
return nil, err
}
after, err := file.Stat()
if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) {
_ = file.Close()
return nil, errors.New("PP source file changed while opening")
}
return file, nil
}
+127
View File
@@ -0,0 +1,127 @@
package sourcescan
import (
"context"
"encoding/json"
"errors"
"sort"
"time"
ppconnector "knowledge-governance/internal/connectors/pp"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
type PPScanner interface {
Scan(context.Context) (ppconnector.Snapshot, string, error)
}
type PPScanStore interface {
StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error)
LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error)
CompleteSourceScan(context.Context, string, governance.SourceScanResult, time.Time) error
FailSourceScan(context.Context, string, string, time.Time) error
}
type PPHandlerConfig struct {
Store PPScanStore
Scanner PPScanner
Now func() time.Time
}
type PPHandler struct {
store PPScanStore
scanner PPScanner
now func() time.Time
}
func NewPPHandler(config PPHandlerConfig) (*PPHandler, error) {
if config.Store == nil || config.Scanner == nil {
return nil, errors.New("PP scan store and scanner are required")
}
if config.Now == nil {
config.Now = time.Now
}
return &PPHandler{store: config.Store, scanner: config.Scanner, now: config.Now}, nil
}
func (h *PPHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload struct {
ScanID string `json:"scan_id"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.ScanID == "" {
return errors.New("invalid source scan payload")
}
now := h.now()
scan, err := h.store.StartSourceScan(ctx, payload.ScanID, now)
if err != nil {
return err
}
if scan.State == governance.SourceScanCompleted {
return nil
}
if scan.State != governance.SourceScanRunning {
return errors.New("source scan is not running")
}
if scan.SourceType != governance.SourceTypePP {
return h.store.FailSourceScan(ctx, scan.ID, "source_type_unsupported", now)
}
snapshot, snapshotSHA, err := h.scanner.Scan(ctx)
if err != nil || !governance.ValidSHA256(snapshotSHA) {
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
}
current, err := h.store.LoadPPCurrentItems(ctx, scan.SourceConnectionID)
if err != nil {
return err
}
reconciliation, err := ppconnector.PlanReconciliation(snapshot, current)
if err != nil {
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
}
observed := make(map[string]struct{}, len(snapshot.Items))
changes := make([]governance.SourceScanChange, 0)
for _, item := range snapshot.Items {
observed[item.SourceURLHash] = struct{}{}
if item.Status == ppconnector.StatusOverLimit {
continue
}
size := item.Bytes
stored, exists := current[item.SourceURLHash]
if !exists {
changes = append(changes, governance.SourceScanChange{
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanAdded,
ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
})
} else if stored.Revision != item.SHA256 {
changes = append(changes, governance.SourceScanChange{
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanUpdated,
PreviousRevision: stored.Revision, ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
})
}
}
missingIDs := make([]string, 0)
for sourceID := range current {
if _, ok := observed[sourceID]; !ok {
missingIDs = append(missingIDs, sourceID)
}
}
sort.Strings(missingIDs)
for _, sourceID := range missingIDs {
changes = append(changes, governance.SourceScanChange{
StableExternalID: sourceID, ChangeType: governance.SourceScanUnavailable,
PreviousRevision: current[sourceID].Revision,
})
}
result := governance.SourceScanResult{
SnapshotSHA256: snapshotSHA,
ObservedItems: len(snapshot.Items),
UnchangedItems: reconciliation.Unchanged,
AddedItems: reconciliation.Adopted,
UpdatedItems: reconciliation.UploadRequired,
OverLimitItems: reconciliation.OverLimit,
MissingItems: len(missingIDs),
PlannedUploads: reconciliation.UploadRequired,
Changes: changes,
}
return h.store.CompleteSourceScan(ctx, scan.ID, result, now)
}
@@ -0,0 +1,208 @@
package sourcescan
import (
"context"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
ppconnector "knowledge-governance/internal/connectors/pp"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
func TestFilePPScannerReadsValidatedSnapshotUnderAllowedRoot(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatalf("NewFilePPScanner() error = %v", err)
}
snapshot, fingerprint, err := scanner.Scan(context.Background())
if err != nil {
t.Fatalf("Scan() error = %v", err)
}
if len(snapshot.Items) != 1 || snapshot.Items[0].SourceURLHash != source {
t.Fatalf("snapshot = %#v", snapshot)
}
if !governance.ValidSHA256(fingerprint) {
t.Fatalf("fingerprint = %q", fingerprint)
}
}
func TestFilePPScannerRejectsManifestItemWithoutLocalPath(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[{"local_path":""}]}`), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatal(err)
}
if _, _, err := scanner.Scan(context.Background()); err == nil || !strings.Contains(err.Error(), "local_path") {
t.Fatalf("Scan() error = %v, want missing local_path error", err)
}
}
func TestFilePPScannerAllowsMissingArtifactForOverLimitRecord(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
manifest := `{"results":[{"url":"https://source.invalid/large","local_path":"` + filepath.Join(root, "not-downloaded.bin") + `","bytes":52428801,"status":"over_limit","source_ids":["pp-large"]}]}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatal(err)
}
snapshot, _, err := scanner.Scan(context.Background())
if err != nil {
t.Fatalf("Scan() error = %v", err)
}
if len(snapshot.Items) != 2 || snapshot.Items[1].Status != ppconnector.StatusOverLimit {
t.Fatalf("snapshot = %#v", snapshot)
}
}
func TestPPHandlerCompletesAggregateReadOnlyReconciliation(t *testing.T) {
now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC)
sameSource := strings.Repeat("a", 64)
updatedSource := strings.Repeat("b", 64)
overLimitSource := strings.Repeat("c", 64)
missingSource := strings.Repeat("d", 64)
addedSource := strings.Repeat("e", 64)
sameSHA := strings.Repeat("1", 64)
newSHA := strings.Repeat("2", 64)
oldSHA := strings.Repeat("3", 64)
addedSHA := strings.Repeat("5", 64)
store := &fakePPScanStore{
scan: governance.SourceScan{ID: "scan-1", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanPending},
current: map[string]ppconnector.CurrentItem{
sameSource: {Revision: sameSHA},
updatedSource: {Revision: oldSHA},
missingSource: {Revision: strings.Repeat("4", 64)},
},
}
scanner := &fakePPScanner{snapshot: ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
{KBID: "kb-1", SourceURLHash: sameSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sameSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
{KBID: "kb-1", SourceURLHash: updatedSource, SourceIDs: []string{"pp-2"}, Bytes: 20, SHA256: newSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-2"},
{KBID: "kb-1", SourceURLHash: overLimitSource, SourceIDs: []string{"pp-3"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
{KBID: "kb-1", SourceURLHash: addedSource, SourceIDs: []string{"pp-4"}, Bytes: 30, SHA256: addedSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-4"},
}}, snapshotSHA: strings.Repeat("f", 64)}
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewPPHandler() error = %v", err)
}
job := jobs.Job{Type: "source.scan.pp", Payload: []byte(`{"scan_id":"scan-1"}`)}
if err := handler.Handle(context.Background(), job); err != nil {
t.Fatalf("Handle() error = %v", err)
}
want := governance.SourceScanResult{
SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 4, UnchangedItems: 1, AddedItems: 1,
UpdatedItems: 1, OverLimitItems: 1, MissingItems: 1, PlannedUploads: 1,
Changes: []governance.SourceScanChange{
{StableExternalID: updatedSource, ChangeType: governance.SourceScanUpdated, PreviousRevision: oldSHA, ObservedRevision: newSHA, ObservedSizeBytes: int64Pointer(20)},
{StableExternalID: addedSource, ChangeType: governance.SourceScanAdded, ObservedRevision: addedSHA, ObservedSizeBytes: int64Pointer(30)},
{StableExternalID: missingSource, ChangeType: governance.SourceScanUnavailable, PreviousRevision: strings.Repeat("4", 64)},
},
}
if store.completedScanID != "scan-1" || !reflect.DeepEqual(store.completed, want) {
t.Fatalf("completed scan = %q %#v, want %#v", store.completedScanID, store.completed, want)
}
if store.completedAt != now || store.failedCode != "" {
t.Fatalf("completedAt=%v failedCode=%q", store.completedAt, store.failedCode)
}
}
func TestPPHandlerNoOpsWhenReclaimedScanAlreadyCompleted(t *testing.T) {
store := &fakePPScanStore{scan: governance.SourceScan{
ID: "scan-completed", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanCompleted,
}}
scanner := &fakePPScanner{}
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"scan_id":"scan-completed"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if scanner.calls != 0 || store.completedScanID != "" || store.failedCode != "" {
t.Fatalf("scanner calls=%d completed=%q failed=%q", scanner.calls, store.completedScanID, store.failedCode)
}
}
func int64Pointer(value int64) *int64 { return &value }
type fakePPScanner struct {
snapshot ppconnector.Snapshot
snapshotSHA string
err error
calls int
}
func (s *fakePPScanner) Scan(context.Context) (ppconnector.Snapshot, string, error) {
s.calls++
return s.snapshot, s.snapshotSHA, s.err
}
type fakePPScanStore struct {
scan governance.SourceScan
current map[string]ppconnector.CurrentItem
completedScanID string
completed governance.SourceScanResult
completedAt time.Time
failedCode string
}
func (s *fakePPScanStore) StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) {
if s.scan.State == governance.SourceScanPending {
s.scan.State = governance.SourceScanRunning
}
return s.scan, nil
}
func (s *fakePPScanStore) LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) {
return s.current, nil
}
func (s *fakePPScanStore) CompleteSourceScan(_ context.Context, scanID string, result governance.SourceScanResult, now time.Time) error {
s.completedScanID, s.completed, s.completedAt = scanID, result, now
return nil
}
func (s *fakePPScanStore) FailSourceScan(_ context.Context, _ string, code string, _ time.Time) error {
s.failedCode = code
return nil
}