feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
+153
View File
@@ -0,0 +1,153 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"log"
"os"
"strings"
"github.com/jackc/pgx/v5/pgxpool"
ppconnector "knowledge-governance/internal/connectors/pp"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func main() {
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError)
flags.SetOutput(io.Discard)
ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL")
manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON")
dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access")
if err := flags.Parse(args); err != nil {
return fmt.Errorf("parse arguments: %w", err)
}
if *ledgerPath == "" || *manifestPath == "" {
return fmt.Errorf("-ledger and -manifest are required")
}
ledger, err := os.Open(*ledgerPath)
if err != nil {
return fmt.Errorf("open PP ledger: %w", err)
}
snapshot, foldErr := ppconnector.FoldLedger(ledger)
closeErr := ledger.Close()
if foldErr != nil {
return foldErr
}
if closeErr != nil {
return fmt.Errorf("close PP ledger: %w", closeErr)
}
manifest, err := os.Open(*manifestPath)
if err != nil {
return fmt.Errorf("open PP download manifest: %w", err)
}
recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest)
closeErr = manifest.Close()
if recoveryErr != nil {
return recoveryErr
}
if closeErr != nil {
return fmt.Errorf("close PP download manifest: %w", closeErr)
}
overLimitManifest, err := os.Open(*manifestPath)
if err != nil {
return fmt.Errorf("reopen PP download manifest: %w", err)
}
complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest)
closeErr = overLimitManifest.Close()
if overLimitErr != nil {
return overLimitErr
}
if closeErr != nil {
return fmt.Errorf("close PP download manifest: %w", closeErr)
}
if err := ppconnector.ValidateSnapshot(complete); err != nil {
return fmt.Errorf("validate PP snapshot: %w", err)
}
plan, err := ppconnector.BuildImportPlan(complete)
if err != nil {
return fmt.Errorf("build PP import plan: %w", err)
}
completed := 0
aliases := 0
for _, item := range complete.Items {
switch item.Status {
case ppconnector.StatusCompleted:
completed++
case ppconnector.StatusContentAlias:
aliases++
}
}
_, _ = fmt.Fprintf(stdout,
"events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n",
complete.EventCount,
len(plan.SourceItems),
completed,
aliases,
overLimit.Added,
recovery.Recovered,
recovery.VerifiedBytes,
len(plan.ContentObjects),
len(plan.Versions),
len(plan.Publications),
)
if *dryRun {
_, _ = fmt.Fprintln(stdout, "dry_run=true")
return nil
}
databaseURL := strings.TrimSpace(getenv("DATABASE_URL"))
if databaseURL == "" {
return fmt.Errorf("DATABASE_URL is required unless -dry-run is used")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return fmt.Errorf("open governance database: %w", err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return fmt.Errorf("ping governance database: %w", err)
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
report, err := postgresstore.ImportPPLegacy(ctx, pool, plan)
if err != nil {
return err
}
_, _ = fmt.Fprintf(stdout,
"inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n",
report.Connections,
report.SourceItems,
report.ContentObjects,
report.Versions,
report.Publications,
)
current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
if err != nil {
return err
}
reconciliation, err := ppconnector.PlanReconciliation(complete, current)
if err != nil {
return err
}
_, _ = fmt.Fprintf(stdout,
"reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n",
reconciliation.Unchanged,
reconciliation.Adopted,
reconciliation.OverLimit,
reconciliation.UploadRequired,
)
return nil
}
+99
View File
@@ -0,0 +1,99 @@
package main
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) {
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sourceHash := strings.Repeat("a", 64)
contentHash := strings.Repeat("c", 64)
ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatalf("write ledger fixture: %v", err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatalf("write manifest fixture: %v", err)
}
var stdout bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout)
if err != nil {
t.Fatalf("run() error = %v", err)
}
output := stdout.String()
for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} {
if !strings.Contains(output, required) {
t.Fatalf("run() output missing %q: %q", required, output)
}
}
for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} {
if strings.Contains(output, forbidden) {
t.Fatalf("run() output disclosed %q", forbidden)
}
}
}
func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) {
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sha := strings.Repeat("c", 64)
ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
t.Fatal(err)
}
var output bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output)
if err != nil {
t.Fatalf("run() error = %v", err)
}
text := output.String()
if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") {
t.Fatalf("dry-run output = %q", text)
}
}
func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sha := strings.Repeat("f", 64)
ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatal(err)
}
var output bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string {
if key == "DATABASE_URL" {
return databaseURL
}
return ""
}, &output)
if err != nil {
t.Fatalf("run() error = %v", err)
}
if !strings.Contains(output.String(), "planned_uploads=0\n") {
t.Fatalf("run() output = %q", output.String())
}
}