feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
|
||||
flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL")
|
||||
manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON")
|
||||
dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return fmt.Errorf("parse arguments: %w", err)
|
||||
}
|
||||
if *ledgerPath == "" || *manifestPath == "" {
|
||||
return fmt.Errorf("-ledger and -manifest are required")
|
||||
}
|
||||
|
||||
ledger, err := os.Open(*ledgerPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP ledger: %w", err)
|
||||
}
|
||||
snapshot, foldErr := ppconnector.FoldLedger(ledger)
|
||||
closeErr := ledger.Close()
|
||||
if foldErr != nil {
|
||||
return foldErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP ledger: %w", closeErr)
|
||||
}
|
||||
|
||||
manifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP download manifest: %w", err)
|
||||
}
|
||||
recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest)
|
||||
closeErr = manifest.Close()
|
||||
if recoveryErr != nil {
|
||||
return recoveryErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
overLimitManifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reopen PP download manifest: %w", err)
|
||||
}
|
||||
complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest)
|
||||
closeErr = overLimitManifest.Close()
|
||||
if overLimitErr != nil {
|
||||
return overLimitErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
if err := ppconnector.ValidateSnapshot(complete); err != nil {
|
||||
return fmt.Errorf("validate PP snapshot: %w", err)
|
||||
}
|
||||
plan, err := ppconnector.BuildImportPlan(complete)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build PP import plan: %w", err)
|
||||
}
|
||||
|
||||
completed := 0
|
||||
aliases := 0
|
||||
for _, item := range complete.Items {
|
||||
switch item.Status {
|
||||
case ppconnector.StatusCompleted:
|
||||
completed++
|
||||
case ppconnector.StatusContentAlias:
|
||||
aliases++
|
||||
}
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n",
|
||||
complete.EventCount,
|
||||
len(plan.SourceItems),
|
||||
completed,
|
||||
aliases,
|
||||
overLimit.Added,
|
||||
recovery.Recovered,
|
||||
recovery.VerifiedBytes,
|
||||
len(plan.ContentObjects),
|
||||
len(plan.Versions),
|
||||
len(plan.Publications),
|
||||
)
|
||||
if *dryRun {
|
||||
_, _ = fmt.Fprintln(stdout, "dry_run=true")
|
||||
return nil
|
||||
}
|
||||
|
||||
databaseURL := strings.TrimSpace(getenv("DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
return fmt.Errorf("DATABASE_URL is required unless -dry-run is used")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open governance database: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("ping governance database: %w", err)
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
report, err := postgresstore.ImportPPLegacy(ctx, pool, plan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n",
|
||||
report.Connections,
|
||||
report.SourceItems,
|
||||
report.ContentObjects,
|
||||
report.Versions,
|
||||
report.Publications,
|
||||
)
|
||||
current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciliation, err := ppconnector.PlanReconciliation(complete, current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n",
|
||||
reconciliation.Unchanged,
|
||||
reconciliation.Adopted,
|
||||
reconciliation.OverLimit,
|
||||
reconciliation.UploadRequired,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sourceHash := strings.Repeat("a", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatalf("write ledger fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatalf("write manifest fixture: %v", err)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} {
|
||||
if !strings.Contains(output, required) {
|
||||
t.Fatalf("run() output missing %q: %q", required, output)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} {
|
||||
if strings.Contains(output, forbidden) {
|
||||
t.Fatalf("run() output disclosed %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
text := output.String()
|
||||
if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") {
|
||||
t.Fatalf("dry-run output = %q", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("f", 64)
|
||||
ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string {
|
||||
if key == "DATABASE_URL" {
|
||||
return databaseURL
|
||||
}
|
||||
return ""
|
||||
}, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(output.String(), "planned_uploads=0\n") {
|
||||
t.Fatalf("run() output = %q", output.String())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user