feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
+156
View File
@@ -0,0 +1,156 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}
+92
View File
@@ -0,0 +1,92 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"knowledge-governance/internal/appconfig"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func TestRunRequiresDatabaseURL(t *testing.T) {
err := run(context.Background(), func(string) string { return "" })
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
}
}
func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) {
handlers, err := workerHandlers(nil, appconfig.Config{})
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
if handlers["demo"] == nil || handlers["demo.sleep"] == nil {
t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers)
}
if handlers["release.probe"] != nil {
t.Fatal("ordinary worker must not register release.probe")
}
if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil {
t.Fatal("ordinary worker must not register test WeKnora jobs when disabled")
}
if handlers["source.scan.pp"] != nil {
t.Fatal("ordinary worker must not register PP scan jobs when disabled")
}
}
func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) {
root := t.TempDir()
config := appconfig.Config{
PPScanEnabled: true,
PPScanAllowedRoot: root,
PPScanLedgerPath: filepath.Join(root, "state.jsonl"),
PPScanManifestPath: filepath.Join(root, "manifest.json"),
}
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
if handlers["source.scan.pp"] == nil {
t.Fatal("source.scan.pp handler is not registered")
}
}
func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) {
config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"}
_, err := workerHandlers(nil, config)
if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") {
t.Fatalf("workerHandlers() error = %v", err)
}
}
func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) {
keyFile := filepath.Join(t.TempDir(), "api-key")
if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil {
t.Fatal(err)
}
config := appconfig.Config{
TestWeKnoraEnabled: true,
TestWeKnoraAllowDockerHTTP: true,
TestWeKnoraBaseURL: "http://app:8080",
TestWeKnoraAPIKeyFile: keyFile,
GovernanceSpoolRoot: t.TempDir(),
TestWeKnoraHTTPTimeout: time.Second,
TestWeKnoraPollInitial: time.Second,
TestWeKnoraStalledAfter: time.Minute,
TestWeKnoraMaxUploadBytes: 1024,
TestAcceptancePolicyVersion: "structural-v1",
}
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} {
if handlers[jobType] == nil {
t.Fatalf("handler %q is not registered", jobType)
}
}
}
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"context"
"errors"
"log"
"time"
)
// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store
// satisfies it and tests can fake it without a database.
type dueSourceClaimer interface {
ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error)
}
// runSourceRefreshTick performs one dispatcher pass and logs only when work
// moved, so an idle system does not spam the worker log.
func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error {
claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now)
if err != nil {
return err
}
if claimed > 0 || enqueued > 0 {
log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued)
}
return nil
}
// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is
// deliberately delayed by one interval: startup already races ApplyMigrations
// and handler registration, and a due backlog enqueue adds nothing on the
// second the process comes up (the worker will drain whatever is queued anyway
// once sources were scheduled before a restart).
func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) {
if claimer == nil || interval <= 0 {
return
}
if onError == nil {
onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) }
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case tick := <-ticker.C:
if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) {
onError(err)
}
}
}
}
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"context"
"errors"
"sync"
"testing"
"time"
)
type fakeClaimer struct {
mu sync.Mutex
calls []time.Time
claimed [2]int
err error
}
func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, now)
if f.err != nil {
return 0, 0, f.err
}
return f.claimed[0], f.claimed[1], nil
}
func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) {
claim := &fakeClaimer{claimed: [2]int{3, 2}}
if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil {
t.Fatal(err)
}
if len(claim.calls) != 1 {
t.Fatalf("calls=%d want 1", len(claim.calls))
}
sentinel := errors.New("boom")
failing := &fakeClaimer{err: sentinel}
if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) {
t.Fatalf("error=%v want sentinel", err)
}
}
func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) {
claim := &fakeClaimer{}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil)
}()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
claim.mu.Lock()
count := len(claim.calls)
claim.mu.Unlock()
if count >= 2 {
break
}
time.Sleep(5 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scheduler did not stop on context cancel")
}
claim.mu.Lock()
defer claim.mu.Unlock()
if len(claim.calls) < 2 {
t.Fatalf("calls=%d want at least 2", len(claim.calls))
}
for _, stamp := range claim.calls {
if stamp.Location() != time.UTC {
t.Fatalf("tick stamp %v not UTC", stamp)
}
}
}
func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) {
claim := &fakeClaimer{err: errors.New("database is offline")}
var errorsSeen int
var mu sync.Mutex
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) {
mu.Lock()
errorsSeen++
mu.Unlock()
})
}()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
mu.Lock()
seen := errorsSeen
mu.Unlock()
if seen >= 2 {
break
}
time.Sleep(5 * time.Millisecond)
}
cancel()
<-done
mu.Lock()
defer mu.Unlock()
if errorsSeen < 2 {
t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen)
}
}
func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) {
// guards against misconfiguration killing the process
runSourceRefreshScheduler(context.Background(), nil, time.Second, nil)
runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil)
}