feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
"knowledge-governance/internal/workflows/sourcescan"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
handlers, err := workerHandlers(store, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
LeaseDuration: config.LeaseDuration,
|
||||
RetryDelay: config.RetryDelay,
|
||||
Handlers: handlers,
|
||||
OnError: func(err error) { log.Printf("worker error: %v", err) },
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
|
||||
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
|
||||
})
|
||||
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
|
||||
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
|
||||
handlers := map[string]jobs.Handler{
|
||||
"demo": func(_ context.Context, job jobs.Job) error {
|
||||
log.Printf("completed demo job id=%s", job.ID)
|
||||
return nil
|
||||
},
|
||||
"demo.sleep": jobs.DemoSleepHandler,
|
||||
}
|
||||
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
|
||||
}
|
||||
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
|
||||
}
|
||||
handlers["source.catalog.refresh"] = catalogHandler.Handle
|
||||
if config.PPScanEnabled {
|
||||
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
|
||||
AllowedRoot: config.PPScanAllowedRoot,
|
||||
LedgerPath: config.PPScanLedgerPath,
|
||||
ManifestPath: config.PPScanManifestPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scanner: %w", err)
|
||||
}
|
||||
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scan handler: %w", err)
|
||||
}
|
||||
handlers["source.scan.pp"] = handler.Handle
|
||||
}
|
||||
if !config.TestWeKnoraEnabled {
|
||||
return handlers, nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.TestWeKnoraBaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: config.TestWeKnoraHTTPTimeout,
|
||||
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test WeKnora client: %w", err)
|
||||
}
|
||||
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
SpoolRoot: config.GovernanceSpoolRoot,
|
||||
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
|
||||
FirstPollDelay: config.TestWeKnoraPollInitial,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test upload handler: %w", err)
|
||||
}
|
||||
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PollDelay: config.TestWeKnoraPollInitial,
|
||||
StallAfter: config.TestWeKnoraStalledAfter,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test poll handler: %w", err)
|
||||
}
|
||||
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PolicyVersion: config.TestAcceptancePolicyVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test validate handler: %w", err)
|
||||
}
|
||||
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test removal handler: %w", err)
|
||||
}
|
||||
handlers["test.upload"] = upload.Handle
|
||||
handlers["test.poll"] = poll.Handle
|
||||
handlers["test.validate"] = validate.Handle
|
||||
handlers["test.remove"] = remove.Handle
|
||||
return handlers, nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" })
|
||||
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) {
|
||||
handlers, err := workerHandlers(nil, appconfig.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["demo"] == nil || handlers["demo.sleep"] == nil {
|
||||
t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers)
|
||||
}
|
||||
if handlers["release.probe"] != nil {
|
||||
t.Fatal("ordinary worker must not register release.probe")
|
||||
}
|
||||
if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil {
|
||||
t.Fatal("ordinary worker must not register test WeKnora jobs when disabled")
|
||||
}
|
||||
if handlers["source.scan.pp"] != nil {
|
||||
t.Fatal("ordinary worker must not register PP scan jobs when disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
config := appconfig.Config{
|
||||
PPScanEnabled: true,
|
||||
PPScanAllowedRoot: root,
|
||||
PPScanLedgerPath: filepath.Join(root, "state.jsonl"),
|
||||
PPScanManifestPath: filepath.Join(root, "manifest.json"),
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["source.scan.pp"] == nil {
|
||||
t.Fatal("source.scan.pp handler is not registered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) {
|
||||
config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"}
|
||||
_, err := workerHandlers(nil, config)
|
||||
if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) {
|
||||
keyFile := filepath.Join(t.TempDir(), "api-key")
|
||||
if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := appconfig.Config{
|
||||
TestWeKnoraEnabled: true,
|
||||
TestWeKnoraAllowDockerHTTP: true,
|
||||
TestWeKnoraBaseURL: "http://app:8080",
|
||||
TestWeKnoraAPIKeyFile: keyFile,
|
||||
GovernanceSpoolRoot: t.TempDir(),
|
||||
TestWeKnoraHTTPTimeout: time.Second,
|
||||
TestWeKnoraPollInitial: time.Second,
|
||||
TestWeKnoraStalledAfter: time.Minute,
|
||||
TestWeKnoraMaxUploadBytes: 1024,
|
||||
TestAcceptancePolicyVersion: "structural-v1",
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} {
|
||||
if handlers[jobType] == nil {
|
||||
t.Fatalf("handler %q is not registered", jobType)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store
|
||||
// satisfies it and tests can fake it without a database.
|
||||
type dueSourceClaimer interface {
|
||||
ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error)
|
||||
}
|
||||
|
||||
// runSourceRefreshTick performs one dispatcher pass and logs only when work
|
||||
// moved, so an idle system does not spam the worker log.
|
||||
func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error {
|
||||
claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if claimed > 0 || enqueued > 0 {
|
||||
log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is
|
||||
// deliberately delayed by one interval: startup already races ApplyMigrations
|
||||
// and handler registration, and a due backlog enqueue adds nothing on the
|
||||
// second the process comes up (the worker will drain whatever is queued anyway
|
||||
// once sources were scheduled before a restart).
|
||||
func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) {
|
||||
if claimer == nil || interval <= 0 {
|
||||
return
|
||||
}
|
||||
if onError == nil {
|
||||
onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) }
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case tick := <-ticker.C:
|
||||
if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeClaimer struct {
|
||||
mu sync.Mutex
|
||||
calls []time.Time
|
||||
claimed [2]int
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, now)
|
||||
if f.err != nil {
|
||||
return 0, 0, f.err
|
||||
}
|
||||
return f.claimed[0], f.claimed[1], nil
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{claimed: [2]int{3, 2}}
|
||||
if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claim.calls) != 1 {
|
||||
t.Fatalf("calls=%d want 1", len(claim.calls))
|
||||
}
|
||||
sentinel := errors.New("boom")
|
||||
failing := &fakeClaimer{err: sentinel}
|
||||
if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) {
|
||||
t.Fatalf("error=%v want sentinel", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) {
|
||||
claim := &fakeClaimer{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
claim.mu.Lock()
|
||||
count := len(claim.calls)
|
||||
claim.mu.Unlock()
|
||||
if count >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scheduler did not stop on context cancel")
|
||||
}
|
||||
claim.mu.Lock()
|
||||
defer claim.mu.Unlock()
|
||||
if len(claim.calls) < 2 {
|
||||
t.Fatalf("calls=%d want at least 2", len(claim.calls))
|
||||
}
|
||||
for _, stamp := range claim.calls {
|
||||
if stamp.Location() != time.UTC {
|
||||
t.Fatalf("tick stamp %v not UTC", stamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{err: errors.New("database is offline")}
|
||||
var errorsSeen int
|
||||
var mu sync.Mutex
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) {
|
||||
mu.Lock()
|
||||
errorsSeen++
|
||||
mu.Unlock()
|
||||
})
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
seen := errorsSeen
|
||||
mu.Unlock()
|
||||
if seen >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if errorsSeen < 2 {
|
||||
t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) {
|
||||
// guards against misconfiguration killing the process
|
||||
runSourceRefreshScheduler(context.Background(), nil, time.Second, nil)
|
||||
runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil)
|
||||
}
|
||||
Reference in New Issue
Block a user