feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"context"
"errors"
"log"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/httpapi"
"knowledge-governance/internal/monitoring"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
releaseAuthorizationToken := ""
if config.ProductionReleaseRequestsEnabled {
releaseAuthorizationToken, err = weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)
if err != nil {
return errors.New("read production release authorization token")
}
}
// P10: with a mounted API key the probe verifies a real knowledge-base read
// (readiness); without one it degrades to the liveness-only HTTP check.
var weknoraReadiness func(context.Context) error
if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" {
if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil {
probeTimeout := config.TestWeKnoraHTTPTimeout
if probeTimeout > 10*time.Second || probeTimeout <= 0 {
probeTimeout = 10 * time.Second
}
if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil {
routingKB := config.InternalRoutingKnowledgeBaseID
weknoraReadiness = func(probeCtx context.Context) error {
_, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1)
return err
}
}
}
}
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
if err != nil {
return err
}
handler, err := httpapi.NewHandler(httpapi.Config{
Store: postgresstore.NewStore(pool),
Monitor: monitorProvider,
InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID,
ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID,
ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled,
ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified,
ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef,
ProductionReleaseAuthorizationToken: releaseAuthorizationToken,
})
if err != nil {
return err
}
server := &http.Server{
Addr: config.AppAddr,
Handler: handler,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
errCh := make(chan error, 1)
go func() { errCh <- server.ListenAndServe() }()
log.Printf("governance API listening on %s", config.AppAddr)
select {
case err := <-errCh:
if errors.Is(err, http.ErrServerClosed) {
return nil
}
return err
case <-ctx.Done():
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return server.Shutdown(shutdownCtx)
}
}
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"os"
"strings"
"testing"
)
func TestRunRequiresDatabaseURL(t *testing.T) {
err := run(context.Background(), func(string) string { return "" })
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
}
}
func TestAPIPropagatesProductionReleaseRequestGate(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
normalized := strings.Join(strings.Fields(string(source)), " ")
for _, required := range []string{
"if config.ProductionReleaseRequestsEnabled {",
"weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)",
"ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled",
"ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified",
"ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef",
"ProductionReleaseAuthorizationToken: releaseAuthorizationToken",
} {
if !strings.Contains(normalized, required) {
t.Fatalf("API production authorization wiring missing %q", required)
}
}
}
func TestComposeDefaultsProductionBackupRestoreGateToFalse(t *testing.T) {
source, err := os.ReadFile("../../compose.yaml")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(source), `PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false}`) {
t.Fatal("compose must pass the production backup restore gate with a false default")
}
}
+153
View File
@@ -0,0 +1,153 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"log"
"os"
"strings"
"github.com/jackc/pgx/v5/pgxpool"
ppconnector "knowledge-governance/internal/connectors/pp"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func main() {
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError)
flags.SetOutput(io.Discard)
ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL")
manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON")
dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access")
if err := flags.Parse(args); err != nil {
return fmt.Errorf("parse arguments: %w", err)
}
if *ledgerPath == "" || *manifestPath == "" {
return fmt.Errorf("-ledger and -manifest are required")
}
ledger, err := os.Open(*ledgerPath)
if err != nil {
return fmt.Errorf("open PP ledger: %w", err)
}
snapshot, foldErr := ppconnector.FoldLedger(ledger)
closeErr := ledger.Close()
if foldErr != nil {
return foldErr
}
if closeErr != nil {
return fmt.Errorf("close PP ledger: %w", closeErr)
}
manifest, err := os.Open(*manifestPath)
if err != nil {
return fmt.Errorf("open PP download manifest: %w", err)
}
recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest)
closeErr = manifest.Close()
if recoveryErr != nil {
return recoveryErr
}
if closeErr != nil {
return fmt.Errorf("close PP download manifest: %w", closeErr)
}
overLimitManifest, err := os.Open(*manifestPath)
if err != nil {
return fmt.Errorf("reopen PP download manifest: %w", err)
}
complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest)
closeErr = overLimitManifest.Close()
if overLimitErr != nil {
return overLimitErr
}
if closeErr != nil {
return fmt.Errorf("close PP download manifest: %w", closeErr)
}
if err := ppconnector.ValidateSnapshot(complete); err != nil {
return fmt.Errorf("validate PP snapshot: %w", err)
}
plan, err := ppconnector.BuildImportPlan(complete)
if err != nil {
return fmt.Errorf("build PP import plan: %w", err)
}
completed := 0
aliases := 0
for _, item := range complete.Items {
switch item.Status {
case ppconnector.StatusCompleted:
completed++
case ppconnector.StatusContentAlias:
aliases++
}
}
_, _ = fmt.Fprintf(stdout,
"events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n",
complete.EventCount,
len(plan.SourceItems),
completed,
aliases,
overLimit.Added,
recovery.Recovered,
recovery.VerifiedBytes,
len(plan.ContentObjects),
len(plan.Versions),
len(plan.Publications),
)
if *dryRun {
_, _ = fmt.Fprintln(stdout, "dry_run=true")
return nil
}
databaseURL := strings.TrimSpace(getenv("DATABASE_URL"))
if databaseURL == "" {
return fmt.Errorf("DATABASE_URL is required unless -dry-run is used")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return fmt.Errorf("open governance database: %w", err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return fmt.Errorf("ping governance database: %w", err)
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
report, err := postgresstore.ImportPPLegacy(ctx, pool, plan)
if err != nil {
return err
}
_, _ = fmt.Fprintf(stdout,
"inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n",
report.Connections,
report.SourceItems,
report.ContentObjects,
report.Versions,
report.Publications,
)
current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
if err != nil {
return err
}
reconciliation, err := ppconnector.PlanReconciliation(complete, current)
if err != nil {
return err
}
_, _ = fmt.Fprintf(stdout,
"reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n",
reconciliation.Unchanged,
reconciliation.Adopted,
reconciliation.OverLimit,
reconciliation.UploadRequired,
)
return nil
}
+99
View File
@@ -0,0 +1,99 @@
package main
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) {
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sourceHash := strings.Repeat("a", 64)
contentHash := strings.Repeat("c", 64)
ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatalf("write ledger fixture: %v", err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatalf("write manifest fixture: %v", err)
}
var stdout bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout)
if err != nil {
t.Fatalf("run() error = %v", err)
}
output := stdout.String()
for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} {
if !strings.Contains(output, required) {
t.Fatalf("run() output missing %q: %q", required, output)
}
}
for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} {
if strings.Contains(output, forbidden) {
t.Fatalf("run() output disclosed %q", forbidden)
}
}
}
func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) {
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sha := strings.Repeat("c", 64)
ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
t.Fatal(err)
}
var output bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output)
if err != nil {
t.Fatalf("run() error = %v", err)
}
text := output.String()
if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") {
t.Fatalf("dry-run output = %q", text)
}
}
func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
directory := t.TempDir()
ledgerPath := filepath.Join(directory, "state.jsonl")
manifestPath := filepath.Join(directory, "manifest.json")
sha := strings.Repeat("f", 64)
ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatal(err)
}
var output bytes.Buffer
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string {
if key == "DATABASE_URL" {
return databaseURL
}
return ""
}, &output)
if err != nil {
t.Fatalf("run() error = %v", err)
}
if !strings.Contains(output.String(), "planned_uploads=0\n") {
t.Fatalf("run() output = %q", output.String())
}
}
+313
View File
@@ -0,0 +1,313 @@
// Command release-diag replays the read-only steps of the release export path
// (source PG snapshot, source ES snapshot, source object read) and additionally
// lists candidate OSS key prefixes, to pinpoint a failing export step with its
// real underlying error. Credentials are read from files only and never printed.
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net/url"
"os"
"time"
"unicode/utf8"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/platform/weknora"
"knowledge-governance/internal/releaseadapter/v071"
)
type objectCfg struct {
Provider string `json:"provider"`
StorageBackendID string `json:"storage_backend_id"`
Endpoint string `json:"endpoint"`
Region string `json:"region"`
Bucket string `json:"bucket"`
AccessKeyFile string `json:"access_key_file"`
SecretKeyFile string `json:"secret_key_file"`
}
type endpointCfg struct {
DatabaseURLFile string `json:"database_url_file"`
Elasticsearch struct {
BaseURL string `json:"base_url"`
Index string `json:"index"`
} `json:"elasticsearch"`
Object objectCfg `json:"object"`
WeKnora struct {
BaseURL string `json:"base_url"`
APIKeyFile string `json:"api_key_file"`
} `json:"weknora"`
}
type config struct {
Routes []struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EmbeddingModel string `json:"embedding_model"`
Source endpointCfg `json:"source"`
Target endpointCfg `json:"target"`
} `json:"routes"`
}
func secret(path string) string {
data, err := os.ReadFile(path)
if err != nil {
fmt.Println("FATAL read secret:", err)
os.Exit(1)
}
return string(data)
}
func main() {
configPath := flag.String("config", "", "release config json")
knowledgeID := flag.String("knowledge", "", "knowledge id to diagnose")
flag.Parse()
raw, err := os.ReadFile(*configPath)
if err != nil {
panic(err)
}
var cfg config
if err := json.Unmarshal(raw, &cfg); err != nil || len(cfg.Routes) != 1 {
panic(err)
}
route := cfg.Routes[0]
fmt.Println("route:", route.KnowledgeBaseID, "tenant:", route.TenantID)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
pool, err := pgxpool.New(ctx, secret(route.Source.DatabaseURLFile))
if err != nil {
panic(err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
fmt.Println("STEP source PG connect: FAIL", err)
return
}
fmt.Println("STEP source PG connect: ok")
postgres, err := v071.NewPostgresExporter(pool)
if err != nil {
panic(err)
}
snapshot, err := postgres.Export(ctx, v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
if err != nil {
fmt.Println("STEP postgres export: FAIL", err)
return
}
names := make([]string, 0, len(snapshot.Tables))
for _, table := range snapshot.Tables {
names = append(names, fmt.Sprintf("%s:%d", table.Name, table.RowCount))
}
fmt.Println("STEP postgres export: ok tables", names)
search, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, PageSize: 500})
if err != nil {
panic(err)
}
searchSnapshot, err := search.Export(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
if err != nil {
fmt.Println("STEP elasticsearch export: FAIL", err)
return
}
fmt.Printf("STEP elasticsearch export: ok documents=%d\n", len(searchSnapshot.Documents))
store, err := newStore(ctx, route.Source.Object)
if err != nil {
fmt.Println("STEP object store build: FAIL", err)
return
}
exporter, err := v071.NewObjectExporter(store, v071.ObjectExporterLimits{})
if err != nil {
panic(err)
}
objectSnapshot, err := exporter.Export(ctx, snapshot)
if err != nil {
fmt.Println("STEP object export: FAIL", err)
} else {
fmt.Printf("STEP object export: ok objects=%d bytes=%d\n", len(objectSnapshot.Objects), objectSnapshot.TotalBytes)
}
client, err := newRawClient(ctx, route.Source.Object)
if err != nil {
panic(err)
}
for _, prefix := range []string{"weknora/", "oss://", ""} {
listed, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(route.Source.Object.Bucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(3)})
if err != nil {
fmt.Printf("STEP list prefix %q: FAIL %v\n", prefix, err)
continue
}
fmt.Printf("STEP list prefix %q: %d objects\n", prefix, len(listed.Contents))
for _, object := range listed.Contents {
fmt.Println(" ", aws.ToString(object.Key))
}
}
knowledge := snapshot.Table("knowledges")
for _, row := range knowledge.Rows {
var typed struct {
Source string `json:"source"`
}
_ = json.Unmarshal(row, &typed)
fmt.Println("knowledge.source:", typed.Source)
}
for _, raw := range snapshot.Table("resources").Rows {
var typed struct {
PhysicalPath string `json:"physical_path"`
}
_ = json.Unmarshal(raw, &typed)
fmt.Println("resource physical_path:", typed.PhysicalPath)
}
fmt.Println("STEP target object stage probe:")
targetStore, err := newStore(ctx, route.Target.Object)
if err != nil {
fmt.Println(" target store build FAIL", err)
return
}
probePath := ".governance-release-diag/probe.txt"
payload := []byte("release-sim-probe")
if err := targetStore.Put(ctx, route.Target.Object.Provider, probePath, bytes.NewReader(payload), int64(len(payload))); err != nil {
fmt.Println(" target Put FAIL:", err)
} else {
size, digest, err := targetStore.Stat(ctx, route.Target.Object.Provider, probePath)
fmt.Println(" target Put ok; Stat:", size, digest[:8], "err:", err)
}
fmt.Println("STEP real ObjectImporter Stage/Promote with exported snapshot:")
importer, err := v071.NewObjectImporter(targetStore, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
if err != nil {
fmt.Println(" importer build FAIL", err)
return
}
stage, err := importer.Stage(ctx, "diag-stage", objectSnapshot)
if err != nil {
fmt.Println(" Stage FAIL:", err)
return
}
fmt.Println(" Stage ok objects:", len(stage.Objects))
if err := importer.Promote(ctx, stage); err != nil {
fmt.Println(" Promote FAIL:", err)
return
}
fmt.Println(" Promote ok")
fmt.Println("STEP target Postgres replace:")
targetPool, err := pgxpool.New(ctx, secret(route.Target.DatabaseURLFile))
if err != nil {
fmt.Println(" target pool FAIL", err)
return
}
defer targetPool.Close()
if err := targetPool.Ping(ctx); err != nil {
fmt.Println(" target PG connect FAIL", err)
return
}
pgImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
if err != nil {
fmt.Println(" importer config FAIL", err)
return
}
identity := v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}
if err := pgImporter.Replace(ctx, identity, snapshot); err != nil {
fmt.Println(" Replace FAIL:", err)
return
}
fmt.Println(" Replace ok")
fmt.Println("STEP target ES replace:")
targetSearch, err := v071.NewSearchImporter(v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, PageSize: 500})
if err != nil {
fmt.Println(" importer build FAIL", err)
return
}
if _, err := targetSearch.Replace(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}, searchSnapshot); err != nil {
fmt.Println(" ES Replace FAIL:", err)
return
}
fmt.Println(" ES Replace ok")
fmt.Println("STEP target WeKnora verification replay:")
wknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: secret(route.Target.WeKnora.APIKeyFile), Timeout: 30 * time.Second, AllowDockerHTTP: true})
if err != nil {
fmt.Println(" client build FAIL", err)
return
}
knowledgeRemote, err := wknoraClient.GetKnowledge(ctx, *knowledgeID)
fmt.Println(" GetKnowledge err:", err, "id:", knowledgeRemote.ID, "kb:", knowledgeRemote.KnowledgeBaseID, "parse:", knowledgeRemote.ParseStatus)
chunksRemote, total, err := wknoraClient.ListChunks(ctx, *knowledgeID, 1, 5, nil)
fmt.Println(" ListChunks err:", err, "total:", total, "page:", len(chunksRemote))
probe := ""
for _, document := range searchSnapshot.Documents {
var fields struct {
Content string `json:"content"`
}
_ = json.Unmarshal(document.Source, &fields)
if utf8.RuneCountInString(fields.Content) >= 12 {
probe = string([]rune(fields.Content)[:min(48, utf8.RuneCountInString(fields.Content))])
break
}
}
results, err := wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: probe, KnowledgeIDs: []string{*knowledgeID}})
fmt.Println(" SearchKnowledge err:", err, "probe_len:", len(probe), "results:", len(results))
for index, result := range results {
if index >= 3 {
break
}
fmt.Printf(" hit knowledge=%s score=%.3f type=%s\n", result.KnowledgeID, result.Score, result.ChunkType)
}
if len(results) == 0 && probe != "" {
results, err = wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: "CRA 620mm", KnowledgeIDs: []string{*knowledgeID}})
fmt.Println(" short-probe SearchKnowledge err:", err, "results:", len(results))
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
func newStore(ctx context.Context, object objectCfg) (*v071.S3ObjectStore, error) {
parsed, err := url.Parse(object.Endpoint)
if err != nil {
return nil, err
}
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{
Provider: object.Provider, Endpoint: object.Endpoint, Region: object.Region,
AccessKeyID: secret(object.AccessKeyFile), SecretAccessKey: secret(object.SecretKeyFile),
Bucket: object.Bucket, Secure: parsed.Scheme == "https",
})
}
func newRawClient(ctx context.Context, object objectCfg) (*s3.Client, error) {
parsed, err := url.Parse(object.Endpoint)
if err != nil {
return nil, err
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(object.Region), awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(secret(object.AccessKeyFile), secret(object.SecretKeyFile), "")))
if err != nil {
return nil, err
}
return s3.NewFromConfig(awsCfg, func(options *s3.Options) {
options.BaseEndpoint = aws.String(object.Endpoint)
options.UsePathStyle = false
_ = parsed
}), nil
}
var _ = io.Discard
+155
View File
@@ -0,0 +1,155 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"syscall"
)
const maxReleaseConfigBytes = 1024 * 1024
type releaseRuntimeConfig struct {
ArchiveRoot string `json:"archive_root"`
Routes []releaseRouteConfig `json:"routes"`
}
type releaseRouteConfig struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EmbeddingModel string `json:"embedding_model"`
Source releaseEndpointConfig `json:"source"`
Target releaseEndpointConfig `json:"target"`
}
type releaseEndpointConfig struct {
DatabaseURLFile string `json:"database_url_file"`
Elasticsearch releaseElasticsearchConfig `json:"elasticsearch"`
Object releaseObjectConfig `json:"object"`
WeKnora releaseWeKnoraConfig `json:"weknora"`
}
type releaseElasticsearchConfig struct {
BaseURL string `json:"base_url"`
Index string `json:"index"`
TokenFile string `json:"token_file"`
}
type releaseObjectConfig struct {
Provider string `json:"provider"`
Endpoint string `json:"endpoint"`
Region string `json:"region"`
Bucket string `json:"bucket"`
StorageBackendID string `json:"storage_backend_id"`
AccessKeyFile string `json:"access_key_file"`
SecretKeyFile string `json:"secret_key_file"`
}
type releaseWeKnoraConfig struct {
BaseURL string `json:"base_url"`
APIKeyFile string `json:"api_key_file"`
AllowDockerHTTP bool `json:"allow_docker_http"`
}
func loadReleaseRuntimeConfig(path string) (releaseRuntimeConfig, error) {
if !filepath.IsAbs(path) {
return releaseRuntimeConfig{}, errors.New("release config path must be absolute")
}
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return releaseRuntimeConfig{}, errors.New("open release config")
}
file := os.NewFile(uintptr(fd), "release-config")
if file == nil {
syscall.Close(fd)
return releaseRuntimeConfig{}, errors.New("open release config")
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() <= 0 || info.Size() > maxReleaseConfigBytes {
return releaseRuntimeConfig{}, errors.New("release config must be a non-empty regular 0600 file")
}
payload, err := io.ReadAll(io.LimitReader(file, maxReleaseConfigBytes+1))
if err != nil || len(payload) > maxReleaseConfigBytes {
return releaseRuntimeConfig{}, errors.New("read release config")
}
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.DisallowUnknownFields()
var config releaseRuntimeConfig
if err := decoder.Decode(&config); err != nil {
return releaseRuntimeConfig{}, errors.New("decode release config")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return releaseRuntimeConfig{}, errors.New("release config contains trailing data")
}
if err := validateReleaseRuntimeConfig(config); err != nil {
return releaseRuntimeConfig{}, err
}
return config, nil
}
func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error {
if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 {
return errors.New("release archive root and routes are required")
}
seen := map[string]bool{}
for _, route := range config.Routes {
if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] {
return errors.New("invalid or duplicate release route")
}
seen[route.KnowledgeBaseID] = true
if !validReleaseEndpoint(route.Source, false) || !validReleaseEndpoint(route.Target, true) {
return errors.New("invalid release route endpoint")
}
if route.Source.DatabaseURLFile == route.Target.DatabaseURLFile ||
route.Source.Elasticsearch.BaseURL == route.Target.Elasticsearch.BaseURL ||
(route.Source.Object.Endpoint == route.Target.Object.Endpoint && route.Source.Object.Bucket == route.Target.Object.Bucket) {
return errors.New("release source and target must be distinct")
}
}
return nil
}
func validReleaseEndpoint(endpoint releaseEndpointConfig, target bool) bool {
if !absoluteFile(endpoint.DatabaseURLFile) || !validHTTPURL(endpoint.Elasticsearch.BaseURL) || !validConfigIdentifier(endpoint.Elasticsearch.Index) ||
(endpoint.Elasticsearch.TokenFile != "" && !absoluteFile(endpoint.Elasticsearch.TokenFile)) ||
!validObjectEndpoint(endpoint.Object) {
return false
}
if target {
return validConfigIdentifier(endpoint.Object.StorageBackendID) && validHTTPURL(endpoint.WeKnora.BaseURL) && absoluteFile(endpoint.WeKnora.APIKeyFile)
}
return endpoint.WeKnora.BaseURL == "" && endpoint.WeKnora.APIKeyFile == ""
}
func validObjectEndpoint(object releaseObjectConfig) bool {
return (object.Provider == "minio" || object.Provider == "s3" || object.Provider == "oss") && validHTTPURL(object.Endpoint) &&
validConfigIdentifier(object.Bucket) && absoluteFile(object.AccessKeyFile) && absoluteFile(object.SecretKeyFile)
}
func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path }
func validHTTPURL(value string) bool {
parsed, err := url.Parse(value)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
}
func validConfigIdentifier(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > 128 {
return false
}
for _, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' {
continue
}
return false
}
return true
}
+65
View File
@@ -0,0 +1,65 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadReleaseRuntimeConfigValidatesRestrictedMultiKBConfig(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "release.json")
payload := validReleaseConfigJSON(root)
if err := os.WriteFile(path, []byte(payload), 0o600); err != nil {
t.Fatal(err)
}
config, err := loadReleaseRuntimeConfig(path)
if err != nil {
t.Fatal(err)
}
if len(config.Routes) != 1 || config.Routes[0].KnowledgeBaseID != "kb-safe" || config.Routes[0].Target.Object.StorageBackendID != "target-backend" {
t.Fatalf("config=%#v", config)
}
}
func TestLoadReleaseRuntimeConfigRejectsUnsafeFilesAndUnknownFields(t *testing.T) {
root := t.TempDir()
payload := validReleaseConfigJSON(root)
for _, test := range []struct {
name string
setup func(string) error
}{
{name: "overbroad", setup: func(path string) error { return os.WriteFile(path, []byte(payload), 0o644) }},
{name: "unknown field", setup: func(path string) error {
return os.WriteFile(path, []byte(strings.Replace(payload, `"archive_root":`, `"unknown":true,"archive_root":`, 1)), 0o600)
}},
} {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
if err := test.setup(path); err != nil {
t.Fatal(err)
}
if _, err := loadReleaseRuntimeConfig(path); err == nil {
t.Fatal("unsafe config accepted")
}
})
}
real := filepath.Join(root, "real.json")
_ = os.WriteFile(real, []byte(payload), 0o600)
link := filepath.Join(root, "link.json")
_ = os.Symlink(real, link)
if _, err := loadReleaseRuntimeConfig(link); err == nil {
t.Fatal("symlink accepted")
}
}
func validReleaseConfigJSON(root string) string {
return `{
"archive_root":"` + filepath.ToSlash(filepath.Join(root, "archive")) + `",
"routes":[{
"tenant_id":10000,"knowledge_base_id":"kb-safe","embedding_model":"bge-m3",
"source":{"database_url_file":"/run/secrets/source-db","elasticsearch":{"base_url":"http://source-es:9200","index":"chunks","token_file":"/run/secrets/source-es"},"object":{"provider":"minio","endpoint":"http://source-minio:9000","region":"us-east-1","bucket":"source","access_key_file":"/run/secrets/source-ak","secret_key_file":"/run/secrets/source-sk"}},
"target":{"database_url_file":"/run/secrets/target-db","elasticsearch":{"base_url":"https://target-es.example","index":"chunks","token_file":"/run/secrets/target-es"},"object":{"provider":"oss","endpoint":"https://oss.example","region":"cn-shenzhen","bucket":"target","storage_backend_id":"target-backend","access_key_file":"/run/secrets/target-ak","secret_key_file":"/run/secrets/target-sk"},"weknora":{"base_url":"https://weknora.example","api_key_file":"/run/secrets/target-weknora"}}
}]}`
}
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"context"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
if !config.ReleaseWorkerEnabled {
log.Print("release worker disabled; no production credentials are loaded")
<-ctx.Done()
return nil
}
runtimeConfig, err := loadReleaseRuntimeConfig(config.ReleaseConfigFile)
if err != nil {
return err
}
releaseRuntime, err := buildReleaseRuntime(ctx, runtimeConfig)
if err != nil {
return err
}
defer releaseRuntime.Close()
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
releaseHandler, err := releaseworkflow.NewHandler(store, releaseRuntime.publisher, time.Now)
if err != nil {
return err
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: map[string]jobs.Handler{
"release.publish": releaseHandler,
"release.rollback": releaseHandler,
},
OnError: func(err error) { log.Printf("release worker error: %v", err) },
})
if err != nil {
return err
}
log.Printf("release worker %s started with %d release route(s)", config.WorkerID, len(runtimeConfig.Routes))
return worker.Run(ctx, config.PollInterval)
}
+44
View File
@@ -0,0 +1,44 @@
package main
import (
"context"
"os"
"strings"
"testing"
)
func TestRunReturnsWithoutDatabaseWhenReleaseWorkerDisabled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
values := map[string]string{
"DATABASE_URL": "postgres://must-not-be-contacted.invalid/governance",
"RELEASE_WORKER_ENABLED": "false",
}
if err := run(ctx, func(key string) string { return values[key] }); err != nil {
t.Fatalf("run() error = %v", err)
}
}
func TestEnabledReleaseWorkerLoadsRestrictedConfigBeforeDatabaseConnections(t *testing.T) {
values := map[string]string{"DATABASE_URL": "not-a-database-url", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/does/not/exist/release.json"}
err := run(context.Background(), func(key string) string { return values[key] })
if err == nil || !strings.Contains(err.Error(), "release config") || strings.Contains(err.Error(), "database") {
t.Fatalf("error=%v", err)
}
}
func TestReleaseWorkerRegistersOnlyPublishAndRollbackHandlers(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
text := string(source)
for _, required := range []string{`"release.publish"`, `"release.rollback"`} {
if !strings.Contains(text, required) {
t.Fatalf("missing handler %s", required)
}
}
if strings.Contains(text, `"release.probe"`) {
t.Fatal("probe-only release handler remains")
}
}
+222
View File
@@ -0,0 +1,222 @@
package main
import (
"context"
"errors"
"net/url"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/platform/weknora"
v071 "knowledge-governance/internal/releaseadapter/v071"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
type releaseRuntime struct {
publisher *releaseworkflow.RoutingPublisher
pools []*pgxpool.Pool
}
func (runtime *releaseRuntime) Close() {
for index := len(runtime.pools) - 1; index >= 0; index-- {
runtime.pools[index].Close()
}
}
func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) {
runtime := &releaseRuntime{}
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
for _, route := range config.Routes {
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
if err != nil {
runtime.Close()
return nil, err
}
runtime.pools = append(runtime.pools, pools...)
publishers[route.KnowledgeBaseID] = publisher
}
router, err := releaseworkflow.NewRoutingPublisher(publishers)
if err != nil {
runtime.Close()
return nil, errors.New("create release publisher routes")
}
runtime.publisher = router
return runtime, nil
}
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
if err != nil {
return nil, nil, errors.New("read source database credential")
}
targetDatabaseURL, err := readRestrictedValue(route.Target.DatabaseURLFile)
if err != nil {
return nil, nil, errors.New("read target database credential")
}
if err := validateDistinctDatabaseURLs(sourceDatabaseURL, targetDatabaseURL); err != nil {
return nil, nil, err
}
sourcePool, err := openReleasePool(ctx, sourceDatabaseURL)
if err != nil {
return nil, nil, errors.New("connect release source database")
}
pools := []*pgxpool.Pool{sourcePool}
targetPool, err := openReleasePool(ctx, targetDatabaseURL)
if err != nil {
sourcePool.Close()
return nil, nil, errors.New("connect release target database")
}
pools = append(pools, targetPool)
cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
for index := len(pools) - 1; index >= 0; index-- {
pools[index].Close()
}
return nil, nil, err
}
sourceSearchToken, err := readOptionalRestrictedValue(route.Source.Elasticsearch.TokenFile)
if err != nil {
return cleanup(errors.New("read source Elasticsearch credential"))
}
targetSearchToken, err := readOptionalRestrictedValue(route.Target.Elasticsearch.TokenFile)
if err != nil {
return cleanup(errors.New("read target Elasticsearch credential"))
}
sourceSearch, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, AuthorizationToken: sourceSearchToken, PageSize: 500})
if err != nil {
return cleanup(errors.New("create source Elasticsearch exporter"))
}
targetSearchConfig := v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, AuthorizationToken: targetSearchToken, PageSize: 500}
targetSearchInspector, err := v071.NewSearchExporter(targetSearchConfig)
if err != nil {
return cleanup(errors.New("create target Elasticsearch inspector"))
}
targetSearch, err := v071.NewSearchImporter(targetSearchConfig)
if err != nil {
return cleanup(errors.New("create target Elasticsearch importer"))
}
sourceObject, err := buildS3ObjectStore(route.Source.Object)
if err != nil {
return cleanup(errors.New("create source object reader"))
}
targetObject, err := buildS3ObjectStore(route.Target.Object)
if err != nil {
return cleanup(errors.New("create target object store"))
}
objectExporter, err := v071.NewObjectExporter(sourceObject, v071.ObjectExporterLimits{})
if err != nil {
return cleanup(errors.New("create source object exporter"))
}
objectImporter, err := v071.NewObjectImporter(targetObject, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
if err != nil {
return cleanup(errors.New("create target object importer"))
}
sourceContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: sourcePool, Mapping: sourceSearch, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Source.Elasticsearch.Index})
if err != nil {
return cleanup(errors.New("create source release contract inspector"))
}
targetContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: targetPool, Mapping: targetSearchInspector, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Target.Elasticsearch.Index})
if err != nil {
return cleanup(errors.New("create target release contract inspector"))
}
postgresExporter, err := v071.NewPostgresExporter(sourcePool)
if err != nil {
return cleanup(errors.New("create source PostgreSQL exporter"))
}
postgresImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
if err != nil {
return cleanup(errors.New("create target PostgreSQL importer"))
}
documentExporter, err := v071.NewDocumentExporter(v071.DocumentExporterConfig{Contract: sourceContract, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: route.TenantID, EmbeddingModel: route.EmbeddingModel, Now: time.Now})
if err != nil {
return cleanup(errors.New("create v0.7.1 document exporter"))
}
archive, err := v071.NewBundleArchive(filepath.Join(archiveRoot, route.KnowledgeBaseID), 1024*1024*1024)
if err != nil {
return cleanup(errors.New("open release bundle archive"))
}
apiKey, err := readRestrictedValue(route.Target.WeKnora.APIKeyFile)
if err != nil {
return cleanup(errors.New("read target WeKnora credential"))
}
weknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: apiKey, Timeout: 60 * time.Second, AllowDockerHTTP: route.Target.WeKnora.AllowDockerHTTP})
if err != nil {
return cleanup(errors.New("create target WeKnora verification client"))
}
verifier, err := v071.NewWeKnoraVerifier(weknoraClient)
if err != nil {
return cleanup(errors.New("create target WeKnora verifier"))
}
documentTarget, err := v071.NewDocumentTarget(v071.DocumentTargetConfig{Archive: archive, Contract: targetContract, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier})
if err != nil {
return cleanup(errors.New("create v0.7.1 document target"))
}
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
if err != nil {
return cleanup(errors.New("create v0.7.1 release publisher"))
}
return publisher, pools, nil
}
func buildS3ObjectStore(config releaseObjectConfig) (*v071.S3ObjectStore, error) {
accessKey, err := readRestrictedValue(config.AccessKeyFile)
if err != nil {
return nil, err
}
secretKey, err := readRestrictedValue(config.SecretKeyFile)
if err != nil {
return nil, err
}
parsed, err := url.Parse(config.Endpoint)
if err != nil {
return nil, errors.New("invalid object endpoint")
}
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: config.Provider, Endpoint: config.Endpoint, Region: config.Region, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: config.Bucket, Secure: parsed.Scheme == "https", ForcePathStyle: config.Provider == "minio"})
}
func readRestrictedValue(path string) (string, error) {
return weknora.ReadAPIKeyFile(path)
}
func readOptionalRestrictedValue(path string) (string, error) {
if path == "" {
return "", nil
}
return readRestrictedValue(path)
}
func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return nil, err
}
if err := pool.Ping(ctx); err != nil {
pool.Close()
return nil, err
}
return pool, nil
}
func validateDistinctDatabaseURLs(sourceURL, targetURL string) error {
source, err := pgxpool.ParseConfig(sourceURL)
if err != nil {
return errors.New("invalid source database credential")
}
target, err := pgxpool.ParseConfig(targetURL)
if err != nil {
return errors.New("invalid target database credential")
}
sourceIdentity := strings.ToLower(source.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(source.ConnConfig.Port), 10) + "|" + source.ConnConfig.Database
targetIdentity := strings.ToLower(target.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(target.ConnConfig.Port), 10) + "|" + target.ConnConfig.Database
if sourceIdentity == targetIdentity {
return errors.New("release source and target database must be distinct")
}
return nil
}
+60
View File
@@ -0,0 +1,60 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
func TestReleaseRuntimeRejectsSameSourceAndTargetDatabaseValues(t *testing.T) {
if err := validateDistinctDatabaseURLs("postgres://user@same/db?sslmode=require", "postgres://user@same/db?sslmode=require"); err == nil {
t.Fatal("same database URL accepted")
}
if err := validateDistinctDatabaseURLs("postgres://user@source/db?sslmode=require", "postgres://user@target/db?sslmode=require"); err != nil {
t.Fatal(err)
}
}
func TestBuildReleaseRuntimeWithRestrictedFilesIntegration(t *testing.T) {
sourceDatabaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL")
targetDatabaseURL := os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL")
minioEndpoint := os.Getenv("TEST_MINIO_ENDPOINT")
if sourceDatabaseURL == "" || targetDatabaseURL == "" || minioEndpoint == "" {
t.Skip("release runtime integration environment is not set")
}
esSource := httptest.NewServer(http.NotFoundHandler())
defer esSource.Close()
esTarget := httptest.NewServer(http.NotFoundHandler())
defer esTarget.Close()
weknoraTarget := httptest.NewServer(http.NotFoundHandler())
defer weknoraTarget.Close()
root := t.TempDir()
archiveRoot := filepath.Join(root, "archive")
archiveRoute := filepath.Join(archiveRoot, "kb-release-safe")
if err := os.MkdirAll(archiveRoute, 0o700); err != nil {
t.Fatal(err)
}
secret := func(name, value string) string {
path := filepath.Join(root, name)
if err := os.WriteFile(path, []byte(value+"\n"), 0o600); err != nil {
t.Fatal(err)
}
return path
}
route := releaseRouteConfig{
TenantID: 10000, KnowledgeBaseID: "kb-release-safe", EmbeddingModel: "bge-m3",
Source: releaseEndpointConfig{DatabaseURLFile: secret("source-db", sourceDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esSource.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-source", AccessKeyFile: secret("source-ak", "release-test"), SecretKeyFile: secret("source-sk", "release-test-secret")}},
Target: releaseEndpointConfig{DatabaseURLFile: secret("target-db", targetDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esTarget.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-target", StorageBackendID: "target-backend", AccessKeyFile: secret("target-ak", "release-test"), SecretKeyFile: secret("target-sk", "release-test-secret")}, WeKnora: releaseWeKnoraConfig{BaseURL: weknoraTarget.URL, APIKeyFile: secret("target-weknora", "test-api-key"), AllowDockerHTTP: true}},
}
runtime, err := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{ArchiveRoot: archiveRoot, Routes: []releaseRouteConfig{route}})
if err != nil {
t.Fatal(err)
}
runtime.Close()
if runtime.publisher == nil || len(runtime.pools) != 2 {
t.Fatalf("runtime=%#v", runtime)
}
}
+131
View File
@@ -0,0 +1,131 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"os"
"strconv"
"time"
"github.com/jackc/pgx/v5/pgxpool"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/spool"
)
type manualSeeder interface {
SeedManualPublication(context.Context, postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error)
}
type seedConfig struct {
SpoolRoot string
SpoolRef string
Format string
Title string
TestKBID string
MaxBytes int64
Now time.Time
}
func main() {
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
flags := flag.NewFlagSet("test-publication-seed", flag.ContinueOnError)
flags.SetOutput(io.Discard)
spoolRef := flags.String("spool-ref", "", "opaque relative spool reference")
format := flags.String("format", "md", "validated artifact format")
title := flags.String("title", "", "safe display title")
if err := flags.Parse(args); err != nil {
return errors.New("invalid seed arguments")
}
databaseURL := getenv("DATABASE_URL")
spoolRoot := getenv("GOVERNANCE_SPOOL_ROOT")
if spoolRoot == "" {
spoolRoot = "/var/lib/governance/spool"
}
testKBID := getenv("TEST_WEKNORA_KB_ID")
maxBytes := int64(50 * 1024 * 1024)
if raw := getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"); raw != "" {
parsed, err := strconv.ParseInt(raw, 10, 64)
if err != nil || parsed <= 0 || parsed > 50*1024*1024 {
return errors.New("invalid TEST_WEKNORA_MAX_UPLOAD_BYTES")
}
maxBytes = parsed
}
if databaseURL == "" || testKBID == "" || *spoolRef == "" || *title == "" {
return errors.New("DATABASE_URL, TEST_WEKNORA_KB_ID, --spool-ref and --title are required")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return errors.New("open governance database")
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return errors.New("connect governance database")
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return errors.New("apply governance migrations")
}
result, err := seedFromSpool(ctx, postgresstore.NewStore(pool), seedConfig{
SpoolRoot: spoolRoot,
SpoolRef: *spoolRef,
Format: *format,
Title: *title,
TestKBID: testKBID,
MaxBytes: maxBytes,
Now: time.Now().UTC(),
})
if err != nil {
return err
}
return json.NewEncoder(output).Encode(result)
}
func seedFromSpool(ctx context.Context, store manualSeeder, config seedConfig) (postgresstore.ManualPublicationSeedResult, error) {
if store == nil {
return postgresstore.ManualPublicationSeedResult{}, errors.New("manual publication store is required")
}
expectation, err := spool.InspectVerified(config.SpoolRoot, config.SpoolRef, config.Format, config.MaxBytes)
if err != nil {
return postgresstore.ManualPublicationSeedResult{}, errors.New("inspect manual spool artifact")
}
mime, ok := mimeForFormat(config.Format)
if !ok {
return postgresstore.ManualPublicationSeedResult{}, errors.New("unsupported manual publication format")
}
result, err := store.SeedManualPublication(ctx, postgresstore.ManualPublicationSeedRequest{
TestKBID: config.TestKBID,
Title: config.Title,
SHA256: expectation.SHA256,
SizeBytes: expectation.SizeBytes,
MIME: mime,
ValidatedFormat: expectation.Format,
LocalSpoolRef: expectation.RelativeRef,
Now: config.Now,
})
if err != nil {
return postgresstore.ManualPublicationSeedResult{}, fmt.Errorf("seed manual publication: %w", err)
}
return result, nil
}
func mimeForFormat(format string) (string, bool) {
mimes := map[string]string{
"md": "text/markdown", "txt": "text/plain", "csv": "text/csv", "json": "application/json",
"pdf": "application/pdf", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"doc": "application/msword", "xls": "application/vnd.ms-excel", "ppt": "application/vnd.ms-powerpoint",
}
value, ok := mimes[format]
return value, ok
}
+58
View File
@@ -0,0 +1,58 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"testing"
"time"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func TestSeedFromSpoolPassesVerifiedArtifactProjection(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("# Governance fixture\n\nThe unique marker is MIKU-P3-5927.\n")
ref := filepath.Join("manual", "fixture.md")
if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil {
t.Fatal(err)
}
store := &manualSeederStub{result: postgresstore.ManualPublicationSeedResult{PublicationID: "tp-1", JobID: "job-1"}}
now := time.Date(2026, 9, 5, 6, 0, 0, 0, time.UTC)
result, err := seedFromSpool(context.Background(), store, seedConfig{
SpoolRoot: root,
SpoolRef: ref,
Format: "md",
Title: "Governance fixture",
TestKBID: "kb-1",
MaxBytes: 1024,
Now: now,
})
if err != nil {
t.Fatalf("seedFromSpool() error = %v", err)
}
digest := sha256.Sum256(body)
if result.PublicationID != "tp-1" || store.request.SHA256 != hex.EncodeToString(digest[:]) || store.request.SizeBytes != int64(len(body)) {
t.Fatalf("result/request = %#v %#v", result, store.request)
}
if store.request.LocalSpoolRef != ref || store.request.ValidatedFormat != "md" || store.request.MIME != "text/markdown" || store.request.Now != now {
t.Fatalf("request = %#v", store.request)
}
}
type manualSeederStub struct {
request postgresstore.ManualPublicationSeedRequest
result postgresstore.ManualPublicationSeedResult
err error
}
func (store *manualSeederStub) SeedManualPublication(_ context.Context, request postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error) {
store.request = request
return store.result, store.err
}
+174
View File
@@ -0,0 +1,174 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"io"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
type reportStore interface {
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error)
}
type reportClient interface {
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error)
}
type e2eReport struct {
PublicationID string `json:"publication_id"`
RemoteStatus string `json:"remote_status"`
ChunkTotal int64 `json:"chunk_total"`
ChunkItemsRead int64 `json:"chunk_items_read"`
AllChunksMatch bool `json:"all_chunks_match"`
SearchResultCount int `json:"search_result_count"`
TargetSearchHits int `json:"target_search_hits"`
MarkerFound bool `json:"marker_found"`
MaxScore float64 `json:"max_score"`
StructuralPassed bool `json:"structural_passed"`
SemanticState string `json:"semantic_state"`
}
func main() {
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
flags := flag.NewFlagSet("test-weknora-e2e-report", flag.ContinueOnError)
flags.SetOutput(io.Discard)
publicationID := flags.String("publication-id", "", "governance publication ID")
query := flags.String("query", "", "synthetic scoped retrieval query")
expectedMarker := flags.String("expected-marker", "", "synthetic marker expected in a scoped result")
if err := flags.Parse(args); err != nil {
return errors.New("invalid report arguments")
}
if *publicationID == "" || *query == "" || *expectedMarker == "" {
return errors.New("--publication-id, --query and --expected-marker are required")
}
databaseURL := getenv("DATABASE_URL")
baseURL := getenv("TEST_WEKNORA_BASE_URL")
keyFile := getenv("TEST_WEKNORA_API_KEY_FILE")
if databaseURL == "" || baseURL == "" || keyFile == "" {
return errors.New("database and test WeKnora configuration are required")
}
allowDockerHTTP := false
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
parsed, err := strconv.ParseBool(raw)
if err != nil {
return errors.New("invalid Docker HTTP setting")
}
allowDockerHTTP = parsed
}
apiKey, err := weknora.ReadAPIKeyFile(keyFile)
if err != nil {
return errors.New("read test WeKnora API key")
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: baseURL,
APIKey: apiKey,
Timeout: 60 * time.Second,
AllowDockerHTTP: allowDockerHTTP,
})
if err != nil {
return errors.New("create test WeKnora client")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return errors.New("open governance database")
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return errors.New("connect governance database")
}
report, err := verifyPublication(ctx, postgresstore.NewStore(pool), client, *publicationID, *query, *expectedMarker)
if err != nil {
return err
}
return json.NewEncoder(output).Encode(report)
}
func verifyPublication(ctx context.Context, store reportStore, client reportClient, publicationID, query, expectedMarker string) (e2eReport, error) {
publication, err := store.LoadPublicationForValidation(ctx, publicationID)
if err != nil {
return e2eReport{}, errors.New("load completed publication")
}
knowledge, err := client.GetKnowledge(ctx, publication.KnowledgeID)
if err != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
return e2eReport{}, errors.New("verify remote knowledge identity")
}
report := e2eReport{
PublicationID: publication.ID,
RemoteStatus: knowledge.ParseStatus,
AllChunksMatch: true,
}
for page := 1; page <= 100; page++ {
chunks, total, err := client.ListChunks(ctx, publication.KnowledgeID, page, 100, []string{"text", "image_ocr"})
if err != nil {
return e2eReport{}, errors.New("read remote chunks")
}
report.ChunkTotal = total
for _, chunk := range chunks {
report.ChunkItemsRead++
if chunk.KnowledgeID != publication.KnowledgeID {
report.AllChunksMatch = false
}
}
if report.ChunkItemsRead >= total || len(chunks) == 0 {
break
}
if page == 100 {
return e2eReport{}, errors.New("remote chunk pagination limit exceeded")
}
}
results, err := client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{publication.KnowledgeID}})
if err != nil {
return e2eReport{}, errors.New("run scoped retrieval probe")
}
report.SearchResultCount = len(results)
for _, result := range results {
if result.KnowledgeID == publication.KnowledgeID {
report.TargetSearchHits++
if strings.Contains(result.Content, expectedMarker) {
report.MarkerFound = true
}
}
if result.Score > report.MaxScore {
report.MaxScore = result.Score
}
}
acceptance, err := store.ListAcceptanceResults(ctx, publication.ID)
if err != nil {
return e2eReport{}, errors.New("read governance acceptance results")
}
structural := map[string]bool{"knowledge_exists": false, "chunks_present": false}
for _, record := range acceptance {
if _, ok := structural[record.CheckType]; ok && record.State == string(governance.AcceptancePassed) {
structural[record.CheckType] = true
}
if record.CheckType == "retrieval_hit" {
report.SemanticState = record.State
}
}
report.StructuralPassed = structural["knowledge_exists"] && structural["chunks_present"]
if report.RemoteStatus != "completed" || report.ChunkTotal <= 0 || report.ChunkItemsRead != report.ChunkTotal || !report.AllChunksMatch || report.TargetSearchHits == 0 || !report.MarkerFound {
return e2eReport{}, errors.New("end-to-end verification failed")
}
return report, nil
}
+87
View File
@@ -0,0 +1,87 @@
package main
import (
"context"
"encoding/json"
"testing"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
func TestVerifyPublicationReturnsSanitizedEvidence(t *testing.T) {
store := &reportStoreStub{
publication: postgresstore.PublicationForValidation{ID: "tp-1", TestKBID: "kb-1", KnowledgeID: "knowledge-secret-id"},
acceptance: []governance.AcceptanceRecord{
{CheckType: "knowledge_exists", State: string(governance.AcceptancePassed)},
{CheckType: "chunks_present", State: string(governance.AcceptancePassed)},
{CheckType: "retrieval_hit", State: string(governance.AcceptanceManualRequired)},
},
}
client := &reportClientStub{
knowledge: weknora.Knowledge{ID: "knowledge-secret-id", KnowledgeBaseID: "kb-1", ParseStatus: "completed"},
chunks: []weknora.Chunk{
{ID: "chunk-1", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
{ID: "chunk-2", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
},
search: []weknora.SearchResult{{KnowledgeID: "knowledge-secret-id", Content: "answer contains MIKU-GOV-P3-5927", Score: 0.02}},
}
report, err := verifyPublication(context.Background(), store, client, "tp-1", "unique marker", "MIKU-GOV-P3-5927")
if err != nil {
t.Fatalf("verifyPublication() error = %v", err)
}
if report.PublicationID != "tp-1" || report.RemoteStatus != "completed" || report.ChunkTotal != 2 || !report.AllChunksMatch || report.SearchResultCount != 1 || report.TargetSearchHits != 1 || !report.MarkerFound || !report.StructuralPassed || report.SemanticState != "manual_required" {
t.Fatalf("report = %#v", report)
}
encoded, err := json.Marshal(report)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"knowledge-secret-id", "chunk-1", "unique marker", "MIKU-GOV-P3-5927"} {
if string(encoded) == forbidden || contains(string(encoded), forbidden) {
t.Fatalf("report leaked %q: %s", forbidden, encoded)
}
}
}
func contains(value, fragment string) bool {
for index := 0; index+len(fragment) <= len(value); index++ {
if value[index:index+len(fragment)] == fragment {
return true
}
}
return false
}
type reportStoreStub struct {
publication postgresstore.PublicationForValidation
acceptance []governance.AcceptanceRecord
}
func (store *reportStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
return store.publication, nil
}
func (store *reportStoreStub) ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) {
return store.acceptance, nil
}
type reportClientStub struct {
knowledge weknora.Knowledge
chunks []weknora.Chunk
search []weknora.SearchResult
}
func (client *reportClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
return client.knowledge, nil
}
func (client *reportClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
return client.chunks, int64(len(client.chunks)), nil
}
func (client *reportClientStub) SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) {
return client.search, nil
}
+130
View File
@@ -0,0 +1,130 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"log"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
"knowledge-governance/internal/workflows/testweknora"
)
type probeConfig struct {
DatabaseURL string
Concurrency int
Online bool
BaseURL string
APIKeyFile string
ChunkSampleEvery int
}
func main() {
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
config, err := parseProbeConfig(args, getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return fmt.Errorf("open database: %w", err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return fmt.Errorf("ping database: %w", err)
}
store := postgresstore.NewStore(pool)
legacy, err := store.ListLegacyVerificationCandidates(ctx)
if err != nil {
return err
}
candidates := make([]testweknora.LegacyCandidate, 0, len(legacy))
for _, candidate := range legacy {
candidates = append(candidates, testweknora.LegacyCandidate{
ID: candidate.ID, TestKBID: candidate.TestKBID, KnowledgeID: candidate.KnowledgeID,
})
}
if !config.Online {
_, _ = io.WriteString(stdout, testweknora.FormatDryRunReport(candidates, config.Concurrency))
return nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.APIKeyFile)
if err != nil {
return fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.BaseURL,
APIKey: apiKey,
Timeout: 30 * time.Second,
})
if err != nil {
return fmt.Errorf("create test WeKnora client: %w", err)
}
report, err := testweknora.VerifyLegacyConcurrent(ctx, client, candidates, config.ChunkSampleEvery, config.Concurrency)
if err != nil {
return fmt.Errorf("verify historical test publications: %w", err)
}
_, _ = io.WriteString(stdout, testweknora.FormatLegacyVerificationReport(report))
return nil
}
func parseProbeConfig(args []string, getenv func(string) string) (probeConfig, error) {
flags := flag.NewFlagSet("test-weknora-probe", flag.ContinueOnError)
flags.SetOutput(io.Discard)
databaseURL := flags.String("database-url", "", "PostgreSQL connection URL (or DATABASE_URL)")
concurrency := flags.Int("concurrency", 4, "planned read-only verify concurrency for the online phase")
online := flags.Bool("online", false, "run authenticated read-only verification")
baseURL := flags.String("base-url", "", "HTTPS WeKnora base URL")
apiKeyFile := flags.String("api-key-file", "", "mode-0600 WeKnora API key file")
chunkSampleEvery := flags.Int("chunk-sample-every", 25, "check one completed Knowledge chunk count every N items; 0 disables")
if err := flags.Parse(args); err != nil {
return probeConfig{}, fmt.Errorf("parse arguments: %w", err)
}
if *databaseURL == "" {
*databaseURL = getenv("DATABASE_URL")
}
if *databaseURL == "" {
return probeConfig{}, fmt.Errorf("-database-url or DATABASE_URL is required")
}
if *concurrency < 1 || *concurrency > 16 || *chunkSampleEvery < 0 {
return probeConfig{}, fmt.Errorf("concurrency must be 1..16 and chunk-sample-every must be non-negative")
}
if *baseURL == "" {
*baseURL = getenv("TEST_WEKNORA_BASE_URL")
}
if *apiKeyFile == "" {
*apiKeyFile = getenv("TEST_WEKNORA_API_KEY_FILE")
}
if *online {
parsed, err := url.Parse(strings.TrimSpace(*baseURL))
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil {
return probeConfig{}, fmt.Errorf("online verification requires an HTTPS base-url")
}
if !filepath.IsAbs(*apiKeyFile) {
return probeConfig{}, fmt.Errorf("online verification requires an absolute api-key-file")
}
}
return probeConfig{
DatabaseURL: *databaseURL,
Concurrency: *concurrency,
Online: *online,
BaseURL: strings.TrimRight(strings.TrimSpace(*baseURL), "/"),
APIKeyFile: *apiKeyFile,
ChunkSampleEvery: *chunkSampleEvery,
}, nil
}
+37
View File
@@ -0,0 +1,37 @@
package main
import (
"context"
"io"
"strings"
"testing"
)
func TestRunRequiresDatabaseURL(t *testing.T) {
err := run(context.Background(), nil, func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "database-url") {
t.Fatalf("run() error = %v, want database-url requirement", err)
}
}
func TestParseProbeConfigSupportsExplicitOnlineMode(t *testing.T) {
config, err := parseProbeConfig([]string{
"--online",
"--base-url", "https://weknora-test.example",
"--api-key-file", "/run/secrets/test_weknora_api_key",
"--concurrency", "4",
"--chunk-sample-every", "25",
}, func(key string) string {
if key == "DATABASE_URL" {
return "postgres://governance@example/governance"
}
return ""
})
if err != nil {
t.Fatalf("parseProbeConfig() error = %v", err)
}
if !config.Online || config.DatabaseURL == "" || config.BaseURL != "https://weknora-test.example" ||
config.APIKeyFile != "/run/secrets/test_weknora_api_key" || config.Concurrency != 4 || config.ChunkSampleEvery != 25 {
t.Fatalf("config = %#v", config)
}
}
+156
View File
@@ -0,0 +1,156 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}
+92
View File
@@ -0,0 +1,92 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"knowledge-governance/internal/appconfig"
postgresstore "knowledge-governance/internal/platform/postgres"
)
func TestRunRequiresDatabaseURL(t *testing.T) {
err := run(context.Background(), func(string) string { return "" })
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
}
}
func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) {
handlers, err := workerHandlers(nil, appconfig.Config{})
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
if handlers["demo"] == nil || handlers["demo.sleep"] == nil {
t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers)
}
if handlers["release.probe"] != nil {
t.Fatal("ordinary worker must not register release.probe")
}
if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil {
t.Fatal("ordinary worker must not register test WeKnora jobs when disabled")
}
if handlers["source.scan.pp"] != nil {
t.Fatal("ordinary worker must not register PP scan jobs when disabled")
}
}
func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) {
root := t.TempDir()
config := appconfig.Config{
PPScanEnabled: true,
PPScanAllowedRoot: root,
PPScanLedgerPath: filepath.Join(root, "state.jsonl"),
PPScanManifestPath: filepath.Join(root, "manifest.json"),
}
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
if handlers["source.scan.pp"] == nil {
t.Fatal("source.scan.pp handler is not registered")
}
}
func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) {
config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"}
_, err := workerHandlers(nil, config)
if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") {
t.Fatalf("workerHandlers() error = %v", err)
}
}
func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) {
keyFile := filepath.Join(t.TempDir(), "api-key")
if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil {
t.Fatal(err)
}
config := appconfig.Config{
TestWeKnoraEnabled: true,
TestWeKnoraAllowDockerHTTP: true,
TestWeKnoraBaseURL: "http://app:8080",
TestWeKnoraAPIKeyFile: keyFile,
GovernanceSpoolRoot: t.TempDir(),
TestWeKnoraHTTPTimeout: time.Second,
TestWeKnoraPollInitial: time.Second,
TestWeKnoraStalledAfter: time.Minute,
TestWeKnoraMaxUploadBytes: 1024,
TestAcceptancePolicyVersion: "structural-v1",
}
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
if err != nil {
t.Fatalf("workerHandlers() error = %v", err)
}
for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} {
if handlers[jobType] == nil {
t.Fatalf("handler %q is not registered", jobType)
}
}
}
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"context"
"errors"
"log"
"time"
)
// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store
// satisfies it and tests can fake it without a database.
type dueSourceClaimer interface {
ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error)
}
// runSourceRefreshTick performs one dispatcher pass and logs only when work
// moved, so an idle system does not spam the worker log.
func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error {
claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now)
if err != nil {
return err
}
if claimed > 0 || enqueued > 0 {
log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued)
}
return nil
}
// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is
// deliberately delayed by one interval: startup already races ApplyMigrations
// and handler registration, and a due backlog enqueue adds nothing on the
// second the process comes up (the worker will drain whatever is queued anyway
// once sources were scheduled before a restart).
func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) {
if claimer == nil || interval <= 0 {
return
}
if onError == nil {
onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) }
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case tick := <-ticker.C:
if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) {
onError(err)
}
}
}
}
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"context"
"errors"
"sync"
"testing"
"time"
)
type fakeClaimer struct {
mu sync.Mutex
calls []time.Time
claimed [2]int
err error
}
func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, now)
if f.err != nil {
return 0, 0, f.err
}
return f.claimed[0], f.claimed[1], nil
}
func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) {
claim := &fakeClaimer{claimed: [2]int{3, 2}}
if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil {
t.Fatal(err)
}
if len(claim.calls) != 1 {
t.Fatalf("calls=%d want 1", len(claim.calls))
}
sentinel := errors.New("boom")
failing := &fakeClaimer{err: sentinel}
if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) {
t.Fatalf("error=%v want sentinel", err)
}
}
func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) {
claim := &fakeClaimer{}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil)
}()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
claim.mu.Lock()
count := len(claim.calls)
claim.mu.Unlock()
if count >= 2 {
break
}
time.Sleep(5 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scheduler did not stop on context cancel")
}
claim.mu.Lock()
defer claim.mu.Unlock()
if len(claim.calls) < 2 {
t.Fatalf("calls=%d want at least 2", len(claim.calls))
}
for _, stamp := range claim.calls {
if stamp.Location() != time.UTC {
t.Fatalf("tick stamp %v not UTC", stamp)
}
}
}
func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) {
claim := &fakeClaimer{err: errors.New("database is offline")}
var errorsSeen int
var mu sync.Mutex
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) {
mu.Lock()
errorsSeen++
mu.Unlock()
})
}()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
mu.Lock()
seen := errorsSeen
mu.Unlock()
if seen >= 2 {
break
}
time.Sleep(5 * time.Millisecond)
}
cancel()
<-done
mu.Lock()
defer mu.Unlock()
if errorsSeen < 2 {
t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen)
}
}
func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) {
// guards against misconfiguration killing the process
runSourceRefreshScheduler(context.Background(), nil, time.Second, nil)
runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil)
}