feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
+112
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/httpapi"
|
||||
"knowledge-governance/internal/monitoring"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
releaseAuthorizationToken := ""
|
||||
if config.ProductionReleaseRequestsEnabled {
|
||||
releaseAuthorizationToken, err = weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)
|
||||
if err != nil {
|
||||
return errors.New("read production release authorization token")
|
||||
}
|
||||
}
|
||||
// P10: with a mounted API key the probe verifies a real knowledge-base read
|
||||
// (readiness); without one it degrades to the liveness-only HTTP check.
|
||||
var weknoraReadiness func(context.Context) error
|
||||
if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" {
|
||||
if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil {
|
||||
probeTimeout := config.TestWeKnoraHTTPTimeout
|
||||
if probeTimeout > 10*time.Second || probeTimeout <= 0 {
|
||||
probeTimeout = 10 * time.Second
|
||||
}
|
||||
if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil {
|
||||
routingKB := config.InternalRoutingKnowledgeBaseID
|
||||
weknoraReadiness = func(probeCtx context.Context) error {
|
||||
_, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler, err := httpapi.NewHandler(httpapi.Config{
|
||||
Store: postgresstore.NewStore(pool),
|
||||
Monitor: monitorProvider,
|
||||
InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID,
|
||||
ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID,
|
||||
ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled,
|
||||
ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified,
|
||||
ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef,
|
||||
ProductionReleaseAuthorizationToken: releaseAuthorizationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
server := &http.Server{
|
||||
Addr: config.AppAddr,
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- server.ListenAndServe() }()
|
||||
log.Printf("governance API listening on %s", config.AppAddr)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return server.Shutdown(shutdownCtx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" })
|
||||
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIPropagatesProductionReleaseRequestGate(t *testing.T) {
|
||||
source, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalized := strings.Join(strings.Fields(string(source)), " ")
|
||||
for _, required := range []string{
|
||||
"if config.ProductionReleaseRequestsEnabled {",
|
||||
"weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)",
|
||||
"ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled",
|
||||
"ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified",
|
||||
"ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef",
|
||||
"ProductionReleaseAuthorizationToken: releaseAuthorizationToken",
|
||||
} {
|
||||
if !strings.Contains(normalized, required) {
|
||||
t.Fatalf("API production authorization wiring missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeDefaultsProductionBackupRestoreGateToFalse(t *testing.T) {
|
||||
source, err := os.ReadFile("../../compose.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(source), `PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false}`) {
|
||||
t.Fatal("compose must pass the production backup restore gate with a false default")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
|
||||
flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL")
|
||||
manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON")
|
||||
dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return fmt.Errorf("parse arguments: %w", err)
|
||||
}
|
||||
if *ledgerPath == "" || *manifestPath == "" {
|
||||
return fmt.Errorf("-ledger and -manifest are required")
|
||||
}
|
||||
|
||||
ledger, err := os.Open(*ledgerPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP ledger: %w", err)
|
||||
}
|
||||
snapshot, foldErr := ppconnector.FoldLedger(ledger)
|
||||
closeErr := ledger.Close()
|
||||
if foldErr != nil {
|
||||
return foldErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP ledger: %w", closeErr)
|
||||
}
|
||||
|
||||
manifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP download manifest: %w", err)
|
||||
}
|
||||
recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest)
|
||||
closeErr = manifest.Close()
|
||||
if recoveryErr != nil {
|
||||
return recoveryErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
overLimitManifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reopen PP download manifest: %w", err)
|
||||
}
|
||||
complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest)
|
||||
closeErr = overLimitManifest.Close()
|
||||
if overLimitErr != nil {
|
||||
return overLimitErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
if err := ppconnector.ValidateSnapshot(complete); err != nil {
|
||||
return fmt.Errorf("validate PP snapshot: %w", err)
|
||||
}
|
||||
plan, err := ppconnector.BuildImportPlan(complete)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build PP import plan: %w", err)
|
||||
}
|
||||
|
||||
completed := 0
|
||||
aliases := 0
|
||||
for _, item := range complete.Items {
|
||||
switch item.Status {
|
||||
case ppconnector.StatusCompleted:
|
||||
completed++
|
||||
case ppconnector.StatusContentAlias:
|
||||
aliases++
|
||||
}
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n",
|
||||
complete.EventCount,
|
||||
len(plan.SourceItems),
|
||||
completed,
|
||||
aliases,
|
||||
overLimit.Added,
|
||||
recovery.Recovered,
|
||||
recovery.VerifiedBytes,
|
||||
len(plan.ContentObjects),
|
||||
len(plan.Versions),
|
||||
len(plan.Publications),
|
||||
)
|
||||
if *dryRun {
|
||||
_, _ = fmt.Fprintln(stdout, "dry_run=true")
|
||||
return nil
|
||||
}
|
||||
|
||||
databaseURL := strings.TrimSpace(getenv("DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
return fmt.Errorf("DATABASE_URL is required unless -dry-run is used")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open governance database: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("ping governance database: %w", err)
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
report, err := postgresstore.ImportPPLegacy(ctx, pool, plan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n",
|
||||
report.Connections,
|
||||
report.SourceItems,
|
||||
report.ContentObjects,
|
||||
report.Versions,
|
||||
report.Publications,
|
||||
)
|
||||
current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciliation, err := ppconnector.PlanReconciliation(complete, current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n",
|
||||
reconciliation.Unchanged,
|
||||
reconciliation.Adopted,
|
||||
reconciliation.OverLimit,
|
||||
reconciliation.UploadRequired,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sourceHash := strings.Repeat("a", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatalf("write ledger fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatalf("write manifest fixture: %v", err)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} {
|
||||
if !strings.Contains(output, required) {
|
||||
t.Fatalf("run() output missing %q: %q", required, output)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} {
|
||||
if strings.Contains(output, forbidden) {
|
||||
t.Fatalf("run() output disclosed %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
text := output.String()
|
||||
if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") {
|
||||
t.Fatalf("dry-run output = %q", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("f", 64)
|
||||
ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string {
|
||||
if key == "DATABASE_URL" {
|
||||
return databaseURL
|
||||
}
|
||||
return ""
|
||||
}, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(output.String(), "planned_uploads=0\n") {
|
||||
t.Fatalf("run() output = %q", output.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
// Command release-diag replays the read-only steps of the release export path
|
||||
// (source PG snapshot, source ES snapshot, source object read) and additionally
|
||||
// lists candidate OSS key prefixes, to pinpoint a failing export step with its
|
||||
// real underlying error. Credentials are read from files only and never printed.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
type objectCfg struct {
|
||||
Provider string `json:"provider"`
|
||||
StorageBackendID string `json:"storage_backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyFile string `json:"access_key_file"`
|
||||
SecretKeyFile string `json:"secret_key_file"`
|
||||
}
|
||||
|
||||
type endpointCfg struct {
|
||||
DatabaseURLFile string `json:"database_url_file"`
|
||||
Elasticsearch struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Index string `json:"index"`
|
||||
} `json:"elasticsearch"`
|
||||
Object objectCfg `json:"object"`
|
||||
WeKnora struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
APIKeyFile string `json:"api_key_file"`
|
||||
} `json:"weknora"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Routes []struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EmbeddingModel string `json:"embedding_model"`
|
||||
Source endpointCfg `json:"source"`
|
||||
Target endpointCfg `json:"target"`
|
||||
} `json:"routes"`
|
||||
}
|
||||
|
||||
func secret(path string) string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Println("FATAL read secret:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "", "release config json")
|
||||
knowledgeID := flag.String("knowledge", "", "knowledge id to diagnose")
|
||||
flag.Parse()
|
||||
raw, err := os.ReadFile(*configPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var cfg config
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || len(cfg.Routes) != 1 {
|
||||
panic(err)
|
||||
}
|
||||
route := cfg.Routes[0]
|
||||
fmt.Println("route:", route.KnowledgeBaseID, "tenant:", route.TenantID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pool, err := pgxpool.New(ctx, secret(route.Source.DatabaseURLFile))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
fmt.Println("STEP source PG connect: FAIL", err)
|
||||
return
|
||||
}
|
||||
fmt.Println("STEP source PG connect: ok")
|
||||
|
||||
postgres, err := v071.NewPostgresExporter(pool)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
snapshot, err := postgres.Export(ctx, v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
|
||||
if err != nil {
|
||||
fmt.Println("STEP postgres export: FAIL", err)
|
||||
return
|
||||
}
|
||||
names := make([]string, 0, len(snapshot.Tables))
|
||||
for _, table := range snapshot.Tables {
|
||||
names = append(names, fmt.Sprintf("%s:%d", table.Name, table.RowCount))
|
||||
}
|
||||
fmt.Println("STEP postgres export: ok tables", names)
|
||||
|
||||
search, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, PageSize: 500})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
searchSnapshot, err := search.Export(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
|
||||
if err != nil {
|
||||
fmt.Println("STEP elasticsearch export: FAIL", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("STEP elasticsearch export: ok documents=%d\n", len(searchSnapshot.Documents))
|
||||
|
||||
store, err := newStore(ctx, route.Source.Object)
|
||||
if err != nil {
|
||||
fmt.Println("STEP object store build: FAIL", err)
|
||||
return
|
||||
}
|
||||
exporter, err := v071.NewObjectExporter(store, v071.ObjectExporterLimits{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
objectSnapshot, err := exporter.Export(ctx, snapshot)
|
||||
if err != nil {
|
||||
fmt.Println("STEP object export: FAIL", err)
|
||||
} else {
|
||||
fmt.Printf("STEP object export: ok objects=%d bytes=%d\n", len(objectSnapshot.Objects), objectSnapshot.TotalBytes)
|
||||
}
|
||||
|
||||
client, err := newRawClient(ctx, route.Source.Object)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, prefix := range []string{"weknora/", "oss://", ""} {
|
||||
listed, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(route.Source.Object.Bucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(3)})
|
||||
if err != nil {
|
||||
fmt.Printf("STEP list prefix %q: FAIL %v\n", prefix, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("STEP list prefix %q: %d objects\n", prefix, len(listed.Contents))
|
||||
for _, object := range listed.Contents {
|
||||
fmt.Println(" ", aws.ToString(object.Key))
|
||||
}
|
||||
}
|
||||
|
||||
knowledge := snapshot.Table("knowledges")
|
||||
for _, row := range knowledge.Rows {
|
||||
var typed struct {
|
||||
Source string `json:"source"`
|
||||
}
|
||||
_ = json.Unmarshal(row, &typed)
|
||||
fmt.Println("knowledge.source:", typed.Source)
|
||||
}
|
||||
for _, raw := range snapshot.Table("resources").Rows {
|
||||
var typed struct {
|
||||
PhysicalPath string `json:"physical_path"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &typed)
|
||||
fmt.Println("resource physical_path:", typed.PhysicalPath)
|
||||
}
|
||||
|
||||
fmt.Println("STEP target object stage probe:")
|
||||
targetStore, err := newStore(ctx, route.Target.Object)
|
||||
if err != nil {
|
||||
fmt.Println(" target store build FAIL", err)
|
||||
return
|
||||
}
|
||||
probePath := ".governance-release-diag/probe.txt"
|
||||
payload := []byte("release-sim-probe")
|
||||
if err := targetStore.Put(ctx, route.Target.Object.Provider, probePath, bytes.NewReader(payload), int64(len(payload))); err != nil {
|
||||
fmt.Println(" target Put FAIL:", err)
|
||||
} else {
|
||||
size, digest, err := targetStore.Stat(ctx, route.Target.Object.Provider, probePath)
|
||||
fmt.Println(" target Put ok; Stat:", size, digest[:8], "err:", err)
|
||||
}
|
||||
|
||||
fmt.Println("STEP real ObjectImporter Stage/Promote with exported snapshot:")
|
||||
importer, err := v071.NewObjectImporter(targetStore, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
fmt.Println(" importer build FAIL", err)
|
||||
return
|
||||
}
|
||||
stage, err := importer.Stage(ctx, "diag-stage", objectSnapshot)
|
||||
if err != nil {
|
||||
fmt.Println(" Stage FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Stage ok objects:", len(stage.Objects))
|
||||
if err := importer.Promote(ctx, stage); err != nil {
|
||||
fmt.Println(" Promote FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Promote ok")
|
||||
|
||||
fmt.Println("STEP target Postgres replace:")
|
||||
targetPool, err := pgxpool.New(ctx, secret(route.Target.DatabaseURLFile))
|
||||
if err != nil {
|
||||
fmt.Println(" target pool FAIL", err)
|
||||
return
|
||||
}
|
||||
defer targetPool.Close()
|
||||
if err := targetPool.Ping(ctx); err != nil {
|
||||
fmt.Println(" target PG connect FAIL", err)
|
||||
return
|
||||
}
|
||||
pgImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
fmt.Println(" importer config FAIL", err)
|
||||
return
|
||||
}
|
||||
identity := v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}
|
||||
if err := pgImporter.Replace(ctx, identity, snapshot); err != nil {
|
||||
fmt.Println(" Replace FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Replace ok")
|
||||
|
||||
fmt.Println("STEP target ES replace:")
|
||||
targetSearch, err := v071.NewSearchImporter(v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, PageSize: 500})
|
||||
if err != nil {
|
||||
fmt.Println(" importer build FAIL", err)
|
||||
return
|
||||
}
|
||||
if _, err := targetSearch.Replace(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}, searchSnapshot); err != nil {
|
||||
fmt.Println(" ES Replace FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" ES Replace ok")
|
||||
|
||||
fmt.Println("STEP target WeKnora verification replay:")
|
||||
wknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: secret(route.Target.WeKnora.APIKeyFile), Timeout: 30 * time.Second, AllowDockerHTTP: true})
|
||||
if err != nil {
|
||||
fmt.Println(" client build FAIL", err)
|
||||
return
|
||||
}
|
||||
knowledgeRemote, err := wknoraClient.GetKnowledge(ctx, *knowledgeID)
|
||||
fmt.Println(" GetKnowledge err:", err, "id:", knowledgeRemote.ID, "kb:", knowledgeRemote.KnowledgeBaseID, "parse:", knowledgeRemote.ParseStatus)
|
||||
chunksRemote, total, err := wknoraClient.ListChunks(ctx, *knowledgeID, 1, 5, nil)
|
||||
fmt.Println(" ListChunks err:", err, "total:", total, "page:", len(chunksRemote))
|
||||
probe := ""
|
||||
for _, document := range searchSnapshot.Documents {
|
||||
var fields struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
_ = json.Unmarshal(document.Source, &fields)
|
||||
if utf8.RuneCountInString(fields.Content) >= 12 {
|
||||
probe = string([]rune(fields.Content)[:min(48, utf8.RuneCountInString(fields.Content))])
|
||||
break
|
||||
}
|
||||
}
|
||||
results, err := wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: probe, KnowledgeIDs: []string{*knowledgeID}})
|
||||
fmt.Println(" SearchKnowledge err:", err, "probe_len:", len(probe), "results:", len(results))
|
||||
for index, result := range results {
|
||||
if index >= 3 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" hit knowledge=%s score=%.3f type=%s\n", result.KnowledgeID, result.Score, result.ChunkType)
|
||||
}
|
||||
if len(results) == 0 && probe != "" {
|
||||
results, err = wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: "CRA 620mm", KnowledgeIDs: []string{*knowledgeID}})
|
||||
fmt.Println(" short-probe SearchKnowledge err:", err, "results:", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func newStore(ctx context.Context, object objectCfg) (*v071.S3ObjectStore, error) {
|
||||
parsed, err := url.Parse(object.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{
|
||||
Provider: object.Provider, Endpoint: object.Endpoint, Region: object.Region,
|
||||
AccessKeyID: secret(object.AccessKeyFile), SecretAccessKey: secret(object.SecretKeyFile),
|
||||
Bucket: object.Bucket, Secure: parsed.Scheme == "https",
|
||||
})
|
||||
}
|
||||
|
||||
func newRawClient(ctx context.Context, object objectCfg) (*s3.Client, error) {
|
||||
parsed, err := url.Parse(object.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(object.Region), awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(secret(object.AccessKeyFile), secret(object.SecretKeyFile), "")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(object.Endpoint)
|
||||
options.UsePathStyle = false
|
||||
_ = parsed
|
||||
}), nil
|
||||
}
|
||||
|
||||
var _ = io.Discard
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const maxReleaseConfigBytes = 1024 * 1024
|
||||
|
||||
type releaseRuntimeConfig struct {
|
||||
ArchiveRoot string `json:"archive_root"`
|
||||
Routes []releaseRouteConfig `json:"routes"`
|
||||
}
|
||||
|
||||
type releaseRouteConfig struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EmbeddingModel string `json:"embedding_model"`
|
||||
Source releaseEndpointConfig `json:"source"`
|
||||
Target releaseEndpointConfig `json:"target"`
|
||||
}
|
||||
|
||||
type releaseEndpointConfig struct {
|
||||
DatabaseURLFile string `json:"database_url_file"`
|
||||
Elasticsearch releaseElasticsearchConfig `json:"elasticsearch"`
|
||||
Object releaseObjectConfig `json:"object"`
|
||||
WeKnora releaseWeKnoraConfig `json:"weknora"`
|
||||
}
|
||||
|
||||
type releaseElasticsearchConfig struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Index string `json:"index"`
|
||||
TokenFile string `json:"token_file"`
|
||||
}
|
||||
|
||||
type releaseObjectConfig struct {
|
||||
Provider string `json:"provider"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
StorageBackendID string `json:"storage_backend_id"`
|
||||
AccessKeyFile string `json:"access_key_file"`
|
||||
SecretKeyFile string `json:"secret_key_file"`
|
||||
}
|
||||
|
||||
type releaseWeKnoraConfig struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
APIKeyFile string `json:"api_key_file"`
|
||||
AllowDockerHTTP bool `json:"allow_docker_http"`
|
||||
}
|
||||
|
||||
func loadReleaseRuntimeConfig(path string) (releaseRuntimeConfig, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return releaseRuntimeConfig{}, errors.New("release config path must be absolute")
|
||||
}
|
||||
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return releaseRuntimeConfig{}, errors.New("open release config")
|
||||
}
|
||||
file := os.NewFile(uintptr(fd), "release-config")
|
||||
if file == nil {
|
||||
syscall.Close(fd)
|
||||
return releaseRuntimeConfig{}, errors.New("open release config")
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() <= 0 || info.Size() > maxReleaseConfigBytes {
|
||||
return releaseRuntimeConfig{}, errors.New("release config must be a non-empty regular 0600 file")
|
||||
}
|
||||
payload, err := io.ReadAll(io.LimitReader(file, maxReleaseConfigBytes+1))
|
||||
if err != nil || len(payload) > maxReleaseConfigBytes {
|
||||
return releaseRuntimeConfig{}, errors.New("read release config")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
var config releaseRuntimeConfig
|
||||
if err := decoder.Decode(&config); err != nil {
|
||||
return releaseRuntimeConfig{}, errors.New("decode release config")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return releaseRuntimeConfig{}, errors.New("release config contains trailing data")
|
||||
}
|
||||
if err := validateReleaseRuntimeConfig(config); err != nil {
|
||||
return releaseRuntimeConfig{}, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error {
|
||||
if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 {
|
||||
return errors.New("release archive root and routes are required")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, route := range config.Routes {
|
||||
if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] {
|
||||
return errors.New("invalid or duplicate release route")
|
||||
}
|
||||
seen[route.KnowledgeBaseID] = true
|
||||
if !validReleaseEndpoint(route.Source, false) || !validReleaseEndpoint(route.Target, true) {
|
||||
return errors.New("invalid release route endpoint")
|
||||
}
|
||||
if route.Source.DatabaseURLFile == route.Target.DatabaseURLFile ||
|
||||
route.Source.Elasticsearch.BaseURL == route.Target.Elasticsearch.BaseURL ||
|
||||
(route.Source.Object.Endpoint == route.Target.Object.Endpoint && route.Source.Object.Bucket == route.Target.Object.Bucket) {
|
||||
return errors.New("release source and target must be distinct")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validReleaseEndpoint(endpoint releaseEndpointConfig, target bool) bool {
|
||||
if !absoluteFile(endpoint.DatabaseURLFile) || !validHTTPURL(endpoint.Elasticsearch.BaseURL) || !validConfigIdentifier(endpoint.Elasticsearch.Index) ||
|
||||
(endpoint.Elasticsearch.TokenFile != "" && !absoluteFile(endpoint.Elasticsearch.TokenFile)) ||
|
||||
!validObjectEndpoint(endpoint.Object) {
|
||||
return false
|
||||
}
|
||||
if target {
|
||||
return validConfigIdentifier(endpoint.Object.StorageBackendID) && validHTTPURL(endpoint.WeKnora.BaseURL) && absoluteFile(endpoint.WeKnora.APIKeyFile)
|
||||
}
|
||||
return endpoint.WeKnora.BaseURL == "" && endpoint.WeKnora.APIKeyFile == ""
|
||||
}
|
||||
|
||||
func validObjectEndpoint(object releaseObjectConfig) bool {
|
||||
return (object.Provider == "minio" || object.Provider == "s3" || object.Provider == "oss") && validHTTPURL(object.Endpoint) &&
|
||||
validConfigIdentifier(object.Bucket) && absoluteFile(object.AccessKeyFile) && absoluteFile(object.SecretKeyFile)
|
||||
}
|
||||
|
||||
func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path }
|
||||
|
||||
func validHTTPURL(value string) bool {
|
||||
parsed, err := url.Parse(value)
|
||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
|
||||
}
|
||||
|
||||
func validConfigIdentifier(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadReleaseRuntimeConfigValidatesRestrictedMultiKBConfig(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "release.json")
|
||||
payload := validReleaseConfigJSON(root)
|
||||
if err := os.WriteFile(path, []byte(payload), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := loadReleaseRuntimeConfig(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Routes) != 1 || config.Routes[0].KnowledgeBaseID != "kb-safe" || config.Routes[0].Target.Object.StorageBackendID != "target-backend" {
|
||||
t.Fatalf("config=%#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReleaseRuntimeConfigRejectsUnsafeFilesAndUnknownFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
payload := validReleaseConfigJSON(root)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
setup func(string) error
|
||||
}{
|
||||
{name: "overbroad", setup: func(path string) error { return os.WriteFile(path, []byte(payload), 0o644) }},
|
||||
{name: "unknown field", setup: func(path string) error {
|
||||
return os.WriteFile(path, []byte(strings.Replace(payload, `"archive_root":`, `"unknown":true,"archive_root":`, 1)), 0o600)
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||
if err := test.setup(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := loadReleaseRuntimeConfig(path); err == nil {
|
||||
t.Fatal("unsafe config accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
real := filepath.Join(root, "real.json")
|
||||
_ = os.WriteFile(real, []byte(payload), 0o600)
|
||||
link := filepath.Join(root, "link.json")
|
||||
_ = os.Symlink(real, link)
|
||||
if _, err := loadReleaseRuntimeConfig(link); err == nil {
|
||||
t.Fatal("symlink accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func validReleaseConfigJSON(root string) string {
|
||||
return `{
|
||||
"archive_root":"` + filepath.ToSlash(filepath.Join(root, "archive")) + `",
|
||||
"routes":[{
|
||||
"tenant_id":10000,"knowledge_base_id":"kb-safe","embedding_model":"bge-m3",
|
||||
"source":{"database_url_file":"/run/secrets/source-db","elasticsearch":{"base_url":"http://source-es:9200","index":"chunks","token_file":"/run/secrets/source-es"},"object":{"provider":"minio","endpoint":"http://source-minio:9000","region":"us-east-1","bucket":"source","access_key_file":"/run/secrets/source-ak","secret_key_file":"/run/secrets/source-sk"}},
|
||||
"target":{"database_url_file":"/run/secrets/target-db","elasticsearch":{"base_url":"https://target-es.example","index":"chunks","token_file":"/run/secrets/target-es"},"object":{"provider":"oss","endpoint":"https://oss.example","region":"cn-shenzhen","bucket":"target","storage_backend_id":"target-backend","access_key_file":"/run/secrets/target-ak","secret_key_file":"/run/secrets/target-sk"},"weknora":{"base_url":"https://weknora.example","api_key_file":"/run/secrets/target-weknora"}}
|
||||
}]}`
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !config.ReleaseWorkerEnabled {
|
||||
log.Print("release worker disabled; no production credentials are loaded")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
runtimeConfig, err := loadReleaseRuntimeConfig(config.ReleaseConfigFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
releaseRuntime, err := buildReleaseRuntime(ctx, runtimeConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer releaseRuntime.Close()
|
||||
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
releaseHandler, err := releaseworkflow.NewHandler(store, releaseRuntime.publisher, time.Now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
LeaseDuration: config.LeaseDuration,
|
||||
RetryDelay: config.RetryDelay,
|
||||
Handlers: map[string]jobs.Handler{
|
||||
"release.publish": releaseHandler,
|
||||
"release.rollback": releaseHandler,
|
||||
},
|
||||
OnError: func(err error) { log.Printf("release worker error: %v", err) },
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("release worker %s started with %d release route(s)", config.WorkerID, len(runtimeConfig.Routes))
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunReturnsWithoutDatabaseWhenReleaseWorkerDisabled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://must-not-be-contacted.invalid/governance",
|
||||
"RELEASE_WORKER_ENABLED": "false",
|
||||
}
|
||||
if err := run(ctx, func(key string) string { return values[key] }); err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnabledReleaseWorkerLoadsRestrictedConfigBeforeDatabaseConnections(t *testing.T) {
|
||||
values := map[string]string{"DATABASE_URL": "not-a-database-url", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/does/not/exist/release.json"}
|
||||
err := run(context.Background(), func(key string) string { return values[key] })
|
||||
if err == nil || !strings.Contains(err.Error(), "release config") || strings.Contains(err.Error(), "database") {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseWorkerRegistersOnlyPublishAndRollbackHandlers(t *testing.T) {
|
||||
source, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(source)
|
||||
for _, required := range []string{`"release.publish"`, `"release.rollback"`} {
|
||||
if !strings.Contains(text, required) {
|
||||
t.Fatalf("missing handler %s", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `"release.probe"`) {
|
||||
t.Fatal("probe-only release handler remains")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
type releaseRuntime struct {
|
||||
publisher *releaseworkflow.RoutingPublisher
|
||||
pools []*pgxpool.Pool
|
||||
}
|
||||
|
||||
func (runtime *releaseRuntime) Close() {
|
||||
for index := len(runtime.pools) - 1; index >= 0; index-- {
|
||||
runtime.pools[index].Close()
|
||||
}
|
||||
}
|
||||
|
||||
func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) {
|
||||
runtime := &releaseRuntime{}
|
||||
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
|
||||
for _, route := range config.Routes {
|
||||
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, err
|
||||
}
|
||||
runtime.pools = append(runtime.pools, pools...)
|
||||
publishers[route.KnowledgeBaseID] = publisher
|
||||
}
|
||||
router, err := releaseworkflow.NewRoutingPublisher(publishers)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, errors.New("create release publisher routes")
|
||||
}
|
||||
runtime.publisher = router
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read source database credential")
|
||||
}
|
||||
targetDatabaseURL, err := readRestrictedValue(route.Target.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read target database credential")
|
||||
}
|
||||
if err := validateDistinctDatabaseURLs(sourceDatabaseURL, targetDatabaseURL); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sourcePool, err := openReleasePool(ctx, sourceDatabaseURL)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("connect release source database")
|
||||
}
|
||||
pools := []*pgxpool.Pool{sourcePool}
|
||||
targetPool, err := openReleasePool(ctx, targetDatabaseURL)
|
||||
if err != nil {
|
||||
sourcePool.Close()
|
||||
return nil, nil, errors.New("connect release target database")
|
||||
}
|
||||
pools = append(pools, targetPool)
|
||||
cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
for index := len(pools) - 1; index >= 0; index-- {
|
||||
pools[index].Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
sourceSearchToken, err := readOptionalRestrictedValue(route.Source.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read source Elasticsearch credential"))
|
||||
}
|
||||
targetSearchToken, err := readOptionalRestrictedValue(route.Target.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target Elasticsearch credential"))
|
||||
}
|
||||
sourceSearch, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, AuthorizationToken: sourceSearchToken, PageSize: 500})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source Elasticsearch exporter"))
|
||||
}
|
||||
targetSearchConfig := v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, AuthorizationToken: targetSearchToken, PageSize: 500}
|
||||
targetSearchInspector, err := v071.NewSearchExporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch inspector"))
|
||||
}
|
||||
targetSearch, err := v071.NewSearchImporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch importer"))
|
||||
}
|
||||
|
||||
sourceObject, err := buildS3ObjectStore(route.Source.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object reader"))
|
||||
}
|
||||
targetObject, err := buildS3ObjectStore(route.Target.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object store"))
|
||||
}
|
||||
objectExporter, err := v071.NewObjectExporter(sourceObject, v071.ObjectExporterLimits{})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object exporter"))
|
||||
}
|
||||
objectImporter, err := v071.NewObjectImporter(targetObject, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object importer"))
|
||||
}
|
||||
|
||||
sourceContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: sourcePool, Mapping: sourceSearch, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Source.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source release contract inspector"))
|
||||
}
|
||||
targetContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: targetPool, Mapping: targetSearchInspector, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Target.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target release contract inspector"))
|
||||
}
|
||||
postgresExporter, err := v071.NewPostgresExporter(sourcePool)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source PostgreSQL exporter"))
|
||||
}
|
||||
postgresImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target PostgreSQL importer"))
|
||||
}
|
||||
documentExporter, err := v071.NewDocumentExporter(v071.DocumentExporterConfig{Contract: sourceContract, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: route.TenantID, EmbeddingModel: route.EmbeddingModel, Now: time.Now})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document exporter"))
|
||||
}
|
||||
|
||||
archive, err := v071.NewBundleArchive(filepath.Join(archiveRoot, route.KnowledgeBaseID), 1024*1024*1024)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("open release bundle archive"))
|
||||
}
|
||||
apiKey, err := readRestrictedValue(route.Target.WeKnora.APIKeyFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target WeKnora credential"))
|
||||
}
|
||||
weknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: apiKey, Timeout: 60 * time.Second, AllowDockerHTTP: route.Target.WeKnora.AllowDockerHTTP})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verification client"))
|
||||
}
|
||||
verifier, err := v071.NewWeKnoraVerifier(weknoraClient)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verifier"))
|
||||
}
|
||||
documentTarget, err := v071.NewDocumentTarget(v071.DocumentTargetConfig{Archive: archive, Contract: targetContract, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document target"))
|
||||
}
|
||||
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 release publisher"))
|
||||
}
|
||||
return publisher, pools, nil
|
||||
}
|
||||
|
||||
func buildS3ObjectStore(config releaseObjectConfig) (*v071.S3ObjectStore, error) {
|
||||
accessKey, err := readRestrictedValue(config.AccessKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secretKey, err := readRestrictedValue(config.SecretKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsed, err := url.Parse(config.Endpoint)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid object endpoint")
|
||||
}
|
||||
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: config.Provider, Endpoint: config.Endpoint, Region: config.Region, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: config.Bucket, Secure: parsed.Scheme == "https", ForcePathStyle: config.Provider == "minio"})
|
||||
}
|
||||
|
||||
func readRestrictedValue(path string) (string, error) {
|
||||
return weknora.ReadAPIKeyFile(path)
|
||||
}
|
||||
|
||||
func readOptionalRestrictedValue(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
return readRestrictedValue(path)
|
||||
}
|
||||
|
||||
func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func validateDistinctDatabaseURLs(sourceURL, targetURL string) error {
|
||||
source, err := pgxpool.ParseConfig(sourceURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid source database credential")
|
||||
}
|
||||
target, err := pgxpool.ParseConfig(targetURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid target database credential")
|
||||
}
|
||||
sourceIdentity := strings.ToLower(source.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(source.ConnConfig.Port), 10) + "|" + source.ConnConfig.Database
|
||||
targetIdentity := strings.ToLower(target.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(target.ConnConfig.Port), 10) + "|" + target.ConnConfig.Database
|
||||
if sourceIdentity == targetIdentity {
|
||||
return errors.New("release source and target database must be distinct")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReleaseRuntimeRejectsSameSourceAndTargetDatabaseValues(t *testing.T) {
|
||||
if err := validateDistinctDatabaseURLs("postgres://user@same/db?sslmode=require", "postgres://user@same/db?sslmode=require"); err == nil {
|
||||
t.Fatal("same database URL accepted")
|
||||
}
|
||||
if err := validateDistinctDatabaseURLs("postgres://user@source/db?sslmode=require", "postgres://user@target/db?sslmode=require"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReleaseRuntimeWithRestrictedFilesIntegration(t *testing.T) {
|
||||
sourceDatabaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL")
|
||||
targetDatabaseURL := os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL")
|
||||
minioEndpoint := os.Getenv("TEST_MINIO_ENDPOINT")
|
||||
if sourceDatabaseURL == "" || targetDatabaseURL == "" || minioEndpoint == "" {
|
||||
t.Skip("release runtime integration environment is not set")
|
||||
}
|
||||
esSource := httptest.NewServer(http.NotFoundHandler())
|
||||
defer esSource.Close()
|
||||
esTarget := httptest.NewServer(http.NotFoundHandler())
|
||||
defer esTarget.Close()
|
||||
weknoraTarget := httptest.NewServer(http.NotFoundHandler())
|
||||
defer weknoraTarget.Close()
|
||||
root := t.TempDir()
|
||||
archiveRoot := filepath.Join(root, "archive")
|
||||
archiveRoute := filepath.Join(archiveRoot, "kb-release-safe")
|
||||
if err := os.MkdirAll(archiveRoute, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := func(name, value string) string {
|
||||
path := filepath.Join(root, name)
|
||||
if err := os.WriteFile(path, []byte(value+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
route := releaseRouteConfig{
|
||||
TenantID: 10000, KnowledgeBaseID: "kb-release-safe", EmbeddingModel: "bge-m3",
|
||||
Source: releaseEndpointConfig{DatabaseURLFile: secret("source-db", sourceDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esSource.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-source", AccessKeyFile: secret("source-ak", "release-test"), SecretKeyFile: secret("source-sk", "release-test-secret")}},
|
||||
Target: releaseEndpointConfig{DatabaseURLFile: secret("target-db", targetDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esTarget.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-target", StorageBackendID: "target-backend", AccessKeyFile: secret("target-ak", "release-test"), SecretKeyFile: secret("target-sk", "release-test-secret")}, WeKnora: releaseWeKnoraConfig{BaseURL: weknoraTarget.URL, APIKeyFile: secret("target-weknora", "test-api-key"), AllowDockerHTTP: true}},
|
||||
}
|
||||
runtime, err := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{ArchiveRoot: archiveRoot, Routes: []releaseRouteConfig{route}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.Close()
|
||||
if runtime.publisher == nil || len(runtime.pools) != 2 {
|
||||
t.Fatalf("runtime=%#v", runtime)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/spool"
|
||||
)
|
||||
|
||||
type manualSeeder interface {
|
||||
SeedManualPublication(context.Context, postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error)
|
||||
}
|
||||
|
||||
type seedConfig struct {
|
||||
SpoolRoot string
|
||||
SpoolRef string
|
||||
Format string
|
||||
Title string
|
||||
TestKBID string
|
||||
MaxBytes int64
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("test-publication-seed", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
spoolRef := flags.String("spool-ref", "", "opaque relative spool reference")
|
||||
format := flags.String("format", "md", "validated artifact format")
|
||||
title := flags.String("title", "", "safe display title")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return errors.New("invalid seed arguments")
|
||||
}
|
||||
databaseURL := getenv("DATABASE_URL")
|
||||
spoolRoot := getenv("GOVERNANCE_SPOOL_ROOT")
|
||||
if spoolRoot == "" {
|
||||
spoolRoot = "/var/lib/governance/spool"
|
||||
}
|
||||
testKBID := getenv("TEST_WEKNORA_KB_ID")
|
||||
maxBytes := int64(50 * 1024 * 1024)
|
||||
if raw := getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"); raw != "" {
|
||||
parsed, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil || parsed <= 0 || parsed > 50*1024*1024 {
|
||||
return errors.New("invalid TEST_WEKNORA_MAX_UPLOAD_BYTES")
|
||||
}
|
||||
maxBytes = parsed
|
||||
}
|
||||
if databaseURL == "" || testKBID == "" || *spoolRef == "" || *title == "" {
|
||||
return errors.New("DATABASE_URL, TEST_WEKNORA_KB_ID, --spool-ref and --title are required")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return errors.New("open governance database")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return errors.New("connect governance database")
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return errors.New("apply governance migrations")
|
||||
}
|
||||
result, err := seedFromSpool(ctx, postgresstore.NewStore(pool), seedConfig{
|
||||
SpoolRoot: spoolRoot,
|
||||
SpoolRef: *spoolRef,
|
||||
Format: *format,
|
||||
Title: *title,
|
||||
TestKBID: testKBID,
|
||||
MaxBytes: maxBytes,
|
||||
Now: time.Now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(output).Encode(result)
|
||||
}
|
||||
|
||||
func seedFromSpool(ctx context.Context, store manualSeeder, config seedConfig) (postgresstore.ManualPublicationSeedResult, error) {
|
||||
if store == nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("manual publication store is required")
|
||||
}
|
||||
expectation, err := spool.InspectVerified(config.SpoolRoot, config.SpoolRef, config.Format, config.MaxBytes)
|
||||
if err != nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("inspect manual spool artifact")
|
||||
}
|
||||
mime, ok := mimeForFormat(config.Format)
|
||||
if !ok {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("unsupported manual publication format")
|
||||
}
|
||||
result, err := store.SeedManualPublication(ctx, postgresstore.ManualPublicationSeedRequest{
|
||||
TestKBID: config.TestKBID,
|
||||
Title: config.Title,
|
||||
SHA256: expectation.SHA256,
|
||||
SizeBytes: expectation.SizeBytes,
|
||||
MIME: mime,
|
||||
ValidatedFormat: expectation.Format,
|
||||
LocalSpoolRef: expectation.RelativeRef,
|
||||
Now: config.Now,
|
||||
})
|
||||
if err != nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, fmt.Errorf("seed manual publication: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func mimeForFormat(format string) (string, bool) {
|
||||
mimes := map[string]string{
|
||||
"md": "text/markdown", "txt": "text/plain", "csv": "text/csv", "json": "application/json",
|
||||
"pdf": "application/pdf", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"doc": "application/msword", "xls": "application/vnd.ms-excel", "ppt": "application/vnd.ms-powerpoint",
|
||||
}
|
||||
value, ok := mimes[format]
|
||||
return value, ok
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func TestSeedFromSpoolPassesVerifiedArtifactProjection(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("# Governance fixture\n\nThe unique marker is MIKU-P3-5927.\n")
|
||||
ref := filepath.Join("manual", "fixture.md")
|
||||
if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := &manualSeederStub{result: postgresstore.ManualPublicationSeedResult{PublicationID: "tp-1", JobID: "job-1"}}
|
||||
now := time.Date(2026, 9, 5, 6, 0, 0, 0, time.UTC)
|
||||
|
||||
result, err := seedFromSpool(context.Background(), store, seedConfig{
|
||||
SpoolRoot: root,
|
||||
SpoolRef: ref,
|
||||
Format: "md",
|
||||
Title: "Governance fixture",
|
||||
TestKBID: "kb-1",
|
||||
MaxBytes: 1024,
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seedFromSpool() error = %v", err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
if result.PublicationID != "tp-1" || store.request.SHA256 != hex.EncodeToString(digest[:]) || store.request.SizeBytes != int64(len(body)) {
|
||||
t.Fatalf("result/request = %#v %#v", result, store.request)
|
||||
}
|
||||
if store.request.LocalSpoolRef != ref || store.request.ValidatedFormat != "md" || store.request.MIME != "text/markdown" || store.request.Now != now {
|
||||
t.Fatalf("request = %#v", store.request)
|
||||
}
|
||||
}
|
||||
|
||||
type manualSeederStub struct {
|
||||
request postgresstore.ManualPublicationSeedRequest
|
||||
result postgresstore.ManualPublicationSeedResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (store *manualSeederStub) SeedManualPublication(_ context.Context, request postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error) {
|
||||
store.request = request
|
||||
return store.result, store.err
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type reportStore interface {
|
||||
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
|
||||
ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error)
|
||||
}
|
||||
|
||||
type reportClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error)
|
||||
}
|
||||
|
||||
type e2eReport struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
RemoteStatus string `json:"remote_status"`
|
||||
ChunkTotal int64 `json:"chunk_total"`
|
||||
ChunkItemsRead int64 `json:"chunk_items_read"`
|
||||
AllChunksMatch bool `json:"all_chunks_match"`
|
||||
SearchResultCount int `json:"search_result_count"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
MarkerFound bool `json:"marker_found"`
|
||||
MaxScore float64 `json:"max_score"`
|
||||
StructuralPassed bool `json:"structural_passed"`
|
||||
SemanticState string `json:"semantic_state"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("test-weknora-e2e-report", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
publicationID := flags.String("publication-id", "", "governance publication ID")
|
||||
query := flags.String("query", "", "synthetic scoped retrieval query")
|
||||
expectedMarker := flags.String("expected-marker", "", "synthetic marker expected in a scoped result")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return errors.New("invalid report arguments")
|
||||
}
|
||||
if *publicationID == "" || *query == "" || *expectedMarker == "" {
|
||||
return errors.New("--publication-id, --query and --expected-marker are required")
|
||||
}
|
||||
databaseURL := getenv("DATABASE_URL")
|
||||
baseURL := getenv("TEST_WEKNORA_BASE_URL")
|
||||
keyFile := getenv("TEST_WEKNORA_API_KEY_FILE")
|
||||
if databaseURL == "" || baseURL == "" || keyFile == "" {
|
||||
return errors.New("database and test WeKnora configuration are required")
|
||||
}
|
||||
allowDockerHTTP := false
|
||||
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return errors.New("invalid Docker HTTP setting")
|
||||
}
|
||||
allowDockerHTTP = parsed
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(keyFile)
|
||||
if err != nil {
|
||||
return errors.New("read test WeKnora API key")
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: 60 * time.Second,
|
||||
AllowDockerHTTP: allowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("create test WeKnora client")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return errors.New("open governance database")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return errors.New("connect governance database")
|
||||
}
|
||||
report, err := verifyPublication(ctx, postgresstore.NewStore(pool), client, *publicationID, *query, *expectedMarker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(output).Encode(report)
|
||||
}
|
||||
|
||||
func verifyPublication(ctx context.Context, store reportStore, client reportClient, publicationID, query, expectedMarker string) (e2eReport, error) {
|
||||
publication, err := store.LoadPublicationForValidation(ctx, publicationID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("load completed publication")
|
||||
}
|
||||
knowledge, err := client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
if err != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
return e2eReport{}, errors.New("verify remote knowledge identity")
|
||||
}
|
||||
report := e2eReport{
|
||||
PublicationID: publication.ID,
|
||||
RemoteStatus: knowledge.ParseStatus,
|
||||
AllChunksMatch: true,
|
||||
}
|
||||
for page := 1; page <= 100; page++ {
|
||||
chunks, total, err := client.ListChunks(ctx, publication.KnowledgeID, page, 100, []string{"text", "image_ocr"})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read remote chunks")
|
||||
}
|
||||
report.ChunkTotal = total
|
||||
for _, chunk := range chunks {
|
||||
report.ChunkItemsRead++
|
||||
if chunk.KnowledgeID != publication.KnowledgeID {
|
||||
report.AllChunksMatch = false
|
||||
}
|
||||
}
|
||||
if report.ChunkItemsRead >= total || len(chunks) == 0 {
|
||||
break
|
||||
}
|
||||
if page == 100 {
|
||||
return e2eReport{}, errors.New("remote chunk pagination limit exceeded")
|
||||
}
|
||||
}
|
||||
results, err := client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{publication.KnowledgeID}})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("run scoped retrieval probe")
|
||||
}
|
||||
report.SearchResultCount = len(results)
|
||||
for _, result := range results {
|
||||
if result.KnowledgeID == publication.KnowledgeID {
|
||||
report.TargetSearchHits++
|
||||
if strings.Contains(result.Content, expectedMarker) {
|
||||
report.MarkerFound = true
|
||||
}
|
||||
}
|
||||
if result.Score > report.MaxScore {
|
||||
report.MaxScore = result.Score
|
||||
}
|
||||
}
|
||||
acceptance, err := store.ListAcceptanceResults(ctx, publication.ID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read governance acceptance results")
|
||||
}
|
||||
structural := map[string]bool{"knowledge_exists": false, "chunks_present": false}
|
||||
for _, record := range acceptance {
|
||||
if _, ok := structural[record.CheckType]; ok && record.State == string(governance.AcceptancePassed) {
|
||||
structural[record.CheckType] = true
|
||||
}
|
||||
if record.CheckType == "retrieval_hit" {
|
||||
report.SemanticState = record.State
|
||||
}
|
||||
}
|
||||
report.StructuralPassed = structural["knowledge_exists"] && structural["chunks_present"]
|
||||
if report.RemoteStatus != "completed" || report.ChunkTotal <= 0 || report.ChunkItemsRead != report.ChunkTotal || !report.AllChunksMatch || report.TargetSearchHits == 0 || !report.MarkerFound {
|
||||
return e2eReport{}, errors.New("end-to-end verification failed")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestVerifyPublicationReturnsSanitizedEvidence(t *testing.T) {
|
||||
store := &reportStoreStub{
|
||||
publication: postgresstore.PublicationForValidation{ID: "tp-1", TestKBID: "kb-1", KnowledgeID: "knowledge-secret-id"},
|
||||
acceptance: []governance.AcceptanceRecord{
|
||||
{CheckType: "knowledge_exists", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "chunks_present", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "retrieval_hit", State: string(governance.AcceptanceManualRequired)},
|
||||
},
|
||||
}
|
||||
client := &reportClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-secret-id", KnowledgeBaseID: "kb-1", ParseStatus: "completed"},
|
||||
chunks: []weknora.Chunk{
|
||||
{ID: "chunk-1", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
{ID: "chunk-2", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
},
|
||||
search: []weknora.SearchResult{{KnowledgeID: "knowledge-secret-id", Content: "answer contains MIKU-GOV-P3-5927", Score: 0.02}},
|
||||
}
|
||||
|
||||
report, err := verifyPublication(context.Background(), store, client, "tp-1", "unique marker", "MIKU-GOV-P3-5927")
|
||||
if err != nil {
|
||||
t.Fatalf("verifyPublication() error = %v", err)
|
||||
}
|
||||
if report.PublicationID != "tp-1" || report.RemoteStatus != "completed" || report.ChunkTotal != 2 || !report.AllChunksMatch || report.SearchResultCount != 1 || report.TargetSearchHits != 1 || !report.MarkerFound || !report.StructuralPassed || report.SemanticState != "manual_required" {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
encoded, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge-secret-id", "chunk-1", "unique marker", "MIKU-GOV-P3-5927"} {
|
||||
if string(encoded) == forbidden || contains(string(encoded), forbidden) {
|
||||
t.Fatalf("report leaked %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(value, fragment string) bool {
|
||||
for index := 0; index+len(fragment) <= len(value); index++ {
|
||||
if value[index:index+len(fragment)] == fragment {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type reportStoreStub struct {
|
||||
publication postgresstore.PublicationForValidation
|
||||
acceptance []governance.AcceptanceRecord
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) {
|
||||
return store.acceptance, nil
|
||||
}
|
||||
|
||||
type reportClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
chunks []weknora.Chunk
|
||||
search []weknora.SearchResult
|
||||
}
|
||||
|
||||
func (client *reportClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
return client.chunks, int64(len(client.chunks)), nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) {
|
||||
return client.search, nil
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
)
|
||||
|
||||
type probeConfig struct {
|
||||
DatabaseURL string
|
||||
Concurrency int
|
||||
Online bool
|
||||
BaseURL string
|
||||
APIKeyFile string
|
||||
ChunkSampleEvery int
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
|
||||
config, err := parseProbeConfig(args, getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("ping database: %w", err)
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
legacy, err := store.ListLegacyVerificationCandidates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates := make([]testweknora.LegacyCandidate, 0, len(legacy))
|
||||
for _, candidate := range legacy {
|
||||
candidates = append(candidates, testweknora.LegacyCandidate{
|
||||
ID: candidate.ID, TestKBID: candidate.TestKBID, KnowledgeID: candidate.KnowledgeID,
|
||||
})
|
||||
}
|
||||
if !config.Online {
|
||||
_, _ = io.WriteString(stdout, testweknora.FormatDryRunReport(candidates, config.Concurrency))
|
||||
return nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.APIKeyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.BaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create test WeKnora client: %w", err)
|
||||
}
|
||||
report, err := testweknora.VerifyLegacyConcurrent(ctx, client, candidates, config.ChunkSampleEvery, config.Concurrency)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify historical test publications: %w", err)
|
||||
}
|
||||
_, _ = io.WriteString(stdout, testweknora.FormatLegacyVerificationReport(report))
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseProbeConfig(args []string, getenv func(string) string) (probeConfig, error) {
|
||||
flags := flag.NewFlagSet("test-weknora-probe", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
databaseURL := flags.String("database-url", "", "PostgreSQL connection URL (or DATABASE_URL)")
|
||||
concurrency := flags.Int("concurrency", 4, "planned read-only verify concurrency for the online phase")
|
||||
online := flags.Bool("online", false, "run authenticated read-only verification")
|
||||
baseURL := flags.String("base-url", "", "HTTPS WeKnora base URL")
|
||||
apiKeyFile := flags.String("api-key-file", "", "mode-0600 WeKnora API key file")
|
||||
chunkSampleEvery := flags.Int("chunk-sample-every", 25, "check one completed Knowledge chunk count every N items; 0 disables")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return probeConfig{}, fmt.Errorf("parse arguments: %w", err)
|
||||
}
|
||||
if *databaseURL == "" {
|
||||
*databaseURL = getenv("DATABASE_URL")
|
||||
}
|
||||
if *databaseURL == "" {
|
||||
return probeConfig{}, fmt.Errorf("-database-url or DATABASE_URL is required")
|
||||
}
|
||||
if *concurrency < 1 || *concurrency > 16 || *chunkSampleEvery < 0 {
|
||||
return probeConfig{}, fmt.Errorf("concurrency must be 1..16 and chunk-sample-every must be non-negative")
|
||||
}
|
||||
if *baseURL == "" {
|
||||
*baseURL = getenv("TEST_WEKNORA_BASE_URL")
|
||||
}
|
||||
if *apiKeyFile == "" {
|
||||
*apiKeyFile = getenv("TEST_WEKNORA_API_KEY_FILE")
|
||||
}
|
||||
if *online {
|
||||
parsed, err := url.Parse(strings.TrimSpace(*baseURL))
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil {
|
||||
return probeConfig{}, fmt.Errorf("online verification requires an HTTPS base-url")
|
||||
}
|
||||
if !filepath.IsAbs(*apiKeyFile) {
|
||||
return probeConfig{}, fmt.Errorf("online verification requires an absolute api-key-file")
|
||||
}
|
||||
}
|
||||
return probeConfig{
|
||||
DatabaseURL: *databaseURL,
|
||||
Concurrency: *concurrency,
|
||||
Online: *online,
|
||||
BaseURL: strings.TrimRight(strings.TrimSpace(*baseURL), "/"),
|
||||
APIKeyFile: *apiKeyFile,
|
||||
ChunkSampleEvery: *chunkSampleEvery,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), nil, func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "database-url") {
|
||||
t.Fatalf("run() error = %v, want database-url requirement", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProbeConfigSupportsExplicitOnlineMode(t *testing.T) {
|
||||
config, err := parseProbeConfig([]string{
|
||||
"--online",
|
||||
"--base-url", "https://weknora-test.example",
|
||||
"--api-key-file", "/run/secrets/test_weknora_api_key",
|
||||
"--concurrency", "4",
|
||||
"--chunk-sample-every", "25",
|
||||
}, func(key string) string {
|
||||
if key == "DATABASE_URL" {
|
||||
return "postgres://governance@example/governance"
|
||||
}
|
||||
return ""
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("parseProbeConfig() error = %v", err)
|
||||
}
|
||||
if !config.Online || config.DatabaseURL == "" || config.BaseURL != "https://weknora-test.example" ||
|
||||
config.APIKeyFile != "/run/secrets/test_weknora_api_key" || config.Concurrency != 4 || config.ChunkSampleEvery != 25 {
|
||||
t.Fatalf("config = %#v", config)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
"knowledge-governance/internal/workflows/sourcescan"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
handlers, err := workerHandlers(store, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
LeaseDuration: config.LeaseDuration,
|
||||
RetryDelay: config.RetryDelay,
|
||||
Handlers: handlers,
|
||||
OnError: func(err error) { log.Printf("worker error: %v", err) },
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
|
||||
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
|
||||
})
|
||||
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
|
||||
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
|
||||
handlers := map[string]jobs.Handler{
|
||||
"demo": func(_ context.Context, job jobs.Job) error {
|
||||
log.Printf("completed demo job id=%s", job.ID)
|
||||
return nil
|
||||
},
|
||||
"demo.sleep": jobs.DemoSleepHandler,
|
||||
}
|
||||
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
|
||||
}
|
||||
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
|
||||
}
|
||||
handlers["source.catalog.refresh"] = catalogHandler.Handle
|
||||
if config.PPScanEnabled {
|
||||
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
|
||||
AllowedRoot: config.PPScanAllowedRoot,
|
||||
LedgerPath: config.PPScanLedgerPath,
|
||||
ManifestPath: config.PPScanManifestPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scanner: %w", err)
|
||||
}
|
||||
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scan handler: %w", err)
|
||||
}
|
||||
handlers["source.scan.pp"] = handler.Handle
|
||||
}
|
||||
if !config.TestWeKnoraEnabled {
|
||||
return handlers, nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.TestWeKnoraBaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: config.TestWeKnoraHTTPTimeout,
|
||||
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test WeKnora client: %w", err)
|
||||
}
|
||||
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
SpoolRoot: config.GovernanceSpoolRoot,
|
||||
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
|
||||
FirstPollDelay: config.TestWeKnoraPollInitial,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test upload handler: %w", err)
|
||||
}
|
||||
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PollDelay: config.TestWeKnoraPollInitial,
|
||||
StallAfter: config.TestWeKnoraStalledAfter,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test poll handler: %w", err)
|
||||
}
|
||||
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PolicyVersion: config.TestAcceptancePolicyVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test validate handler: %w", err)
|
||||
}
|
||||
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test removal handler: %w", err)
|
||||
}
|
||||
handlers["test.upload"] = upload.Handle
|
||||
handlers["test.poll"] = poll.Handle
|
||||
handlers["test.validate"] = validate.Handle
|
||||
handlers["test.remove"] = remove.Handle
|
||||
return handlers, nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" })
|
||||
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) {
|
||||
handlers, err := workerHandlers(nil, appconfig.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["demo"] == nil || handlers["demo.sleep"] == nil {
|
||||
t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers)
|
||||
}
|
||||
if handlers["release.probe"] != nil {
|
||||
t.Fatal("ordinary worker must not register release.probe")
|
||||
}
|
||||
if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil {
|
||||
t.Fatal("ordinary worker must not register test WeKnora jobs when disabled")
|
||||
}
|
||||
if handlers["source.scan.pp"] != nil {
|
||||
t.Fatal("ordinary worker must not register PP scan jobs when disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
config := appconfig.Config{
|
||||
PPScanEnabled: true,
|
||||
PPScanAllowedRoot: root,
|
||||
PPScanLedgerPath: filepath.Join(root, "state.jsonl"),
|
||||
PPScanManifestPath: filepath.Join(root, "manifest.json"),
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["source.scan.pp"] == nil {
|
||||
t.Fatal("source.scan.pp handler is not registered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) {
|
||||
config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"}
|
||||
_, err := workerHandlers(nil, config)
|
||||
if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) {
|
||||
keyFile := filepath.Join(t.TempDir(), "api-key")
|
||||
if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := appconfig.Config{
|
||||
TestWeKnoraEnabled: true,
|
||||
TestWeKnoraAllowDockerHTTP: true,
|
||||
TestWeKnoraBaseURL: "http://app:8080",
|
||||
TestWeKnoraAPIKeyFile: keyFile,
|
||||
GovernanceSpoolRoot: t.TempDir(),
|
||||
TestWeKnoraHTTPTimeout: time.Second,
|
||||
TestWeKnoraPollInitial: time.Second,
|
||||
TestWeKnoraStalledAfter: time.Minute,
|
||||
TestWeKnoraMaxUploadBytes: 1024,
|
||||
TestAcceptancePolicyVersion: "structural-v1",
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} {
|
||||
if handlers[jobType] == nil {
|
||||
t.Fatalf("handler %q is not registered", jobType)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store
|
||||
// satisfies it and tests can fake it without a database.
|
||||
type dueSourceClaimer interface {
|
||||
ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error)
|
||||
}
|
||||
|
||||
// runSourceRefreshTick performs one dispatcher pass and logs only when work
|
||||
// moved, so an idle system does not spam the worker log.
|
||||
func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error {
|
||||
claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if claimed > 0 || enqueued > 0 {
|
||||
log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is
|
||||
// deliberately delayed by one interval: startup already races ApplyMigrations
|
||||
// and handler registration, and a due backlog enqueue adds nothing on the
|
||||
// second the process comes up (the worker will drain whatever is queued anyway
|
||||
// once sources were scheduled before a restart).
|
||||
func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) {
|
||||
if claimer == nil || interval <= 0 {
|
||||
return
|
||||
}
|
||||
if onError == nil {
|
||||
onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) }
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case tick := <-ticker.C:
|
||||
if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeClaimer struct {
|
||||
mu sync.Mutex
|
||||
calls []time.Time
|
||||
claimed [2]int
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, now)
|
||||
if f.err != nil {
|
||||
return 0, 0, f.err
|
||||
}
|
||||
return f.claimed[0], f.claimed[1], nil
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{claimed: [2]int{3, 2}}
|
||||
if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claim.calls) != 1 {
|
||||
t.Fatalf("calls=%d want 1", len(claim.calls))
|
||||
}
|
||||
sentinel := errors.New("boom")
|
||||
failing := &fakeClaimer{err: sentinel}
|
||||
if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) {
|
||||
t.Fatalf("error=%v want sentinel", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) {
|
||||
claim := &fakeClaimer{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
claim.mu.Lock()
|
||||
count := len(claim.calls)
|
||||
claim.mu.Unlock()
|
||||
if count >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scheduler did not stop on context cancel")
|
||||
}
|
||||
claim.mu.Lock()
|
||||
defer claim.mu.Unlock()
|
||||
if len(claim.calls) < 2 {
|
||||
t.Fatalf("calls=%d want at least 2", len(claim.calls))
|
||||
}
|
||||
for _, stamp := range claim.calls {
|
||||
if stamp.Location() != time.UTC {
|
||||
t.Fatalf("tick stamp %v not UTC", stamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{err: errors.New("database is offline")}
|
||||
var errorsSeen int
|
||||
var mu sync.Mutex
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) {
|
||||
mu.Lock()
|
||||
errorsSeen++
|
||||
mu.Unlock()
|
||||
})
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
seen := errorsSeen
|
||||
mu.Unlock()
|
||||
if seen >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if errorsSeen < 2 {
|
||||
t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) {
|
||||
// guards against misconfiguration killing the process
|
||||
runSourceRefreshScheduler(context.Background(), nil, time.Second, nil)
|
||||
runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil)
|
||||
}
|
||||
Reference in New Issue
Block a user