Files
knowledge-governance/cmd/worker/main.go
T
姚章浩 6400fd1bbe feat(library): 快照搬运先于维护、双向校验与云端版本回传
为什么改:云端 release-executor 与目标同机后,本机封存的快照必须"先到、先验证"
才允许开维护窗口。此前只有 pack/unpack 两条人工命令:UI 点发布时执行器读到的是
本机归档路径,云端根本没有这份快照,作业只能在装配阶段失败;而预备份落在云端,
本机页面也读不到。用户已批准补齐这条链路。

搬运(新包 internal/libraryhandoff):本机 Load 逐 blob 重算 → Pack → SSH 流式写入
云端暂存 → 原子改名 → 云端 snapshot-unpack 再逐 blob 校验 → 读回版本清单摘要自证 →
scope/tenant/知识库身份与治理库作业一致。只有全部通过才投递 library.sync|restore,
失败时发布作业根本不存在(目标不进维护、不写知识)。远端命令是固定 argv,容器名与
路径只来自部署配置;同 ID 异内容拒绝;同一快照本机 flock 串行化;暂存 bundle 用后即删,
归档本体永不自动清理。

回传:capture 角色按已成功作业的 snapshot_id 扫描,把本机缺失的云端快照 Fetch 回来
(云端 pack → SSH 流回 → 本机 Unpack 逐 blob 校验)。回传成功后 API/页面既有的列表、
详情、差异、存储与历史恢复路径读到的就是真实字节;失败则版本不出现在本机,不会被误报成可恢复。

角色:搬运与回传只注册在 capture;若接线到 full(同时消费发布作业)启动即失败,
避免与云端执行器抢同一批作业。release-executor 仍不拥有治理库 DDL,也不消费
QA/邮件/来源扫描。

验证:internal/libraryhandoff 全绿(跨进程 push/fetch、身份不符拒绝、幂等复用、
并发串行、篡改与未知快照不落版本、路径穿越拒绝、远端错误不当作"没有");
cmd/worker 在隔离 fixture 库(独立测试 PG)上验证"校验通过才入队发布、失败不入队";
httpapi 队列决策单测;相关包 go test 全绿;go vet 全仓通过。

未部署、未推送、未构建镜像:云端接线与验收仍由父执行。真实云链路(含 SSH 通道)
本轮没有云侧写操作,只有本机跨进程与隔离 fixture 证据。
2026-10-30 18:58:34 +08:00

519 lines
24 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryagent"
"knowledge-governance/internal/libraryhandoff"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
"knowledge-governance/internal/versionhistory"
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
// libraryStageRecorder 把执行阶段写进持久作业,页面刷新/worker 重启后仍可复现。
// libraryStageStore 是阶段/证据记录用到的存储面:由 libraryAttemptWriter 实现,
// 所以阶段进度与版本登记天然带执行代次(失租代次写不进去)。
type libraryStageStore interface {
UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind, sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error)
RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error
}
type libraryStageRecorder struct {
store libraryStageStore
jobID string
scope string
// operationID 让版本事件 ID 带执行意图:不同 operation 引用同一快照是两条事件,
// 不会互相覆盖证据(F02)。为空表示历史/测试路径,退回旧 ID 形状。
operationID string
totals map[string]stageSize
// mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。
mappingDigest string
// onBackup 在预备份完成后、写目标之前调用:把"预备份身份 + 内容风险"落盘。
onBackup func(context.Context, libraryrelease.BackupRef) error
}
func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) {
// 真实总量来自快照 manifest:已知就写真实数字,未知就保持 NULL(不写 0、不编百分比)。
var total, completed, bytesTotal, bytesDone *int64
if size, ok := r.totals[stage]; ok {
total, bytesTotal = int64Ptr(size.count), int64Ptr(size.bytes)
if state == "succeeded" {
completed, bytesDone = total, bytesTotal
}
}
if err := r.store.UpdateLibraryStage(ctx, r.jobID, stage, state, total, completed, bytesTotal, bytesDone,
detail, time.Now().UTC()); err != nil {
log.Printf("library stage %s/%s update failed: %v", r.jobID, stage, err)
}
}
func int64Ptr(value int64) *int64 { return &value }
// stageSize 是某一类数据在快照里的真实总量(行数/字节)。
type stageSize struct {
count int64
bytes int64
}
// stageSizesFrom 从 manifest 汇总四类进度的真实总量:
// database/resources/index/wiki 各自的行数与字节,全部来自归档部件清单。
func stageSizesFrom(manifest librarysnapshot.Manifest) map[string]stageSize {
sizes := map[string]stageSize{}
kindFor := map[string]string{"database": "database", "resource": "resources", "index": "index", "wiki": "wiki"}
for _, part := range manifest.Parts {
stage, ok := kindFor[part.Kind]
if !ok {
continue
}
current := sizes[stage]
current.count += part.Count
current.bytes += part.Bytes
sizes[stage] = current
}
return sizes
}
// libraryCaptureJobType is shared with the API which enqueues it.
const libraryCaptureJobType = "library.capture"
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型;与 API 侧常量同值。
const libraryRecoverJobType = "library.recover"
// libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。
const libraryRestoreJobType = "library.restore"
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
const libraryBaselineJobType = "library.baseline"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
// 迁移也是"后台副作用":只有控制面那个进程拥有 DDL。云端发布执行器只读校验
// schema 版本,落后即拒绝启动——它不该成为治理库的第二个结构决策者。
if config.LibraryWorkerRole.ExecutesMigrations() {
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
} else if pending, pendingErr := postgresstore.PendingMigrations(ctx, pool); pendingErr != nil {
return fmt.Errorf("治理库 schema 版本检查失败: %w", pendingErr)
} else if len(pending) > 0 {
return fmt.Errorf("治理库 schema 落后(待应用迁移 %v):本角色不拥有 DDL,先由控制面进程完成迁移", pending)
}
store := postgresstore.NewStore(pool)
// 启动先接管被中断的写作业(复用已持久化的预备份),再把剩下的孤儿判失败:
// 顺序反了会把可接管的作业先标失败,丢掉已完成的备份。
//
// 只由"发布执行器"角色接管:release 作业的目标来自本进程配置,capture 角色的
// 进程按本机维护配置去接管云端目标的作业,等于把快照写到另一套目标上。
if config.LibraryWorkerRole.ExecutesRelease() {
if resumed, failed, resumeErr := resumeInterruptedLibraryJobs(context.Background(), store, config); resumeErr != nil {
log.Printf("library job resume failed: %v", resumeErr)
} else if resumed > 0 || failed > 0 {
log.Printf("library job resume: %d resumed, %d failed without backup", resumed, failed)
}
// 运行期也要持续接管:只在启动扫一次会漏掉"worker 在两次重启之间被杀"的情况。
go func() {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
resumed, failed, err := resumeInterruptedLibraryJobs(ctx, store, config)
if err != nil {
log.Printf("library job takeover sweep: %v", err)
} else if resumed > 0 || failed > 0 {
log.Printf("library job takeover sweep: %d resumed, %d failed without backup", resumed, failed)
}
}
}
}()
}
// 只收拾本角色负责的 kind:跨域"顺手收拾"会把另一台执行器的在飞作业标失败。
if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC(),
config.LibraryWorkerRole.LibraryJobKinds()); reconcileErr != nil {
log.Printf("library job reconcile failed: %v", reconcileErr)
} else if repaired > 0 {
log.Printf("library job reconcile: %d orphan job(s) marked failed", repaired)
}
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
if config.LibraryWorkerRole.ExecutesSourceWork() {
// 快照搬运(本机 → 云端执行器)与云端版本回传都只有 capture 角色做:
// 它持有源归档与 SSH 凭据;release-executor 在云端没有回本机的通道。
if handoffConfig, configured, handoffErr := libraryhandoff.FromEnv(getenv); handoffErr != nil {
return fmt.Errorf("快照搬运接线无效: %w", handoffErr)
} else if configured {
if config.LibraryWorkerRole.ExecutesRelease() {
return errors.New("快照搬运只能接在 capture 角色上:full 角色同时消费发布作业,会与云端执行器抢同一批作业")
}
handoff, buildErr := libraryhandoff.New(handoffConfig, nil)
if buildErr != nil {
return buildErr
}
registerLibraryHandoffHandlers(handlers, store, config, handoff)
go runLibraryArchivePullLoop(ctx, store, handoff, config.LibraryArchiveRoot, handoffConfig.PullInterval,
func(err error) { log.Printf("library archive pull: %v", err) })
log.Printf("library handoff enabled: executor=%s archive=%s pull_every=%s",
handoffConfig.ExecutorContainer, handoffConfig.RemoteArchive, handoffConfig.PullInterval)
}
if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil {
log.Print("QA directory synchronization unavailable; handler was not registered")
}
mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers)
if mirrorErr != nil {
log.Print("QA local mirror configuration invalid; handler was not registered")
}
if mirrorStore != nil {
go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") })
}
}
// 生产同步/恢复的执行器不依赖主机侧封存 agent:没有 agent socket 也要能跑。
//
// 执行域准入(R4)在这里统一判定:目标实况、旧 release 路由绑定、未释放维护窗口
// 任何一项未知或重叠 ⇒ 这些 handler 直接拒绝执行(明确失败,无任何维护动作,
// 也不停止任何既有服务)。
//
// 只有"发布执行器"角色注册它们:capture 角色的进程按本机目标配置执行一条
// 指向别处目标的作业,后果是跨目标写入,不是失败重试能纠正的事。
if config.LibraryWorkerRole.ExecutesRelease() {
admissionErr := verifyLibraryExecutionDomain(context.Background(), config, store)
if admissionErr != nil {
log.Printf("library execution domain NOT admitted: %v", admissionErr)
}
registerLibraryReleaseHandlers(handlers, store, config, admissionErr)
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
if config.LibraryWorkerRole.ExecutesSourceWork() {
if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil {
go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") })
}
go weeklyMailLoop(ctx, pool, getenv)
// 封存执行器状态:worker 是唯一能连 agent socket 的一方,它写观测、页面读观测。
go runLibraryExecutorProbe(ctx, store, config, libraryExecutorProbeInterval)
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
}
log.Printf("worker %s started role=%s release=%t source=%t (source refresh scheduler every %s)",
config.WorkerID, config.LibraryWorkerRole, config.LibraryWorkerRole.ExecutesRelease(),
config.LibraryWorkerRole.ExecutesSourceWork(), config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
// 目录刷新与附件抓取属于"源侧"作业:它们读本机 spool、调本机脚本。
// 发布执行器角色在云端没有这些挂载,注册它们只会抢走本机队列里的作业。
if config.LibraryWorkerRole.ExecutesSourceWork() {
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
ingestionRoot := config.IngestionSpoolRoot
if ingestionRoot == "" {
ingestionRoot = config.GovernanceSpoolRoot
}
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
ingestionRoot = "/var/lib/governance/ingestion"
}
fetchMaxBytes := config.ArtifactFetchMaxBytes
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
}
if fetchMaxBytes <= 0 {
fetchMaxBytes = 50 * 1024 * 1024
}
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
ingestionRoot, "ingestion", fetchMaxBytes)
if err != nil {
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
}
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
if err != nil {
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
}
handlers["artifact.fetch"] = artifactHandler.Handle
}
if config.LibraryAgentSocket != "" && config.LibraryWorkerRole.ExecutesSourceWork() {
// 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧
// agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。
agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute}
handlers[libraryCaptureJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
Scope string `json:"scope"`
CaptureJob string `json:"capture_job"`
RequestedBy string `json:"requested_by"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.CaptureJob == "" {
return fmt.Errorf("invalid library capture payload")
}
// 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。
libraryJobID := "libseal-" + payload.CaptureJob
startedAt := time.Now().UTC()
if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil {
log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr)
}
if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil,
"主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil {
log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr)
}
// 平台审计过的精确死信解除指纹:读取失败必须阻断(fail closed),
// 不能因为存储不可用就静默按"没有解除"或"全部解除"处理。
released, releaseErr := store.DeadLetterReleaseFingerprints(ctx, config.SourceInstance())
if releaseErr != nil {
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, releaseErr.Error(), time.Now().UTC())
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", "dead letter release storage unavailable", "", "", "", time.Now().UTC())
return fmt.Errorf("load dead letter releases: %w", releaseErr)
}
// 期望执行器身份随请求一起送进 agent,由接收方在操作互斥内、任何源操作之前
// 比对:这样即使 socket 在这之前被换掉,错的执行器也一次 run 都不会跑。
expectation, expectErr := captureExpectation(config)
if expectErr != nil {
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, expectErr.Error(), time.Now().UTC())
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", expectErr.Error(),
"未停任何写入者;补齐来源身份配置后重试", "", "", time.Now().UTC())
return expectErr
}
result, err := libraryagent.CaptureReleased(ctx, agentConfig, payload.Scope, payload.CaptureJob, released, expectation)
finishedAt := time.Now().UTC()
if err != nil {
// 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, err.Error(), finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
return err
}
// 执行后复核:结果里 agent 自报的身份必须还是期望实例,否则这次封存
// 可能来自别的栈,绝不能登记成"本实例的测试封存"。
if identityErr := captureExecutorPostcheck(result.Payload, expectation); identityErr != nil {
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, identityErr.Error(), finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", identityErr.Error(),
"快照已产生但来源身份不符,未登记为测试封存;核对执行器后重新封存", "", "", finishedAt)
return identityErr
}
// 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionTestSealed, payload.Scope, result.SnapshotID),
Scope: payload.Scope,
Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID,
Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay,
"requested_by": payload.RequestedBy, "expectation_fingerprint": expectationFingerprint(expectation),
"agent_result": json.RawMessage(result.Payload)},
CreatedBy: payload.RequestedBy, CreatedAt: finishedAt,
})
if versionErr != nil {
log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr)
}
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil,
"已封存快照 "+result.SnapshotID, finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "",
result.SnapshotID, "", finishedAt)
log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s",
payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy)
return nil
}
// A crash can leave the source writer stopped: recover before taking work,
// and keep the failure visible instead of pretending the worker is ready.
go func() {
recoveryCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if _, err := libraryagent.Recover(recoveryCtx, agentConfig); err != nil {
log.Printf("library agent recovery check failed: %v", err)
return
}
log.Printf("library agent reachable; freeze journals recovered")
}()
}
if config.VersionHistoryGiteaBaseURL != "" && config.LibraryWorkerRole.ExecutesSourceWork() {
// 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册,
// 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。
client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{
BaseURL: config.VersionHistoryGiteaBaseURL,
AllowedBaseURLs: config.VersionHistoryAllowedOrigins,
LinkOrigin: config.VersionHistoryLinkOrigin,
Repo: config.VersionHistoryRepo,
Timeout: config.VersionHistoryHTTPTimeout,
})
if err != nil {
return nil, fmt.Errorf("create version history client: %w", err)
}
syncer, err := versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store,
RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages})
if err != nil {
return nil, fmt.Errorf("create version history syncer: %w", err)
}
handlers[versionhistory.JobType] = func(ctx context.Context, job jobs.Job) error {
triggeredBy := ""
var payload struct {
TriggeredBy string `json:"triggered_by"`
TargetSHA string `json:"target_sha"`
}
if err := json.Unmarshal(job.Payload, &payload); err == nil {
triggeredBy = payload.TriggeredBy
}
activeSyncer := syncer
if payload.TargetSHA != "" {
var e error
activeSyncer, e = versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages, TargetSHAs: []string{payload.TargetSHA}})
if e != nil {
return e
}
}
run, err := activeSyncer.Sync(ctx, job.ID, triggeredBy)
if err != nil {
log.Printf("version history sync failed job=%s run=%d code=%s: %v", job.ID, run.ID, run.ErrorCode, err)
return err
}
log.Printf("version history sync completed job=%s run=%d fetched=%d expected=%v complete=%v",
job.ID, run.ID, run.FetchedCount, run.ExpectedCount, run.Complete)
return nil
}
}
if config.PPScanEnabled && config.LibraryWorkerRole.ExecutesSourceWork() {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled || !config.LibraryWorkerRole.ExecutesSourceWork() {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
policy, err := governance.ParseAutoReviewPolicy(config.TestAutoReviewPolicy)
if err != nil {
return nil, fmt.Errorf("resolve test auto review policy: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
AutoReviewPolicy: policy,
Report: func(err error) { log.Printf("auto review error: %v", err) },
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}