Files
knowledge-governance/cmd/worker/main.go
T
姚章浩 281699a431
build / build (push) Successful in 1m21s
feat(library): 生产整库真实读写/校验/恢复与本机真实服务验收
为什么:生产目标此前是"无条件失败"的占位,编排再完整也不能落地;本轮把它做成真实实现,
并在本机 WeKnora 测试栈的真实 PG/ES 上跑通导入、撤销、预备份与故障恢复。

- ProductionLibraryTarget:Current 读行数+身份摘要;Apply 按阶段真实写入(database/wiki
  同事务先删本库再插、resources 传对象、index 搬运原始文档与向量);Verify 比对身份摘要;
  Restore 按预备份快照回放。
- 撤销真正生效:索引阶段写入后 PruneScope 删除快照里没有的文档。
- ProductionBackupStore:导出前独立完整预备份,导出前后行数不一致即失败。
- worker 用注入配置装配目标与预备份;缺项把变量名写进阶段并失败,不再有占位对象。
- capacity 安全加法防溢出;保留策略组合语义明确(各维度都超限才候选)。
- 新增本机真实服务集成测试(一次性 kg_library_test_* 库 + 随机 ES 索引),
  覆盖导入/校验/撤销/预备份/故障注入/恢复。
2026-10-28 15:54:01 +08:00

434 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryagent"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
"knowledge-governance/internal/versionhistory"
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
// libraryStageRecorder 把执行阶段写进持久作业,页面刷新/worker 重启后仍可复现。
type libraryStageRecorder struct {
store *postgresstore.Store
jobID string
scope string
}
func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) {
if err := r.store.UpdateLibraryStage(ctx, r.jobID, stage, state, nil, nil, nil, nil, detail, time.Now().UTC()); err != nil {
log.Printf("library stage %s/%s update failed: %v", r.jobID, stage, err)
}
}
// libraryCaptureJobType is shared with the API which enqueues it.
const libraryCaptureJobType = "library.capture"
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
// 启动先收拾孤儿业务作业:它们会永久占住 (scope,kind) 唯一索引,
// 不清掉的话页面会一直显示"在飞",同一目标再也发不出新作业。
if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC()); reconcileErr != nil {
log.Printf("library job reconcile failed: %v", reconcileErr)
} else if repaired > 0 {
log.Printf("library job reconcile: %d orphan job(s) marked failed", repaired)
}
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil {
log.Print("QA directory synchronization unavailable; handler was not registered")
}
mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers)
if mirrorErr != nil {
log.Print("QA local mirror configuration invalid; handler was not registered")
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil {
go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") })
}
if mirrorStore != nil {
go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") })
}
go weeklyMailLoop(ctx, pool, getenv)
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
ingestionRoot := config.IngestionSpoolRoot
if ingestionRoot == "" {
ingestionRoot = config.GovernanceSpoolRoot
}
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
ingestionRoot = "/var/lib/governance/ingestion"
}
fetchMaxBytes := config.ArtifactFetchMaxBytes
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
}
if fetchMaxBytes <= 0 {
fetchMaxBytes = 50 * 1024 * 1024
}
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
ingestionRoot, "ingestion", fetchMaxBytes)
if err != nil {
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
}
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
if err != nil {
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
}
handlers["artifact.fetch"] = artifactHandler.Handle
if config.LibraryAgentSocket != "" {
// 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧
// agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。
agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute}
handlers[libraryCaptureJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
Scope string `json:"scope"`
CaptureJob string `json:"capture_job"`
RequestedBy string `json:"requested_by"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.CaptureJob == "" {
return fmt.Errorf("invalid library capture payload")
}
// 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。
libraryJobID := "libseal-" + payload.CaptureJob
startedAt := time.Now().UTC()
if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil {
log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr)
}
if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil,
"主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil {
log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr)
}
result, err := libraryagent.Capture(ctx, agentConfig, payload.Scope, payload.CaptureJob)
finishedAt := time.Now().UTC()
if err != nil {
// 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, err.Error(), finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
return err
}
// 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: "libver-" + payload.Scope + "-" + result.SnapshotID, Scope: payload.Scope,
Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID,
Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay,
"requested_by": payload.RequestedBy, "agent_result": json.RawMessage(result.Payload)},
CreatedBy: payload.RequestedBy, CreatedAt: finishedAt,
})
if versionErr != nil {
log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr)
}
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil,
"已封存快照 "+result.SnapshotID, finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "",
result.SnapshotID, "", finishedAt)
log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s",
payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy)
return nil
}
// 生产同步:唯一入口在 API(闸口未齐不建作业)。这里仍实现执行器,
// 但默认 fail-closed:没有闸口证据就明确失败,绝不向生产写入半个字节。
handlers[librarySyncJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" {
return fmt.Errorf("invalid library sync payload")
}
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
// 闸口本身关闭时不必进入执行编排:先给出可操作的缺项。
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
}
if len(gateReasons) > 0 {
message := "生产同步被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
digest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobSync,
map[string]any{"target_snapshot_id": payload.TargetSnapshotID})
if digestErr != nil {
message := "无法计算请求摘要"
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message, "", "", "", time.Now().UTC())
return errors.New(message)
}
grant, found, grantErr := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if grantErr != nil || !found {
// 授权缺失/过期/参数不符都会走到这里:不能写生产。
message := "没有与目标快照匹配的有效发布授权"
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
request := libraryrelease.Request{
Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy},
}
// 生产目标与预备份存储来自部署注入的真实连接;未配置就在第一步失败并写明缺项,
// 不会出现"写了一半没有回滚目标"的状态。
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope)
if wireErr != nil {
message := wireErr.Error()
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
defer target.Close()
outcome := libraryrelease.Run(ctx, request, target, backups, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "verify", "", "",
"", payload.TargetSnapshotID, finishedAt)
log.Printf("library sync succeeded job=%s scope=%s snapshot=%s backup=%s",
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.ID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error,
outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error,
outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", outcome.Error,
"未对生产执行任何写入,无需恢复", "", "", finishedAt)
return errors.New(outcome.Error)
}
}
// A crash can leave the source writer stopped: recover before taking work,
// and keep the failure visible instead of pretending the worker is ready.
go func() {
recoveryCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if _, err := libraryagent.Recover(recoveryCtx, agentConfig); err != nil {
log.Printf("library agent recovery check failed: %v", err)
return
}
log.Printf("library agent reachable; freeze journals recovered")
}()
}
if config.VersionHistoryGiteaBaseURL != "" {
// 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册,
// 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。
client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{
BaseURL: config.VersionHistoryGiteaBaseURL,
AllowedBaseURLs: config.VersionHistoryAllowedOrigins,
LinkOrigin: config.VersionHistoryLinkOrigin,
Repo: config.VersionHistoryRepo,
Timeout: config.VersionHistoryHTTPTimeout,
})
if err != nil {
return nil, fmt.Errorf("create version history client: %w", err)
}
syncer, err := versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store,
RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages})
if err != nil {
return nil, fmt.Errorf("create version history syncer: %w", err)
}
handlers[versionhistory.JobType] = func(ctx context.Context, job jobs.Job) error {
triggeredBy := ""
var payload struct {
TriggeredBy string `json:"triggered_by"`
TargetSHA string `json:"target_sha"`
}
if err := json.Unmarshal(job.Payload, &payload); err == nil {
triggeredBy = payload.TriggeredBy
}
activeSyncer := syncer
if payload.TargetSHA != "" {
var e error
activeSyncer, e = versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages, TargetSHAs: []string{payload.TargetSHA}})
if e != nil {
return e
}
}
run, err := activeSyncer.Sync(ctx, job.ID, triggeredBy)
if err != nil {
log.Printf("version history sync failed job=%s run=%d code=%s: %v", job.ID, run.ID, run.ErrorCode, err)
return err
}
log.Printf("version history sync completed job=%s run=%d fetched=%d expected=%v complete=%v",
job.ID, run.ID, run.FetchedCount, run.ExpectedCount, run.Complete)
return nil
}
}
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
policy, err := governance.ParseAutoReviewPolicy(config.TestAutoReviewPolicy)
if err != nil {
return nil, fmt.Errorf("resolve test auto review policy: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
AutoReviewPolicy: policy,
Report: func(err error) { log.Printf("auto review error: %v", err) },
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}