Files
knowledge-governance/cmd/worker/main.go
T
姚章浩 dc9e827656
build / build (push) Failing after 17s
feat(library): 整库版本/作业持久化与状态读取契约,生产同步默认拒绝
- 新增 library_versions:测试封存、生产生效、生产预备份三类分开登记,
  生产生效版本唯一(旧版本必须先 supersede),测试快照不能顶替生产版本
- 新增 library_jobs + library_job_stages:业务作业身份、阶段、错误与恢复说明,
  分项计数用 NULL 表示"总量未知";互斥与幂等由数据库唯一索引保证
- 新增 GET /api/v1/library/overview、/jobs、/jobs/{id};POST /syncs 闸口未齐
  返回 503 缺项且不建作业
- 封存作业落库:先建业务作业再入队,worker 推进 export 阶段并登记 test_sealed
- 发布管理读真实状态:生产未知不再顶替、差异 unknown、启动同步按缺项禁用、
  同步明细/四类分项来自持久阶段;阶段勾选只认当前作业证据
- 准备信息标注封存时间与范围,不声称实时测试库已就绪
2026-10-28 13:58:51 +08:00

354 lines
15 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryagent"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
"knowledge-governance/internal/versionhistory"
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
// libraryCaptureJobType is shared with the API which enqueues it.
const libraryCaptureJobType = "library.capture"
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil {
log.Print("QA directory synchronization unavailable; handler was not registered")
}
mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers)
if mirrorErr != nil {
log.Print("QA local mirror configuration invalid; handler was not registered")
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil {
go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") })
}
if mirrorStore != nil {
go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") })
}
go weeklyMailLoop(ctx, pool, getenv)
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
ingestionRoot := config.IngestionSpoolRoot
if ingestionRoot == "" {
ingestionRoot = config.GovernanceSpoolRoot
}
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
ingestionRoot = "/var/lib/governance/ingestion"
}
fetchMaxBytes := config.ArtifactFetchMaxBytes
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
}
if fetchMaxBytes <= 0 {
fetchMaxBytes = 50 * 1024 * 1024
}
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
ingestionRoot, "ingestion", fetchMaxBytes)
if err != nil {
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
}
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
if err != nil {
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
}
handlers["artifact.fetch"] = artifactHandler.Handle
if config.LibraryAgentSocket != "" {
// 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧
// agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。
agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute}
handlers[libraryCaptureJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
Scope string `json:"scope"`
CaptureJob string `json:"capture_job"`
RequestedBy string `json:"requested_by"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.CaptureJob == "" {
return fmt.Errorf("invalid library capture payload")
}
// 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。
libraryJobID := "libseal-" + payload.CaptureJob
startedAt := time.Now().UTC()
if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil {
log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr)
}
if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil,
"主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil {
log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr)
}
result, err := libraryagent.Capture(ctx, agentConfig, payload.Scope, payload.CaptureJob)
finishedAt := time.Now().UTC()
if err != nil {
// 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, err.Error(), finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
return err
}
// 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: "libver-" + payload.Scope + "-" + result.SnapshotID, Scope: payload.Scope,
Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID,
Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay,
"requested_by": payload.RequestedBy, "agent_result": json.RawMessage(result.Payload)},
CreatedBy: payload.RequestedBy, CreatedAt: finishedAt,
})
if versionErr != nil {
log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr)
}
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil,
"已封存快照 "+result.SnapshotID, finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "",
result.SnapshotID, "", finishedAt)
log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s",
payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy)
return nil
}
// 生产同步:唯一入口在 API(闸口未齐不建作业)。这里仍实现执行器,
// 但默认 fail-closed:没有闸口证据就明确失败,绝不向生产写入半个字节。
handlers[librarySyncJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" {
return fmt.Errorf("invalid library sync payload")
}
now := time.Now().UTC()
reasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
// 生产写入适配与预备份实现之前,执行链本身就是缺项:直接拒绝而不是空转。
reasons = append(reasons, "生产预备份与写入适配尚未实现")
message := "生产同步执行链未接通:" + strings.Join(reasons, ";")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
_ = store.UpdateLibraryStage(ctx, payload.LibraryJobID, "prebackup", "failed", nil, nil, nil, nil, message, now)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", now)
log.Printf("library sync refused job=%s scope=%s: %s", payload.LibraryJobID, payload.Scope, message)
return errors.New(message)
}
// A crash can leave the source writer stopped: recover before taking work,
// and keep the failure visible instead of pretending the worker is ready.
go func() {
recoveryCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if _, err := libraryagent.Recover(recoveryCtx, agentConfig); err != nil {
log.Printf("library agent recovery check failed: %v", err)
return
}
log.Printf("library agent reachable; freeze journals recovered")
}()
}
if config.VersionHistoryGiteaBaseURL != "" {
// 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册,
// 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。
client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{
BaseURL: config.VersionHistoryGiteaBaseURL,
AllowedBaseURLs: config.VersionHistoryAllowedOrigins,
LinkOrigin: config.VersionHistoryLinkOrigin,
Repo: config.VersionHistoryRepo,
Timeout: config.VersionHistoryHTTPTimeout,
})
if err != nil {
return nil, fmt.Errorf("create version history client: %w", err)
}
syncer, err := versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store,
RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages})
if err != nil {
return nil, fmt.Errorf("create version history syncer: %w", err)
}
handlers[versionhistory.JobType] = func(ctx context.Context, job jobs.Job) error {
triggeredBy := ""
var payload struct {
TriggeredBy string `json:"triggered_by"`
TargetSHA string `json:"target_sha"`
}
if err := json.Unmarshal(job.Payload, &payload); err == nil {
triggeredBy = payload.TriggeredBy
}
activeSyncer := syncer
if payload.TargetSHA != "" {
var e error
activeSyncer, e = versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages, TargetSHAs: []string{payload.TargetSHA}})
if e != nil {
return e
}
}
run, err := activeSyncer.Sync(ctx, job.ID, triggeredBy)
if err != nil {
log.Printf("version history sync failed job=%s run=%d code=%s: %v", job.ID, run.ID, run.ErrorCode, err)
return err
}
log.Printf("version history sync completed job=%s run=%d fetched=%d expected=%v complete=%v",
job.ID, run.ID, run.FetchedCount, run.ExpectedCount, run.Complete)
return nil
}
}
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
policy, err := governance.ParseAutoReviewPolicy(config.TestAutoReviewPolicy)
if err != nil {
return nil, fmt.Errorf("resolve test auto review policy: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
AutoReviewPolicy: policy,
Report: func(err error) { log.Printf("auto review error: %v", err) },
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}