Files
knowledge-governance/cmd/worker/main.go
T
姚章浩 7b8cc0384b
build / build (push) Successful in 1m19s
feat(library): 生产发布授权与同步/恢复编排,worker 接线
为什么:同步入口此前只会拒绝;授权、预备份、四类写入、验证与失败恢复都没有代码。
本轮把编排与授权实现出来,真实生产执行仍按授权闸口关闭。

- internal/libraryrelease:预备份 → 数据库/资源/索引/Wiki → 验证 → 失败恢复;
  授权过期/目标不符/快照不符/参数变化在动生产前拒绝;恢复失败单独上报。
- 迁移 000067 library_release_authorizations(目标+快照+参数摘要+有效期,可撤销)。
- POST/GET /api/v1/library/authorizations 与 /{id}/revoke,仅管理员签发,5–1440 分钟。
- worker library.sync 接编排:先闸口再授权,阶段落库,只有编排成功才标成功。
- 生产目标适配器未配置时显式失败(不返回零值状态),预备份存储同。
- overview 返回授权状态,页面"生产保护与恢复"显示签发人与有效期。
- 验收矩阵区分"已实现·隔离测试通过"与"真实执行未授权"。
2026-10-28 15:11:27 +08:00

427 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryagent"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
v071 "knowledge-governance/internal/releaseadapter/v071"
"knowledge-governance/internal/versionhistory"
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
)
// libraryStageRecorder 把执行阶段写进持久作业,页面刷新/worker 重启后仍可复现。
type libraryStageRecorder struct {
store *postgresstore.Store
jobID string
scope string
}
func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) {
if err := r.store.UpdateLibraryStage(ctx, r.jobID, stage, state, nil, nil, nil, nil, detail, time.Now().UTC()); err != nil {
log.Printf("library stage %s/%s update failed: %v", r.jobID, stage, err)
}
}
// libraryCaptureJobType is shared with the API which enqueues it.
const libraryCaptureJobType = "library.capture"
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, getenv func(string) string) error {
config, err := appconfig.Load(getenv)
if err != nil {
return err
}
pool, err := pgxpool.New(ctx, config.DatabaseURL)
if err != nil {
return err
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
return err
}
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
return err
}
store := postgresstore.NewStore(pool)
// 启动先收拾孤儿业务作业:它们会永久占住 (scope,kind) 唯一索引,
// 不清掉的话页面会一直显示"在飞",同一目标再也发不出新作业。
if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC()); reconcileErr != nil {
log.Printf("library job reconcile failed: %v", reconcileErr)
} else if repaired > 0 {
log.Printf("library job reconcile: %d orphan job(s) marked failed", repaired)
}
handlers, err := workerHandlers(store, config)
if err != nil {
return err
}
if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil {
log.Print("QA directory synchronization unavailable; handler was not registered")
}
mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers)
if mirrorErr != nil {
log.Print("QA local mirror configuration invalid; handler was not registered")
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
WorkerID: config.WorkerID,
LeaseDuration: config.LeaseDuration,
RetryDelay: config.RetryDelay,
Handlers: handlers,
OnError: func(err error) { log.Printf("worker error: %v", err) },
})
if err != nil {
return err
}
if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil {
go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") })
}
if mirrorStore != nil {
go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") })
}
go weeklyMailLoop(ctx, pool, getenv)
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
})
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
return worker.Run(ctx, config.PollInterval)
}
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
handlers := map[string]jobs.Handler{
"demo": func(_ context.Context, job jobs.Job) error {
log.Printf("completed demo job id=%s", job.ID)
return nil
},
"demo.sleep": jobs.DemoSleepHandler,
}
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
if err != nil {
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
}
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
if err != nil {
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
ingestionRoot := config.IngestionSpoolRoot
if ingestionRoot == "" {
ingestionRoot = config.GovernanceSpoolRoot
}
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
ingestionRoot = "/var/lib/governance/ingestion"
}
fetchMaxBytes := config.ArtifactFetchMaxBytes
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
}
if fetchMaxBytes <= 0 {
fetchMaxBytes = 50 * 1024 * 1024
}
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
ingestionRoot, "ingestion", fetchMaxBytes)
if err != nil {
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
}
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
if err != nil {
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
}
handlers["artifact.fetch"] = artifactHandler.Handle
if config.LibraryAgentSocket != "" {
// 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧
// agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。
agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute}
handlers[libraryCaptureJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
Scope string `json:"scope"`
CaptureJob string `json:"capture_job"`
RequestedBy string `json:"requested_by"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.CaptureJob == "" {
return fmt.Errorf("invalid library capture payload")
}
// 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。
libraryJobID := "libseal-" + payload.CaptureJob
startedAt := time.Now().UTC()
if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil {
log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr)
}
if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil,
"主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil {
log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr)
}
result, err := libraryagent.Capture(ctx, agentConfig, payload.Scope, payload.CaptureJob)
finishedAt := time.Now().UTC()
if err != nil {
// 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, err.Error(), finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
return err
}
// 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: "libver-" + payload.Scope + "-" + result.SnapshotID, Scope: payload.Scope,
Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID,
Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay,
"requested_by": payload.RequestedBy, "agent_result": json.RawMessage(result.Payload)},
CreatedBy: payload.RequestedBy, CreatedAt: finishedAt,
})
if versionErr != nil {
log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr)
}
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil,
"已封存快照 "+result.SnapshotID, finishedAt)
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "",
result.SnapshotID, "", finishedAt)
log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s",
payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy)
return nil
}
// 生产同步:唯一入口在 API(闸口未齐不建作业)。这里仍实现执行器,
// 但默认 fail-closed:没有闸口证据就明确失败,绝不向生产写入半个字节。
handlers[librarySyncJobType] = func(ctx context.Context, job jobs.Job) error {
var payload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" {
return fmt.Errorf("invalid library sync payload")
}
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
// 闸口本身关闭时不必进入执行编排:先给出可操作的缺项。
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
}
if len(gateReasons) > 0 {
message := "生产同步被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
digest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobSync,
map[string]any{"target_snapshot_id": payload.TargetSnapshotID})
if digestErr != nil {
message := "无法计算请求摘要"
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message, "", "", "", time.Now().UTC())
return errors.New(message)
}
grant, found, grantErr := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if grantErr != nil || !found {
// 授权缺失/过期/参数不符都会走到这里:不能写生产。
message := "没有与目标快照匹配的有效发布授权"
recorder.Stage(ctx, "prebackup", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
request := libraryrelease.Request{
Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy},
}
// 生产目标与预备份存储都来自适配层:未配置时整条链路在第一步就以明确缺项失败,
// 不会出现"写了一半没有回滚目标"的状态。
target, _ := v071.NewProductionLibraryTarget(config.ProductionWeKnoraBaseURL, config.ProductionWeKnoraAPIKeyFile)
outcome := libraryrelease.Run(ctx, request, target, v071.ArchiveBackupStore{}, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "verify", "", "",
"", payload.TargetSnapshotID, finishedAt)
log.Printf("library sync succeeded job=%s scope=%s snapshot=%s backup=%s",
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.ID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error,
outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error,
outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", outcome.Error,
"未对生产执行任何写入,无需恢复", "", "", finishedAt)
return errors.New(outcome.Error)
}
}
// A crash can leave the source writer stopped: recover before taking work,
// and keep the failure visible instead of pretending the worker is ready.
go func() {
recoveryCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if _, err := libraryagent.Recover(recoveryCtx, agentConfig); err != nil {
log.Printf("library agent recovery check failed: %v", err)
return
}
log.Printf("library agent reachable; freeze journals recovered")
}()
}
if config.VersionHistoryGiteaBaseURL != "" {
// 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册,
// 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。
client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{
BaseURL: config.VersionHistoryGiteaBaseURL,
AllowedBaseURLs: config.VersionHistoryAllowedOrigins,
LinkOrigin: config.VersionHistoryLinkOrigin,
Repo: config.VersionHistoryRepo,
Timeout: config.VersionHistoryHTTPTimeout,
})
if err != nil {
return nil, fmt.Errorf("create version history client: %w", err)
}
syncer, err := versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store,
RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages})
if err != nil {
return nil, fmt.Errorf("create version history syncer: %w", err)
}
handlers[versionhistory.JobType] = func(ctx context.Context, job jobs.Job) error {
triggeredBy := ""
var payload struct {
TriggeredBy string `json:"triggered_by"`
TargetSHA string `json:"target_sha"`
}
if err := json.Unmarshal(job.Payload, &payload); err == nil {
triggeredBy = payload.TriggeredBy
}
activeSyncer := syncer
if payload.TargetSHA != "" {
var e error
activeSyncer, e = versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages, TargetSHAs: []string{payload.TargetSHA}})
if e != nil {
return e
}
}
run, err := activeSyncer.Sync(ctx, job.ID, triggeredBy)
if err != nil {
log.Printf("version history sync failed job=%s run=%d code=%s: %v", job.ID, run.ID, run.ErrorCode, err)
return err
}
log.Printf("version history sync completed job=%s run=%d fetched=%d expected=%v complete=%v",
job.ID, run.ID, run.FetchedCount, run.ExpectedCount, run.Complete)
return nil
}
}
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
LedgerPath: config.PPScanLedgerPath,
ManifestPath: config.PPScanManifestPath,
})
if err != nil {
return nil, fmt.Errorf("create PP source scanner: %w", err)
}
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
return nil, fmt.Errorf("create PP source scan handler: %w", err)
}
handlers["source.scan.pp"] = handler.Handle
}
if !config.TestWeKnoraEnabled {
return handlers, nil
}
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
if err != nil {
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
}
client, err := weknora.NewClient(weknora.ClientConfig{
BaseURL: config.TestWeKnoraBaseURL,
APIKey: apiKey,
Timeout: config.TestWeKnoraHTTPTimeout,
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
})
if err != nil {
return nil, fmt.Errorf("create test WeKnora client: %w", err)
}
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
Store: store,
Client: client,
SpoolRoot: config.GovernanceSpoolRoot,
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
FirstPollDelay: config.TestWeKnoraPollInitial,
})
if err != nil {
return nil, fmt.Errorf("create test upload handler: %w", err)
}
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
Store: store,
Client: client,
PollDelay: config.TestWeKnoraPollInitial,
StallAfter: config.TestWeKnoraStalledAfter,
})
if err != nil {
return nil, fmt.Errorf("create test poll handler: %w", err)
}
policy, err := governance.ParseAutoReviewPolicy(config.TestAutoReviewPolicy)
if err != nil {
return nil, fmt.Errorf("resolve test auto review policy: %w", err)
}
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
Store: store,
Client: client,
PolicyVersion: config.TestAcceptancePolicyVersion,
AutoReviewPolicy: policy,
Report: func(err error) { log.Printf("auto review error: %v", err) },
})
if err != nil {
return nil, fmt.Errorf("create test validate handler: %w", err)
}
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
Store: store,
Client: client,
})
if err != nil {
return nil, fmt.Errorf("create test removal handler: %w", err)
}
handlers["test.upload"] = upload.Handle
handlers["test.poll"] = poll.Handle
handlers["test.validate"] = validate.Handle
handlers["test.remove"] = remove.Handle
return handlers, nil
}