package main import ( "context" "encoding/json" "fmt" "log" "os" "os/signal" "path/filepath" "syscall" "time" "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryagent" postgresstore "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/platform/weknora" "knowledge-governance/internal/versionhistory" artifactfetch "knowledge-governance/internal/workflows/artifactfetch" catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh" "knowledge-governance/internal/workflows/sourcescan" "knowledge-governance/internal/workflows/testweknora" ) // libraryStageRecorder 把执行阶段写进持久作业,页面刷新/worker 重启后仍可复现。 type libraryStageRecorder struct { store *postgresstore.Store jobID string scope string // mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。 mappingDigest string } func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) { if err := r.store.UpdateLibraryStage(ctx, r.jobID, stage, state, nil, nil, nil, nil, detail, time.Now().UTC()); err != nil { log.Printf("library stage %s/%s update failed: %v", r.jobID, stage, err) } } // libraryCaptureJobType is shared with the API which enqueues it. const libraryCaptureJobType = "library.capture" // librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。 const librarySyncJobType = "library.sync" // libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。 const libraryRestoreJobType = "library.restore" func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() if err := run(ctx, os.Getenv); err != nil { log.Fatal(err) } } func run(ctx context.Context, getenv func(string) string) error { config, err := appconfig.Load(getenv) if err != nil { return err } pool, err := pgxpool.New(ctx, config.DatabaseURL) if err != nil { return err } defer pool.Close() if err := pool.Ping(ctx); err != nil { return err } if err := postgresstore.ApplyMigrations(ctx, pool); err != nil { return err } store := postgresstore.NewStore(pool) // 启动先接管被中断的写作业(复用已持久化的预备份),再把剩下的孤儿判失败: // 顺序反了会把可接管的作业先标失败,丢掉已完成的备份。 if resumed, failed, resumeErr := resumeInterruptedLibraryJobs(context.Background(), store, config); resumeErr != nil { log.Printf("library job resume failed: %v", resumeErr) } else if resumed > 0 || failed > 0 { log.Printf("library job resume: %d resumed, %d failed without backup", resumed, failed) } // 运行期也要持续接管:只在启动扫一次会漏掉"worker 在两次重启之间被杀"的情况。 go func() { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: resumed, failed, err := resumeInterruptedLibraryJobs(ctx, store, config) if err != nil { log.Printf("library job takeover sweep: %v", err) } else if resumed > 0 || failed > 0 { log.Printf("library job takeover sweep: %d resumed, %d failed without backup", resumed, failed) } } } }() if repaired, reconcileErr := store.ReconcileLibraryJobs(context.Background(), time.Now().UTC()); reconcileErr != nil { log.Printf("library job reconcile failed: %v", reconcileErr) } else if repaired > 0 { log.Printf("library job reconcile: %d orphan job(s) marked failed", repaired) } handlers, err := workerHandlers(store, config) if err != nil { return err } if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil { log.Print("QA directory synchronization unavailable; handler was not registered") } mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers) if mirrorErr != nil { log.Print("QA local mirror configuration invalid; handler was not registered") } // 生产同步/恢复的执行器不依赖主机侧封存 agent:没有 agent socket 也要能跑。 // 这里在全部注册之后无条件覆盖一次,确保 library.sync/library.restore 一定在队列类型里。 handlers[librarySyncJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobSync) } handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) } worker, err := jobs.NewWorker(jobs.WorkerConfig{ Queue: store, WorkerID: config.WorkerID, LeaseDuration: config.LeaseDuration, RetryDelay: config.RetryDelay, Handlers: handlers, OnError: func(err error) { log.Printf("worker error: %v", err) }, }) if err != nil { return err } if ai := configureQAProcessing(pool, getenv, config.DatabaseURL); ai != nil { go ai.Run(ctx, func(error) { log.Print("QA AI worker unavailable; durable work and original labels retained") }) } if mirrorStore != nil { go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") }) } go weeklyMailLoop(ctx, pool, getenv) go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) { log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err) }) log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval) return worker.Run(ctx, config.PollInterval) } func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) { handlers := map[string]jobs.Handler{ "demo": func(_ context.Context, job jobs.Job) error { log.Printf("completed demo job id=%s", job.ID) return nil }, "demo.sleep": jobs.DemoSleepHandler, } catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py") if err != nil { return nil, fmt.Errorf("create catalog refresh runner: %w", err) } catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner}) if err != nil { return nil, fmt.Errorf("create catalog refresh handler: %w", err) } handlers["source.catalog.refresh"] = catalogHandler.Handle ingestionRoot := config.IngestionSpoolRoot if ingestionRoot == "" { ingestionRoot = config.GovernanceSpoolRoot } if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" { ingestionRoot = "/var/lib/governance/ingestion" } fetchMaxBytes := config.ArtifactFetchMaxBytes if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes { fetchMaxBytes = config.TestWeKnoraMaxUploadBytes } if fetchMaxBytes <= 0 { fetchMaxBytes = 50 * 1024 * 1024 } artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", ingestionRoot, "ingestion", fetchMaxBytes) if err != nil { return nil, fmt.Errorf("create artifact fetch runner: %w", err) } artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner}) if err != nil { return nil, fmt.Errorf("create artifact fetch handler: %w", err) } handlers["artifact.fetch"] = artifactHandler.Handle if config.LibraryAgentSocket != "" { // 整库封存:worker 只投递请求并记录结果,真正的停机/导出/恢复发生在主机侧 // agent(同 uid unix socket)。容器里既没有 Docker socket,也没有来源凭据。 agentConfig := libraryagent.ClientConfig{Socket: config.LibraryAgentSocket, Timeout: 15 * time.Minute} handlers[libraryCaptureJobType] = func(ctx context.Context, job jobs.Job) error { var payload struct { Scope string `json:"scope"` CaptureJob string `json:"capture_job"` RequestedBy string `json:"requested_by"` } if err := json.Unmarshal(job.Payload, &payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.CaptureJob == "" { return fmt.Errorf("invalid library capture payload") } // 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。 libraryJobID := "libseal-" + payload.CaptureJob startedAt := time.Now().UTC() if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil { log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr) } if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil, "主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil { log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr) } result, err := libraryagent.Capture(ctx, agentConfig, payload.Scope, payload.CaptureJob) finishedAt := time.Now().UTC() if err != nil { // 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。 _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, err.Error(), finishedAt) _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(), "agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt) log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err) return err } // 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。 versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionTestSealed, payload.Scope, result.SnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID, Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay, "requested_by": payload.RequestedBy, "agent_result": json.RawMessage(result.Payload)}, CreatedBy: payload.RequestedBy, CreatedAt: finishedAt, }) if versionErr != nil { log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr) } _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil, "已封存快照 "+result.SnapshotID, finishedAt) _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "", result.SnapshotID, "", finishedAt) log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s", payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy) return nil } // 生产同步:唯一入口在 API(闸口未齐不建作业)。这里仍实现执行器, // 但默认 fail-closed:没有闸口证据就明确失败,绝不向生产写入半个字节。 handlers[librarySyncJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobSync) } handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) } // A crash can leave the source writer stopped: recover before taking work, // and keep the failure visible instead of pretending the worker is ready. go func() { recoveryCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() if _, err := libraryagent.Recover(recoveryCtx, agentConfig); err != nil { log.Printf("library agent recovery check failed: %v", err) return } log.Printf("library agent reachable; freeze journals recovered") }() } if config.VersionHistoryGiteaBaseURL != "" { // 「版本记录」同步:匿名只读远端仓库的提交/tag,写本库快照。未配置来源就不注册, // 界面上的同步按钮也会显示"未接线",不会入队一个没人处理的作业。 client, err := versionhistory.NewGiteaClient(versionhistory.GiteaConfig{ BaseURL: config.VersionHistoryGiteaBaseURL, AllowedBaseURLs: config.VersionHistoryAllowedOrigins, LinkOrigin: config.VersionHistoryLinkOrigin, Repo: config.VersionHistoryRepo, Timeout: config.VersionHistoryHTTPTimeout, }) if err != nil { return nil, fmt.Errorf("create version history client: %w", err) } syncer, err := versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages}) if err != nil { return nil, fmt.Errorf("create version history syncer: %w", err) } handlers[versionhistory.JobType] = func(ctx context.Context, job jobs.Job) error { triggeredBy := "" var payload struct { TriggeredBy string `json:"triggered_by"` TargetSHA string `json:"target_sha"` } if err := json.Unmarshal(job.Payload, &payload); err == nil { triggeredBy = payload.TriggeredBy } activeSyncer := syncer if payload.TargetSHA != "" { var e error activeSyncer, e = versionhistory.NewSyncer(versionhistory.SyncerConfig{Source: client, Store: store, RepoKey: config.VersionHistoryRepo, MaxPages: config.VersionHistoryMaxPages, TargetSHAs: []string{payload.TargetSHA}}) if e != nil { return e } } run, err := activeSyncer.Sync(ctx, job.ID, triggeredBy) if err != nil { log.Printf("version history sync failed job=%s run=%d code=%s: %v", job.ID, run.ID, run.ErrorCode, err) return err } log.Printf("version history sync completed job=%s run=%d fetched=%d expected=%v complete=%v", job.ID, run.ID, run.FetchedCount, run.ExpectedCount, run.Complete) return nil } } if config.PPScanEnabled { scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{ AllowedRoot: config.PPScanAllowedRoot, LedgerPath: config.PPScanLedgerPath, ManifestPath: config.PPScanManifestPath, }) if err != nil { return nil, fmt.Errorf("create PP source scanner: %w", err) } handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner}) if err != nil { return nil, fmt.Errorf("create PP source scan handler: %w", err) } handlers["source.scan.pp"] = handler.Handle } if !config.TestWeKnoraEnabled { return handlers, nil } apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) if err != nil { return nil, fmt.Errorf("read test WeKnora API key: %w", err) } client, err := weknora.NewClient(weknora.ClientConfig{ BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: config.TestWeKnoraHTTPTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP, }) if err != nil { return nil, fmt.Errorf("create test WeKnora client: %w", err) } upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{ Store: store, Client: client, SpoolRoot: config.GovernanceSpoolRoot, MaxUploadBytes: config.TestWeKnoraMaxUploadBytes, FirstPollDelay: config.TestWeKnoraPollInitial, }) if err != nil { return nil, fmt.Errorf("create test upload handler: %w", err) } poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{ Store: store, Client: client, PollDelay: config.TestWeKnoraPollInitial, StallAfter: config.TestWeKnoraStalledAfter, }) if err != nil { return nil, fmt.Errorf("create test poll handler: %w", err) } policy, err := governance.ParseAutoReviewPolicy(config.TestAutoReviewPolicy) if err != nil { return nil, fmt.Errorf("resolve test auto review policy: %w", err) } validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{ Store: store, Client: client, PolicyVersion: config.TestAcceptancePolicyVersion, AutoReviewPolicy: policy, Report: func(err error) { log.Printf("auto review error: %v", err) }, }) if err != nil { return nil, fmt.Errorf("create test validate handler: %w", err) } remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{ Store: store, Client: client, }) if err != nil { return nil, fmt.Errorf("create test removal handler: %w", err) } handlers["test.upload"] = upload.Handle handlers["test.poll"] = poll.Handle handlers["test.validate"] = validate.Handle handlers["test.remove"] = remove.Handle return handlers, nil }