Files
knowledge-governance/internal/jobs/worker.go
T
姚章浩 98294b9583
build / build (push) Failing after 24s
fix(library): 统一执行身份、终态收尾、准入与数据身份
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
  · 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
    A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
  · 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
    upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
  · 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
    写业务状态、释放别人的维护锁;
  · 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
    服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
  · 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
    location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
  · 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。

what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
  · operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
    治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
  · 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
    心跳失租立刻取消 handler;
  · 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
    收尾;恢复成功登记 recovery 事件并推进 current;
  · worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
  · 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
    发布前核目标实时模型契约、容量门禁前置;
  · 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
  · 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。

how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
2026-10-30 16:14:54 +08:00

188 lines
5.2 KiB
Go

package jobs
import (
"context"
"errors"
"sort"
"time"
)
type Queue interface {
Claim(ctx context.Context, workerID string, allowedTypes []string, now time.Time, leaseDuration time.Duration) (*Job, error)
Complete(ctx context.Context, id, owner string, now time.Time) error
Fail(ctx context.Context, id, owner, message, code string, now, retryAt time.Time) error
Heartbeat(ctx context.Context, id, owner string, now time.Time, leaseDuration time.Duration) error
}
// TerminalFailer is an optional Queue extension (P6): a permanent failure goes
// straight to the dead-letter state with a stable error code instead of
// consuming the retry budget.
type TerminalFailer interface {
FailTerminal(ctx context.Context, id, owner, message, code string, now time.Time) error
}
type Handler func(context.Context, Job) error
type WorkerConfig struct {
Queue Queue
WorkerID string
LeaseDuration time.Duration
HeartbeatInterval time.Duration
RetryDelay time.Duration
Now func() time.Time
Handlers map[string]Handler
OnError func(error)
}
type Worker struct {
queue Queue
workerID string
leaseDuration time.Duration
heartbeatInterval time.Duration
retryDelay time.Duration
now func() time.Time
handlers map[string]Handler
allowedTypes []string
onError func(error)
}
func NewWorker(config WorkerConfig) (*Worker, error) {
if config.Queue == nil {
return nil, errors.New("queue is required")
}
if config.WorkerID == "" {
return nil, errors.New("worker ID is required")
}
if config.LeaseDuration <= 0 {
return nil, errors.New("lease duration must be positive")
}
if config.Now == nil {
config.Now = time.Now
}
if config.RetryDelay <= 0 {
config.RetryDelay = time.Second
}
if config.HeartbeatInterval <= 0 {
config.HeartbeatInterval = config.LeaseDuration / 3
}
if config.OnError == nil {
config.OnError = func(error) {}
}
allowedTypes := make([]string, 0, len(config.Handlers))
for jobType := range config.Handlers {
allowedTypes = append(allowedTypes, jobType)
}
sort.Strings(allowedTypes)
return &Worker{
queue: config.Queue,
workerID: config.WorkerID,
leaseDuration: config.LeaseDuration,
heartbeatInterval: config.HeartbeatInterval,
retryDelay: config.RetryDelay,
now: config.Now,
handlers: config.Handlers,
allowedTypes: allowedTypes,
onError: config.OnError,
}, nil
}
func (w *Worker) Run(ctx context.Context, pollInterval time.Duration) error {
if pollInterval <= 0 {
return errors.New("poll interval must be positive")
}
for {
claimed, err := w.RunOnce(ctx)
if err != nil {
w.onError(err)
}
if claimed && err == nil {
continue
}
timer := time.NewTimer(pollInterval)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return nil
case <-timer.C:
}
}
}
func (w *Worker) failJob(ctx context.Context, job Job, cause error, failedAt time.Time) error {
code := ""
if terminal, c := Terminal(cause); terminal {
code = c
if failer, ok := w.queue.(TerminalFailer); ok {
return failer.FailTerminal(ctx, job.ID, w.workerID, cause.Error(), code, failedAt)
}
}
return w.queue.Fail(ctx, job.ID, w.workerID, cause.Error(), code, failedAt, failedAt.Add(w.retryDelay))
}
func (w *Worker) RunOnce(ctx context.Context) (bool, error) {
claimedAt := w.now()
job, err := w.queue.Claim(ctx, w.workerID, w.allowedTypes, claimedAt, w.leaseDuration)
if err != nil {
return false, err
}
if job == nil {
return false, nil
}
handler := w.handlers[job.Type]
if handler == nil {
err := Permanent(ErrCodeNoHandler, errors.New("no handler registered for job type "+job.Type))
failedAt := w.now()
if failErr := w.failJob(ctx, *job, err, failedAt); failErr != nil {
return true, failErr
}
return true, err
}
// handler 必须能被"失租"取消:心跳报错说明这个代次已经不再拥有作业,
// 若还让 handler 继续跑,它会以旧代次身份继续写业务状态/释放维护锁(F13)。
handlerCtx, cancelHandler := context.WithCancel(ctx)
defer cancelHandler()
heartbeatCtx, stopHeartbeat := context.WithCancel(ctx)
heartbeatResult := make(chan error, 1)
go func() {
ticker := time.NewTicker(w.heartbeatInterval)
defer ticker.Stop()
for {
select {
case <-heartbeatCtx.Done():
heartbeatResult <- nil
return
case <-ticker.C:
if err := w.queue.Heartbeat(ctx, job.ID, w.workerID, w.now(), w.leaseDuration); err != nil {
// 先撤销 handler 的授权,再把错误交给下面统一处理:
// 顺序反过来就会出现"错误已记录、handler 还在写"的空窗。
cancelHandler()
heartbeatResult <- err
return
}
}
}
}()
handlerErr := handler(handlerCtx, *job)
stopHeartbeat()
cancelHandler()
heartbeatErr := <-heartbeatResult
if handlerErr == nil {
handlerErr = heartbeatErr
}
if handlerErr != nil {
failedAt := w.now()
if failErr := w.failJob(ctx, *job, handlerErr, failedAt); failErr != nil {
return true, failErr
}
return true, handlerErr
}
if err := w.queue.Complete(ctx, job.ID, w.workerID, w.now()); err != nil {
return true, err
}
return true, nil
}