Files
knowledge-governance/internal/librarymaintenance/quiesce.go
T
姚章浩 0ef79846e8
build / build (push) Successful in 1m19s
fix(library): 后台静默必须在关入口之后证明,并在停写者后复核
原顺序先证明后台空、再关入口排空:排空期间还在服务的在途请求会派生新任务,
那一刻的空到停写者时已经失效,而 app 收到 SIGTERM 会 exit(1) 把任务腰斩。

现在固定为:关入口并排空 → 证明后台静默(连续为空 + 稳定窗口 + 任务指纹)→
停写入者 → 停完再复核(队列非空/指纹变化即判 background_truncated)。
scheduled/retry 自行产生任务或任务卡住时 fail closed;旧 journal 的
background_idle 布尔不再当证明;入口关到一半失败时恢复入口并保持维护。
2026-10-29 04:02:59 +08:00

329 lines
9.0 KiB
Go

package librarymaintenance
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net"
"sort"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryfreeze"
)
// targetQuiescer 回答"后台写入者是不是已经空了":
// - 目标库:未结清的后台操作、未完成的解析(这两张表就是 app 的后台账本);
// - 目标队列:asynq 里还在排队/在跑/待重试的任务。
//
// 为什么要单独做:HTTP 入口排空只覆盖连接;app 内的 asynq worker 会被上游的
// SIGTERM exit(1) 切断,正在跑的任务就这样被腰斩。窗口必须先等它们结束。
type targetQuiescer struct {
pool *pgxpool.Pool
redisAddr string
redisPass string
redisDB int
timeout time.Duration
queueMatch string
}
func newTargetQuiescer(ctx context.Context, databaseURL, redisAddr, redisPass string, redisDB int, timeout time.Duration) (*targetQuiescer, error) {
if databaseURL == "" || redisAddr == "" {
return nil, errors.New("后台静默检查缺少目标库或队列地址")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return nil, errors.New("后台静默检查连不上目标库")
}
if timeout <= 0 {
timeout = 10 * time.Second
}
return &targetQuiescer{pool: pool, redisAddr: redisAddr, redisPass: redisPass, redisDB: redisDB,
timeout: timeout, queueMatch: "asynq:*"}, nil
}
func (q *targetQuiescer) Close() { q.pool.Close() }
// Snapshot 是一次"后台任务集合"取样:既给出深度,也给出任务指纹。
//
// 指纹是把"这一瞬间恰好空"和"一段时间内没有新任务产生"分开的唯一手段:
// scheduled/retry 会自己冒出新任务,只看瞬时 0 无法证明停写者时的切点。
func (q *targetQuiescer) Snapshot(ctx context.Context) (libraryfreeze.QuiesceSnapshot, error) {
var pendingOps, unsettled int64
// task_pending_ops / pending_subtasks_count 是 app 自己的"正在做"账本:
// 不看它们只看 HTTP 会漏。注意不能拿 parse_status<>'completed' 当"在跑"——
// 解析失败/卡住的文档会永远停在那里,那是数据状态,不是会跑动的任务;
// 用它当判据会让维护窗口永久开不了(真跑踩过)。
if err := q.pool.QueryRow(ctx, `SELECT
(SELECT count(*) FROM task_pending_ops),
(SELECT count(*) FROM knowledges WHERE deleted_at IS NULL
AND coalesce(pending_subtasks_count,0) <> 0)`).
Scan(&pendingOps, &unsettled); err != nil {
return libraryfreeze.QuiesceSnapshot{}, fmt.Errorf("读不到后台账本:%w", err)
}
tasks, err := q.queueTasks(ctx)
if err != nil {
return libraryfreeze.QuiesceSnapshot{}, err
}
sort.Strings(tasks)
sum := sha256.Sum256([]byte(strings.Join(tasks, "\n")))
return libraryfreeze.QuiesceSnapshot{
Depth: int64(len(tasks)),
PendingOps: pendingOps,
Unsettled: unsettled,
Digest: hex.EncodeToString(sum[:]),
Detail: fmt.Sprintf("pending_ops=%d unsettled_knowledges=%d asynq_tasks=%d",
pendingOps, unsettled, len(tasks)),
}, nil
}
func (q *targetQuiescer) Idle(ctx context.Context) (bool, string, error) {
snapshot, err := q.Snapshot(ctx)
if err != nil {
return false, "", err
}
idle := snapshot.Depth == 0 && snapshot.PendingOps == 0 && snapshot.Unsettled == 0
return idle, snapshot.Detail, nil
}
// queueTasks 列出"还没结束"的 asynq 任务(排队/在跑/待重试),每项是 键|任务ID。
// archived/completed 是历史,不算忙。
func (q *targetQuiescer) queueTasks(ctx context.Context) ([]string, error) {
conn, err := dialRedis(q.redisAddr, q.redisPass, q.redisDB, q.timeout)
if err != nil {
return nil, fmt.Errorf("连不上目标队列:%w", err)
}
defer conn.Close()
keys, err := conn.scan(ctx, q.queueMatch)
if err != nil {
return nil, fmt.Errorf("扫描目标队列失败:%w", err)
}
sort.Strings(keys)
tasks := make([]string, 0, 32)
for _, key := range keys {
suffix := key[strings.LastIndex(key, ":"):]
switch suffix {
case ":pending", ":inprogress", ":active", ":scheduled", ":retry":
default:
continue
}
items, err := conn.rangeAll(ctx, key)
if err != nil {
return nil, fmt.Errorf("读取目标队列 %s 失败:%w", key, err)
}
for _, item := range items {
tasks = append(tasks, key+"|"+taskIdentity(item))
}
}
return tasks, nil
}
// taskIdentity 从 asynq 的任务 JSON 里取稳定 ID;取不到就用整条内容摘要,
// 目的都是"同一批任务 -> 同一个指纹"。
func taskIdentity(raw string) string {
var payload map[string]json.RawMessage
if err := json.Unmarshal([]byte(raw), &payload); err == nil {
for _, key := range []string{"ID", "id"} {
if value, ok := payload[key]; ok {
var id string
if err := json.Unmarshal(value, &id); err == nil && id != "" {
return id
}
}
}
}
sum := sha256.Sum256([]byte(raw))
return hex.EncodeToString(sum[:])
}
// ---- 最小 RESP 客户端:只用到 AUTH/SELECT/SCAN/LLEN/ZCARD,不引第三方库 ----
type redisConn struct {
conn net.Conn
reader *bufio.Reader
timeout time.Duration
}
func dialRedis(addr, password string, db int, timeout time.Duration) (*redisConn, error) {
dialer := &net.Dialer{Timeout: timeout}
conn, err := dialer.Dial("tcp", addr)
if err != nil {
return nil, err
}
_ = conn.SetDeadline(time.Now().Add(timeout))
c := &redisConn{conn: conn, reader: bufio.NewReader(conn), timeout: timeout}
if password != "" {
if _, err := c.do("AUTH", password); err != nil {
conn.Close()
return nil, err
}
}
if db != 0 {
if _, err := c.do("SELECT", strconv.Itoa(db)); err != nil {
conn.Close()
return nil, err
}
}
return c, nil
}
func (c *redisConn) Close() error { return c.conn.Close() }
// rangeAll 读整个 list/zset。必须先问 TYPE 再选命令:
// 对 zset 发 LRANGE 会返回错误,而不是"不是 list"的普通回复。
func (c *redisConn) rangeAll(ctx context.Context, key string) ([]string, error) {
kind, err := c.do("TYPE", key)
if err != nil {
return nil, err
}
var command string
switch kind {
case "list":
command = "LRANGE"
case "zset":
command = "ZRANGE"
default:
return nil, nil
}
reply, err := c.do(command, key, "0", "-1")
if err != nil {
return nil, err
}
items, ok := reply.([]any)
if !ok {
return nil, errors.New("unexpected queue reply")
}
out := make([]string, 0, len(items))
for _, item := range items {
if text, ok := item.(string); ok {
out = append(out, text)
}
}
return out, nil
}
func (c *redisConn) do(args ...string) (any, error) {
_ = c.conn.SetDeadline(time.Now().Add(c.timeout))
var builder strings.Builder
builder.WriteString("*" + strconv.Itoa(len(args)) + "\r\n")
for _, arg := range args {
builder.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n")
}
if _, err := c.conn.Write([]byte(builder.String())); err != nil {
return nil, err
}
return c.read()
}
func (c *redisConn) read() (any, error) {
line, err := c.reader.ReadString('\n')
if err != nil {
return nil, err
}
line = strings.TrimRight(line, "\r\n")
if line == "" {
return nil, errors.New("empty redis reply")
}
switch line[0] {
case '+':
return line[1:], nil
case '-':
return nil, errors.New(line[1:])
case ':':
value, err := strconv.ParseInt(line[1:], 10, 64)
return value, err
case '$':
length, err := strconv.Atoi(line[1:])
if err != nil {
return nil, err
}
if length < 0 {
return nil, nil
}
buf := make([]byte, length+2)
if _, err := ioReadFull(c.reader, buf); err != nil {
return nil, err
}
return string(buf[:length]), nil
case '*':
count, err := strconv.Atoi(line[1:])
if err != nil {
return nil, err
}
items := make([]any, 0, count)
for i := 0; i < count; i++ {
item, err := c.read()
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, nil
default:
return nil, errors.New("unsupported redis reply")
}
}
func ioReadFull(reader *bufio.Reader, buf []byte) (int, error) {
total := 0
for total < len(buf) {
n, err := reader.Read(buf[total:])
total += n
if err != nil {
return total, err
}
}
return total, nil
}
func (c *redisConn) scan(ctx context.Context, match string) ([]string, error) {
var keys []string
cursor := "0"
for {
if err := ctx.Err(); err != nil {
return nil, err
}
reply, err := c.do("SCAN", cursor, "MATCH", match, "COUNT", "1000")
if err != nil {
return nil, err
}
items, ok := reply.([]any)
if !ok || len(items) != 2 {
return nil, errors.New("unexpected SCAN reply")
}
next, _ := items[0].(string)
batch, _ := items[1].([]any)
for _, item := range batch {
if key, ok := item.(string); ok {
keys = append(keys, key)
}
}
if next == "0" || next == "" {
return keys, nil
}
cursor = next
}
}
func (c *redisConn) intCmd(ctx context.Context, cmd string, args ...string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
reply, err := c.do(append([]string{cmd}, args...)...)
if err != nil {
return 0, err
}
value, ok := reply.(int64)
if !ok {
return 0, errors.New("unexpected integer reply")
}
return value, nil
}