0ef79846e8
build / build (push) Successful in 1m19s
原顺序先证明后台空、再关入口排空:排空期间还在服务的在途请求会派生新任务, 那一刻的空到停写者时已经失效,而 app 收到 SIGTERM 会 exit(1) 把任务腰斩。 现在固定为:关入口并排空 → 证明后台静默(连续为空 + 稳定窗口 + 任务指纹)→ 停写入者 → 停完再复核(队列非空/指纹变化即判 background_truncated)。 scheduled/retry 自行产生任务或任务卡住时 fail closed;旧 journal 的 background_idle 布尔不再当证明;入口关到一半失败时恢复入口并保持维护。
329 lines
9.0 KiB
Go
329 lines
9.0 KiB
Go
package librarymaintenance
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"knowledge-governance/internal/libraryfreeze"
|
|
)
|
|
|
|
// targetQuiescer 回答"后台写入者是不是已经空了":
|
|
// - 目标库:未结清的后台操作、未完成的解析(这两张表就是 app 的后台账本);
|
|
// - 目标队列:asynq 里还在排队/在跑/待重试的任务。
|
|
//
|
|
// 为什么要单独做:HTTP 入口排空只覆盖连接;app 内的 asynq worker 会被上游的
|
|
// SIGTERM exit(1) 切断,正在跑的任务就这样被腰斩。窗口必须先等它们结束。
|
|
type targetQuiescer struct {
|
|
pool *pgxpool.Pool
|
|
redisAddr string
|
|
redisPass string
|
|
redisDB int
|
|
timeout time.Duration
|
|
queueMatch string
|
|
}
|
|
|
|
func newTargetQuiescer(ctx context.Context, databaseURL, redisAddr, redisPass string, redisDB int, timeout time.Duration) (*targetQuiescer, error) {
|
|
if databaseURL == "" || redisAddr == "" {
|
|
return nil, errors.New("后台静默检查缺少目标库或队列地址")
|
|
}
|
|
pool, err := pgxpool.New(ctx, databaseURL)
|
|
if err != nil {
|
|
return nil, errors.New("后台静默检查连不上目标库")
|
|
}
|
|
if timeout <= 0 {
|
|
timeout = 10 * time.Second
|
|
}
|
|
return &targetQuiescer{pool: pool, redisAddr: redisAddr, redisPass: redisPass, redisDB: redisDB,
|
|
timeout: timeout, queueMatch: "asynq:*"}, nil
|
|
}
|
|
|
|
func (q *targetQuiescer) Close() { q.pool.Close() }
|
|
|
|
// Snapshot 是一次"后台任务集合"取样:既给出深度,也给出任务指纹。
|
|
//
|
|
// 指纹是把"这一瞬间恰好空"和"一段时间内没有新任务产生"分开的唯一手段:
|
|
// scheduled/retry 会自己冒出新任务,只看瞬时 0 无法证明停写者时的切点。
|
|
func (q *targetQuiescer) Snapshot(ctx context.Context) (libraryfreeze.QuiesceSnapshot, error) {
|
|
var pendingOps, unsettled int64
|
|
// task_pending_ops / pending_subtasks_count 是 app 自己的"正在做"账本:
|
|
// 不看它们只看 HTTP 会漏。注意不能拿 parse_status<>'completed' 当"在跑"——
|
|
// 解析失败/卡住的文档会永远停在那里,那是数据状态,不是会跑动的任务;
|
|
// 用它当判据会让维护窗口永久开不了(真跑踩过)。
|
|
if err := q.pool.QueryRow(ctx, `SELECT
|
|
(SELECT count(*) FROM task_pending_ops),
|
|
(SELECT count(*) FROM knowledges WHERE deleted_at IS NULL
|
|
AND coalesce(pending_subtasks_count,0) <> 0)`).
|
|
Scan(&pendingOps, &unsettled); err != nil {
|
|
return libraryfreeze.QuiesceSnapshot{}, fmt.Errorf("读不到后台账本:%w", err)
|
|
}
|
|
tasks, err := q.queueTasks(ctx)
|
|
if err != nil {
|
|
return libraryfreeze.QuiesceSnapshot{}, err
|
|
}
|
|
sort.Strings(tasks)
|
|
sum := sha256.Sum256([]byte(strings.Join(tasks, "\n")))
|
|
return libraryfreeze.QuiesceSnapshot{
|
|
Depth: int64(len(tasks)),
|
|
PendingOps: pendingOps,
|
|
Unsettled: unsettled,
|
|
Digest: hex.EncodeToString(sum[:]),
|
|
Detail: fmt.Sprintf("pending_ops=%d unsettled_knowledges=%d asynq_tasks=%d",
|
|
pendingOps, unsettled, len(tasks)),
|
|
}, nil
|
|
}
|
|
|
|
func (q *targetQuiescer) Idle(ctx context.Context) (bool, string, error) {
|
|
snapshot, err := q.Snapshot(ctx)
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
idle := snapshot.Depth == 0 && snapshot.PendingOps == 0 && snapshot.Unsettled == 0
|
|
return idle, snapshot.Detail, nil
|
|
}
|
|
|
|
// queueTasks 列出"还没结束"的 asynq 任务(排队/在跑/待重试),每项是 键|任务ID。
|
|
// archived/completed 是历史,不算忙。
|
|
func (q *targetQuiescer) queueTasks(ctx context.Context) ([]string, error) {
|
|
conn, err := dialRedis(q.redisAddr, q.redisPass, q.redisDB, q.timeout)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("连不上目标队列:%w", err)
|
|
}
|
|
defer conn.Close()
|
|
keys, err := conn.scan(ctx, q.queueMatch)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("扫描目标队列失败:%w", err)
|
|
}
|
|
sort.Strings(keys)
|
|
tasks := make([]string, 0, 32)
|
|
for _, key := range keys {
|
|
suffix := key[strings.LastIndex(key, ":"):]
|
|
switch suffix {
|
|
case ":pending", ":inprogress", ":active", ":scheduled", ":retry":
|
|
default:
|
|
continue
|
|
}
|
|
items, err := conn.rangeAll(ctx, key)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("读取目标队列 %s 失败:%w", key, err)
|
|
}
|
|
for _, item := range items {
|
|
tasks = append(tasks, key+"|"+taskIdentity(item))
|
|
}
|
|
}
|
|
return tasks, nil
|
|
}
|
|
|
|
// taskIdentity 从 asynq 的任务 JSON 里取稳定 ID;取不到就用整条内容摘要,
|
|
// 目的都是"同一批任务 -> 同一个指纹"。
|
|
func taskIdentity(raw string) string {
|
|
var payload map[string]json.RawMessage
|
|
if err := json.Unmarshal([]byte(raw), &payload); err == nil {
|
|
for _, key := range []string{"ID", "id"} {
|
|
if value, ok := payload[key]; ok {
|
|
var id string
|
|
if err := json.Unmarshal(value, &id); err == nil && id != "" {
|
|
return id
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sum := sha256.Sum256([]byte(raw))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// ---- 最小 RESP 客户端:只用到 AUTH/SELECT/SCAN/LLEN/ZCARD,不引第三方库 ----
|
|
|
|
type redisConn struct {
|
|
conn net.Conn
|
|
reader *bufio.Reader
|
|
timeout time.Duration
|
|
}
|
|
|
|
func dialRedis(addr, password string, db int, timeout time.Duration) (*redisConn, error) {
|
|
dialer := &net.Dialer{Timeout: timeout}
|
|
conn, err := dialer.Dial("tcp", addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_ = conn.SetDeadline(time.Now().Add(timeout))
|
|
c := &redisConn{conn: conn, reader: bufio.NewReader(conn), timeout: timeout}
|
|
if password != "" {
|
|
if _, err := c.do("AUTH", password); err != nil {
|
|
conn.Close()
|
|
return nil, err
|
|
}
|
|
}
|
|
if db != 0 {
|
|
if _, err := c.do("SELECT", strconv.Itoa(db)); err != nil {
|
|
conn.Close()
|
|
return nil, err
|
|
}
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
func (c *redisConn) Close() error { return c.conn.Close() }
|
|
|
|
// rangeAll 读整个 list/zset。必须先问 TYPE 再选命令:
|
|
// 对 zset 发 LRANGE 会返回错误,而不是"不是 list"的普通回复。
|
|
func (c *redisConn) rangeAll(ctx context.Context, key string) ([]string, error) {
|
|
kind, err := c.do("TYPE", key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var command string
|
|
switch kind {
|
|
case "list":
|
|
command = "LRANGE"
|
|
case "zset":
|
|
command = "ZRANGE"
|
|
default:
|
|
return nil, nil
|
|
}
|
|
reply, err := c.do(command, key, "0", "-1")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, ok := reply.([]any)
|
|
if !ok {
|
|
return nil, errors.New("unexpected queue reply")
|
|
}
|
|
out := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
if text, ok := item.(string); ok {
|
|
out = append(out, text)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (c *redisConn) do(args ...string) (any, error) {
|
|
_ = c.conn.SetDeadline(time.Now().Add(c.timeout))
|
|
var builder strings.Builder
|
|
builder.WriteString("*" + strconv.Itoa(len(args)) + "\r\n")
|
|
for _, arg := range args {
|
|
builder.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n")
|
|
}
|
|
if _, err := c.conn.Write([]byte(builder.String())); err != nil {
|
|
return nil, err
|
|
}
|
|
return c.read()
|
|
}
|
|
|
|
func (c *redisConn) read() (any, error) {
|
|
line, err := c.reader.ReadString('\n')
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
line = strings.TrimRight(line, "\r\n")
|
|
if line == "" {
|
|
return nil, errors.New("empty redis reply")
|
|
}
|
|
switch line[0] {
|
|
case '+':
|
|
return line[1:], nil
|
|
case '-':
|
|
return nil, errors.New(line[1:])
|
|
case ':':
|
|
value, err := strconv.ParseInt(line[1:], 10, 64)
|
|
return value, err
|
|
case '$':
|
|
length, err := strconv.Atoi(line[1:])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if length < 0 {
|
|
return nil, nil
|
|
}
|
|
buf := make([]byte, length+2)
|
|
if _, err := ioReadFull(c.reader, buf); err != nil {
|
|
return nil, err
|
|
}
|
|
return string(buf[:length]), nil
|
|
case '*':
|
|
count, err := strconv.Atoi(line[1:])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items := make([]any, 0, count)
|
|
for i := 0; i < count; i++ {
|
|
item, err := c.read()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
return items, nil
|
|
default:
|
|
return nil, errors.New("unsupported redis reply")
|
|
}
|
|
}
|
|
|
|
func ioReadFull(reader *bufio.Reader, buf []byte) (int, error) {
|
|
total := 0
|
|
for total < len(buf) {
|
|
n, err := reader.Read(buf[total:])
|
|
total += n
|
|
if err != nil {
|
|
return total, err
|
|
}
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func (c *redisConn) scan(ctx context.Context, match string) ([]string, error) {
|
|
var keys []string
|
|
cursor := "0"
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
reply, err := c.do("SCAN", cursor, "MATCH", match, "COUNT", "1000")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, ok := reply.([]any)
|
|
if !ok || len(items) != 2 {
|
|
return nil, errors.New("unexpected SCAN reply")
|
|
}
|
|
next, _ := items[0].(string)
|
|
batch, _ := items[1].([]any)
|
|
for _, item := range batch {
|
|
if key, ok := item.(string); ok {
|
|
keys = append(keys, key)
|
|
}
|
|
}
|
|
if next == "0" || next == "" {
|
|
return keys, nil
|
|
}
|
|
cursor = next
|
|
}
|
|
}
|
|
|
|
func (c *redisConn) intCmd(ctx context.Context, cmd string, args ...string) (int64, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
reply, err := c.do(append([]string{cmd}, args...)...)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
value, ok := reply.(int64)
|
|
if !ok {
|
|
return 0, errors.New("unexpected integer reply")
|
|
}
|
|
return value, nil
|
|
}
|