fix(scripts): 演练判据改用认领与决策行并校验预备份复用

v4 判据(都带作业身份,且时间戳必须晚于杀点):

- `scripts/takeover_log_contract.py` 是唯一判据:认领行 + 决策行都要有本作业 id、
  target/backup 一致;只有汇总行、别的作业的认领行、杀之前的旧日志三种一律红;
  `--self-test` 11 条(正例 2 / 负例 9)进 `make verify`(新增第 16 项必需检查);
- `prebackup` 不再强制 `skipped`:真实接管是复用后重跑整条链路,阶段记 succeeded。
  但**不取消备份校验**,改为证明复用同一份——快照 id、清单字节与 mtime、
  版本行首次登记时刻都不变,且证据里有 `backup_reused=true`;
- 杀之前把预备份清单 sha256/mtime 与版本行登记时刻拍进 evidence;
- summary.json 增加 takeover_decision / claim_at / backup_registered_at。

verify: 判据自检 11 条全过;用真实 Go 输出行形状的正例日志 rc=0;旧形状(只有
汇总行)rc=2 且理由点名"汇总";`--check` 前置体检 13 项全 pass(含新增
preflight_log_contract);make verify overall success。
This commit is contained in:
姚章浩
2026-10-30 07:56:16 +08:00
parent 85b3596e0e
commit 8a3aa800a3
3 changed files with 404 additions and 17 deletions
+5
View File
@@ -251,6 +251,10 @@ def checks() -> list[Check]:
[py3, "scripts/ci_checks.py", "--internal-web-test"], env=env),
Check("contracts", "版本与变更记录契约(不许 SKIP;开发构建不许自称正式版)",
[py3, "scripts/ci_checks.py", "--internal-contracts", g], env=env),
# 演练判据本身也要被验:认领/决策行缺一条、只有汇总行、杀前日志都不许算通过。
# 2026-10-29 那次演练 exit 5 就是"脚本要的老文案只有一条代码路径会打印"。
Check("takeover-log-contract", "接管日志契约(认领/决策行判据 + 正负例自检)",
[py3, "scripts/takeover_log_contract.py", "--self-test"]),
Check("version", "CHANGELOG 与版本记录(四类小节无空栏目 + 正式 tag 与提交内版本一致)",
[py3, "scripts/check_changelog.py", "--tags"]),
Check("security-secrets", "密钥泄漏扫描(含扫描器自检)",
@@ -439,6 +443,7 @@ CHECK_MANIFEST: dict[str, tuple[str, ...]] = {
"tests-go": ("local", "ci"),
"tests-web": ("local", "ci"),
"contracts": ("local", "ci"),
"takeover-log-contract": ("local", "ci"),
"version": ("local", "ci"),
"security-secrets": ("local", "ci"),
"security-deps": ("local", "ci"),
+104 -17
View File
@@ -1,10 +1,21 @@
#!/usr/bin/env bash
# library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈)
#
# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被接管并
# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被认领并
# 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**,全成立才 exit 0。
# missed / notrun / timeout / 只同快照收尾 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。
#
# ⚠️ 2026-10-29T19:56Z 这次真演练(exit 5)教会两件事,都是判据缺陷:
# 1) 脚本要的文案 `library job <id> resumed after worker restart` 只有"新建队列行"
# 那条代码路径会打印;真实走的是"队列行已存在"那条。于是 worker 只留下汇总行
# `library job takeover sweep: 1 resumed`,脚本判不出是哪个作业、也判不出它是否被认领。
# 现在改由 scripts/takeover_log_contract.py 判**认领行 + 决策行**(都带作业 id 与
# target/backup,且时间戳必须晚于杀点);只有汇总行/别的作业/杀前日志一律不算证据。
# 2) 老判据要求 `prebackup:skipped`。真实接管是"复用崩溃前那份预备份重跑整条链路",
# 阶段会写成 succeeded(release.go 记的是"预备份 <id>")。**不能因此取消备份校验**:
# 现在要求预备份快照身份与崩溃前一致、清单文件字节/mtime 未变、版本行 created_at
# 仍是首次登记时刻(复用而非重做),写集合摘要也不变。
#
# 为什么不再用 `docker kill`:
# docker kill/stop 走 Docker API 的"手动停止",restart policy 不生效(`unless-stopped`
# 也不生效),容器就一直躺着。2026-10-29T19:16Z 的现场就是这样:脚本等了 420s 判失败,
@@ -16,7 +27,7 @@
#
# 命中点位仍是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入):
# worker 在指定阶段写完后写 <barrier>/after-database.hit 并停住 → 本脚本看到 .hit
# → SIGKILL → 按策略重启/显式拉起 → 启动扫描接管同一作业。
# → SIGKILL → 按策略重启/显式拉起 → 启动扫描认领同一作业。
# 屏障目录在本机归档根下(<archive>/barrier),没人放行就自行继续(记 not_run)。
#
# 期望内容按**目标快照自己的 manifest** 取(knowledges/chunks/wiki_pages),不是写死
@@ -33,12 +44,15 @@
# --snapshot <id>(校验命中的作业目标就是它)
# --job <业务作业 id>|--since <RFC3339|unix>
# --expect "knowledges=..,chunks=..,wiki_pages=.."(覆盖 manifest 推导)
# --expect-decision fresh|pending_final(默认 fresh;at after:database 之后
# 只能是 fresh——pending_final 等于"只差收尾",正是上一轮的假完成)
# --allow-same-snapshot(允许目标快照 == 中断前生效版本;这条**不能**当完整续作验收)
# --manual-start 0|1(策略未拉起时是否显式 docker start,默认 1)
#
set -uo pipefail
SCRIPT_VERSION=3
SCRIPT_VERSION=4
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODE=""
SCOPE="internal"
KIND="auto"
@@ -48,8 +62,10 @@ SINCE=""
CONFIRM_KILL=0
ALLOW_SAME_SNAPSHOT=0
MANUAL_START=1
EXPECT_DECISION="${EXPECT_DECISION:-fresh}"
EXPECT_COUNTS="${EXPECT_COUNTS:-}"
VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}"
LOG_CONTRACT="${TAKEOVER_LOG_CONTRACT:-$SCRIPT_DIR/takeover_log_contract.py}"
WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}"
GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}"
@@ -77,6 +93,7 @@ while [ $# -gt 0 ]; do
--job) EXPECT_JOB="${2:-}"; shift ;;
--since) SINCE="${2:-}"; shift ;;
--expect) EXPECT_COUNTS="${2:-}"; shift ;;
--expect-decision) EXPECT_DECISION="${2:-}"; shift ;;
--point) BARRIER_POINT="${2:-}"; shift ;;
--manual-start) MANUAL_START="${2:-}"; shift ;;
--allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;;
@@ -99,6 +116,10 @@ case "$MANUAL_START" in
0|1) ;;
*) echo "--manual-start must be 0|1" >&2; exit 64 ;;
esac
case "$EXPECT_DECISION" in
fresh|pending_final) ;;
*) echo "--expect-decision must be fresh|pending_final (got $EXPECT_DECISION)" >&2; exit 64 ;;
esac
# set -u:所有跨函数使用的变量先给默认值
WORKER_IMAGE_ID=""
@@ -129,6 +150,14 @@ WS_REUSED=""
WS_TARGET=""
WS_BACKUP=""
WS_FILE=""
WS_FILE_SHA=""
WS_BACKUP_SHA=""
WS_BACKUP_MTIME=""
WS_BACKUP_CREATED=""
WS_REUSED=""
DECISION=""
CLAIM_AT=""
LOG_RAW=""
KILLED_AT=""
KILL_PID=""
KILL_PROOF=""
@@ -241,6 +270,11 @@ finish() { # finish <exit code>
"killed_pid": "$KILL_PID",
"kill_proof": "$KILL_PROOF",
"restart_source": "$RESTART_SOURCE",
"takeover_decision": "$DECISION",
"takeover_claim_at": "$CLAIM_AT",
"expected_decision": "$EXPECT_DECISION",
"backup_snapshot": "$WS_BACKUP",
"backup_registered_at": "$WS_BACKUP_CREATED",
"writeset_digest": "$WS_DIGEST",
"evidence_dir": "$EVIDENCE"
}
@@ -352,6 +386,12 @@ preflight() {
fi
record preflight_verifier pass "verifier=$VERIFIER"
# 判据本身也要被验:脚本缺失或自检红,演练中途就只能靠人肉判断了。
[ -f "$LOG_CONTRACT" ] || die 2 preflight_log_contract "接管日志判据脚本不存在:$LOG_CONTRACT"
python3 "$LOG_CONTRACT" --self-test >>"$EVIDENCE/log-contract.log" 2>&1 || die 2 preflight_log_contract \
"接管日志判据自检没通过(见 log-contract.log):判据坏了不许当演练通过"
record preflight_log_contract pass "contract=$LOG_CONTRACT self-test=ok"
[ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \
"worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore"
record preflight_not_armed pass "barrier not armed"
@@ -518,19 +558,27 @@ lock_job() {
read_writeset_evidence() {
local row
row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','')||'|'||created_at::text||'|'||coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
[ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据"
IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row"
IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP WS_BACKUP_CREATED WS_REUSED <<<"$row"
[ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest:接管会失败,这次不算可接管"
[ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致"
WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json"
[ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE"
grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)"
grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID"
# 崩溃前那份预备份的身份与字节也拍下来:接管是否"复用同一份、没被重做覆盖"
# 只能靠事后比对(快照 id + 清单字节 + 清单 mtime + 版本行首次登记时刻)。
local backup_manifest
backup_manifest="$ARCHIVE_HOST/versions/$WS_BACKUP.json"
[ -s "$backup_manifest" ] || die 4 read_writeset "预备份快照清单不存在或为空:$backup_manifest"
WS_BACKUP_SHA="$(sha256sum "$backup_manifest" | awk '{print $1}')"
WS_BACKUP_MTIME="$(stat -c %Y "$backup_manifest")"
WS_FILE_SHA="$(sha256sum "$WS_FILE" | awk '{print $1}')"
# 独立校验器是必需的(preflight 已确认存在):它判定不符就不能继续。
"$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \
>>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)"
record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=verified"
record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP backup_manifest_sha=$WS_BACKUP_SHA backup_manifest_mtime=$WS_BACKUP_MTIME backup_registered_at=$WS_BACKUP_CREATED writeset_sha=$WS_FILE_SHA canonical=verified"
}
kill_worker() {
@@ -615,9 +663,9 @@ wait_restart() {
}
wait_takeover() {
local deadline state="" last_note=0
local deadline state="" last_note=0 out rc CONTRACT_OUT
deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S ))
log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…"
log "等待同一个作业被认领(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…"
while [ "$(now_epoch)" -lt "$deadline" ]; do
state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
case "$state" in succeeded|failed|cancelled) break ;; esac
@@ -634,11 +682,25 @@ wait_takeover() {
failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;;
*) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;;
esac
# 人工拉起/策略拉起都要能对得上"启动扫描真的接了这个作业"。
local resumed
resumed="$(docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | grep -c "library job $JOB_ID resumed after worker restart")"
[ "${resumed:-0}" -ge 1 ] || die 5 wait_takeover "worker 日志里没有本作业的 resume 记录:接管来源不明"
record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; resume_log=$resumed"
# 认领与决策都要有**本作业 id** 的行,且时间戳晚于杀点。
# 汇总行(`library job takeover sweep: 1 resumed`)不算证据:它既不说是哪个作业,
# 也不说是否真的被认领(别的 worker 抢走、attach 失败都只影响计数)。
LOG_RAW="$EVIDENCE/worker-log-raw.txt"
# -t 才有每行时间戳:判据要证明"这些行出现在杀之后",没有时间戳就只能判红。
docker logs -t --since "$KILLED_AT" "$WORKER_CONTAINER" >"$LOG_RAW" 2>&1
CONTRACT_OUT="$EVIDENCE/log-contract.log"
python3 "$LOG_CONTRACT" --log "$LOG_RAW" --job "$JOB_ID" --target "$JOB_TARGET" \
--backup "$WS_BACKUP" --since "$KILLED_AT" --expect-decision "$EXPECT_DECISION" \
>"$CONTRACT_OUT" 2>&1; rc=$?
out="$(cat "$CONTRACT_OUT")"
if [ "$rc" -ne 0 ]; then
die 5 wait_takeover "接管日志契约不成立(rc=$rc,见 log-contract.log):$(tr '\n' ' ' <"$CONTRACT_OUT" | cut -c1-400)"
fi
CLAIM_AT="$(sed -n 's/.*claim_at=\([^ ]*\).*/\1/p' <<<"$out" | head -1)"
DECISION="$(sed -n 's/.*decision=\([^ ]*\).*/\1/p' <<<"$out" | head -1)"
[ -n "$DECISION" ] || die 5 wait_takeover "判据脚本没给出决策(输出:$out)"
record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; claim_at=${CLAIM_AT:-?}; decision=$DECISION; contract=ok"
}
verify_stages_complete() {
@@ -653,13 +715,38 @@ verify_stages_complete() {
}
verify_takeover_semantics() {
local stages finalize_at eff after_digest
local stages pre_state finalize_at eff after_digest
local backup_now backup_created_now backup_sha_now backup_mtime_now pre_reused_now
stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")"
grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \
"阶段里没有 prebackup:skipped:无法证明走的是接管路径(stages=$stages)"
pre_state="$(gov_sql "select coalesce(state,'') from library_job_stages where job_id='$JOB_ID' and stage='prebackup'")"
# 接管复用崩溃前那份预备份:阶段既可能写成 skipped(只标记不重跑),也可能写成
# succeeded(release.go 记"预备份 <id>"——真实 2026-10-29 现场就是这种)。
# 两种都接受,但**不能因此取消备份校验**:必须证明复用的是同一份、没被重做覆盖。
case "$pre_state" in
skipped|succeeded) ;;
*) die 6 verify_takeover "prebackup 阶段是 ${pre_state:-缺失}:既不是复用(skipped)也不是复用后重跑(succeeded),接管路径不成立(stages=$stages)" ;;
esac
finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")"
[ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功(stages=$stages)"
record verify_takeover pass "stages=$stages finalize_at=$finalize_at"
backup_now="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
[ "$backup_now" = "$WS_BACKUP" ] || die 6 verify_takeover \
"预备份身份变了:崩溃前 $WS_BACKUP → 现在 ${backup_now:-空}(重做了备份 = 把半写状态当成可回滚基线)"
backup_created_now="$(gov_sql "select created_at::text from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
[ "$backup_created_now" = "$WS_BACKUP_CREATED" ] || die 6 verify_takeover \
"预备份版本行的首次登记时刻被改写:$WS_BACKUP_CREATED → $backup_created_now(复用被洗成重做)"
backup_sha_now="$(sha256sum "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null | awk '{print $1}')"
[ "$backup_sha_now" = "$WS_BACKUP_SHA" ] || die 6 verify_takeover \
"预备份清单字节变了:$WS_BACKUP_SHA → ${backup_sha_now:-缺失}(同一快照 id 下内容被重写)"
backup_mtime_now="$(stat -c %Y "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null)"
[ "$backup_mtime_now" = "$WS_BACKUP_MTIME" ] || die 6 verify_takeover \
"预备份清单文件被重新写过(mtime $WS_BACKUP_MTIME → ${backup_mtime_now:-缺失}):崩溃后重做了备份"
pre_reused_now="$(gov_sql "select coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
if [ "$pre_state" = "succeeded" ] && [ "$pre_reused_now" != "true" ]; then
die 6 verify_takeover \
"prebackup 记成 succeeded 但版本证据没有 backup_reused=true:分不清'复用'与'重做了备份'(evidence=$pre_reused_now)"
fi
record verify_takeover pass "stages=$stages decision=$DECISION finalize_at=$finalize_at prebackup=$pre_state backup=$backup_now registered_at=$backup_created_now manifest_sha=$backup_sha_now reused=${pre_reused_now:-no}"
eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")"
local eff_snapshot eff_job eff_superseded
+295
View File
@@ -0,0 +1,295 @@
#!/usr/bin/env python3
"""接管日志契约:从真实 worker 日志里证明"这个作业被认领了、并且做了哪个接管决策"。
为什么要有这份独立判据
----------------------
2026-10-29T19:56Z 的 `after:database` 演练里,worker 只打印了汇总行
`library job takeover sweep: 1 resumed, 0 failed without backup`:汇总既没说
是**哪个**作业,也没说它是不是真的被本进程认领(被别的 worker 抢走、attach 失败
都只影响计数)。演练脚本当时要求的老文案 `library job <id> resumed after worker restart`
只在"新建队列行"那条路径打印,真实走的却是"队列行已存在"那条 ⇒ 判据与实现各说一套,
演练 exit 5。这里把契约固化成可执行判据。
判据(三条缺一不可):
1. 一行 `library job <id> takeover claimed ... key=value ...`,且 target/backup 与本次作业一致;
2. 一行 `library job <id> takeover decision=<d> ...`,且 <d> 等于期望决策;
3. 两行的时间戳都 >= 杀点(取自 `docker logs -t` 的行首时间戳):杀之前的旧日志不算。
只接受**带本作业 id** 的行:汇总行(takeover sweep / 没有 id 的 resume 统计)一律不算证据;
别的作业的认领行也不算。
用法:
python3 scripts/takeover_log_contract.py --log worker.log --job <id> \
--target <快照> --backup <预备份快照> --since 2026-10-29T19:56:49Z
python3 scripts/takeover_log_contract.py --self-test # 正负例自检(门禁 CI 里跑)
退出码:0 成立 / 2 判据不成立(原因逐条打印)/ 3 日志读不到 / 64 参数不合法
"""
from __future__ import annotations
import argparse
import datetime as dt
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
LINE_TS = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z) (.*)$")
# Go 的 log 默认带 LstdFlags(`2026/10/29 19:57:21 `),`docker logs -t` 又在最前面加
# 自己的 RFC3339 时间戳 ⇒ 行体里还有一层"年/月/日 时:分:秒"。判据只认行首那一个。
GO_TS = re.compile(r"^\d{4}/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d+)? ")
CLAIM_PREFIX = "library job {job} takeover claimed "
DECISION_PREFIX = "library job {job} takeover decision="
ANY_CLAIM = re.compile(r"library job (\S+) takeover claimed ")
ANY_DECISION = re.compile(r"library job (\S+) takeover decision=(\S+)")
REQUIRED_CLAIM_KEYS = ("at", "kind", "scope", "target", "backup", "queue", "path")
TAKEOVER_DECISIONS = ("fresh", "pending_final", "superseded", "foreign_effective", "unknown")
def parse_ts(value: str) -> dt.datetime | None:
"""解析 `docker logs -t` 的 RFC3339(可能带纳秒)时间戳;解析不了就是 None。"""
text = value.strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
if "." in text:
head, tail = text.split(".", 1)
# 纳秒 → 微秒:Python 只认 6 位,直接截断(比较用,不需要更高精度)
frac = re.match(r"(\d+)", tail)
zone = tail[frac.end():] if frac else ""
text = head + "." + (frac.group(1)[:6].ljust(6, "0") if frac else "000000") + zone
try:
parsed = dt.datetime.fromisoformat(text)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.timezone.utc)
def key_values(body: str) -> dict[str, str]:
"""按 key=value 取字段(不按位置猜):Go 侧改字段名会让演练直接红,而不是静默放过。"""
out: dict[str, str] = {}
for token in body.split():
if "=" in token:
key, _, value = token.partition("=")
if key and value:
out[key] = value
return out
@dataclass
class Result:
ok: bool
reasons: list[str] = field(default_factory=list)
info: dict[str, str] = field(default_factory=dict)
@dataclass
class Entry:
ts: dt.datetime | None
body: str
def split_lines(lines: list[str]) -> list[Entry]:
entries: list[Entry] = []
for raw in lines:
line = raw.rstrip("\n")
if not line.strip():
continue
match = LINE_TS.match(line)
if match:
entries.append(Entry(parse_ts(match.group(1)), GO_TS.sub("", match.group(2), count=1)))
else:
# 没有 docker -t 前缀就无法证明"这是杀之后的日志":仍然保留(去掉内层
# Go 时间戳后照常识别),由判据显式判红,而不是当成通过。
entries.append(Entry(None, GO_TS.sub("", line, count=1)))
return entries
def evaluate(lines: list[str], job: str, target: str, backup: str, since: dt.datetime,
expect_decision: str = "fresh") -> Result:
entries = split_lines(lines)
claim_prefix = CLAIM_PREFIX.format(job=job)
decision_prefix = DECISION_PREFIX.format(job=job)
claims = [e for e in entries if e.body.startswith(claim_prefix)]
decisions = [e for e in entries if e.body.startswith(decision_prefix)]
after = lambda e: e.ts is not None and e.ts >= since # noqa: E731
fresh_claims = [e for e in claims if after(e)]
fresh_decisions = [e for e in decisions if after(e)]
aggregate = [e for e in entries if "takeover sweep" in e.body or re.search(
r"library job resume: \d+ resumed", e.body)]
other_claims = [(m.group(1), e) for e in entries for m in [ANY_CLAIM.search(e.body)]
if m and m.group(1) != job]
other_decisions = [(m.group(1), m.group(2), e) for e in entries
for m in [ANY_DECISION.search(e.body)] if m and m.group(1) != job]
reasons: list[str] = []
info: dict[str, str] = {}
if not fresh_claims:
if claims:
reasons.append("本作业的认领行全都出现在杀点 %s 之前:杀之前的日志不能证明这次接管"
% since.isoformat())
elif other_claims:
reasons.append("只有别的作业的认领行(%s):不是本作业被认领"
% ", ".join(sorted({j for j, _ in other_claims})))
elif aggregate:
reasons.append("只看到汇总行(%s):汇总既不说哪个作业,也不说它是否被认领"
% aggregate[0].body.strip()[:120])
else:
reasons.append("日志里没有本作业的认领行 `%s…`" % claim_prefix.strip())
else:
kv = key_values(fresh_claims[-1].body)
info["claim_at"] = kv.get("at", "")
info["claim_path"] = kv.get("path", "")
info["claim_queue"] = kv.get("queue", "")
missing = [k for k in REQUIRED_CLAIM_KEYS if not kv.get(k)]
if missing:
reasons.append("认领行缺字段 %s(key=value 契约):%s"
% (",".join(missing), fresh_claims[-1].body.strip()))
if kv.get("target") and kv["target"] != target:
reasons.append("认领行的 target=%s 不是本次目标 %s(别的作业/别的快照的认领)"
% (kv["target"], target))
if kv.get("backup") and backup and kv["backup"] != backup:
reasons.append("认领行的 backup=%s 不是崩溃前登记的那份预备份 %s(换了基线)"
% (kv["backup"], backup))
if not fresh_decisions:
if decisions:
reasons.append("本作业的决策行全都出现在杀点之前:不能证明这次接管的决策")
else:
reasons.append("日志里没有本作业的接管决策行 `%s<decision>…`" % decision_prefix)
else:
decision = fresh_decisions[-1].body[len(decision_prefix):].split()[0]
info["decision"] = decision
if decision not in TAKEOVER_DECISIONS:
reasons.append("未知的接管决策 %r(期望 %s)" % (decision, expect_decision))
elif decision != expect_decision:
reasons.append("接管决策是 %s,期望 %s(在 after:database 之后不可能是"
"「只差收尾」,同快照收尾=上一轮的假完成)" % (decision, expect_decision))
kv = key_values(fresh_decisions[-1].body)
if kv.get("target") and kv["target"] != target:
reasons.append("决策行的 target=%s 不是本次目标 %s" % (kv["target"], target))
return Result(ok=not reasons, reasons=reasons, info=info)
def read_log(path: Path) -> list[str] | None:
if not path.is_file():
return None
return path.read_text(encoding="utf-8", errors="replace").splitlines()
# ── 自检:正例必须先绿,负例必须逐条红(否则判据是死的)──────────────────
FIXTURE_JOB = "librestore-internal-1793303791259342087"
FIXTURE_TARGET = "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7"
FIXTURE_BACKUP = "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510"
FIXTURE_SINCE = "2026-10-29T19:56:49Z"
FIXTURE_AGGREGATE = ("2026-10-29T19:57:20.123456789Z 2026/10/29 19:57:20 "
"library job takeover sweep: 1 resumed, 0 failed without backup")
FIXTURE_CLAIM = ("2026-10-29T19:57:20.500000000Z 2026/10/29 19:57:20 library job %s takeover claimed "
"at=2026-10-29T19:57:20Z kind=restore scope=internal target=%s backup=%s "
"queue=q65e9fd4d932b262659cdb0dcbedb92ce path=existing-queue"
% (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP))
FIXTURE_DECISION = ("2026-10-29T19:57:22.000000000Z 2026/10/29 19:57:22 library job %s takeover decision=fresh "
"at=2026-10-29T19:57:22Z kind=restore scope=internal target=%s backup=%s "
"finalize_only=false" % (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP))
PRE_KILL_CLAIM = FIXTURE_CLAIM.replace("2026-10-29T19:57:20.500000000Z", "2026-10-29T19:56:40.000000000Z")
PRE_KILL_DECISION = FIXTURE_DECISION.replace("2026-10-29T19:57:22.000000000Z", "2026-10-29T19:56:41.000000000Z")
def self_test() -> int:
since = parse_ts(FIXTURE_SINCE)
assert since is not None
cases: list[tuple[str, list[str], bool, str, str]] = [
("正例:认领+决策都在杀之后", [FIXTURE_AGGREGATE, FIXTURE_CLAIM, FIXTURE_DECISION],
True, "fresh", ""),
("负例:只有汇总行(上一轮的真实形状)", [FIXTURE_AGGREGATE, FIXTURE_DECISION],
False, "fresh", "汇总"),
("负例:认领属于别的作业",
[FIXTURE_CLAIM.replace(FIXTURE_JOB, "libsync-internal-other"), FIXTURE_DECISION],
False, "fresh", "别的作业"),
("负例:认领/决策都在杀之前(杀前日志伪通过)", [PRE_KILL_CLAIM, PRE_KILL_DECISION],
False, "fresh", "杀点"),
("负例:有认领没决策", [FIXTURE_CLAIM], False, "fresh", "决策行"),
("负例:有决策没认领", [FIXTURE_DECISION], False, "fresh", "认领行"),
("负例:认领的 target 不是本次目标",
[FIXTURE_CLAIM.replace(FIXTURE_TARGET, "fc595511c7388d8e6cab3de37af8a2b6a64ede47241918684e81f2fb5d0e1f16"),
FIXTURE_DECISION], False, "fresh", "target="),
("负例:认领的 backup 换了(重做了备份)",
[FIXTURE_CLAIM.replace(FIXTURE_BACKUP, "0fea01923a7c6073d06d7310588e737359f44ecee5ab1fd8f5892eba8e208b63"),
FIXTURE_DECISION], False, "fresh", "backup="),
("负例:决策是 pending_final(同快照收尾)",
[FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")],
False, "fresh", "pending_final"),
("负例:没有 docker -t 时间戳(无法证明在杀之后)",
[FIXTURE_CLAIM.split(" ", 1)[1], FIXTURE_DECISION.split(" ", 1)[1]],
False, "fresh", "杀点"),
("正例:期望决策可显式放宽为 pending_final",
[FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")],
True, "pending_final", ""),
]
failed = 0
for name, lines, want_ok, expect, want_reason in cases:
result = evaluate(lines, FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP, since, expect)
if result.ok != want_ok:
print("❌ %s:期望 ok=%s,实际 ok=%s(%s)"
% (name, want_ok, result.ok, ";".join(result.reasons)), file=sys.stderr)
failed += 1
continue
if not want_ok and want_reason not in ";".join(result.reasons):
print("❌ %s:红了但理由不对(期望含 %r,实际 %r)"
% (name, want_reason, result.reasons), file=sys.stderr)
failed += 1
continue
if want_ok and want_reason:
print("❌ 自检夹具写错了:正例不该要求失败理由 %r" % want_reason, file=sys.stderr)
failed += 1
if failed:
print("❌ 接管日志契约自检:%d 条不成立" % failed, file=sys.stderr)
return 1
print("✅ 接管日志契约自检通过:%d 条(正例 %d / 负例 %d),负例逐条红"
% (len(cases), sum(1 for c in cases if c[2]), sum(1 for c in cases if not c[2])))
return 0
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description="接管日志契约判据")
parser.add_argument("--log", type=Path, help="docker logs -t 的输出")
parser.add_argument("--job", help="业务作业 id")
parser.add_argument("--target", help="目标快照(作业的 target_snapshot_id)")
parser.add_argument("--backup", default="", help="崩溃前登记的预备份快照(可省略)")
parser.add_argument("--since", help="杀点时刻(RFC3339);早于它的日志不算证据")
parser.add_argument("--expect-decision", default="fresh",
choices=TAKEOVER_DECISIONS, help="期望的接管决策(默认 fresh)")
parser.add_argument("--self-test", action="store_true", help="只跑正负例自检")
args = parser.parse_args(argv)
if args.self_test:
return self_test()
if not (args.log and args.job and args.target and args.since):
print("必须给 --log/--job/--target/--since(或用 --self-test)", file=sys.stderr)
return 64
since = parse_ts(args.since)
if since is None:
print("--since 不是 RFC3339:%r" % args.since, file=sys.stderr)
return 64
lines = read_log(args.log)
if lines is None:
print("读不到日志:%s" % args.log, file=sys.stderr)
return 3
result = evaluate(lines, args.job, args.target, args.backup, since, args.expect_decision)
if not result.ok:
for reason in result.reasons:
print("❌ " + reason, file=sys.stderr)
return 2
print("OK claim_at=%s path=%s queue=%s decision=%s target=%s backup=%s"
% (result.info.get("claim_at", "?"), result.info.get("claim_path", "?"),
result.info.get("claim_queue", "?"), result.info.get("decision", "?"),
args.target, args.backup or "?"))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))