fix(scripts): 演练判据改用认领与决策行并校验预备份复用
v4 判据(都带作业身份,且时间戳必须晚于杀点): - `scripts/takeover_log_contract.py` 是唯一判据:认领行 + 决策行都要有本作业 id、 target/backup 一致;只有汇总行、别的作业的认领行、杀之前的旧日志三种一律红; `--self-test` 11 条(正例 2 / 负例 9)进 `make verify`(新增第 16 项必需检查); - `prebackup` 不再强制 `skipped`:真实接管是复用后重跑整条链路,阶段记 succeeded。 但**不取消备份校验**,改为证明复用同一份——快照 id、清单字节与 mtime、 版本行首次登记时刻都不变,且证据里有 `backup_reused=true`; - 杀之前把预备份清单 sha256/mtime 与版本行登记时刻拍进 evidence; - summary.json 增加 takeover_decision / claim_at / backup_registered_at。 verify: 判据自检 11 条全过;用真实 Go 输出行形状的正例日志 rc=0;旧形状(只有 汇总行)rc=2 且理由点名"汇总";`--check` 前置体检 13 项全 pass(含新增 preflight_log_contract);make verify overall success。
This commit is contained in:
@@ -251,6 +251,10 @@ def checks() -> list[Check]:
|
||||
[py3, "scripts/ci_checks.py", "--internal-web-test"], env=env),
|
||||
Check("contracts", "版本与变更记录契约(不许 SKIP;开发构建不许自称正式版)",
|
||||
[py3, "scripts/ci_checks.py", "--internal-contracts", g], env=env),
|
||||
# 演练判据本身也要被验:认领/决策行缺一条、只有汇总行、杀前日志都不许算通过。
|
||||
# 2026-10-29 那次演练 exit 5 就是"脚本要的老文案只有一条代码路径会打印"。
|
||||
Check("takeover-log-contract", "接管日志契约(认领/决策行判据 + 正负例自检)",
|
||||
[py3, "scripts/takeover_log_contract.py", "--self-test"]),
|
||||
Check("version", "CHANGELOG 与版本记录(四类小节无空栏目 + 正式 tag 与提交内版本一致)",
|
||||
[py3, "scripts/check_changelog.py", "--tags"]),
|
||||
Check("security-secrets", "密钥泄漏扫描(含扫描器自检)",
|
||||
@@ -439,6 +443,7 @@ CHECK_MANIFEST: dict[str, tuple[str, ...]] = {
|
||||
"tests-go": ("local", "ci"),
|
||||
"tests-web": ("local", "ci"),
|
||||
"contracts": ("local", "ci"),
|
||||
"takeover-log-contract": ("local", "ci"),
|
||||
"version": ("local", "ci"),
|
||||
"security-secrets": ("local", "ci"),
|
||||
"security-deps": ("local", "ci"),
|
||||
|
||||
@@ -1,10 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈)
|
||||
#
|
||||
# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被接管并
|
||||
# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被认领并
|
||||
# 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**,全成立才 exit 0。
|
||||
# missed / notrun / timeout / 只同快照收尾 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。
|
||||
#
|
||||
# ⚠️ 2026-10-29T19:56Z 这次真演练(exit 5)教会两件事,都是判据缺陷:
|
||||
# 1) 脚本要的文案 `library job <id> resumed after worker restart` 只有"新建队列行"
|
||||
# 那条代码路径会打印;真实走的是"队列行已存在"那条。于是 worker 只留下汇总行
|
||||
# `library job takeover sweep: 1 resumed`,脚本判不出是哪个作业、也判不出它是否被认领。
|
||||
# 现在改由 scripts/takeover_log_contract.py 判**认领行 + 决策行**(都带作业 id 与
|
||||
# target/backup,且时间戳必须晚于杀点);只有汇总行/别的作业/杀前日志一律不算证据。
|
||||
# 2) 老判据要求 `prebackup:skipped`。真实接管是"复用崩溃前那份预备份重跑整条链路",
|
||||
# 阶段会写成 succeeded(release.go 记的是"预备份 <id>")。**不能因此取消备份校验**:
|
||||
# 现在要求预备份快照身份与崩溃前一致、清单文件字节/mtime 未变、版本行 created_at
|
||||
# 仍是首次登记时刻(复用而非重做),写集合摘要也不变。
|
||||
#
|
||||
# 为什么不再用 `docker kill`:
|
||||
# docker kill/stop 走 Docker API 的"手动停止",restart policy 不生效(`unless-stopped`
|
||||
# 也不生效),容器就一直躺着。2026-10-29T19:16Z 的现场就是这样:脚本等了 420s 判失败,
|
||||
@@ -16,7 +27,7 @@
|
||||
#
|
||||
# 命中点位仍是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入):
|
||||
# worker 在指定阶段写完后写 <barrier>/after-database.hit 并停住 → 本脚本看到 .hit
|
||||
# → SIGKILL → 按策略重启/显式拉起 → 启动扫描接管同一作业。
|
||||
# → SIGKILL → 按策略重启/显式拉起 → 启动扫描认领同一作业。
|
||||
# 屏障目录在本机归档根下(<archive>/barrier),没人放行就自行继续(记 not_run)。
|
||||
#
|
||||
# 期望内容按**目标快照自己的 manifest** 取(knowledges/chunks/wiki_pages),不是写死
|
||||
@@ -33,12 +44,15 @@
|
||||
# --snapshot <id>(校验命中的作业目标就是它)
|
||||
# --job <业务作业 id>|--since <RFC3339|unix>
|
||||
# --expect "knowledges=..,chunks=..,wiki_pages=.."(覆盖 manifest 推导)
|
||||
# --expect-decision fresh|pending_final(默认 fresh;at after:database 之后
|
||||
# 只能是 fresh——pending_final 等于"只差收尾",正是上一轮的假完成)
|
||||
# --allow-same-snapshot(允许目标快照 == 中断前生效版本;这条**不能**当完整续作验收)
|
||||
# --manual-start 0|1(策略未拉起时是否显式 docker start,默认 1)
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_VERSION=3
|
||||
SCRIPT_VERSION=4
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MODE=""
|
||||
SCOPE="internal"
|
||||
KIND="auto"
|
||||
@@ -48,8 +62,10 @@ SINCE=""
|
||||
CONFIRM_KILL=0
|
||||
ALLOW_SAME_SNAPSHOT=0
|
||||
MANUAL_START=1
|
||||
EXPECT_DECISION="${EXPECT_DECISION:-fresh}"
|
||||
EXPECT_COUNTS="${EXPECT_COUNTS:-}"
|
||||
VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}"
|
||||
LOG_CONTRACT="${TAKEOVER_LOG_CONTRACT:-$SCRIPT_DIR/takeover_log_contract.py}"
|
||||
|
||||
WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}"
|
||||
GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}"
|
||||
@@ -77,6 +93,7 @@ while [ $# -gt 0 ]; do
|
||||
--job) EXPECT_JOB="${2:-}"; shift ;;
|
||||
--since) SINCE="${2:-}"; shift ;;
|
||||
--expect) EXPECT_COUNTS="${2:-}"; shift ;;
|
||||
--expect-decision) EXPECT_DECISION="${2:-}"; shift ;;
|
||||
--point) BARRIER_POINT="${2:-}"; shift ;;
|
||||
--manual-start) MANUAL_START="${2:-}"; shift ;;
|
||||
--allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;;
|
||||
@@ -99,6 +116,10 @@ case "$MANUAL_START" in
|
||||
0|1) ;;
|
||||
*) echo "--manual-start must be 0|1" >&2; exit 64 ;;
|
||||
esac
|
||||
case "$EXPECT_DECISION" in
|
||||
fresh|pending_final) ;;
|
||||
*) echo "--expect-decision must be fresh|pending_final (got $EXPECT_DECISION)" >&2; exit 64 ;;
|
||||
esac
|
||||
|
||||
# set -u:所有跨函数使用的变量先给默认值
|
||||
WORKER_IMAGE_ID=""
|
||||
@@ -129,6 +150,14 @@ WS_REUSED=""
|
||||
WS_TARGET=""
|
||||
WS_BACKUP=""
|
||||
WS_FILE=""
|
||||
WS_FILE_SHA=""
|
||||
WS_BACKUP_SHA=""
|
||||
WS_BACKUP_MTIME=""
|
||||
WS_BACKUP_CREATED=""
|
||||
WS_REUSED=""
|
||||
DECISION=""
|
||||
CLAIM_AT=""
|
||||
LOG_RAW=""
|
||||
KILLED_AT=""
|
||||
KILL_PID=""
|
||||
KILL_PROOF=""
|
||||
@@ -241,6 +270,11 @@ finish() { # finish <exit code>
|
||||
"killed_pid": "$KILL_PID",
|
||||
"kill_proof": "$KILL_PROOF",
|
||||
"restart_source": "$RESTART_SOURCE",
|
||||
"takeover_decision": "$DECISION",
|
||||
"takeover_claim_at": "$CLAIM_AT",
|
||||
"expected_decision": "$EXPECT_DECISION",
|
||||
"backup_snapshot": "$WS_BACKUP",
|
||||
"backup_registered_at": "$WS_BACKUP_CREATED",
|
||||
"writeset_digest": "$WS_DIGEST",
|
||||
"evidence_dir": "$EVIDENCE"
|
||||
}
|
||||
@@ -352,6 +386,12 @@ preflight() {
|
||||
fi
|
||||
record preflight_verifier pass "verifier=$VERIFIER"
|
||||
|
||||
# 判据本身也要被验:脚本缺失或自检红,演练中途就只能靠人肉判断了。
|
||||
[ -f "$LOG_CONTRACT" ] || die 2 preflight_log_contract "接管日志判据脚本不存在:$LOG_CONTRACT"
|
||||
python3 "$LOG_CONTRACT" --self-test >>"$EVIDENCE/log-contract.log" 2>&1 || die 2 preflight_log_contract \
|
||||
"接管日志判据自检没通过(见 log-contract.log):判据坏了不许当演练通过"
|
||||
record preflight_log_contract pass "contract=$LOG_CONTRACT self-test=ok"
|
||||
|
||||
[ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \
|
||||
"worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore"
|
||||
record preflight_not_armed pass "barrier not armed"
|
||||
@@ -518,19 +558,27 @@ lock_job() {
|
||||
|
||||
read_writeset_evidence() {
|
||||
local row
|
||||
row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
|
||||
row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','')||'|'||created_at::text||'|'||coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
|
||||
[ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据"
|
||||
IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row"
|
||||
IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP WS_BACKUP_CREATED WS_REUSED <<<"$row"
|
||||
[ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest:接管会失败,这次不算可接管"
|
||||
[ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致"
|
||||
WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json"
|
||||
[ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE"
|
||||
grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)"
|
||||
grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID"
|
||||
# 崩溃前那份预备份的身份与字节也拍下来:接管是否"复用同一份、没被重做覆盖"
|
||||
# 只能靠事后比对(快照 id + 清单字节 + 清单 mtime + 版本行首次登记时刻)。
|
||||
local backup_manifest
|
||||
backup_manifest="$ARCHIVE_HOST/versions/$WS_BACKUP.json"
|
||||
[ -s "$backup_manifest" ] || die 4 read_writeset "预备份快照清单不存在或为空:$backup_manifest"
|
||||
WS_BACKUP_SHA="$(sha256sum "$backup_manifest" | awk '{print $1}')"
|
||||
WS_BACKUP_MTIME="$(stat -c %Y "$backup_manifest")"
|
||||
WS_FILE_SHA="$(sha256sum "$WS_FILE" | awk '{print $1}')"
|
||||
# 独立校验器是必需的(preflight 已确认存在):它判定不符就不能继续。
|
||||
"$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \
|
||||
>>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)"
|
||||
record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=verified"
|
||||
record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP backup_manifest_sha=$WS_BACKUP_SHA backup_manifest_mtime=$WS_BACKUP_MTIME backup_registered_at=$WS_BACKUP_CREATED writeset_sha=$WS_FILE_SHA canonical=verified"
|
||||
}
|
||||
|
||||
kill_worker() {
|
||||
@@ -615,9 +663,9 @@ wait_restart() {
|
||||
}
|
||||
|
||||
wait_takeover() {
|
||||
local deadline state="" last_note=0
|
||||
local deadline state="" last_note=0 out rc CONTRACT_OUT
|
||||
deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S ))
|
||||
log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…"
|
||||
log "等待同一个作业被认领(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…"
|
||||
while [ "$(now_epoch)" -lt "$deadline" ]; do
|
||||
state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
|
||||
case "$state" in succeeded|failed|cancelled) break ;; esac
|
||||
@@ -634,11 +682,25 @@ wait_takeover() {
|
||||
failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;;
|
||||
*) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;;
|
||||
esac
|
||||
# 人工拉起/策略拉起都要能对得上"启动扫描真的接了这个作业"。
|
||||
local resumed
|
||||
resumed="$(docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | grep -c "library job $JOB_ID resumed after worker restart")"
|
||||
[ "${resumed:-0}" -ge 1 ] || die 5 wait_takeover "worker 日志里没有本作业的 resume 记录:接管来源不明"
|
||||
record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; resume_log=$resumed"
|
||||
|
||||
# 认领与决策都要有**本作业 id** 的行,且时间戳晚于杀点。
|
||||
# 汇总行(`library job takeover sweep: 1 resumed`)不算证据:它既不说是哪个作业,
|
||||
# 也不说是否真的被认领(别的 worker 抢走、attach 失败都只影响计数)。
|
||||
LOG_RAW="$EVIDENCE/worker-log-raw.txt"
|
||||
# -t 才有每行时间戳:判据要证明"这些行出现在杀之后",没有时间戳就只能判红。
|
||||
docker logs -t --since "$KILLED_AT" "$WORKER_CONTAINER" >"$LOG_RAW" 2>&1
|
||||
CONTRACT_OUT="$EVIDENCE/log-contract.log"
|
||||
python3 "$LOG_CONTRACT" --log "$LOG_RAW" --job "$JOB_ID" --target "$JOB_TARGET" \
|
||||
--backup "$WS_BACKUP" --since "$KILLED_AT" --expect-decision "$EXPECT_DECISION" \
|
||||
>"$CONTRACT_OUT" 2>&1; rc=$?
|
||||
out="$(cat "$CONTRACT_OUT")"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
die 5 wait_takeover "接管日志契约不成立(rc=$rc,见 log-contract.log):$(tr '\n' ' ' <"$CONTRACT_OUT" | cut -c1-400)"
|
||||
fi
|
||||
CLAIM_AT="$(sed -n 's/.*claim_at=\([^ ]*\).*/\1/p' <<<"$out" | head -1)"
|
||||
DECISION="$(sed -n 's/.*decision=\([^ ]*\).*/\1/p' <<<"$out" | head -1)"
|
||||
[ -n "$DECISION" ] || die 5 wait_takeover "判据脚本没给出决策(输出:$out)"
|
||||
record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; claim_at=${CLAIM_AT:-?}; decision=$DECISION; contract=ok"
|
||||
}
|
||||
|
||||
verify_stages_complete() {
|
||||
@@ -653,13 +715,38 @@ verify_stages_complete() {
|
||||
}
|
||||
|
||||
verify_takeover_semantics() {
|
||||
local stages finalize_at eff after_digest
|
||||
local stages pre_state finalize_at eff after_digest
|
||||
local backup_now backup_created_now backup_sha_now backup_mtime_now pre_reused_now
|
||||
stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")"
|
||||
grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \
|
||||
"阶段里没有 prebackup:skipped:无法证明走的是接管路径(stages=$stages)"
|
||||
pre_state="$(gov_sql "select coalesce(state,'') from library_job_stages where job_id='$JOB_ID' and stage='prebackup'")"
|
||||
# 接管复用崩溃前那份预备份:阶段既可能写成 skipped(只标记不重跑),也可能写成
|
||||
# succeeded(release.go 记"预备份 <id>"——真实 2026-10-29 现场就是这种)。
|
||||
# 两种都接受,但**不能因此取消备份校验**:必须证明复用的是同一份、没被重做覆盖。
|
||||
case "$pre_state" in
|
||||
skipped|succeeded) ;;
|
||||
*) die 6 verify_takeover "prebackup 阶段是 ${pre_state:-缺失}:既不是复用(skipped)也不是复用后重跑(succeeded),接管路径不成立(stages=$stages)" ;;
|
||||
esac
|
||||
finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")"
|
||||
[ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功(stages=$stages)"
|
||||
record verify_takeover pass "stages=$stages finalize_at=$finalize_at"
|
||||
|
||||
backup_now="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
|
||||
[ "$backup_now" = "$WS_BACKUP" ] || die 6 verify_takeover \
|
||||
"预备份身份变了:崩溃前 $WS_BACKUP → 现在 ${backup_now:-空}(重做了备份 = 把半写状态当成可回滚基线)"
|
||||
backup_created_now="$(gov_sql "select created_at::text from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
|
||||
[ "$backup_created_now" = "$WS_BACKUP_CREATED" ] || die 6 verify_takeover \
|
||||
"预备份版本行的首次登记时刻被改写:$WS_BACKUP_CREATED → $backup_created_now(复用被洗成重做)"
|
||||
backup_sha_now="$(sha256sum "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null | awk '{print $1}')"
|
||||
[ "$backup_sha_now" = "$WS_BACKUP_SHA" ] || die 6 verify_takeover \
|
||||
"预备份清单字节变了:$WS_BACKUP_SHA → ${backup_sha_now:-缺失}(同一快照 id 下内容被重写)"
|
||||
backup_mtime_now="$(stat -c %Y "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null)"
|
||||
[ "$backup_mtime_now" = "$WS_BACKUP_MTIME" ] || die 6 verify_takeover \
|
||||
"预备份清单文件被重新写过(mtime $WS_BACKUP_MTIME → ${backup_mtime_now:-缺失}):崩溃后重做了备份"
|
||||
pre_reused_now="$(gov_sql "select coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
|
||||
if [ "$pre_state" = "succeeded" ] && [ "$pre_reused_now" != "true" ]; then
|
||||
die 6 verify_takeover \
|
||||
"prebackup 记成 succeeded 但版本证据没有 backup_reused=true:分不清'复用'与'重做了备份'(evidence=$pre_reused_now)"
|
||||
fi
|
||||
record verify_takeover pass "stages=$stages decision=$DECISION finalize_at=$finalize_at prebackup=$pre_state backup=$backup_now registered_at=$backup_created_now manifest_sha=$backup_sha_now reused=${pre_reused_now:-no}"
|
||||
|
||||
eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")"
|
||||
local eff_snapshot eff_job eff_superseded
|
||||
|
||||
Executable
+295
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""接管日志契约:从真实 worker 日志里证明"这个作业被认领了、并且做了哪个接管决策"。
|
||||
|
||||
为什么要有这份独立判据
|
||||
----------------------
|
||||
2026-10-29T19:56Z 的 `after:database` 演练里,worker 只打印了汇总行
|
||||
`library job takeover sweep: 1 resumed, 0 failed without backup`:汇总既没说
|
||||
是**哪个**作业,也没说它是不是真的被本进程认领(被别的 worker 抢走、attach 失败
|
||||
都只影响计数)。演练脚本当时要求的老文案 `library job <id> resumed after worker restart`
|
||||
只在"新建队列行"那条路径打印,真实走的却是"队列行已存在"那条 ⇒ 判据与实现各说一套,
|
||||
演练 exit 5。这里把契约固化成可执行判据。
|
||||
|
||||
判据(三条缺一不可):
|
||||
1. 一行 `library job <id> takeover claimed ... key=value ...`,且 target/backup 与本次作业一致;
|
||||
2. 一行 `library job <id> takeover decision=<d> ...`,且 <d> 等于期望决策;
|
||||
3. 两行的时间戳都 >= 杀点(取自 `docker logs -t` 的行首时间戳):杀之前的旧日志不算。
|
||||
只接受**带本作业 id** 的行:汇总行(takeover sweep / 没有 id 的 resume 统计)一律不算证据;
|
||||
别的作业的认领行也不算。
|
||||
|
||||
用法:
|
||||
python3 scripts/takeover_log_contract.py --log worker.log --job <id> \
|
||||
--target <快照> --backup <预备份快照> --since 2026-10-29T19:56:49Z
|
||||
python3 scripts/takeover_log_contract.py --self-test # 正负例自检(门禁 CI 里跑)
|
||||
退出码:0 成立 / 2 判据不成立(原因逐条打印)/ 3 日志读不到 / 64 参数不合法
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
LINE_TS = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z) (.*)$")
|
||||
# Go 的 log 默认带 LstdFlags(`2026/10/29 19:57:21 `),`docker logs -t` 又在最前面加
|
||||
# 自己的 RFC3339 时间戳 ⇒ 行体里还有一层"年/月/日 时:分:秒"。判据只认行首那一个。
|
||||
GO_TS = re.compile(r"^\d{4}/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d+)? ")
|
||||
CLAIM_PREFIX = "library job {job} takeover claimed "
|
||||
DECISION_PREFIX = "library job {job} takeover decision="
|
||||
ANY_CLAIM = re.compile(r"library job (\S+) takeover claimed ")
|
||||
ANY_DECISION = re.compile(r"library job (\S+) takeover decision=(\S+)")
|
||||
|
||||
REQUIRED_CLAIM_KEYS = ("at", "kind", "scope", "target", "backup", "queue", "path")
|
||||
TAKEOVER_DECISIONS = ("fresh", "pending_final", "superseded", "foreign_effective", "unknown")
|
||||
|
||||
|
||||
def parse_ts(value: str) -> dt.datetime | None:
|
||||
"""解析 `docker logs -t` 的 RFC3339(可能带纳秒)时间戳;解析不了就是 None。"""
|
||||
text = value.strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
if "." in text:
|
||||
head, tail = text.split(".", 1)
|
||||
# 纳秒 → 微秒:Python 只认 6 位,直接截断(比较用,不需要更高精度)
|
||||
frac = re.match(r"(\d+)", tail)
|
||||
zone = tail[frac.end():] if frac else ""
|
||||
text = head + "." + (frac.group(1)[:6].ljust(6, "0") if frac else "000000") + zone
|
||||
try:
|
||||
parsed = dt.datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.timezone.utc)
|
||||
|
||||
|
||||
def key_values(body: str) -> dict[str, str]:
|
||||
"""按 key=value 取字段(不按位置猜):Go 侧改字段名会让演练直接红,而不是静默放过。"""
|
||||
out: dict[str, str] = {}
|
||||
for token in body.split():
|
||||
if "=" in token:
|
||||
key, _, value = token.partition("=")
|
||||
if key and value:
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
ok: bool
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
info: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
ts: dt.datetime | None
|
||||
body: str
|
||||
|
||||
|
||||
def split_lines(lines: list[str]) -> list[Entry]:
|
||||
entries: list[Entry] = []
|
||||
for raw in lines:
|
||||
line = raw.rstrip("\n")
|
||||
if not line.strip():
|
||||
continue
|
||||
match = LINE_TS.match(line)
|
||||
if match:
|
||||
entries.append(Entry(parse_ts(match.group(1)), GO_TS.sub("", match.group(2), count=1)))
|
||||
else:
|
||||
# 没有 docker -t 前缀就无法证明"这是杀之后的日志":仍然保留(去掉内层
|
||||
# Go 时间戳后照常识别),由判据显式判红,而不是当成通过。
|
||||
entries.append(Entry(None, GO_TS.sub("", line, count=1)))
|
||||
return entries
|
||||
|
||||
|
||||
def evaluate(lines: list[str], job: str, target: str, backup: str, since: dt.datetime,
|
||||
expect_decision: str = "fresh") -> Result:
|
||||
entries = split_lines(lines)
|
||||
claim_prefix = CLAIM_PREFIX.format(job=job)
|
||||
decision_prefix = DECISION_PREFIX.format(job=job)
|
||||
|
||||
claims = [e for e in entries if e.body.startswith(claim_prefix)]
|
||||
decisions = [e for e in entries if e.body.startswith(decision_prefix)]
|
||||
after = lambda e: e.ts is not None and e.ts >= since # noqa: E731
|
||||
|
||||
fresh_claims = [e for e in claims if after(e)]
|
||||
fresh_decisions = [e for e in decisions if after(e)]
|
||||
aggregate = [e for e in entries if "takeover sweep" in e.body or re.search(
|
||||
r"library job resume: \d+ resumed", e.body)]
|
||||
other_claims = [(m.group(1), e) for e in entries for m in [ANY_CLAIM.search(e.body)]
|
||||
if m and m.group(1) != job]
|
||||
other_decisions = [(m.group(1), m.group(2), e) for e in entries
|
||||
for m in [ANY_DECISION.search(e.body)] if m and m.group(1) != job]
|
||||
|
||||
reasons: list[str] = []
|
||||
info: dict[str, str] = {}
|
||||
|
||||
if not fresh_claims:
|
||||
if claims:
|
||||
reasons.append("本作业的认领行全都出现在杀点 %s 之前:杀之前的日志不能证明这次接管"
|
||||
% since.isoformat())
|
||||
elif other_claims:
|
||||
reasons.append("只有别的作业的认领行(%s):不是本作业被认领"
|
||||
% ", ".join(sorted({j for j, _ in other_claims})))
|
||||
elif aggregate:
|
||||
reasons.append("只看到汇总行(%s):汇总既不说哪个作业,也不说它是否被认领"
|
||||
% aggregate[0].body.strip()[:120])
|
||||
else:
|
||||
reasons.append("日志里没有本作业的认领行 `%s…`" % claim_prefix.strip())
|
||||
else:
|
||||
kv = key_values(fresh_claims[-1].body)
|
||||
info["claim_at"] = kv.get("at", "")
|
||||
info["claim_path"] = kv.get("path", "")
|
||||
info["claim_queue"] = kv.get("queue", "")
|
||||
missing = [k for k in REQUIRED_CLAIM_KEYS if not kv.get(k)]
|
||||
if missing:
|
||||
reasons.append("认领行缺字段 %s(key=value 契约):%s"
|
||||
% (",".join(missing), fresh_claims[-1].body.strip()))
|
||||
if kv.get("target") and kv["target"] != target:
|
||||
reasons.append("认领行的 target=%s 不是本次目标 %s(别的作业/别的快照的认领)"
|
||||
% (kv["target"], target))
|
||||
if kv.get("backup") and backup and kv["backup"] != backup:
|
||||
reasons.append("认领行的 backup=%s 不是崩溃前登记的那份预备份 %s(换了基线)"
|
||||
% (kv["backup"], backup))
|
||||
|
||||
if not fresh_decisions:
|
||||
if decisions:
|
||||
reasons.append("本作业的决策行全都出现在杀点之前:不能证明这次接管的决策")
|
||||
else:
|
||||
reasons.append("日志里没有本作业的接管决策行 `%s<decision>…`" % decision_prefix)
|
||||
else:
|
||||
decision = fresh_decisions[-1].body[len(decision_prefix):].split()[0]
|
||||
info["decision"] = decision
|
||||
if decision not in TAKEOVER_DECISIONS:
|
||||
reasons.append("未知的接管决策 %r(期望 %s)" % (decision, expect_decision))
|
||||
elif decision != expect_decision:
|
||||
reasons.append("接管决策是 %s,期望 %s(在 after:database 之后不可能是"
|
||||
"「只差收尾」,同快照收尾=上一轮的假完成)" % (decision, expect_decision))
|
||||
kv = key_values(fresh_decisions[-1].body)
|
||||
if kv.get("target") and kv["target"] != target:
|
||||
reasons.append("决策行的 target=%s 不是本次目标 %s" % (kv["target"], target))
|
||||
|
||||
return Result(ok=not reasons, reasons=reasons, info=info)
|
||||
|
||||
|
||||
def read_log(path: Path) -> list[str] | None:
|
||||
if not path.is_file():
|
||||
return None
|
||||
return path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
|
||||
|
||||
# ── 自检:正例必须先绿,负例必须逐条红(否则判据是死的)──────────────────
|
||||
FIXTURE_JOB = "librestore-internal-1793303791259342087"
|
||||
FIXTURE_TARGET = "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7"
|
||||
FIXTURE_BACKUP = "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510"
|
||||
FIXTURE_SINCE = "2026-10-29T19:56:49Z"
|
||||
|
||||
FIXTURE_AGGREGATE = ("2026-10-29T19:57:20.123456789Z 2026/10/29 19:57:20 "
|
||||
"library job takeover sweep: 1 resumed, 0 failed without backup")
|
||||
FIXTURE_CLAIM = ("2026-10-29T19:57:20.500000000Z 2026/10/29 19:57:20 library job %s takeover claimed "
|
||||
"at=2026-10-29T19:57:20Z kind=restore scope=internal target=%s backup=%s "
|
||||
"queue=q65e9fd4d932b262659cdb0dcbedb92ce path=existing-queue"
|
||||
% (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP))
|
||||
FIXTURE_DECISION = ("2026-10-29T19:57:22.000000000Z 2026/10/29 19:57:22 library job %s takeover decision=fresh "
|
||||
"at=2026-10-29T19:57:22Z kind=restore scope=internal target=%s backup=%s "
|
||||
"finalize_only=false" % (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP))
|
||||
PRE_KILL_CLAIM = FIXTURE_CLAIM.replace("2026-10-29T19:57:20.500000000Z", "2026-10-29T19:56:40.000000000Z")
|
||||
PRE_KILL_DECISION = FIXTURE_DECISION.replace("2026-10-29T19:57:22.000000000Z", "2026-10-29T19:56:41.000000000Z")
|
||||
|
||||
|
||||
def self_test() -> int:
|
||||
since = parse_ts(FIXTURE_SINCE)
|
||||
assert since is not None
|
||||
cases: list[tuple[str, list[str], bool, str, str]] = [
|
||||
("正例:认领+决策都在杀之后", [FIXTURE_AGGREGATE, FIXTURE_CLAIM, FIXTURE_DECISION],
|
||||
True, "fresh", ""),
|
||||
("负例:只有汇总行(上一轮的真实形状)", [FIXTURE_AGGREGATE, FIXTURE_DECISION],
|
||||
False, "fresh", "汇总"),
|
||||
("负例:认领属于别的作业",
|
||||
[FIXTURE_CLAIM.replace(FIXTURE_JOB, "libsync-internal-other"), FIXTURE_DECISION],
|
||||
False, "fresh", "别的作业"),
|
||||
("负例:认领/决策都在杀之前(杀前日志伪通过)", [PRE_KILL_CLAIM, PRE_KILL_DECISION],
|
||||
False, "fresh", "杀点"),
|
||||
("负例:有认领没决策", [FIXTURE_CLAIM], False, "fresh", "决策行"),
|
||||
("负例:有决策没认领", [FIXTURE_DECISION], False, "fresh", "认领行"),
|
||||
("负例:认领的 target 不是本次目标",
|
||||
[FIXTURE_CLAIM.replace(FIXTURE_TARGET, "fc595511c7388d8e6cab3de37af8a2b6a64ede47241918684e81f2fb5d0e1f16"),
|
||||
FIXTURE_DECISION], False, "fresh", "target="),
|
||||
("负例:认领的 backup 换了(重做了备份)",
|
||||
[FIXTURE_CLAIM.replace(FIXTURE_BACKUP, "0fea01923a7c6073d06d7310588e737359f44ecee5ab1fd8f5892eba8e208b63"),
|
||||
FIXTURE_DECISION], False, "fresh", "backup="),
|
||||
("负例:决策是 pending_final(同快照收尾)",
|
||||
[FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")],
|
||||
False, "fresh", "pending_final"),
|
||||
("负例:没有 docker -t 时间戳(无法证明在杀之后)",
|
||||
[FIXTURE_CLAIM.split(" ", 1)[1], FIXTURE_DECISION.split(" ", 1)[1]],
|
||||
False, "fresh", "杀点"),
|
||||
("正例:期望决策可显式放宽为 pending_final",
|
||||
[FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")],
|
||||
True, "pending_final", ""),
|
||||
]
|
||||
failed = 0
|
||||
for name, lines, want_ok, expect, want_reason in cases:
|
||||
result = evaluate(lines, FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP, since, expect)
|
||||
if result.ok != want_ok:
|
||||
print("❌ %s:期望 ok=%s,实际 ok=%s(%s)"
|
||||
% (name, want_ok, result.ok, ";".join(result.reasons)), file=sys.stderr)
|
||||
failed += 1
|
||||
continue
|
||||
if not want_ok and want_reason not in ";".join(result.reasons):
|
||||
print("❌ %s:红了但理由不对(期望含 %r,实际 %r)"
|
||||
% (name, want_reason, result.reasons), file=sys.stderr)
|
||||
failed += 1
|
||||
continue
|
||||
if want_ok and want_reason:
|
||||
print("❌ 自检夹具写错了:正例不该要求失败理由 %r" % want_reason, file=sys.stderr)
|
||||
failed += 1
|
||||
if failed:
|
||||
print("❌ 接管日志契约自检:%d 条不成立" % failed, file=sys.stderr)
|
||||
return 1
|
||||
print("✅ 接管日志契约自检通过:%d 条(正例 %d / 负例 %d),负例逐条红"
|
||||
% (len(cases), sum(1 for c in cases if c[2]), sum(1 for c in cases if not c[2])))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(description="接管日志契约判据")
|
||||
parser.add_argument("--log", type=Path, help="docker logs -t 的输出")
|
||||
parser.add_argument("--job", help="业务作业 id")
|
||||
parser.add_argument("--target", help="目标快照(作业的 target_snapshot_id)")
|
||||
parser.add_argument("--backup", default="", help="崩溃前登记的预备份快照(可省略)")
|
||||
parser.add_argument("--since", help="杀点时刻(RFC3339);早于它的日志不算证据")
|
||||
parser.add_argument("--expect-decision", default="fresh",
|
||||
choices=TAKEOVER_DECISIONS, help="期望的接管决策(默认 fresh)")
|
||||
parser.add_argument("--self-test", action="store_true", help="只跑正负例自检")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.self_test:
|
||||
return self_test()
|
||||
if not (args.log and args.job and args.target and args.since):
|
||||
print("必须给 --log/--job/--target/--since(或用 --self-test)", file=sys.stderr)
|
||||
return 64
|
||||
since = parse_ts(args.since)
|
||||
if since is None:
|
||||
print("--since 不是 RFC3339:%r" % args.since, file=sys.stderr)
|
||||
return 64
|
||||
lines = read_log(args.log)
|
||||
if lines is None:
|
||||
print("读不到日志:%s" % args.log, file=sys.stderr)
|
||||
return 3
|
||||
|
||||
result = evaluate(lines, args.job, args.target, args.backup, since, args.expect_decision)
|
||||
if not result.ok:
|
||||
for reason in result.reasons:
|
||||
print("❌ " + reason, file=sys.stderr)
|
||||
return 2
|
||||
print("OK claim_at=%s path=%s queue=%s decision=%s target=%s backup=%s"
|
||||
% (result.info.get("claim_at", "?"), result.info.get("claim_path", "?"),
|
||||
result.info.get("claim_queue", "?"), result.info.get("decision", "?"),
|
||||
args.target, args.backup or "?"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user