From 8a3aa800a3e0ecd59a4878557f3ec47ee99f891c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Fri, 30 Oct 2026 07:56:16 +0800 Subject: [PATCH] =?UTF-8?q?fix(scripts):=20=E6=BC=94=E7=BB=83=E5=88=A4?= =?UTF-8?q?=E6=8D=AE=E6=94=B9=E7=94=A8=E8=AE=A4=E9=A2=86=E4=B8=8E=E5=86=B3?= =?UTF-8?q?=E7=AD=96=E8=A1=8C=E5=B9=B6=E6=A0=A1=E9=AA=8C=E9=A2=84=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=A4=8D=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v4 判据(都带作业身份,且时间戳必须晚于杀点): - `scripts/takeover_log_contract.py` 是唯一判据:认领行 + 决策行都要有本作业 id、 target/backup 一致;只有汇总行、别的作业的认领行、杀之前的旧日志三种一律红; `--self-test` 11 条(正例 2 / 负例 9)进 `make verify`(新增第 16 项必需检查); - `prebackup` 不再强制 `skipped`:真实接管是复用后重跑整条链路,阶段记 succeeded。 但**不取消备份校验**,改为证明复用同一份——快照 id、清单字节与 mtime、 版本行首次登记时刻都不变,且证据里有 `backup_reused=true`; - 杀之前把预备份清单 sha256/mtime 与版本行登记时刻拍进 evidence; - summary.json 增加 takeover_decision / claim_at / backup_registered_at。 verify: 判据自检 11 条全过;用真实 Go 输出行形状的正例日志 rc=0;旧形状(只有 汇总行)rc=2 且理由点名"汇总";`--check` 前置体检 13 项全 pass(含新增 preflight_log_contract);make verify overall success。 --- scripts/ci_checks.py | 5 + scripts/library-kill-takeover.sh | 121 +++++++++++-- scripts/takeover_log_contract.py | 295 +++++++++++++++++++++++++++++++ 3 files changed, 404 insertions(+), 17 deletions(-) create mode 100755 scripts/takeover_log_contract.py diff --git a/scripts/ci_checks.py b/scripts/ci_checks.py index 8150a39..8f59112 100644 --- a/scripts/ci_checks.py +++ b/scripts/ci_checks.py @@ -251,6 +251,10 @@ def checks() -> list[Check]: [py3, "scripts/ci_checks.py", "--internal-web-test"], env=env), Check("contracts", "版本与变更记录契约(不许 SKIP;开发构建不许自称正式版)", [py3, "scripts/ci_checks.py", "--internal-contracts", g], env=env), + # 演练判据本身也要被验:认领/决策行缺一条、只有汇总行、杀前日志都不许算通过。 + # 2026-10-29 那次演练 exit 5 就是"脚本要的老文案只有一条代码路径会打印"。 + Check("takeover-log-contract", "接管日志契约(认领/决策行判据 + 正负例自检)", + [py3, "scripts/takeover_log_contract.py", "--self-test"]), Check("version", "CHANGELOG 与版本记录(四类小节无空栏目 + 正式 tag 与提交内版本一致)", [py3, "scripts/check_changelog.py", "--tags"]), Check("security-secrets", "密钥泄漏扫描(含扫描器自检)", @@ -439,6 +443,7 @@ CHECK_MANIFEST: dict[str, tuple[str, ...]] = { "tests-go": ("local", "ci"), "tests-web": ("local", "ci"), "contracts": ("local", "ci"), + "takeover-log-contract": ("local", "ci"), "version": ("local", "ci"), "security-secrets": ("local", "ci"), "security-deps": ("local", "ci"), diff --git a/scripts/library-kill-takeover.sh b/scripts/library-kill-takeover.sh index 4bee8e7..cacea09 100755 --- a/scripts/library-kill-takeover.sh +++ b/scripts/library-kill-takeover.sh @@ -1,10 +1,21 @@ #!/usr/bin/env bash # library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈) # -# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被接管并 +# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被认领并 # 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**,全成立才 exit 0。 # missed / notrun / timeout / 只同快照收尾 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。 # +# ⚠️ 2026-10-29T19:56Z 这次真演练(exit 5)教会两件事,都是判据缺陷: +# 1) 脚本要的文案 `library job resumed after worker restart` 只有"新建队列行" +# 那条代码路径会打印;真实走的是"队列行已存在"那条。于是 worker 只留下汇总行 +# `library job takeover sweep: 1 resumed`,脚本判不出是哪个作业、也判不出它是否被认领。 +# 现在改由 scripts/takeover_log_contract.py 判**认领行 + 决策行**(都带作业 id 与 +# target/backup,且时间戳必须晚于杀点);只有汇总行/别的作业/杀前日志一律不算证据。 +# 2) 老判据要求 `prebackup:skipped`。真实接管是"复用崩溃前那份预备份重跑整条链路", +# 阶段会写成 succeeded(release.go 记的是"预备份 ")。**不能因此取消备份校验**: +# 现在要求预备份快照身份与崩溃前一致、清单文件字节/mtime 未变、版本行 created_at +# 仍是首次登记时刻(复用而非重做),写集合摘要也不变。 +# # 为什么不再用 `docker kill`: # docker kill/stop 走 Docker API 的"手动停止",restart policy 不生效(`unless-stopped` # 也不生效),容器就一直躺着。2026-10-29T19:16Z 的现场就是这样:脚本等了 420s 判失败, @@ -16,7 +27,7 @@ # # 命中点位仍是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入): # worker 在指定阶段写完后写 /after-database.hit 并停住 → 本脚本看到 .hit -# → SIGKILL → 按策略重启/显式拉起 → 启动扫描接管同一作业。 +# → SIGKILL → 按策略重启/显式拉起 → 启动扫描认领同一作业。 # 屏障目录在本机归档根下(/barrier),没人放行就自行继续(记 not_run)。 # # 期望内容按**目标快照自己的 manifest** 取(knowledges/chunks/wiki_pages),不是写死 @@ -33,12 +44,15 @@ # --snapshot (校验命中的作业目标就是它) # --job <业务作业 id>|--since # --expect "knowledges=..,chunks=..,wiki_pages=.."(覆盖 manifest 推导) +# --expect-decision fresh|pending_final(默认 fresh;at after:database 之后 +# 只能是 fresh——pending_final 等于"只差收尾",正是上一轮的假完成) # --allow-same-snapshot(允许目标快照 == 中断前生效版本;这条**不能**当完整续作验收) # --manual-start 0|1(策略未拉起时是否显式 docker start,默认 1) # set -uo pipefail -SCRIPT_VERSION=3 +SCRIPT_VERSION=4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODE="" SCOPE="internal" KIND="auto" @@ -48,8 +62,10 @@ SINCE="" CONFIRM_KILL=0 ALLOW_SAME_SNAPSHOT=0 MANUAL_START=1 +EXPECT_DECISION="${EXPECT_DECISION:-fresh}" EXPECT_COUNTS="${EXPECT_COUNTS:-}" VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}" +LOG_CONTRACT="${TAKEOVER_LOG_CONTRACT:-$SCRIPT_DIR/takeover_log_contract.py}" WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}" GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}" @@ -77,6 +93,7 @@ while [ $# -gt 0 ]; do --job) EXPECT_JOB="${2:-}"; shift ;; --since) SINCE="${2:-}"; shift ;; --expect) EXPECT_COUNTS="${2:-}"; shift ;; + --expect-decision) EXPECT_DECISION="${2:-}"; shift ;; --point) BARRIER_POINT="${2:-}"; shift ;; --manual-start) MANUAL_START="${2:-}"; shift ;; --allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;; @@ -99,6 +116,10 @@ case "$MANUAL_START" in 0|1) ;; *) echo "--manual-start must be 0|1" >&2; exit 64 ;; esac +case "$EXPECT_DECISION" in + fresh|pending_final) ;; + *) echo "--expect-decision must be fresh|pending_final (got $EXPECT_DECISION)" >&2; exit 64 ;; +esac # set -u:所有跨函数使用的变量先给默认值 WORKER_IMAGE_ID="" @@ -129,6 +150,14 @@ WS_REUSED="" WS_TARGET="" WS_BACKUP="" WS_FILE="" +WS_FILE_SHA="" +WS_BACKUP_SHA="" +WS_BACKUP_MTIME="" +WS_BACKUP_CREATED="" +WS_REUSED="" +DECISION="" +CLAIM_AT="" +LOG_RAW="" KILLED_AT="" KILL_PID="" KILL_PROOF="" @@ -241,6 +270,11 @@ finish() { # finish "killed_pid": "$KILL_PID", "kill_proof": "$KILL_PROOF", "restart_source": "$RESTART_SOURCE", + "takeover_decision": "$DECISION", + "takeover_claim_at": "$CLAIM_AT", + "expected_decision": "$EXPECT_DECISION", + "backup_snapshot": "$WS_BACKUP", + "backup_registered_at": "$WS_BACKUP_CREATED", "writeset_digest": "$WS_DIGEST", "evidence_dir": "$EVIDENCE" } @@ -352,6 +386,12 @@ preflight() { fi record preflight_verifier pass "verifier=$VERIFIER" + # 判据本身也要被验:脚本缺失或自检红,演练中途就只能靠人肉判断了。 + [ -f "$LOG_CONTRACT" ] || die 2 preflight_log_contract "接管日志判据脚本不存在:$LOG_CONTRACT" + python3 "$LOG_CONTRACT" --self-test >>"$EVIDENCE/log-contract.log" 2>&1 || die 2 preflight_log_contract \ + "接管日志判据自检没通过(见 log-contract.log):判据坏了不许当演练通过" + record preflight_log_contract pass "contract=$LOG_CONTRACT self-test=ok" + [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \ "worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore" record preflight_not_armed pass "barrier not armed" @@ -518,19 +558,27 @@ lock_job() { read_writeset_evidence() { local row - row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" + row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','')||'|'||created_at::text||'|'||coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据" - IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row" + IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP WS_BACKUP_CREATED WS_REUSED <<<"$row" [ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest:接管会失败,这次不算可接管" [ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致" WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json" [ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE" grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)" grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID" + # 崩溃前那份预备份的身份与字节也拍下来:接管是否"复用同一份、没被重做覆盖" + # 只能靠事后比对(快照 id + 清单字节 + 清单 mtime + 版本行首次登记时刻)。 + local backup_manifest + backup_manifest="$ARCHIVE_HOST/versions/$WS_BACKUP.json" + [ -s "$backup_manifest" ] || die 4 read_writeset "预备份快照清单不存在或为空:$backup_manifest" + WS_BACKUP_SHA="$(sha256sum "$backup_manifest" | awk '{print $1}')" + WS_BACKUP_MTIME="$(stat -c %Y "$backup_manifest")" + WS_FILE_SHA="$(sha256sum "$WS_FILE" | awk '{print $1}')" # 独立校验器是必需的(preflight 已确认存在):它判定不符就不能继续。 "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \ >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)" - record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=verified" + record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP backup_manifest_sha=$WS_BACKUP_SHA backup_manifest_mtime=$WS_BACKUP_MTIME backup_registered_at=$WS_BACKUP_CREATED writeset_sha=$WS_FILE_SHA canonical=verified" } kill_worker() { @@ -615,9 +663,9 @@ wait_restart() { } wait_takeover() { - local deadline state="" last_note=0 + local deadline state="" last_note=0 out rc CONTRACT_OUT deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S )) - log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…" + log "等待同一个作业被认领(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…" while [ "$(now_epoch)" -lt "$deadline" ]; do state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" case "$state" in succeeded|failed|cancelled) break ;; esac @@ -634,11 +682,25 @@ wait_takeover() { failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;; *) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;; esac - # 人工拉起/策略拉起都要能对得上"启动扫描真的接了这个作业"。 - local resumed - resumed="$(docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | grep -c "library job $JOB_ID resumed after worker restart")" - [ "${resumed:-0}" -ge 1 ] || die 5 wait_takeover "worker 日志里没有本作业的 resume 记录:接管来源不明" - record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; resume_log=$resumed" + + # 认领与决策都要有**本作业 id** 的行,且时间戳晚于杀点。 + # 汇总行(`library job takeover sweep: 1 resumed`)不算证据:它既不说是哪个作业, + # 也不说是否真的被认领(别的 worker 抢走、attach 失败都只影响计数)。 + LOG_RAW="$EVIDENCE/worker-log-raw.txt" + # -t 才有每行时间戳:判据要证明"这些行出现在杀之后",没有时间戳就只能判红。 + docker logs -t --since "$KILLED_AT" "$WORKER_CONTAINER" >"$LOG_RAW" 2>&1 + CONTRACT_OUT="$EVIDENCE/log-contract.log" + python3 "$LOG_CONTRACT" --log "$LOG_RAW" --job "$JOB_ID" --target "$JOB_TARGET" \ + --backup "$WS_BACKUP" --since "$KILLED_AT" --expect-decision "$EXPECT_DECISION" \ + >"$CONTRACT_OUT" 2>&1; rc=$? + out="$(cat "$CONTRACT_OUT")" + if [ "$rc" -ne 0 ]; then + die 5 wait_takeover "接管日志契约不成立(rc=$rc,见 log-contract.log):$(tr '\n' ' ' <"$CONTRACT_OUT" | cut -c1-400)" + fi + CLAIM_AT="$(sed -n 's/.*claim_at=\([^ ]*\).*/\1/p' <<<"$out" | head -1)" + DECISION="$(sed -n 's/.*decision=\([^ ]*\).*/\1/p' <<<"$out" | head -1)" + [ -n "$DECISION" ] || die 5 wait_takeover "判据脚本没给出决策(输出:$out)" + record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; claim_at=${CLAIM_AT:-?}; decision=$DECISION; contract=ok" } verify_stages_complete() { @@ -653,13 +715,38 @@ verify_stages_complete() { } verify_takeover_semantics() { - local stages finalize_at eff after_digest + local stages pre_state finalize_at eff after_digest + local backup_now backup_created_now backup_sha_now backup_mtime_now pre_reused_now stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")" - grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \ - "阶段里没有 prebackup:skipped:无法证明走的是接管路径(stages=$stages)" + pre_state="$(gov_sql "select coalesce(state,'') from library_job_stages where job_id='$JOB_ID' and stage='prebackup'")" + # 接管复用崩溃前那份预备份:阶段既可能写成 skipped(只标记不重跑),也可能写成 + # succeeded(release.go 记"预备份 "——真实 2026-10-29 现场就是这种)。 + # 两种都接受,但**不能因此取消备份校验**:必须证明复用的是同一份、没被重做覆盖。 + case "$pre_state" in + skipped|succeeded) ;; + *) die 6 verify_takeover "prebackup 阶段是 ${pre_state:-缺失}:既不是复用(skipped)也不是复用后重跑(succeeded),接管路径不成立(stages=$stages)" ;; + esac finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")" [ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功(stages=$stages)" - record verify_takeover pass "stages=$stages finalize_at=$finalize_at" + + backup_now="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" + [ "$backup_now" = "$WS_BACKUP" ] || die 6 verify_takeover \ + "预备份身份变了:崩溃前 $WS_BACKUP → 现在 ${backup_now:-空}(重做了备份 = 把半写状态当成可回滚基线)" + backup_created_now="$(gov_sql "select created_at::text from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" + [ "$backup_created_now" = "$WS_BACKUP_CREATED" ] || die 6 verify_takeover \ + "预备份版本行的首次登记时刻被改写:$WS_BACKUP_CREATED → $backup_created_now(复用被洗成重做)" + backup_sha_now="$(sha256sum "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null | awk '{print $1}')" + [ "$backup_sha_now" = "$WS_BACKUP_SHA" ] || die 6 verify_takeover \ + "预备份清单字节变了:$WS_BACKUP_SHA → ${backup_sha_now:-缺失}(同一快照 id 下内容被重写)" + backup_mtime_now="$(stat -c %Y "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null)" + [ "$backup_mtime_now" = "$WS_BACKUP_MTIME" ] || die 6 verify_takeover \ + "预备份清单文件被重新写过(mtime $WS_BACKUP_MTIME → ${backup_mtime_now:-缺失}):崩溃后重做了备份" + pre_reused_now="$(gov_sql "select coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" + if [ "$pre_state" = "succeeded" ] && [ "$pre_reused_now" != "true" ]; then + die 6 verify_takeover \ + "prebackup 记成 succeeded 但版本证据没有 backup_reused=true:分不清'复用'与'重做了备份'(evidence=$pre_reused_now)" + fi + record verify_takeover pass "stages=$stages decision=$DECISION finalize_at=$finalize_at prebackup=$pre_state backup=$backup_now registered_at=$backup_created_now manifest_sha=$backup_sha_now reused=${pre_reused_now:-no}" eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")" local eff_snapshot eff_job eff_superseded diff --git a/scripts/takeover_log_contract.py b/scripts/takeover_log_contract.py new file mode 100755 index 0000000..373df0a --- /dev/null +++ b/scripts/takeover_log_contract.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""接管日志契约:从真实 worker 日志里证明"这个作业被认领了、并且做了哪个接管决策"。 + +为什么要有这份独立判据 +---------------------- +2026-10-29T19:56Z 的 `after:database` 演练里,worker 只打印了汇总行 +`library job takeover sweep: 1 resumed, 0 failed without backup`:汇总既没说 +是**哪个**作业,也没说它是不是真的被本进程认领(被别的 worker 抢走、attach 失败 +都只影响计数)。演练脚本当时要求的老文案 `library job resumed after worker restart` +只在"新建队列行"那条路径打印,真实走的却是"队列行已存在"那条 ⇒ 判据与实现各说一套, +演练 exit 5。这里把契约固化成可执行判据。 + +判据(三条缺一不可): + 1. 一行 `library job takeover claimed ... key=value ...`,且 target/backup 与本次作业一致; + 2. 一行 `library job takeover decision= ...`,且 等于期望决策; + 3. 两行的时间戳都 >= 杀点(取自 `docker logs -t` 的行首时间戳):杀之前的旧日志不算。 +只接受**带本作业 id** 的行:汇总行(takeover sweep / 没有 id 的 resume 统计)一律不算证据; +别的作业的认领行也不算。 + +用法: + python3 scripts/takeover_log_contract.py --log worker.log --job \ + --target <快照> --backup <预备份快照> --since 2026-10-29T19:56:49Z + python3 scripts/takeover_log_contract.py --self-test # 正负例自检(门禁 CI 里跑) +退出码:0 成立 / 2 判据不成立(原因逐条打印)/ 3 日志读不到 / 64 参数不合法 +""" +from __future__ import annotations + +import argparse +import datetime as dt +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +LINE_TS = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z) (.*)$") +# Go 的 log 默认带 LstdFlags(`2026/10/29 19:57:21 `),`docker logs -t` 又在最前面加 +# 自己的 RFC3339 时间戳 ⇒ 行体里还有一层"年/月/日 时:分:秒"。判据只认行首那一个。 +GO_TS = re.compile(r"^\d{4}/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d+)? ") +CLAIM_PREFIX = "library job {job} takeover claimed " +DECISION_PREFIX = "library job {job} takeover decision=" +ANY_CLAIM = re.compile(r"library job (\S+) takeover claimed ") +ANY_DECISION = re.compile(r"library job (\S+) takeover decision=(\S+)") + +REQUIRED_CLAIM_KEYS = ("at", "kind", "scope", "target", "backup", "queue", "path") +TAKEOVER_DECISIONS = ("fresh", "pending_final", "superseded", "foreign_effective", "unknown") + + +def parse_ts(value: str) -> dt.datetime | None: + """解析 `docker logs -t` 的 RFC3339(可能带纳秒)时间戳;解析不了就是 None。""" + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + if "." in text: + head, tail = text.split(".", 1) + # 纳秒 → 微秒:Python 只认 6 位,直接截断(比较用,不需要更高精度) + frac = re.match(r"(\d+)", tail) + zone = tail[frac.end():] if frac else "" + text = head + "." + (frac.group(1)[:6].ljust(6, "0") if frac else "000000") + zone + try: + parsed = dt.datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.timezone.utc) + + +def key_values(body: str) -> dict[str, str]: + """按 key=value 取字段(不按位置猜):Go 侧改字段名会让演练直接红,而不是静默放过。""" + out: dict[str, str] = {} + for token in body.split(): + if "=" in token: + key, _, value = token.partition("=") + if key and value: + out[key] = value + return out + + +@dataclass +class Result: + ok: bool + reasons: list[str] = field(default_factory=list) + info: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Entry: + ts: dt.datetime | None + body: str + + +def split_lines(lines: list[str]) -> list[Entry]: + entries: list[Entry] = [] + for raw in lines: + line = raw.rstrip("\n") + if not line.strip(): + continue + match = LINE_TS.match(line) + if match: + entries.append(Entry(parse_ts(match.group(1)), GO_TS.sub("", match.group(2), count=1))) + else: + # 没有 docker -t 前缀就无法证明"这是杀之后的日志":仍然保留(去掉内层 + # Go 时间戳后照常识别),由判据显式判红,而不是当成通过。 + entries.append(Entry(None, GO_TS.sub("", line, count=1))) + return entries + + +def evaluate(lines: list[str], job: str, target: str, backup: str, since: dt.datetime, + expect_decision: str = "fresh") -> Result: + entries = split_lines(lines) + claim_prefix = CLAIM_PREFIX.format(job=job) + decision_prefix = DECISION_PREFIX.format(job=job) + + claims = [e for e in entries if e.body.startswith(claim_prefix)] + decisions = [e for e in entries if e.body.startswith(decision_prefix)] + after = lambda e: e.ts is not None and e.ts >= since # noqa: E731 + + fresh_claims = [e for e in claims if after(e)] + fresh_decisions = [e for e in decisions if after(e)] + aggregate = [e for e in entries if "takeover sweep" in e.body or re.search( + r"library job resume: \d+ resumed", e.body)] + other_claims = [(m.group(1), e) for e in entries for m in [ANY_CLAIM.search(e.body)] + if m and m.group(1) != job] + other_decisions = [(m.group(1), m.group(2), e) for e in entries + for m in [ANY_DECISION.search(e.body)] if m and m.group(1) != job] + + reasons: list[str] = [] + info: dict[str, str] = {} + + if not fresh_claims: + if claims: + reasons.append("本作业的认领行全都出现在杀点 %s 之前:杀之前的日志不能证明这次接管" + % since.isoformat()) + elif other_claims: + reasons.append("只有别的作业的认领行(%s):不是本作业被认领" + % ", ".join(sorted({j for j, _ in other_claims}))) + elif aggregate: + reasons.append("只看到汇总行(%s):汇总既不说哪个作业,也不说它是否被认领" + % aggregate[0].body.strip()[:120]) + else: + reasons.append("日志里没有本作业的认领行 `%s…`" % claim_prefix.strip()) + else: + kv = key_values(fresh_claims[-1].body) + info["claim_at"] = kv.get("at", "") + info["claim_path"] = kv.get("path", "") + info["claim_queue"] = kv.get("queue", "") + missing = [k for k in REQUIRED_CLAIM_KEYS if not kv.get(k)] + if missing: + reasons.append("认领行缺字段 %s(key=value 契约):%s" + % (",".join(missing), fresh_claims[-1].body.strip())) + if kv.get("target") and kv["target"] != target: + reasons.append("认领行的 target=%s 不是本次目标 %s(别的作业/别的快照的认领)" + % (kv["target"], target)) + if kv.get("backup") and backup and kv["backup"] != backup: + reasons.append("认领行的 backup=%s 不是崩溃前登记的那份预备份 %s(换了基线)" + % (kv["backup"], backup)) + + if not fresh_decisions: + if decisions: + reasons.append("本作业的决策行全都出现在杀点之前:不能证明这次接管的决策") + else: + reasons.append("日志里没有本作业的接管决策行 `%s…`" % decision_prefix) + else: + decision = fresh_decisions[-1].body[len(decision_prefix):].split()[0] + info["decision"] = decision + if decision not in TAKEOVER_DECISIONS: + reasons.append("未知的接管决策 %r(期望 %s)" % (decision, expect_decision)) + elif decision != expect_decision: + reasons.append("接管决策是 %s,期望 %s(在 after:database 之后不可能是" + "「只差收尾」,同快照收尾=上一轮的假完成)" % (decision, expect_decision)) + kv = key_values(fresh_decisions[-1].body) + if kv.get("target") and kv["target"] != target: + reasons.append("决策行的 target=%s 不是本次目标 %s" % (kv["target"], target)) + + return Result(ok=not reasons, reasons=reasons, info=info) + + +def read_log(path: Path) -> list[str] | None: + if not path.is_file(): + return None + return path.read_text(encoding="utf-8", errors="replace").splitlines() + + +# ── 自检:正例必须先绿,负例必须逐条红(否则判据是死的)────────────────── +FIXTURE_JOB = "librestore-internal-1793303791259342087" +FIXTURE_TARGET = "0d982f7667be35227c18b89ca08e83d37a44e5d08a1582a8874be08eee5b61e7" +FIXTURE_BACKUP = "69eb5a90f303b3fb9e0ebffececb92dcf2f62701fd459d58871ac305053bd510" +FIXTURE_SINCE = "2026-10-29T19:56:49Z" + +FIXTURE_AGGREGATE = ("2026-10-29T19:57:20.123456789Z 2026/10/29 19:57:20 " + "library job takeover sweep: 1 resumed, 0 failed without backup") +FIXTURE_CLAIM = ("2026-10-29T19:57:20.500000000Z 2026/10/29 19:57:20 library job %s takeover claimed " + "at=2026-10-29T19:57:20Z kind=restore scope=internal target=%s backup=%s " + "queue=q65e9fd4d932b262659cdb0dcbedb92ce path=existing-queue" + % (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP)) +FIXTURE_DECISION = ("2026-10-29T19:57:22.000000000Z 2026/10/29 19:57:22 library job %s takeover decision=fresh " + "at=2026-10-29T19:57:22Z kind=restore scope=internal target=%s backup=%s " + "finalize_only=false" % (FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP)) +PRE_KILL_CLAIM = FIXTURE_CLAIM.replace("2026-10-29T19:57:20.500000000Z", "2026-10-29T19:56:40.000000000Z") +PRE_KILL_DECISION = FIXTURE_DECISION.replace("2026-10-29T19:57:22.000000000Z", "2026-10-29T19:56:41.000000000Z") + + +def self_test() -> int: + since = parse_ts(FIXTURE_SINCE) + assert since is not None + cases: list[tuple[str, list[str], bool, str, str]] = [ + ("正例:认领+决策都在杀之后", [FIXTURE_AGGREGATE, FIXTURE_CLAIM, FIXTURE_DECISION], + True, "fresh", ""), + ("负例:只有汇总行(上一轮的真实形状)", [FIXTURE_AGGREGATE, FIXTURE_DECISION], + False, "fresh", "汇总"), + ("负例:认领属于别的作业", + [FIXTURE_CLAIM.replace(FIXTURE_JOB, "libsync-internal-other"), FIXTURE_DECISION], + False, "fresh", "别的作业"), + ("负例:认领/决策都在杀之前(杀前日志伪通过)", [PRE_KILL_CLAIM, PRE_KILL_DECISION], + False, "fresh", "杀点"), + ("负例:有认领没决策", [FIXTURE_CLAIM], False, "fresh", "决策行"), + ("负例:有决策没认领", [FIXTURE_DECISION], False, "fresh", "认领行"), + ("负例:认领的 target 不是本次目标", + [FIXTURE_CLAIM.replace(FIXTURE_TARGET, "fc595511c7388d8e6cab3de37af8a2b6a64ede47241918684e81f2fb5d0e1f16"), + FIXTURE_DECISION], False, "fresh", "target="), + ("负例:认领的 backup 换了(重做了备份)", + [FIXTURE_CLAIM.replace(FIXTURE_BACKUP, "0fea01923a7c6073d06d7310588e737359f44ecee5ab1fd8f5892eba8e208b63"), + FIXTURE_DECISION], False, "fresh", "backup="), + ("负例:决策是 pending_final(同快照收尾)", + [FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")], + False, "fresh", "pending_final"), + ("负例:没有 docker -t 时间戳(无法证明在杀之后)", + [FIXTURE_CLAIM.split(" ", 1)[1], FIXTURE_DECISION.split(" ", 1)[1]], + False, "fresh", "杀点"), + ("正例:期望决策可显式放宽为 pending_final", + [FIXTURE_CLAIM, FIXTURE_DECISION.replace("decision=fresh", "decision=pending_final")], + True, "pending_final", ""), + ] + failed = 0 + for name, lines, want_ok, expect, want_reason in cases: + result = evaluate(lines, FIXTURE_JOB, FIXTURE_TARGET, FIXTURE_BACKUP, since, expect) + if result.ok != want_ok: + print("❌ %s:期望 ok=%s,实际 ok=%s(%s)" + % (name, want_ok, result.ok, ";".join(result.reasons)), file=sys.stderr) + failed += 1 + continue + if not want_ok and want_reason not in ";".join(result.reasons): + print("❌ %s:红了但理由不对(期望含 %r,实际 %r)" + % (name, want_reason, result.reasons), file=sys.stderr) + failed += 1 + continue + if want_ok and want_reason: + print("❌ 自检夹具写错了:正例不该要求失败理由 %r" % want_reason, file=sys.stderr) + failed += 1 + if failed: + print("❌ 接管日志契约自检:%d 条不成立" % failed, file=sys.stderr) + return 1 + print("✅ 接管日志契约自检通过:%d 条(正例 %d / 负例 %d),负例逐条红" + % (len(cases), sum(1 for c in cases if c[2]), sum(1 for c in cases if not c[2]))) + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description="接管日志契约判据") + parser.add_argument("--log", type=Path, help="docker logs -t 的输出") + parser.add_argument("--job", help="业务作业 id") + parser.add_argument("--target", help="目标快照(作业的 target_snapshot_id)") + parser.add_argument("--backup", default="", help="崩溃前登记的预备份快照(可省略)") + parser.add_argument("--since", help="杀点时刻(RFC3339);早于它的日志不算证据") + parser.add_argument("--expect-decision", default="fresh", + choices=TAKEOVER_DECISIONS, help="期望的接管决策(默认 fresh)") + parser.add_argument("--self-test", action="store_true", help="只跑正负例自检") + args = parser.parse_args(argv) + + if args.self_test: + return self_test() + if not (args.log and args.job and args.target and args.since): + print("必须给 --log/--job/--target/--since(或用 --self-test)", file=sys.stderr) + return 64 + since = parse_ts(args.since) + if since is None: + print("--since 不是 RFC3339:%r" % args.since, file=sys.stderr) + return 64 + lines = read_log(args.log) + if lines is None: + print("读不到日志:%s" % args.log, file=sys.stderr) + return 3 + + result = evaluate(lines, args.job, args.target, args.backup, since, args.expect_decision) + if not result.ok: + for reason in result.reasons: + print("❌ " + reason, file=sys.stderr) + return 2 + print("OK claim_at=%s path=%s queue=%s decision=%s target=%s backup=%s" + % (result.info.get("claim_at", "?"), result.info.get("claim_path", "?"), + result.info.get("claim_queue", "?"), result.info.get("decision", "?"), + args.target, args.backup or "?")) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) \ No newline at end of file