From 1d70059d328977fcae01589992d16f5d407dbe5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Fri, 30 Oct 2026 03:40:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(scripts):=20=E4=B8=AD=E6=96=AD=E6=BC=94?= =?UTF-8?q?=E7=BB=83=E6=94=B9=E7=94=A8=E8=BF=9B=E7=A8=8B=E5=B4=A9=E6=BA=83?= =?UTF-8?q?=E8=AF=AD=E4=B9=89=E5=B9=B6=E6=8C=89=E5=BF=AB=E7=85=A7=E6=8E=A8?= =?UTF-8?q?=E5=AF=BC=E6=9C=9F=E6=9C=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一轮为什么空等 420s:docker kill 走 Docker API 的"手动停止",unless-stopped 也不拉起,容器一直躺着,最后由人工 docker start 才接管 —— 那不是崩溃自恢复。 改法: - 命中后从宿主机向容器 PID1 发 SIGKILL(同 uid 才允许,先证明能杀);退出码证据取 docker events 的 die exitCode=137 —— restart 后 State.ExitCode 会被重置成 0, 事后 inspect 读不到 137。 - 限期等 restart policy 拉起(同容器/同镜像才算);策略没起来才显式 docker start, 并记 restart_source=manual_start,绝不当自动恢复。 - 期望内容按目标快照 manifest 推导(knowledges/chunks/wiki_pages),resources 不进 默认期望(同租户其它知识库资源会保留,trap-16)。 - --kind sync|restore + --snapshot 强校验;目标快照等于中断前生效版本时**杀之前 就拒绝**(同快照收尾不能当完整续作验收)。 - 新增 verify_stages_complete:database/resources/index/wiki/verify 全部 succeeded 才算完整续作(上一轮这四个还是 pending,却被判成功)。 - 独立写集合校验器缺失即 fail(旧版记 canonical=not_run 还 pass)。 - save_evidence 读实例维护用 owner_scope(旧版读 scope 列,SQL 报错被吞)。 验证(已跑): - bash -n 通过;--check 全绿(READY_FOR_PARENT,含 preflight_verifier) - 负例:WRITESET_VERIFIER=/nonexistent → preflight_verifier fail / exit 2 - 期望推导:fc5955→4/38/31、0d982→1/2/9、0fea→4/38/31;不存在的快照 → exit 1 - 宿主 SIGKILL 能力实测(scratch 容器,非本栈):kill -9 PID1 → 策略自动重启, RestartCount 0→1,docker events 记 exitCode=137 --- scripts/library-kill-takeover.sh | 293 ++++++++++++++++++++++++++----- 1 file changed, 246 insertions(+), 47 deletions(-) diff --git a/scripts/library-kill-takeover.sh b/scripts/library-kill-takeover.sh index 35bbcf1..4fcd003 100755 --- a/scripts/library-kill-takeover.sh +++ b/scripts/library-kill-takeover.sh @@ -1,36 +1,54 @@ #!/usr/bin/env bash # library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈) # -# 旧脚本(handoffs/release-reuse-18683ca/kill-takeover.sh)为什么不能用: -# 1. 前置要求"作业已 running 且 ledger 齐"——可它又要先启动观察,永远错过; -# 2. 维护窗口里要求业务 health 200——那时 app 被有意停掉,必然被拒; -# 3. 没杀到任何进程也 exit 0——"没做"和"做成"分不开。 +# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被接管并 +# 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**,全成立才 exit 0。 +# missed / notrun / timeout / 只同快照收尾 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。 # -# 这一版的判据只有一条:**真的 SIGKILL 了真实 worker 容器 → 同一个作业被接管 → -# 目标按期望内容恢复并校验 → 服务/入口恢复**,四条全成立才 exit 0。 -# missed / notrun / timeout 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。 +# 为什么不再用 `docker kill`: +# docker kill/stop 走 Docker API 的"手动停止",restart policy 不生效(`unless-stopped` +# 也不生效),容器就一直躺着。2026-10-29T19:16Z 的现场就是这样:脚本等了 420s 判失败, +# 最后由人工 `docker start` 才接管。那不是"崩溃自恢复",只是"有人按了开始"。 +# 现在的命中手段是**宿主机向容器 PID1 发 SIGKILL**(同一 uid 才允许):这才是进程崩溃, +# Docker 看到的是进程死亡 ⇒ restart policy 按策略拉起,语义与真实故障一致。 +# 若策略没在限期内拉起,脚本可以显式 `docker start`,但必须在 evidence 里记成 +# restart_source=manual_start,**不许**算成"自动恢复"。 # -# 命中手段是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入): +# 命中点位仍是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入): # worker 在指定阶段写完后写 /after-database.hit 并停住 → 本脚本看到 .hit -# → SIGKILL 容器 → 容器按 restart 策略重启 → 启动/周期扫描接管同一作业。 -# 屏障目录在本机归档根下(/barrier),没人放行就 300s 自行继续。 +# → SIGKILL → 按策略重启/显式拉起 → 启动扫描接管同一作业。 +# 屏障目录在本机归档根下(/barrier),没人放行就自行继续(记 not_run)。 +# +# 期望内容按**目标快照自己的 manifest** 取(knowledges/chunks/wiki_pages),不是写死 +# 4/38/31:同一个脚本既能跑 sync 也能跑 restore,期望随快照变。resources 不进默认 +# 期望——同租户其它知识库的资源不在预备份范围内(trap-16),目标里会保留它们。 # # 用法: # bash scripts/library-kill-takeover.sh --check # 前置体检(不是演练通过) -# bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill -# → 打印 ARMED 后,父在页面上发起同步作业;脚本自己命中、杀、验证、收尾 +# bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill \ +# --kind restore --snapshot <目标快照 id> +# → 打印 ARMED 后,父在页面上发起该作业;脚本自己命中、杀、验证、收尾 # bash scripts/library-kill-takeover.sh --restore # 撤掉屏障注入 -# 可选:--job <业务作业 id>|--since |--expect "knowledges=4,chunks=38,..." +# 可选:--kind sync|restore|auto(默认 auto,给了就强校验作业类型) +# --snapshot (校验命中的作业目标就是它) +# --job <业务作业 id>|--since +# --expect "knowledges=..,chunks=..,wiki_pages=.."(覆盖 manifest 推导) +# --allow-same-snapshot(允许目标快照 == 中断前生效版本;这条**不能**当完整续作验收) +# --manual-start 0|1(策略未拉起时是否显式 docker start,默认 1) # set -uo pipefail -SCRIPT_VERSION=2 +SCRIPT_VERSION=3 MODE="" SCOPE="internal" +KIND="auto" +EXPECT_SNAPSHOT="" EXPECT_JOB="" SINCE="" CONFIRM_KILL=0 -EXPECT_COUNTS="${EXPECT_COUNTS:-knowledges=4,chunks=38,resources=22,wiki_pages=31}" +ALLOW_SAME_SNAPSHOT=0 +MANUAL_START=1 +EXPECT_COUNTS="${EXPECT_COUNTS:-}" VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}" WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}" @@ -42,7 +60,9 @@ COMPOSE_SERVICE="${COMPOSE_SERVICE:-worker}" BARRIER_POINT="${BARRIER_POINT:-after:database}" BARRIER_TIMEOUT="${BARRIER_TIMEOUT:-300s}" HIT_WAIT_S="${HIT_WAIT_S:-900}" -RESTART_WAIT_S="${RESTART_WAIT_S:-180}" +KILL_PROOF_S="${KILL_PROOF_S:-30}" +POLICY_RESTART_WAIT_S="${POLICY_RESTART_WAIT_S:-20}" +RESTART_WAIT_S="${RESTART_WAIT_S:-90}" TAKEOVER_WAIT_S="${TAKEOVER_WAIT_S:-420}" DRILL_HOME="${DRILL_HOME:-$HOME/.local/share/knowledge-governance/drills}" @@ -52,12 +72,16 @@ while [ $# -gt 0 ]; do --run) MODE=run ;; --restore) MODE=restore ;; --scope) SCOPE="${2:-}"; shift ;; + --kind) KIND="${2:-}"; shift ;; + --snapshot) EXPECT_SNAPSHOT="${2:-}"; shift ;; --job) EXPECT_JOB="${2:-}"; shift ;; --since) SINCE="${2:-}"; shift ;; --expect) EXPECT_COUNTS="${2:-}"; shift ;; --point) BARRIER_POINT="${2:-}"; shift ;; + --manual-start) MANUAL_START="${2:-}"; shift ;; + --allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;; --confirm-kill) CONFIRM_KILL=1 ;; - -h|--help) sed -n '1,30p' "$0"; exit 0 ;; + -h|--help) sed -n '1,42p' "$0"; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 64 ;; esac shift @@ -67,10 +91,19 @@ if [ -z "$MODE" ]; then echo "must pass one of --check | --run | --restore" >&2 exit 64 fi +case "$KIND" in + auto|sync|restore) ;; + *) echo "--kind must be auto|sync|restore (got $KIND)" >&2; exit 64 ;; +esac +case "$MANUAL_START" in + 0|1) ;; + *) echo "--manual-start must be 0|1" >&2; exit 64 ;; +esac # set -u:所有跨函数使用的变量先给默认值 WORKER_IMAGE_ID="" WORKER_IMAGE="" +WORKER_CONTAINER_ID="" ARCHIVE_ROOT_IN_CONTAINER="" ARCHIVE_HOST="" BARRIER_HOST="" @@ -90,12 +123,18 @@ JOB_KIND="" JOB_SNAPSHOT="" JOB_TARGET="" JOB_CREATED="" +PRE_EFFECTIVE="" WS_DIGEST="" WS_REUSED="" WS_TARGET="" WS_BACKUP="" WS_FILE="" KILLED_AT="" +KILL_PID="" +KILL_PROOF="" +RESTART_SOURCE="" +RESTART_COUNT_BEFORE="" +EVENTS_PID="" STATUS_OVERRIDE="" ARMED=0 COMPOSE_ARGS=() @@ -149,8 +188,33 @@ url_part() { # now_epoch() { date -u +%s; } utc_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } +# 目标快照自己的期望内容:只认归档里的 manifest(不写死 4/38/31)。 +# resources 故意不进默认期望:同租户其它知识库的资源不在预备份范围(trap-16), +# 目标里会保留它们,把 21 写成期望就会永远红。 +snapshot_expectations() { # + python3 - "$ARCHIVE_HOST/versions/$1.json" "$SCOPE" <<'PY' +import json, sys, pathlib +path, scope = sys.argv[1], sys.argv[2] +try: + manifest = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) +except Exception as exc: + sys.exit("读不到目标快照 manifest(%s):%s" % (path, exc)) +if manifest.get("format") != "weknora-library-snapshot/v1": + sys.exit("manifest format=%r 不是 weknora-library-snapshot/v1" % manifest.get("format")) +if manifest.get("scope") != scope: + sys.exit("manifest scope=%r 与请求 %r 不一致" % (manifest.get("scope"), scope)) +counts = {p["name"]: p["count"] for p in manifest.get("parts", []) if p.get("kind") == "database"} +for table in ("knowledges", "chunks"): + if table not in counts: + sys.exit("manifest 缺少 database.%s:不能推导期望" % table) +wiki = [p["count"] for p in manifest.get("parts", []) if p.get("kind") == "wiki" and p.get("name") == "wiki_pages"] +print("knowledges=%d,chunks=%d,wiki_pages=%d" % (counts["knowledges"], counts["chunks"], wiki[0] if wiki else 0)) +PY +} + finish() { # finish local code="$1" status="PASS" + if [ -n "$EVENTS_PID" ]; then kill "$EVENTS_PID" 2>/dev/null; fi if grep -q $'\tfail\t' "$CHECKS" 2>/dev/null; then status="FAIL"; fi if grep -q $'\tnot_run\t' "$CHECKS" 2>/dev/null && [ "$code" -eq 0 ]; then status="FAIL"; fi [ -n "$STATUS_OVERRIDE" ] && status="$STATUS_OVERRIDE" @@ -162,11 +226,21 @@ finish() { # finish "status": "$status", "exit_code": $code, "scope": "$SCOPE", + "requested_kind": "$KIND", + "job_kind": "$JOB_KIND", "worker_container": "$WORKER_CONTAINER", + "worker_container_id": "$WORKER_CONTAINER_ID", "worker_image": "$WORKER_IMAGE", + "worker_image_id": "$WORKER_IMAGE_ID", "barrier_point": "$BARRIER_POINT", "job": "$JOB_ID", + "target_snapshot": "$JOB_TARGET", + "pre_effective_snapshot": "$PRE_EFFECTIVE", + "expected_counts": "$EXPECT_COUNTS", "killed_at": "$KILLED_AT", + "killed_pid": "$KILL_PID", + "kill_proof": "$KILL_PROOF", + "restart_source": "$RESTART_SOURCE", "writeset_digest": "$WS_DIGEST", "evidence_dir": "$EVIDENCE" } @@ -186,11 +260,13 @@ preflight() { command -v docker >/dev/null 2>&1 || die 2 preflight_docker "docker 命令不可用" docker info >/dev/null 2>&1 || die 2 preflight_docker "docker daemon 不可用" docker compose version >/dev/null 2>&1 || die 2 preflight_docker "docker compose 不可用" - record preflight_docker pass "docker + compose 可用" + command -v python3 >/dev/null 2>&1 || die 2 preflight_docker "python3 不可用(推导期望内容要用)" + record preflight_docker pass "docker + compose + python3 可用" docker inspect "$WORKER_CONTAINER" >/dev/null 2>&1 || die 2 preflight_worker "worker 容器不存在:$WORKER_CONTAINER" WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" WORKER_IMAGE="$(container_field "$WORKER_CONTAINER" '{{.Config.Image}}')" + WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" local running running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" [ "$running" = "true" ] || die 2 preflight_worker "worker 没在运行(先把它起回来)" @@ -209,6 +285,7 @@ preflight() { ARCHIVE_HOST="$(archive_root_host "$ARCHIVE_ROOT_IN_CONTAINER")" { [ -n "$ARCHIVE_HOST" ] && [ -d "$ARCHIVE_HOST" ]; } || die 2 preflight_archive "归档根在宿主机上找不到(容器内 $ARCHIVE_ROOT_IN_CONTAINER)" [ -d "$ARCHIVE_HOST/writesets" ] || die 2 preflight_archive "归档根下没有 writesets/:台账不在预期位置" + [ -d "$ARCHIVE_HOST/versions" ] || die 2 preflight_archive "归档根下没有 versions/:快照清单不在预期位置" [ -w "$ARCHIVE_HOST" ] || die 2 preflight_archive "归档根宿主机路径不可写:$ARCHIVE_HOST" BARRIER_HOST="$ARCHIVE_HOST/barrier" record preflight_archive pass "archive=$ARCHIVE_HOST barrier=$BARRIER_HOST" @@ -268,6 +345,13 @@ preflight() { docker inspect "$TARGET_DB_CONTAINER" >/dev/null 2>&1 || die 2 preflight_target_db "目标库容器不存在:$TARGET_DB_CONTAINER" record preflight_target_db pass "target db=$TARGET_DB_CONTAINER/$TARGET_DB_NAME user=$TARGET_DB_USER" + # 独立写集合校验器缺失 ⇒ 失败。上一版在缺失时把 canonical 记成 not_run 还 pass, + # 等于"没有校验也算校验过"。 + if [ ! -x "$VERIFIER" ]; then + die 2 preflight_verifier "写集合独立校验器不存在或不可执行:$VERIFIER(缺校验不许当通过;用 WRITESET_VERIFIER 指向它)" + fi + record preflight_verifier pass "verifier=$VERIFIER" + [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \ "worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore" record preflight_not_armed pass "barrier not armed" @@ -278,6 +362,9 @@ preflight() { *) die 2 preflight_job_arg "--job 不是整库作业 id:$EXPECT_JOB" ;; esac fi + if [ -n "$EXPECT_SNAPSHOT" ]; then + [ "${#EXPECT_SNAPSHOT}" -ge 16 ] || die 2 preflight_job_arg "--snapshot 不像快照 id:$EXPECT_SNAPSHOT" + fi record preflight_done pass "OK" } @@ -337,8 +424,11 @@ arm_barrier() { local armed armed="$(worker_env KG_TEST_BARRIER_DIR)" [ "$armed" = "$ARCHIVE_ROOT_IN_CONTAINER/barrier" ] || die 2 arm_barrier "屏障注入没生效(KG_TEST_BARRIER_DIR=${armed:-空})" + # 注入会重建容器:镜像/容器身份以重建后的为准(后面要比对"重启前后同镜像")。 + WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" + WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" ARMED=1 - record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT" + record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT container=$WORKER_CONTAINER_ID" } restore_barrier() { @@ -359,7 +449,7 @@ wait_for_hit() { local base deadline base="$BARRIER_HOST/$(sed 's/[:/ ]/-/g' <<<"$BARRIER_POINT")" deadline=$(( $(now_epoch) + HIT_WAIT_S )) - log "ARMED: 现在请在页面上发起同步作业(scope=$SCOPE)。等待屏障点位 $BARRIER_POINT 命中 …" + log "ARMED: 现在请在页面上发起作业(scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定})。等待屏障点位 $BARRIER_POINT 命中 …" while [ "$(now_epoch)" -lt "$deadline" ]; do if [ -f "$base.timeout" ]; then cp "$base.timeout" "$EVIDENCE/" 2>/dev/null @@ -388,6 +478,12 @@ lock_job() { IFS='|' read -r JOB_STATE JOB_SCOPE JOB_KIND JOB_SNAPSHOT JOB_TARGET JOB_CREATED <<<"$row" [ "$JOB_SCOPE" = "$SCOPE" ] || die 3 lock_job "作业 scope=$JOB_SCOPE 与请求 $SCOPE 不一致" [ "$JOB_STATE" = "running" ] || die 3 lock_job "作业状态是 $JOB_STATE 不是 running:命中时它已经不在写入路径上了" + if [ "$KIND" != "auto" ] && [ "$JOB_KIND" != "$KIND" ]; then + die 3 lock_job "作业 kind=$JOB_KIND 与请求 --kind $KIND 不一致(期望内容也会按错的那一类推导)" + fi + if [ -n "$EXPECT_SNAPSHOT" ] && [ "$JOB_TARGET" != "$EXPECT_SNAPSHOT" ]; then + die 3 lock_job "作业目标快照 $JOB_TARGET 与 --snapshot $EXPECT_SNAPSHOT 不一致:拒绝把它当这次演练" + fi if [ -n "$SINCE" ]; then local since_epoch created_epoch if [[ "$SINCE" =~ ^[0-9]+$ ]]; then since_epoch="$SINCE"; else since_epoch="$(date -u -d "$SINCE" +%s 2>/dev/null || echo 0)"; fi @@ -395,13 +491,33 @@ lock_job() { { [ "$since_epoch" -gt 0 ] && [ "$created_epoch" -ge "$since_epoch" ]; } || die 3 lock_job "作业创建时间 $JOB_CREATED 不满足 --since $SINCE(或 --since 解析不了)" fi + # 目标快照 == 中断前的当前生效版本 ⇒ 这次"接管"只可能走同快照快速收尾, + # 证明不了完整续作(2026-10-29 的现场正是如此)。杀之前就拒绝,别浪费一次中断。 + PRE_EFFECTIVE="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")" + if [ -n "$PRE_EFFECTIVE" ] && [ "$JOB_TARGET" = "$PRE_EFFECTIVE" ] && [ "$ALLOW_SAME_SNAPSHOT" -ne 1 ]; then + die 3 lock_job "目标快照 $JOB_TARGET 就是中断前的当前生效版本:同快照收尾不能当完整续作的验收证据(要演练请换一个没生效过的快照;确要同快照加 --allow-same-snapshot)" + fi others="$(gov_sql "select coalesce(string_agg(id, ','),'') from library_jobs where state in ('pending','running') and id<>'$JOB_ID'")" [ -z "$others" ] || die 3 lock_job "还有别的作业在飞:$others(只允许杀这一个作业的执行者)" - record lock_job pass "job=$JOB_ID state=running scope=$JOB_SCOPE kind=$JOB_KIND target=$JOB_TARGET created=$JOB_CREATED" + + local expected + if [ -n "$EXPECT_COUNTS" ]; then + expected="$EXPECT_COUNTS" + log "期望内容来自 --expect:$expected" + else + expected="$(snapshot_expectations "$JOB_TARGET")" || die 3 lock_job "推导目标快照的期望内容失败($JOB_TARGET)" + EXPECT_COUNTS="$expected" + log "期望内容来自快照 manifest($JOB_TARGET):$expected;resources 不在默认期望内(同租户其它知识库资源会保留)" + fi + case "$expected" in + *=*) ;; + *) die 3 lock_job "期望内容形状不对:$expected" ;; + esac + record lock_job pass "kind=$JOB_KIND job=$JOB_ID state=running scope=$JOB_SCOPE target=$JOB_TARGET pre_effective=${PRE_EFFECTIVE:-none} expect=$EXPECT_COUNTS created=$JOB_CREATED" } read_writeset_evidence() { - local row canonical="not_run" + local row row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据" IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row" @@ -411,40 +527,103 @@ read_writeset_evidence() { [ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE" grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)" grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID" - if [ -e "$VERIFIER" ]; then - "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \ - >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)" - canonical="verified" - fi - record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=$canonical" + # 独立校验器是必需的(preflight 已确认存在):它判定不符就不能继续。 + "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \ + >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)" + record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=verified" } kill_worker() { [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill:杀容器必须是显式动作" - local before_restart job_state deadline exit_code="" + local before_restart job_state deadline owner_uid before_restart="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" job_state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" [ "$job_state" = "running" ] || die 4 kill "杀之前作业状态变成 $job_state:这次不构成写入中的中断" + KILL_PID="$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')" + [ -n "$KILL_PID" ] && [ "$KILL_PID" -gt 1 ] || die 4 kill "读不到 worker 的宿主机 PID(${KILL_PID:-空})" + + # 只有"同一 uid"或 root 才能向容器 PID1 发信号;先证明能杀,再谈演练。 + owner_uid="$(stat -c %u "/proc/$KILL_PID" 2>/dev/null)" + if [ "$(id -u)" != "0" ] && [ "${owner_uid:-x}" != "$(id -u)" ]; then + die 4 kill "宿主机 PID $KILL_PID 属于 uid=${owner_uid:-?},当前 uid=$(id -u) 无权 SIGKILL:不能用「进程崩溃」语义演练(换 --manual-start 方案前先解决权限)" + fi + kill -0 "$KILL_PID" 2>/dev/null || die 4 kill "PID $KILL_PID 不存活,杀不到真实进程" + + # 退出码证据从 docker events 取:restart policy 会很快把容器拉起来, + # State.ExitCode 随即被重置成 0,事后 inspect 读不到 137(这正是上一版没留下 + # "真的 137"证据的原因之一)。 + : >"$EVIDENCE/docker-events.log" + timeout 180 docker events --filter "container=$WORKER_CONTAINER_ID" --since "$(now_epoch)" \ + --format '{{json .}}' >>"$EVIDENCE/docker-events.log" 2>>"$EVIDENCE/docker.log" & + EVENTS_PID=$! + KILLED_AT="$(utc_now)" - log "SIGKILL $WORKER_CONTAINER (job=$JOB_ID at $KILLED_AT)" - docker kill --signal=KILL "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 4 kill "docker kill 失败" - deadline=$(( $(now_epoch) + 30 )) + RESTART_COUNT_BEFORE="$before_restart" + log "SIGKILL worker container PID1 host_pid=$KILL_PID (job=$JOB_ID at $KILLED_AT) — 进程崩溃语义,不是 docker kill" + kill -9 "$KILL_PID" 2>>"$EVIDENCE/docker.log" || die 4 kill "向 PID $KILL_PID 发送 SIGKILL 失败" + + deadline=$(( $(now_epoch) + KILL_PROOF_S )) + KILL_PROOF="" while [ "$(now_epoch)" -lt "$deadline" ]; do - exit_code="$(container_field "$WORKER_CONTAINER" '{{.State.ExitCode}}')" - [ "$exit_code" = "137" ] && break + if grep '"Action":"die"' "$EVIDENCE/docker-events.log" 2>/dev/null | grep -q '"exitCode":"137"'; then + KILL_PROOF="docker-events die exitCode=137" + break + fi sleep 1 done - [ "$exit_code" = "137" ] || die 4 kill "容器退出码是 ${exit_code:-?},不是 137(SIGKILL)" - record kill pass "container=$WORKER_CONTAINER exit=137 killed_at=$KILLED_AT restart_before=$before_restart" + [ -n "$KILL_PROOF" ] || die 4 kill "docker events 里没有 exitCode=137 的 die 事件(${KILL_PROOF_S}s 内):这次没有证明真被杀" + record kill pass "container=$WORKER_CONTAINER id=$WORKER_CONTAINER_ID host_pid=$KILL_PID exit=137 proof=\"$KILL_PROOF\" killed_at=$KILLED_AT restart_before=$RESTART_COUNT_BEFORE semantics=process-crash-sigkill" +} + +wait_restart() { + local deadline running + deadline=$(( $(now_epoch) + POLICY_RESTART_WAIT_S )) + RESTART_SOURCE="" + while [ "$(now_epoch)" -lt "$deadline" ]; do + running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" + if [ "$running" = "true" ] && [ "$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')" != "$KILL_PID" ]; then + RESTART_SOURCE="policy" + break + fi + sleep 1 + done + if [ -z "$RESTART_SOURCE" ]; then + if [ "$MANUAL_START" -ne 1 ]; then + die 5 restart "restart policy 在 ${POLICY_RESTART_WAIT_S}s 内没有拉起(未允许显式 docker start):这次不算自动接管" + fi + log "restart policy 没有在 ${POLICY_RESTART_WAIT_S}s 内拉起:显式 docker start(记 restart_source=manual_start,不是自动恢复)" + docker start "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 5 restart "显式 docker start 失败" + RESTART_SOURCE="manual_start" + deadline=$(( $(now_epoch) + RESTART_WAIT_S )) + while [ "$(now_epoch)" -lt "$deadline" ]; do + running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" + [ "$running" = "true" ] && break + sleep 1 + done + [ "$running" = "true" ] || die 5 restart "显式启动后容器仍没运行" + fi + local image_after cid_after restarts_after + image_after="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" + cid_after="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" + restarts_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" + [ "$image_after" = "$WORKER_IMAGE_ID" ] || die 5 restart "重启后镜像变了:$WORKER_IMAGE_ID -> $image_after(不是同一个产物)" + [ "$cid_after" = "$WORKER_CONTAINER_ID" ] || die 5 restart "重启后容器身份变了:$WORKER_CONTAINER_ID -> $cid_after" + record restart pass "source=$RESTART_SOURCE image_unchanged=1 container_unchanged=1 restart_count=$RESTART_COUNT_BEFORE->$restarts_after" } wait_takeover() { - local deadline state="" restart_after + local deadline state="" last_note=0 deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S )) log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…" while [ "$(now_epoch)" -lt "$deadline" ]; do state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" case "$state" in succeeded|failed|cancelled) break ;; esac + # 60s 没动静就把现场抓下来(别再空等到底才知道为什么)。 + if [ $(( $(now_epoch) - last_note )) -ge 60 ]; then + last_note="$(now_epoch)" + docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | tail -5 >>"$EVIDENCE/worker.log" + log "接管等待中:job state=${state:-?} worker_running=$(container_field "$WORKER_CONTAINER" '{{.State.Running}}') queue=$(gov_sql "select coalesce(string_agg(type||':'||state,','),'') from jobs where type like 'library.%' and state in ('pending','running','retry')")" + fi sleep 2 done case "$state" in @@ -452,8 +631,22 @@ wait_takeover() { failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;; *) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;; esac - restart_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" - record wait_takeover pass "same job succeeded; RestartCount=$restart_after" + # 人工拉起/策略拉起都要能对得上"启动扫描真的接了这个作业"。 + local resumed + resumed="$(docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | grep -c "library job $JOB_ID resumed after worker restart")" + [ "${resumed:-0}" -ge 1 ] || die 5 wait_takeover "worker 日志里没有本作业的 resume 记录:接管来源不明" + record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; resume_log=$resumed" +} + +verify_stages_complete() { + local stages missing="" stage + stages="$(gov_sql "select stage||':'||state from library_job_stages where job_id='$JOB_ID'")" + for stage in database resources index wiki verify; do + grep -qx "$stage:succeeded" <<<"$stages" || missing+="$stage " + done + [ -z "$missing" ] || die 6 verify_stages \ + "作业没有真跑完写入链路:未 succeeded 的阶段 = $missing(stages=$(tr '\n' ',' <<<"$stages"))" + record verify_stages pass "database/resources/index/wiki/verify 全部 succeeded" } verify_takeover_semantics() { @@ -469,7 +662,7 @@ verify_takeover_semantics() { local eff_snapshot eff_job eff_superseded IFS='|' read -r eff_snapshot eff_job eff_superseded <<<"$eff" [ "$eff_snapshot" = "$JOB_TARGET" ] || die 6 verify_effective "当前生效版本是 ${eff_snapshot:-无},不是本作业目标 $JOB_TARGET" - [ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无},不是 $JOB_ID" + [ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无},不是 $JOB_ID(同快照由别作业登记不算本次完成)" [ -z "$eff_superseded" ] || die 6 verify_effective "生效版本已被顶替($eff_superseded)" after_digest="$(gov_sql "select coalesce(evidence->>'writeset_digest','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ "$after_digest" = "$WS_DIGEST" ] || die 6 verify_effective "接管后写集合摘要变了($WS_DIGEST -> ${after_digest:-空})" @@ -477,12 +670,16 @@ verify_takeover_semantics() { } verify_target_counts() { - local part table want got sql="" failed=0 detail="" + local part table want got failed=0 detail="" sql="" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}" + case "$table" in + *[!a-z_]*) die 6 verify_counts "期望里的表名不合法:$table" ;; + esac sql+="select '$table='||count(*) from $table; " done - got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" 2>>"$EVIDENCE/psql.log")" + got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" \ + 2>>"$EVIDENCE/psql.log")" [ -n "$got" ] || die 6 verify_counts "目标库读数失败($TARGET_DB_CONTAINER/$TARGET_DB_NAME)" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}"; want="${part##*=}" @@ -509,29 +706,29 @@ verify_service_restored() { "还有未释放的维护占用(scope=$held instance=$held_instance)" { [ "$released" -ge 1 ] && [ "$released_instance" -ge 1 ]; } || die 6 verify_service \ "本作业的维护记录没有释放证据(scope=$released instance=$released_instance)" - # 业务 health 只在服务已恢复时作为附加证据:维护期间 app 被停是预期,不作为拒绝理由。 record verify_service pass "instance_running=$instance_running ingress_running=$ingress_running health(instance)=$(health_of "$TARGET_INSTANCE") health(ingress)=$(health_of "$INGRESS_CONTAINER") released=$released/$released_instance" } save_evidence() { { - echo "# 脱敏证据(job=$JOB_ID scope=$SCOPE killed_at=$KILLED_AT)" + echo "# 脱敏证据(job=$JOB_ID kind=$JOB_KIND scope=$SCOPE target=$JOB_TARGET killed_at=$KILLED_AT restart_source=$RESTART_SOURCE)" echo "## library_jobs" gov_sql "select id||E'\t'||scope||E'\t'||kind||E'\t'||state||E'\t'||stage||E'\t'||coalesce(snapshot_id,'')||E'\t'||coalesce(target_snapshot_id,'')||E'\t'||created_at::text||E'\t'||coalesce(finished_at::text,'') from library_jobs where id='$JOB_ID'" echo "## library_job_stages" gov_sql "select stage||E'\t'||state||E'\t'||coalesce(total::text,'')||E'\t'||coalesce(completed::text,'')||E'\t'||coalesce(detail,'')||E'\t'||updated_at::text from library_job_stages where job_id='$JOB_ID' order by updated_at" echo "## library_versions(本作业)" gov_sql "select kind||E'\t'||snapshot_id||E'\t'||source_job_id||E'\t'||created_at::text||E'\t'||coalesce(superseded_at::text,'')||E'\t'||evidence::text from library_versions where source_job_id='$JOB_ID' order by created_at" - echo "## 维护占用(本作业)" - gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'" + echo "## 维护占用(本作业;实例表列是 owner_scope)" + gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||owner_scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'" echo "## 目标内容计数" docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "select 'knowledges='||count(*) from knowledges union all select 'chunks='||count(*) from chunks union all select 'resources='||count(*) from resources union all select 'wiki_pages='||count(*) from wiki_pages" 2>/dev/null } >"$EVIDENCE/job-evidence.txt" + docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" >"$EVIDENCE/worker.log" 2>&1 log "evidence written: $EVIDENCE/job-evidence.txt" } # ---------- 主流程 ---------- -log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE evidence=$EVIDENCE" +log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定} evidence=$EVIDENCE" preflight case "$MODE" in @@ -550,7 +747,9 @@ case "$MODE" in lock_job read_writeset_evidence kill_worker + wait_restart wait_takeover + verify_stages_complete verify_takeover_semantics verify_target_counts verify_service_restored