#!/usr/bin/env bash # library-kill-takeover.sh —— 真实 worker 中断演练（只在本机测试栈） # # 判据只有一条：**真的把 worker 进程 SIGKILL 掉（exit 137）→ 同一个作业被认领并 # 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**，全成立才 exit 0。 # missed / notrun / timeout / 只同快照收尾 一律非 0，并在 evidence 里写成 not_run 而不是"通过"。 # # ⚠️ 2026-10-29T19:56Z 这次真演练（exit 5）教会两件事，都是判据缺陷： # 1) 脚本要的文案 `library job resumed after worker restart` 只有"新建队列行" # 那条代码路径会打印；真实走的是"队列行已存在"那条。于是 worker 只留下汇总行 # `library job takeover sweep: 1 resumed`，脚本判不出是哪个作业、也判不出它是否被认领。 # 现在改由 scripts/takeover_log_contract.py 判**认领行 + 决策行**（都带作业 id 与 # target/backup，且时间戳必须晚于杀点）；只有汇总行/别的作业/杀前日志一律不算证据。 # 2) 老判据要求 `prebackup:skipped`。真实接管是"复用崩溃前那份预备份重跑整条链路"， # 阶段会写成 succeeded（release.go 记的是"预备份 "）。**不能因此取消备份校验**： # 现在要求预备份快照身份与崩溃前一致、清单文件字节/mtime 未变、版本行 created_at # 仍是首次登记时刻（复用而非重做），写集合摘要也不变。 # # 为什么不再用 `docker kill`： # docker kill/stop 走 Docker API 的"手动停止"，restart policy 不生效（`unless-stopped` # 也不生效），容器就一直躺着。2026-10-29T19:16Z 的现场就是这样：脚本等了 420s 判失败， # 最后由人工 `docker start` 才接管。那不是"崩溃自恢复"，只是"有人按了开始"。 # 现在的命中手段是**宿主机向容器 PID1 发 SIGKILL**（同一 uid 才允许）：这才是进程崩溃， # Docker 看到的是进程死亡 ⇒ restart policy 按策略拉起，语义与真实故障一致。 # 若策略没在限期内拉起，脚本可以显式 `docker start`，但必须在 evidence 里记成 # restart_source=manual_start，**不许**算成"自动恢复"。 # # 命中点位仍是"固定同步屏障"（internal/testbarrier，默认关闭，仅本机注入）： # worker 在指定阶段写完后写 /after-database.hit 并停住 → 本脚本看到 .hit # → SIGKILL → 按策略重启/显式拉起 → 启动扫描认领同一作业。 # 屏障目录在本机归档根下（/barrier），没人放行就自行继续（记 not_run）。 # # 期望内容按**目标快照自己的 manifest** 取（knowledges/chunks/wiki_pages），不是写死 # 4/38/31：同一个脚本既能跑 sync 也能跑 restore，期望随快照变。resources 不进默认 # 期望——同租户其它知识库的资源不在预备份范围内（trap-16），目标里会保留它们。 # # 用法： # bash scripts/library-kill-takeover.sh --check # 前置体检（不是演练通过） # bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill \ # --kind restore --snapshot <目标快照 id> # → 打印 ARMED 后，父在页面上发起该作业；脚本自己命中、杀、验证、收尾 # bash scripts/library-kill-takeover.sh --restore # 撤掉屏障注入 # 可选：--kind sync|restore|auto（默认 auto，给了就强校验作业类型） # --snapshot （校验命中的作业目标就是它） # --job <业务作业 id>｜--since # --expect "knowledges=..,chunks=..,wiki_pages=.."（覆盖 manifest 推导） # --expect-decision fresh|pending_final（默认 fresh；at after:database 之后 # 只能是 fresh——pending_final 等于"只差收尾"，正是上一轮的假完成） # --allow-same-snapshot（允许目标快照 == 中断前生效版本；这条**不能**当完整续作验收） # --manual-start 0|1（策略未拉起时是否显式 docker start，默认 1） # set -uo pipefail SCRIPT_VERSION=4 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODE="" SCOPE="internal" KIND="auto" EXPECT_SNAPSHOT="" EXPECT_JOB="" SINCE="" CONFIRM_KILL=0 ALLOW_SAME_SNAPSHOT=0 MANUAL_START=1 EXPECT_DECISION="${EXPECT_DECISION:-fresh}" EXPECT_COUNTS="${EXPECT_COUNTS:-}" VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}" LOG_CONTRACT="${TAKEOVER_LOG_CONTRACT:-$SCRIPT_DIR/takeover_log_contract.py}" WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}" GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}" GOV_DB_USER="${GOV_DB_USER:-governance}" GOV_DB_NAME="${GOV_DB_NAME:-governance}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-knowledge-governance-p1}" COMPOSE_SERVICE="${COMPOSE_SERVICE:-worker}" BARRIER_POINT="${BARRIER_POINT:-after:database}" BARRIER_TIMEOUT="${BARRIER_TIMEOUT:-300s}" HIT_WAIT_S="${HIT_WAIT_S:-900}" KILL_PROOF_S="${KILL_PROOF_S:-30}" POLICY_RESTART_WAIT_S="${POLICY_RESTART_WAIT_S:-20}" RESTART_WAIT_S="${RESTART_WAIT_S:-90}" TAKEOVER_WAIT_S="${TAKEOVER_WAIT_S:-420}" DRILL_HOME="${DRILL_HOME:-$HOME/.local/share/knowledge-governance/drills}" while [ $# -gt 0 ]; do case "$1" in --check) MODE=check ;; --run) MODE=run ;; --restore) MODE=restore ;; --scope) SCOPE="${2:-}"; shift ;; --kind) KIND="${2:-}"; shift ;; --snapshot) EXPECT_SNAPSHOT="${2:-}"; shift ;; --job) EXPECT_JOB="${2:-}"; shift ;; --since) SINCE="${2:-}"; shift ;; --expect) EXPECT_COUNTS="${2:-}"; shift ;; --expect-decision) EXPECT_DECISION="${2:-}"; shift ;; --point) BARRIER_POINT="${2:-}"; shift ;; --manual-start) MANUAL_START="${2:-}"; shift ;; --allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;; --confirm-kill) CONFIRM_KILL=1 ;; -h|--help) sed -n '1,42p' "$0"; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 64 ;; esac shift done if [ -z "$MODE" ]; then echo "must pass one of --check | --run | --restore" >&2 exit 64 fi case "$KIND" in auto|sync|restore) ;; *) echo "--kind must be auto|sync|restore (got $KIND)" >&2; exit 64 ;; esac case "$MANUAL_START" in 0|1) ;; *) echo "--manual-start must be 0|1" >&2; exit 64 ;; esac case "$EXPECT_DECISION" in fresh|pending_final) ;; *) echo "--expect-decision must be fresh|pending_final (got $EXPECT_DECISION)" >&2; exit 64 ;; esac # set -u：所有跨函数使用的变量先给默认值 WORKER_IMAGE_ID="" WORKER_IMAGE="" WORKER_CONTAINER_ID="" ARCHIVE_ROOT_IN_CONTAINER="" ARCHIVE_HOST="" BARRIER_HOST="" TARGET_INSTANCE="" TARGET_PROJECT="" TARGET_INGRESS_SERVICE="" TARGET_WRITERS="" INGRESS_CONTAINER="" TARGET_DB_CONTAINER="" TARGET_DB_USER="" TARGET_DB_NAME="" HIT_SUBJECT="" JOB_ID="" JOB_STATE="" JOB_SCOPE="" JOB_KIND="" JOB_SNAPSHOT="" JOB_TARGET="" JOB_CREATED="" PRE_EFFECTIVE="" WS_DIGEST="" WS_REUSED="" WS_TARGET="" WS_BACKUP="" WS_FILE="" WS_FILE_SHA="" WS_BACKUP_SHA="" WS_BACKUP_MTIME="" WS_BACKUP_CREATED="" WS_REUSED="" DECISION="" CLAIM_AT="" LOG_RAW="" KILLED_AT="" KILL_PID="" KILL_PROOF="" RESTART_SOURCE="" RESTART_COUNT_BEFORE="" EVENTS_PID="" STATUS_OVERRIDE="" ARMED=0 COMPOSE_ARGS=() STAMP="$(date -u +%Y%m%dT%H%M%SZ)" EVIDENCE="$DRILL_HOME/$STAMP" mkdir -p "$EVIDENCE" || { echo "cannot create evidence dir $EVIDENCE" >&2; exit 70; } CHECKS="$EVIDENCE/checks.tsv" : >"$CHECKS" log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$EVIDENCE/phases.log"; } # record record() { printf '%s\t%s\t%s\n' "$1" "$2" "${3:-}" >>"$CHECKS" log "CHECK $1 = $2${3:+ ($3)}" } die() { # die record "$2" fail "$3" log "ABORT: $2 — $3" finish "$1" } # ---------- 小工具 ---------- worker_env() { # 只读非敏感变量 docker inspect "$WORKER_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n "s/^$1=//p" | head -1 } gov_sql() { docker exec "$GOV_DB_CONTAINER" psql -U "$GOV_DB_USER" -d "$GOV_DB_NAME" -Atc "$1" 2>>"$EVIDENCE/psql.log"; } container_field() { docker inspect "$1" --format "$2" 2>/dev/null; } archive_root_host() { docker inspect "$WORKER_CONTAINER" --format '{{range .Mounts}}{{.Destination}}={{.Source}}{{println}}{{end}}' 2>/dev/null | sed -n "s#^$1=##p" | head -1 } container_of_service() { # docker ps -a --filter "label=com.docker.compose.project=$1" \ --format '{{.Names}}|{{.Label "com.docker.compose.service"}}' | awk -F'|' -v svc="$2" '$2==svc{print $1; exit}' } health_of() { local status status="$(docker inspect "$1" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no_healthcheck{{end}}' 2>/dev/null)" echo "${status:-unknown}" } url_part() { # case "$2" in user) sed -E 's#^[a-zA-Z0-9+]+://([^:/@]+).*#\1#' <<<"$1" ;; host) sed -E 's#^[a-zA-Z0-9+]+://[^@]*@([^:/]+).*#\1#' <<<"$1" ;; db) sed -E 's#^[a-zA-Z0-9+]+://[^/]+/([^?]+).*#\1#' <<<"$1" ;; *) echo "" ;; esac } now_epoch() { date -u +%s; } utc_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } # 目标快照自己的期望内容：只认归档里的 manifest（不写死 4/38/31）。 # resources 故意不进默认期望：同租户其它知识库的资源不在预备份范围（trap-16）， # 目标里会保留它们，把 21 写成期望就会永远红。 snapshot_expectations() { # python3 - "$ARCHIVE_HOST/versions/$1.json" "$SCOPE" <<'PY' import json, sys, pathlib path, scope = sys.argv[1], sys.argv[2] try: manifest = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) except Exception as exc: sys.exit("读不到目标快照 manifest（%s）：%s" % (path, exc)) if manifest.get("format") != "weknora-library-snapshot/v1": sys.exit("manifest format=%r 不是 weknora-library-snapshot/v1" % manifest.get("format")) if manifest.get("scope") != scope: sys.exit("manifest scope=%r 与请求 %r 不一致" % (manifest.get("scope"), scope)) counts = {p["name"]: p["count"] for p in manifest.get("parts", []) if p.get("kind") == "database"} for table in ("knowledges", "chunks"): if table not in counts: sys.exit("manifest 缺少 database.%s：不能推导期望" % table) wiki = [p["count"] for p in manifest.get("parts", []) if p.get("kind") == "wiki" and p.get("name") == "wiki_pages"] print("knowledges=%d,chunks=%d,wiki_pages=%d" % (counts["knowledges"], counts["chunks"], wiki[0] if wiki else 0)) PY } finish() { # finish local code="$1" status="PASS" if [ -n "$EVENTS_PID" ]; then kill "$EVENTS_PID" 2>/dev/null; fi if grep -q $'\tfail\t' "$CHECKS" 2>/dev/null; then status="FAIL"; fi if grep -q $'\tnot_run\t' "$CHECKS" 2>/dev/null && [ "$code" -eq 0 ]; then status="FAIL"; fi [ -n "$STATUS_OVERRIDE" ] && status="$STATUS_OVERRIDE" cp "$CHECKS" "$EVIDENCE/checks.snapshot.tsv" 2>/dev/null cat >"$EVIDENCE/summary.json" </dev/null 2>&1 || die 2 preflight_docker "docker 命令不可用" docker info >/dev/null 2>&1 || die 2 preflight_docker "docker daemon 不可用" docker compose version >/dev/null 2>&1 || die 2 preflight_docker "docker compose 不可用" command -v python3 >/dev/null 2>&1 || die 2 preflight_docker "python3 不可用（推导期望内容要用）" record preflight_docker pass "docker + compose + python3 可用" docker inspect "$WORKER_CONTAINER" >/dev/null 2>&1 || die 2 preflight_worker "worker 容器不存在：$WORKER_CONTAINER" WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" WORKER_IMAGE="$(container_field "$WORKER_CONTAINER" '{{.Config.Image}}')" WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" local running running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" [ "$running" = "true" ] || die 2 preflight_worker "worker 没在运行（先把它起回来）" record preflight_worker pass "image=$WORKER_IMAGE id=$WORKER_IMAGE_ID running=$running" # 仅本机：worker 连的治理库必须就是本机这个容器（拒绝把屏障注入到远程栈）。 local gov_host gov_host="$(url_part "$(worker_env DATABASE_URL)" host)" [ "$gov_host" = "$GOV_DB_CONTAINER" ] || die 2 preflight_local_only \ "worker 连的治理库是 ${gov_host:-?}，不是本机 $GOV_DB_CONTAINER：拒绝在非本机栈注入屏障" [ "$(gov_sql 'select 1')" = "1" ] || die 2 preflight_gov_db "治理库读不到（$GOV_DB_CONTAINER/$GOV_DB_NAME）" record preflight_local_only pass "governance db=$GOV_DB_CONTAINER" ARCHIVE_ROOT_IN_CONTAINER="$(worker_env LIBRARY_ARCHIVE_ROOT)" [ -n "$ARCHIVE_ROOT_IN_CONTAINER" ] || die 2 preflight_archive "worker 没有 LIBRARY_ARCHIVE_ROOT" ARCHIVE_HOST="$(archive_root_host "$ARCHIVE_ROOT_IN_CONTAINER")" { [ -n "$ARCHIVE_HOST" ] && [ -d "$ARCHIVE_HOST" ]; } || die 2 preflight_archive "归档根在宿主机上找不到（容器内 $ARCHIVE_ROOT_IN_CONTAINER）" [ -d "$ARCHIVE_HOST/writesets" ] || die 2 preflight_archive "归档根下没有 writesets/：台账不在预期位置" [ -d "$ARCHIVE_HOST/versions" ] || die 2 preflight_archive "归档根下没有 versions/：快照清单不在预期位置" [ -w "$ARCHIVE_HOST" ] || die 2 preflight_archive "归档根宿主机路径不可写：$ARCHIVE_HOST" BARRIER_HOST="$ARCHIVE_HOST/barrier" record preflight_archive pass "archive=$ARCHIVE_HOST barrier=$BARRIER_HOST" # --restore 只负责把注入撤掉：它必须能在"上一轮没有收尾"（甚至还有作业在飞）时也跑通， # 否则 preflight_not_armed 会把它自己锁死（实测过）。 if [ "$MODE" = "restore" ]; then record preflight_done pass "restore 模式：只校验 docker/worker/归档根，不检查在飞作业与维护窗口" return 0 fi TARGET_INSTANCE="$(worker_env LIBRARY_TARGET_INSTANCE)" TARGET_PROJECT="$(worker_env LIBRARY_TARGET_PROJECT)" TARGET_INGRESS_SERVICE="$(worker_env LIBRARY_TARGET_INGRESS_SERVICE)" TARGET_WRITERS="$(worker_env LIBRARY_TARGET_WRITERS)" [ -n "$TARGET_INSTANCE" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_INSTANCE（维护目标未接线）" [ -n "$TARGET_PROJECT" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_PROJECT" docker inspect "$TARGET_INSTANCE" >/dev/null 2>&1 || die 2 preflight_topology "目标写入者容器缺失：$TARGET_INSTANCE" INGRESS_CONTAINER="$(container_of_service "$TARGET_PROJECT" "$TARGET_INGRESS_SERVICE")" [ -n "$INGRESS_CONTAINER" ] || die 2 preflight_topology "目标项目 $TARGET_PROJECT 下找不到入口服务 $TARGET_INGRESS_SERVICE" local instance_running instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')" [ "$instance_running" = "true" ] || die 2 preflight_topology \ "目标写入者 $TARGET_INSTANCE 没在运行：先按预备份恢复并收尾，再来演练" record preflight_topology pass "instance=$TARGET_INSTANCE running=$instance_running project=$TARGET_PROJECT writers=$TARGET_WRITERS ingress=$INGRESS_CONTAINER" local held held_instance held="$(gov_sql "select count(*) from library_maintenance where released_at is null")" held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")" if [ "$held" != "0" ] || [ "$held_instance" != "0" ]; then local owner owner="$(gov_sql "select coalesce(owner_job_id,'?') from library_maintenance where released_at is null limit 1")" die 2 preflight_no_open_window "已有未释放的维护窗口（owner=${owner:-?} scope=$held instance=$held_instance）：先恢复/收尾" fi record preflight_no_open_window pass "no unreleased maintenance" local in_flight queued in_flight="$(gov_sql "select coalesce(string_agg(id||'/'||state, ','),'') from library_jobs where state in ('pending','running')")" [ -z "$in_flight" ] || die 2 preflight_no_inflight "已有在飞的整库作业：$in_flight（一次只演练一个作业）" queued="$(gov_sql "select count(*) from jobs where type like 'library.%' and state in ('pending','running','retry')")" [ "$queued" = "0" ] || die 2 preflight_no_inflight "队列里还有 $queued 个 library.* 作业在飞/排队" record preflight_no_inflight pass "no in-flight library jobs" # 镜像必须真的带屏障代码：只拷脚本不换镜像，等待会永远落空（旧脚本的病）。 local supported supported="$(docker run --rm --entrypoint grep "$WORKER_IMAGE_ID" -ac KG_TEST_BARRIER_DIR /app/worker 2>/dev/null | tr -dc '0-9')" [ "${supported:-0}" -ge 1 ] || die 2 preflight_barrier_supported \ "镜像 $WORKER_IMAGE 里没有屏障代码（grep KG_TEST_BARRIER_DIR=${supported:-0}）：先部署带屏障的 worker 镜像" record preflight_barrier_supported pass "image $WORKER_IMAGE contains barrier" local target_db_url target_db_url="$(worker_env LIBRARY_PRODUCTION_DATABASE_URL)" TARGET_DB_CONTAINER="$(url_part "$target_db_url" host)" TARGET_DB_USER="$(url_part "$target_db_url" user)" TARGET_DB_NAME="$(url_part "$target_db_url" db)" [ -n "$TARGET_DB_CONTAINER" ] || die 2 preflight_target_db "worker 没有 LIBRARY_PRODUCTION_DATABASE_URL：无法独立核对目标内容" docker inspect "$TARGET_DB_CONTAINER" >/dev/null 2>&1 || die 2 preflight_target_db "目标库容器不存在：$TARGET_DB_CONTAINER" record preflight_target_db pass "target db=$TARGET_DB_CONTAINER/$TARGET_DB_NAME user=$TARGET_DB_USER" # 独立写集合校验器缺失 ⇒ 失败。上一版在缺失时把 canonical 记成 not_run 还 pass， # 等于"没有校验也算校验过"。 if [ ! -x "$VERIFIER" ]; then die 2 preflight_verifier "写集合独立校验器不存在或不可执行：$VERIFIER（缺校验不许当通过；用 WRITESET_VERIFIER 指向它）" fi record preflight_verifier pass "verifier=$VERIFIER" # 判据本身也要被验：脚本缺失或自检红，演练中途就只能靠人肉判断了。 [ -f "$LOG_CONTRACT" ] || die 2 preflight_log_contract "接管日志判据脚本不存在：$LOG_CONTRACT" python3 "$LOG_CONTRACT" --self-test >>"$EVIDENCE/log-contract.log" 2>&1 || die 2 preflight_log_contract \ "接管日志判据自检没通过（见 log-contract.log）：判据坏了不许当演练通过" record preflight_log_contract pass "contract=$LOG_CONTRACT self-test=ok" [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \ "worker 已经带着屏障注入在跑（上一轮没收尾）：先 --restore" record preflight_not_armed pass "barrier not armed" if [ -n "$EXPECT_JOB" ]; then case "$EXPECT_JOB" in libsync-*|librestore-*|libbase-*) ;; *) die 2 preflight_job_arg "--job 不是整库作业 id：$EXPECT_JOB" ;; esac fi if [ -n "$EXPECT_SNAPSHOT" ]; then [ "${#EXPECT_SNAPSHOT}" -ge 16 ] || die 2 preflight_job_arg "--snapshot 不像快照 id：$EXPECT_SNAPSHOT" fi record preflight_done pass "OK" } # ---------- 屏障注入 / 撤销 ---------- compose_argv() { # local override="$1" cfg project_dir file cfg="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.config_files"}}')" [ -n "$cfg" ] || { echo "worker 不是 compose 起的：找不到 config_files 标签" >&2; return 1; } project_dir="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.working_dir"}}')" COMPOSE_ARGS=(-p "$COMPOSE_PROJECT") [ -n "$project_dir" ] && COMPOSE_ARGS+=(--project-directory "$project_dir") while IFS= read -r file; do # 注入用的 overlay 会被 compose 记进容器的 config_files 标签；原样重放就等于 # 自己把注入又加回去（实测 --restore 撤不掉）。这里按文件名排除演练 overlay。 case "$file" in ""|*/barrier.override.yaml) continue ;; esac COMPOSE_ARGS+=(-f "$file") done < <(tr ',' '\n' <<<"$cfg") [ -n "$override" ] && COMPOSE_ARGS+=(-f "$override") return 0 } write_override() { cat >"$EVIDENCE/barrier.override.yaml" <&1 | grep -q "worker-1 started"; then return 0 fi sleep 1 done return 1 } arm_barrier() { rm -rf "$BARRIER_HOST" mkdir -p "$BARRIER_HOST" || die 2 arm_barrier "屏障目录建不出来：$BARRIER_HOST" write_override compose_argv "$EVIDENCE/barrier.override.yaml" || die 2 arm_barrier "compose 配置取不到" log "recreating $WORKER_CONTAINER with barrier point=$BARRIER_POINT" docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 || die 2 arm_barrier "compose up 失败（见 compose.log）" wait_worker_ready || die 2 arm_barrier "worker 重启后没有就绪" local armed armed="$(worker_env KG_TEST_BARRIER_DIR)" [ "$armed" = "$ARCHIVE_ROOT_IN_CONTAINER/barrier" ] || die 2 arm_barrier "屏障注入没生效（KG_TEST_BARRIER_DIR=${armed:-空}）" # 注入会重建容器：镜像/容器身份以重建后的为准（后面要比对"重启前后同镜像"）。 WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" ARMED=1 record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT container=$WORKER_CONTAINER_ID" } restore_barrier() { rm -rf "$BARRIER_HOST" compose_argv "" || die 7 restore "compose 配置取不到" docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 || die 7 restore "compose up（撤销注入）失败" wait_worker_ready || die 7 restore "worker 撤销注入后没有就绪" [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 7 restore "屏障注入还在（KG_TEST_BARRIER_DIR 非空）" [ "$(container_field "$WORKER_CONTAINER" '{{.Image}}')" = "$WORKER_IMAGE_ID" ] || die 7 restore \ "撤销注入后镜像变了：$WORKER_IMAGE_ID -> $(container_field "$WORKER_CONTAINER" '{{.Image}}')" ARMED=0 record restore pass "barrier env removed, image unchanged" } # ---------- 命中、锁作业、取证 ---------- wait_for_hit() { local base deadline base="$BARRIER_HOST/$(sed 's/[:/ ]/-/g' <<<"$BARRIER_POINT")" deadline=$(( $(now_epoch) + HIT_WAIT_S )) log "ARMED: 现在请在页面上发起作业（scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定}）。等待屏障点位 $BARRIER_POINT 命中 …" while [ "$(now_epoch)" -lt "$deadline" ]; do if [ -f "$base.timeout" ]; then cp "$base.timeout" "$EVIDENCE/" 2>/dev/null die 3 wait_for_hit "屏障超时（worker 已自行继续）：这次没有停在写入中间，判 not_run" fi if [ -f "$base.hit" ]; then cp "$base.hit" "$EVIDENCE/" HIT_SUBJECT="$(sed -n 's/.*"subject":"\([^"]*\)".*/\1/p' "$base.hit")" record wait_for_hit pass "hit=$BARRIER_POINT subject=${HIT_SUBJECT:-?}" return 0 fi sleep 1 done die 3 wait_for_hit "等待屏障命中超时（${HIT_WAIT_S}s）：作业没有到达 $BARRIER_POINT，判 not_run" } lock_job() { JOB_ID="${EXPECT_JOB:-$HIT_SUBJECT}" [ -n "$JOB_ID" ] || die 3 lock_job "屏障现场没有作业身份，也没有 --job" if [ -n "$EXPECT_JOB" ] && [ "$HIT_SUBJECT" != "$EXPECT_JOB" ]; then die 3 lock_job "屏障命中的是别的作业（hit=$HIT_SUBJECT 期望=$EXPECT_JOB）：拒绝继续" fi local row others row="$(gov_sql "select state||'|'||scope||'|'||kind||'|'||coalesce(snapshot_id,'')||'|'||coalesce(target_snapshot_id,'')||'|'||created_at::text from library_jobs where id='$JOB_ID'")" [ -n "$row" ] || die 3 lock_job "治理库里没有作业 $JOB_ID" IFS='|' read -r JOB_STATE JOB_SCOPE JOB_KIND JOB_SNAPSHOT JOB_TARGET JOB_CREATED <<<"$row" [ "$JOB_SCOPE" = "$SCOPE" ] || die 3 lock_job "作业 scope=$JOB_SCOPE 与请求 $SCOPE 不一致" [ "$JOB_STATE" = "running" ] || die 3 lock_job "作业状态是 $JOB_STATE 不是 running：命中时它已经不在写入路径上了" if [ "$KIND" != "auto" ] && [ "$JOB_KIND" != "$KIND" ]; then die 3 lock_job "作业 kind=$JOB_KIND 与请求 --kind $KIND 不一致（期望内容也会按错的那一类推导）" fi if [ -n "$EXPECT_SNAPSHOT" ] && [ "$JOB_TARGET" != "$EXPECT_SNAPSHOT" ]; then die 3 lock_job "作业目标快照 $JOB_TARGET 与 --snapshot $EXPECT_SNAPSHOT 不一致：拒绝把它当这次演练" fi if [ -n "$SINCE" ]; then local since_epoch created_epoch if [[ "$SINCE" =~ ^[0-9]+$ ]]; then since_epoch="$SINCE"; else since_epoch="$(date -u -d "$SINCE" +%s 2>/dev/null || echo 0)"; fi created_epoch="$(date -u -d "$JOB_CREATED" +%s 2>/dev/null || echo 0)" { [ "$since_epoch" -gt 0 ] && [ "$created_epoch" -ge "$since_epoch" ]; } || die 3 lock_job "作业创建时间 $JOB_CREATED 不满足 --since $SINCE（或 --since 解析不了）" fi # 目标快照 == 中断前的当前生效版本 ⇒ 这次"接管"只可能走同快照快速收尾， # 证明不了完整续作（2026-10-29 的现场正是如此）。杀之前就拒绝，别浪费一次中断。 PRE_EFFECTIVE="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")" if [ -n "$PRE_EFFECTIVE" ] && [ "$JOB_TARGET" = "$PRE_EFFECTIVE" ] && [ "$ALLOW_SAME_SNAPSHOT" -ne 1 ]; then die 3 lock_job "目标快照 $JOB_TARGET 就是中断前的当前生效版本：同快照收尾不能当完整续作的验收证据（要演练请换一个没生效过的快照；确要同快照加 --allow-same-snapshot）" fi others="$(gov_sql "select coalesce(string_agg(id, ','),'') from library_jobs where state in ('pending','running') and id<>'$JOB_ID'")" [ -z "$others" ] || die 3 lock_job "还有别的作业在飞：$others（只允许杀这一个作业的执行者）" local expected if [ -n "$EXPECT_COUNTS" ]; then expected="$EXPECT_COUNTS" log "期望内容来自 --expect：$expected" else expected="$(snapshot_expectations "$JOB_TARGET")" || die 3 lock_job "推导目标快照的期望内容失败（$JOB_TARGET）" EXPECT_COUNTS="$expected" log "期望内容来自快照 manifest（$JOB_TARGET）：$expected；resources 不在默认期望内（同租户其它知识库资源会保留）" fi case "$expected" in *=*) ;; *) die 3 lock_job "期望内容形状不对：$expected" ;; esac record lock_job pass "kind=$JOB_KIND job=$JOB_ID state=running scope=$JOB_SCOPE target=$JOB_TARGET pre_effective=${PRE_EFFECTIVE:-none} expect=$EXPECT_COUNTS created=$JOB_CREATED" } read_writeset_evidence() { local row row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','')||'|'||created_at::text||'|'||coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录：屏障点位早于台账登记，不能当「写集合已登记」的证据" IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP WS_BACKUP_CREATED WS_REUSED <<<"$row" [ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest：接管会失败，这次不算可接管" [ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致" WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json" [ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空：$WS_FILE" grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致（$WS_DIGEST）" grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID" # 崩溃前那份预备份的身份与字节也拍下来：接管是否"复用同一份、没被重做覆盖" # 只能靠事后比对（快照 id + 清单字节 + 清单 mtime + 版本行首次登记时刻）。 local backup_manifest backup_manifest="$ARCHIVE_HOST/versions/$WS_BACKUP.json" [ -s "$backup_manifest" ] || die 4 read_writeset "预备份快照清单不存在或为空：$backup_manifest" WS_BACKUP_SHA="$(sha256sum "$backup_manifest" | awk '{print $1}')" WS_BACKUP_MTIME="$(stat -c %Y "$backup_manifest")" WS_FILE_SHA="$(sha256sum "$WS_FILE" | awk '{print $1}')" # 独立校验器是必需的（preflight 已确认存在）：它判定不符就不能继续。 "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \ >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符（见 verifier.log）" record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP backup_manifest_sha=$WS_BACKUP_SHA backup_manifest_mtime=$WS_BACKUP_MTIME backup_registered_at=$WS_BACKUP_CREATED writeset_sha=$WS_FILE_SHA canonical=verified" } kill_worker() { [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill：杀容器必须是显式动作" local before_restart job_state deadline owner_uid before_restart="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" job_state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" [ "$job_state" = "running" ] || die 4 kill "杀之前作业状态变成 $job_state：这次不构成写入中的中断" KILL_PID="$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')" [ -n "$KILL_PID" ] && [ "$KILL_PID" -gt 1 ] || die 4 kill "读不到 worker 的宿主机 PID（${KILL_PID:-空}）" # 只有"同一 uid"或 root 才能向容器 PID1 发信号；先证明能杀，再谈演练。 owner_uid="$(stat -c %u "/proc/$KILL_PID" 2>/dev/null)" if [ "$(id -u)" != "0" ] && [ "${owner_uid:-x}" != "$(id -u)" ]; then die 4 kill "宿主机 PID $KILL_PID 属于 uid=${owner_uid:-?}，当前 uid=$(id -u) 无权 SIGKILL：不能用「进程崩溃」语义演练（换 --manual-start 方案前先解决权限）" fi kill -0 "$KILL_PID" 2>/dev/null || die 4 kill "PID $KILL_PID 不存活，杀不到真实进程" # 退出码证据从 docker events 取：restart policy 会很快把容器拉起来， # State.ExitCode 随即被重置成 0，事后 inspect 读不到 137（这正是上一版没留下 # "真的 137"证据的原因之一）。 : >"$EVIDENCE/docker-events.log" timeout 180 docker events --filter "container=$WORKER_CONTAINER_ID" --since "$(now_epoch)" \ --format '{{json .}}' >>"$EVIDENCE/docker-events.log" 2>>"$EVIDENCE/docker.log" & EVENTS_PID=$! # 给监听器 1s 就位：抢在同一秒里杀，die 事件可能早于 attach 被错过， # 那样只会误报"没有 137 证据"（假红），不会假绿。 sleep 1 KILLED_AT="$(utc_now)" RESTART_COUNT_BEFORE="$before_restart" log "SIGKILL worker container PID1 host_pid=$KILL_PID (job=$JOB_ID at $KILLED_AT) — 进程崩溃语义，不是 docker kill" kill -9 "$KILL_PID" 2>>"$EVIDENCE/docker.log" || die 4 kill "向 PID $KILL_PID 发送 SIGKILL 失败" deadline=$(( $(now_epoch) + KILL_PROOF_S )) KILL_PROOF="" while [ "$(now_epoch)" -lt "$deadline" ]; do if grep '"Action":"die"' "$EVIDENCE/docker-events.log" 2>/dev/null | grep -q '"exitCode":"137"'; then KILL_PROOF="docker-events die exitCode=137" break fi sleep 1 done [ -n "$KILL_PROOF" ] || die 4 kill "docker events 里没有 exitCode=137 的 die 事件（${KILL_PROOF_S}s 内）：这次没有证明真被杀" record kill pass "container=$WORKER_CONTAINER id=$WORKER_CONTAINER_ID host_pid=$KILL_PID exit=137 proof=\"$KILL_PROOF\" killed_at=$KILLED_AT restart_before=$RESTART_COUNT_BEFORE semantics=process-crash-sigkill" } wait_restart() { local deadline running deadline=$(( $(now_epoch) + POLICY_RESTART_WAIT_S )) RESTART_SOURCE="" while [ "$(now_epoch)" -lt "$deadline" ]; do running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" if [ "$running" = "true" ] && [ "$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')" != "$KILL_PID" ]; then RESTART_SOURCE="policy" break fi sleep 1 done if [ -z "$RESTART_SOURCE" ]; then if [ "$MANUAL_START" -ne 1 ]; then die 5 restart "restart policy 在 ${POLICY_RESTART_WAIT_S}s 内没有拉起（未允许显式 docker start）：这次不算自动接管" fi log "restart policy 没有在 ${POLICY_RESTART_WAIT_S}s 内拉起：显式 docker start（记 restart_source=manual_start，不是自动恢复）" docker start "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 5 restart "显式 docker start 失败" RESTART_SOURCE="manual_start" deadline=$(( $(now_epoch) + RESTART_WAIT_S )) while [ "$(now_epoch)" -lt "$deadline" ]; do running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" [ "$running" = "true" ] && break sleep 1 done [ "$running" = "true" ] || die 5 restart "显式启动后容器仍没运行" fi local image_after cid_after restarts_after image_after="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" cid_after="$(container_field "$WORKER_CONTAINER" '{{.Id}}')" restarts_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" [ "$image_after" = "$WORKER_IMAGE_ID" ] || die 5 restart "重启后镜像变了：$WORKER_IMAGE_ID -> $image_after（不是同一个产物）" [ "$cid_after" = "$WORKER_CONTAINER_ID" ] || die 5 restart "重启后容器身份变了：$WORKER_CONTAINER_ID -> $cid_after" record restart pass "source=$RESTART_SOURCE image_unchanged=1 container_unchanged=1 restart_count=$RESTART_COUNT_BEFORE->$restarts_after" } wait_takeover() { local deadline state="" last_note=0 out rc CONTRACT_OUT deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S )) log "等待同一个作业被认领（lease 30s + 扫描 30s，上限 ${TAKEOVER_WAIT_S}s）…" while [ "$(now_epoch)" -lt "$deadline" ]; do state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" case "$state" in succeeded|failed|cancelled) break ;; esac # 60s 没动静就把现场抓下来（别再空等到底才知道为什么）。 if [ $(( $(now_epoch) - last_note )) -ge 60 ]; then last_note="$(now_epoch)" docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | tail -5 >>"$EVIDENCE/worker.log" log "接管等待中：job state=${state:-?} worker_running=$(container_field "$WORKER_CONTAINER" '{{.State.Running}}') queue=$(gov_sql "select coalesce(string_agg(type||':'||state,','),'') from jobs where type like 'library.%' and state in ('pending','running','retry')")" fi sleep 2 done case "$state" in succeeded) ;; failed|cancelled) die 5 wait_takeover "作业终态是 $state（接管失败）：$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;; *) die 5 wait_takeover "等待接管超时（最后状态 ${state:-?}）：不算成功" ;; esac # 认领与决策都要有**本作业 id** 的行，且时间戳晚于杀点。 # 汇总行（`library job takeover sweep: 1 resumed`）不算证据：它既不说是哪个作业， # 也不说是否真的被认领（别的 worker 抢走、attach 失败都只影响计数）。 LOG_RAW="$EVIDENCE/worker-log-raw.txt" # -t 才有每行时间戳：判据要证明"这些行出现在杀之后"，没有时间戳就只能判红。 docker logs -t --since "$KILLED_AT" "$WORKER_CONTAINER" >"$LOG_RAW" 2>&1 CONTRACT_OUT="$EVIDENCE/log-contract.log" python3 "$LOG_CONTRACT" --log "$LOG_RAW" --job "$JOB_ID" --target "$JOB_TARGET" \ --backup "$WS_BACKUP" --since "$KILLED_AT" --expect-decision "$EXPECT_DECISION" \ >"$CONTRACT_OUT" 2>&1; rc=$? out="$(cat "$CONTRACT_OUT")" if [ "$rc" -ne 0 ]; then die 5 wait_takeover "接管日志契约不成立（rc=$rc，见 log-contract.log）：$(tr '\n' ' ' <"$CONTRACT_OUT" | cut -c1-400)" fi CLAIM_AT="$(sed -n 's/.*claim_at=\([^ ]*\).*/\1/p' <<<"$out" | head -1)" DECISION="$(sed -n 's/.*decision=\([^ ]*\).*/\1/p' <<<"$out" | head -1)" [ -n "$DECISION" ] || die 5 wait_takeover "判据脚本没给出决策（输出：$out）" record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; claim_at=${CLAIM_AT:-?}; decision=$DECISION; contract=ok" } verify_stages_complete() { local stages missing="" stage stages="$(gov_sql "select stage||':'||state from library_job_stages where job_id='$JOB_ID'")" for stage in database resources index wiki verify; do grep -qx "$stage:succeeded" <<<"$stages" || missing+="$stage " done [ -z "$missing" ] || die 6 verify_stages \ "作业没有真跑完写入链路：未 succeeded 的阶段 = $missing（stages=$(tr '\n' ',' <<<"$stages")）" record verify_stages pass "database/resources/index/wiki/verify 全部 succeeded" } verify_takeover_semantics() { local stages pre_state finalize_at eff after_digest local backup_now backup_created_now backup_sha_now backup_mtime_now pre_reused_now stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")" pre_state="$(gov_sql "select coalesce(state,'') from library_job_stages where job_id='$JOB_ID' and stage='prebackup'")" # 接管复用崩溃前那份预备份：阶段既可能写成 skipped（只标记不重跑），也可能写成 # succeeded（release.go 记"预备份 "——真实 2026-10-29 现场就是这种）。 # 两种都接受，但**不能因此取消备份校验**：必须证明复用的是同一份、没被重做覆盖。 case "$pre_state" in skipped|succeeded) ;; *) die 6 verify_takeover "prebackup 阶段是 ${pre_state:-缺失}：既不是复用（skipped）也不是复用后重跑（succeeded），接管路径不成立（stages=$stages）" ;; esac finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")" [ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功（stages=$stages）" backup_now="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ "$backup_now" = "$WS_BACKUP" ] || die 6 verify_takeover \ "预备份身份变了：崩溃前 $WS_BACKUP → 现在 ${backup_now:-空}（重做了备份 = 把半写状态当成可回滚基线）" backup_created_now="$(gov_sql "select created_at::text from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ "$backup_created_now" = "$WS_BACKUP_CREATED" ] || die 6 verify_takeover \ "预备份版本行的首次登记时刻被改写：$WS_BACKUP_CREATED → $backup_created_now（复用被洗成重做）" backup_sha_now="$(sha256sum "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null | awk '{print $1}')" [ "$backup_sha_now" = "$WS_BACKUP_SHA" ] || die 6 verify_takeover \ "预备份清单字节变了：$WS_BACKUP_SHA → ${backup_sha_now:-缺失}（同一快照 id 下内容被重写）" backup_mtime_now="$(stat -c %Y "$ARCHIVE_HOST/versions/$backup_now.json" 2>/dev/null)" [ "$backup_mtime_now" = "$WS_BACKUP_MTIME" ] || die 6 verify_takeover \ "预备份清单文件被重新写过（mtime $WS_BACKUP_MTIME → ${backup_mtime_now:-缺失}）：崩溃后重做了备份" pre_reused_now="$(gov_sql "select coalesce(evidence->>'backup_reused','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" if [ "$pre_state" = "succeeded" ] && [ "$pre_reused_now" != "true" ]; then die 6 verify_takeover \ "prebackup 记成 succeeded 但版本证据没有 backup_reused=true：分不清'复用'与'重做了备份'（evidence=$pre_reused_now）" fi record verify_takeover pass "stages=$stages decision=$DECISION finalize_at=$finalize_at prebackup=$pre_state backup=$backup_now registered_at=$backup_created_now manifest_sha=$backup_sha_now reused=${pre_reused_now:-no}" eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")" local eff_snapshot eff_job eff_superseded IFS='|' read -r eff_snapshot eff_job eff_superseded <<<"$eff" [ "$eff_snapshot" = "$JOB_TARGET" ] || die 6 verify_effective "当前生效版本是 ${eff_snapshot:-无}，不是本作业目标 $JOB_TARGET" [ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无}，不是 $JOB_ID（同快照由别作业登记不算本次完成）" [ -z "$eff_superseded" ] || die 6 verify_effective "生效版本已被顶替（$eff_superseded）" after_digest="$(gov_sql "select coalesce(evidence->>'writeset_digest','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ "$after_digest" = "$WS_DIGEST" ] || die 6 verify_effective "接管后写集合摘要变了（$WS_DIGEST -> ${after_digest:-空}）" record verify_effective pass "effective=$eff_snapshot job=$JOB_ID writeset=$after_digest" } verify_target_counts() { local part table want got failed=0 detail="" sql="" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}" case "$table" in *[!a-z_]*) die 6 verify_counts "期望里的表名不合法：$table" ;; esac sql+="select '$table='||count(*) from $table; " done got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" \ 2>>"$EVIDENCE/psql.log")" [ -n "$got" ] || die 6 verify_counts "目标库读数失败（$TARGET_DB_CONTAINER/$TARGET_DB_NAME）" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}"; want="${part##*=}" if ! grep -qx "$table=$want" <<<"$got"; then failed=$((failed + 1)) detail+="$table=$(sed -n "s/^$table=//p" <<<"$got")(want $want) " fi done [ "$failed" -eq 0 ] || die 6 verify_counts "目标内容与期望不符：$detail（实际 $(tr '\n' ' ' <<<"$got")）" record verify_counts pass "counts=$(tr '\n' ' ' <<<"$got")" } verify_service_restored() { local instance_running ingress_running held held_instance released released_instance instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')" ingress_running="$(container_field "$INGRESS_CONTAINER" '{{.State.Running}}')" [ "$instance_running" = "true" ] || die 6 verify_service "目标写入者 $TARGET_INSTANCE 没有恢复运行" [ "$ingress_running" = "true" ] || die 6 verify_service "入口 $INGRESS_CONTAINER 没有恢复运行" held="$(gov_sql "select count(*) from library_maintenance where released_at is null")" held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")" released="$(gov_sql "select count(*) from library_maintenance where owner_job_id='$JOB_ID' and released_at is not null")" released_instance="$(gov_sql "select count(*) from library_instance_maintenance where owner_job_id='$JOB_ID' and released_at is not null")" { [ "$held" = "0" ] && [ "$held_instance" = "0" ]; } || die 6 verify_service \ "还有未释放的维护占用（scope=$held instance=$held_instance）" { [ "$released" -ge 1 ] && [ "$released_instance" -ge 1 ]; } || die 6 verify_service \ "本作业的维护记录没有释放证据（scope=$released instance=$released_instance）" record verify_service pass "instance_running=$instance_running ingress_running=$ingress_running health(instance)=$(health_of "$TARGET_INSTANCE") health(ingress)=$(health_of "$INGRESS_CONTAINER") released=$released/$released_instance" } save_evidence() { { echo "# 脱敏证据（job=$JOB_ID kind=$JOB_KIND scope=$SCOPE target=$JOB_TARGET killed_at=$KILLED_AT restart_source=$RESTART_SOURCE）" echo "## library_jobs" gov_sql "select id||E'\t'||scope||E'\t'||kind||E'\t'||state||E'\t'||stage||E'\t'||coalesce(snapshot_id,'')||E'\t'||coalesce(target_snapshot_id,'')||E'\t'||created_at::text||E'\t'||coalesce(finished_at::text,'') from library_jobs where id='$JOB_ID'" echo "## library_job_stages" gov_sql "select stage||E'\t'||state||E'\t'||coalesce(total::text,'')||E'\t'||coalesce(completed::text,'')||E'\t'||coalesce(detail,'')||E'\t'||updated_at::text from library_job_stages where job_id='$JOB_ID' order by updated_at" echo "## library_versions（本作业）" gov_sql "select kind||E'\t'||snapshot_id||E'\t'||source_job_id||E'\t'||created_at::text||E'\t'||coalesce(superseded_at::text,'')||E'\t'||evidence::text from library_versions where source_job_id='$JOB_ID' order by created_at" echo "## 维护占用（本作业；实例表列是 owner_scope）" gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||owner_scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'" echo "## 目标内容计数" docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "select 'knowledges='||count(*) from knowledges union all select 'chunks='||count(*) from chunks union all select 'resources='||count(*) from resources union all select 'wiki_pages='||count(*) from wiki_pages" 2>/dev/null } >"$EVIDENCE/job-evidence.txt" docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" >"$EVIDENCE/worker.log" 2>&1 log "evidence written: $EVIDENCE/job-evidence.txt" } # ---------- 主流程 ---------- log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定} evidence=$EVIDENCE" preflight case "$MODE" in check) STATUS_OVERRIDE="READY_FOR_PARENT" finish 0 ;; restore) restore_barrier finish 0 ;; run) [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill" arm_barrier wait_for_hit lock_job read_writeset_evidence kill_worker wait_restart wait_takeover verify_stages_complete verify_takeover_semantics verify_target_counts verify_service_restored save_evidence restore_barrier STATUS_OVERRIDE="DRILL_PASSED" finish 0 ;; esac