#!/usr/bin/env bash # library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈) # # 旧脚本(handoffs/release-reuse-18683ca/kill-takeover.sh)为什么不能用: # 1. 前置要求"作业已 running 且 ledger 齐"——可它又要先启动观察,永远错过; # 2. 维护窗口里要求业务 health 200——那时 app 被有意停掉,必然被拒; # 3. 没杀到任何进程也 exit 0——"没做"和"做成"分不开。 # # 这一版的判据只有一条:**真的 SIGKILL 了真实 worker 容器 → 同一个作业被接管 → # 目标按期望内容恢复并校验 → 服务/入口恢复**,四条全成立才 exit 0。 # missed / notrun / timeout 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。 # # 命中手段是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入): # worker 在指定阶段写完后写 /after-database.hit 并停住 → 本脚本看到 .hit # → SIGKILL 容器 → 容器按 restart 策略重启 → 启动/周期扫描接管同一作业。 # 屏障目录在本机归档根下(/barrier),没人放行就 300s 自行继续。 # # 用法: # bash scripts/library-kill-takeover.sh --check # 前置体检(不是演练通过) # bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill # → 打印 ARMED 后,父在页面上发起同步作业;脚本自己命中、杀、验证、收尾 # bash scripts/library-kill-takeover.sh --restore # 撤掉屏障注入 # 可选:--job <业务作业 id>|--since |--expect "knowledges=4,chunks=38,..." # set -uo pipefail SCRIPT_VERSION=2 MODE="" SCOPE="internal" EXPECT_JOB="" SINCE="" CONFIRM_KILL=0 EXPECT_COUNTS="${EXPECT_COUNTS:-knowledges=4,chunks=38,resources=22,wiki_pages=31}" VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}" WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}" GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}" GOV_DB_USER="${GOV_DB_USER:-governance}" GOV_DB_NAME="${GOV_DB_NAME:-governance}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-knowledge-governance-p1}" COMPOSE_SERVICE="${COMPOSE_SERVICE:-worker}" BARRIER_POINT="${BARRIER_POINT:-after:database}" BARRIER_TIMEOUT="${BARRIER_TIMEOUT:-300s}" HIT_WAIT_S="${HIT_WAIT_S:-900}" RESTART_WAIT_S="${RESTART_WAIT_S:-180}" TAKEOVER_WAIT_S="${TAKEOVER_WAIT_S:-420}" DRILL_HOME="${DRILL_HOME:-$HOME/.local/share/knowledge-governance/drills}" while [ $# -gt 0 ]; do case "$1" in --check) MODE=check ;; --run) MODE=run ;; --restore) MODE=restore ;; --scope) SCOPE="${2:-}"; shift ;; --job) EXPECT_JOB="${2:-}"; shift ;; --since) SINCE="${2:-}"; shift ;; --expect) EXPECT_COUNTS="${2:-}"; shift ;; --point) BARRIER_POINT="${2:-}"; shift ;; --confirm-kill) CONFIRM_KILL=1 ;; -h|--help) sed -n '1,30p' "$0"; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 64 ;; esac shift done if [ -z "$MODE" ]; then echo "must pass one of --check | --run | --restore" >&2 exit 64 fi # set -u:所有跨函数使用的变量先给默认值 WORKER_IMAGE_ID="" WORKER_IMAGE="" ARCHIVE_ROOT_IN_CONTAINER="" ARCHIVE_HOST="" BARRIER_HOST="" TARGET_INSTANCE="" TARGET_PROJECT="" TARGET_INGRESS_SERVICE="" TARGET_WRITERS="" INGRESS_CONTAINER="" TARGET_DB_CONTAINER="" TARGET_DB_USER="" TARGET_DB_NAME="" HIT_SUBJECT="" JOB_ID="" JOB_STATE="" JOB_SCOPE="" JOB_KIND="" JOB_SNAPSHOT="" JOB_TARGET="" JOB_CREATED="" WS_DIGEST="" WS_REUSED="" WS_TARGET="" WS_BACKUP="" WS_FILE="" KILLED_AT="" STATUS_OVERRIDE="" COMPOSE_ARGS=() STAMP="$(date -u +%Y%m%dT%H%M%SZ)" EVIDENCE="$DRILL_HOME/$STAMP" mkdir -p "$EVIDENCE" || { echo "cannot create evidence dir $EVIDENCE" >&2; exit 70; } CHECKS="$EVIDENCE/checks.tsv" : >"$CHECKS" log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$EVIDENCE/phases.log"; } # record record() { printf '%s\t%s\t%s\n' "$1" "$2" "${3:-}" >>"$CHECKS" log "CHECK $1 = $2${3:+ ($3)}" } die() { # die record "$2" fail "$3" log "ABORT: $2 — $3" finish "$1" } # ---------- 小工具 ---------- worker_env() { # 只读非敏感变量 docker inspect "$WORKER_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n "s/^$1=//p" | head -1 } gov_sql() { docker exec "$GOV_DB_CONTAINER" psql -U "$GOV_DB_USER" -d "$GOV_DB_NAME" -Atc "$1" 2>>"$EVIDENCE/psql.log"; } container_field() { docker inspect "$1" --format "$2" 2>/dev/null; } archive_root_host() { docker inspect "$WORKER_CONTAINER" --format '{{range .Mounts}}{{.Destination}}={{.Source}}{{println}}{{end}}' 2>/dev/null | sed -n "s#^$1=##p" | head -1 } container_of_service() { # docker ps -a --filter "label=com.docker.compose.project=$1" \ --format '{{.Names}}|{{.Label "com.docker.compose.service"}}' | awk -F'|' -v svc="$2" '$2==svc{print $1; exit}' } health_of() { local status status="$(docker inspect "$1" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no_healthcheck{{end}}' 2>/dev/null)" echo "${status:-unknown}" } url_part() { # case "$2" in user) sed -E 's#^[a-zA-Z0-9+]+://([^:/@]+).*#\1#' <<<"$1" ;; host) sed -E 's#^[a-zA-Z0-9+]+://[^@]*@([^:/]+).*#\1#' <<<"$1" ;; db) sed -E 's#^[a-zA-Z0-9+]+://[^/]+/([^?]+).*#\1#' <<<"$1" ;; *) echo "" ;; esac } now_epoch() { date -u +%s; } utc_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } finish() { # finish local code="$1" status="PASS" if grep -q $'\tfail\t' "$CHECKS" 2>/dev/null; then status="FAIL"; fi if grep -q $'\tnot_run\t' "$CHECKS" 2>/dev/null && [ "$code" -eq 0 ]; then status="FAIL"; fi [ -n "$STATUS_OVERRIDE" ] && status="$STATUS_OVERRIDE" cp "$CHECKS" "$EVIDENCE/checks.snapshot.tsv" 2>/dev/null cat >"$EVIDENCE/summary.json" </dev/null 2>&1 || die 2 preflight_docker "docker 命令不可用" docker info >/dev/null 2>&1 || die 2 preflight_docker "docker daemon 不可用" docker compose version >/dev/null 2>&1 || die 2 preflight_docker "docker compose 不可用" record preflight_docker pass "docker + compose 可用" docker inspect "$WORKER_CONTAINER" >/dev/null 2>&1 || die 2 preflight_worker "worker 容器不存在:$WORKER_CONTAINER" WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')" WORKER_IMAGE="$(container_field "$WORKER_CONTAINER" '{{.Config.Image}}')" local running running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" [ "$running" = "true" ] || die 2 preflight_worker "worker 没在运行(先把它起回来)" record preflight_worker pass "image=$WORKER_IMAGE id=$WORKER_IMAGE_ID running=$running" # 仅本机:worker 连的治理库必须就是本机这个容器(拒绝把屏障注入到远程栈)。 local gov_host gov_host="$(url_part "$(worker_env DATABASE_URL)" host)" [ "$gov_host" = "$GOV_DB_CONTAINER" ] || die 2 preflight_local_only \ "worker 连的治理库是 ${gov_host:-?},不是本机 $GOV_DB_CONTAINER:拒绝在非本机栈注入屏障" [ "$(gov_sql 'select 1')" = "1" ] || die 2 preflight_gov_db "治理库读不到($GOV_DB_CONTAINER/$GOV_DB_NAME)" record preflight_local_only pass "governance db=$GOV_DB_CONTAINER" ARCHIVE_ROOT_IN_CONTAINER="$(worker_env LIBRARY_ARCHIVE_ROOT)" [ -n "$ARCHIVE_ROOT_IN_CONTAINER" ] || die 2 preflight_archive "worker 没有 LIBRARY_ARCHIVE_ROOT" ARCHIVE_HOST="$(archive_root_host "$ARCHIVE_ROOT_IN_CONTAINER")" { [ -n "$ARCHIVE_HOST" ] && [ -d "$ARCHIVE_HOST" ]; } || die 2 preflight_archive "归档根在宿主机上找不到(容器内 $ARCHIVE_ROOT_IN_CONTAINER)" [ -d "$ARCHIVE_HOST/writesets" ] || die 2 preflight_archive "归档根下没有 writesets/:台账不在预期位置" [ -w "$ARCHIVE_HOST" ] || die 2 preflight_archive "归档根宿主机路径不可写:$ARCHIVE_HOST" BARRIER_HOST="$ARCHIVE_HOST/barrier" record preflight_archive pass "archive=$ARCHIVE_HOST barrier=$BARRIER_HOST" TARGET_INSTANCE="$(worker_env LIBRARY_TARGET_INSTANCE)" TARGET_PROJECT="$(worker_env LIBRARY_TARGET_PROJECT)" TARGET_INGRESS_SERVICE="$(worker_env LIBRARY_TARGET_INGRESS_SERVICE)" TARGET_WRITERS="$(worker_env LIBRARY_TARGET_WRITERS)" [ -n "$TARGET_INSTANCE" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_INSTANCE(维护目标未接线)" [ -n "$TARGET_PROJECT" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_PROJECT" docker inspect "$TARGET_INSTANCE" >/dev/null 2>&1 || die 2 preflight_topology "目标写入者容器缺失:$TARGET_INSTANCE" INGRESS_CONTAINER="$(container_of_service "$TARGET_PROJECT" "$TARGET_INGRESS_SERVICE")" [ -n "$INGRESS_CONTAINER" ] || die 2 preflight_topology "目标项目 $TARGET_PROJECT 下找不到入口服务 $TARGET_INGRESS_SERVICE" local instance_running instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')" [ "$instance_running" = "true" ] || die 2 preflight_topology \ "目标写入者 $TARGET_INSTANCE 没在运行:先按预备份恢复并收尾,再来演练" record preflight_topology pass "instance=$TARGET_INSTANCE running=$instance_running project=$TARGET_PROJECT writers=$TARGET_WRITERS ingress=$INGRESS_CONTAINER" local held held_instance held="$(gov_sql "select count(*) from library_maintenance where released_at is null")" held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")" if [ "$held" != "0" ] || [ "$held_instance" != "0" ]; then local owner owner="$(gov_sql "select coalesce(owner_job_id,'?') from library_maintenance where released_at is null limit 1")" die 2 preflight_no_open_window "已有未释放的维护窗口(owner=${owner:-?} scope=$held instance=$held_instance):先恢复/收尾" fi record preflight_no_open_window pass "no unreleased maintenance" local in_flight queued in_flight="$(gov_sql "select coalesce(string_agg(id||'/'||state, ','),'') from library_jobs where state in ('pending','running')")" [ -z "$in_flight" ] || die 2 preflight_no_inflight "已有在飞的整库作业:$in_flight(一次只演练一个作业)" queued="$(gov_sql "select count(*) from jobs where type like 'library.%' and state in ('pending','running','retry')")" [ "$queued" = "0" ] || die 2 preflight_no_inflight "队列里还有 $queued 个 library.* 作业在飞/排队" record preflight_no_inflight pass "no in-flight library jobs" # 镜像必须真的带屏障代码:只拷脚本不换镜像,等待会永远落空(旧脚本的病)。 local supported supported="$(docker run --rm --entrypoint grep "$WORKER_IMAGE_ID" -ac KG_TEST_BARRIER_DIR /app/worker 2>/dev/null | tr -dc '0-9')" [ "${supported:-0}" -ge 1 ] || die 2 preflight_barrier_supported \ "镜像 $WORKER_IMAGE 里没有屏障代码(grep KG_TEST_BARRIER_DIR=${supported:-0}):先部署带屏障的 worker 镜像" record preflight_barrier_supported pass "image $WORKER_IMAGE contains barrier" local target_db_url target_db_url="$(worker_env LIBRARY_PRODUCTION_DATABASE_URL)" TARGET_DB_CONTAINER="$(url_part "$target_db_url" host)" TARGET_DB_USER="$(url_part "$target_db_url" user)" TARGET_DB_NAME="$(url_part "$target_db_url" db)" [ -n "$TARGET_DB_CONTAINER" ] || die 2 preflight_target_db "worker 没有 LIBRARY_PRODUCTION_DATABASE_URL:无法独立核对目标内容" docker inspect "$TARGET_DB_CONTAINER" >/dev/null 2>&1 || die 2 preflight_target_db "目标库容器不存在:$TARGET_DB_CONTAINER" record preflight_target_db pass "target db=$TARGET_DB_CONTAINER/$TARGET_DB_NAME user=$TARGET_DB_USER" [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \ "worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore" record preflight_not_armed pass "barrier not armed" if [ -n "$EXPECT_JOB" ]; then case "$EXPECT_JOB" in libsync-*|librestore-*|libbase-*) ;; *) die 2 preflight_job_arg "--job 不是整库作业 id:$EXPECT_JOB" ;; esac fi record preflight_done pass "OK" } # ---------- 屏障注入 / 撤销 ---------- compose_argv() { # local override="$1" cfg project_dir file cfg="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.config_files"}}')" [ -n "$cfg" ] || { echo "worker 不是 compose 起的:找不到 config_files 标签" >&2; return 1; } project_dir="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.working_dir"}}')" COMPOSE_ARGS=(-p "$COMPOSE_PROJECT") [ -n "$project_dir" ] && COMPOSE_ARGS+=(--project-directory "$project_dir") while IFS= read -r file; do [ -n "$file" ] && COMPOSE_ARGS+=(-f "$file") done < <(tr ',' '\n' <<<"$cfg") [ -n "$override" ] && COMPOSE_ARGS+=(-f "$override") return 0 } write_override() { cat >"$EVIDENCE/barrier.override.yaml" <&1 | grep -q "worker-1 started"; then return 0 fi sleep 1 done return 1 } arm_barrier() { rm -rf "$BARRIER_HOST" mkdir -p "$BARRIER_HOST" || die 2 arm_barrier "屏障目录建不出来:$BARRIER_HOST" write_override compose_argv "$EVIDENCE/barrier.override.yaml" || die 2 arm_barrier "compose 配置取不到" log "recreating $WORKER_CONTAINER with barrier point=$BARRIER_POINT" docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 || die 2 arm_barrier "compose up 失败(见 compose.log)" wait_worker_ready || die 2 arm_barrier "worker 重启后没有就绪" local armed armed="$(worker_env KG_TEST_BARRIER_DIR)" [ "$armed" = "$ARCHIVE_ROOT_IN_CONTAINER/barrier" ] || die 2 arm_barrier "屏障注入没生效(KG_TEST_BARRIER_DIR=${armed:-空})" record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT" } restore_barrier() { rm -rf "$BARRIER_HOST" compose_argv "" || die 7 restore "compose 配置取不到" docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 || die 7 restore "compose up(撤销注入)失败" wait_worker_ready || die 7 restore "worker 撤销注入后没有就绪" [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 7 restore "屏障注入还在(KG_TEST_BARRIER_DIR 非空)" [ "$(container_field "$WORKER_CONTAINER" '{{.Image}}')" = "$WORKER_IMAGE_ID" ] || die 7 restore \ "撤销注入后镜像变了:$WORKER_IMAGE_ID -> $(container_field "$WORKER_CONTAINER" '{{.Image}}')" record restore pass "barrier env removed, image unchanged" } # ---------- 命中、锁作业、取证 ---------- wait_for_hit() { local base deadline base="$BARRIER_HOST/$(sed 's/[:/ ]/-/g' <<<"$BARRIER_POINT")" deadline=$(( $(now_epoch) + HIT_WAIT_S )) log "ARMED: 现在请在页面上发起同步作业(scope=$SCOPE)。等待屏障点位 $BARRIER_POINT 命中 …" while [ "$(now_epoch)" -lt "$deadline" ]; do if [ -f "$base.timeout" ]; then cp "$base.timeout" "$EVIDENCE/" 2>/dev/null die 3 wait_for_hit "屏障超时(worker 已自行继续):这次没有停在写入中间,判 not_run" fi if [ -f "$base.hit" ]; then cp "$base.hit" "$EVIDENCE/" HIT_SUBJECT="$(sed -n 's/.*"subject":"\([^"]*\)".*/\1/p' "$base.hit")" record wait_for_hit pass "hit=$BARRIER_POINT subject=${HIT_SUBJECT:-?}" return 0 fi sleep 1 done die 3 wait_for_hit "等待屏障命中超时(${HIT_WAIT_S}s):作业没有到达 $BARRIER_POINT,判 not_run" } lock_job() { JOB_ID="${EXPECT_JOB:-$HIT_SUBJECT}" [ -n "$JOB_ID" ] || die 3 lock_job "屏障现场没有作业身份,也没有 --job" if [ -n "$EXPECT_JOB" ] && [ "$HIT_SUBJECT" != "$EXPECT_JOB" ]; then die 3 lock_job "屏障命中的是别的作业(hit=$HIT_SUBJECT 期望=$EXPECT_JOB):拒绝继续" fi local row others row="$(gov_sql "select state||'|'||scope||'|'||kind||'|'||coalesce(snapshot_id,'')||'|'||coalesce(target_snapshot_id,'')||'|'||created_at::text from library_jobs where id='$JOB_ID'")" [ -n "$row" ] || die 3 lock_job "治理库里没有作业 $JOB_ID" IFS='|' read -r JOB_STATE JOB_SCOPE JOB_KIND JOB_SNAPSHOT JOB_TARGET JOB_CREATED <<<"$row" [ "$JOB_SCOPE" = "$SCOPE" ] || die 3 lock_job "作业 scope=$JOB_SCOPE 与请求 $SCOPE 不一致" [ "$JOB_STATE" = "running" ] || die 3 lock_job "作业状态是 $JOB_STATE 不是 running:命中时它已经不在写入路径上了" if [ -n "$SINCE" ]; then local since_epoch created_epoch if [[ "$SINCE" =~ ^[0-9]+$ ]]; then since_epoch="$SINCE"; else since_epoch="$(date -u -d "$SINCE" +%s 2>/dev/null || echo 0)"; fi created_epoch="$(date -u -d "$JOB_CREATED" +%s 2>/dev/null || echo 0)" { [ "$since_epoch" -gt 0 ] && [ "$created_epoch" -ge "$since_epoch" ]; } || die 3 lock_job "作业创建时间 $JOB_CREATED 不满足 --since $SINCE(或 --since 解析不了)" fi others="$(gov_sql "select coalesce(string_agg(id, ','),'') from library_jobs where state in ('pending','running') and id<>'$JOB_ID'")" [ -z "$others" ] || die 3 lock_job "还有别的作业在飞:$others(只允许杀这一个作业的执行者)" record lock_job pass "job=$JOB_ID state=running scope=$JOB_SCOPE kind=$JOB_KIND target=$JOB_TARGET created=$JOB_CREATED" } read_writeset_evidence() { local row canonical="not_run" row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据" IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row" [ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest:接管会失败,这次不算可接管" [ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致" WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json" [ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE" grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)" grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID" if [ -e "$VERIFIER" ]; then "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \ >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)" canonical="verified" fi record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=$canonical" } kill_worker() { [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill:杀容器必须是显式动作" local before_restart job_state deadline exit_code="" before_restart="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" job_state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" [ "$job_state" = "running" ] || die 4 kill "杀之前作业状态变成 $job_state:这次不构成写入中的中断" KILLED_AT="$(utc_now)" log "SIGKILL $WORKER_CONTAINER (job=$JOB_ID at $KILLED_AT)" docker kill --signal=KILL "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 4 kill "docker kill 失败" deadline=$(( $(now_epoch) + 30 )) while [ "$(now_epoch)" -lt "$deadline" ]; do exit_code="$(container_field "$WORKER_CONTAINER" '{{.State.ExitCode}}')" [ "$exit_code" = "137" ] && break sleep 1 done [ "$exit_code" = "137" ] || die 4 kill "容器退出码是 ${exit_code:-?},不是 137(SIGKILL)" record kill pass "container=$WORKER_CONTAINER exit=137 killed_at=$KILLED_AT restart_before=$before_restart" } wait_takeover() { local deadline state="" restart_after deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S )) log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…" while [ "$(now_epoch)" -lt "$deadline" ]; do state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")" case "$state" in succeeded|failed|cancelled) break ;; esac sleep 2 done case "$state" in succeeded) ;; failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;; *) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;; esac restart_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')" record wait_takeover pass "same job succeeded; RestartCount=$restart_after" } verify_takeover_semantics() { local stages finalize_at eff after_digest stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")" grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \ "阶段里没有 prebackup:skipped:无法证明走的是接管路径(stages=$stages)" finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")" [ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功(stages=$stages)" record verify_takeover pass "stages=$stages finalize_at=$finalize_at" eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")" local eff_snapshot eff_job eff_superseded IFS='|' read -r eff_snapshot eff_job eff_superseded <<<"$eff" [ "$eff_snapshot" = "$JOB_TARGET" ] || die 6 verify_effective "当前生效版本是 ${eff_snapshot:-无},不是本作业目标 $JOB_TARGET" [ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无},不是 $JOB_ID" [ -z "$eff_superseded" ] || die 6 verify_effective "生效版本已被顶替($eff_superseded)" after_digest="$(gov_sql "select coalesce(evidence->>'writeset_digest','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")" [ "$after_digest" = "$WS_DIGEST" ] || die 6 verify_effective "接管后写集合摘要变了($WS_DIGEST -> ${after_digest:-空})" record verify_effective pass "effective=$eff_snapshot job=$JOB_ID writeset=$after_digest" } verify_target_counts() { local part table want got sql="" failed=0 detail="" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}" sql+="select '$table='||count(*) from $table; " done got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" 2>>"$EVIDENCE/psql.log")" [ -n "$got" ] || die 6 verify_counts "目标库读数失败($TARGET_DB_CONTAINER/$TARGET_DB_NAME)" for part in ${EXPECT_COUNTS//,/ }; do table="${part%%=*}"; want="${part##*=}" if ! grep -qx "$table=$want" <<<"$got"; then failed=$((failed + 1)) detail+="$table=$(sed -n "s/^$table=//p" <<<"$got")(want $want) " fi done [ "$failed" -eq 0 ] || die 6 verify_counts "目标内容与期望不符:$detail(实际 $(tr '\n' ' ' <<<"$got"))" record verify_counts pass "counts=$(tr '\n' ' ' <<<"$got")" } verify_service_restored() { local instance_running ingress_running held held_instance released released_instance instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')" ingress_running="$(container_field "$INGRESS_CONTAINER" '{{.State.Running}}')" [ "$instance_running" = "true" ] || die 6 verify_service "目标写入者 $TARGET_INSTANCE 没有恢复运行" [ "$ingress_running" = "true" ] || die 6 verify_service "入口 $INGRESS_CONTAINER 没有恢复运行" held="$(gov_sql "select count(*) from library_maintenance where released_at is null")" held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")" released="$(gov_sql "select count(*) from library_maintenance where owner_job_id='$JOB_ID' and released_at is not null")" released_instance="$(gov_sql "select count(*) from library_instance_maintenance where owner_job_id='$JOB_ID' and released_at is not null")" { [ "$held" = "0" ] && [ "$held_instance" = "0" ]; } || die 6 verify_service \ "还有未释放的维护占用(scope=$held instance=$held_instance)" { [ "$released" -ge 1 ] && [ "$released_instance" -ge 1 ]; } || die 6 verify_service \ "本作业的维护记录没有释放证据(scope=$released instance=$released_instance)" # 业务 health 只在服务已恢复时作为附加证据:维护期间 app 被停是预期,不作为拒绝理由。 record verify_service pass "instance_running=$instance_running ingress_running=$ingress_running health(instance)=$(health_of "$TARGET_INSTANCE") health(ingress)=$(health_of "$INGRESS_CONTAINER") released=$released/$released_instance" } save_evidence() { { echo "# 脱敏证据(job=$JOB_ID scope=$SCOPE killed_at=$KILLED_AT)" echo "## library_jobs" gov_sql "select id||E'\t'||scope||E'\t'||kind||E'\t'||state||E'\t'||stage||E'\t'||coalesce(snapshot_id,'')||E'\t'||coalesce(target_snapshot_id,'')||E'\t'||created_at::text||E'\t'||coalesce(finished_at::text,'') from library_jobs where id='$JOB_ID'" echo "## library_job_stages" gov_sql "select stage||E'\t'||state||E'\t'||coalesce(total::text,'')||E'\t'||coalesce(completed::text,'')||E'\t'||coalesce(detail,'')||E'\t'||updated_at::text from library_job_stages where job_id='$JOB_ID' order by updated_at" echo "## library_versions(本作业)" gov_sql "select kind||E'\t'||snapshot_id||E'\t'||source_job_id||E'\t'||created_at::text||E'\t'||coalesce(superseded_at::text,'')||E'\t'||evidence::text from library_versions where source_job_id='$JOB_ID' order by created_at" echo "## 维护占用(本作业)" gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'" echo "## 目标内容计数" docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "select 'knowledges='||count(*) from knowledges union all select 'chunks='||count(*) from chunks union all select 'resources='||count(*) from resources union all select 'wiki_pages='||count(*) from wiki_pages" 2>/dev/null } >"$EVIDENCE/job-evidence.txt" log "evidence written: $EVIDENCE/job-evidence.txt" } # ---------- 主流程 ---------- log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE evidence=$EVIDENCE" preflight case "$MODE" in check) STATUS_OVERRIDE="READY_FOR_PARENT" finish 0 ;; restore) restore_barrier finish 0 ;; run) [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill" arm_barrier wait_for_hit lock_job read_writeset_evidence kill_worker wait_takeover verify_takeover_semantics verify_target_counts verify_service_restored save_evidence restore_barrier STATUS_OVERRIDE="DRILL_PASSED" finish 0 ;; esac