knowledge-governance/scripts/library-kill-takeover.sh

#!/usr/bin/env bash
# library-kill-takeover.sh —— 真实 worker 中断演练（只在本机测试栈）
#
# 旧脚本（handoffs/release-reuse-18683ca/kill-takeover.sh）为什么不能用：
#   1. 前置要求"作业已 running 且 ledger 齐"——可它又要先启动观察，永远错过；
#   2. 维护窗口里要求业务 health 200——那时 app 被有意停掉，必然被拒；
#   3. 没杀到任何进程也 exit 0——"没做"和"做成"分不开。
#
# 这一版的判据只有一条：**真的 SIGKILL 了真实 worker 容器 → 同一个作业被接管 →
# 目标按期望内容恢复并校验 → 服务/入口恢复**，四条全成立才 exit 0。
# missed / notrun / timeout 一律非 0，并在 evidence 里写成 not_run 而不是"通过"。
#
# 命中手段是"固定同步屏障"（internal/testbarrier，默认关闭，仅本机注入）：
#   worker 在指定阶段写完后写 <barrier>/after-database.hit 并停住 → 本脚本看到 .hit
#   → SIGKILL 容器 → 容器按 restart 策略重启 → 启动/周期扫描接管同一作业。
#   屏障目录在本机归档根下（<archive>/barrier），没人放行就 300s 自行继续。
#
# 用法：
#   bash scripts/library-kill-takeover.sh --check                 # 前置体检（不是演练通过）
#   bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill
#        → 打印 ARMED 后，父在页面上发起同步作业；脚本自己命中、杀、验证、收尾
#   bash scripts/library-kill-takeover.sh --restore               # 撤掉屏障注入
#   可选：--job <业务作业 id>｜--since <RFC3339|unix>｜--expect "knowledges=4,chunks=38,..."
#
set -uo pipefail

SCRIPT_VERSION=2
MODE=""
SCOPE="internal"
EXPECT_JOB=""
SINCE=""
CONFIRM_KILL=0
EXPECT_COUNTS="${EXPECT_COUNTS:-knowledges=4,chunks=38,resources=22,wiki_pages=31}"
VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}"

WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}"
GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}"
GOV_DB_USER="${GOV_DB_USER:-governance}"
GOV_DB_NAME="${GOV_DB_NAME:-governance}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-knowledge-governance-p1}"
COMPOSE_SERVICE="${COMPOSE_SERVICE:-worker}"
BARRIER_POINT="${BARRIER_POINT:-after:database}"
BARRIER_TIMEOUT="${BARRIER_TIMEOUT:-300s}"
HIT_WAIT_S="${HIT_WAIT_S:-900}"
RESTART_WAIT_S="${RESTART_WAIT_S:-180}"
TAKEOVER_WAIT_S="${TAKEOVER_WAIT_S:-420}"
DRILL_HOME="${DRILL_HOME:-$HOME/.local/share/knowledge-governance/drills}"

while [ $# -gt 0 ]; do
  case "$1" in
    --check) MODE=check ;;
    --run) MODE=run ;;
    --restore) MODE=restore ;;
    --scope) SCOPE="${2:-}"; shift ;;
    --job) EXPECT_JOB="${2:-}"; shift ;;
    --since) SINCE="${2:-}"; shift ;;
    --expect) EXPECT_COUNTS="${2:-}"; shift ;;
    --point) BARRIER_POINT="${2:-}"; shift ;;
    --confirm-kill) CONFIRM_KILL=1 ;;
    -h|--help) sed -n '1,30p' "$0"; exit 0 ;;
    *) echo "unknown argument: $1" >&2; exit 64 ;;
  esac
  shift
done

if [ -z "$MODE" ]; then
  echo "must pass one of --check | --run | --restore" >&2
  exit 64
fi

# set -u：所有跨函数使用的变量先给默认值
WORKER_IMAGE_ID=""
WORKER_IMAGE=""
ARCHIVE_ROOT_IN_CONTAINER=""
ARCHIVE_HOST=""
BARRIER_HOST=""
TARGET_INSTANCE=""
TARGET_PROJECT=""
TARGET_INGRESS_SERVICE=""
TARGET_WRITERS=""
INGRESS_CONTAINER=""
TARGET_DB_CONTAINER=""
TARGET_DB_USER=""
TARGET_DB_NAME=""
HIT_SUBJECT=""
JOB_ID=""
JOB_STATE=""
JOB_SCOPE=""
JOB_KIND=""
JOB_SNAPSHOT=""
JOB_TARGET=""
JOB_CREATED=""
WS_DIGEST=""
WS_REUSED=""
WS_TARGET=""
WS_BACKUP=""
WS_FILE=""
KILLED_AT=""
STATUS_OVERRIDE=""
COMPOSE_ARGS=()

STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
EVIDENCE="$DRILL_HOME/$STAMP"
mkdir -p "$EVIDENCE" || { echo "cannot create evidence dir $EVIDENCE" >&2; exit 70; }
CHECKS="$EVIDENCE/checks.tsv"
: >"$CHECKS"

log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$EVIDENCE/phases.log"; }
# record <name> <pass|fail|not_run> <detail>
record() {
  printf '%s\t%s\t%s\n' "$1" "$2" "${3:-}" >>"$CHECKS"
  log "CHECK $1 = $2${3:+ ($3)}"
}
die() { # die <code> <name> <detail>
  record "$2" fail "$3"
  log "ABORT: $2 — $3"
  finish "$1"
}

# ---------- 小工具 ----------
worker_env() { # 只读非敏感变量
  docker inspect "$WORKER_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null |
    sed -n "s/^$1=//p" | head -1
}
gov_sql() { docker exec "$GOV_DB_CONTAINER" psql -U "$GOV_DB_USER" -d "$GOV_DB_NAME" -Atc "$1" 2>>"$EVIDENCE/psql.log"; }
container_field() { docker inspect "$1" --format "$2" 2>/dev/null; }
archive_root_host() {
  docker inspect "$WORKER_CONTAINER" --format '{{range .Mounts}}{{.Destination}}={{.Source}}{{println}}{{end}}' 2>/dev/null |
    sed -n "s#^$1=##p" | head -1
}
container_of_service() { # <compose project> <service>
  docker ps -a --filter "label=com.docker.compose.project=$1" \
    --format '{{.Names}}|{{.Label "com.docker.compose.service"}}' | awk -F'|' -v svc="$2" '$2==svc{print $1; exit}'
}
health_of() {
  local status
  status="$(docker inspect "$1" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no_healthcheck{{end}}' 2>/dev/null)"
  echo "${status:-unknown}"
}
url_part() { # <url> <user|host|db>
  case "$2" in
    user) sed -E 's#^[a-zA-Z0-9+]+://([^:/@]+).*#\1#' <<<"$1" ;;
    host) sed -E 's#^[a-zA-Z0-9+]+://[^@]*@([^:/]+).*#\1#' <<<"$1" ;;
    db)   sed -E 's#^[a-zA-Z0-9+]+://[^/]+/([^?]+).*#\1#' <<<"$1" ;;
    *) echo "" ;;
  esac
}
now_epoch() { date -u +%s; }
utc_now() { date -u +%Y-%m-%dT%H:%M:%SZ; }

finish() { # finish <exit code>
  local code="$1" status="PASS"
  if grep -q $'\tfail\t' "$CHECKS" 2>/dev/null; then status="FAIL"; fi
  if grep -q $'\tnot_run\t' "$CHECKS" 2>/dev/null && [ "$code" -eq 0 ]; then status="FAIL"; fi
  [ -n "$STATUS_OVERRIDE" ] && status="$STATUS_OVERRIDE"
  cp "$CHECKS" "$EVIDENCE/checks.snapshot.tsv" 2>/dev/null
  cat >"$EVIDENCE/summary.json" <<JSON
{
  "script_version": $SCRIPT_VERSION,
  "mode": "$MODE",
  "status": "$status",
  "exit_code": $code,
  "scope": "$SCOPE",
  "worker_container": "$WORKER_CONTAINER",
  "worker_image": "$WORKER_IMAGE",
  "barrier_point": "$BARRIER_POINT",
  "job": "$JOB_ID",
  "killed_at": "$KILLED_AT",
  "writeset_digest": "$WS_DIGEST",
  "evidence_dir": "$EVIDENCE"
}
JSON
  log "SUMMARY status=$status exit=$code evidence=$EVIDENCE"
  if [ "$MODE" = "check" ] && [ "$status" = "READY_FOR_PARENT" ]; then
    log "READY_FOR_PARENT（只是前置体检通过，不是演练通过）"
  fi
  exit "$code"
}

# ---------- 前置体检（--check 与 --run 共用） ----------
preflight() {
  command -v docker >/dev/null 2>&1 || die 2 preflight_docker "docker 命令不可用"
  docker info >/dev/null 2>&1 || die 2 preflight_docker "docker daemon 不可用"
  docker compose version >/dev/null 2>&1 || die 2 preflight_docker "docker compose 不可用"
  record preflight_docker pass "docker + compose 可用"

  docker inspect "$WORKER_CONTAINER" >/dev/null 2>&1 || die 2 preflight_worker "worker 容器不存在：$WORKER_CONTAINER"
  WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')"
  WORKER_IMAGE="$(container_field "$WORKER_CONTAINER" '{{.Config.Image}}')"
  local running
  running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')"
  [ "$running" = "true" ] || die 2 preflight_worker "worker 没在运行（先把它起回来）"
  record preflight_worker pass "image=$WORKER_IMAGE id=$WORKER_IMAGE_ID running=$running"

  # 仅本机：worker 连的治理库必须就是本机这个容器（拒绝把屏障注入到远程栈）。
  local gov_host
  gov_host="$(url_part "$(worker_env DATABASE_URL)" host)"
  [ "$gov_host" = "$GOV_DB_CONTAINER" ] || die 2 preflight_local_only \
    "worker 连的治理库是 ${gov_host:-?}，不是本机 $GOV_DB_CONTAINER：拒绝在非本机栈注入屏障"
  [ "$(gov_sql 'select 1')" = "1" ] || die 2 preflight_gov_db "治理库读不到（$GOV_DB_CONTAINER/$GOV_DB_NAME）"
  record preflight_local_only pass "governance db=$GOV_DB_CONTAINER"

  ARCHIVE_ROOT_IN_CONTAINER="$(worker_env LIBRARY_ARCHIVE_ROOT)"
  [ -n "$ARCHIVE_ROOT_IN_CONTAINER" ] || die 2 preflight_archive "worker 没有 LIBRARY_ARCHIVE_ROOT"
  ARCHIVE_HOST="$(archive_root_host "$ARCHIVE_ROOT_IN_CONTAINER")"
  { [ -n "$ARCHIVE_HOST" ] && [ -d "$ARCHIVE_HOST" ]; } || die 2 preflight_archive "归档根在宿主机上找不到（容器内 $ARCHIVE_ROOT_IN_CONTAINER）"
  [ -d "$ARCHIVE_HOST/writesets" ] || die 2 preflight_archive "归档根下没有 writesets/：台账不在预期位置"
  [ -w "$ARCHIVE_HOST" ] || die 2 preflight_archive "归档根宿主机路径不可写：$ARCHIVE_HOST"
  BARRIER_HOST="$ARCHIVE_HOST/barrier"
  record preflight_archive pass "archive=$ARCHIVE_HOST barrier=$BARRIER_HOST"

  TARGET_INSTANCE="$(worker_env LIBRARY_TARGET_INSTANCE)"
  TARGET_PROJECT="$(worker_env LIBRARY_TARGET_PROJECT)"
  TARGET_INGRESS_SERVICE="$(worker_env LIBRARY_TARGET_INGRESS_SERVICE)"
  TARGET_WRITERS="$(worker_env LIBRARY_TARGET_WRITERS)"
  [ -n "$TARGET_INSTANCE" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_INSTANCE（维护目标未接线）"
  [ -n "$TARGET_PROJECT" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_PROJECT"
  docker inspect "$TARGET_INSTANCE" >/dev/null 2>&1 || die 2 preflight_topology "目标写入者容器缺失：$TARGET_INSTANCE"
  INGRESS_CONTAINER="$(container_of_service "$TARGET_PROJECT" "$TARGET_INGRESS_SERVICE")"
  [ -n "$INGRESS_CONTAINER" ] || die 2 preflight_topology "目标项目 $TARGET_PROJECT 下找不到入口服务 $TARGET_INGRESS_SERVICE"
  local instance_running
  instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')"
  [ "$instance_running" = "true" ] || die 2 preflight_topology \
    "目标写入者 $TARGET_INSTANCE 没在运行：先按预备份恢复并收尾，再来演练"
  record preflight_topology pass "instance=$TARGET_INSTANCE running=$instance_running project=$TARGET_PROJECT writers=$TARGET_WRITERS ingress=$INGRESS_CONTAINER"

  local held held_instance
  held="$(gov_sql "select count(*) from library_maintenance where released_at is null")"
  held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")"
  if [ "$held" != "0" ] || [ "$held_instance" != "0" ]; then
    local owner
    owner="$(gov_sql "select coalesce(owner_job_id,'?') from library_maintenance where released_at is null limit 1")"
    die 2 preflight_no_open_window "已有未释放的维护窗口（owner=${owner:-?} scope=$held instance=$held_instance）：先恢复/收尾"
  fi
  record preflight_no_open_window pass "no unreleased maintenance"

  local in_flight queued
  in_flight="$(gov_sql "select coalesce(string_agg(id||'/'||state, ','),'') from library_jobs where state in ('pending','running')")"
  [ -z "$in_flight" ] || die 2 preflight_no_inflight "已有在飞的整库作业：$in_flight（一次只演练一个作业）"
  queued="$(gov_sql "select count(*) from jobs where type like 'library.%' and state in ('pending','running','retry')")"
  [ "$queued" = "0" ] || die 2 preflight_no_inflight "队列里还有 $queued 个 library.* 作业在飞/排队"
  record preflight_no_inflight pass "no in-flight library jobs"

  # 镜像必须真的带屏障代码：只拷脚本不换镜像，等待会永远落空（旧脚本的病）。
  local supported
  supported="$(docker run --rm --entrypoint grep "$WORKER_IMAGE_ID" -ac KG_TEST_BARRIER_DIR /app/worker 2>/dev/null | tr -dc '0-9')"
  [ "${supported:-0}" -ge 1 ] || die 2 preflight_barrier_supported \
    "镜像 $WORKER_IMAGE 里没有屏障代码（grep KG_TEST_BARRIER_DIR=${supported:-0}）：先部署带屏障的 worker 镜像"
  record preflight_barrier_supported pass "image $WORKER_IMAGE contains barrier"

  local target_db_url
  target_db_url="$(worker_env LIBRARY_PRODUCTION_DATABASE_URL)"
  TARGET_DB_CONTAINER="$(url_part "$target_db_url" host)"
  TARGET_DB_USER="$(url_part "$target_db_url" user)"
  TARGET_DB_NAME="$(url_part "$target_db_url" db)"
  [ -n "$TARGET_DB_CONTAINER" ] || die 2 preflight_target_db "worker 没有 LIBRARY_PRODUCTION_DATABASE_URL：无法独立核对目标内容"
  docker inspect "$TARGET_DB_CONTAINER" >/dev/null 2>&1 || die 2 preflight_target_db "目标库容器不存在：$TARGET_DB_CONTAINER"
  record preflight_target_db pass "target db=$TARGET_DB_CONTAINER/$TARGET_DB_NAME user=$TARGET_DB_USER"

  [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \
    "worker 已经带着屏障注入在跑（上一轮没收尾）：先 --restore"
  record preflight_not_armed pass "barrier not armed"

  if [ -n "$EXPECT_JOB" ]; then
    case "$EXPECT_JOB" in
      libsync-*|librestore-*|libbase-*) ;;
      *) die 2 preflight_job_arg "--job 不是整库作业 id：$EXPECT_JOB" ;;
    esac
  fi
  record preflight_done pass "OK"
}

# ---------- 屏障注入 / 撤销 ----------
compose_argv() { # <override or "">
  local override="$1" cfg project_dir file
  cfg="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.config_files"}}')"
  [ -n "$cfg" ] || { echo "worker 不是 compose 起的：找不到 config_files 标签" >&2; return 1; }
  project_dir="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.working_dir"}}')"
  COMPOSE_ARGS=(-p "$COMPOSE_PROJECT")
  [ -n "$project_dir" ] && COMPOSE_ARGS+=(--project-directory "$project_dir")
  while IFS= read -r file; do
    [ -n "$file" ] && COMPOSE_ARGS+=(-f "$file")
  done < <(tr ',' '\n' <<<"$cfg")
  [ -n "$override" ] && COMPOSE_ARGS+=(-f "$override")
  return 0
}

write_override() {
  cat >"$EVIDENCE/barrier.override.yaml" <<YAML
# 仅本机演练：给 worker 注入固定同步屏障（默认关闭；--restore 撤掉）
services:
  $COMPOSE_SERVICE:
    environment:
      KG_TEST_BARRIER_DIR: "$ARCHIVE_ROOT_IN_CONTAINER/barrier"
      KG_TEST_BARRIER_POINTS: "$BARRIER_POINT"
      KG_TEST_BARRIER_TIMEOUT: "$BARRIER_TIMEOUT"
YAML
}

wait_worker_ready() {
  local deadline=$(( $(now_epoch) + RESTART_WAIT_S ))
  while [ "$(now_epoch)" -lt "$deadline" ]; do
    if [ "$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" = "true" ] &&
       docker logs --since 180s "$WORKER_CONTAINER" 2>&1 | grep -q "worker-1 started"; then
      return 0
    fi
    sleep 1
  done
  return 1
}

arm_barrier() {
  rm -rf "$BARRIER_HOST"
  mkdir -p "$BARRIER_HOST" || die 2 arm_barrier "屏障目录建不出来：$BARRIER_HOST"
  write_override
  compose_argv "$EVIDENCE/barrier.override.yaml" || die 2 arm_barrier "compose 配置取不到"
  log "recreating $WORKER_CONTAINER with barrier point=$BARRIER_POINT"
  docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 ||
    die 2 arm_barrier "compose up 失败（见 compose.log）"
  wait_worker_ready || die 2 arm_barrier "worker 重启后没有就绪"
  local armed
  armed="$(worker_env KG_TEST_BARRIER_DIR)"
  [ "$armed" = "$ARCHIVE_ROOT_IN_CONTAINER/barrier" ] || die 2 arm_barrier "屏障注入没生效（KG_TEST_BARRIER_DIR=${armed:-空}）"
  record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT"
}

restore_barrier() {
  rm -rf "$BARRIER_HOST"
  compose_argv "" || die 7 restore "compose 配置取不到"
  docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 ||
    die 7 restore "compose up（撤销注入）失败"
  wait_worker_ready || die 7 restore "worker 撤销注入后没有就绪"
  [ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 7 restore "屏障注入还在（KG_TEST_BARRIER_DIR 非空）"
  [ "$(container_field "$WORKER_CONTAINER" '{{.Image}}')" = "$WORKER_IMAGE_ID" ] || die 7 restore \
    "撤销注入后镜像变了：$WORKER_IMAGE_ID -> $(container_field "$WORKER_CONTAINER" '{{.Image}}')"
  record restore pass "barrier env removed, image unchanged"
}

# ---------- 命中、锁作业、取证 ----------
wait_for_hit() {
  local base deadline
  base="$BARRIER_HOST/$(sed 's/[:/ ]/-/g' <<<"$BARRIER_POINT")"
  deadline=$(( $(now_epoch) + HIT_WAIT_S ))
  log "ARMED: 现在请在页面上发起同步作业（scope=$SCOPE）。等待屏障点位 $BARRIER_POINT 命中 …"
  while [ "$(now_epoch)" -lt "$deadline" ]; do
    if [ -f "$base.timeout" ]; then
      cp "$base.timeout" "$EVIDENCE/" 2>/dev/null
      die 3 wait_for_hit "屏障超时（worker 已自行继续）：这次没有停在写入中间，判 not_run"
    fi
    if [ -f "$base.hit" ]; then
      cp "$base.hit" "$EVIDENCE/"
      HIT_SUBJECT="$(sed -n 's/.*"subject":"\([^"]*\)".*/\1/p' "$base.hit")"
      record wait_for_hit pass "hit=$BARRIER_POINT subject=${HIT_SUBJECT:-?}"
      return 0
    fi
    sleep 1
  done
  die 3 wait_for_hit "等待屏障命中超时（${HIT_WAIT_S}s）：作业没有到达 $BARRIER_POINT，判 not_run"
}

lock_job() {
  JOB_ID="${EXPECT_JOB:-$HIT_SUBJECT}"
  [ -n "$JOB_ID" ] || die 3 lock_job "屏障现场没有作业身份，也没有 --job"
  if [ -n "$EXPECT_JOB" ] && [ "$HIT_SUBJECT" != "$EXPECT_JOB" ]; then
    die 3 lock_job "屏障命中的是别的作业（hit=$HIT_SUBJECT 期望=$EXPECT_JOB）：拒绝继续"
  fi
  local row others
  row="$(gov_sql "select state||'|'||scope||'|'||kind||'|'||coalesce(snapshot_id,'')||'|'||coalesce(target_snapshot_id,'')||'|'||created_at::text from library_jobs where id='$JOB_ID'")"
  [ -n "$row" ] || die 3 lock_job "治理库里没有作业 $JOB_ID"
  IFS='|' read -r JOB_STATE JOB_SCOPE JOB_KIND JOB_SNAPSHOT JOB_TARGET JOB_CREATED <<<"$row"
  [ "$JOB_SCOPE" = "$SCOPE" ] || die 3 lock_job "作业 scope=$JOB_SCOPE 与请求 $SCOPE 不一致"
  [ "$JOB_STATE" = "running" ] || die 3 lock_job "作业状态是 $JOB_STATE 不是 running：命中时它已经不在写入路径上了"
  if [ -n "$SINCE" ]; then
    local since_epoch created_epoch
    if [[ "$SINCE" =~ ^[0-9]+$ ]]; then since_epoch="$SINCE"; else since_epoch="$(date -u -d "$SINCE" +%s 2>/dev/null || echo 0)"; fi
    created_epoch="$(date -u -d "$JOB_CREATED" +%s 2>/dev/null || echo 0)"
    { [ "$since_epoch" -gt 0 ] && [ "$created_epoch" -ge "$since_epoch" ]; } ||
      die 3 lock_job "作业创建时间 $JOB_CREATED 不满足 --since $SINCE（或 --since 解析不了）"
  fi
  others="$(gov_sql "select coalesce(string_agg(id, ','),'') from library_jobs where state in ('pending','running') and id<>'$JOB_ID'")"
  [ -z "$others" ] || die 3 lock_job "还有别的作业在飞：$others（只允许杀这一个作业的执行者）"
  record lock_job pass "job=$JOB_ID state=running scope=$JOB_SCOPE kind=$JOB_KIND target=$JOB_TARGET created=$JOB_CREATED"
}

read_writeset_evidence() {
  local row canonical="not_run"
  row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
  [ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录：屏障点位早于台账登记，不能当「写集合已登记」的证据"
  IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row"
  [ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest：接管会失败，这次不算可接管"
  [ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致"
  WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json"
  [ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空：$WS_FILE"
  grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致（$WS_DIGEST）"
  grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID"
  if [ -e "$VERIFIER" ]; then
    "$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \
      >>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符（见 verifier.log）"
    canonical="verified"
  fi
  record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=$canonical"
}

kill_worker() {
  [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill：杀容器必须是显式动作"
  local before_restart job_state deadline exit_code=""
  before_restart="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')"
  job_state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
  [ "$job_state" = "running" ] || die 4 kill "杀之前作业状态变成 $job_state：这次不构成写入中的中断"
  KILLED_AT="$(utc_now)"
  log "SIGKILL $WORKER_CONTAINER (job=$JOB_ID at $KILLED_AT)"
  docker kill --signal=KILL "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 4 kill "docker kill 失败"
  deadline=$(( $(now_epoch) + 30 ))
  while [ "$(now_epoch)" -lt "$deadline" ]; do
    exit_code="$(container_field "$WORKER_CONTAINER" '{{.State.ExitCode}}')"
    [ "$exit_code" = "137" ] && break
    sleep 1
  done
  [ "$exit_code" = "137" ] || die 4 kill "容器退出码是 ${exit_code:-?}，不是 137（SIGKILL）"
  record kill pass "container=$WORKER_CONTAINER exit=137 killed_at=$KILLED_AT restart_before=$before_restart"
}

wait_takeover() {
  local deadline state="" restart_after
  deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S ))
  log "等待同一个作业被接管（lease 30s + 扫描 30s，上限 ${TAKEOVER_WAIT_S}s）…"
  while [ "$(now_epoch)" -lt "$deadline" ]; do
    state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
    case "$state" in succeeded|failed|cancelled) break ;; esac
    sleep 2
  done
  case "$state" in
    succeeded) ;;
    failed|cancelled) die 5 wait_takeover "作业终态是 $state（接管失败）：$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;;
    *) die 5 wait_takeover "等待接管超时（最后状态 ${state:-?}）：不算成功" ;;
  esac
  restart_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')"
  record wait_takeover pass "same job succeeded; RestartCount=$restart_after"
}

verify_takeover_semantics() {
  local stages finalize_at eff after_digest
  stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")"
  grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \
    "阶段里没有 prebackup:skipped：无法证明走的是接管路径（stages=$stages）"
  finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")"
  [ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功（stages=$stages）"
  record verify_takeover pass "stages=$stages finalize_at=$finalize_at"

  eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")"
  local eff_snapshot eff_job eff_superseded
  IFS='|' read -r eff_snapshot eff_job eff_superseded <<<"$eff"
  [ "$eff_snapshot" = "$JOB_TARGET" ] || die 6 verify_effective "当前生效版本是 ${eff_snapshot:-无}，不是本作业目标 $JOB_TARGET"
  [ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无}，不是 $JOB_ID"
  [ -z "$eff_superseded" ] || die 6 verify_effective "生效版本已被顶替（$eff_superseded）"
  after_digest="$(gov_sql "select coalesce(evidence->>'writeset_digest','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
  [ "$after_digest" = "$WS_DIGEST" ] || die 6 verify_effective "接管后写集合摘要变了（$WS_DIGEST -> ${after_digest:-空}）"
  record verify_effective pass "effective=$eff_snapshot job=$JOB_ID writeset=$after_digest"
}

verify_target_counts() {
  local part table want got sql="" failed=0 detail=""
  for part in ${EXPECT_COUNTS//,/ }; do
    table="${part%%=*}"
    sql+="select '$table='||count(*) from $table; "
  done
  got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" 2>>"$EVIDENCE/psql.log")"
  [ -n "$got" ] || die 6 verify_counts "目标库读数失败（$TARGET_DB_CONTAINER/$TARGET_DB_NAME）"
  for part in ${EXPECT_COUNTS//,/ }; do
    table="${part%%=*}"; want="${part##*=}"
    if ! grep -qx "$table=$want" <<<"$got"; then
      failed=$((failed + 1))
      detail+="$table=$(sed -n "s/^$table=//p" <<<"$got")(want $want) "
    fi
  done
  [ "$failed" -eq 0 ] || die 6 verify_counts "目标内容与期望不符：$detail（实际 $(tr '\n' ' ' <<<"$got")）"
  record verify_counts pass "counts=$(tr '\n' ' ' <<<"$got")"
}

verify_service_restored() {
  local instance_running ingress_running held held_instance released released_instance
  instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')"
  ingress_running="$(container_field "$INGRESS_CONTAINER" '{{.State.Running}}')"
  [ "$instance_running" = "true" ] || die 6 verify_service "目标写入者 $TARGET_INSTANCE 没有恢复运行"
  [ "$ingress_running" = "true" ] || die 6 verify_service "入口 $INGRESS_CONTAINER 没有恢复运行"
  held="$(gov_sql "select count(*) from library_maintenance where released_at is null")"
  held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")"
  released="$(gov_sql "select count(*) from library_maintenance where owner_job_id='$JOB_ID' and released_at is not null")"
  released_instance="$(gov_sql "select count(*) from library_instance_maintenance where owner_job_id='$JOB_ID' and released_at is not null")"
  { [ "$held" = "0" ] && [ "$held_instance" = "0" ]; } || die 6 verify_service \
    "还有未释放的维护占用（scope=$held instance=$held_instance）"
  { [ "$released" -ge 1 ] && [ "$released_instance" -ge 1 ]; } || die 6 verify_service \
    "本作业的维护记录没有释放证据（scope=$released instance=$released_instance）"
  # 业务 health 只在服务已恢复时作为附加证据：维护期间 app 被停是预期，不作为拒绝理由。
  record verify_service pass "instance_running=$instance_running ingress_running=$ingress_running health(instance)=$(health_of "$TARGET_INSTANCE") health(ingress)=$(health_of "$INGRESS_CONTAINER") released=$released/$released_instance"
}

save_evidence() {
  {
    echo "# 脱敏证据（job=$JOB_ID scope=$SCOPE killed_at=$KILLED_AT）"
    echo "## library_jobs"
    gov_sql "select id||E'\t'||scope||E'\t'||kind||E'\t'||state||E'\t'||stage||E'\t'||coalesce(snapshot_id,'')||E'\t'||coalesce(target_snapshot_id,'')||E'\t'||created_at::text||E'\t'||coalesce(finished_at::text,'') from library_jobs where id='$JOB_ID'"
    echo "## library_job_stages"
    gov_sql "select stage||E'\t'||state||E'\t'||coalesce(total::text,'')||E'\t'||coalesce(completed::text,'')||E'\t'||coalesce(detail,'')||E'\t'||updated_at::text from library_job_stages where job_id='$JOB_ID' order by updated_at"
    echo "## library_versions（本作业）"
    gov_sql "select kind||E'\t'||snapshot_id||E'\t'||source_job_id||E'\t'||created_at::text||E'\t'||coalesce(superseded_at::text,'')||E'\t'||evidence::text from library_versions where source_job_id='$JOB_ID' order by created_at"
    echo "## 维护占用（本作业）"
    gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'"
    echo "## 目标内容计数"
    docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "select 'knowledges='||count(*) from knowledges union all select 'chunks='||count(*) from chunks union all select 'resources='||count(*) from resources union all select 'wiki_pages='||count(*) from wiki_pages" 2>/dev/null
  } >"$EVIDENCE/job-evidence.txt"
  log "evidence written: $EVIDENCE/job-evidence.txt"
}

# ---------- 主流程 ----------
log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE evidence=$EVIDENCE"
preflight

case "$MODE" in
  check)
    STATUS_OVERRIDE="READY_FOR_PARENT"
    finish 0
    ;;
  restore)
    restore_barrier
    finish 0
    ;;
  run)
    [ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill"
    arm_barrier
    wait_for_hit
    lock_job
    read_writeset_evidence
    kill_worker
    wait_takeover
    verify_takeover_semantics
    verify_target_counts
    verify_service_restored
    save_evidence
    restore_barrier
    STATUS_OVERRIDE="DRILL_PASSED"
    finish 0
    ;;
esac