Files
knowledge-governance/scripts/library-kill-takeover.sh
T
姚章浩 1d70059d32 fix(scripts): 中断演练改用进程崩溃语义并按快照推导期望
上一轮为什么空等 420s:docker kill 走 Docker API 的"手动停止",unless-stopped
也不拉起,容器一直躺着,最后由人工 docker start 才接管 —— 那不是崩溃自恢复。

改法:
- 命中后从宿主机向容器 PID1 发 SIGKILL(同 uid 才允许,先证明能杀);退出码证据取
  docker events 的 die exitCode=137 —— restart 后 State.ExitCode 会被重置成 0,
  事后 inspect 读不到 137。
- 限期等 restart policy 拉起(同容器/同镜像才算);策略没起来才显式 docker start,
  并记 restart_source=manual_start,绝不当自动恢复。
- 期望内容按目标快照 manifest 推导(knowledges/chunks/wiki_pages),resources 不进
  默认期望(同租户其它知识库资源会保留,trap-16)。
- --kind sync|restore + --snapshot 强校验;目标快照等于中断前生效版本时**杀之前
  就拒绝**(同快照收尾不能当完整续作验收)。
- 新增 verify_stages_complete:database/resources/index/wiki/verify 全部 succeeded
  才算完整续作(上一轮这四个还是 pending,却被判成功)。
- 独立写集合校验器缺失即 fail(旧版记 canonical=not_run 还 pass)。
- save_evidence 读实例维护用 owner_scope(旧版读 scope 列,SQL 报错被吞)。

验证(已跑):
- bash -n 通过;--check 全绿(READY_FOR_PARENT,含 preflight_verifier)
- 负例:WRITESET_VERIFIER=/nonexistent → preflight_verifier fail / exit 2
- 期望推导:fc5955→4/38/31、0d982→1/2/9、0fea→4/38/31;不存在的快照 → exit 1
- 宿主 SIGKILL 能力实测(scratch 容器,非本栈):kill -9 PID1 → 策略自动重启,
  RestartCount 0→1,docker events 记 exitCode=137
2026-10-30 03:40:18 +08:00

761 lines
41 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# library-kill-takeover.sh —— 真实 worker 中断演练(只在本机测试栈)
#
# 判据只有一条:**真的把 worker 进程 SIGKILL 掉(exit 137)→ 同一个作业被接管并
# 真跑完写入链路 → 目标按该快照的期望内容恢复并校验 → 服务/入口恢复**,全成立才 exit 0。
# missed / notrun / timeout / 只同快照收尾 一律非 0,并在 evidence 里写成 not_run 而不是"通过"。
#
# 为什么不再用 `docker kill`:
# docker kill/stop 走 Docker API 的"手动停止",restart policy 不生效(`unless-stopped`
# 也不生效),容器就一直躺着。2026-10-29T19:16Z 的现场就是这样:脚本等了 420s 判失败,
# 最后由人工 `docker start` 才接管。那不是"崩溃自恢复",只是"有人按了开始"。
# 现在的命中手段是**宿主机向容器 PID1 发 SIGKILL**(同一 uid 才允许):这才是进程崩溃,
# Docker 看到的是进程死亡 ⇒ restart policy 按策略拉起,语义与真实故障一致。
# 若策略没在限期内拉起,脚本可以显式 `docker start`,但必须在 evidence 里记成
# restart_source=manual_start,**不许**算成"自动恢复"。
#
# 命中点位仍是"固定同步屏障"(internal/testbarrier,默认关闭,仅本机注入):
# worker 在指定阶段写完后写 <barrier>/after-database.hit 并停住 → 本脚本看到 .hit
# → SIGKILL → 按策略重启/显式拉起 → 启动扫描接管同一作业。
# 屏障目录在本机归档根下(<archive>/barrier),没人放行就自行继续(记 not_run)。
#
# 期望内容按**目标快照自己的 manifest** 取(knowledges/chunks/wiki_pages),不是写死
# 4/38/31:同一个脚本既能跑 sync 也能跑 restore,期望随快照变。resources 不进默认
# 期望——同租户其它知识库的资源不在预备份范围内(trap-16),目标里会保留它们。
#
# 用法:
# bash scripts/library-kill-takeover.sh --check # 前置体检(不是演练通过)
# bash scripts/library-kill-takeover.sh --run --scope internal --confirm-kill \
# --kind restore --snapshot <目标快照 id>
# → 打印 ARMED 后,父在页面上发起该作业;脚本自己命中、杀、验证、收尾
# bash scripts/library-kill-takeover.sh --restore # 撤掉屏障注入
# 可选:--kind sync|restore|auto(默认 auto,给了就强校验作业类型)
# --snapshot <id>(校验命中的作业目标就是它)
# --job <业务作业 id>|--since <RFC3339|unix>
# --expect "knowledges=..,chunks=..,wiki_pages=.."(覆盖 manifest 推导)
# --allow-same-snapshot(允许目标快照 == 中断前生效版本;这条**不能**当完整续作验收)
# --manual-start 0|1(策略未拉起时是否显式 docker start,默认 1)
#
set -uo pipefail
SCRIPT_VERSION=3
MODE=""
SCOPE="internal"
KIND="auto"
EXPECT_SNAPSHOT=""
EXPECT_JOB=""
SINCE=""
CONFIRM_KILL=0
ALLOW_SAME_SNAPSHOT=0
MANUAL_START=1
EXPECT_COUNTS="${EXPECT_COUNTS:-}"
VERIFIER="${WRITESET_VERIFIER:-$HOME/.hermes/handoffs/release-current-version-takeover/verify-writeset}"
WORKER_CONTAINER="${WORKER_CONTAINER:-knowledge-governance-p1-worker-1}"
GOV_DB_CONTAINER="${GOV_DB_CONTAINER:-knowledge-governance-p1-postgres-1}"
GOV_DB_USER="${GOV_DB_USER:-governance}"
GOV_DB_NAME="${GOV_DB_NAME:-governance}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-knowledge-governance-p1}"
COMPOSE_SERVICE="${COMPOSE_SERVICE:-worker}"
BARRIER_POINT="${BARRIER_POINT:-after:database}"
BARRIER_TIMEOUT="${BARRIER_TIMEOUT:-300s}"
HIT_WAIT_S="${HIT_WAIT_S:-900}"
KILL_PROOF_S="${KILL_PROOF_S:-30}"
POLICY_RESTART_WAIT_S="${POLICY_RESTART_WAIT_S:-20}"
RESTART_WAIT_S="${RESTART_WAIT_S:-90}"
TAKEOVER_WAIT_S="${TAKEOVER_WAIT_S:-420}"
DRILL_HOME="${DRILL_HOME:-$HOME/.local/share/knowledge-governance/drills}"
while [ $# -gt 0 ]; do
case "$1" in
--check) MODE=check ;;
--run) MODE=run ;;
--restore) MODE=restore ;;
--scope) SCOPE="${2:-}"; shift ;;
--kind) KIND="${2:-}"; shift ;;
--snapshot) EXPECT_SNAPSHOT="${2:-}"; shift ;;
--job) EXPECT_JOB="${2:-}"; shift ;;
--since) SINCE="${2:-}"; shift ;;
--expect) EXPECT_COUNTS="${2:-}"; shift ;;
--point) BARRIER_POINT="${2:-}"; shift ;;
--manual-start) MANUAL_START="${2:-}"; shift ;;
--allow-same-snapshot) ALLOW_SAME_SNAPSHOT=1 ;;
--confirm-kill) CONFIRM_KILL=1 ;;
-h|--help) sed -n '1,42p' "$0"; exit 0 ;;
*) echo "unknown argument: $1" >&2; exit 64 ;;
esac
shift
done
if [ -z "$MODE" ]; then
echo "must pass one of --check | --run | --restore" >&2
exit 64
fi
case "$KIND" in
auto|sync|restore) ;;
*) echo "--kind must be auto|sync|restore (got $KIND)" >&2; exit 64 ;;
esac
case "$MANUAL_START" in
0|1) ;;
*) echo "--manual-start must be 0|1" >&2; exit 64 ;;
esac
# set -u:所有跨函数使用的变量先给默认值
WORKER_IMAGE_ID=""
WORKER_IMAGE=""
WORKER_CONTAINER_ID=""
ARCHIVE_ROOT_IN_CONTAINER=""
ARCHIVE_HOST=""
BARRIER_HOST=""
TARGET_INSTANCE=""
TARGET_PROJECT=""
TARGET_INGRESS_SERVICE=""
TARGET_WRITERS=""
INGRESS_CONTAINER=""
TARGET_DB_CONTAINER=""
TARGET_DB_USER=""
TARGET_DB_NAME=""
HIT_SUBJECT=""
JOB_ID=""
JOB_STATE=""
JOB_SCOPE=""
JOB_KIND=""
JOB_SNAPSHOT=""
JOB_TARGET=""
JOB_CREATED=""
PRE_EFFECTIVE=""
WS_DIGEST=""
WS_REUSED=""
WS_TARGET=""
WS_BACKUP=""
WS_FILE=""
KILLED_AT=""
KILL_PID=""
KILL_PROOF=""
RESTART_SOURCE=""
RESTART_COUNT_BEFORE=""
EVENTS_PID=""
STATUS_OVERRIDE=""
ARMED=0
COMPOSE_ARGS=()
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
EVIDENCE="$DRILL_HOME/$STAMP"
mkdir -p "$EVIDENCE" || { echo "cannot create evidence dir $EVIDENCE" >&2; exit 70; }
CHECKS="$EVIDENCE/checks.tsv"
: >"$CHECKS"
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$EVIDENCE/phases.log"; }
# record <name> <pass|fail|not_run> <detail>
record() {
printf '%s\t%s\t%s\n' "$1" "$2" "${3:-}" >>"$CHECKS"
log "CHECK $1 = $2${3:+ ($3)}"
}
die() { # die <code> <name> <detail>
record "$2" fail "$3"
log "ABORT: $2 — $3"
finish "$1"
}
# ---------- 小工具 ----------
worker_env() { # 只读非敏感变量
docker inspect "$WORKER_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null |
sed -n "s/^$1=//p" | head -1
}
gov_sql() { docker exec "$GOV_DB_CONTAINER" psql -U "$GOV_DB_USER" -d "$GOV_DB_NAME" -Atc "$1" 2>>"$EVIDENCE/psql.log"; }
container_field() { docker inspect "$1" --format "$2" 2>/dev/null; }
archive_root_host() {
docker inspect "$WORKER_CONTAINER" --format '{{range .Mounts}}{{.Destination}}={{.Source}}{{println}}{{end}}' 2>/dev/null |
sed -n "s#^$1=##p" | head -1
}
container_of_service() { # <compose project> <service>
docker ps -a --filter "label=com.docker.compose.project=$1" \
--format '{{.Names}}|{{.Label "com.docker.compose.service"}}' | awk -F'|' -v svc="$2" '$2==svc{print $1; exit}'
}
health_of() {
local status
status="$(docker inspect "$1" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no_healthcheck{{end}}' 2>/dev/null)"
echo "${status:-unknown}"
}
url_part() { # <url> <user|host|db>
case "$2" in
user) sed -E 's#^[a-zA-Z0-9+]+://([^:/@]+).*#\1#' <<<"$1" ;;
host) sed -E 's#^[a-zA-Z0-9+]+://[^@]*@([^:/]+).*#\1#' <<<"$1" ;;
db) sed -E 's#^[a-zA-Z0-9+]+://[^/]+/([^?]+).*#\1#' <<<"$1" ;;
*) echo "" ;;
esac
}
now_epoch() { date -u +%s; }
utc_now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
# 目标快照自己的期望内容:只认归档里的 manifest(不写死 4/38/31)。
# resources 故意不进默认期望:同租户其它知识库的资源不在预备份范围(trap-16),
# 目标里会保留它们,把 21 写成期望就会永远红。
snapshot_expectations() { # <snapshot id>
python3 - "$ARCHIVE_HOST/versions/$1.json" "$SCOPE" <<'PY'
import json, sys, pathlib
path, scope = sys.argv[1], sys.argv[2]
try:
manifest = json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
except Exception as exc:
sys.exit("读不到目标快照 manifest(%s):%s" % (path, exc))
if manifest.get("format") != "weknora-library-snapshot/v1":
sys.exit("manifest format=%r 不是 weknora-library-snapshot/v1" % manifest.get("format"))
if manifest.get("scope") != scope:
sys.exit("manifest scope=%r 与请求 %r 不一致" % (manifest.get("scope"), scope))
counts = {p["name"]: p["count"] for p in manifest.get("parts", []) if p.get("kind") == "database"}
for table in ("knowledges", "chunks"):
if table not in counts:
sys.exit("manifest 缺少 database.%s:不能推导期望" % table)
wiki = [p["count"] for p in manifest.get("parts", []) if p.get("kind") == "wiki" and p.get("name") == "wiki_pages"]
print("knowledges=%d,chunks=%d,wiki_pages=%d" % (counts["knowledges"], counts["chunks"], wiki[0] if wiki else 0))
PY
}
finish() { # finish <exit code>
local code="$1" status="PASS"
if [ -n "$EVENTS_PID" ]; then kill "$EVENTS_PID" 2>/dev/null; fi
if grep -q $'\tfail\t' "$CHECKS" 2>/dev/null; then status="FAIL"; fi
if grep -q $'\tnot_run\t' "$CHECKS" 2>/dev/null && [ "$code" -eq 0 ]; then status="FAIL"; fi
[ -n "$STATUS_OVERRIDE" ] && status="$STATUS_OVERRIDE"
cp "$CHECKS" "$EVIDENCE/checks.snapshot.tsv" 2>/dev/null
cat >"$EVIDENCE/summary.json" <<JSON
{
"script_version": $SCRIPT_VERSION,
"mode": "$MODE",
"status": "$status",
"exit_code": $code,
"scope": "$SCOPE",
"requested_kind": "$KIND",
"job_kind": "$JOB_KIND",
"worker_container": "$WORKER_CONTAINER",
"worker_container_id": "$WORKER_CONTAINER_ID",
"worker_image": "$WORKER_IMAGE",
"worker_image_id": "$WORKER_IMAGE_ID",
"barrier_point": "$BARRIER_POINT",
"job": "$JOB_ID",
"target_snapshot": "$JOB_TARGET",
"pre_effective_snapshot": "$PRE_EFFECTIVE",
"expected_counts": "$EXPECT_COUNTS",
"killed_at": "$KILLED_AT",
"killed_pid": "$KILL_PID",
"kill_proof": "$KILL_PROOF",
"restart_source": "$RESTART_SOURCE",
"writeset_digest": "$WS_DIGEST",
"evidence_dir": "$EVIDENCE"
}
JSON
log "SUMMARY status=$status exit=$code evidence=$EVIDENCE"
if [ "$ARMED" -eq 1 ] && [ "$status" != "DRILL_PASSED" ]; then
log "注意:屏障注入仍在 worker 上(本次没有走到撤注入那一步)。请运行:bash $0 --restore"
fi
if [ "$MODE" = "check" ] && [ "$status" = "READY_FOR_PARENT" ]; then
log "READY_FOR_PARENT(只是前置体检通过,不是演练通过)"
fi
exit "$code"
}
# ---------- 前置体检(--check 与 --run 共用) ----------
preflight() {
command -v docker >/dev/null 2>&1 || die 2 preflight_docker "docker 命令不可用"
docker info >/dev/null 2>&1 || die 2 preflight_docker "docker daemon 不可用"
docker compose version >/dev/null 2>&1 || die 2 preflight_docker "docker compose 不可用"
command -v python3 >/dev/null 2>&1 || die 2 preflight_docker "python3 不可用(推导期望内容要用)"
record preflight_docker pass "docker + compose + python3 可用"
docker inspect "$WORKER_CONTAINER" >/dev/null 2>&1 || die 2 preflight_worker "worker 容器不存在:$WORKER_CONTAINER"
WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')"
WORKER_IMAGE="$(container_field "$WORKER_CONTAINER" '{{.Config.Image}}')"
WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')"
local running
running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')"
[ "$running" = "true" ] || die 2 preflight_worker "worker 没在运行(先把它起回来)"
record preflight_worker pass "image=$WORKER_IMAGE id=$WORKER_IMAGE_ID running=$running"
# 仅本机:worker 连的治理库必须就是本机这个容器(拒绝把屏障注入到远程栈)。
local gov_host
gov_host="$(url_part "$(worker_env DATABASE_URL)" host)"
[ "$gov_host" = "$GOV_DB_CONTAINER" ] || die 2 preflight_local_only \
"worker 连的治理库是 ${gov_host:-?},不是本机 $GOV_DB_CONTAINER:拒绝在非本机栈注入屏障"
[ "$(gov_sql 'select 1')" = "1" ] || die 2 preflight_gov_db "治理库读不到($GOV_DB_CONTAINER/$GOV_DB_NAME)"
record preflight_local_only pass "governance db=$GOV_DB_CONTAINER"
ARCHIVE_ROOT_IN_CONTAINER="$(worker_env LIBRARY_ARCHIVE_ROOT)"
[ -n "$ARCHIVE_ROOT_IN_CONTAINER" ] || die 2 preflight_archive "worker 没有 LIBRARY_ARCHIVE_ROOT"
ARCHIVE_HOST="$(archive_root_host "$ARCHIVE_ROOT_IN_CONTAINER")"
{ [ -n "$ARCHIVE_HOST" ] && [ -d "$ARCHIVE_HOST" ]; } || die 2 preflight_archive "归档根在宿主机上找不到(容器内 $ARCHIVE_ROOT_IN_CONTAINER)"
[ -d "$ARCHIVE_HOST/writesets" ] || die 2 preflight_archive "归档根下没有 writesets/:台账不在预期位置"
[ -d "$ARCHIVE_HOST/versions" ] || die 2 preflight_archive "归档根下没有 versions/:快照清单不在预期位置"
[ -w "$ARCHIVE_HOST" ] || die 2 preflight_archive "归档根宿主机路径不可写:$ARCHIVE_HOST"
BARRIER_HOST="$ARCHIVE_HOST/barrier"
record preflight_archive pass "archive=$ARCHIVE_HOST barrier=$BARRIER_HOST"
# --restore 只负责把注入撤掉:它必须能在"上一轮没有收尾"(甚至还有作业在飞)时也跑通,
# 否则 preflight_not_armed 会把它自己锁死(实测过)。
if [ "$MODE" = "restore" ]; then
record preflight_done pass "restore 模式:只校验 docker/worker/归档根,不检查在飞作业与维护窗口"
return 0
fi
TARGET_INSTANCE="$(worker_env LIBRARY_TARGET_INSTANCE)"
TARGET_PROJECT="$(worker_env LIBRARY_TARGET_PROJECT)"
TARGET_INGRESS_SERVICE="$(worker_env LIBRARY_TARGET_INGRESS_SERVICE)"
TARGET_WRITERS="$(worker_env LIBRARY_TARGET_WRITERS)"
[ -n "$TARGET_INSTANCE" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_INSTANCE(维护目标未接线)"
[ -n "$TARGET_PROJECT" ] || die 2 preflight_topology "worker 没有 LIBRARY_TARGET_PROJECT"
docker inspect "$TARGET_INSTANCE" >/dev/null 2>&1 || die 2 preflight_topology "目标写入者容器缺失:$TARGET_INSTANCE"
INGRESS_CONTAINER="$(container_of_service "$TARGET_PROJECT" "$TARGET_INGRESS_SERVICE")"
[ -n "$INGRESS_CONTAINER" ] || die 2 preflight_topology "目标项目 $TARGET_PROJECT 下找不到入口服务 $TARGET_INGRESS_SERVICE"
local instance_running
instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')"
[ "$instance_running" = "true" ] || die 2 preflight_topology \
"目标写入者 $TARGET_INSTANCE 没在运行:先按预备份恢复并收尾,再来演练"
record preflight_topology pass "instance=$TARGET_INSTANCE running=$instance_running project=$TARGET_PROJECT writers=$TARGET_WRITERS ingress=$INGRESS_CONTAINER"
local held held_instance
held="$(gov_sql "select count(*) from library_maintenance where released_at is null")"
held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")"
if [ "$held" != "0" ] || [ "$held_instance" != "0" ]; then
local owner
owner="$(gov_sql "select coalesce(owner_job_id,'?') from library_maintenance where released_at is null limit 1")"
die 2 preflight_no_open_window "已有未释放的维护窗口(owner=${owner:-?} scope=$held instance=$held_instance):先恢复/收尾"
fi
record preflight_no_open_window pass "no unreleased maintenance"
local in_flight queued
in_flight="$(gov_sql "select coalesce(string_agg(id||'/'||state, ','),'') from library_jobs where state in ('pending','running')")"
[ -z "$in_flight" ] || die 2 preflight_no_inflight "已有在飞的整库作业:$in_flight(一次只演练一个作业)"
queued="$(gov_sql "select count(*) from jobs where type like 'library.%' and state in ('pending','running','retry')")"
[ "$queued" = "0" ] || die 2 preflight_no_inflight "队列里还有 $queued 个 library.* 作业在飞/排队"
record preflight_no_inflight pass "no in-flight library jobs"
# 镜像必须真的带屏障代码:只拷脚本不换镜像,等待会永远落空(旧脚本的病)。
local supported
supported="$(docker run --rm --entrypoint grep "$WORKER_IMAGE_ID" -ac KG_TEST_BARRIER_DIR /app/worker 2>/dev/null | tr -dc '0-9')"
[ "${supported:-0}" -ge 1 ] || die 2 preflight_barrier_supported \
"镜像 $WORKER_IMAGE 里没有屏障代码(grep KG_TEST_BARRIER_DIR=${supported:-0}):先部署带屏障的 worker 镜像"
record preflight_barrier_supported pass "image $WORKER_IMAGE contains barrier"
local target_db_url
target_db_url="$(worker_env LIBRARY_PRODUCTION_DATABASE_URL)"
TARGET_DB_CONTAINER="$(url_part "$target_db_url" host)"
TARGET_DB_USER="$(url_part "$target_db_url" user)"
TARGET_DB_NAME="$(url_part "$target_db_url" db)"
[ -n "$TARGET_DB_CONTAINER" ] || die 2 preflight_target_db "worker 没有 LIBRARY_PRODUCTION_DATABASE_URL:无法独立核对目标内容"
docker inspect "$TARGET_DB_CONTAINER" >/dev/null 2>&1 || die 2 preflight_target_db "目标库容器不存在:$TARGET_DB_CONTAINER"
record preflight_target_db pass "target db=$TARGET_DB_CONTAINER/$TARGET_DB_NAME user=$TARGET_DB_USER"
# 独立写集合校验器缺失 ⇒ 失败。上一版在缺失时把 canonical 记成 not_run 还 pass,
# 等于"没有校验也算校验过"。
if [ ! -x "$VERIFIER" ]; then
die 2 preflight_verifier "写集合独立校验器不存在或不可执行:$VERIFIER(缺校验不许当通过;用 WRITESET_VERIFIER 指向它)"
fi
record preflight_verifier pass "verifier=$VERIFIER"
[ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 2 preflight_not_armed \
"worker 已经带着屏障注入在跑(上一轮没收尾):先 --restore"
record preflight_not_armed pass "barrier not armed"
if [ -n "$EXPECT_JOB" ]; then
case "$EXPECT_JOB" in
libsync-*|librestore-*|libbase-*) ;;
*) die 2 preflight_job_arg "--job 不是整库作业 id:$EXPECT_JOB" ;;
esac
fi
if [ -n "$EXPECT_SNAPSHOT" ]; then
[ "${#EXPECT_SNAPSHOT}" -ge 16 ] || die 2 preflight_job_arg "--snapshot 不像快照 id:$EXPECT_SNAPSHOT"
fi
record preflight_done pass "OK"
}
# ---------- 屏障注入 / 撤销 ----------
compose_argv() { # <override or "">
local override="$1" cfg project_dir file
cfg="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.config_files"}}')"
[ -n "$cfg" ] || { echo "worker 不是 compose 起的:找不到 config_files 标签" >&2; return 1; }
project_dir="$(container_field "$WORKER_CONTAINER" '{{index .Config.Labels "com.docker.compose.project.working_dir"}}')"
COMPOSE_ARGS=(-p "$COMPOSE_PROJECT")
[ -n "$project_dir" ] && COMPOSE_ARGS+=(--project-directory "$project_dir")
while IFS= read -r file; do
# 注入用的 overlay 会被 compose 记进容器的 config_files 标签;原样重放就等于
# 自己把注入又加回去(实测 --restore 撤不掉)。这里按文件名排除演练 overlay。
case "$file" in
""|*/barrier.override.yaml) continue ;;
esac
COMPOSE_ARGS+=(-f "$file")
done < <(tr ',' '\n' <<<"$cfg")
[ -n "$override" ] && COMPOSE_ARGS+=(-f "$override")
return 0
}
write_override() {
cat >"$EVIDENCE/barrier.override.yaml" <<YAML
# 仅本机演练:给 worker 注入固定同步屏障(默认关闭;--restore 撤掉)
services:
$COMPOSE_SERVICE:
environment:
KG_TEST_BARRIER_DIR: "$ARCHIVE_ROOT_IN_CONTAINER/barrier"
KG_TEST_BARRIER_POINTS: "$BARRIER_POINT"
KG_TEST_BARRIER_TIMEOUT: "$BARRIER_TIMEOUT"
YAML
}
wait_worker_ready() {
local deadline=$(( $(now_epoch) + RESTART_WAIT_S ))
while [ "$(now_epoch)" -lt "$deadline" ]; do
if [ "$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')" = "true" ] &&
docker logs --since 180s "$WORKER_CONTAINER" 2>&1 | grep -q "worker-1 started"; then
return 0
fi
sleep 1
done
return 1
}
arm_barrier() {
rm -rf "$BARRIER_HOST"
mkdir -p "$BARRIER_HOST" || die 2 arm_barrier "屏障目录建不出来:$BARRIER_HOST"
write_override
compose_argv "$EVIDENCE/barrier.override.yaml" || die 2 arm_barrier "compose 配置取不到"
log "recreating $WORKER_CONTAINER with barrier point=$BARRIER_POINT"
docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 ||
die 2 arm_barrier "compose up 失败(见 compose.log)"
wait_worker_ready || die 2 arm_barrier "worker 重启后没有就绪"
local armed
armed="$(worker_env KG_TEST_BARRIER_DIR)"
[ "$armed" = "$ARCHIVE_ROOT_IN_CONTAINER/barrier" ] || die 2 arm_barrier "屏障注入没生效(KG_TEST_BARRIER_DIR=${armed:-空})"
# 注入会重建容器:镜像/容器身份以重建后的为准(后面要比对"重启前后同镜像")。
WORKER_CONTAINER_ID="$(container_field "$WORKER_CONTAINER" '{{.Id}}')"
WORKER_IMAGE_ID="$(container_field "$WORKER_CONTAINER" '{{.Image}}')"
ARMED=1
record arm_barrier pass "dir=$armed point=$BARRIER_POINT timeout=$BARRIER_TIMEOUT container=$WORKER_CONTAINER_ID"
}
restore_barrier() {
rm -rf "$BARRIER_HOST"
compose_argv "" || die 7 restore "compose 配置取不到"
docker compose "${COMPOSE_ARGS[@]}" up -d --no-deps --force-recreate "$COMPOSE_SERVICE" >>"$EVIDENCE/compose.log" 2>&1 ||
die 7 restore "compose up(撤销注入)失败"
wait_worker_ready || die 7 restore "worker 撤销注入后没有就绪"
[ -z "$(worker_env KG_TEST_BARRIER_DIR)" ] || die 7 restore "屏障注入还在(KG_TEST_BARRIER_DIR 非空)"
[ "$(container_field "$WORKER_CONTAINER" '{{.Image}}')" = "$WORKER_IMAGE_ID" ] || die 7 restore \
"撤销注入后镜像变了:$WORKER_IMAGE_ID -> $(container_field "$WORKER_CONTAINER" '{{.Image}}')"
ARMED=0
record restore pass "barrier env removed, image unchanged"
}
# ---------- 命中、锁作业、取证 ----------
wait_for_hit() {
local base deadline
base="$BARRIER_HOST/$(sed 's/[:/ ]/-/g' <<<"$BARRIER_POINT")"
deadline=$(( $(now_epoch) + HIT_WAIT_S ))
log "ARMED: 现在请在页面上发起作业(scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定})。等待屏障点位 $BARRIER_POINT 命中 …"
while [ "$(now_epoch)" -lt "$deadline" ]; do
if [ -f "$base.timeout" ]; then
cp "$base.timeout" "$EVIDENCE/" 2>/dev/null
die 3 wait_for_hit "屏障超时(worker 已自行继续):这次没有停在写入中间,判 not_run"
fi
if [ -f "$base.hit" ]; then
cp "$base.hit" "$EVIDENCE/"
HIT_SUBJECT="$(sed -n 's/.*"subject":"\([^"]*\)".*/\1/p' "$base.hit")"
record wait_for_hit pass "hit=$BARRIER_POINT subject=${HIT_SUBJECT:-?}"
return 0
fi
sleep 1
done
die 3 wait_for_hit "等待屏障命中超时(${HIT_WAIT_S}s):作业没有到达 $BARRIER_POINT,判 not_run"
}
lock_job() {
JOB_ID="${EXPECT_JOB:-$HIT_SUBJECT}"
[ -n "$JOB_ID" ] || die 3 lock_job "屏障现场没有作业身份,也没有 --job"
if [ -n "$EXPECT_JOB" ] && [ "$HIT_SUBJECT" != "$EXPECT_JOB" ]; then
die 3 lock_job "屏障命中的是别的作业(hit=$HIT_SUBJECT 期望=$EXPECT_JOB):拒绝继续"
fi
local row others
row="$(gov_sql "select state||'|'||scope||'|'||kind||'|'||coalesce(snapshot_id,'')||'|'||coalesce(target_snapshot_id,'')||'|'||created_at::text from library_jobs where id='$JOB_ID'")"
[ -n "$row" ] || die 3 lock_job "治理库里没有作业 $JOB_ID"
IFS='|' read -r JOB_STATE JOB_SCOPE JOB_KIND JOB_SNAPSHOT JOB_TARGET JOB_CREATED <<<"$row"
[ "$JOB_SCOPE" = "$SCOPE" ] || die 3 lock_job "作业 scope=$JOB_SCOPE 与请求 $SCOPE 不一致"
[ "$JOB_STATE" = "running" ] || die 3 lock_job "作业状态是 $JOB_STATE 不是 running:命中时它已经不在写入路径上了"
if [ "$KIND" != "auto" ] && [ "$JOB_KIND" != "$KIND" ]; then
die 3 lock_job "作业 kind=$JOB_KIND 与请求 --kind $KIND 不一致(期望内容也会按错的那一类推导)"
fi
if [ -n "$EXPECT_SNAPSHOT" ] && [ "$JOB_TARGET" != "$EXPECT_SNAPSHOT" ]; then
die 3 lock_job "作业目标快照 $JOB_TARGET 与 --snapshot $EXPECT_SNAPSHOT 不一致:拒绝把它当这次演练"
fi
if [ -n "$SINCE" ]; then
local since_epoch created_epoch
if [[ "$SINCE" =~ ^[0-9]+$ ]]; then since_epoch="$SINCE"; else since_epoch="$(date -u -d "$SINCE" +%s 2>/dev/null || echo 0)"; fi
created_epoch="$(date -u -d "$JOB_CREATED" +%s 2>/dev/null || echo 0)"
{ [ "$since_epoch" -gt 0 ] && [ "$created_epoch" -ge "$since_epoch" ]; } ||
die 3 lock_job "作业创建时间 $JOB_CREATED 不满足 --since $SINCE(或 --since 解析不了)"
fi
# 目标快照 == 中断前的当前生效版本 ⇒ 这次"接管"只可能走同快照快速收尾,
# 证明不了完整续作(2026-10-29 的现场正是如此)。杀之前就拒绝,别浪费一次中断。
PRE_EFFECTIVE="$(gov_sql "select coalesce(snapshot_id,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")"
if [ -n "$PRE_EFFECTIVE" ] && [ "$JOB_TARGET" = "$PRE_EFFECTIVE" ] && [ "$ALLOW_SAME_SNAPSHOT" -ne 1 ]; then
die 3 lock_job "目标快照 $JOB_TARGET 就是中断前的当前生效版本:同快照收尾不能当完整续作的验收证据(要演练请换一个没生效过的快照;确要同快照加 --allow-same-snapshot)"
fi
others="$(gov_sql "select coalesce(string_agg(id, ','),'') from library_jobs where state in ('pending','running') and id<>'$JOB_ID'")"
[ -z "$others" ] || die 3 lock_job "还有别的作业在飞:$others(只允许杀这一个作业的执行者)"
local expected
if [ -n "$EXPECT_COUNTS" ]; then
expected="$EXPECT_COUNTS"
log "期望内容来自 --expect:$expected"
else
expected="$(snapshot_expectations "$JOB_TARGET")" || die 3 lock_job "推导目标快照的期望内容失败($JOB_TARGET)"
EXPECT_COUNTS="$expected"
log "期望内容来自快照 manifest($JOB_TARGET):$expected;resources 不在默认期望内(同租户其它知识库资源会保留)"
fi
case "$expected" in
*=*) ;;
*) die 3 lock_job "期望内容形状不对:$expected" ;;
esac
record lock_job pass "kind=$JOB_KIND job=$JOB_ID state=running scope=$JOB_SCOPE target=$JOB_TARGET pre_effective=${PRE_EFFECTIVE:-none} expect=$EXPECT_COUNTS created=$JOB_CREATED"
}
read_writeset_evidence() {
local row
row="$(gov_sql "select coalesce(evidence->>'writeset_digest','')||'|'||coalesce(evidence->>'writeset_reused_rows','')||'|'||coalesce(evidence->>'writeset_target_snapshot','')||'|'||coalesce(evidence->>'backup_snapshot','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
[ -n "$row" ] || die 4 read_writeset "作业 $JOB_ID 还没有预备份版本记录:屏障点位早于台账登记,不能当「写集合已登记」的证据"
IFS='|' read -r WS_DIGEST WS_REUSED WS_TARGET WS_BACKUP <<<"$row"
[ -n "$WS_DIGEST" ] || die 4 read_writeset "预备份记录里没有 writeset_digest:接管会失败,这次不算可接管"
[ "$WS_TARGET" = "$JOB_TARGET" ] || die 4 read_writeset "写集合绑定目标 $WS_TARGET 与作业目标 $JOB_TARGET 不一致"
WS_FILE="$ARCHIVE_HOST/writesets/$WS_DIGEST.json"
[ -s "$WS_FILE" ] || die 4 read_writeset "写集合文件不存在或为空:$WS_FILE"
grep -q "\"digest\":\"$WS_DIGEST\"" "$WS_FILE" || die 4 read_writeset "写集合文件自述摘要与登记不一致($WS_DIGEST)"
grep -q "\"job_id\":\"$JOB_ID\"" "$WS_FILE" || die 4 read_writeset "写集合文件不属于作业 $JOB_ID"
# 独立校验器是必需的(preflight 已确认存在):它判定不符就不能继续。
"$VERIFIER" -archive-root "$ARCHIVE_HOST" -digest "$WS_DIGEST" -job "$JOB_ID" -scope "$SCOPE" \
>>"$EVIDENCE/verifier.log" 2>&1 || die 4 read_writeset "独立校验器判定写集合摘要不符(见 verifier.log)"
record read_writeset pass "digest=$WS_DIGEST reused=${WS_REUSED:-0} backup=$WS_BACKUP canonical=verified"
}
kill_worker() {
[ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill:杀容器必须是显式动作"
local before_restart job_state deadline owner_uid
before_restart="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')"
job_state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
[ "$job_state" = "running" ] || die 4 kill "杀之前作业状态变成 $job_state:这次不构成写入中的中断"
KILL_PID="$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')"
[ -n "$KILL_PID" ] && [ "$KILL_PID" -gt 1 ] || die 4 kill "读不到 worker 的宿主机 PID(${KILL_PID:-空})"
# 只有"同一 uid"或 root 才能向容器 PID1 发信号;先证明能杀,再谈演练。
owner_uid="$(stat -c %u "/proc/$KILL_PID" 2>/dev/null)"
if [ "$(id -u)" != "0" ] && [ "${owner_uid:-x}" != "$(id -u)" ]; then
die 4 kill "宿主机 PID $KILL_PID 属于 uid=${owner_uid:-?},当前 uid=$(id -u) 无权 SIGKILL:不能用「进程崩溃」语义演练(换 --manual-start 方案前先解决权限)"
fi
kill -0 "$KILL_PID" 2>/dev/null || die 4 kill "PID $KILL_PID 不存活,杀不到真实进程"
# 退出码证据从 docker events 取:restart policy 会很快把容器拉起来,
# State.ExitCode 随即被重置成 0,事后 inspect 读不到 137(这正是上一版没留下
# "真的 137"证据的原因之一)。
: >"$EVIDENCE/docker-events.log"
timeout 180 docker events --filter "container=$WORKER_CONTAINER_ID" --since "$(now_epoch)" \
--format '{{json .}}' >>"$EVIDENCE/docker-events.log" 2>>"$EVIDENCE/docker.log" &
EVENTS_PID=$!
KILLED_AT="$(utc_now)"
RESTART_COUNT_BEFORE="$before_restart"
log "SIGKILL worker container PID1 host_pid=$KILL_PID (job=$JOB_ID at $KILLED_AT) — 进程崩溃语义,不是 docker kill"
kill -9 "$KILL_PID" 2>>"$EVIDENCE/docker.log" || die 4 kill "向 PID $KILL_PID 发送 SIGKILL 失败"
deadline=$(( $(now_epoch) + KILL_PROOF_S ))
KILL_PROOF=""
while [ "$(now_epoch)" -lt "$deadline" ]; do
if grep '"Action":"die"' "$EVIDENCE/docker-events.log" 2>/dev/null | grep -q '"exitCode":"137"'; then
KILL_PROOF="docker-events die exitCode=137"
break
fi
sleep 1
done
[ -n "$KILL_PROOF" ] || die 4 kill "docker events 里没有 exitCode=137 的 die 事件(${KILL_PROOF_S}s 内):这次没有证明真被杀"
record kill pass "container=$WORKER_CONTAINER id=$WORKER_CONTAINER_ID host_pid=$KILL_PID exit=137 proof=\"$KILL_PROOF\" killed_at=$KILLED_AT restart_before=$RESTART_COUNT_BEFORE semantics=process-crash-sigkill"
}
wait_restart() {
local deadline running
deadline=$(( $(now_epoch) + POLICY_RESTART_WAIT_S ))
RESTART_SOURCE=""
while [ "$(now_epoch)" -lt "$deadline" ]; do
running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')"
if [ "$running" = "true" ] && [ "$(container_field "$WORKER_CONTAINER" '{{.State.Pid}}')" != "$KILL_PID" ]; then
RESTART_SOURCE="policy"
break
fi
sleep 1
done
if [ -z "$RESTART_SOURCE" ]; then
if [ "$MANUAL_START" -ne 1 ]; then
die 5 restart "restart policy 在 ${POLICY_RESTART_WAIT_S}s 内没有拉起(未允许显式 docker start):这次不算自动接管"
fi
log "restart policy 没有在 ${POLICY_RESTART_WAIT_S}s 内拉起:显式 docker start(记 restart_source=manual_start,不是自动恢复)"
docker start "$WORKER_CONTAINER" >>"$EVIDENCE/docker.log" 2>&1 || die 5 restart "显式 docker start 失败"
RESTART_SOURCE="manual_start"
deadline=$(( $(now_epoch) + RESTART_WAIT_S ))
while [ "$(now_epoch)" -lt "$deadline" ]; do
running="$(container_field "$WORKER_CONTAINER" '{{.State.Running}}')"
[ "$running" = "true" ] && break
sleep 1
done
[ "$running" = "true" ] || die 5 restart "显式启动后容器仍没运行"
fi
local image_after cid_after restarts_after
image_after="$(container_field "$WORKER_CONTAINER" '{{.Image}}')"
cid_after="$(container_field "$WORKER_CONTAINER" '{{.Id}}')"
restarts_after="$(container_field "$WORKER_CONTAINER" '{{.RestartCount}}')"
[ "$image_after" = "$WORKER_IMAGE_ID" ] || die 5 restart "重启后镜像变了:$WORKER_IMAGE_ID -> $image_after(不是同一个产物)"
[ "$cid_after" = "$WORKER_CONTAINER_ID" ] || die 5 restart "重启后容器身份变了:$WORKER_CONTAINER_ID -> $cid_after"
record restart pass "source=$RESTART_SOURCE image_unchanged=1 container_unchanged=1 restart_count=$RESTART_COUNT_BEFORE->$restarts_after"
}
wait_takeover() {
local deadline state="" last_note=0
deadline=$(( $(now_epoch) + TAKEOVER_WAIT_S ))
log "等待同一个作业被接管(lease 30s + 扫描 30s,上限 ${TAKEOVER_WAIT_S}s)…"
while [ "$(now_epoch)" -lt "$deadline" ]; do
state="$(gov_sql "select state from library_jobs where id='$JOB_ID'")"
case "$state" in succeeded|failed|cancelled) break ;; esac
# 60s 没动静就把现场抓下来(别再空等到底才知道为什么)。
if [ $(( $(now_epoch) - last_note )) -ge 60 ]; then
last_note="$(now_epoch)"
docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | tail -5 >>"$EVIDENCE/worker.log"
log "接管等待中:job state=${state:-?} worker_running=$(container_field "$WORKER_CONTAINER" '{{.State.Running}}') queue=$(gov_sql "select coalesce(string_agg(type||':'||state,','),'') from jobs where type like 'library.%' and state in ('pending','running','retry')")"
fi
sleep 2
done
case "$state" in
succeeded) ;;
failed|cancelled) die 5 wait_takeover "作业终态是 $state(接管失败):$(gov_sql "select coalesce(error,'') from library_jobs where id='$JOB_ID'")" ;;
*) die 5 wait_takeover "等待接管超时(最后状态 ${state:-?}):不算成功" ;;
esac
# 人工拉起/策略拉起都要能对得上"启动扫描真的接了这个作业"。
local resumed
resumed="$(docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" 2>&1 | grep -c "library job $JOB_ID resumed after worker restart")"
[ "${resumed:-0}" -ge 1 ] || die 5 wait_takeover "worker 日志里没有本作业的 resume 记录:接管来源不明"
record wait_takeover pass "same job succeeded; restart_source=$RESTART_SOURCE; resume_log=$resumed"
}
verify_stages_complete() {
local stages missing="" stage
stages="$(gov_sql "select stage||':'||state from library_job_stages where job_id='$JOB_ID'")"
for stage in database resources index wiki verify; do
grep -qx "$stage:succeeded" <<<"$stages" || missing+="$stage "
done
[ -z "$missing" ] || die 6 verify_stages \
"作业没有真跑完写入链路:未 succeeded 的阶段 = $missing(stages=$(tr '\n' ',' <<<"$stages"))"
record verify_stages pass "database/resources/index/wiki/verify 全部 succeeded"
}
verify_takeover_semantics() {
local stages finalize_at eff after_digest
stages="$(gov_sql "select coalesce(string_agg(stage||':'||state, ',' order by updated_at),'') from library_job_stages where job_id='$JOB_ID'")"
grep -q "prebackup:skipped" <<<"$stages" || die 6 verify_takeover \
"阶段里没有 prebackup:skipped:无法证明走的是接管路径(stages=$stages)"
finalize_at="$(gov_sql "select coalesce(max(updated_at)::text,'') from library_job_stages where job_id='$JOB_ID' and stage='finalize' and state='succeeded'")"
[ -n "$finalize_at" ] || die 6 verify_takeover "finalize 没有成功(stages=$stages)"
record verify_takeover pass "stages=$stages finalize_at=$finalize_at"
eff="$(gov_sql "select coalesce(snapshot_id,'')||'|'||coalesce(source_job_id,'')||'|'||coalesce(superseded_at::text,'') from library_versions where scope='$SCOPE' and kind='production_effective' and superseded_at is null")"
local eff_snapshot eff_job eff_superseded
IFS='|' read -r eff_snapshot eff_job eff_superseded <<<"$eff"
[ "$eff_snapshot" = "$JOB_TARGET" ] || die 6 verify_effective "当前生效版本是 ${eff_snapshot:-无},不是本作业目标 $JOB_TARGET"
[ "$eff_job" = "$JOB_ID" ] || die 6 verify_effective "生效版本来源作业是 ${eff_job:-无},不是 $JOB_ID(同快照由别作业登记不算本次完成)"
[ -z "$eff_superseded" ] || die 6 verify_effective "生效版本已被顶替($eff_superseded)"
after_digest="$(gov_sql "select coalesce(evidence->>'writeset_digest','') from library_versions where scope='$SCOPE' and kind='production_prebackup' and source_job_id='$JOB_ID'")"
[ "$after_digest" = "$WS_DIGEST" ] || die 6 verify_effective "接管后写集合摘要变了($WS_DIGEST -> ${after_digest:-空})"
record verify_effective pass "effective=$eff_snapshot job=$JOB_ID writeset=$after_digest"
}
verify_target_counts() {
local part table want got failed=0 detail="" sql=""
for part in ${EXPECT_COUNTS//,/ }; do
table="${part%%=*}"
case "$table" in
*[!a-z_]*) die 6 verify_counts "期望里的表名不合法:$table" ;;
esac
sql+="select '$table='||count(*) from $table; "
done
got="$(docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "$sql" \
2>>"$EVIDENCE/psql.log")"
[ -n "$got" ] || die 6 verify_counts "目标库读数失败($TARGET_DB_CONTAINER/$TARGET_DB_NAME)"
for part in ${EXPECT_COUNTS//,/ }; do
table="${part%%=*}"; want="${part##*=}"
if ! grep -qx "$table=$want" <<<"$got"; then
failed=$((failed + 1))
detail+="$table=$(sed -n "s/^$table=//p" <<<"$got")(want $want) "
fi
done
[ "$failed" -eq 0 ] || die 6 verify_counts "目标内容与期望不符:$detail(实际 $(tr '\n' ' ' <<<"$got"))"
record verify_counts pass "counts=$(tr '\n' ' ' <<<"$got")"
}
verify_service_restored() {
local instance_running ingress_running held held_instance released released_instance
instance_running="$(container_field "$TARGET_INSTANCE" '{{.State.Running}}')"
ingress_running="$(container_field "$INGRESS_CONTAINER" '{{.State.Running}}')"
[ "$instance_running" = "true" ] || die 6 verify_service "目标写入者 $TARGET_INSTANCE 没有恢复运行"
[ "$ingress_running" = "true" ] || die 6 verify_service "入口 $INGRESS_CONTAINER 没有恢复运行"
held="$(gov_sql "select count(*) from library_maintenance where released_at is null")"
held_instance="$(gov_sql "select count(*) from library_instance_maintenance where released_at is null")"
released="$(gov_sql "select count(*) from library_maintenance where owner_job_id='$JOB_ID' and released_at is not null")"
released_instance="$(gov_sql "select count(*) from library_instance_maintenance where owner_job_id='$JOB_ID' and released_at is not null")"
{ [ "$held" = "0" ] && [ "$held_instance" = "0" ]; } || die 6 verify_service \
"还有未释放的维护占用(scope=$held instance=$held_instance)"
{ [ "$released" -ge 1 ] && [ "$released_instance" -ge 1 ]; } || die 6 verify_service \
"本作业的维护记录没有释放证据(scope=$released instance=$released_instance)"
record verify_service pass "instance_running=$instance_running ingress_running=$ingress_running health(instance)=$(health_of "$TARGET_INSTANCE") health(ingress)=$(health_of "$INGRESS_CONTAINER") released=$released/$released_instance"
}
save_evidence() {
{
echo "# 脱敏证据(job=$JOB_ID kind=$JOB_KIND scope=$SCOPE target=$JOB_TARGET killed_at=$KILLED_AT restart_source=$RESTART_SOURCE)"
echo "## library_jobs"
gov_sql "select id||E'\t'||scope||E'\t'||kind||E'\t'||state||E'\t'||stage||E'\t'||coalesce(snapshot_id,'')||E'\t'||coalesce(target_snapshot_id,'')||E'\t'||created_at::text||E'\t'||coalesce(finished_at::text,'') from library_jobs where id='$JOB_ID'"
echo "## library_job_stages"
gov_sql "select stage||E'\t'||state||E'\t'||coalesce(total::text,'')||E'\t'||coalesce(completed::text,'')||E'\t'||coalesce(detail,'')||E'\t'||updated_at::text from library_job_stages where job_id='$JOB_ID' order by updated_at"
echo "## library_versions(本作业)"
gov_sql "select kind||E'\t'||snapshot_id||E'\t'||source_job_id||E'\t'||created_at::text||E'\t'||coalesce(superseded_at::text,'')||E'\t'||evidence::text from library_versions where source_job_id='$JOB_ID' order by created_at"
echo "## 维护占用(本作业;实例表列是 owner_scope)"
gov_sql "select 'scope|'||scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_maintenance where owner_job_id='$JOB_ID' union all select 'instance|'||owner_scope||'|'||state||'|'||coalesce(released_at::text,'not_released') from library_instance_maintenance where owner_job_id='$JOB_ID'"
echo "## 目标内容计数"
docker exec "$TARGET_DB_CONTAINER" psql -U "$TARGET_DB_USER" -d "$TARGET_DB_NAME" -Atc "select 'knowledges='||count(*) from knowledges union all select 'chunks='||count(*) from chunks union all select 'resources='||count(*) from resources union all select 'wiki_pages='||count(*) from wiki_pages" 2>/dev/null
} >"$EVIDENCE/job-evidence.txt"
docker logs --since "$KILLED_AT" "$WORKER_CONTAINER" >"$EVIDENCE/worker.log" 2>&1
log "evidence written: $EVIDENCE/job-evidence.txt"
}
# ---------- 主流程 ----------
log "library-kill-takeover v$SCRIPT_VERSION mode=$MODE scope=$SCOPE kind=$KIND target=${EXPECT_SNAPSHOT:-未指定} evidence=$EVIDENCE"
preflight
case "$MODE" in
check)
STATUS_OVERRIDE="READY_FOR_PARENT"
finish 0
;;
restore)
restore_barrier
finish 0
;;
run)
[ "$CONFIRM_KILL" -eq 1 ] || die 4 kill "缺少 --confirm-kill"
arm_barrier
wait_for_hit
lock_job
read_writeset_evidence
kill_worker
wait_restart
wait_takeover
verify_stages_complete
verify_takeover_semantics
verify_target_counts
verify_service_restored
save_evidence
restore_barrier
STATUS_OVERRIDE="DRILL_PASSED"
finish 0
;;
esac