Files
knowledge-governance/scripts/logical-backup.sh
T
姚章浩 b32755b35f feat(backup): 备份目录契约 + 导入前留档覆盖 ES 层
- scripts/backup_layout.sh:定义 $KB_BACKUP_ROOT/{target-kb-snapshots,rds-logical,es-snapshots,logs}
  与 /etc/knowledge-governance/backup.env,任意主机跑同一份(测试云主机 → 未来生产云主机)
- scripts/es_kb_export.py:只读 scroll 导出目标库 ES 文档(gzip JSONL,_id/_source,可按 _id 重建)
- scripts/target_kb_snapshot.sh:留档改为 PG 行 + ES 文档两层;PULL_ES 默认 0(692 MB / 家用上行 19m40s,
  留在目标主机,清单记 location=cloud:<host>:<path>);挂载源与落点统一走 KB_BACKUP_ROOT
- scripts/logical-backup.sh:自管整库逻辑备份(pg_dump -Fd -j4 -Z6,按天轮转)
- internal/releasesnapshot:门禁要求 PG 与 ES 两层齐备(ES 计数 >0);云侧留档核声明与哈希、
  声明本地却缺失则 fail-closed;新增 3 条用例(共 10)
- cmd/guardcheck:用真实产物复核门禁判断
- docs:PRODUCTION_BACKUP_RECOVERY.md 重写(契约/RDS 控制台/恢复路径/边界);P6 同步留档语义
2026-10-09 12:19:15 +08:00

190 lines
8.4 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# 目标 RDS PostgreSQL 的**自管逻辑备份**:手动触发,文件落在你自己这台云主机上,
# 不经过 RDS 托管备份服务。
#
# 为什么要它:RDS 托管的备份只存在阿里云内部存储,你既看不到也拿不走,只能"恢复成
# 新实例";这份是你能随手取用、能在任何地方 pg_restore 的自有副本。两者互补:
# 托管备份 + 日志备份负责"按时间点恢复",这份负责"带走 / 异地 / 独立校验"。
#
# 用法(在云主机上):
# bash logical-backup.sh # 全库逻辑备份 + 轮转
# BACKUP_ROOT=/data/rds-backups bash logical-backup.sh
# DBS="weknora_test" bash logical-backup.sh # 只备指定库
# KEEP_DAILY=14 KEEP_MONTHS=6 bash logical-backup.sh
# DRY_RUN=1 bash logical-backup.sh # 只看会做什么
#
# 环境变量:
# BACKUP_ROOT 备份根目录(默认 /opt/weknora-test/rds-backups)
# APP_CONTAINER 提供 DB 凭据的容器(默认 weknora-test-v071-app-1)
# PG_IMAGE pg_dump 容器镜像(默认 postgres:17.9-alpine,必须 ≥ 服务端 17.9)
# DBS 逗号分隔的库名;默认自动列出非模板库
# JOBS 并行度(默认 4)
# KEEP_DAILY 保留最近几份每日备份(默认 7)
# KEEP_MONTHS 额外保留最近几个自然月各一份(默认 3)
# EXCLUDE_DBS 逗号分隔的跳过列表(默认空)
set -euo pipefail
# 备份目录契约:由 scripts/backup_layout.sh 写到 /etc/knowledge-governance/backup.env。
# 换主机(测试云主机 → 生产云主机)时只改这一处,脚本本身不动。
if [ -z "${KB_BACKUP_ROOT:-}" ] && [ -f /etc/knowledge-governance/backup.env ]; then
# shellcheck disable=SC1091
. /etc/knowledge-governance/backup.env
fi
KB_BACKUP_ROOT="${KB_BACKUP_ROOT:-/srv/kb-backups}"
BACKUP_ROOT="${BACKUP_ROOT:-$KB_BACKUP_ROOT/rds-logical}"
APP_CONTAINER="${APP_CONTAINER:-weknora-test-v071-app-1}"
PG_IMAGE="${PG_IMAGE:-postgres:17.9-alpine}"
JOBS="${JOBS:-4}"
KEEP_DAILY="${KEEP_DAILY:-7}"
KEEP_MONTHS="${KEEP_MONTHS:-3}"
DRY_RUN="${DRY_RUN:-0}"
DBS="${DBS:-}"
EXCLUDE_DBS="${EXCLUDE_DBS:-}"
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
env_value() {
docker inspect "$APP_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' \
| grep -E "^$1=" | head -1 | cut -d= -f2-
}
DB_HOST_VALUE="$(env_value DB_HOST)"; DB_PORT_VALUE="$(env_value DB_PORT)"
DB_USER_VALUE="$(env_value DB_USER)"; DB_PASS_VALUE="$(env_value DB_PASSWORD)"
if [ -z "${DB_HOST_VALUE}${DB_PORT_VALUE}${DB_USER_VALUE}${DB_PASS_VALUE}" ]; then
echo "缺少数据库凭据:容器 $APP_CONTAINER 里没有 DB_* 环境变量" >&2; exit 1
fi
PGURL="$(python3 - "$DB_USER_VALUE" "$DB_PASS_VALUE" "$DB_HOST_VALUE" "$DB_PORT_VALUE" "$(env_value DB_NAME)" <<'PY'
import sys, urllib.parse as u
user, password, host, port, name = sys.argv[1:6]
print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")
PY
)"
# 库列表:默认所有非模板库
if [ -z "$DBS" ]; then
DBS="$(docker run --rm -e PGURL="$PGURL" "$PG_IMAGE" sh -c \
"psql \"\$PGURL\" -tAc \"select datname from pg_database where datistemplate = false and datallowconn order by datname\"" \
| tr -d '\r' | paste -sd, -)"
fi
if [ -n "$EXCLUDE_DBS" ]; then
DBS="$(python3 -c "
import sys
exclude = set(filter(None, sys.argv[2].split(',')))
print(','.join(d for d in sys.argv[1].split(',') if d and d not in exclude))
" "$DBS" "$EXCLUDE_DBS")"
fi
if [ -z "$DBS" ]; then echo "没有可备份的库" >&2; exit 1; fi
TARGET="$BACKUP_ROOT/$STAMP"
echo "备份目标: $TARGET"
echo "库列表 : $DBS"
echo "镜像 : $PG_IMAGE (并行 $JOBS)"
if [ "$DRY_RUN" = "1" ]; then echo "DRY_RUN=1,不执行"; exit 0; fi
mkdir -p "$TARGET"
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
START_SECONDS="$(date +%s)"
# 全局对象(角色/表空间):RDS 上通常受限,尽力而为,失败不阻塞。
docker run --rm -e PGURL="$PGURL" "$PG_IMAGE" sh -c \
"pg_dumpall --globals-only --no-role-passwords --dbname=\"\$PGURL\"" >"$TARGET/globals.sql" 2>"$TARGET/globals.err" || true
[ -s "$TARGET/globals.err" ] || rm -f "$TARGET/globals.err"
IFS=',' read -r -a DB_ARRAY <<<"$DBS"
for db in "${DB_ARRAY[@]}"; do
db="$(printf '%s' "$db" | tr -d ' ')"
[ -n "$db" ] || continue
DB_URL="$(python3 - "$DB_USER_VALUE" "$DB_PASS_VALUE" "$DB_HOST_VALUE" "$DB_PORT_VALUE" "$db" <<'PY'
import sys, urllib.parse as u
user, password, host, port, name = sys.argv[1:6]
print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")
PY
)"
echo " - $db ..."
docker run --rm -e DB_URL="$DB_URL" -v "$TARGET:/backup" "$PG_IMAGE" sh -c \
"pg_dump -Fd -j $JOBS -Z 6 --no-sync -d \"\$DB_URL\" -f /backup/${db}.dir"
# pg_dump -Fd 不接受已有目录;目录名冲突说明本次重复执行,直接报错退出
done
# dump 之后写清单与说明(先写清单,再算哈希,避免把哈希写进自己的输入)
DURATION="$(( $(date +%s) - START_SECONDS ))"
python3 - "$TARGET" "$STAMP" "$STARTED_AT" "$DURATION" "$DBS" "$PG_IMAGE" "$JOBS" <<'PY'
import hashlib, json, os, pathlib, sys
target = pathlib.Path(sys.argv[1]); stamp, started, duration = sys.argv[2], sys.argv[3], int(sys.argv[4])
databases, image, jobs = sys.argv[5], sys.argv[6], int(sys.argv[7])
files = []
for path in sorted(target.rglob("*")):
if path.is_dir() or path.name in ("MANIFEST.json",):
continue
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
digest.update(block)
files.append({"file": str(path.relative_to(target)), "bytes": path.stat().st_size,
"sha256": digest.hexdigest()})
manifest = {
"kind": "rds-logical-backup",
"created_at": stamp,
"started_at": started,
"duration_seconds": duration,
"databases": [name for name in databases.split(",") if name],
"format": "pg_dump -Fd (directory, parallel, zlib)",
"pg_dump_image": image,
"parallel_jobs": jobs,
"restore_hint": "pg_restore -d <目标库> -j4 <库名>.dir(先建库;跨实例恢复可能需要 --no-owner/--no-acl)",
"total_bytes": sum(item["bytes"] for item in files),
"files": files,
}
(target / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
print(f"duration={duration}s total={manifest['total_bytes']/1048576:.1f}MiB files={len(files)}")
PY
cat >"$TARGET/README.md" <<'MD'
# 自管逻辑备份(pg_dump 目录格式)
- 谁做的:`logical-backup.sh`,手动触发,文件在本机磁盘(不经 RDS 托管备份服务)。
- 每个库一个 `.dir` 目录(`pg_dump -Fd -jN -Z6`):**目录格式支持并行恢复与按表选择性恢复**。
- `MANIFEST.json`:库列表、耗时、每个文件的 sha256/size。
- `globals.sql`:角色/表空间(RDS 上通常受限,尽力而为,可能为空)。
## 怎么恢复
```bash
# 1) 建目标库(已在别处建好则跳过)
psql "$TARGET_URL" -c 'CREATE DATABASE restore_check'
# 2) 并行恢复(-j 至少 4;目录格式才能并行)
docker run --rm -e TARGET_URL="$TARGET_URL" -v "$PWD:/backup" postgres:17.9-alpine \
pg_restore -d "$TARGET_URL" -j4 --clean --if-exists /backup/<库名>.dir
# 3) 只验某张表(不动生产)
docker run --rm -v "$PWD:/backup" postgres:17.9-alpine \
pg_restore -l /backup/<库名>.dir | grep -i wiki_pages
```
注意:
- **pg_restore 版本必须 ≥ dump 时的 pg_dump**(本次是 17.9)→ 用 `postgres:17.9-alpine`。
- 跨实例恢复若报角色/权限错误,加 `--no-owner --no-acl`。
- 这份备份**不能替代按时间点恢复(PITR)**:它只能回到"备份那一刻"。
MD
# 轮转:保留最近 KEEP_DAILY 份,另在最近 KEEP_MONTHS 个自然月各留一份
python3 - "$BACKUP_ROOT" "$KEEP_DAILY" "$KEEP_MONTHS" <<'PY'
import pathlib, re, shutil, sys
root = pathlib.Path(sys.argv[1]); keep_daily = int(sys.argv[2]); keep_months = int(sys.argv[3])
stamps = sorted([p for p in root.iterdir() if p.is_dir() and re.fullmatch(r"\d{8}T\d{6}Z", p.name)], reverse=True)
keep = {p.name for p in stamps[:keep_daily]}
months = []
for path in stamps:
month = path.name[:6]
if month not in months:
months.append(month)
keep.add(path.name)
if len(months) >= keep_months:
break
for path in stamps:
if path.name not in keep:
shutil.rmtree(path)
print("rotated out:", path.name)
print("kept:", len([p for p in stamps if p.name in keep]))
PY
echo "done: $TARGET"