姚章浩
19e0596e1d
单库 pg_dump 失败（权限/编码/管理库）不再中止整次备份：清掉半个目录、记进 MANIFEST.failed_databases、 轮转照常执行、最后按失败情况给退出码。云主机实跑：33 秒 / 742.6 MiB / 8 库 / 失败 0。
202 lines
8.8 KiB
Bash
202 lines
8.8 KiB
Bash
#!/usr/bin/env bash
|
||
# 目标 RDS PostgreSQL 的**自管逻辑备份**：手动触发，文件落在你自己这台云主机上，
|
||
# 不经过 RDS 托管备份服务。
|
||
#
|
||
# 为什么要它：RDS 托管的备份只存在阿里云内部存储，你既看不到也拿不走，只能"恢复成
|
||
# 新实例"；这份是你能随手取用、能在任何地方 pg_restore 的自有副本。两者互补：
|
||
# 托管备份 + 日志备份负责"按时间点恢复"，这份负责"带走 / 异地 / 独立校验"。
|
||
#
|
||
# 用法（在云主机上）：
|
||
# bash logical-backup.sh # 全库逻辑备份 + 轮转
|
||
# BACKUP_ROOT=/data/rds-backups bash logical-backup.sh
|
||
# DBS="weknora_test" bash logical-backup.sh # 只备指定库
|
||
# KEEP_DAILY=14 KEEP_MONTHS=6 bash logical-backup.sh
|
||
# DRY_RUN=1 bash logical-backup.sh # 只看会做什么
|
||
#
|
||
# 环境变量：
|
||
# BACKUP_ROOT 备份根目录（默认 /opt/weknora-test/rds-backups）
|
||
# APP_CONTAINER 提供 DB 凭据的容器（默认 weknora-test-v071-app-1）
|
||
# PG_IMAGE pg_dump 容器镜像（默认 postgres:17.9-alpine，必须 ≥ 服务端 17.9）
|
||
# DBS 逗号分隔的库名；默认自动列出非模板库
|
||
# JOBS 并行度（默认 4）
|
||
# KEEP_DAILY 保留最近几份每日备份（默认 7）
|
||
# KEEP_MONTHS 额外保留最近几个自然月各一份（默认 3）
|
||
# EXCLUDE_DBS 逗号分隔的跳过列表（默认 rdsadmin：RDS 管理库，普通账号导不出来）
|
||
set -euo pipefail
|
||
|
||
# 备份目录契约：由 scripts/backup_layout.sh 写到 /etc/knowledge-governance/backup.env。
|
||
# 换主机（测试云主机 → 生产云主机）时只改这一处，脚本本身不动。
|
||
if [ -z "${KB_BACKUP_ROOT:-}" ] && [ -f /etc/knowledge-governance/backup.env ]; then
|
||
# shellcheck disable=SC1091
|
||
. /etc/knowledge-governance/backup.env
|
||
fi
|
||
KB_BACKUP_ROOT="${KB_BACKUP_ROOT:-/srv/kb-backups}"
|
||
BACKUP_ROOT="${BACKUP_ROOT:-$KB_BACKUP_ROOT/rds-logical}"
|
||
APP_CONTAINER="${APP_CONTAINER:-weknora-test-v071-app-1}"
|
||
PG_IMAGE="${PG_IMAGE:-postgres:17.9-alpine}"
|
||
JOBS="${JOBS:-4}"
|
||
KEEP_DAILY="${KEEP_DAILY:-7}"
|
||
KEEP_MONTHS="${KEEP_MONTHS:-3}"
|
||
DRY_RUN="${DRY_RUN:-0}"
|
||
DBS="${DBS:-}"
|
||
EXCLUDE_DBS="${EXCLUDE_DBS:-rdsadmin}"
|
||
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||
|
||
env_value() {
|
||
docker inspect "$APP_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' \
|
||
| grep -E "^$1=" | head -1 | cut -d= -f2-
|
||
}
|
||
DB_HOST_VALUE="$(env_value DB_HOST)"; DB_PORT_VALUE="$(env_value DB_PORT)"
|
||
DB_USER_VALUE="$(env_value DB_USER)"; DB_PASS_VALUE="$(env_value DB_PASSWORD)"
|
||
if [ -z "${DB_HOST_VALUE}${DB_PORT_VALUE}${DB_USER_VALUE}${DB_PASS_VALUE}" ]; then
|
||
echo "缺少数据库凭据：容器 $APP_CONTAINER 里没有 DB_* 环境变量" >&2; exit 1
|
||
fi
|
||
PGURL="$(python3 - "$DB_USER_VALUE" "$DB_PASS_VALUE" "$DB_HOST_VALUE" "$DB_PORT_VALUE" "$(env_value DB_NAME)" <<'PY'
|
||
import sys, urllib.parse as u
|
||
user, password, host, port, name = sys.argv[1:6]
|
||
print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")
|
||
PY
|
||
)"
|
||
|
||
# 库列表：默认所有非模板库
|
||
if [ -z "$DBS" ]; then
|
||
DBS="$(docker run --rm -e PGURL="$PGURL" "$PG_IMAGE" sh -c \
|
||
"psql \"\$PGURL\" -tAc \"select datname from pg_database where datistemplate = false and datallowconn order by datname\"" \
|
||
| tr -d '\r' | paste -sd, -)"
|
||
fi
|
||
if [ -n "$EXCLUDE_DBS" ]; then
|
||
DBS="$(python3 -c "
|
||
import sys
|
||
exclude = set(filter(None, sys.argv[2].split(',')))
|
||
print(','.join(d for d in sys.argv[1].split(',') if d and d not in exclude))
|
||
" "$DBS" "$EXCLUDE_DBS")"
|
||
fi
|
||
if [ -z "$DBS" ]; then echo "没有可备份的库" >&2; exit 1; fi
|
||
|
||
TARGET="$BACKUP_ROOT/$STAMP"
|
||
echo "备份目标: $TARGET"
|
||
echo "库列表 : $DBS"
|
||
echo "镜像 : $PG_IMAGE (并行 $JOBS)"
|
||
if [ "$DRY_RUN" = "1" ]; then echo "DRY_RUN=1，不执行"; exit 0; fi
|
||
|
||
mkdir -p "$TARGET"
|
||
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
START_SECONDS="$(date +%s)"
|
||
|
||
# 全局对象（角色/表空间）：RDS 上通常受限，尽力而为，失败不阻塞。
|
||
docker run --rm -e PGURL="$PGURL" "$PG_IMAGE" sh -c \
|
||
"pg_dumpall --globals-only --no-role-passwords --dbname=\"\$PGURL\"" >"$TARGET/globals.sql" 2>"$TARGET/globals.err" || true
|
||
[ -s "$TARGET/globals.err" ] || rm -f "$TARGET/globals.err"
|
||
|
||
IFS=',' read -r -a DB_ARRAY <<<"$DBS"
|
||
FAILED_DBS=""
|
||
for db in "${DB_ARRAY[@]}"; do
|
||
db="$(printf '%s' "$db" | tr -d ' ')"
|
||
[ -n "$db" ] || continue
|
||
DB_URL="$(python3 - "$DB_USER_VALUE" "$DB_PASS_VALUE" "$DB_HOST_VALUE" "$DB_PORT_VALUE" "$db" <<'PY'
|
||
import sys, urllib.parse as u
|
||
user, password, host, port, name = sys.argv[1:6]
|
||
print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")
|
||
PY
|
||
)"
|
||
echo " - $db ..."
|
||
if docker run --rm -e DB_URL="$DB_URL" -v "$TARGET:/backup" "$PG_IMAGE" sh -c \
|
||
"pg_dump -Fd -j $JOBS -Z 6 --no-sync -d \"\$DB_URL\" -f /backup/${db}.dir"; then
|
||
:
|
||
else
|
||
# 一个库失败（权限/编码/管理库）不能把整次备份作废：清掉半个目录并记账
|
||
echo " 失败，跳过并记账: $db" >&2
|
||
rm -rf "$TARGET/${db}.dir"
|
||
FAILED_DBS="${FAILED_DBS}${db} "
|
||
fi
|
||
done
|
||
|
||
# dump 之后写清单与说明（先写清单，再算哈希，避免把哈希写进自己的输入）
|
||
DURATION="$(( $(date +%s) - START_SECONDS ))"
|
||
python3 - "$TARGET" "$STAMP" "$STARTED_AT" "$DURATION" "$DBS" "$PG_IMAGE" "$JOBS" "$FAILED_DBS" <<'PY'
|
||
import hashlib, json, os, pathlib, sys
|
||
target = pathlib.Path(sys.argv[1]); stamp, started, duration = sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||
databases, image, jobs = sys.argv[5], sys.argv[6], int(sys.argv[7]); failed = sys.argv[8].split()
|
||
files = []
|
||
for path in sorted(target.rglob("*")):
|
||
if path.is_dir() or path.name in ("MANIFEST.json",):
|
||
continue
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||
digest.update(block)
|
||
files.append({"file": str(path.relative_to(target)), "bytes": path.stat().st_size,
|
||
"sha256": digest.hexdigest()})
|
||
manifest = {
|
||
"kind": "rds-logical-backup",
|
||
"created_at": stamp,
|
||
"started_at": started,
|
||
"duration_seconds": duration,
|
||
"databases": [name for name in databases.split(",") if name],
|
||
"failed_databases": failed,
|
||
"format": "pg_dump -Fd (directory, parallel, zlib)",
|
||
"pg_dump_image": image,
|
||
"parallel_jobs": jobs,
|
||
"restore_hint": "pg_restore -d <目标库> -j4 <库名>.dir（先建库；跨实例恢复可能需要 --no-owner/--no-acl）",
|
||
"total_bytes": sum(item["bytes"] for item in files),
|
||
"files": files,
|
||
}
|
||
(target / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
|
||
print(f"duration={duration}s total={manifest['total_bytes']/1048576:.1f}MiB files={len(files)}")
|
||
PY
|
||
|
||
cat >"$TARGET/README.md" <<'MD'
|
||
# 自管逻辑备份（pg_dump 目录格式）
|
||
|
||
- 谁做的：`logical-backup.sh`，手动触发，文件在本机磁盘（不经 RDS 托管备份服务）。
|
||
- 每个库一个 `.dir` 目录（`pg_dump -Fd -jN -Z6`）：**目录格式支持并行恢复与按表选择性恢复**。
|
||
- `MANIFEST.json`：库列表、耗时、每个文件的 sha256/size。
|
||
- `globals.sql`：角色/表空间（RDS 上通常受限，尽力而为，可能为空）。
|
||
|
||
## 怎么恢复
|
||
|
||
```bash
|
||
# 1) 建目标库（已在别处建好则跳过）
|
||
psql "$TARGET_URL" -c 'CREATE DATABASE restore_check'
|
||
|
||
# 2) 并行恢复（-j 至少 4；目录格式才能并行）
|
||
docker run --rm -e TARGET_URL="$TARGET_URL" -v "$PWD:/backup" postgres:17.9-alpine \
|
||
pg_restore -d "$TARGET_URL" -j4 --clean --if-exists /backup/<库名>.dir
|
||
|
||
# 3) 只验某张表（不动生产）
|
||
docker run --rm -v "$PWD:/backup" postgres:17.9-alpine \
|
||
pg_restore -l /backup/<库名>.dir | grep -i wiki_pages
|
||
```
|
||
|
||
注意：
|
||
- **pg_restore 版本必须 ≥ dump 时的 pg_dump**（本次是 17.9）→ 用 `postgres:17.9-alpine`。
|
||
- 跨实例恢复若报角色/权限错误，加 `--no-owner --no-acl`。
|
||
- 这份备份**不能替代按时间点恢复（PITR）**：它只能回到"备份那一刻"。
|
||
MD
|
||
|
||
# 轮转：保留最近 KEEP_DAILY 份，另在最近 KEEP_MONTHS 个自然月各留一份
|
||
python3 - "$BACKUP_ROOT" "$KEEP_DAILY" "$KEEP_MONTHS" <<'PY'
|
||
import pathlib, re, shutil, sys
|
||
root = pathlib.Path(sys.argv[1]); keep_daily = int(sys.argv[2]); keep_months = int(sys.argv[3])
|
||
stamps = sorted([p for p in root.iterdir() if p.is_dir() and re.fullmatch(r"\d{8}T\d{6}Z", p.name)], reverse=True)
|
||
keep = {p.name for p in stamps[:keep_daily]}
|
||
months = []
|
||
for path in stamps:
|
||
month = path.name[:6]
|
||
if month not in months:
|
||
months.append(month)
|
||
keep.add(path.name)
|
||
if len(months) >= keep_months:
|
||
break
|
||
for path in stamps:
|
||
if path.name not in keep:
|
||
shutil.rmtree(path)
|
||
print("rotated out:", path.name)
|
||
print("kept:", len([p for p in stamps if p.name in keep]))
|
||
PY
|
||
|
||
if [ -n "$FAILED_DBS" ]; then
|
||
echo "done（有失败的库）: $TARGET | failed: $FAILED_DBS" >&2
|
||
exit 1
|
||
fi
|
||
echo "done: $TARGET" |