#!/usr/bin/env bash # 目标 WeKnora 知识库的只读行级快照(KB 维度)。 # # 用途:发布 / 撤权 / 回滚前,把目标库**当前状态**留档,出事后有可回退的现场。 # 只做 SELECT + \copy,不写目标库任何一行。 # # 用法: # bash scripts/target_kb_snapshot.sh # 用 production_kb_bindings 里已绑定的目标库 # bash scripts/target_kb_snapshot.sh [] # 指定目标库 # # 环境变量: # SNAPSHOT_ROOT 输出根目录(默认 backups/target-kb-snapshots) # TUNNEL_HOST 目标 PG 隧道地址(默认 172.17.0.1) # TUNNEL_PORT 目标 PG 隧道端口(默认 25432) # INCLUDE_CHUNKS 1=连 chunks 一起导出(较大,默认 1) # WITH_ES 1=同时记录目标 ES 的文档数(默认 1) # NOTE 附加到清单的备注(例如发布单号) # # 覆盖范围(PG 行级):knowledge_bases / knowledges / knowledge_tags / # knowledge_tag_relations / chunks / resources / resource_bindings / # wiki_pages / wiki_folders(含软删行,不含表结构)。 # # 不覆盖:Elasticsearch 文档内容与向量、OSS 对象本体(只在 resources.jsonl 里 # 记路径/大小/哈希做清单)、表结构与索引。整库级恢复请用阿里云 RDS 快照/PITR。 set -euo pipefail WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$WORKDIR" SNAPSHOT_ROOT="${SNAPSHOT_ROOT:-$WORKDIR/backups/target-kb-snapshots}" TUNNEL_HOST="${TUNNEL_HOST:-172.17.0.1}" TUNNEL_PORT="${TUNNEL_PORT:-25432}" INCLUDE_CHUNKS="${INCLUDE_CHUNKS:-1}" WITH_ES="${WITH_ES:-1}" NOTE="${NOTE:-}" KB_IDS=("$@") if [ "${#KB_IDS[@]}" -eq 0 ]; then mapfile -t KB_IDS < <(docker exec knowledge-governance-p1-postgres-1 psql -U governance -d governance -tA -c \ "SELECT unnest(ARRAY[internal_kb_id, external_kb_id]) FROM production_kb_bindings WHERE internal_kb_id <> '' OR external_kb_id <> ''" \ | sed '/^$/d' | sort -u) fi if [ "${#KB_IDS[@]}" -eq 0 ]; then echo "no target knowledge base id given and none bound"; exit 1 fi for id in "${KB_IDS[@]}"; do case "$id" in [0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*) ;; *) echo "invalid knowledge base id: $id"; exit 1 ;; esac done STAMP="$(date -u +%Y%m%dT%H%M%SZ)" PREFIX="$(printf '%s' "${KB_IDS[0]}" | cut -c1-8)" OUT="$SNAPSHOT_ROOT/${STAMP}-${PREFIX}${PREFIX:+_}" OUT="${OUT%_}" mkdir -p "$OUT" # ---------- 连接串(不打印) ---------- SECRET_FILE="secrets/release-sim/target_database_url" [ -f "$SECRET_FILE" ] || { echo "missing $SECRET_FILE"; exit 1; } PGURL="$(python3 - "$SECRET_FILE" "$TUNNEL_HOST" "$TUNNEL_PORT" <<'PY' import re, sys, pathlib url = pathlib.Path(sys.argv[1]).read_text().strip() url = re.sub(r'@[^/@]+:\d+/', '@%s:%s/' % (sys.argv[2], sys.argv[3]), url) print(url) PY )" ENDPOINT="$TUNNEL_HOST:$TUNNEL_PORT ($(python3 - "$SECRET_FILE" <<'PY' import sys, pathlib, re, urllib.parse url = pathlib.Path(sys.argv[1]).read_text().strip() parsed = urllib.parse.urlparse(url) print("host=%s db=%s" % (parsed.hostname or "?", (parsed.path or "/?").lstrip("/") or "?")) PY ))" # ---------- 生成导出脚本(本身也是留档的一部分) ---------- KB_SQL="'$(printf '%s' "${KB_IDS[*]}" | sed "s/ /','/g")'" cat >"$OUT/export.sql" <>"$OUT/export.sql" </dev/null # ---------- ES 文档数(只读,失败不阻塞) ---------- ES_SUMMARY="" if [ "$WITH_ES" = "1" ]; then ES_URL="http://${TUNNEL_HOST}:29200" for id in "${KB_IDS[@]}"; do count="$(curl -s --max-time 15 -H 'Content-Type: application/json' \ -X POST "$ES_URL/weknora_test_chunks/_count" \ -d "{\"query\":{\"term\":{\"knowledge_base_id\":\"$id\"}}}" \ | python3 -c 'import json,sys; print(json.load(sys.stdin).get("count","?"))' 2>/dev/null || echo "?")" ES_SUMMARY="${ES_SUMMARY}${id}=${count} " done fi # ---------- 清单 + 说明 ---------- python3 - "$OUT" "$STAMP" "$ENDPOINT" "$INCLUDE_CHUNKS" "$ES_SUMMARY" "$NOTE" "${KB_IDS[@]}" <<'PY' import hashlib, json, pathlib, sys out = pathlib.Path(sys.argv[1]); stamp = sys.argv[2]; endpoint = sys.argv[3] include_chunks = sys.argv[4] == "1"; es_summary = sys.argv[5].strip(); note = sys.argv[6] kb_ids = sys.argv[7:] files = [] for path in sorted(out.glob("*.jsonl")): digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1 << 20), b""): digest.update(block) rows = sum(1 for _ in path.open("rb")) files.append({"file": path.name, "rows": rows, "bytes": path.stat().st_size, "sha256": digest.hexdigest()}) manifest = { "kind": "weknora-target-kb-snapshot", "created_at": stamp, "target_endpoint": endpoint, "knowledge_base_ids": kb_ids, "profile": "kb-rows" + ("" if include_chunks else " (chunks excluded)"), "covers": ["knowledge_bases", "knowledges", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "resource_bindings", "resources"] + (["chunks"] if include_chunks else []), "does_not_cover": ["Elasticsearch documents and vectors", "OSS object bodies (listed by reference in resources.jsonl)", "table structure / indexes", "rows outside the listed knowledge bases"], "es_document_counts": es_summary or None, "note": note or None, "files": files, } (out / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n") summary = " ".join(f"{item['file']}={item['rows']}" for item in files) print("snapshot:", out) print("files:", summary) print("bytes:", sum(item["bytes"] for item in files)) if es_summary: print("es docs:", es_summary) PY cat >"$OUT/README.md" <<'MD' # 目标知识库快照(只读行级) ## 它是什么 发布 / 撤权 / 回滚**之前**的目标库现场:KB 维度的 PG 行(含软删行),每张表一个 JSONL(`to_jsonb(row)` 一行一条),附 `MANIFEST.json`(行数 / 字节 / sha256 / 覆盖范围) 与 `export.sql`(导出用的原始查询)。生成过程只有 SELECT + `\copy`,不修改目标库。 ## 它不是什么 - 不含 Elasticsearch 文档与向量、不含 OSS 对象本体(`resources.jsonl` 里只有 路径 / 大小 / 哈希清单)、不含表结构与索引。 - 不是整库级灾备。**整库级恢复请用阿里云 RDS 的快照 / 按时间点恢复(PITR)**: 恢复到临时实例后,再按 KB 导出需要的行,这样连索引和约束一起回到某个时间点。 ## 怎么恢复(行级) 先确认目标库当前状态已被另一份快照覆盖(避免二次覆盖),然后按外键顺序灌回: ```bash URL=<目标库连接串> # 例如经隧道:postgres://...@172.17.0.1:25432/ OUT=<本快照目录> docker run --rm --network host -e PGURL="$URL" -v "$OUT:/snap:ro" postgres:16 \ psql "$PGURL" -v ON_ERROR_STOP=1 <<'SQL' BEGIN; -- 每张表:把 JSONL 灌进临时表,再按类型还原成行 DO $$ DECLARE t text; BEGIN FOREACH t IN ARRAY ARRAY[ 'knowledge_bases','knowledges','chunks','knowledge_tags','knowledge_tag_relations', 'resources','resource_bindings','wiki_folders','wiki_pages'] LOOP EXECUTE format('CREATE TEMP TABLE stage_%s (raw jsonb)', t); EXECUTE format($f$\copy stage_%s(raw) FROM '/snap/%s.jsonl'$f$, t, t); EXECUTE format('INSERT INTO %I SELECT (jsonb_populate_record(NULL::%I, raw)).* FROM stage_%s ON CONFLICT (id) DO NOTHING', t, t, t); EXECUTE format('DROP TABLE stage_%s', t); END LOOP; END $$; COMMIT; SQL ``` 注意: - 只恢复了**行**;缺行时不会删、也不会改(`ON CONFLICT DO NOTHING`)。 - 与目标库既有唯一索引冲突的行会被跳过,恢复后请核对 `MANIFEST.json` 的行数。 - 向量在 Elasticsearch 侧,行恢复后必须重建索引 / 重放 ES 文档,否则检索不到。 MD echo "done: $OUT"