diff --git a/docs/P6_RELEASE_WORKER_ACCEPTANCE.md b/docs/P6_RELEASE_WORKER_ACCEPTANCE.md index e363db6..cdbb76f 100644 --- a/docs/P6_RELEASE_WORKER_ACCEPTANCE.md +++ b/docs/P6_RELEASE_WORKER_ACCEPTANCE.md @@ -276,6 +276,33 @@ WeKnora 的 wiki 图谱与页面同表(`wiki_pages`,节点/边靠 slug + `in - 已知取舍:降级路径上共享页的正文仍由目标库的库级合成结果决定,无法在不重算(不额外烧 token)的 前提下与本包一致;这类页只做引用级差集,并在作用域日志里计入 `foreign_ref_stripped`。 +### 发布前备份(2026-10-08 起) + +发布链本身**不做**发布前备份:`DocumentTarget.Prepare` 只归档本包、解析 `previousRevision` +(首次发布为 `absent`,此时回滚只能停用、无法恢复),`PRODUCTION_BACKUP_RESTORE_VERIFIED` +只是"备份/恢复能力已验证"的人工开关。目标库现场必须显式留档: + +```bash +bash scripts/target_kb_snapshot.sh # 用 production_kb_bindings 里绑定的目标库 +bash scripts/target_kb_snapshot.sh [] # 指定目标库 +INCLUDE_CHUNKS=0 bash scripts/target_kb_snapshot.sh # 不导 chunks(快照很小,适合每次发布前) +``` + +输出 `backups/target-kb-snapshots/-/`:KB 维度行级 JSONL(`to_jsonb(row)`, +含软删行)+ `MANIFEST.json`(行数/字节/sha256/覆盖范围/ES 文档数)+ `export.sql`(原始查询) ++ `README.md`(恢复步骤)。全过程只有 SELECT + `\copy`,不修改目标库;`backups/` 已在 .gitignore。 + +覆盖范围与边界: + +- 覆盖:`knowledge_bases`、`knowledges`、`chunks`、`knowledge_tags`、`knowledge_tag_relations`、 + `resources`、`resource_bindings`、`wiki_pages`、`wiki_folders`(PG 行;不含表结构)。 +- 不覆盖:Elasticsearch 文档与向量、OSS 对象本体(`resources.jsonl` 只给路径/大小/哈希清单)、 + 列出的知识库之外的行。整库级恢复用阿里云 RDS 快照/按时间点恢复(PITR),恢复到临时实例后按 KB 取行。 + +约定:对**共享真实知识库**发布、撤权、回滚前必须先跑一次快照,并把快照目录与 +`MANIFEST.json` 的 sha256 记进发布单;`previous_production_revision=absent` 时必须写明 +"无前版可回退,只有这份快照"。待办:把"快照存在"从人工约定做成发布前置硬门禁。 + ## Elasticsearch 通道 源导出: diff --git a/scripts/target_kb_snapshot.sh b/scripts/target_kb_snapshot.sh new file mode 100755 index 0000000..4c24401 --- /dev/null +++ b/scripts/target_kb_snapshot.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# 目标 WeKnora 知识库的只读行级快照(KB 维度)。 +# +# 用途:发布 / 撤权 / 回滚前,把目标库**当前状态**留档,出事后有可回退的现场。 +# 只做 SELECT + \copy,不写目标库任何一行。 +# +# 用法: +# bash scripts/target_kb_snapshot.sh # 用 production_kb_bindings 里已绑定的目标库 +# bash scripts/target_kb_snapshot.sh [] # 指定目标库 +# +# 环境变量: +# SNAPSHOT_ROOT 输出根目录(默认 backups/target-kb-snapshots) +# TUNNEL_HOST 目标 PG 隧道地址(默认 172.17.0.1) +# TUNNEL_PORT 目标 PG 隧道端口(默认 25432) +# INCLUDE_CHUNKS 1=连 chunks 一起导出(较大,默认 1) +# WITH_ES 1=同时记录目标 ES 的文档数(默认 1) +# NOTE 附加到清单的备注(例如发布单号) +# +# 覆盖范围(PG 行级):knowledge_bases / knowledges / knowledge_tags / +# knowledge_tag_relations / chunks / resources / resource_bindings / +# wiki_pages / wiki_folders(含软删行,不含表结构)。 +# +# 不覆盖:Elasticsearch 文档内容与向量、OSS 对象本体(只在 resources.jsonl 里 +# 记路径/大小/哈希做清单)、表结构与索引。整库级恢复请用阿里云 RDS 快照/PITR。 +set -euo pipefail + +WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$WORKDIR" + +SNAPSHOT_ROOT="${SNAPSHOT_ROOT:-$WORKDIR/backups/target-kb-snapshots}" +TUNNEL_HOST="${TUNNEL_HOST:-172.17.0.1}" +TUNNEL_PORT="${TUNNEL_PORT:-25432}" +INCLUDE_CHUNKS="${INCLUDE_CHUNKS:-1}" +WITH_ES="${WITH_ES:-1}" +NOTE="${NOTE:-}" + +KB_IDS=("$@") +if [ "${#KB_IDS[@]}" -eq 0 ]; then + mapfile -t KB_IDS < <(docker exec knowledge-governance-p1-postgres-1 psql -U governance -d governance -tA -c \ + "SELECT unnest(ARRAY[internal_kb_id, external_kb_id]) FROM production_kb_bindings WHERE internal_kb_id <> '' OR external_kb_id <> ''" \ + | sed '/^$/d' | sort -u) +fi +if [ "${#KB_IDS[@]}" -eq 0 ]; then + echo "no target knowledge base id given and none bound"; exit 1 +fi + +for id in "${KB_IDS[@]}"; do + case "$id" in + [0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*) ;; + *) echo "invalid knowledge base id: $id"; exit 1 ;; + esac +done + +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +PREFIX="$(printf '%s' "${KB_IDS[0]}" | cut -c1-8)" +OUT="$SNAPSHOT_ROOT/${STAMP}-${PREFIX}${PREFIX:+_}" +OUT="${OUT%_}" +mkdir -p "$OUT" + +# ---------- 连接串(不打印) ---------- +SECRET_FILE="secrets/release-sim/target_database_url" +[ -f "$SECRET_FILE" ] || { echo "missing $SECRET_FILE"; exit 1; } +PGURL="$(python3 - "$SECRET_FILE" "$TUNNEL_HOST" "$TUNNEL_PORT" <<'PY' +import re, sys, pathlib +url = pathlib.Path(sys.argv[1]).read_text().strip() +url = re.sub(r'@[^/@]+:\d+/', '@%s:%s/' % (sys.argv[2], sys.argv[3]), url) +print(url) +PY +)" +ENDPOINT="$TUNNEL_HOST:$TUNNEL_PORT ($(python3 - "$SECRET_FILE" <<'PY' +import sys, pathlib, re, urllib.parse +url = pathlib.Path(sys.argv[1]).read_text().strip() +parsed = urllib.parse.urlparse(url) +print("host=%s db=%s" % (parsed.hostname or "?", (parsed.path or "/?").lstrip("/") or "?")) +PY +))" + +# ---------- 生成导出脚本(本身也是留档的一部分) ---------- +KB_SQL="'$(printf '%s' "${KB_IDS[*]}" | sed "s/ /','/g")'" + +cat >"$OUT/export.sql" <>"$OUT/export.sql" </dev/null + +# ---------- ES 文档数(只读,失败不阻塞) ---------- +ES_SUMMARY="" +if [ "$WITH_ES" = "1" ]; then + ES_URL="http://${TUNNEL_HOST}:29200" + for id in "${KB_IDS[@]}"; do + count="$(curl -s --max-time 15 -H 'Content-Type: application/json' \ + -X POST "$ES_URL/weknora_test_chunks/_count" \ + -d "{\"query\":{\"term\":{\"knowledge_base_id\":\"$id\"}}}" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("count","?"))' 2>/dev/null || echo "?")" + ES_SUMMARY="${ES_SUMMARY}${id}=${count} " + done +fi + +# ---------- 清单 + 说明 ---------- +python3 - "$OUT" "$STAMP" "$ENDPOINT" "$INCLUDE_CHUNKS" "$ES_SUMMARY" "$NOTE" "${KB_IDS[@]}" <<'PY' +import hashlib, json, pathlib, sys +out = pathlib.Path(sys.argv[1]); stamp = sys.argv[2]; endpoint = sys.argv[3] +include_chunks = sys.argv[4] == "1"; es_summary = sys.argv[5].strip(); note = sys.argv[6] +kb_ids = sys.argv[7:] +files = [] +for path in sorted(out.glob("*.jsonl")): + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + rows = sum(1 for _ in path.open("rb")) + files.append({"file": path.name, "rows": rows, "bytes": path.stat().st_size, + "sha256": digest.hexdigest()}) +manifest = { + "kind": "weknora-target-kb-snapshot", + "created_at": stamp, + "target_endpoint": endpoint, + "knowledge_base_ids": kb_ids, + "profile": "kb-rows" + ("" if include_chunks else " (chunks excluded)"), + "covers": ["knowledge_bases", "knowledges", "knowledge_tags", "knowledge_tag_relations", + "wiki_pages", "wiki_folders", "resource_bindings", "resources"] + + (["chunks"] if include_chunks else []), + "does_not_cover": ["Elasticsearch documents and vectors", "OSS object bodies (listed by reference in resources.jsonl)", + "table structure / indexes", "rows outside the listed knowledge bases"], + "es_document_counts": es_summary or None, + "note": note or None, + "files": files, +} +(out / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n") +summary = " ".join(f"{item['file']}={item['rows']}" for item in files) +print("snapshot:", out) +print("files:", summary) +print("bytes:", sum(item["bytes"] for item in files)) +if es_summary: + print("es docs:", es_summary) +PY + +cat >"$OUT/README.md" <<'MD' +# 目标知识库快照(只读行级) + +## 它是什么 +发布 / 撤权 / 回滚**之前**的目标库现场:KB 维度的 PG 行(含软删行),每张表一个 +JSONL(`to_jsonb(row)` 一行一条),附 `MANIFEST.json`(行数 / 字节 / sha256 / 覆盖范围) +与 `export.sql`(导出用的原始查询)。生成过程只有 SELECT + `\copy`,不修改目标库。 + +## 它不是什么 +- 不含 Elasticsearch 文档与向量、不含 OSS 对象本体(`resources.jsonl` 里只有 + 路径 / 大小 / 哈希清单)、不含表结构与索引。 +- 不是整库级灾备。**整库级恢复请用阿里云 RDS 的快照 / 按时间点恢复(PITR)**: + 恢复到临时实例后,再按 KB 导出需要的行,这样连索引和约束一起回到某个时间点。 + +## 怎么恢复(行级) +先确认目标库当前状态已被另一份快照覆盖(避免二次覆盖),然后按外键顺序灌回: + +```bash +URL=<目标库连接串> # 例如经隧道:postgres://...@172.17.0.1:25432/ +OUT=<本快照目录> +docker run --rm --network host -e PGURL="$URL" -v "$OUT:/snap:ro" postgres:16 \ + psql "$PGURL" -v ON_ERROR_STOP=1 <<'SQL' +BEGIN; +-- 每张表:把 JSONL 灌进临时表,再按类型还原成行 +DO $$ +DECLARE t text; +BEGIN + FOREACH t IN ARRAY ARRAY[ + 'knowledge_bases','knowledges','chunks','knowledge_tags','knowledge_tag_relations', + 'resources','resource_bindings','wiki_folders','wiki_pages'] LOOP + EXECUTE format('CREATE TEMP TABLE stage_%s (raw jsonb)', t); + EXECUTE format($f$\copy stage_%s(raw) FROM '/snap/%s.jsonl'$f$, t, t); + EXECUTE format('INSERT INTO %I SELECT (jsonb_populate_record(NULL::%I, raw)).* FROM stage_%s + ON CONFLICT (id) DO NOTHING', t, t, t); + EXECUTE format('DROP TABLE stage_%s', t); + END LOOP; +END $$; +COMMIT; +SQL +``` + +注意: +- 只恢复了**行**;缺行时不会删、也不会改(`ON CONFLICT DO NOTHING`)。 +- 与目标库既有唯一索引冲突的行会被跳过,恢复后请核对 `MANIFEST.json` 的行数。 +- 向量在 Elasticsearch 侧,行恢复后必须重建索引 / 重放 ES 文档,否则检索不到。 +MD + +echo "done: $OUT" \ No newline at end of file