feat(release): 发布前快照硬门禁 + 云内快照 + ES/OSS 保护
按生产级备份组合落地:
- internal/releasesnapshot:新增门禁包,要求 root 下存在覆盖目标库、未过期
(默认 24h)、含 wiki_pages.jsonl 与 sha256 的快照;本地文件再校验哈希。
7 条单测(过期/换库/篡改/缺页/未接线/半成品)。
- 发布器:写目标库之前(Publish/Delete/Rollback 三条路)先过门禁;映射目标
(源库→目标库,共享真实库)或 policy=always 时必须有快照;门禁未接线
同样拒绝(fail-closed)。新错误码 release_target_snapshot_missing(不可重试)
+ 页面中文提示。6 条发布器单测。
- worker 配置新增 target_snapshot{root,max_age_hours,policy},校验 policy/
max_age/绝对路径;compose.release-sim.yaml 只读挂载快照目录。
- scripts/target_kb_snapshot.sh:默认云内执行(与 DB 同 VPC,凭据取自 WeKnora
容器)+ 默认不导 chunks(44s 出 8 张表);修复 \copy 不能跨行/不能带 WITH、
ES 计数改用 knowledge_base_id.keyword。
- ES:注册 kb_backup 仓库(path.repo),首个快照 36s/4.7GB/0 分片失败,
SLM kb-hourly(3d)+kb-daily(7d) 已建并执行成功。
- OSS:目标桶开启版本控制(此前未开启)。
- 文档 P6 增补硬门禁与云内执行;真实快照目录上门禁实测:两个绑定库通过、
未覆盖库拒绝、未接线拒绝。
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""目标 OSS 桶的保护设置:读取状态,或开启版本控制 / 跨区域复制检查。
|
||||
|
||||
只处理目标环境的桶(secrets/release-sim 下的 key 文件),凭据不打印。
|
||||
|
||||
用法:
|
||||
python3 scripts/oss_bucket_settings.py status # 版本控制 / 复制 / 生命周期
|
||||
python3 scripts/oss_bucket_settings.py enable-versioning # 写操作:开启版本控制
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import pathlib
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from email.utils import formatdate
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets" / "release-sim"
|
||||
BUCKET = "dobot-weknora-test"
|
||||
ENDPOINT = "oss-cn-shenzhen.aliyuncs.com"
|
||||
|
||||
|
||||
def read_secret(name):
|
||||
return (SECRETS / name).read_text().strip()
|
||||
|
||||
|
||||
def request(method, subresource, body=""):
|
||||
access_key = read_secret("target_object_access_key")
|
||||
secret_key = read_secret("target_object_secret_key")
|
||||
date = formatdate(usegmt=True)
|
||||
content_type = "application/xml" if body else ""
|
||||
canonical = f"/{BUCKET}/?{subresource}"
|
||||
string_to_sign = f"{method}\n\n{content_type}\n{date}\n{canonical}"
|
||||
signature = base64.b64encode(
|
||||
hmac.new(secret_key.encode(), string_to_sign.encode(), hashlib.sha1).digest()
|
||||
).decode()
|
||||
headers = {"Date": date, "Authorization": f"OSS {access_key}:{signature}"}
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
http_request = urllib.request.Request(
|
||||
f"https://{BUCKET}.{ENDPOINT}/?{subresource}",
|
||||
data=body.encode() if body else None, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(http_request, timeout=20) as response:
|
||||
return response.status, response.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def versioning_state():
|
||||
status, body = request("GET", "versioning")
|
||||
if status != 200:
|
||||
return f"HTTP {status}"
|
||||
return "Enabled" if "<Status>Enabled</Status>" in body else "Disabled(未开启)"
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1] if len(sys.argv) > 1 else "status"
|
||||
if action == "status":
|
||||
print("versioning:", versioning_state())
|
||||
for name in ("replication", "lifecycle"):
|
||||
status, body = request("GET", name)
|
||||
summary = "none" if status == 404 else " ".join(body.split())[:120]
|
||||
print(f"{name}: HTTP {status} {summary}")
|
||||
return
|
||||
if action == "enable-versioning":
|
||||
print("before:", versioning_state())
|
||||
body = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"
|
||||
status, response = request("PUT", "versioning", body)
|
||||
print("PUT versioning:", f"HTTP {status}", " ".join(response.split())[:160])
|
||||
print("after:", versioning_state())
|
||||
return
|
||||
print("unknown action:", action, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+126
-60
@@ -5,32 +5,47 @@
|
||||
# 只做 SELECT + \copy,不写目标库任何一行。
|
||||
#
|
||||
# 用法:
|
||||
# bash scripts/target_kb_snapshot.sh # 用 production_kb_bindings 里已绑定的目标库
|
||||
# bash scripts/target_kb_snapshot.sh <kb_id> [<kb_id>] # 指定目标库
|
||||
# bash scripts/target_kb_snapshot.sh # 已绑定的目标库;云内执行(快)
|
||||
# EXECUTION=tunnel bash scripts/target_kb_snapshot.sh # 走本机 SSH 隧道(慢,兜底)
|
||||
# bash scripts/target_kb_snapshot.sh <kb_id> [<kb_id>] # 指定目标库
|
||||
# INCLUDE_CHUNKS=1 bash scripts/target_kb_snapshot.sh # 连 chunks 一起导(大,按需)
|
||||
# PULL_CHUNKS=1 bash scripts/target_kb_snapshot.sh # 云内执行时把 chunks 结果也拉回本机
|
||||
#
|
||||
# 环境变量:
|
||||
# EXECUTION cloud(默认,与 DB 同 VPC 的云主机上跑)| tunnel(本机经 SSH 隧道)
|
||||
# SNAPSHOT_ROOT 输出根目录(默认 backups/target-kb-snapshots)
|
||||
# TUNNEL_HOST 目标 PG 隧道地址(默认 172.17.0.1)
|
||||
# TUNNEL_PORT 目标 PG 隧道端口(默认 25432)
|
||||
# INCLUDE_CHUNKS 1=连 chunks 一起导出(较大,默认 1)
|
||||
# INCLUDE_CHUNKS 1=连 chunks 一起导(默认 0:秒级完成;向量在 ES 侧,行可从源端重发)
|
||||
# PULL_CHUNKS 1=云内执行时把 chunks 结果也拉回本机(默认 0,留在云主机)
|
||||
# TUNNEL_HOST/TUNNEL_PORT 隧道模式的目标 PG 地址(默认 172.17.0.1:25432)
|
||||
# SSH_CONFIG SSH 配置路径(默认 ~/.ssh/config.weknora-cloud-test)
|
||||
# SSH_HOST 云主机别名(默认 weknora-test)
|
||||
# APP_CONTAINER 云上提供 DB 凭据的容器(默认 weknora-test-v071-app-1)
|
||||
# REMOTE_ROOT 云上快照目录(默认 /opt/weknora-test/kb-snapshots)
|
||||
# WITH_ES 1=同时记录目标 ES 的文档数(默认 1)
|
||||
# NOTE 附加到清单的备注(例如发布单号)
|
||||
#
|
||||
# 覆盖范围(PG 行级):knowledge_bases / knowledges / knowledge_tags /
|
||||
# knowledge_tag_relations / chunks / resources / resource_bindings /
|
||||
# wiki_pages / wiki_folders(含软删行,不含表结构)。
|
||||
# knowledge_tag_relations / wiki_pages / wiki_folders / resources /
|
||||
# resource_bindings(+ chunks,可选)。含软删行,不含表结构。
|
||||
#
|
||||
# 不覆盖:Elasticsearch 文档内容与向量、OSS 对象本体(只在 resources.jsonl 里
|
||||
# 记路径/大小/哈希做清单)、表结构与索引。整库级恢复请用阿里云 RDS 快照/PITR。
|
||||
# 不覆盖:Elasticsearch 文档与向量(见 ES 仓库 kb_backup 与 SLM 策略 kb-hourly/kb-daily)、
|
||||
# OSS 对象本体(只在 resources.jsonl 里记路径/大小/哈希做清单)、表结构与索引。
|
||||
# 整库级恢复请用阿里云 RDS 快照/PITR。
|
||||
set -euo pipefail
|
||||
|
||||
WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$WORKDIR"
|
||||
|
||||
EXECUTION="${EXECUTION:-cloud}"
|
||||
SNAPSHOT_ROOT="${SNAPSHOT_ROOT:-$WORKDIR/backups/target-kb-snapshots}"
|
||||
TUNNEL_HOST="${TUNNEL_HOST:-172.17.0.1}"
|
||||
TUNNEL_PORT="${TUNNEL_PORT:-25432}"
|
||||
INCLUDE_CHUNKS="${INCLUDE_CHUNKS:-1}"
|
||||
INCLUDE_CHUNKS="${INCLUDE_CHUNKS:-0}"
|
||||
PULL_CHUNKS="${PULL_CHUNKS:-0}"
|
||||
SSH_CONFIG="${SSH_CONFIG:-$HOME/.ssh/config.weknora-cloud-test}"
|
||||
SSH_HOST="${SSH_HOST:-weknora-test}"
|
||||
APP_CONTAINER="${APP_CONTAINER:-weknora-test-v071-app-1}"
|
||||
REMOTE_ROOT="${REMOTE_ROOT:-/opt/weknora-test/kb-snapshots}"
|
||||
WITH_ES="${WITH_ES:-1}"
|
||||
NOTE="${NOTE:-}"
|
||||
|
||||
@@ -53,34 +68,15 @@ done
|
||||
|
||||
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
PREFIX="$(printf '%s' "${KB_IDS[0]}" | cut -c1-8)"
|
||||
OUT="$SNAPSHOT_ROOT/${STAMP}-${PREFIX}${PREFIX:+_}"
|
||||
OUT="${OUT%_}"
|
||||
OUT="$SNAPSHOT_ROOT/${STAMP}-${PREFIX}"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
# ---------- 连接串(不打印) ----------
|
||||
SECRET_FILE="secrets/release-sim/target_database_url"
|
||||
[ -f "$SECRET_FILE" ] || { echo "missing $SECRET_FILE"; exit 1; }
|
||||
PGURL="$(python3 - "$SECRET_FILE" "$TUNNEL_HOST" "$TUNNEL_PORT" <<'PY'
|
||||
import re, sys, pathlib
|
||||
url = pathlib.Path(sys.argv[1]).read_text().strip()
|
||||
url = re.sub(r'@[^/@]+:\d+/', '@%s:%s/' % (sys.argv[2], sys.argv[3]), url)
|
||||
print(url)
|
||||
PY
|
||||
)"
|
||||
ENDPOINT="$TUNNEL_HOST:$TUNNEL_PORT ($(python3 - "$SECRET_FILE" <<'PY'
|
||||
import sys, pathlib, re, urllib.parse
|
||||
url = pathlib.Path(sys.argv[1]).read_text().strip()
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
print("host=%s db=%s" % (parsed.hostname or "?", (parsed.path or "/?").lstrip("/") or "?"))
|
||||
PY
|
||||
))"
|
||||
|
||||
# ---------- 生成导出脚本(本身也是留档的一部分) ----------
|
||||
KB_SQL="'$(printf '%s' "${KB_IDS[*]}" | sed "s/ /','/g")'"
|
||||
|
||||
# ---------- 生成导出脚本(本身也是留档的一部分) ----------
|
||||
cat >"$OUT/export.sql" <<SQL
|
||||
-- 目标知识库只读行级快照:$STAMP
|
||||
-- 生成者:scripts/target_kb_snapshot.sh
|
||||
-- 生成者:scripts/target_kb_snapshot.sh(EXECUTION=$EXECUTION)
|
||||
-- 目标库:${KB_IDS[*]}
|
||||
-- 说明:全部为 SELECT + \copy,不修改目标库。
|
||||
\pset pager off
|
||||
@@ -92,8 +88,10 @@ cat >"$OUT/export.sql" <<SQL
|
||||
\copy (SELECT to_jsonb(r) FROM knowledge_tag_relations r WHERE r.knowledge_id IN (SELECT id FROM knowledges WHERE knowledge_base_id IN ($KB_SQL))) TO 'out/knowledge_tag_relations.jsonl'
|
||||
\copy (SELECT to_jsonb(t) FROM wiki_pages t WHERE t.knowledge_base_id IN ($KB_SQL)) TO 'out/wiki_pages.jsonl'
|
||||
\copy (SELECT to_jsonb(t) FROM wiki_folders t WHERE t.knowledge_base_id IN ($KB_SQL)) TO 'out/wiki_folders.jsonl'
|
||||
\copy (SELECT to_jsonb(b) FROM resource_bindings b WHERE b.owner_id IN (SELECT id FROM knowledges WHERE knowledge_base_id IN ($KB_SQL)) OR b.owner_id IN (SELECT id FROM chunks WHERE knowledge_base_id IN ($KB_SQL))) TO 'out/resource_bindings.jsonl'
|
||||
\copy (SELECT to_jsonb(r) FROM resources r WHERE r.id IN (SELECT resource_id FROM resource_bindings WHERE owner_id IN (SELECT id FROM knowledges WHERE knowledge_base_id IN ($KB_SQL)) OR owner_id IN (SELECT id FROM chunks WHERE knowledge_base_id IN ($KB_SQL)))) TO 'out/resources.jsonl'
|
||||
-- 归属集合用 UNION 子查询(云上实测 0.6~0.8s;OR+IN 会退化成慢查询)。
|
||||
-- 注意:psql 的 \copy 是单行元命令,且不接受 WITH 开头,整条语句必须写成一行。
|
||||
\copy (SELECT to_jsonb(b) FROM resource_bindings b WHERE b.owner_id IN (SELECT id FROM knowledges WHERE knowledge_base_id IN ($KB_SQL) UNION SELECT id FROM chunks WHERE knowledge_base_id IN ($KB_SQL))) TO 'out/resource_bindings.jsonl'
|
||||
\copy (SELECT to_jsonb(r) FROM resources r WHERE r.id IN (SELECT b.resource_id FROM resource_bindings b WHERE b.owner_id IN (SELECT id FROM knowledges WHERE knowledge_base_id IN ($KB_SQL) UNION SELECT id FROM chunks WHERE knowledge_base_id IN ($KB_SQL)))) TO 'out/resources.jsonl'
|
||||
SQL
|
||||
if [ "$INCLUDE_CHUNKS" = "1" ]; then
|
||||
cat >>"$OUT/export.sql" <<SQL
|
||||
@@ -101,61 +99,130 @@ if [ "$INCLUDE_CHUNKS" = "1" ]; then
|
||||
SQL
|
||||
fi
|
||||
|
||||
# ---------- 在容器里执行(只读导出到挂载目录) ----------
|
||||
REMOTE_DIR=""
|
||||
if [ "$EXECUTION" = "cloud" ]; then
|
||||
# ---------- 云内执行:与 DB 同 VPC,内网直连,不受家用带宽限制 ----------
|
||||
REMOTE_DIR="$REMOTE_ROOT/$STAMP-${PREFIX}"
|
||||
ssh -F "$SSH_CONFIG" "$SSH_HOST" "mkdir -p '$REMOTE_DIR'"
|
||||
scp -q -F "$SSH_CONFIG" "$OUT/export.sql" "$SSH_HOST:$REMOTE_DIR/export.sql"
|
||||
ssh -F "$SSH_CONFIG" "$SSH_HOST" \
|
||||
APP_CONTAINER="$APP_CONTAINER" REMOTE_DIR="$REMOTE_DIR" INCLUDE_CHUNKS="$INCLUDE_CHUNKS" \
|
||||
'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
env_value() { docker inspect "$APP_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -E "^$1=" | head -1 | cut -d= -f2-; }
|
||||
DB_HOST_VALUE=$(env_value DB_HOST); DB_PORT_VALUE=$(env_value DB_PORT)
|
||||
DB_NAME_VALUE=$(env_value DB_NAME); DB_USER_VALUE=$(env_value DB_USER)
|
||||
DB_PASS_VALUE=$(env_value DB_PASSWORD)
|
||||
PGURL=$(python3 - "$DB_USER_VALUE" "$DB_PASS_VALUE" "$DB_HOST_VALUE" "$DB_PORT_VALUE" "$DB_NAME_VALUE" <<'PY'
|
||||
import sys, urllib.parse as u
|
||||
user, password, host, port, name = sys.argv[1:6]
|
||||
print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")
|
||||
PY
|
||||
)
|
||||
docker run --rm --network host -e PGURL="$PGURL" \
|
||||
-v "$OUT:/out" -v "$OUT/export.sql:/scripts/export.sql:ro" postgres:16 \
|
||||
-v "$REMOTE_DIR:/out" -v "$REMOTE_DIR/export.sql:/scripts/export.sql:ro" postgres:16-alpine \
|
||||
sh -c 'psql "$PGURL" -q -f /scripts/export.sql' >/dev/null
|
||||
rm -f "$REMOTE_DIR/export.sql"
|
||||
if [ "$INCLUDE_CHUNKS" = "1" ] && [ -f "$REMOTE_DIR/chunks.jsonl" ]; then
|
||||
gzip -f "$REMOTE_DIR/chunks.jsonl"
|
||||
fi
|
||||
python3 - "$REMOTE_DIR" <<'PY'
|
||||
import hashlib, json, pathlib, sys
|
||||
out = pathlib.Path(sys.argv[1]); files = []
|
||||
for path in sorted(out.glob("*.jsonl*")):
|
||||
digest = hashlib.sha256(); size = 0
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(block); size += len(block)
|
||||
files.append({"file": path.name, "bytes": size, "sha256": digest.hexdigest()})
|
||||
(out / "REMOTE_MANIFEST.json").write_text(json.dumps({"host_files": files}, indent=2) + "\n")
|
||||
print("remote files:", " ".join(f"{f['file']}={f['bytes']}" for f in files))
|
||||
PY
|
||||
exit 0
|
||||
REMOTE
|
||||
# 小表拉回本机;chunks 默认留在云主机(PULL_CHUNKS=1 才拉)
|
||||
scp -q -F "$SSH_CONFIG" "$SSH_HOST:$REMOTE_DIR/*.jsonl" "$OUT/" 2>/dev/null || true
|
||||
scp -q -F "$SSH_CONFIG" "$SSH_HOST:$REMOTE_DIR/REMOTE_MANIFEST.json" "$OUT/" 2>/dev/null || true
|
||||
if [ "$PULL_CHUNKS" = "1" ]; then
|
||||
scp -q -F "$SSH_CONFIG" "$SSH_HOST:$REMOTE_DIR/chunks.jsonl.gz" "$OUT/" 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
# ---------- 隧道模式(兜底):本机经 SSH 隧道直连 ----------
|
||||
SECRET_FILE="secrets/release-sim/target_database_url"
|
||||
[ -f "$SECRET_FILE" ] || { echo "missing $SECRET_FILE"; exit 1; }
|
||||
PGURL="$(python3 - "$SECRET_FILE" "$TUNNEL_HOST" "$TUNNEL_PORT" <<'PY'
|
||||
import re, sys, pathlib
|
||||
url = pathlib.Path(sys.argv[1]).read_text().strip()
|
||||
print(re.sub(r'@[^/@]+:\d+/', '@%s:%s/' % (sys.argv[2], sys.argv[3]), url))
|
||||
PY
|
||||
)"
|
||||
docker run --rm --network host -e PGURL="$PGURL" \
|
||||
-v "$OUT:/out" -v "$OUT/export.sql:/scripts/export.sql:ro" postgres:16 \
|
||||
sh -c 'psql "$PGURL" -q -f /scripts/export.sql' >/dev/null
|
||||
fi
|
||||
|
||||
# ---------- ES 文档数(只读,失败不阻塞) ----------
|
||||
ES_SUMMARY=""
|
||||
if [ "$WITH_ES" = "1" ]; then
|
||||
ES_URL="http://${TUNNEL_HOST}:29200"
|
||||
for id in "${KB_IDS[@]}"; do
|
||||
count="$(curl -s --max-time 15 -H 'Content-Type: application/json' \
|
||||
-X POST "$ES_URL/weknora_test_chunks/_count" \
|
||||
-d "{\"query\":{\"term\":{\"knowledge_base_id\":\"$id\"}}}" \
|
||||
-X POST "http://${TUNNEL_HOST}:29200/weknora_test_chunks/_count" \
|
||||
-d "{\"query\":{\"term\":{\"knowledge_base_id.keyword\":\"$id\"}}}" \
|
||||
| python3 -c 'import json,sys; print(json.load(sys.stdin).get("count","?"))' 2>/dev/null || echo "?")"
|
||||
ES_SUMMARY="${ES_SUMMARY}${id}=${count} "
|
||||
done
|
||||
fi
|
||||
|
||||
# ---------- 清单 + 说明 ----------
|
||||
python3 - "$OUT" "$STAMP" "$ENDPOINT" "$INCLUDE_CHUNKS" "$ES_SUMMARY" "$NOTE" "${KB_IDS[@]}" <<'PY'
|
||||
python3 - "$OUT" "$STAMP" "$EXECUTION" "$INCLUDE_CHUNKS" "$ES_SUMMARY" "$NOTE" "$REMOTE_DIR" "${KB_IDS[@]}" <<'PY'
|
||||
import hashlib, json, pathlib, sys
|
||||
out = pathlib.Path(sys.argv[1]); stamp = sys.argv[2]; endpoint = sys.argv[3]
|
||||
out = pathlib.Path(sys.argv[1]); stamp, execution = sys.argv[2], sys.argv[3]
|
||||
include_chunks = sys.argv[4] == "1"; es_summary = sys.argv[5].strip(); note = sys.argv[6]
|
||||
kb_ids = sys.argv[7:]
|
||||
remote_dir = sys.argv[7]; kb_ids = sys.argv[8:]
|
||||
files = []
|
||||
for path in sorted(out.glob("*.jsonl")):
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
rows = sum(1 for _ in path.open("rb"))
|
||||
files.append({"file": path.name, "rows": rows, "bytes": path.stat().st_size,
|
||||
"sha256": digest.hexdigest()})
|
||||
files.append({"file": path.name, "rows": sum(1 for _ in path.open("rb")),
|
||||
"bytes": path.stat().st_size, "sha256": digest.hexdigest(), "location": "local"})
|
||||
for path in sorted(out.glob("chunks.jsonl.gz")):
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
files.append({"file": path.name, "bytes": path.stat().st_size,
|
||||
"sha256": digest.hexdigest(), "location": "local"})
|
||||
remote_manifest = out / "REMOTE_MANIFEST.json"
|
||||
if remote_manifest.exists():
|
||||
for item in json.loads(remote_manifest.read_text()).get("host_files", []):
|
||||
if not any(existing["file"] == item["file"] for existing in files):
|
||||
item["location"] = f"cloud:{remote_dir}"
|
||||
files.append(item)
|
||||
manifest = {
|
||||
"kind": "weknora-target-kb-snapshot",
|
||||
"created_at": stamp,
|
||||
"target_endpoint": endpoint,
|
||||
"execution": execution,
|
||||
"knowledge_base_ids": kb_ids,
|
||||
"profile": "kb-rows" + ("" if include_chunks else " (chunks excluded)"),
|
||||
"profile": "kb-rows" + ("" if include_chunks else " (chunks excluded: 向量在 ES 侧,行可从源端重发)"),
|
||||
"covers": ["knowledge_bases", "knowledges", "knowledge_tags", "knowledge_tag_relations",
|
||||
"wiki_pages", "wiki_folders", "resource_bindings", "resources"]
|
||||
+ (["chunks"] if include_chunks else []),
|
||||
"does_not_cover": ["Elasticsearch documents and vectors", "OSS object bodies (listed by reference in resources.jsonl)",
|
||||
"table structure / indexes", "rows outside the listed knowledge bases"],
|
||||
"does_not_cover": ["Elasticsearch documents and vectors (see ES repository kb_backup, SLM kb-hourly/kb-daily)",
|
||||
"OSS object bodies (listed by reference in resources.jsonl)",
|
||||
"table structure / indexes", "rows outside the listed knowledge bases",
|
||||
"whole-database recovery (use Aliyun RDS snapshot / PITR)"],
|
||||
"es_document_counts": es_summary or None,
|
||||
"remote_directory": remote_dir or None,
|
||||
"note": note or None,
|
||||
"files": files,
|
||||
}
|
||||
(out / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
|
||||
summary = " ".join(f"{item['file']}={item['rows']}" for item in files)
|
||||
print("snapshot:", out)
|
||||
print("files:", summary)
|
||||
print("bytes:", sum(item["bytes"] for item in files))
|
||||
if es_summary:
|
||||
print("es docs:", es_summary)
|
||||
print("files:", " ".join(f"{item['file']}={item.get('rows', '-')}" for item in files))
|
||||
print("local bytes:", sum(item["bytes"] for item in files if item.get("location") == "local"))
|
||||
print("es docs:", es_summary or "-")
|
||||
PY
|
||||
|
||||
cat >"$OUT/README.md" <<'MD'
|
||||
@@ -163,12 +230,12 @@ cat >"$OUT/README.md" <<'MD'
|
||||
|
||||
## 它是什么
|
||||
发布 / 撤权 / 回滚**之前**的目标库现场:KB 维度的 PG 行(含软删行),每张表一个
|
||||
JSONL(`to_jsonb(row)` 一行一条),附 `MANIFEST.json`(行数 / 字节 / sha256 / 覆盖范围)
|
||||
与 `export.sql`(导出用的原始查询)。生成过程只有 SELECT + `\copy`,不修改目标库。
|
||||
JSONL(`to_jsonb(row)` 一行一条),附 `MANIFEST.json`(行数 / 字节 / sha256 / 覆盖范围 /
|
||||
位置)与 `export.sql`(导出用的原始查询)。生成过程只有 SELECT + `\copy`,不修改目标库。
|
||||
|
||||
## 它不是什么
|
||||
- 不含 Elasticsearch 文档与向量、不含 OSS 对象本体(`resources.jsonl` 里只有
|
||||
路径 / 大小 / 哈希清单)、不含表结构与索引。
|
||||
- 不含 Elasticsearch 文档与向量(见 ES 仓库 `kb_backup` 与 SLM 策略 `kb-hourly`/`kb-daily`)、
|
||||
不含 OSS 对象本体(`resources.jsonl` 里只有路径 / 大小 / 哈希清单)、不含表结构与索引。
|
||||
- 不是整库级灾备。**整库级恢复请用阿里云 RDS 的快照 / 按时间点恢复(PITR)**:
|
||||
恢复到临时实例后,再按 KB 导出需要的行,这样连索引和约束一起回到某个时间点。
|
||||
|
||||
@@ -176,12 +243,11 @@ JSONL(`to_jsonb(row)` 一行一条),附 `MANIFEST.json`(行数 / 字节
|
||||
先确认目标库当前状态已被另一份快照覆盖(避免二次覆盖),然后按外键顺序灌回:
|
||||
|
||||
```bash
|
||||
URL=<目标库连接串> # 例如经隧道:postgres://...@172.17.0.1:25432/<db>
|
||||
URL=<目标库连接串>
|
||||
OUT=<本快照目录>
|
||||
docker run --rm --network host -e PGURL="$URL" -v "$OUT:/snap:ro" postgres:16 \
|
||||
psql "$PGURL" -v ON_ERROR_STOP=1 <<'SQL'
|
||||
BEGIN;
|
||||
-- 每张表:把 JSONL 灌进临时表,再按类型还原成行
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
|
||||
Reference in New Issue
Block a user