Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 47428a8b58 | |||
| 91988a0d3b | |||
| 07a0ee2644 | |||
| 0b5de6ef7e | |||
| d7d636792d | |||
| 89c484cdc4 |
@@ -0,0 +1,82 @@
|
|||||||
|
name: deploy-production
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
action:
|
||||||
|
description: deploy 或 writeback
|
||||||
|
required: true
|
||||||
|
default: deploy
|
||||||
|
sha:
|
||||||
|
description: 验收过的完整提交 SHA
|
||||||
|
required: true
|
||||||
|
bundle:
|
||||||
|
description: release.json 的完整 SHA256
|
||||||
|
required: true
|
||||||
|
task:
|
||||||
|
description: 对应的 HiFox 任务 UUID
|
||||||
|
required: true
|
||||||
|
confirm:
|
||||||
|
description: deploy-production 或 retry-writeback
|
||||||
|
required: true
|
||||||
|
approval_event:
|
||||||
|
description: HiFox 自动触发填写的状态事件 UUID
|
||||||
|
required: false
|
||||||
|
acceptance_digest:
|
||||||
|
description: HiFox 自动触发填写的验收记录摘要
|
||||||
|
required: false
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: cloud-test-governance-main-production
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
guard:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 5
|
||||||
|
steps:
|
||||||
|
- name: 核对手动入口及发布输入
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
test "$GITHUB_EVENT_NAME" = workflow_dispatch
|
||||||
|
test "$GITHUB_REPOSITORY" = Weknora/knowledge-governance
|
||||||
|
test "$GITHUB_REF" = refs/heads/main
|
||||||
|
case "$GITHUB_SERVER_URL" in
|
||||||
|
https://gitea.zheromiku.top|http://192.168.50.223:3000) ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
|
git init -q .
|
||||||
|
git fetch -q --depth 200 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" refs/heads/main
|
||||||
|
git checkout -q --detach "$GITHUB_SHA"
|
||||||
|
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||||
|
python3 scripts/production_cd.py guard --event "$GITHUB_EVENT_PATH"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [guard]
|
||||||
|
runs-on: kg-production-cd
|
||||||
|
timeout-minutes: 40
|
||||||
|
steps:
|
||||||
|
- name: 取同一提交的控制端代码
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
test "$GITHUB_EVENT_NAME" = workflow_dispatch
|
||||||
|
test "$GITHUB_REPOSITORY" = Weknora/knowledge-governance
|
||||||
|
test "$GITHUB_REF" = refs/heads/main
|
||||||
|
case "$GITHUB_SERVER_URL" in
|
||||||
|
https://gitea.zheromiku.top|http://192.168.50.223:3000) ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
|
git init -q .
|
||||||
|
git fetch -q --depth 200 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" refs/heads/main
|
||||||
|
git checkout -q --detach "$GITHUB_SHA"
|
||||||
|
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||||
|
- name: 调用云主站受限发布入口并读回回执
|
||||||
|
env:
|
||||||
|
KG_CD_SSH_KEY: ${{ secrets.KG_CD_SSH_KEY }}
|
||||||
|
KG_CD_KNOWN_HOSTS: ${{ secrets.KG_CD_KNOWN_HOSTS }}
|
||||||
|
KG_CD_SSH_HOST: ${{ vars.KG_CD_SSH_HOST }}
|
||||||
|
KG_CD_SSH_PORT: ${{ vars.KG_CD_SSH_PORT }}
|
||||||
|
run: python3 scripts/production_cd.py send --event "$GITHUB_EVENT_PATH"
|
||||||
@@ -27,6 +27,14 @@ BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
|||||||
verify:
|
verify:
|
||||||
$(PY) scripts/ci_checks.py
|
$(PY) scripts/ci_checks.py
|
||||||
|
|
||||||
|
.PHONY: qa qa-doctor qa-self-test
|
||||||
|
qa:
|
||||||
|
$(PY) scripts/qa_env.py run
|
||||||
|
qa-doctor:
|
||||||
|
$(PY) scripts/qa_env.py doctor
|
||||||
|
qa-self-test:
|
||||||
|
$(PY) -m unittest discover -s scripts -p test_qa_env.py -v
|
||||||
|
|
||||||
# 清单:哪些是必需项、在哪个档案(local / ci)下必需
|
# 清单:哪些是必需项、在哪个档案(local / ci)下必需
|
||||||
list:
|
list:
|
||||||
$(PY) scripts/ci_checks.py --list
|
$(PY) scripts/ci_checks.py --list
|
||||||
|
|||||||
@@ -2,6 +2,31 @@
|
|||||||
|
|
||||||
以下命令从仓库根目录运行。日常开发可在 Ubuntu 宿主进行,指定提交的回归在 `hifox-qa` 的独立检出中运行。部署到阿里云测试服后的验收按 [操作 Runbook](OPERATIONS_RUNBOOK.md) 和 [用户流程验收](USER_FLOWS_ACCEPTANCE.md) 执行。
|
以下命令从仓库根目录运行。日常开发可在 Ubuntu 宿主进行,指定提交的回归在 `hifox-qa` 的独立检出中运行。部署到阿里云测试服后的验收按 [操作 Runbook](OPERATIONS_RUNBOOK.md) 和 [用户流程验收](USER_FLOWS_ACCEPTANCE.md) 执行。
|
||||||
|
|
||||||
|
## QA 虚拟机一键回归
|
||||||
|
|
||||||
|
在 `hifox-qa` 检出提测提交后,运行下列入口。将 SHA 占位值换成提测记录里的完整提交 SHA。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make qa-doctor
|
||||||
|
KG_QA_SHA='<完整提交 SHA>' make qa
|
||||||
|
```
|
||||||
|
|
||||||
|
`make qa-doctor` 检查工具链、HiFox 机器在线状态、OMP 模型调用、测试库身份和已安装的迁移。`make qa` 安装锁定依赖,在专用测试库上通过 API 初始化和迁移,随后核对完整迁移序列、API 就绪状态、Web 首页及 Vite 健康代理。通过后执行 Go 和 Vitest 测试,结束时关闭本次启动的服务。每次运行的服务日志保存在 `~/.local/state/knowledge-governance/qa/`。需要提交前完整门禁时执行 `make verify`。
|
||||||
|
|
||||||
|
初次配置由负责人在 QA 虚拟机准备 `~/.config/knowledge-governance/`。以下示例使用虚构的 RDS 主机名。
|
||||||
|
|
||||||
|
| 文件 | 内容 |
|
||||||
|
| --- | --- |
|
||||||
|
| `qa.json` | 当前用户所有、权限 `0600`,字段 `host`、`port`、`database`、`user` |
|
||||||
|
| `qa.pgpass` | 当前用户所有、权限 `0600`,填写与 `qa.json` 匹配的口令条目 |
|
||||||
|
| `rds-ca.pem` | 对应实例的 CA 证书,用于校验 RDS 主机名 |
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"host":"rds.example.test","port":5432,"database":"kg_qa_app","user":"kg_qa_app"}
|
||||||
|
```
|
||||||
|
|
||||||
|
专用非生产 RDS 库和角色以 `kg_qa_` 开头,权限和网络准备见第 2 节。首次运行接受空库,后续运行接受与当前提交兼容的迁移序列。缺库、认证失败、迁移不兼容和端口冲突均会使命令非零退出。浏览器业务 E2E 和专用集成库的 Go 测试按第 4 节执行。
|
||||||
|
|
||||||
## 1. 检出代码与安装依赖
|
## 1. 检出代码与安装依赖
|
||||||
|
|
||||||
准备 Git、Make、Python 3、Go 1.26(含同一工具链的 `gofmt`)、Node.js 22 和 npm。数据库检查另需 PostgreSQL 客户端 `psql`。首次安装依赖需要能访问 Go 和 npm 的依赖源。
|
准备 Git、Make、Python 3、Go 1.26(含同一工具链的 `gofmt`)、Node.js 22 和 npm。数据库检查另需 PostgreSQL 客户端 `psql`。首次安装依赖需要能访问 Go 和 npm 的依赖源。
|
||||||
|
|||||||
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
管理员配置连接和发布授权,文档工程师维护资料,只读成员查看进度。先确定要操作的环境,再按下面的接线与验收步骤进入业务流程。
|
管理员配置连接和发布授权,文档工程师维护资料,只读成员查看进度。先确定要操作的环境,再按下面的接线与验收步骤进入业务流程。
|
||||||
|
|
||||||
|
云测试治理主站自身的应用升级见[主站应用 CD](PRODUCTION_CD_RUNBOOK.md)。
|
||||||
|
|
||||||
## 入口与反向代理
|
## 入口与反向代理
|
||||||
|
|
||||||
| 请求从哪里发出 | 地址或路径 | 实际目的地 |
|
| 请求从哪里发出 | 地址或路径 | 实际目的地 |
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# 云测试治理主站应用 CD
|
||||||
|
|
||||||
|
正式入口为 `http://100.85.111.121:8088/`。云端 Compose 项目为 `knowledge-governance-cloud-test`,工作目录 `/opt/knowledge-governance`,当前配置链为 `compose.json`,服务为 `api`、`web`、`worker`。Web 容器提供 `8088`,通过挂载的 nginx 配置代理 API 的 `127.0.0.1:18098`。应用 CD 与知识库内容发布各自保留原有授权流程。
|
||||||
|
|
||||||
|
## 建立验收产物
|
||||||
|
|
||||||
|
1. 从干净检出的完整提交构建一次 API、worker、web 镜像。构建调用方对三个镜像都注入 `org.opencontainers.image.revision=<完整 SHA>`。API 构建身份沿用 `scripts/version.py --identity` 输出,传给 Dockerfile 的版本、提交和构建时间参数。API 与 worker 各自记录镜像 ID,即使它们来自同一个 Dockerfile。
|
||||||
|
2. 封存后用同一份归档部署到隔离验收环境。输出目录必须是新路径,父目录应已存在。三个 `--api/--web/--worker` 参数是本地镜像的完整 ID。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/production_cd.py pack --sha "$SHA" --task "$TASK_UUID" \
|
||||||
|
--api "$API_ID" --web "$WEB_ID" --worker "$WORKER_ID" --output "$BUNDLE_DIR"
|
||||||
|
docker load -i "$BUNDLE_DIR/api.tar"
|
||||||
|
docker load -i "$BUNDLE_DIR/web.tar"
|
||||||
|
docker load -i "$BUNDLE_DIR/worker.tar"
|
||||||
|
```
|
||||||
|
|
||||||
|
3. 用验收环境自身的 Compose 和数据库启动这些镜像。用下面的命令读回验收环境运行身份。此处三个服务参数是实际的**测试容器名**。在业务验收入口执行实际用例并将输出、结果与报告链接放在同一份验收记录里。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/production_cd.py attest --sha "$SHA" --task "$TASK_UUID" \
|
||||||
|
--api "$QA_API" --web "$QA_WEB" --worker "$QA_WORKER" --test-url "$QA_URL" --directory "$BUNDLE_DIR"
|
||||||
|
BUNDLE_SHA=$(sha256sum "$BUNDLE_DIR/release.json" | cut -d' ' -f1)
|
||||||
|
```
|
||||||
|
|
||||||
|
4. 由验收人使用获授权的 HiFox CLI 登记业务结论,读回任务、验收字段和状态。`cd_acceptance` 是 JSON 字符串类型的 metadata,其解析结果为 `{"verdict":"passed","sha":"…","bundle":"…","evidence":"https://…"}`。录入人应先核对真实报告与运行身份。自动发布时,`evidence` 使用同一 HiFox 任务的验收报告评论链接(带 `#comment-<UUID>`)。评论须正文包含完整提交和归档清单摘要,并在发布人确认前完成。用 `task view --output json` 核对实际任务 UUID、空间、状态和验收字段。验收登记重试复用原归档摘要与原记录。以当前版本只接受一个任务,多个任务分别验收。
|
||||||
|
5. 在首次发布前,由运维把四个文件放入云主站的 `/opt/knowledge-governance/cd/bundles/<release.json SHA256>/`。通过安全文件通道传到临时目录,核对摘要与内容后用同一文件系统上的目录重命名发布;已有目录先核对内容,匹配时复用。此目录仅供目标控制器读取,文件由该账号持有且组与其他用户无写权限。
|
||||||
|
|
||||||
|
## 在 HiFox 确认发布
|
||||||
|
|
||||||
|
验收、登记、上传归档都完成后,由发布人把原任务移入「待发布」。HiFox 的「知识治理|待发布自动执行 CD」把任务 UUID 交给专用发布 Agent,Agent 调用白名单内的 `scripts/hifox_release_trigger.py`。桥接脚本读取 HiFox 时间线,校验最近一次状态变更的操作者、状态、同任务验收报告及产物摘要,随后调度 Gitea `deploy-production@main`。Gitea 仍只开放 `workflow_dispatch`。目标控制器按完整输入校验 Gitea 运行,并在切换前复查 HiFox 状态和验收摘要。
|
||||||
|
|
||||||
|
相同状态事件复用已记录的 Gitea 运行。如果调度结果未知,核查对应 Gitea 运行与本机调度记录后再处理。旧任务已处于「待发布」时,在验收登记完成后重新进入该状态可发起新确认。
|
||||||
|
|
||||||
|
也可由发布人使用 Gitea Actions 中原有人工入口应急。分支选 `main`,使用 `action=deploy`、完整 `sha`、`bundle`、任务 UUID `task` 和 `confirm=deploy-production`,仅 HiFox Agent 填写 `approval_event` 和 `acceptance_digest`。
|
||||||
|
|
||||||
|
主站侧从 Gitea 读取运行、操作者和通过的校验作业,并验证输入摘要及工作流代码。候选代码可来自仓库内的任务分支,以实际验收 SHA 为准。切换前还核验必需 CI 状态、HiFox 验收记录、三份归档、线上源代码血缘及数据库迁移账本。切换复用 `scripts/deploy-api-web.sh`,仅更换三项 Compose `image`;保留原配置字节以备回滚。切换后自动读回三镜像、API 二进制 SHA、主站健康、API readiness 及实际提供的前端资源。HiFox 回写包含原始运行 URL、完整提交、摘要、镜像 ID 及核验证据。任务评论另含主站登录地址和已核验的运行版本,带 OCI 版本标签时显示其值,其余镜像显示完整提交。
|
||||||
|
|
||||||
|
目标机已有的 `schema_migrations` 必须与候选完全相同,API 与 worker 的库连接均须核对。应用会在启动时检查迁移。新增或改变的迁移先按独立流程审查、备份、执行并复验;CD 自身只读迁移账本。发布时运营方应保留 prior tag,保障用原 Compose 字节回滚。
|
||||||
|
|
||||||
|
## 失败恢复
|
||||||
|
|
||||||
|
- `rolled_back` 表示自动恢复了原 Compose、旧的镜像 ID、API SHA 与站点健康。`rollback_failed` 或 `cutover_started` 要求巡检现场、按回执恢复并复验。目标机互斥锁覆盖切换及回写。Gitea 的取消记录只能说明流水线结束,回滚以目标机回执和运行时读回为准。
|
||||||
|
- `DEPLOYED_HIFOX_PENDING` 表示镜像切换和核验已经完成。以同一 `sha`、`bundle`、`task` 人工发起 `action=writeback`、`confirm=retry-writeback`。脚本先读原目标回执并复验仍在运行的版本,随后补写 HiFox,保留原部署链接。状态与 metadata 的当前值以 HiFox 读回为准。重复调用核对记录后结束。
|
||||||
|
- 目标回执 `/opt/knowledge-governance/cd/receipts/<bundle>.json`,镜像切换前的 Compose 保存为同名 `.compose.backup`。SSH 超时或强制中断后,先读回执和当前运行态。已存在未决或失败回执时,重复部署被拒。保留原回执作为事故证据。
|
||||||
|
|
||||||
|
## 一次性配置与权限
|
||||||
|
|
||||||
|
- 目标机安装仓库 `scripts/{production_cd.py,cd_migration_probe.py,deploy-api-web.sh,production-cd-ssh.sh}` 到 `/opt/knowledge-governance/cd/bin/`。创建私有 `receipts/`、`bundles/`、`hifox-home/`,初始化 `source.git` 为 bare Git 仓,并将 `.gitea/required-contexts.txt` 安装到该目录。仅发布运维可更新这些文件。
|
||||||
|
- 填写目标机 `/opt/knowledge-governance/cd/deploy-production.yml`,内容与 Gitea 主分支的工作流逐字节相同。控制器用 Gitea 发布人账号 ID 和只读日志验证确认。目标端 `authorized_keys` 为 CI 单独公钥设置 `restrict,command="/opt/knowledge-governance/cd/bin/production-cd-ssh.sh"`。校验云主机 SSH host key 后将其按 `kg-governance-cd` 别名写入 `KG_CD_KNOWN_HOSTS`。对应独立 SSH 私钥保存在 `KG_CD_SSH_KEY`。Runner 标签为 `kg-production-cd`,使用 `KG_CD_SSH_HOST` 和 `KG_CD_SSH_PORT` 接入批准的 Tailscale 或本机 IPv6 隧道。
|
||||||
|
- 将受控 HiFox CLI 安装至 `bin/hifox`,由账号持有人在目标主机使用 `hifox-home` 对应的私有配置目录完成登录。身份须能读取指定空间的任务、写 `cd_release` metadata 并完成对应状态迁移。状态绑定前分别读回组织状态和实际任务类型,复用原有待发布、已发布 ID,保留其余工作流配置。
|
||||||
|
- 在专用发布 Agent 的 Ubuntu 开发机安装 `scripts/{production_cd.py,hifox_release_trigger.py}` 到 `~/.local/lib/kg-hifox-release/`。该机使用受控 HiFox 登录和已配置的 Gitea Git 凭据助手,凭据不传入自动化指令。使用私有目录 `~/.local/state/kg-hifox-release/` 存放 `active-since`(UTC,`YYYY-MM-DDTHH:MM:SSZ`)及调度记录。首次接通时,将 `active-since` 设为通道启用时刻,历史状态事件不追发。发布期间需要 Agent 执行机和 HiFox Daemon 在线。
|
||||||
|
- 常规 push CI 负责验证和静态预览。这个 workflow 的唯一事件是 `workflow_dispatch`。`make verify` 的 `contracts` 子项运行 CD 用例。真实切换的验收要求对应候选、现场前置条件和发布人确认。
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
> 这份文档是**破坏性变化与数据迁移步骤的落脚点**:`CHANGELOG.md` 里以 `**破坏性**` 开头的条目
|
> 这份文档是**破坏性变化与数据迁移步骤的落脚点**:`CHANGELOG.md` 里以 `**破坏性**` 开头的条目
|
||||||
> **必须**链接到本文(或本文的某个小节)。发布记录本身仍在 `CHANGELOG.md` —— 本文不维护第二份变更列表。
|
> **必须**链接到本文(或本文的某个小节)。发布记录本身仍在 `CHANGELOG.md` —— 本文不维护第二份变更列表。
|
||||||
>
|
>
|
||||||
> 本仓 CI **只做 CI**:不打 tag、不发 Release、不部署。发布与部署都是**独立的人工审批动作**。
|
> 常规 CI 负责检查、构建与静态预览。版本 tag 与 Release 由人批准后执行。主站应用部署使用独立的[人工确认 CD](PRODUCTION_CD_RUNBOOK.md)。
|
||||||
|
|
||||||
## 一、正式发布(由人批准后执行)
|
## 一、正式发布(由人批准后执行)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""Run inside a governance container. Read the real application DB ledger only."""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from urllib.parse import parse_qsl, unquote, urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
def connection_env(env):
|
||||||
|
# Keep only the connection properties actually shared by libpq and pgx.
|
||||||
|
# Unknown DSN options can change the database identity or schema search path.
|
||||||
|
u = urlsplit(env['DATABASE' + '_URL'])
|
||||||
|
if u.scheme not in ('postgres', 'postgresql') or not u.hostname or not u.username or not u.path[1:] or u.fragment:
|
||||||
|
raise ValueError('invalid database URL')
|
||||||
|
query = parse_qsl(u.query, strict_parsing=True, keep_blank_values=True)
|
||||||
|
supported = {'sslmode': 'PGSSLMODE', 'sslrootcert': 'PGSSLROOTCERT', 'sslcert': 'PGSSLCERT', 'sslkey': 'PGSSLKEY'}
|
||||||
|
if len(dict(query)) != len(query) or any(key not in supported or not val for key, val in query):
|
||||||
|
raise ValueError('unknown database connection options')
|
||||||
|
safe = {'PATH': '/usr/bin:/bin', 'HOME': env.get('HOME', '/nonexistent'), 'LC_ALL': 'C',
|
||||||
|
'PGHOST': u.hostname, 'PGPORT': str(u.port or 5432), 'PGDATABASE': unquote(u.path[1:]),
|
||||||
|
'PGUSER': unquote(u.username), 'PGCONNECT_TIMEOUT': '5',
|
||||||
|
'PGOPTIONS': '-c default_transaction_read_only=on -c statement_timeout=5000'}
|
||||||
|
for name in ('PGPASSFILE', 'PGSSLMODE', 'PGSSLROOTCERT', 'PGSSLCERT', 'PGSSLKEY'):
|
||||||
|
if env.get(name):
|
||||||
|
safe[name] = env[name]
|
||||||
|
for name in env:
|
||||||
|
if name.startswith('PG') and name not in safe:
|
||||||
|
raise ValueError('unknown inherited database connection options')
|
||||||
|
safe.update({supported[key]: value for key, value in query})
|
||||||
|
if u.password:
|
||||||
|
safe['PGPASSWORD'] = unquote(u.password)
|
||||||
|
for name in ('PGDATABASE', 'PGUSER'):
|
||||||
|
if any(char in safe[name] for char in '\r\n\x00='):
|
||||||
|
raise ValueError('unsafe database identifier')
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
env = connection_env(os.environ)
|
||||||
|
sql = """SELECT json_build_object('database', current_database(), 'user', current_user,
|
||||||
|
'read_only', current_setting('transaction_read_only'),
|
||||||
|
'versions', (SELECT json_agg(version ORDER BY version) FROM schema_migrations));"""
|
||||||
|
p = subprocess.run(['psql', '-XAtw', '-v', 'ON_ERROR_STOP=1', '-c', sql],
|
||||||
|
env=env, capture_output=True, timeout=12)
|
||||||
|
if p.returncode:
|
||||||
|
raise ValueError('query failed')
|
||||||
|
data = json.loads(p.stdout)
|
||||||
|
if data['read_only'] != 'on' or data['database'] != env['PGDATABASE'] or data['user'] != env['PGUSER']:
|
||||||
|
raise ValueError('database identity mismatch')
|
||||||
|
print(json.dumps(data['versions']))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except (OSError, ValueError, KeyError, subprocess.TimeoutExpired):
|
||||||
|
raise SystemExit('read-only migration probe failed') from None
|
||||||
@@ -356,6 +356,14 @@ def cmd_internal_web_test() -> int:
|
|||||||
def cmd_internal_contracts(go: str) -> int:
|
def cmd_internal_contracts(go: str) -> int:
|
||||||
"""版本契约:结构 + 注入身份必须**真 PASS**(SKIP 判红),
|
"""版本契约:结构 + 注入身份必须**真 PASS**(SKIP 判红),
|
||||||
并且**开发构建不许自称正式版**(负例:标成 release ⇒ 必须红)。"""
|
并且**开发构建不许自称正式版**(负例:标成 release ⇒ 必须红)。"""
|
||||||
|
qa_rc = subprocess.run(py("-m", "unittest", "discover", "-s", "scripts", "-p", "test_qa_env.py", "-v"),
|
||||||
|
cwd=ROOT).returncode
|
||||||
|
if qa_rc:
|
||||||
|
return qa_rc
|
||||||
|
cd_rc = subprocess.run(py("-m", "unittest", "discover", "-s", "scripts", "-p", "test_production_cd.py", "-v"),
|
||||||
|
cwd=ROOT).returncode
|
||||||
|
if cd_rc:
|
||||||
|
return cd_rc
|
||||||
snapshot_rc = subprocess.run(py("scripts/development_snapshot.py","--self-test"),cwd=ROOT).returncode
|
snapshot_rc = subprocess.run(py("scripts/development_snapshot.py","--self-test"),cwd=ROOT).returncode
|
||||||
if snapshot_rc:
|
if snapshot_rc:
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
@@ -1,6 +1,20 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# 重建并上线 governance api + web(本机 docker build 需要宿主代理 + --network host)
|
# 重建并上线 governance api + web(本机 docker build 需要宿主代理 + --network host)
|
||||||
set -u
|
set -u
|
||||||
|
|
||||||
|
# Share the image-only Compose switch with CD without entering the legacy build path.
|
||||||
|
compose_up() {
|
||||||
|
"${compose[@]}" up -d --no-build --no-deps "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${1:-}" == --verified-compose ]]; then
|
||||||
|
[[ "$#" == 7 && "$5" == api && "$6" == web && "$7" == worker && "$4" == /* ]] || exit 2
|
||||||
|
cd -- "$2" || exit 1
|
||||||
|
compose=(docker compose -p "$3" -f "$4")
|
||||||
|
compose_up --pull never "$5" "$6" "$7"
|
||||||
|
exit $?
|
||||||
|
fi
|
||||||
|
|
||||||
cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1
|
cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1
|
||||||
[ -z "$(git status --porcelain --untracked-files=normal)" ] || { printf "%s\n" "REFUSED_DIRTY_SOURCE"; exit 1; }
|
[ -z "$(git status --porcelain --untracked-files=normal)" ] || { printf "%s\n" "REFUSED_DIRTY_SOURCE"; exit 1; }
|
||||||
TS=$(date +%Y%m%d-%H%M%S)
|
TS=$(date +%Y%m%d-%H%M%S)
|
||||||
@@ -103,7 +117,9 @@ if [ "${GATES_PREV}" = "true" ] && [[ "${COMPOSE_FILES_EFFECTIVE}" != *compose.r
|
|||||||
echo " 确实要关闸:ALLOW_GATE_OFF=1 bash scripts/deploy-api-web.sh"
|
echo " 确实要关闸:ALLOW_GATE_OFF=1 bash scripts/deploy-api-web.sh"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
docker compose ${COMPOSE_FILES_EFFECTIVE} up -d --no-build --no-deps api web || { echo "UP_FAILED"; exit 1; }
|
read -r -a compose_file_args <<< "$COMPOSE_FILES_EFFECTIVE"
|
||||||
|
compose=(docker compose "${compose_file_args[@]}")
|
||||||
|
compose_up api web || { echo "UP_FAILED"; exit 1; }
|
||||||
|
|
||||||
echo "=== 5. 状态"
|
echo "=== 5. 状态"
|
||||||
docker ps --filter name=knowledge-governance-p1 --format '{{.Names}} {{.Status}} {{.Image}}'
|
docker ps --filter name=knowledge-governance-p1 --format '{{.Names}} {{.Status}} {{.Image}}'
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Submit a HiFox-approved release from the designated CD Agent host."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.request import ProxyHandler, Request, build_opener
|
||||||
|
|
||||||
|
import production_cd as cd
|
||||||
|
|
||||||
|
|
||||||
|
DISPATCH = f'/api/v1/repos/{cd.REPO}/actions/workflows/deploy-production.yml/dispatches?return_run_details=true'
|
||||||
|
PATH = Path.home() / '.local/state/kg-hifox-release'
|
||||||
|
RUN_ID = re.compile('[1-9][0-9]{0,14}')
|
||||||
|
|
||||||
|
|
||||||
|
def forge(method, path, payload=None):
|
||||||
|
request = Request(cd.FORGE + path, method=method,
|
||||||
|
data=json.dumps(payload).encode() if payload is not None else None)
|
||||||
|
env = {**os.environ, 'GIT_TERMINAL_PROMPT': '0', 'GIT_ASKPASS': '/bin/false'}
|
||||||
|
fields = dict(line.split('=', 1) for line in cd.run(
|
||||||
|
['/usr/bin/git', 'credential', 'fill'], 'read Gitea credential', cwd='/', env=env,
|
||||||
|
input_data=f'url={cd.FORGE}/{cd.REPO}.git\n\n'.encode()).decode().splitlines() if '=' in line)
|
||||||
|
cd.require(fields.get('username') and fields.get('password'), 'Gitea credential unavailable')
|
||||||
|
auth = base64.b64encode(f'{fields["username"]}:{fields["password"]}'.encode()).decode()
|
||||||
|
request.add_header('Authorization', 'Basic ' + auth)
|
||||||
|
request.add_header('Content-Type', 'application/json')
|
||||||
|
with build_opener(ProxyHandler({}), cd.NoRedirect()).open(request, timeout=20) as response:
|
||||||
|
return json.load(response)
|
||||||
|
|
||||||
|
|
||||||
|
def hifox(cli, *args):
|
||||||
|
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(Path.home()), 'CI': '1'}
|
||||||
|
raw = cd.run([cli, '--organization-id', cd.ORG, *args, '--output', 'json'], 'read HiFox release record', env=env)
|
||||||
|
result = json.JSONDecoder().raw_decode(raw.decode().strip())[0]
|
||||||
|
cd.require(isinstance(result, (dict, list)) and (not isinstance(result, dict)
|
||||||
|
or (result.get('success') is not False and not result.get('error'))), 'HiFox record unavailable')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def queue(task_id, cli, base=PATH):
|
||||||
|
cd.require(cd.matches(cd.UUID, task_id), 'use the triggering task UUID')
|
||||||
|
task = hifox(cli, 'task', 'view', task_id)
|
||||||
|
if not isinstance(task, dict) or task.get('id') != task_id or task.get('spaceId') != cd.SPACE:
|
||||||
|
raise cd.CDError('task is outside the release space')
|
||||||
|
acceptance = cd.metadata(task, 'cd_acceptance') or {}
|
||||||
|
cd.require(isinstance(acceptance, dict) and cd.matches(cd.SHA, acceptance.get('sha'))
|
||||||
|
and cd.matches(cd.HASH, acceptance.get('bundle')), 'skip: no registered candidate acceptance')
|
||||||
|
inputs = dict(action='deploy', confirm='deploy-production', sha=acceptance['sha'], bundle=acceptance['bundle'], task=task_id)
|
||||||
|
timeline = hifox(cli, 'api', 'get', f'/rest/v1/tasks/{task_id}/timeline')
|
||||||
|
if not isinstance(timeline, list):
|
||||||
|
raise cd.CDError('HiFox timeline unavailable')
|
||||||
|
binding, approved_at = cd.release_approval(task, timeline, inputs)
|
||||||
|
inputs.update(binding)
|
||||||
|
since = cd.immutable_file(base / 'active-since').read_text().strip()
|
||||||
|
cd.require(re.fullmatch(r'\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ', since) and approved_at > since,
|
||||||
|
'status approval predates activation; enter 待发布 again after acceptance')
|
||||||
|
|
||||||
|
claim = base / (binding['approval_event'] + '.json')
|
||||||
|
with cd.deploy_lock(base / 'dispatch.lock'):
|
||||||
|
if claim.exists():
|
||||||
|
result = cd.read_json(cd.immutable_file(claim))
|
||||||
|
cd.require(result.get('inputs') == inputs, 'HiFox approval already used for a different release')
|
||||||
|
cd.require(result.get('state') == 'queued', 'dispatch outcome unknown; reconcile Gitea before retry')
|
||||||
|
return result
|
||||||
|
result = {'state': 'submitting', 'inputs': inputs, 'started_at': cd.stamp()}
|
||||||
|
cd.record(claim, result)
|
||||||
|
response = forge('POST', DISPATCH, {'ref': 'main', 'inputs': inputs})
|
||||||
|
run_id = str(response.get('workflow_run_id', ''))
|
||||||
|
cd.require(cd.matches(RUN_ID, run_id), 'dispatch response has no valid run ID; reconcile before retry')
|
||||||
|
result.update(state='queued', run_id=run_id, run_url=f'{cd.FORGE}/{cd.REPO}/actions/runs/{run_id}')
|
||||||
|
cd.record(claim, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('task', help='UUID of the task that entered 待发布')
|
||||||
|
args = parser.parse_args()
|
||||||
|
executable = Path.home() / '.local/bin/hifox'
|
||||||
|
try:
|
||||||
|
print(json.dumps(queue(args.task, str(executable)), ensure_ascii=False))
|
||||||
|
except HTTPError as exc:
|
||||||
|
sys.exit(f'CD: Gitea dispatch HTTP {exc.code}; reconcile before retry')
|
||||||
|
except cd.CDError as exc:
|
||||||
|
sys.exit(f'CD: {exc}')
|
||||||
|
except (OSError, KeyError, ValueError, TypeError) as exc:
|
||||||
|
sys.exit(f'CD: {type(exc).__name__}; inspect the dispatch record')
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
umask 077
|
||||||
|
[ "${SSH_ORIGINAL_COMMAND:-}" = kg-cd-v1 ] || exit 1
|
||||||
|
exec /usr/bin/env -i HOME=/opt/knowledge-governance/cd PATH=/usr/bin:/bin LANG=C.UTF-8 \
|
||||||
|
GIT_TERMINAL_PROMPT=0 /usr/bin/python3 -I /opt/knowledge-governance/cd/bin/production_cd.py serve
|
||||||
@@ -0,0 +1,849 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Governance application CD. The installed host copy owns the release boundary."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import copy
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from urllib.error import URLError
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
from urllib.request import HTTPRedirectHandler, ProxyHandler, build_opener
|
||||||
|
|
||||||
|
REPO = 'Weknora/knowledge-governance'
|
||||||
|
FORGE = 'https://gitea.zheromiku.top'
|
||||||
|
TARGET = 'cloud-test-governance-main'
|
||||||
|
ORG = '169e6f7f-e508-4a1f-9a9b-63d240439a3d'
|
||||||
|
SPACE = '106e05d1-4091-40b9-8c57-f76dc94f7825'
|
||||||
|
READY = '3e15556f-ca19-417c-8bfd-666634ea5d29'
|
||||||
|
PUBLISHED = '16185166-b8e9-4761-9926-845a1f41c96d'
|
||||||
|
RELEASE_OWNER = '39ae59fb-876c-4587-ab5d-a12a841c6c09'
|
||||||
|
LOGIN_URL = 'http://100.85.111.121:8088/'
|
||||||
|
ROLES = ('api', 'web', 'worker')
|
||||||
|
REVISION = 'org.opencontainers.image.revision'
|
||||||
|
DATA = Path('/opt/knowledge-governance/cd')
|
||||||
|
APP = DATA.parent
|
||||||
|
PROJECT = 'knowledge-governance-cloud-test'
|
||||||
|
SHA = re.compile('[0-9a-f]{40}')
|
||||||
|
HASH = re.compile('[0-9a-f]{64}')
|
||||||
|
IMAGE = re.compile('sha256:[0-9a-f]{64}')
|
||||||
|
UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}')
|
||||||
|
GUARD_PREFIX = 'KG_CD_CONFIRM_V1 '
|
||||||
|
|
||||||
|
|
||||||
|
class CDError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def require(ok, message):
|
||||||
|
if not ok:
|
||||||
|
raise CDError(message)
|
||||||
|
|
||||||
|
|
||||||
|
def stamp():
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def read_json(path):
|
||||||
|
return json.loads(path.read_bytes())
|
||||||
|
|
||||||
|
|
||||||
|
def sync_dir(directory):
|
||||||
|
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
|
||||||
|
try:
|
||||||
|
os.fsync(fd)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_write(path, data, mode=0o600):
|
||||||
|
fd, temp = tempfile.mkstemp(dir=path.parent)
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, 'wb') as out:
|
||||||
|
os.fchmod(out.fileno(), mode)
|
||||||
|
out.write(data)
|
||||||
|
out.flush()
|
||||||
|
os.fsync(out.fileno())
|
||||||
|
os.replace(temp, path)
|
||||||
|
sync_dir(path.parent)
|
||||||
|
finally:
|
||||||
|
Path(temp).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def record(path, value):
|
||||||
|
atomic_write(path, (json.dumps(value, ensure_ascii=False, sort_keys=True) + '\n').encode())
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def deploy_lock(path):
|
||||||
|
fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||||
|
with os.fdopen(fd, 'r+') as handle:
|
||||||
|
try:
|
||||||
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError as exc:
|
||||||
|
raise CDError('deployment lock busy') from exc
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
def matches(pattern, text):
|
||||||
|
return isinstance(text, str) and pattern.fullmatch(text) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def validate_request(request):
|
||||||
|
actions = {'deploy': 'deploy-production', 'writeback': 'retry-writeback'}
|
||||||
|
action = request.get('action')
|
||||||
|
require(action in actions and request.get('confirm') == actions[action], 'explicit confirmation required')
|
||||||
|
require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task'))
|
||||||
|
and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID')
|
||||||
|
require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID')
|
||||||
|
event, digest = request.get('approval_event', ''), request.get('acceptance_digest', '')
|
||||||
|
require((event == '' and digest == '') or
|
||||||
|
(action == 'deploy' and matches(UUID, event) and matches(HASH, digest)),
|
||||||
|
'invalid HiFox approval binding')
|
||||||
|
|
||||||
|
|
||||||
|
def guard_proof(request, control_sha):
|
||||||
|
require(matches(SHA, control_sha), 'invalid workflow source SHA')
|
||||||
|
binding = {key: request[key] for key in ('run_id', 'action', 'confirm', 'sha', 'bundle', 'task')}
|
||||||
|
if request.get('approval_event'):
|
||||||
|
binding.update(approval_event=request['approval_event'], acceptance_digest=request['acceptance_digest'])
|
||||||
|
binding.update(repository=REPO, target=TARGET, workflow_sha=control_sha)
|
||||||
|
serialized = json.dumps(binding, sort_keys=True, separators=(',', ':')).encode()
|
||||||
|
return GUARD_PREFIX + hashlib.sha256(serialized).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def verify_guard_log(log, request, control_sha):
|
||||||
|
expected = guard_proof(request, control_sha)
|
||||||
|
pattern = re.compile(r'\d{4}-\d{2}-\d{2}T[0-9:.]+Z (' + GUARD_PREFIX + r'[0-9a-f]{64})')
|
||||||
|
proofs = [match.group(1) for line in log.decode().splitlines() if (match := pattern.fullmatch(line))]
|
||||||
|
require(proofs == [expected], 'Gitea guard dispatch proof missing or different')
|
||||||
|
|
||||||
|
|
||||||
|
def hash_file(path):
|
||||||
|
with path.open('rb') as src:
|
||||||
|
return hashlib.file_digest(src, 'sha256').hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def immutable_file(path):
|
||||||
|
info = path.lstat()
|
||||||
|
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and not info.st_mode & 0o022,
|
||||||
|
'bundle requires regular files owned by the deployment account')
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def verify_bundle(directory, request, archives=True):
|
||||||
|
raw = immutable_file(directory / 'release.json').read_bytes()
|
||||||
|
require(hashlib.sha256(raw).hexdigest() == request['bundle'], 'release manifest digest mismatch')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
require(manifest.get('schema') == 1 and manifest.get('target') == TARGET
|
||||||
|
and manifest.get('sha') == request['sha'] and manifest.get('task') == request['task'],
|
||||||
|
'bundle commit, task or target mismatch')
|
||||||
|
require(set(manifest.get('images', {})) == set(ROLES), 'bundle must include API, web and worker')
|
||||||
|
for role, image in manifest['images'].items():
|
||||||
|
require(matches(IMAGE, image.get('id')) and matches(HASH, image.get('sha256')), 'invalid image record')
|
||||||
|
if archives:
|
||||||
|
archive = immutable_file(directory / (role + '.tar'))
|
||||||
|
require(hash_file(archive) == image['sha256'], f'{role} archive checksum mismatch')
|
||||||
|
versions = manifest.get('migrations')
|
||||||
|
require(isinstance(versions, list) and versions and all(type(n) is int and n > 0 for n in versions)
|
||||||
|
and versions == sorted(set(versions)), 'invalid migration ledger in manifest')
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def verify_contexts(result, contexts):
|
||||||
|
latest = {}
|
||||||
|
for item in result.get('statuses') or []:
|
||||||
|
latest.setdefault(item.get('context'), item.get('status'))
|
||||||
|
require(bool(contexts) and all(latest.get(context) == 'success' for context in contexts),
|
||||||
|
'required CI status is missing or unsuccessful')
|
||||||
|
|
||||||
|
|
||||||
|
def metadata(task, key):
|
||||||
|
value = (task.get('metadata') or {}).get(key)
|
||||||
|
return json.loads(value) if isinstance(value, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
def check_task(task, request, receipt=None):
|
||||||
|
require(task.get('id') == request['task'] and task.get('spaceId') == SPACE, 'HiFox task or space mismatch')
|
||||||
|
status = (task.get('status') or {}).get('id')
|
||||||
|
same_publication = receipt and metadata(task, 'cd_release') == receipt.get('hifox')
|
||||||
|
require(status == READY or (status == PUBLISHED and same_publication), 'HiFox task is not ready for release')
|
||||||
|
evidence = metadata(task, 'cd_acceptance') or {}
|
||||||
|
url = urlsplit(evidence.get('evidence', ''))
|
||||||
|
require(evidence.get('verdict') == 'passed' and evidence.get('sha') == request['sha']
|
||||||
|
and evidence.get('bundle') == request['bundle'] and url.scheme == 'https' and url.hostname
|
||||||
|
and not (url.username or url.password or url.query)
|
||||||
|
and (not url.fragment or (url.netloc == 'app.hifox.com' and url.fragment.startswith('comment-')
|
||||||
|
and matches(UUID, url.fragment.removeprefix('comment-')))),
|
||||||
|
'missing or changed business acceptance for this commit and bundle')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def acceptance_digest(acceptance):
|
||||||
|
return hashlib.sha256(json.dumps(acceptance, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def release_approval(task, timeline, request):
|
||||||
|
evidence = check_task(task, request)
|
||||||
|
changes = [row for row in timeline if row.get('event') == 'status_changed']
|
||||||
|
require(bool(changes), 'missing HiFox status history')
|
||||||
|
approval = changes[-1]
|
||||||
|
approver = approval.get('actor') or {}
|
||||||
|
require((approval.get('metadata') or {}).get('to') == '待发布'
|
||||||
|
and approval.get('createdAt') and approval['createdAt'] == task.get('statusChangedAt')
|
||||||
|
and approver.get('id') == RELEASE_OWNER and approver.get('type') == 'human'
|
||||||
|
and matches(UUID, approval.get('id')), 'the latest transition must be an approval by the release owner')
|
||||||
|
task_url = task.get('url', '')
|
||||||
|
prefix = task_url + '#comment-'
|
||||||
|
require(task_url.startswith('https://app.hifox.com/') and evidence['evidence'].startswith(prefix),
|
||||||
|
'automated release requires the same-task acceptance report')
|
||||||
|
comment_id = evidence['evidence'].removeprefix(prefix)
|
||||||
|
require(matches(UUID, comment_id), 'invalid acceptance report ID')
|
||||||
|
reports = [row for row in timeline if row.get('event') == 'commented' and row.get('id') == comment_id]
|
||||||
|
require(len(reports) == 1, 'acceptance report missing from task history')
|
||||||
|
report = reports[0]
|
||||||
|
body = report.get('content') or ''
|
||||||
|
require(report.get('createdAt') and (report.get('updatedAt') or report['createdAt']) < approval['createdAt']
|
||||||
|
and request['sha'] in body and request['bundle'] in body,
|
||||||
|
'the unchanged report must identify the candidate and predate approval')
|
||||||
|
binding = {'approval_event': approval['id'], 'acceptance_digest': acceptance_digest(evidence)}
|
||||||
|
require(all(request.get(key, value) == value for key, value in binding.items()),
|
||||||
|
'HiFox approval or acceptance changed after dispatch')
|
||||||
|
return binding, approval['createdAt']
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_task(host, request):
|
||||||
|
task = host.task(request['task'])
|
||||||
|
if request.get('approval_event'):
|
||||||
|
release_approval(task, host.timeline(request['task']), request)
|
||||||
|
return check_task(task, request)
|
||||||
|
|
||||||
|
|
||||||
|
def release(host, request):
|
||||||
|
validate_request(request)
|
||||||
|
path = host.receipt_path
|
||||||
|
existing = read_json(path) if path.exists() else None
|
||||||
|
if request['action'] == 'writeback':
|
||||||
|
require(existing and existing['state'] in ('deployed_hifox_pending', 'published'),
|
||||||
|
'writeback requires a verified deployment receipt')
|
||||||
|
require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch')
|
||||||
|
manifest = verify_bundle(host.bundle_path(request), request, archives=False)
|
||||||
|
host.approve(request)
|
||||||
|
host.verify(manifest)
|
||||||
|
return writeback(host, request, existing)
|
||||||
|
|
||||||
|
require(existing is None, 'existing release receipt: use writeback or inspect interrupted/failed deployment')
|
||||||
|
host.approve(request)
|
||||||
|
manifest = verify_bundle(host.bundle_path(request), request)
|
||||||
|
host.check_ci(request)
|
||||||
|
accepted = accepted_task(host, request)
|
||||||
|
before = host.baseline()
|
||||||
|
host.pre_switch(before, manifest)
|
||||||
|
receipt = {k: request[k] for k in ('sha', 'task', 'bundle', 'run_id')}
|
||||||
|
receipt.update(state='preparing', target=TARGET, started_at=stamp(), previous=before)
|
||||||
|
record(path, receipt)
|
||||||
|
try:
|
||||||
|
host.load(host.bundle_path(request), manifest)
|
||||||
|
host.pre_switch(before, manifest)
|
||||||
|
host.approve(request)
|
||||||
|
require(accepted_task(host, request) == accepted, 'business acceptance changed before switch')
|
||||||
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
||||||
|
receipt.update(state='failed_before_cutover')
|
||||||
|
record(path, receipt)
|
||||||
|
raise CDError(f'failed_before_cutover: {exc}' if isinstance(exc, CDError) else 'failed_before_cutover') from exc
|
||||||
|
|
||||||
|
receipt.update(state='cutover_started')
|
||||||
|
record(path, receipt)
|
||||||
|
try:
|
||||||
|
host.switch(before, manifest)
|
||||||
|
evidence = host.verify(manifest)
|
||||||
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
||||||
|
status = 'rollback_failed'
|
||||||
|
try:
|
||||||
|
host.rollback(before)
|
||||||
|
status = 'rolled_back'
|
||||||
|
except (CDError, OSError, ValueError, KeyError):
|
||||||
|
pass
|
||||||
|
receipt.update(state=status, finished_at=stamp(), failure=str(exc) if isinstance(exc, CDError) else 'operation failed')
|
||||||
|
record(path, receipt)
|
||||||
|
raise CDError(f'{status}: deployment verification failed') from exc
|
||||||
|
|
||||||
|
receipt.update(state='deployed_hifox_pending', deployed_at=stamp(), evidence=evidence)
|
||||||
|
receipt['hifox'] = {**evidence, 'target': TARGET, 'bundle': request['bundle'], 'deployed_at': receipt['deployed_at'],
|
||||||
|
'login_url': LOGIN_URL,
|
||||||
|
'run_url': f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'}
|
||||||
|
record(path, receipt)
|
||||||
|
return writeback(host, request, receipt)
|
||||||
|
|
||||||
|
|
||||||
|
def writeback(host, request, receipt):
|
||||||
|
try:
|
||||||
|
task = host.task(request['task'])
|
||||||
|
check_task(task, request, receipt)
|
||||||
|
host.callback(request['task'], task, receipt.get('hifox'))
|
||||||
|
receipt.update(state='published')
|
||||||
|
record(host.receipt_path, receipt)
|
||||||
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
||||||
|
raise CDError('DEPLOYED_HIFOX_PENDING: retry writeback for this receipt') from exc
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
|
||||||
|
def with_images(config, images):
|
||||||
|
result = copy.deepcopy(config)
|
||||||
|
require(set(result['services']) == set(ROLES), 'unexpected Compose service set')
|
||||||
|
for role in ROLES:
|
||||||
|
result['services'][role]['image'] = images[role]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def run(argv, purpose, *, cwd=None, input_data=None, timeout=30, allowed=(0,), env=None):
|
||||||
|
try:
|
||||||
|
result = subprocess.run(argv, cwd=cwd, input=input_data, capture_output=True, timeout=timeout, env=env)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise CDError(f'{purpose} unavailable or timed out') from exc
|
||||||
|
require(result.returncode in allowed, f'{purpose} failed (exit {result.returncode})')
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *_args, **_kwargs):
|
||||||
|
raise CDError('unexpected HTTP redirect')
|
||||||
|
|
||||||
|
|
||||||
|
def http(url, *, direct=False, limit=2**20):
|
||||||
|
handlers = [NoRedirect()]
|
||||||
|
if direct:
|
||||||
|
handlers.append(ProxyHandler({}))
|
||||||
|
try:
|
||||||
|
with build_opener(*handlers).open(url, timeout=10) as response:
|
||||||
|
data = response.read(limit + 1)
|
||||||
|
require(len(data) <= limit, 'HTTP response exceeds limit')
|
||||||
|
return data
|
||||||
|
except (URLError, TimeoutError) as exc:
|
||||||
|
raise CDError('HTTP read failed') from exc
|
||||||
|
|
||||||
|
|
||||||
|
class Assets(HTMLParser):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.paths = []
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
attrs = dict(attrs)
|
||||||
|
path = attrs.get('src' if tag == 'script' else 'href', '')
|
||||||
|
if path and re.fullmatch(r'/assets/[A-Za-z0-9_.-]+\.(?:js|css)', path):
|
||||||
|
self.paths.append(path)
|
||||||
|
|
||||||
|
|
||||||
|
def disk_assets(html):
|
||||||
|
parser = Assets()
|
||||||
|
parser.feed(html.decode())
|
||||||
|
require(any(p.endswith('.js') for p in parser.paths), 'web index has no compiled entry point')
|
||||||
|
return sorted(set(parser.paths))
|
||||||
|
|
||||||
|
|
||||||
|
def image_inspect(image):
|
||||||
|
return json.loads(run(['docker', 'image', 'inspect', image], 'read image identity'))[0]
|
||||||
|
|
||||||
|
|
||||||
|
def inspect(name):
|
||||||
|
obj = json.loads(run(['docker', 'container', 'inspect', name], 'read running container'))[0]
|
||||||
|
require(obj['State']['Status'] == 'running' and matches(IMAGE, obj.get('Image')), 'expected container is not running')
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
def image_label(image):
|
||||||
|
return image_inspect(image)['Config']['Labels'].get(REVISION, '')
|
||||||
|
|
||||||
|
|
||||||
|
def api_identity(container):
|
||||||
|
raw = run(['docker', 'exec', container, '/app/api', '--version-record-status'], 'read API build identity', allowed=(0, 1))
|
||||||
|
obj = json.loads(raw)
|
||||||
|
require(matches(SHA, obj.get('commit_sha')), 'API binary has no full build identity')
|
||||||
|
return obj['commit_sha']
|
||||||
|
|
||||||
|
|
||||||
|
def migration_files(git, sha):
|
||||||
|
paths = git('ls-tree', '-r', '--name-only', sha, 'internal/platform/postgres/migrations').decode().splitlines()
|
||||||
|
return sorted(int(Path(line).name.split('_', 1)[0]) for line in paths if line.endswith('.up.sql'))
|
||||||
|
|
||||||
|
|
||||||
|
def ledger(container):
|
||||||
|
probe = Path(__file__).with_name('cd_migration_probe.py').read_text()
|
||||||
|
data = run(['docker', 'exec', container, 'python3', '-c', probe], 'read migration ledger')
|
||||||
|
result = json.loads(data)
|
||||||
|
require(isinstance(result, list) and result, 'unexpected migration ledger result')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def verify_archive(archive, expected):
|
||||||
|
# docker save by ID omits mutable RepoTags. Refuse archives that could retag another app.
|
||||||
|
with tarfile.open(archive, 'r:') as t:
|
||||||
|
entry = t.getmember('manifest.json')
|
||||||
|
require(entry.isfile() and entry.size < 65536, 'unexpected Docker archive manifest')
|
||||||
|
containers = json.load(t.extractfile(entry))
|
||||||
|
require(len(containers) == 1 and not containers[0].get('RepoTags'), 'archive must contain one untagged image')
|
||||||
|
config = t.getmember(containers[0]['Config'])
|
||||||
|
require(config.isfile() and config.size < 2**20, 'unexpected Docker image config')
|
||||||
|
require('sha256:' + hashlib.sha256(t.extractfile(config).read()).hexdigest() == expected,
|
||||||
|
'archive config does not match approved image ID')
|
||||||
|
for member in t:
|
||||||
|
require((member.isdir() or member.isfile()) and not member.name.startswith('/')
|
||||||
|
and '..' not in Path(member.name).parts, 'unexpected Docker archive path')
|
||||||
|
if 'index.json' in t.getnames():
|
||||||
|
index = t.getmember('index.json')
|
||||||
|
require(index.isfile() and index.size < 65536, 'unexpected OCI archive index')
|
||||||
|
oci = json.load(t.extractfile(index))
|
||||||
|
require(len(oci.get('manifests', [])) == 1 and all(
|
||||||
|
key not in descriptor.get('annotations', {}) for descriptor in oci['manifests']
|
||||||
|
for key in ('org.opencontainers.image.ref.name', 'io.containerd.image.name')),
|
||||||
|
'archive OCI index contains a mutable name')
|
||||||
|
|
||||||
|
|
||||||
|
class LiveHost:
|
||||||
|
def __init__(self, request):
|
||||||
|
self.receipt_path = DATA / 'receipts' / (request['bundle'] + '.json')
|
||||||
|
self.compose = APP / 'compose.json'
|
||||||
|
self.git_path = DATA / 'source.git'
|
||||||
|
self.containers = {role: 'kg-cloud-' + role for role in ROLES}
|
||||||
|
self.health_url = 'http://127.0.0.1:8088'
|
||||||
|
self.request = request
|
||||||
|
|
||||||
|
def bundle_path(self, request):
|
||||||
|
return DATA / 'bundles' / request['bundle']
|
||||||
|
|
||||||
|
def git(self, *args):
|
||||||
|
return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90)
|
||||||
|
|
||||||
|
def approve(self, request):
|
||||||
|
repo_url = f'{FORGE}/{REPO}'
|
||||||
|
run_id = request['run_id']
|
||||||
|
run_api = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{run_id}'
|
||||||
|
dispatch = json.loads(http(run_api))
|
||||||
|
control_sha = dispatch.get('head_sha')
|
||||||
|
trigger_actor = dispatch.get('trigger_actor') or dispatch.get('actor') or {}
|
||||||
|
require(str(dispatch.get('id')) == run_id and dispatch.get('event') == 'workflow_dispatch'
|
||||||
|
and dispatch.get('path') == 'deploy-production.yml@refs/heads/main'
|
||||||
|
and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running')
|
||||||
|
and (dispatch.get('actor') or {}).get('id') == 1 and trigger_actor.get('id') == 1
|
||||||
|
and matches(SHA, control_sha), 'Gitea run is not an active deployment dispatched by the release owner')
|
||||||
|
# Repo secrets can be read by other workflows. Only a completed guard from
|
||||||
|
# the installed workflow and controller may authorize the exact inputs.
|
||||||
|
try:
|
||||||
|
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
|
||||||
|
except CDError:
|
||||||
|
self.fetch_source()
|
||||||
|
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
|
||||||
|
trusted = {'scripts/production_cd.py': Path(__file__).resolve(),
|
||||||
|
'.gitea/workflows/deploy-production.yml': DATA / 'deploy-production.yml'}
|
||||||
|
for name, local in trusted.items():
|
||||||
|
require(self.git('show', f'{control_sha}:{name}') == immutable_file(local).read_bytes(),
|
||||||
|
'Gitea approval workflow differs from installed control files')
|
||||||
|
jobs = json.loads(http(run_api + '/jobs')).get('jobs') or []
|
||||||
|
guards = [job for job in jobs if job.get('name') == 'guard' and job.get('conclusion') == 'success'
|
||||||
|
and str(job.get('run_id')) == run_id and job.get('head_sha') == control_sha
|
||||||
|
and matches(re.compile('[1-9][0-9]{0,14}'), str(job.get('id')))]
|
||||||
|
require(len(guards) == 1, 'Gitea approval guard has not succeeded')
|
||||||
|
guard_id = guards[0]['id']
|
||||||
|
verify_guard_log(http(f'{repo_url}/actions/runs/{run_id}/jobs/{guard_id}/logs'), request, control_sha)
|
||||||
|
|
||||||
|
def fetch_source(self):
|
||||||
|
self.git('fetch', '--no-tags', '--prune', f'{FORGE}/{REPO}.git', '+refs/heads/*:refs/remotes/origin/*')
|
||||||
|
|
||||||
|
def check_ci(self, request):
|
||||||
|
self.fetch_source()
|
||||||
|
refs = self.git('for-each-ref', '--format=%(refname)', '--contains', request['sha'], 'refs/remotes/origin/')
|
||||||
|
require(refs.strip(), 'candidate commit is not reachable from a repository branch')
|
||||||
|
contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines()
|
||||||
|
verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')),
|
||||||
|
[line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')])
|
||||||
|
|
||||||
|
def hifox(self, *args):
|
||||||
|
cli = DATA / 'bin/hifox'
|
||||||
|
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(DATA / 'hifox-home'), 'CI': '1'}
|
||||||
|
raw = run([str(cli), '--organization-id', ORG, *args, '--output', 'json'], 'HiFox operation', env=env)
|
||||||
|
result = json.JSONDecoder().raw_decode(raw.decode().strip())[0]
|
||||||
|
require(isinstance(result, (dict, list)) and (not isinstance(result, dict)
|
||||||
|
or (result.get('success') is not False and not result.get('error'))),
|
||||||
|
'HiFox did not return a successful result')
|
||||||
|
return result
|
||||||
|
|
||||||
|
def task(self, task):
|
||||||
|
return self.hifox('task', 'view', task)
|
||||||
|
|
||||||
|
def timeline(self, task):
|
||||||
|
return self.hifox('api', 'get', f'/rest/v1/tasks/{task}/timeline')
|
||||||
|
|
||||||
|
def comment_once(self, task, key, text):
|
||||||
|
marker = f'KG-CD {key}'
|
||||||
|
body = f'{text}\n\n`{marker}`'
|
||||||
|
|
||||||
|
def present():
|
||||||
|
return any(row.get('event') == 'commented' and row.get('content') == body
|
||||||
|
and (row.get('actor') or {}).get('id') == RELEASE_OWNER for row in self.timeline(task))
|
||||||
|
|
||||||
|
if not present():
|
||||||
|
self.hifox('task', 'comment', task, '--body', body)
|
||||||
|
require(present(), 'HiFox release comment readback mismatch')
|
||||||
|
|
||||||
|
def report_failure(self, request):
|
||||||
|
if not self.receipt_path.exists():
|
||||||
|
return
|
||||||
|
receipt = read_json(self.receipt_path)
|
||||||
|
if any(receipt.get(key) != request[key] for key in ('task', 'sha', 'bundle', 'run_id')):
|
||||||
|
return
|
||||||
|
messages = {
|
||||||
|
'failed_before_cutover': '发布检查失败,未切换主站。',
|
||||||
|
'rolled_back': '发布核验失败,已恢复原版本并复验。',
|
||||||
|
'rollback_failed': '发布核验失败,回滚未通过复验,请检查主站。',
|
||||||
|
'deployed_hifox_pending': '主站已部署并核验,HiFox 回写待完成。请重试回写。',
|
||||||
|
}
|
||||||
|
state = receipt['state']
|
||||||
|
if state in messages:
|
||||||
|
run_url = f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'
|
||||||
|
self.comment_once(request['task'], f'{state} #{request["run_id"]}',
|
||||||
|
f'{messages[state]}\n\n候选 `{request["sha"]}`\n\n[发布运行]({run_url})')
|
||||||
|
|
||||||
|
def callback(self, task_id, task, expected):
|
||||||
|
if metadata(task, 'cd_release') != expected:
|
||||||
|
self.hifox('task', 'metadata', 'set', task_id, 'cd_release', json.dumps(json.dumps(expected)))
|
||||||
|
fresh = self.task(task_id)
|
||||||
|
check_task(fresh, self.request, {'hifox': expected})
|
||||||
|
require(metadata(fresh, 'cd_release') == expected, 'HiFox release receipt readback mismatch')
|
||||||
|
images = '\n'.join(f'- {role} `{expected["images"][role]}`' for role in ROLES)
|
||||||
|
self.comment_once(task_id, f'published {expected["bundle"]}',
|
||||||
|
f'云治理主站发布完成,健康检查通过。\n\n'
|
||||||
|
f'运行版本 `{expected.get("version") or expected["sha"]}`\n\n'
|
||||||
|
f'提交 `{expected["sha"]}`\n\n{images}\n\n'
|
||||||
|
f'[登录主站]({LOGIN_URL}) · [发布运行]({expected["run_url"]})')
|
||||||
|
fresh = self.task(task_id)
|
||||||
|
check_task(fresh, self.request, {'hifox': expected})
|
||||||
|
require(metadata(fresh, 'cd_release') == expected, 'HiFox release changed during comment writeback')
|
||||||
|
if fresh['status']['id'] != PUBLISHED:
|
||||||
|
self.hifox('task', 'edit', task_id, '--status', PUBLISHED)
|
||||||
|
final = self.task(task_id)
|
||||||
|
final_status = (final.get('status') or {}).get('id')
|
||||||
|
require(final_status == PUBLISHED and metadata(final, 'cd_release') == expected, 'HiFox published readback mismatch')
|
||||||
|
|
||||||
|
def compose_cmd(self, *args):
|
||||||
|
return ['docker', 'compose', '-p', PROJECT, '-f', str(self.compose), *args]
|
||||||
|
|
||||||
|
def rendered_config(self):
|
||||||
|
raw = run(self.compose_cmd('config', '--format', 'json'), 'render Compose configuration', cwd=APP)
|
||||||
|
return json.loads(raw)['services']
|
||||||
|
|
||||||
|
def runtime(self):
|
||||||
|
rendered = self.rendered_config()
|
||||||
|
require(set(rendered) == set(ROLES), 'unexpected Compose service set')
|
||||||
|
result = {}
|
||||||
|
for role, name in self.containers.items():
|
||||||
|
c = inspect(name)
|
||||||
|
labels = c['Config']['Labels']
|
||||||
|
require(labels.get('com.docker.compose.project') == PROJECT
|
||||||
|
and labels.get('com.docker.compose.service') == role
|
||||||
|
and labels.get('com.docker.compose.project.working_dir') == str(APP)
|
||||||
|
and labels.get('com.docker.compose.project.config_files') == str(self.compose),
|
||||||
|
f'{role} does not belong to the approved Compose chain')
|
||||||
|
config = rendered[role]
|
||||||
|
image = image_inspect(c['Image'])
|
||||||
|
require(image_inspect(config['image'])['Id'] == c['Image'], f'{role} Compose image has drifted')
|
||||||
|
env = lambda entries: dict(item.split('=', 1) for item in entries or [])
|
||||||
|
expected_env = env(image['Config'].get('Env'))
|
||||||
|
expected_env.update({key: str(value) for key, value in (config.get('environment') or {}).items()
|
||||||
|
if value is not None})
|
||||||
|
require(env(c['Config'].get('Env')) == expected_env, f'{role} running environment has drifted')
|
||||||
|
expected_mounts = config.get('volumes') or []
|
||||||
|
require(all(mount['type'] == 'bind' for mount in expected_mounts), 'unexpected non-bind Compose volume')
|
||||||
|
mounts = {(v['Source'], v['Destination'], not v['RW']) for v in c.get('Mounts', []) if v['Type'] == 'bind'}
|
||||||
|
require(mounts == {(v['source'], v['target'], v.get('read_only', False)) for v in expected_mounts}
|
||||||
|
and len(mounts) == len(c.get('Mounts', [])), f'{role} running mounts have drifted')
|
||||||
|
require(not config.get('network_mode') or config['network_mode'] == c['HostConfig']['NetworkMode'],
|
||||||
|
f'{role} running network has drifted')
|
||||||
|
result[role] = c['Image']
|
||||||
|
return result
|
||||||
|
|
||||||
|
def baseline(self):
|
||||||
|
images = self.runtime()
|
||||||
|
self.probe_web()
|
||||||
|
return {'compose_sha256': hash_file(self.compose), 'images': images,
|
||||||
|
'revisions': {role: image_label(image) for role, image in images.items()}}
|
||||||
|
|
||||||
|
def pre_switch(self, previous, manifest):
|
||||||
|
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
|
||||||
|
'target baseline changed during deployment')
|
||||||
|
for revision in set(previous['revisions'].values()):
|
||||||
|
require(matches(SHA, revision), 'running image has no source revision')
|
||||||
|
self.git('merge-base', '--is-ancestor', revision, manifest['sha'])
|
||||||
|
require(migration_files(self.git, manifest['sha']) == manifest['migrations'], 'source migration set differs from bundle')
|
||||||
|
for role in ('api', 'worker'):
|
||||||
|
require(ledger(self.containers[role]) == manifest['migrations'],
|
||||||
|
f'{role} migration ledger mismatch; verify schema changes outside CD')
|
||||||
|
|
||||||
|
def load(self, directory, manifest):
|
||||||
|
archives = {role: directory / (role + '.tar') for role in ROLES}
|
||||||
|
for role, archive in archives.items():
|
||||||
|
verify_archive(archive, manifest['images'][role]['id'])
|
||||||
|
for role, archive in archives.items():
|
||||||
|
image = manifest['images'][role]['id']
|
||||||
|
run(['docker', 'image', 'load', '-i', str(archive)], f'load {role} image', timeout=600)
|
||||||
|
require(image_inspect(image)['Id'] == image and image_label(image) == manifest['sha'],
|
||||||
|
f'{role} loaded image differs from approved build')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def backup(self):
|
||||||
|
return self.receipt_path.with_suffix('.compose.backup')
|
||||||
|
|
||||||
|
def switch(self, previous, manifest):
|
||||||
|
current = self.compose.read_bytes()
|
||||||
|
require(hashlib.sha256(current).hexdigest() == previous['compose_sha256'], 'Compose changed before switch')
|
||||||
|
images = {role: manifest['images'][role]['id'] for role in ROLES}
|
||||||
|
changed = with_images(json.loads(current), images)
|
||||||
|
for role, image in previous['images'].items():
|
||||||
|
run(['docker', 'image', 'tag', image, f'knowledge-governance-cd-rollback:{self.request["bundle"]}-{role}'],
|
||||||
|
'retain rollback image')
|
||||||
|
atomic_write(self.backup, current)
|
||||||
|
atomic_write(self.compose, (json.dumps(changed, indent=2) + '\n').encode(), self.compose.stat().st_mode & 0o777)
|
||||||
|
self.recreate()
|
||||||
|
|
||||||
|
def recreate(self):
|
||||||
|
script = Path(__file__).with_name('deploy-api-web.sh')
|
||||||
|
run(['bash', str(script), '--verified-compose', str(APP), PROJECT, str(self.compose), 'api', 'web', 'worker'],
|
||||||
|
'switch governance Compose services', timeout=120)
|
||||||
|
|
||||||
|
def probe_web(self):
|
||||||
|
origin = self.health_url
|
||||||
|
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'web proxy health probe failed')
|
||||||
|
require(json.loads(http('http://127.0.0.1:18098/readyz', direct=True)).get('status') in ('ok', 'ready'),
|
||||||
|
'API readiness probe failed')
|
||||||
|
index = http(origin + '/', direct=True)
|
||||||
|
return index, {path: hashlib.sha256(http(origin + path, direct=True, limit=32 * 2**20)).hexdigest()
|
||||||
|
for path in disk_assets(index)}
|
||||||
|
|
||||||
|
def verify_web_image(self):
|
||||||
|
origin_index, assets = self.probe_web()
|
||||||
|
name = self.containers['web']
|
||||||
|
def read_asset(path):
|
||||||
|
return run(['docker', 'exec', name, 'cat', '/usr/share/nginx/html' + path], 'read running web asset')
|
||||||
|
|
||||||
|
require(origin_index == read_asset('/index.html'), 'served index differs from running web image')
|
||||||
|
for path, served_digest in assets.items():
|
||||||
|
require(hashlib.sha256(read_asset(path)).hexdigest() == served_digest, 'served asset differs from running web image')
|
||||||
|
return assets
|
||||||
|
|
||||||
|
def verify(self, manifest):
|
||||||
|
images = {role: manifest['images'][role]['id'] for role in ROLES}
|
||||||
|
def check():
|
||||||
|
require(self.runtime() == images, 'running image IDs differ from approved bundle')
|
||||||
|
require(api_identity(self.containers['api']) == manifest['sha'], 'running API SHA mismatches approval')
|
||||||
|
assets = self.verify_web_image()
|
||||||
|
version = image_inspect(images['api'])['Config']['Labels'].get('org.opencontainers.image.version')
|
||||||
|
if not isinstance(version, str) or not re.fullmatch(r'[0-9A-Za-z.+_-]{1,128}', version):
|
||||||
|
version = manifest['sha']
|
||||||
|
return {'sha': manifest['sha'], 'version': version, 'images': images, 'health': 'ok', 'asset_sha256': assets}
|
||||||
|
|
||||||
|
return wait_ready(check)
|
||||||
|
|
||||||
|
def rollback(self, previous):
|
||||||
|
if not self.backup.exists():
|
||||||
|
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
|
||||||
|
'rollback baseline cannot be established')
|
||||||
|
self.verify_web_image()
|
||||||
|
return
|
||||||
|
original = self.backup.read_bytes()
|
||||||
|
require(hashlib.sha256(original).hexdigest() == previous['compose_sha256'], 'rollback backup checksum mismatch')
|
||||||
|
source = json.loads(original)
|
||||||
|
for role, image in previous['images'].items():
|
||||||
|
reference = source['services'][role]['image']
|
||||||
|
require(image_inspect(reference)['Id'] == image, 'rollback source tag has drifted')
|
||||||
|
atomic_write(self.compose, original, self.compose.stat().st_mode & 0o777)
|
||||||
|
self.recreate()
|
||||||
|
|
||||||
|
def check():
|
||||||
|
require(self.runtime() == previous['images'], 'rollback running images mismatch')
|
||||||
|
require(api_identity(self.containers['api']) == previous['revisions']['api'], 'rollback API SHA mismatch')
|
||||||
|
self.verify_web_image()
|
||||||
|
|
||||||
|
wait_ready(check)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_ready(check, seconds=90):
|
||||||
|
deadline = time.monotonic() + seconds
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return check()
|
||||||
|
except (CDError, OSError, KeyError, ValueError):
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
|
||||||
|
def event_request(event, env):
|
||||||
|
require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main'
|
||||||
|
and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required')
|
||||||
|
inputs = event.get('inputs') or {}
|
||||||
|
request = {name: inputs.get(name) or '' for name in
|
||||||
|
('action', 'sha', 'task', 'bundle', 'confirm', 'approval_event', 'acceptance_digest')}
|
||||||
|
request['run_id'] = env.get('GITHUB_RUN_ID', '')
|
||||||
|
validate_request(request)
|
||||||
|
return request
|
||||||
|
|
||||||
|
|
||||||
|
def pack(args):
|
||||||
|
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
|
||||||
|
require(not args.output.exists(), 'bundle output already exists')
|
||||||
|
source = lambda *argv: run(['git', *argv], 'read candidate source')
|
||||||
|
require(source('rev-parse', args.sha + '^{commit}').decode().strip() == args.sha, 'source commit not found')
|
||||||
|
images = {}
|
||||||
|
versions = migration_files(source, args.sha)
|
||||||
|
manifest = dict(schema=1, target=TARGET, sha=args.sha, task=args.task, images=images, migrations=versions)
|
||||||
|
with tempfile.TemporaryDirectory(dir=args.output.parent) as directory:
|
||||||
|
build = Path(directory) / "sealed"
|
||||||
|
build.mkdir(mode=0o700)
|
||||||
|
for role in ROLES:
|
||||||
|
image = image_inspect(getattr(args, role).removeprefix('sha256:'))
|
||||||
|
image_id = image['Id']
|
||||||
|
require(matches(IMAGE, image_id) and image_label(image_id) == args.sha, f'{role} source label mismatch')
|
||||||
|
archive = build / (role + '.tar')
|
||||||
|
run(['docker', 'image', 'save', '-o', str(archive), image_id], f'archive {role} image', timeout=900)
|
||||||
|
verify_archive(archive, image_id)
|
||||||
|
archive.chmod(0o600)
|
||||||
|
images[role] = {'id': image_id, 'sha256': hash_file(archive)}
|
||||||
|
record(build / 'release.json', manifest)
|
||||||
|
for item in build.iterdir():
|
||||||
|
with item.open("rb") as artifact:
|
||||||
|
os.fsync(artifact.fileno())
|
||||||
|
sync_dir(build)
|
||||||
|
build.rename(args.output)
|
||||||
|
sync_dir(args.output.parent)
|
||||||
|
print(hash_file(args.output / 'release.json'))
|
||||||
|
|
||||||
|
|
||||||
|
def attest(args):
|
||||||
|
request = {'sha': args.sha, 'task': args.task, 'bundle': hash_file(args.directory / 'release.json')}
|
||||||
|
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
|
||||||
|
manifest = verify_bundle(args.directory, request)
|
||||||
|
before = {role: inspect(getattr(args, role)) for role in ROLES}
|
||||||
|
for role, obj in before.items():
|
||||||
|
require(obj['Image'] == manifest['images'][role]['id'] and obj['Name'] != '/kg-cloud-' + role,
|
||||||
|
'isolated test runtime image differs from bundle')
|
||||||
|
require(api_identity(args.api) == args.sha and ledger(args.api) == manifest['migrations']
|
||||||
|
and ledger(args.worker) == manifest['migrations'], 'test runtime SHA or schema differs')
|
||||||
|
base = urlsplit(args.test_url)
|
||||||
|
require(base.scheme in ('http', 'https') and base.hostname and not (base.username or base.password or base.query or base.fragment)
|
||||||
|
and base.path in ('', '/') and base.port != 8088, 'provide isolated test endpoint')
|
||||||
|
origin = args.test_url.rstrip('/')
|
||||||
|
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'isolated test health probe failed')
|
||||||
|
index = http(origin + '/', direct=True)
|
||||||
|
require(index == run(['docker', 'exec', args.web, 'cat', '/usr/share/nginx/html/index.html'], 'read test web index'),
|
||||||
|
'test endpoint is serving another web image')
|
||||||
|
for role, obj in before.items():
|
||||||
|
require(inspect(getattr(args, role))['Id'] == obj['Id'], 'test instance changed during readback')
|
||||||
|
print(json.dumps({'sha': args.sha, 'bundle': request['bundle'], 'test_url': args.test_url,
|
||||||
|
'images': {role: obj['Image'] for role, obj in before.items()}}, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
def cli():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
commands = parser.add_subparsers(dest='command', required=True)
|
||||||
|
for command in ('guard', 'send', 'serve', 'pack', 'attest'):
|
||||||
|
sub = commands.add_parser(command)
|
||||||
|
if command in ('guard', 'send'):
|
||||||
|
sub.add_argument('--event', type=Path, required=True)
|
||||||
|
if command in ('pack', 'attest'):
|
||||||
|
sub.add_argument('--sha', required=True)
|
||||||
|
sub.add_argument('--task', required=True)
|
||||||
|
if command in ('pack', 'attest'):
|
||||||
|
for role in ROLES:
|
||||||
|
sub.add_argument('--' + role, required=True)
|
||||||
|
if command == 'pack':
|
||||||
|
sub.add_argument('--output', type=Path, required=True)
|
||||||
|
if command == 'attest':
|
||||||
|
sub.add_argument('--directory', type=Path, required=True)
|
||||||
|
sub.add_argument('--test-url', required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.command in ('guard', 'send'):
|
||||||
|
request = event_request(read_json(args.event), os.environ)
|
||||||
|
if args.command == 'guard':
|
||||||
|
print(guard_proof(request, os.environ.get('GITHUB_SHA')))
|
||||||
|
else:
|
||||||
|
send(request)
|
||||||
|
elif args.command == 'pack':
|
||||||
|
pack(args)
|
||||||
|
elif args.command == 'attest':
|
||||||
|
attest(args)
|
||||||
|
else:
|
||||||
|
raw = sys.stdin.buffer.read(8193)
|
||||||
|
require(len(raw) <= 8192, 'request exceeds size limit')
|
||||||
|
request = json.loads(raw)
|
||||||
|
validate_request(request)
|
||||||
|
with deploy_lock(DATA / 'release.lock'):
|
||||||
|
host = LiveHost(request)
|
||||||
|
for item in (DATA / 'receipts').glob('*.json'):
|
||||||
|
if read_json(item)['state'] in ('cutover_started', 'rollback_failed'):
|
||||||
|
raise CDError('previous deployment outcome requires operator reconciliation')
|
||||||
|
try:
|
||||||
|
result = release(host, request)
|
||||||
|
except (CDError, OSError, ValueError, KeyError):
|
||||||
|
try:
|
||||||
|
host.report_failure(request)
|
||||||
|
except (CDError, OSError, ValueError, KeyError):
|
||||||
|
print('CD: HiFox failure comment pending', file=sys.stderr)
|
||||||
|
raise
|
||||||
|
print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
def send(request):
|
||||||
|
key = os.environ.get('KG_CD_SSH_KEY', '')
|
||||||
|
known_hosts = os.environ.get('KG_CD_KNOWN_HOSTS', '')
|
||||||
|
host = os.environ.get('KG_CD_SSH_HOST', '')
|
||||||
|
port = os.environ.get('KG_CD_SSH_PORT', '22') or '22'
|
||||||
|
require(key.strip() and known_hosts.strip(), 'missing KG_CD_SSH_KEY or KG_CD_KNOWN_HOSTS')
|
||||||
|
require(host in ('100.85.111.121', '::1') and port.isdecimal() and 1 <= int(port) <= 65535,
|
||||||
|
'configure the direct or approved tunnel address for the target release endpoint')
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
temp = Path(directory)
|
||||||
|
for filename, value in (('key', key), ('known_hosts', known_hosts)):
|
||||||
|
atomic_write(temp / filename, (value.strip() + '\n').encode())
|
||||||
|
argv = ['ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'IdentitiesOnly=yes',
|
||||||
|
'-o', 'ClearAllForwardings=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'HostKeyAlias=kg-governance-cd',
|
||||||
|
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', f'UserKnownHostsFile={temp / "known_hosts"}',
|
||||||
|
'-i', str(temp / 'key'), '-p', port, '-T', 'root@' + host, 'kg-cd-v1']
|
||||||
|
try:
|
||||||
|
result = subprocess.run(argv, input=json.dumps(request).encode(), capture_output=True, timeout=2100)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise CDError('target outcome unknown; inspect durable receipt before retry') from exc
|
||||||
|
if result.returncode:
|
||||||
|
error = result.stderr.decode(errors='replace')
|
||||||
|
for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:',
|
||||||
|
'target preflight rejected', 'deployment lock busy', 'writeback requires a verified deployment receipt',
|
||||||
|
'Gitea approval guard has not succeeded', 'Gitea guard dispatch proof missing or different'):
|
||||||
|
if marker in error:
|
||||||
|
raise CDError(error.strip().splitlines()[-1])
|
||||||
|
raise CDError('target rejected request or outcome unknown; inspect durable receipt')
|
||||||
|
response = json.loads(result.stdout)
|
||||||
|
require(response.get('state') == 'published' and all(response.get(k) == request[k] for k in ('sha', 'task', 'bundle')),
|
||||||
|
'invalid or unverified deployment receipt')
|
||||||
|
print(json.dumps(response, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
cli()
|
||||||
|
except CDError as error:
|
||||||
|
print(f'CD: {error}', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
except (OSError, KeyError, ValueError, TypeError, tarfile.TarError) as error:
|
||||||
|
print(f'CD: operation could not be verified ({type(error).__name__})', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
@@ -0,0 +1,374 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Prepare and exercise an isolated Knowledge Governance QA run."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from contextlib import ExitStack
|
||||||
|
import fcntl
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
CONFIG = Path.home() / '.config/knowledge-governance/qa.json'
|
||||||
|
API_PORT = 8088
|
||||||
|
WEB_PORT = 5173
|
||||||
|
RUNTIME_RESPONSE = 'KG_QA_RUNTIME_OK'
|
||||||
|
|
||||||
|
|
||||||
|
class QAError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def report(stage, detail):
|
||||||
|
print(f'[qa:{stage}] {detail}', flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def capture(argv, stage, timeout=10, env=None):
|
||||||
|
try:
|
||||||
|
return subprocess.run(argv, cwd=ROOT, env=env, text=True, capture_output=True,
|
||||||
|
timeout=timeout, stdin=subprocess.DEVNULL)
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise QAError(f'{stage} 缺少命令 {argv[0]}') from exc
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise QAError(f'{stage} 超过 {timeout} 秒') from exc
|
||||||
|
|
||||||
|
|
||||||
|
def private_file(path):
|
||||||
|
try:
|
||||||
|
info = path.lstat()
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise QAError(f'缺少测试配置文件 {path}') from exc
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600:
|
||||||
|
raise QAError(f'{path} 必须是当前用户所有的 0600 普通文件')
|
||||||
|
|
||||||
|
|
||||||
|
def database_config(path):
|
||||||
|
private_file(path)
|
||||||
|
try:
|
||||||
|
config = json.loads(path.read_text())
|
||||||
|
except (UnicodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise QAError(f'{path} 不是有效 JSON') from exc
|
||||||
|
if not isinstance(config, dict) or config.keys() != {'host', 'port', 'database', 'user'}:
|
||||||
|
raise QAError('qa.json 只接受 host、port、database、user 四个字段')
|
||||||
|
for field in ('database', 'user'):
|
||||||
|
if not re.fullmatch(r'kg_qa_[a-z0-9_]{1,56}', str(config[field])):
|
||||||
|
raise QAError(f'RDS {field} 必须以 kg_qa_ 开头并使用小写字母、数字或下划线')
|
||||||
|
host = config['host']
|
||||||
|
if not isinstance(host, str) or not re.fullmatch(r'[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+', host):
|
||||||
|
raise QAError('RDS host 必须填写证书对应的 DNS 主机名')
|
||||||
|
if type(config['port']) is not int or not 1 <= config['port'] <= 65535:
|
||||||
|
raise QAError('RDS port 必须是 1–65535 的整数')
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def tool_env():
|
||||||
|
allowed = ('HOME', 'PATH', 'LANG', 'TMPDIR', 'GOPROXY', 'GOSUMDB', 'GOPRIVATE', 'GONOSUMDB',
|
||||||
|
'GOMODCACHE', 'GOCACHE', 'npm_config_cache', 'npm_config_registry',
|
||||||
|
'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY',
|
||||||
|
'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')
|
||||||
|
env = {key: os.environ[key] for key in allowed if key in os.environ}
|
||||||
|
env['GOFLAGS'] = '-mod=readonly'
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def database_env(config, directory):
|
||||||
|
# Keep inherited PG variables and integration switches out of application children.
|
||||||
|
env = tool_env()
|
||||||
|
env.update(PGHOST=config['host'], PGPORT=str(config['port']), PGDATABASE=config['database'],
|
||||||
|
PGUSER=config['user'], PGSSLMODE='verify-full', PGCONNECT_TIMEOUT='5',
|
||||||
|
PGPASSFILE=str(directory / 'qa.pgpass'), PGSSLROOTCERT=str(directory / 'rds-ca.pem'),
|
||||||
|
PGOPTIONS='-c search_path=public', LC_ALL='C',
|
||||||
|
TEST_WEKNORA_ENABLED='false', PP_SCAN_ENABLED='false',
|
||||||
|
RELEASE_WORKER_ENABLED='false', PRODUCTION_RELEASE_REQUESTS_ENABLED='false',
|
||||||
|
DOBOTAI_OPS_READ_ENABLED='false', DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED='false')
|
||||||
|
env['APP_ADDR'] = f'127.0.0.1:{API_PORT}'
|
||||||
|
env['DATA' + 'BASE_URL'] = (
|
||||||
|
f"postgresql://{config['user']}@{config['host']}:{config['port']}/{config['database']}"
|
||||||
|
'?sslmode=verify-full&connect_timeout=5&search_path=public'
|
||||||
|
)
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def psql(env, sql):
|
||||||
|
result = capture(['psql', '-XAtw', '-v', 'ON_ERROR_STOP=1', '-c', sql], 'RDS 探针', env=env)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return result.stdout.strip()
|
||||||
|
detail = result.stderr.lower()
|
||||||
|
hints = (
|
||||||
|
('password', '口令认证失败,检查 qa.pgpass 和登录角色'),
|
||||||
|
('certificate', 'TLS 校验失败,检查 RDS 主机名和 CA'),
|
||||||
|
('does not exist', '数据库或角色不存在,联系管理员确认专用 QA 库'),
|
||||||
|
('permission denied', 'RDS 权限不足,检查专用 QA 角色的 CONNECT、schema USAGE/CREATE 权限'),
|
||||||
|
('could not translate host name', 'RDS 主机名无法解析,检查私网 DNS'),
|
||||||
|
('timeout', '连接 RDS 超时,检查 QA 虚拟机到 RDS 的私网和安全组'),
|
||||||
|
('connection refused', 'RDS 拒绝 TCP 连接,检查主机名、端口和私网接入'),
|
||||||
|
)
|
||||||
|
hint = next((message for marker, message in hints if marker in detail), '连接或 SQL 探针失败,检查 RDS 接入和权限')
|
||||||
|
raise QAError(f'RDS {hint}(psql 退出 {result.returncode})')
|
||||||
|
|
||||||
|
|
||||||
|
def database_state(env):
|
||||||
|
identity = psql(env, """
|
||||||
|
SELECT concat_ws('|', current_database(), current_user,
|
||||||
|
COALESCE((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false),
|
||||||
|
(SELECT NOT (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls)
|
||||||
|
FROM pg_roles WHERE rolname = current_user),
|
||||||
|
(has_schema_privilege(current_user, 'public', 'USAGE') AND
|
||||||
|
has_schema_privilege(current_user, 'public', 'CREATE')),
|
||||||
|
to_regclass('public.schema_migrations') IS NOT NULL,
|
||||||
|
EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||||
|
WHERE n.nspname = 'public' AND c.relname <> 'schema_migrations')
|
||||||
|
OR EXISTS (SELECT FROM pg_namespace
|
||||||
|
WHERE nspname NOT IN ('public', 'information_schema') AND nspname !~ '^pg_')
|
||||||
|
OR EXISTS (SELECT FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace
|
||||||
|
WHERE n.nspname = 'public'));
|
||||||
|
""")
|
||||||
|
fields = identity.split('|')
|
||||||
|
if len(fields) != 7 or fields[:4] != [env['PGDATABASE'], env['PGUSER'], 't', 't']:
|
||||||
|
raise QAError('RDS 实际库名、角色、TLS 或角色权限与 QA 配置不符')
|
||||||
|
if fields[4] != 't':
|
||||||
|
raise QAError('RDS 角色需要 public schema 的 USAGE 和 CREATE 权限')
|
||||||
|
rows = psql(env, 'SELECT version FROM public.schema_migrations ORDER BY version;') if fields[5] == 't' else ''
|
||||||
|
if any(not re.fullmatch(r'[0-9]+', row) for row in rows.splitlines()):
|
||||||
|
raise QAError('RDS 迁移记录包含无效版本')
|
||||||
|
return [int(row) for row in rows.splitlines()], fields[6] == 't'
|
||||||
|
|
||||||
|
|
||||||
|
def expected_migrations():
|
||||||
|
paths = (ROOT / 'internal/platform/postgres/migrations').glob('*.up.sql')
|
||||||
|
try:
|
||||||
|
versions = sorted(int(path.name.split('_', 1)[0]) for path in paths)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise QAError('仓库存在无效的迁移文件名') from exc
|
||||||
|
if not versions or len(versions) != len(set(versions)):
|
||||||
|
raise QAError('仓库迁移文件缺失或版本重复')
|
||||||
|
return versions
|
||||||
|
|
||||||
|
|
||||||
|
def check_migrations(expected, applied, nonempty, *, complete):
|
||||||
|
if not applied and nonempty:
|
||||||
|
raise QAError('RDS 非空且缺少有效迁移记录,停止初始化')
|
||||||
|
if expected[:len(applied)] != applied:
|
||||||
|
raise QAError('RDS 迁移序列与当前提交不兼容,使用专用空库或匹配的测试库')
|
||||||
|
if complete and applied != expected:
|
||||||
|
raise QAError(f'RDS 迁移未完成,缺少版本 {expected[len(applied)]} 至 {expected[-1]}')
|
||||||
|
|
||||||
|
|
||||||
|
def check_hifox(output):
|
||||||
|
try:
|
||||||
|
state = json.loads(output)
|
||||||
|
online = state['connected'] is True and state['remoteLiveness']['status'] == 'online'
|
||||||
|
except (ValueError, KeyError, TypeError) as exc:
|
||||||
|
raise QAError('HiFox 未返回有效的在线状态,检查本机注册和接收服务') from exc
|
||||||
|
if not online:
|
||||||
|
raise QAError('HiFox 未确认测试机在线,检查本机接收服务和网络')
|
||||||
|
|
||||||
|
|
||||||
|
def check_checkout(expected):
|
||||||
|
if not re.fullmatch(r'[0-9a-f]{40}', expected):
|
||||||
|
raise QAError('设置 KG_QA_SHA 为提测的完整 40 位 Git SHA')
|
||||||
|
actual = capture(['git', 'rev-parse', 'HEAD'], 'Git 检出')
|
||||||
|
if actual.returncode or actual.stdout.strip() != expected:
|
||||||
|
raise QAError('Git 检出与提测提交不一致')
|
||||||
|
state = capture(['git', 'status', '--porcelain'], 'Git 状态')
|
||||||
|
if state.returncode or state.stdout.strip():
|
||||||
|
raise QAError('QA 需要干净的独立检出,请检查 Git 状态')
|
||||||
|
|
||||||
|
|
||||||
|
def preflight(env):
|
||||||
|
for cmd in ('git', 'make', 'go', 'node', 'npm', 'psql', 'hifox', 'omp'):
|
||||||
|
if not shutil.which(cmd, path=env.get('PATH')):
|
||||||
|
raise QAError(f'工具链缺少 {cmd},检查 QA 机器的 PATH')
|
||||||
|
go = capture(['go', 'env', 'GOROOT', 'GOVERSION'], 'Go 工具链', env=env)
|
||||||
|
fields = go.stdout.splitlines()
|
||||||
|
if go.returncode or len(fields) != 2 or not fields[1].startswith('go1.26.'):
|
||||||
|
raise QAError('QA 需要 Go 1.26 工具链')
|
||||||
|
root = Path(fields[0]) / 'bin'
|
||||||
|
if not os.access(root / 'gofmt', os.X_OK):
|
||||||
|
raise QAError('Go 工具链缺少 gofmt,请安装完整的 Go 1.26')
|
||||||
|
env['PATH'] = f"{root}:{env['PATH']}"
|
||||||
|
node = capture(['node', '--version'], 'Node.js 工具链', env=env)
|
||||||
|
if node.returncode or not node.stdout.startswith('v22.'):
|
||||||
|
raise QAError('QA 需要 Node.js 22 工具链')
|
||||||
|
report('toolchain', f'{fields[1]} / Node.js {node.stdout.strip()}')
|
||||||
|
|
||||||
|
hifox = capture(['hifox', 'computer', 'status', '--output', 'json'], 'HiFox 在线检查', timeout=15)
|
||||||
|
if hifox.returncode:
|
||||||
|
raise QAError(f'HiFox 状态命令失败(退出 {hifox.returncode})')
|
||||||
|
check_hifox(hifox.stdout)
|
||||||
|
report('machine', f'{socket.gethostname()} / HiFox online')
|
||||||
|
prompt = f'Reply with exactly {RUNTIME_RESPONSE}. Do not use tools.'
|
||||||
|
omp = capture(['omp', '--no-session', '-p', prompt], 'OMP Runtime 探针', timeout=35)
|
||||||
|
if omp.returncode or RUNTIME_RESPONSE not in {line.strip() for line in omp.stdout.splitlines()}:
|
||||||
|
raise QAError('OMP Runtime 没有返回探针结果,检查 OMP 模型配置和 Magpie 网关')
|
||||||
|
report('runtime', 'OMP 模型请求通过')
|
||||||
|
|
||||||
|
|
||||||
|
def check_ports():
|
||||||
|
for port in (API_PORT, WEB_PORT):
|
||||||
|
with socket.socket() as listener:
|
||||||
|
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
try:
|
||||||
|
listener.bind(('127.0.0.1', port))
|
||||||
|
except OSError as exc:
|
||||||
|
raise QAError(f'QA 端口 {port} 无法绑定,检查已有监听进程') from exc
|
||||||
|
|
||||||
|
|
||||||
|
def run_step(argv, stage, env, timeout):
|
||||||
|
report(stage, ' '.join(argv[:2]))
|
||||||
|
process = subprocess.Popen(argv, cwd=ROOT, env=env, stdin=subprocess.DEVNULL, start_new_session=True)
|
||||||
|
try:
|
||||||
|
code = process.wait(timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise QAError(f'{stage} 超过 {timeout} 秒') from exc
|
||||||
|
finally:
|
||||||
|
stop_process(process)
|
||||||
|
if code:
|
||||||
|
raise QAError(f'{stage} 失败(退出 {code}),查看上方命令输出')
|
||||||
|
|
||||||
|
|
||||||
|
def stop_process(process):
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGTERM)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
process.wait(timeout=5)
|
||||||
|
|
||||||
|
|
||||||
|
def start_service(stack, argv, env, log):
|
||||||
|
output = stack.enter_context(log.open('w', encoding='utf-8'))
|
||||||
|
process = subprocess.Popen(argv, cwd=ROOT, env=env, stdin=subprocess.DEVNULL,
|
||||||
|
stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
|
||||||
|
stack.callback(stop_process, process)
|
||||||
|
return process
|
||||||
|
|
||||||
|
|
||||||
|
def request(port, path):
|
||||||
|
connection = http.client.HTTPConnection('127.0.0.1', port, timeout=2)
|
||||||
|
try:
|
||||||
|
connection.request('GET', path)
|
||||||
|
response = connection.getresponse()
|
||||||
|
return response.status, response.getheader('Content-Type', ''), response.read(32768)
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def probe(port, path, expected):
|
||||||
|
try:
|
||||||
|
code, content_type, body = request(port, path)
|
||||||
|
if code != 200:
|
||||||
|
return False
|
||||||
|
if expected == 'html':
|
||||||
|
return 'text/html' in content_type and b'id="root"' in body
|
||||||
|
return 'application/json' in content_type and json.loads(body)['status'] == expected
|
||||||
|
except (OSError, http.client.HTTPException, ValueError, KeyError, TypeError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def wait_service(process, port, path, expected, log):
|
||||||
|
deadline = time.monotonic() + 45
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if process.poll() is not None:
|
||||||
|
raise QAError(f'服务端口 {port} 启动进程提前退出(退出 {process.returncode}),日志 {log}')
|
||||||
|
if probe(port, path, expected):
|
||||||
|
return
|
||||||
|
time.sleep(0.25)
|
||||||
|
raise QAError(f'服务探针超时 127.0.0.1:{port}{path},日志 {log}')
|
||||||
|
|
||||||
|
|
||||||
|
def qa_run(commit, env, db_env, versions, state):
|
||||||
|
with tempfile.TemporaryDirectory(prefix='run-', dir=state) as directory, ExitStack() as services:
|
||||||
|
binary = Path(directory) / 'api'
|
||||||
|
run_step(['make', 'deps-install'], 'deps', env, 300)
|
||||||
|
ldflags = f'-X knowledge-governance/internal/version.CommitSHA={commit}'
|
||||||
|
run_step(['go', 'build', '-ldflags', ldflags, '-o', str(binary), './cmd/api'], 'build-api', env, 300)
|
||||||
|
check_ports()
|
||||||
|
log_dir = Path(tempfile.mkdtemp(prefix=f'{commit[:12]}-', dir=state))
|
||||||
|
report('logs', str(log_dir))
|
||||||
|
app_env = db_env | {'PATH': env['PATH'], 'GOVERNANCE_SPOOL_ROOT': str(Path(directory) / 'spool')}
|
||||||
|
api_log = log_dir / 'api.log'
|
||||||
|
api = start_service(services, [str(binary)], app_env, api_log)
|
||||||
|
wait_service(api, API_PORT, '/readyz', 'ready', api_log)
|
||||||
|
check_migrations(versions, *database_state(db_env), complete=True)
|
||||||
|
report('migration', f'已核对全部 {len(versions)} 条迁移,最新版本 {versions[-1]}')
|
||||||
|
|
||||||
|
web_log = log_dir / 'web.log'
|
||||||
|
web = start_service(services, ['npm', '--prefix', 'web', 'run', 'dev', '--', '--host', '127.0.0.1',
|
||||||
|
'--port', str(WEB_PORT), '--strictPort'], env, web_log)
|
||||||
|
wait_service(web, WEB_PORT, '/', 'html', web_log)
|
||||||
|
wait_service(web, WEB_PORT, '/health', 'ok', web_log)
|
||||||
|
report('services', 'API /readyz、Web 首页与 Vite /health 代理通过')
|
||||||
|
run_step(['make', 'tests-go', 'tests-web'], 'tests', env, 300)
|
||||||
|
check_checkout(commit)
|
||||||
|
if api.poll() is not None or web.poll() is not None or not probe(WEB_PORT, '/health', 'ok'):
|
||||||
|
raise QAError(f'服务在测试期间失去健康状态,日志目录 {log_dir}')
|
||||||
|
report('result', f'PASS {commit},已停止本次 API 和 Web 进程')
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action', choices=('doctor', 'run'))
|
||||||
|
args = parser.parse_args()
|
||||||
|
env = tool_env()
|
||||||
|
try:
|
||||||
|
commit = os.environ.get('KG_QA_SHA', '')
|
||||||
|
if args.action == 'run':
|
||||||
|
check_checkout(commit)
|
||||||
|
check_ports()
|
||||||
|
preflight(env)
|
||||||
|
config_path = Path(os.environ.get('KG_QA_CONFIG', CONFIG)).expanduser().absolute()
|
||||||
|
config = database_config(config_path)
|
||||||
|
private_file(config_path.parent / 'qa.pgpass')
|
||||||
|
if not (config_path.parent / 'rds-ca.pem').is_file():
|
||||||
|
raise QAError(f'缺少 RDS CA 文件 {config_path.parent / "rds-ca.pem"}')
|
||||||
|
db_env = database_env(config, config_path.parent)
|
||||||
|
versions = expected_migrations()
|
||||||
|
if args.action == 'doctor':
|
||||||
|
applied, nonempty = database_state(db_env)
|
||||||
|
check_migrations(versions, applied, nonempty, complete=False)
|
||||||
|
report('database', f'连接与身份正常,已应用 {len(applied)}/{len(versions)} 条迁移')
|
||||||
|
return 0
|
||||||
|
|
||||||
|
state = Path.home() / '.local/state/knowledge-governance/qa'
|
||||||
|
state.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
with (state / 'run.lock').open('a') as lock:
|
||||||
|
try:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError as exc:
|
||||||
|
raise QAError('本机已有 QA 任务,等待当前任务退出') from exc
|
||||||
|
check_ports()
|
||||||
|
check_migrations(versions, *database_state(db_env), complete=False)
|
||||||
|
qa_run(commit, env, db_env, versions, state)
|
||||||
|
return 0
|
||||||
|
except (QAError, OSError, ValueError) as exc:
|
||||||
|
print(f'[qa:FAIL] {exc}', file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print('[qa:FAIL] QA 任务中断', file=sys.stderr)
|
||||||
|
return 130
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
def interrupt(_signal, _frame):
|
||||||
|
raise KeyboardInterrupt
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, interrupt)
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
import production_cd as cd
|
||||||
|
import hifox_release_trigger as trigger
|
||||||
|
|
||||||
|
|
||||||
|
SHA = 'a' * 40
|
||||||
|
TASK = 'd5abdf29-d0b5-4992-8073-0cb8fffd84ed'
|
||||||
|
IMAGES = {role: 'sha256:' + str(i) * 64 for i, role in enumerate(('api', 'web', 'worker'), 1)}
|
||||||
|
CTX = 'build / build (push)'
|
||||||
|
|
||||||
|
|
||||||
|
class ProductionCD(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(temp.cleanup)
|
||||||
|
self.root = Path(temp.name)
|
||||||
|
self.request = dict(sha=SHA, task=TASK, bundle='b' * 64, confirm='deploy-production',
|
||||||
|
action='deploy', run_id='42')
|
||||||
|
self.manifest = dict(schema=1, target=cd.TARGET, sha=SHA, task=TASK, migrations=[1, 2], images={})
|
||||||
|
for role, image in IMAGES.items():
|
||||||
|
archive = self.root / (role + '.tar')
|
||||||
|
archive.write_text(role)
|
||||||
|
self.manifest['images'][role] = dict(id=image, sha256=hashlib.sha256(role.encode()).hexdigest())
|
||||||
|
data = json.dumps(self.manifest).encode()
|
||||||
|
(self.root / 'release.json').write_bytes(data)
|
||||||
|
for path in self.root.iterdir():
|
||||||
|
path.chmod(0o600)
|
||||||
|
self.request['bundle'] = hashlib.sha256(data).hexdigest()
|
||||||
|
self.acceptance = {'verdict': 'passed', 'sha': SHA, 'bundle': self.request['bundle'],
|
||||||
|
'evidence': 'https://example.test/result/42'}
|
||||||
|
self.host = Mock()
|
||||||
|
self.host.task.return_value = {'id': TASK, 'spaceId': cd.SPACE, 'status': {'id': cd.READY},
|
||||||
|
'metadata': {'cd_acceptance': json.dumps(self.acceptance)}}
|
||||||
|
self.host.baseline.return_value = {'before': 'baseline'}
|
||||||
|
self.host.pre_switch.return_value = None
|
||||||
|
self.host.verify.return_value = {'sha': SHA, 'images': IMAGES, 'health': 'ok'}
|
||||||
|
self.host.receipt_path = self.root / 'receipt.json'
|
||||||
|
self.host.bundle_path.return_value = self.root
|
||||||
|
|
||||||
|
def execute(self):
|
||||||
|
return cd.release(self.host, self.request)
|
||||||
|
|
||||||
|
def assert_no_deploy(self):
|
||||||
|
self.host.load.assert_not_called()
|
||||||
|
self.host.switch.assert_not_called()
|
||||||
|
self.host.callback.assert_not_called()
|
||||||
|
|
||||||
|
def test_no_confirmation_has_zero_side_effects(self):
|
||||||
|
self.request['confirm'] = ''
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'confirmation'):
|
||||||
|
self.execute()
|
||||||
|
self.host.assert_not_called()
|
||||||
|
self.assert_no_deploy()
|
||||||
|
|
||||||
|
def test_preconditions_prevent_loading_and_switching(self):
|
||||||
|
for mode in ('approval', 'ci', 'test', 'changed_bundle', 'tampered_archive', 'migration'):
|
||||||
|
with self.subTest(mode=mode):
|
||||||
|
self.setUp()
|
||||||
|
if mode in ('approval', 'ci', 'migration'):
|
||||||
|
step = {'approval': self.host.approve, 'ci': self.host.check_ci,
|
||||||
|
'migration': self.host.pre_switch}[mode]
|
||||||
|
step.side_effect = cd.CDError(mode)
|
||||||
|
elif mode == 'test':
|
||||||
|
self.host.task.return_value['metadata'].clear()
|
||||||
|
elif mode == 'changed_bundle':
|
||||||
|
self.request['bundle'] = 'f' * 64
|
||||||
|
else:
|
||||||
|
(self.root / 'web.tar').write_bytes(b'other image')
|
||||||
|
with self.assertRaises(cd.CDError):
|
||||||
|
self.execute()
|
||||||
|
self.assert_no_deploy()
|
||||||
|
|
||||||
|
def test_bare_ci_success_cannot_replace_required_context(self):
|
||||||
|
for statuses in ([], [{'context': 'different', 'status': 'success'}],
|
||||||
|
[{'context': CTX, 'status': 'failure'}, {'context': CTX, 'status': 'success'}]):
|
||||||
|
with self.assertRaises(cd.CDError):
|
||||||
|
cd.verify_contexts({'state': 'success', 'statuses': statuses}, [CTX])
|
||||||
|
cd.verify_contexts({'state': 'success', 'statuses': [{'context': CTX, 'status': 'success'}]}, [CTX])
|
||||||
|
|
||||||
|
def test_approval_and_acceptance_rechecked_before_switch(self):
|
||||||
|
self.host.approve.side_effect = [None, cd.CDError('revoked')]
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'revoked'):
|
||||||
|
self.execute()
|
||||||
|
self.host.switch.assert_not_called()
|
||||||
|
self.host.callback.assert_not_called()
|
||||||
|
|
||||||
|
def test_success_and_writeback_retry_do_not_switch_twice(self):
|
||||||
|
self.host.callback.side_effect = [cd.CDError('HiFox timeout'), None]
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'DEPLOYED_HIFOX_PENDING'):
|
||||||
|
self.execute()
|
||||||
|
self.assertEqual(cd.read_json(self.host.receipt_path)['state'], 'deployed_hifox_pending')
|
||||||
|
self.request.update(action='writeback', confirm='retry-writeback')
|
||||||
|
self.assertEqual(self.execute()['state'], 'published')
|
||||||
|
self.host.switch.assert_called_once()
|
||||||
|
self.host.load.assert_called_once()
|
||||||
|
self.assertEqual(self.host.verify.call_count, 2)
|
||||||
|
|
||||||
|
def test_writeback_without_receipt_refuses(self):
|
||||||
|
self.request.update(action='writeback', confirm='retry-writeback')
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'receipt'):
|
||||||
|
self.execute()
|
||||||
|
self.assert_no_deploy()
|
||||||
|
|
||||||
|
def test_failed_cutover_rolls_back_and_never_marks_published(self):
|
||||||
|
for result in ('rolled_back', 'rollback_failed'):
|
||||||
|
with self.subTest(result=result):
|
||||||
|
self.setUp()
|
||||||
|
self.host.verify.side_effect = cd.CDError('new unhealthy')
|
||||||
|
if result == 'rollback_failed':
|
||||||
|
self.host.rollback.side_effect = cd.CDError('old unhealthy')
|
||||||
|
with self.assertRaisesRegex(cd.CDError, result):
|
||||||
|
self.execute()
|
||||||
|
self.assertEqual(cd.read_json(self.host.receipt_path)['state'], result)
|
||||||
|
self.host.rollback.assert_called_once()
|
||||||
|
self.host.callback.assert_not_called()
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'receipt'):
|
||||||
|
self.execute()
|
||||||
|
self.host.switch.assert_called_once()
|
||||||
|
|
||||||
|
def test_gitea_guard_log_binds_the_exact_dispatch(self):
|
||||||
|
trusted = 'f' * 40
|
||||||
|
proof = cd.guard_proof(self.request, trusted)
|
||||||
|
log = f'2026-11-22T17:31:54.013Z {proof}\n'.encode()
|
||||||
|
cd.verify_guard_log(log, self.request, trusted)
|
||||||
|
for name, altered in (('sha', 'b' * 40), ('bundle', 'c' * 64), ('run_id', '43'),
|
||||||
|
('task', 'd5abdf29-d0b5-4992-8073-0cb8fffd84ee2'), ('confirm', 'retry-writeback')):
|
||||||
|
with self.subTest(name=name), self.assertRaisesRegex(cd.CDError, 'dispatch proof'):
|
||||||
|
cd.verify_guard_log(log, {**self.request, name: altered}, trusted)
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'dispatch proof'):
|
||||||
|
cd.verify_guard_log(log + log, self.request, trusted)
|
||||||
|
|
||||||
|
def hifox_fixture(self):
|
||||||
|
task = self.host.task.return_value
|
||||||
|
report = 'ba3ced78-e648-47de-9c17-bccb8f31f36f'
|
||||||
|
event = 'a3aa1dbe-4082-4972-8e29-ed3e39e26b30'
|
||||||
|
task.update(url='https://app.hifox.com/example/task/TEST-42', statusChangedAt='2026-11-22T18:00:00Z')
|
||||||
|
self.acceptance['evidence'] = task['url'] + '#comment-' + report
|
||||||
|
task['metadata']['cd_acceptance'] = json.dumps(self.acceptance)
|
||||||
|
timeline = [dict(id=report, event='commented', createdAt='2026-11-22T17:59:00Z',
|
||||||
|
content=f'验收通过 {SHA} {self.request["bundle"]}'),
|
||||||
|
dict(id=event, event='status_changed', createdAt=task['statusChangedAt'],
|
||||||
|
actor={'type': 'human', 'id': cd.RELEASE_OWNER}, metadata={'to': '待发布'})]
|
||||||
|
return task, timeline
|
||||||
|
|
||||||
|
def test_hifox_dispatch_claims_one_transition(self):
|
||||||
|
task, timeline = self.hifox_fixture()
|
||||||
|
cd.atomic_write(self.root / 'active-since', b'2026-11-22T17:55:00Z\n')
|
||||||
|
with patch.object(trigger, 'hifox', side_effect=[task, timeline, task, timeline]), \
|
||||||
|
patch.object(trigger, 'forge', return_value={'workflow_run_id': 42}) as dispatch:
|
||||||
|
first = trigger.queue(TASK, 'unused', self.root)
|
||||||
|
replay = trigger.queue(TASK, 'unused', self.root)
|
||||||
|
self.assertEqual(first, replay)
|
||||||
|
self.assertEqual(first['run_id'], '42')
|
||||||
|
dispatch.assert_called_once_with('POST', trigger.DISPATCH, {'ref': 'main', 'inputs': first['inputs']})
|
||||||
|
self.assertEqual(first['inputs']['approval_event'], timeline[-1]['id'])
|
||||||
|
|
||||||
|
def test_unknown_dispatch_outcome_is_held(self):
|
||||||
|
task, timeline = self.hifox_fixture()
|
||||||
|
cd.atomic_write(self.root / 'active-since', b'2026-11-22T17:55:00Z\n')
|
||||||
|
with patch.object(trigger, 'hifox', side_effect=[task, timeline, task, timeline]), \
|
||||||
|
patch.object(trigger, 'forge', side_effect=TimeoutError()) as dispatch:
|
||||||
|
with self.assertRaises(TimeoutError):
|
||||||
|
trigger.queue(TASK, 'unused', self.root)
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'outcome unknown'):
|
||||||
|
trigger.queue(TASK, 'unused', self.root)
|
||||||
|
dispatch.assert_called_once()
|
||||||
|
|
||||||
|
def test_hifox_gate_requires_owner_and_existing_candidate_report(self):
|
||||||
|
for failure in ('agent', 'other human', 'mismatched report', 'updated report', 'stale transition', 'no evidence'):
|
||||||
|
with self.subTest(failure=failure):
|
||||||
|
task, timeline = self.hifox_fixture()
|
||||||
|
if failure == 'agent':
|
||||||
|
timeline[-1]['actor']['type'] = 'agent'
|
||||||
|
if failure == 'other human':
|
||||||
|
timeline[-1]['actor']['id'] = TASK
|
||||||
|
if failure == 'mismatched report':
|
||||||
|
timeline[0]['content'] = SHA
|
||||||
|
if failure == 'updated report':
|
||||||
|
timeline[0]['updatedAt'] = '2026-11-22T19:00:00Z'
|
||||||
|
if failure == 'stale transition':
|
||||||
|
task['statusChangedAt'] = '2026-11-22T19:00:00Z'
|
||||||
|
if failure == 'no evidence':
|
||||||
|
task['metadata'] = {}
|
||||||
|
with patch.object(trigger, 'hifox', side_effect=[task, timeline]), patch.object(trigger, 'forge') as forge:
|
||||||
|
with self.assertRaises(cd.CDError):
|
||||||
|
trigger.queue(TASK, 'unused', self.root)
|
||||||
|
forge.assert_not_called()
|
||||||
|
|
||||||
|
def test_auto_approval_is_checked_again_before_switch(self):
|
||||||
|
task, timeline = self.hifox_fixture()
|
||||||
|
approval, _ = cd.release_approval(task, timeline, self.request)
|
||||||
|
self.request.update(approval)
|
||||||
|
self.host.timeline.side_effect = [timeline, timeline[:-1]]
|
||||||
|
proof = cd.guard_proof(self.request, 'f' * 40)
|
||||||
|
self.assertNotEqual(proof, cd.guard_proof({**self.request, 'acceptance_digest': 'e' * 64}, 'f' * 40))
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'status history'):
|
||||||
|
self.execute()
|
||||||
|
self.host.switch.assert_not_called()
|
||||||
|
|
||||||
|
def test_target_lock_is_exclusive(self):
|
||||||
|
path = self.root / 'lock'
|
||||||
|
with cd.deploy_lock(path):
|
||||||
|
with self.assertRaisesRegex(cd.CDError, 'deployment lock'):
|
||||||
|
with cd.deploy_lock(path):
|
||||||
|
self.fail('lock was acquired twice')
|
||||||
|
|
||||||
|
def test_persisted_compose_changes_only_images(self):
|
||||||
|
original = {'services': {role: {'image': role + ':old', 'env_file': [{'path': '/kept', 'format': 'raw'}],
|
||||||
|
'volumes': ['/kept:/run/key:ro'], 'environment': {'GATE': 'true'}}
|
||||||
|
for role in IMAGES}, 'name': 'governance'}
|
||||||
|
changed = cd.with_images(original, IMAGES)
|
||||||
|
for role, new in changed['services'].items():
|
||||||
|
self.assertEqual(new.pop('image'), IMAGES[role])
|
||||||
|
old = original['services'][role].copy()
|
||||||
|
old.pop('image')
|
||||||
|
self.assertEqual(new, old)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from threading import Thread
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import qa_env
|
||||||
|
|
||||||
|
|
||||||
|
class QAEnvironmentTests(unittest.TestCase):
|
||||||
|
def test_missing_approval_config_names_file(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
config = Path(directory) / 'qa.json'
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, 'qa.json'):
|
||||||
|
qa_env.database_config(config)
|
||||||
|
|
||||||
|
def test_rejects_non_qa_database_before_connection(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
config = Path(directory) / 'qa.json'
|
||||||
|
config.write_text(json.dumps(dict(host='rds.example.test', port=5432,
|
||||||
|
database='governance', user='kg_qa_app')))
|
||||||
|
config.chmod(0o600)
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, 'kg_qa_'):
|
||||||
|
qa_env.database_config(config)
|
||||||
|
|
||||||
|
def test_database_environment_uses_only_approved_identity(self):
|
||||||
|
settings = dict(host='rds.example.test', port=5432, database='kg_qa_app', user='kg_qa_app')
|
||||||
|
with patch.dict(os.environ, {'PGHOSTADDR': '127.0.0.1', 'PGPASSWORD': 'fixture',
|
||||||
|
'PGSERVICE': 'other', 'PRODUCTION_RELEASE_REQUESTS_ENABLED': 'true'}):
|
||||||
|
env = qa_env.database_env(settings, Path('/approved'))
|
||||||
|
self.assertEqual(env['PGHOST'], 'rds.example.test')
|
||||||
|
self.assertEqual(env['PGSSLMODE'], 'verify-full')
|
||||||
|
self.assertEqual(env['PRODUCTION_RELEASE_REQUESTS_ENABLED'], 'false')
|
||||||
|
self.assertFalse({'PGHOSTADDR', 'PGPASSWORD', 'PGSERVICE'} & env.keys())
|
||||||
|
|
||||||
|
def test_empty_database_or_known_migration_prefix_can_bootstrap(self):
|
||||||
|
for applied in ([], [1], [1, 2]):
|
||||||
|
with self.subTest(applied=applied):
|
||||||
|
qa_env.check_migrations([1, 2], applied, False, complete=False)
|
||||||
|
|
||||||
|
def test_pending_migrations_block_testing(self):
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, '迁移未完成'):
|
||||||
|
qa_env.check_migrations([1, 2], [1], False, complete=True)
|
||||||
|
|
||||||
|
def test_schema_without_ledger_is_rejected(self):
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, '非空'):
|
||||||
|
qa_env.check_migrations([1, 2], [], True, complete=False)
|
||||||
|
|
||||||
|
def test_unknown_or_missing_middle_migrations_are_rejected(self):
|
||||||
|
for applied in ([1, 3], [1, 2, 3], [2]):
|
||||||
|
with self.subTest(applied=applied), self.assertRaisesRegex(qa_env.QAError, '迁移序列'):
|
||||||
|
qa_env.check_migrations([1, 2], applied, False, complete=False)
|
||||||
|
|
||||||
|
def test_hifox_exit_zero_does_not_override_offline_state(self):
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, '在线'):
|
||||||
|
qa_env.check_hifox('{"connected":true,"remoteLiveness":{"status":"offline"}}')
|
||||||
|
qa_env.check_hifox('{"connected":true,"remoteLiveness":{"status":"online"}}')
|
||||||
|
|
||||||
|
def test_health_probe_rejects_spa_fallback(self):
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header('Content-Type', 'text/html')
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b'<html><div id="root"></div></html>')
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with ThreadingHTTPServer(('127.0.0.1', 0), Handler) as server:
|
||||||
|
thread = Thread(target=server.serve_forever)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
self.assertTrue(qa_env.probe(server.server_port, '/', 'html'))
|
||||||
|
self.assertFalse(qa_env.probe(server.server_port, '/health', 'ok'))
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
thread.join()
|
||||||
|
|
||||||
|
def test_database_identity_mismatch_stops_before_reading_migrations(self):
|
||||||
|
env = {'PGDATABASE': 'kg_qa_app', 'PGUSER': 'kg_qa_app'}
|
||||||
|
with patch.object(qa_env, 'psql', return_value='other|kg_qa_app|t|t|t|t|t') as probe:
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, '实际库名'):
|
||||||
|
qa_env.database_state(env)
|
||||||
|
self.assertEqual(probe.call_count, 1)
|
||||||
|
|
||||||
|
def test_full_sha_mismatch_rejects_current_checkout(self):
|
||||||
|
full_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=qa_env.ROOT, text=True).strip()
|
||||||
|
wrong = ('0' if full_sha[0] != '0' else '1') + full_sha[1:]
|
||||||
|
with self.assertRaisesRegex(qa_env.QAError, '提交不一致'):
|
||||||
|
qa_env.check_checkout(wrong)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user