From 07a0ee2644e4a94fd4b3c7674e00c2c1299dce25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 23 Nov 2026 02:24:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(deploy):=20=E4=BD=BF=E7=94=A8=20Gitea=20?= =?UTF-8?q?=E5=8D=95=E6=AC=A1=E4=BA=BA=E5=B7=A5=E7=A1=AE=E8=AE=A4=E5=8F=91?= =?UTF-8?q?=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/deploy-production.yml | 3 - docs/PRODUCTION_CD_RUNBOOK.md | 16 ++--- scripts/production_cd.py | 92 +++++++++++++++----------- scripts/test_production_cd.py | 18 ++++- 4 files changed, 74 insertions(+), 55 deletions(-) diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml index a6ce48a..3aafd88 100644 --- a/.gitea/workflows/deploy-production.yml +++ b/.gitea/workflows/deploy-production.yml @@ -19,9 +19,6 @@ on: confirm: description: deploy-production 或 retry-writeback required: true - approval: - description: 首次部署的发布人签名(Base64);仅补回写时留空 - required: false concurrency: group: cloud-test-governance-main-production diff --git a/docs/PRODUCTION_CD_RUNBOOK.md b/docs/PRODUCTION_CD_RUNBOOK.md index d74c0e1..3c83d9d 100644 --- a/docs/PRODUCTION_CD_RUNBOOK.md +++ b/docs/PRODUCTION_CD_RUNBOOK.md @@ -28,29 +28,21 @@ ## 人工确认并自动部署 -系统模板的末端为人工切换。此项目按已确认的流程由发布人签署精确的提交、任务和归档摘要,发起 Gitea workflow 后由控制器完成自动切换、核验和回写。 +在 Gitea Actions 打开 `deploy-production` 并运行工作流。工作流分支选择 `main`,填写 `action=deploy`、已验收的完整 `sha`、归档清单摘要 `bundle`、HiFox 任务 UUID `task` 和 `confirm=deploy-production`。发布人用自己的 Gitea 账号提交一次确认。 -发布人用受信任的 OpenSSH 签名密钥签署准确候选。签名只覆盖本仓、本目标和本次验收身份。`KG_SIGNING_KEY` 是发布人本地密钥路径,可由 SSH agent 提供;签名输出供发起 workflow 使用。 - -```bash -python3 scripts/production_cd.py approval-message --sha "$SHA" --task "$TASK_UUID" --bundle "$BUNDLE_SHA" \ - | ssh-keygen -Y sign -f "$KG_SIGNING_KEY" -n knowledge-governance-release-v1 \ - | base64 -w0 -``` - -Gitea Actions → `deploy-production` → Run workflow,选择 `main`,填写 `action=deploy`、`sha`、`bundle`、`task`、`confirm=deploy-production` 和签名 `approval`。主站侧复核发起者及运行、签名、提交在 `main` 的可达性、必需 CI 状态、HiFox 验收记录、三份归档及实际数据库账本,确认线上镜像提交是候选的祖先。切换复用 `scripts/deploy-api-web.sh`,仅更换三项 Compose `image`;保留原配置字节以备回滚。切换后自动读回三镜像、API 二进制 SHA、主站健康、API readiness 及实际提供的前端资源。HiFox 回写包含原始运行 URL、完整提交、摘要、镜像 ID 及核验证据。 +主站侧从 Gitea 读取运行、操作者和通过的校验作业,并验证输入摘要及工作流代码。候选代码可来自仓库内的任务分支,以实际验收 SHA 为准。切换前还核验必需 CI 状态、HiFox 验收记录、三份归档、线上源代码血缘及数据库迁移账本。切换复用 `scripts/deploy-api-web.sh`,仅更换三项 Compose `image`;保留原配置字节以备回滚。切换后自动读回三镜像、API 二进制 SHA、主站健康、API readiness 及实际提供的前端资源。HiFox 回写包含原始运行 URL、完整提交、摘要、镜像 ID 及核验证据。 目标机已有的 `schema_migrations` 必须与候选完全相同,API 与 worker 的库连接均须核对。应用会在启动时检查迁移。新增或改变的迁移先按独立流程审查、备份、执行并复验;CD 自身只读迁移账本。发布时运营方应保留 prior tag,保障用原 Compose 字节回滚。 ## 失败恢复 - `rolled_back` 表示自动恢复了原 Compose、旧的镜像 ID、API SHA 与站点健康。`rollback_failed` 或 `cutover_started` 要求巡检现场、按回执恢复并复验。目标机互斥锁覆盖切换及回写。Gitea 的取消记录只能说明流水线结束,回滚以目标机回执和运行时读回为准。 -- `DEPLOYED_HIFOX_PENDING` 表示镜像切换和核验已经完成。以同一 `sha`、`bundle`、`task` 人工发起 `action=writeback`、`confirm=retry-writeback`,`approval` 留空。脚本先读原目标回执并复验仍在运行的版本,随后补写 HiFox,保留原部署链接。状态与 metadata 的当前值以 HiFox 读回为准。重复调用核对记录后结束。 +- `DEPLOYED_HIFOX_PENDING` 表示镜像切换和核验已经完成。以同一 `sha`、`bundle`、`task` 人工发起 `action=writeback`、`confirm=retry-writeback`。脚本先读原目标回执并复验仍在运行的版本,随后补写 HiFox,保留原部署链接。状态与 metadata 的当前值以 HiFox 读回为准。重复调用核对记录后结束。 - 目标回执 `/opt/knowledge-governance/cd/receipts/.json`,镜像切换前的 Compose 保存为同名 `.compose.backup`。SSH 超时或强制中断后,先读回执和当前运行态。已存在未决或失败回执时,重复部署被拒。保留原回执作为事故证据。 ## 一次性配置与权限 - 目标机安装仓库 `scripts/{production_cd.py,cd_migration_probe.py,deploy-api-web.sh,production-cd-ssh.sh}` 到 `/opt/knowledge-governance/cd/bin/`。创建私有 `receipts/`、`bundles/`、`hifox-home/`,初始化 `source.git` 为 bare Git 仓,并将 `.gitea/required-contexts.txt` 安装到该目录。仅发布运维可更新这些文件。 -- 把发布人的**公钥**登记在 `allowed_signers`,principal 为 `release-owner`。目标端 `authorized_keys` 为 CI 单独公钥设置 `restrict,command="/opt/knowledge-governance/cd/bin/production-cd-ssh.sh"`。校验云主机 SSH host key 后将其按 `kg-governance-cd` 别名写入 `KG_CD_KNOWN_HOSTS`。对应独立 SSH 私钥保存在 `KG_CD_SSH_KEY`。Runner 标签为 `kg-production-cd`,使用 `KG_CD_SSH_HOST` 和 `KG_CD_SSH_PORT` 接入批准的 Tailscale 或本机 IPv6 隧道。 +- 填写目标机 `/opt/knowledge-governance/cd/deploy-production.yml`,内容与 Gitea 主分支的工作流逐字节相同。控制器用 Gitea 发布人账号 ID 和只读日志验证确认。目标端 `authorized_keys` 为 CI 单独公钥设置 `restrict,command="/opt/knowledge-governance/cd/bin/production-cd-ssh.sh"`。校验云主机 SSH host key 后将其按 `kg-governance-cd` 别名写入 `KG_CD_KNOWN_HOSTS`。对应独立 SSH 私钥保存在 `KG_CD_SSH_KEY`。Runner 标签为 `kg-production-cd`,使用 `KG_CD_SSH_HOST` 和 `KG_CD_SSH_PORT` 接入批准的 Tailscale 或本机 IPv6 隧道。 - 将受控 HiFox CLI 安装至 `bin/hifox`,由账号持有人在目标主机使用 `hifox-home` 对应的私有配置目录完成登录。身份须能读取指定空间的任务、写 `cd_release` metadata 并完成对应状态迁移。状态绑定前分别读回组织状态和实际任务类型,复用原有待发布、已发布 ID,保留其余工作流配置。 - 常规 push CI 负责验证和静态预览。这个 workflow 的唯一事件是 `workflow_dispatch`。`make verify` 的 `contracts` 子项运行 CD 用例。真实切换的验收要求对应候选、现场前置条件和发布人确认。 diff --git a/scripts/production_cd.py b/scripts/production_cd.py index 22ef948..71ee4bd 100644 --- a/scripts/production_cd.py +++ b/scripts/production_cd.py @@ -2,8 +2,6 @@ """Governance application CD. The installed host copy owns the release boundary.""" import argparse -import base64 -import binascii from contextlib import contextmanager import copy from datetime import datetime, timezone @@ -40,7 +38,7 @@ SHA = re.compile('[0-9a-f]{40}') HASH = re.compile('[0-9a-f]{64}') IMAGE = re.compile('sha256:[0-9a-f]{64}') UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}') -NAMESPACE = 'knowledge-governance-release-v1' +GUARD_PREFIX = 'KG_CD_CONFIRM_V1 ' class CDError(RuntimeError): @@ -108,13 +106,21 @@ def validate_request(request): require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task')) and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID') require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID') - approval = request.get('approval') - require(isinstance(approval, str) and len(approval) <= 4096 and (action == 'writeback' or approval), - 'missing or invalid release approval') -def approval_message(request): - return (f"{NAMESPACE}\n{REPO}\n{TARGET}\n{request['sha']}\n{request['bundle']}\n{request['task']}\n").encode() +def guard_proof(request, control_sha): + require(matches(SHA, control_sha), 'invalid workflow source SHA') + binding = {key: request[key] for key in ('run_id', 'action', 'confirm', 'sha', 'bundle', 'task')} + binding.update(repository=REPO, target=TARGET, workflow_sha=control_sha) + serialized = json.dumps(binding, sort_keys=True, separators=(',', ':')).encode() + return GUARD_PREFIX + hashlib.sha256(serialized).hexdigest() + + +def verify_guard_log(log, request, control_sha): + expected = guard_proof(request, control_sha) + pattern = re.compile(r'\d{4}-\d{2}-\d{2}T[0-9:.]+Z (' + GUARD_PREFIX + r'[0-9a-f]{64})') + proofs = [match.group(1) for line in log.decode().splitlines() if (match := pattern.fullmatch(line))] + require(proofs == [expected], 'Gitea guard dispatch proof missing or different') def hash_file(path): @@ -184,6 +190,7 @@ def release(host, request): 'writeback requires a verified deployment receipt') require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch') manifest = verify_bundle(host.bundle_path(request), request, archives=False) + host.approve(request) host.verify(manifest) return writeback(host, request, existing) @@ -370,28 +377,44 @@ class LiveHost: return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90) def approve(self, request): - try: - signature = base64.b64decode(request['approval'], validate=True) - except (binascii.Error, ValueError) as exc: - raise CDError('approval signature is invalid') from exc - with tempfile.TemporaryDirectory(dir=DATA) as directory: - signature_file = Path(directory) / 'approval.sig' - signature_file.write_bytes(signature) - run(['ssh-keygen', '-Y', 'verify', '-f', str(DATA / 'allowed_signers'), '-I', 'release-owner', - '-n', NAMESPACE, '-s', str(signature_file)], 'release approval', input_data=approval_message(request)) - - def check_dispatch_run(self, request): - path = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{request["run_id"]}' - dispatch = json.loads(http(path)) - require(str(dispatch.get('id')) == request['run_id'] and dispatch.get('event') == 'workflow_dispatch' + repo_url = f'{FORGE}/{REPO}' + run_id = request['run_id'] + run_api = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{run_id}' + dispatch = json.loads(http(run_api)) + control_sha = dispatch.get('head_sha') + trigger_actor = dispatch.get('trigger_actor') or dispatch.get('actor') or {} + require(str(dispatch.get('id')) == run_id and dispatch.get('event') == 'workflow_dispatch' and dispatch.get('path') == 'deploy-production.yml@refs/heads/main' and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running') - and (dispatch.get('actor') or {}).get('id') == 1, - 'Gitea run is not an active deployment dispatched by the release owner') + and (dispatch.get('actor') or {}).get('id') == 1 and trigger_actor.get('id') == 1 + and matches(SHA, control_sha), 'Gitea run is not an active deployment dispatched by the release owner') + # Repo secrets can be read by other workflows. Only a completed guard from + # the installed workflow and controller may authorize the exact inputs. + try: + self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main') + except CDError: + self.fetch_source() + self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main') + trusted = {'scripts/production_cd.py': Path(__file__).resolve(), + '.gitea/workflows/deploy-production.yml': DATA / 'deploy-production.yml'} + for name, local in trusted.items(): + require(self.git('show', f'{control_sha}:{name}') == immutable_file(local).read_bytes(), + 'Gitea approval workflow differs from installed control files') + jobs = json.loads(http(run_api + '/jobs')).get('jobs') or [] + guards = [job for job in jobs if job.get('name') == 'guard' and job.get('conclusion') == 'success' + and str(job.get('run_id')) == run_id and job.get('head_sha') == control_sha + and matches(re.compile('[1-9][0-9]{0,14}'), str(job.get('id')))] + require(len(guards) == 1, 'Gitea approval guard has not succeeded') + guard_id = guards[0]['id'] + verify_guard_log(http(f'{repo_url}/actions/runs/{run_id}/jobs/{guard_id}/logs'), request, control_sha) + + def fetch_source(self): + self.git('fetch', '--no-tags', '--prune', f'{FORGE}/{REPO}.git', '+refs/heads/*:refs/remotes/origin/*') def check_ci(self, request): - self.git('fetch', '--no-tags', f'{FORGE}/{REPO}.git', '+refs/heads/main:refs/remotes/origin/main') - self.git('merge-base', '--is-ancestor', request['sha'], 'refs/remotes/origin/main') + self.fetch_source() + refs = self.git('for-each-ref', '--format=%(refname)', '--contains', request['sha'], 'refs/remotes/origin/') + require(refs.strip(), 'candidate commit is not reachable from a repository branch') contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines() verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')), [line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')]) @@ -573,7 +596,7 @@ def event_request(event, env): require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main' and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required') inputs = event.get('inputs') or {} - request = {name: inputs.get(name, '') for name in ('action', 'sha', 'task', 'bundle', 'confirm', 'approval')} + request = {name: inputs.get(name, '') for name in ('action', 'sha', 'task', 'bundle', 'confirm')} request['run_id'] = env.get('GITHUB_RUN_ID', '') validate_request(request) return request @@ -636,15 +659,13 @@ def attest(args): def cli(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) - for command in ('guard', 'send', 'approval-message', 'serve', 'pack', 'attest'): + for command in ('guard', 'send', 'serve', 'pack', 'attest'): sub = commands.add_parser(command) if command in ('guard', 'send'): sub.add_argument('--event', type=Path, required=True) - if command in ('approval-message', 'pack', 'attest'): + if command in ('pack', 'attest'): sub.add_argument('--sha', required=True) sub.add_argument('--task', required=True) - if command == 'approval-message': - sub.add_argument('--bundle', required=True) if command in ('pack', 'attest'): for role in ROLES: sub.add_argument('--' + role, required=True) @@ -657,12 +678,9 @@ def cli(): if args.command in ('guard', 'send'): request = event_request(read_json(args.event), os.environ) if args.command == 'guard': - print('dispatch parameters verified') + print(guard_proof(request, os.environ.get('GITHUB_SHA'))) else: send(request) - elif args.command == 'approval-message': - require(matches(SHA, args.sha) and matches(UUID, args.task) and matches(HASH, args.bundle), 'invalid approval binding') - sys.stdout.buffer.write(approval_message(vars(args))) elif args.command == 'pack': pack(args) elif args.command == 'attest': @@ -677,7 +695,6 @@ def cli(): for item in (DATA / 'receipts').glob('*.json'): if read_json(item)['state'] in ('cutover_started', 'rollback_failed'): raise CDError('previous deployment outcome requires operator reconciliation') - host.check_dispatch_run(request) result = release(host, request) print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False)) @@ -705,7 +722,8 @@ def send(request): if result.returncode: error = result.stderr.decode(errors='replace') for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:', - 'target preflight rejected', 'deployment lock busy'): + 'target preflight rejected', 'deployment lock busy', 'writeback requires a verified deployment receipt', + 'Gitea approval guard has not succeeded', 'Gitea guard dispatch proof missing or different'): if marker in error: raise CDError(error.strip().splitlines()[-1]) raise CDError('target rejected request or outcome unknown; inspect durable receipt') diff --git a/scripts/test_production_cd.py b/scripts/test_production_cd.py index e8f7416..628e368 100644 --- a/scripts/test_production_cd.py +++ b/scripts/test_production_cd.py @@ -20,7 +20,7 @@ class ProductionCD(unittest.TestCase): self.addCleanup(temp.cleanup) self.root = Path(temp.name) self.request = dict(sha=SHA, task=TASK, bundle='b' * 64, confirm='deploy-production', - action='deploy', approval='signed', run_id='42') + action='deploy', run_id='42') self.manifest = dict(schema=1, target=cd.TARGET, sha=SHA, task=TASK, migrations=[1, 2], images={}) for role, image in IMAGES.items(): archive = self.root / (role + '.tar') @@ -94,14 +94,14 @@ class ProductionCD(unittest.TestCase): with self.assertRaisesRegex(cd.CDError, 'DEPLOYED_HIFOX_PENDING'): self.execute() self.assertEqual(cd.read_json(self.host.receipt_path)['state'], 'deployed_hifox_pending') - self.request.update(action='writeback', confirm='retry-writeback', approval='') + self.request.update(action='writeback', confirm='retry-writeback') self.assertEqual(self.execute()['state'], 'published') self.host.switch.assert_called_once() self.host.load.assert_called_once() self.assertEqual(self.host.verify.call_count, 2) def test_writeback_without_receipt_refuses(self): - self.request.update(action='writeback', confirm='retry-writeback', approval='') + self.request.update(action='writeback', confirm='retry-writeback') with self.assertRaisesRegex(cd.CDError, 'receipt'): self.execute() self.assert_no_deploy() @@ -122,6 +122,18 @@ class ProductionCD(unittest.TestCase): self.execute() self.host.switch.assert_called_once() + def test_gitea_guard_log_binds_the_exact_dispatch(self): + trusted = 'f' * 40 + proof = cd.guard_proof(self.request, trusted) + log = f'2026-11-22T17:31:54.013Z {proof}\n'.encode() + cd.verify_guard_log(log, self.request, trusted) + for name, altered in (('sha', 'b' * 40), ('bundle', 'c' * 64), ('run_id', '43'), + ('task', 'd5abdf29-d0b5-4992-8073-0cb8fffd84ee2'), ('confirm', 'retry-writeback')): + with self.subTest(name=name), self.assertRaisesRegex(cd.CDError, 'dispatch proof'): + cd.verify_guard_log(log, {**self.request, name: altered}, trusted) + with self.assertRaisesRegex(cd.CDError, 'dispatch proof'): + cd.verify_guard_log(log + log, self.request, trusted) + def test_target_lock_is_exclusive(self): path = self.root / 'lock' with cd.deploy_lock(path):