#!/usr/bin/env python3 """Governance application CD. The installed host copy owns the release boundary.""" import argparse from contextlib import contextmanager import copy from datetime import datetime, timezone import fcntl import hashlib from html.parser import HTMLParser import json import os from pathlib import Path import re import stat import subprocess import sys import tarfile import tempfile import time from urllib.error import URLError from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, ProxyHandler, build_opener REPO = 'Weknora/knowledge-governance' FORGE = 'https://gitea.zheromiku.top' TARGET = 'cloud-test-governance-main' ORG = '169e6f7f-e508-4a1f-9a9b-63d240439a3d' SPACE = '106e05d1-4091-40b9-8c57-f76dc94f7825' READY = '3e15556f-ca19-417c-8bfd-666634ea5d29' PUBLISHED = '16185166-b8e9-4761-9926-845a1f41c96d' RELEASE_OWNER = '39ae59fb-876c-4587-ab5d-a12a841c6c09' LOGIN_URL = 'http://100.85.111.121:8088/' ROLES = ('api', 'web', 'worker') REVISION = 'org.opencontainers.image.revision' DATA = Path('/opt/knowledge-governance/cd') APP = DATA.parent PROJECT = 'knowledge-governance-cloud-test' SHA = re.compile('[0-9a-f]{40}') HASH = re.compile('[0-9a-f]{64}') IMAGE = re.compile('sha256:[0-9a-f]{64}') UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}') GUARD_PREFIX = 'KG_CD_CONFIRM_V1 ' class CDError(RuntimeError): pass def require(ok, message): if not ok: raise CDError(message) def stamp(): return datetime.now(timezone.utc).isoformat() def read_json(path): return json.loads(path.read_bytes()) def sync_dir(directory): fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(fd) finally: os.close(fd) def atomic_write(path, data, mode=0o600): fd, temp = tempfile.mkstemp(dir=path.parent) try: with os.fdopen(fd, 'wb') as out: os.fchmod(out.fileno(), mode) out.write(data) out.flush() os.fsync(out.fileno()) os.replace(temp, path) sync_dir(path.parent) finally: Path(temp).unlink(missing_ok=True) def record(path, value): atomic_write(path, (json.dumps(value, ensure_ascii=False, sort_keys=True) + '\n').encode()) @contextmanager def deploy_lock(path): fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'r+') as handle: try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: raise CDError('deployment lock busy') from exc yield def matches(pattern, text): return isinstance(text, str) and pattern.fullmatch(text) is not None def validate_request(request): actions = {'deploy': 'deploy-production', 'writeback': 'retry-writeback'} action = request.get('action') require(action in actions and request.get('confirm') == actions[action], 'explicit confirmation required') require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task')) and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID') require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID') event, digest = request.get('approval_event', ''), request.get('acceptance_digest', '') require((event == '' and digest == '') or (action == 'deploy' and matches(UUID, event) and matches(HASH, digest)), 'invalid HiFox approval binding') def guard_proof(request, control_sha): require(matches(SHA, control_sha), 'invalid workflow source SHA') binding = {key: request[key] for key in ('run_id', 'action', 'confirm', 'sha', 'bundle', 'task')} if request.get('approval_event'): binding.update(approval_event=request['approval_event'], acceptance_digest=request['acceptance_digest']) binding.update(repository=REPO, target=TARGET, workflow_sha=control_sha) serialized = json.dumps(binding, sort_keys=True, separators=(',', ':')).encode() return GUARD_PREFIX + hashlib.sha256(serialized).hexdigest() def verify_guard_log(log, request, control_sha): expected = guard_proof(request, control_sha) pattern = re.compile(r'\d{4}-\d{2}-\d{2}T[0-9:.]+Z (' + GUARD_PREFIX + r'[0-9a-f]{64})') proofs = [match.group(1) for line in log.decode().splitlines() if (match := pattern.fullmatch(line))] require(proofs == [expected], 'Gitea guard dispatch proof missing or different') def hash_file(path): with path.open('rb') as src: return hashlib.file_digest(src, 'sha256').hexdigest() def immutable_file(path): info = path.lstat() require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and not info.st_mode & 0o022, 'bundle requires regular files owned by the deployment account') return path def verify_bundle(directory, request, archives=True): raw = immutable_file(directory / 'release.json').read_bytes() require(hashlib.sha256(raw).hexdigest() == request['bundle'], 'release manifest digest mismatch') manifest = json.loads(raw) require(manifest.get('schema') == 1 and manifest.get('target') == TARGET and manifest.get('sha') == request['sha'] and manifest.get('task') == request['task'], 'bundle commit, task or target mismatch') require(set(manifest.get('images', {})) == set(ROLES), 'bundle must include API, web and worker') for role, image in manifest['images'].items(): require(matches(IMAGE, image.get('id')) and matches(HASH, image.get('sha256')), 'invalid image record') if archives: archive = immutable_file(directory / (role + '.tar')) require(hash_file(archive) == image['sha256'], f'{role} archive checksum mismatch') versions = manifest.get('migrations') require(isinstance(versions, list) and versions and all(type(n) is int and n > 0 for n in versions) and versions == sorted(set(versions)), 'invalid migration ledger in manifest') return manifest def verify_contexts(result, contexts): latest = {} for item in result.get('statuses') or []: latest.setdefault(item.get('context'), item.get('status')) require(bool(contexts) and all(latest.get(context) == 'success' for context in contexts), 'required CI status is missing or unsuccessful') def metadata(task, key): value = (task.get('metadata') or {}).get(key) return json.loads(value) if isinstance(value, str) else None def check_task(task, request, receipt=None): require(task.get('id') == request['task'] and task.get('spaceId') == SPACE, 'HiFox task or space mismatch') status = (task.get('status') or {}).get('id') same_publication = receipt and metadata(task, 'cd_release') == receipt.get('hifox') require(status == READY or (status == PUBLISHED and same_publication), 'HiFox task is not ready for release') evidence = metadata(task, 'cd_acceptance') or {} url = urlsplit(evidence.get('evidence', '')) require(evidence.get('verdict') == 'passed' and evidence.get('sha') == request['sha'] and evidence.get('bundle') == request['bundle'] and url.scheme == 'https' and url.hostname and not (url.username or url.password or url.query) and (not url.fragment or (url.netloc == 'app.hifox.com' and url.fragment.startswith('comment-') and matches(UUID, url.fragment.removeprefix('comment-')))), 'missing or changed business acceptance for this commit and bundle') return evidence def acceptance_digest(acceptance): return hashlib.sha256(json.dumps(acceptance, sort_keys=True, separators=(',', ':')).encode()).hexdigest() def release_approval(task, timeline, request): evidence = check_task(task, request) changes = [row for row in timeline if row.get('event') == 'status_changed'] require(bool(changes), 'missing HiFox status history') approval = changes[-1] approver = approval.get('actor') or {} require((approval.get('metadata') or {}).get('to') == '待发布' and approval.get('createdAt') and approval['createdAt'] == task.get('statusChangedAt') and approver.get('id') == RELEASE_OWNER and approver.get('type') == 'human' and matches(UUID, approval.get('id')), 'the latest transition must be an approval by the release owner') task_url = task.get('url', '') prefix = task_url + '#comment-' require(task_url.startswith('https://app.hifox.com/') and evidence['evidence'].startswith(prefix), 'automated release requires the same-task acceptance report') comment_id = evidence['evidence'].removeprefix(prefix) require(matches(UUID, comment_id), 'invalid acceptance report ID') reports = [row for row in timeline if row.get('event') == 'commented' and row.get('id') == comment_id] require(len(reports) == 1, 'acceptance report missing from task history') report = reports[0] body = report.get('content') or '' require(report.get('createdAt') and (report.get('updatedAt') or report['createdAt']) < approval['createdAt'] and request['sha'] in body and request['bundle'] in body, 'the unchanged report must identify the candidate and predate approval') binding = {'approval_event': approval['id'], 'acceptance_digest': acceptance_digest(evidence)} require(all(request.get(key, value) == value for key, value in binding.items()), 'HiFox approval or acceptance changed after dispatch') return binding, approval['createdAt'] def accepted_task(host, request): task = host.task(request['task']) if request.get('approval_event'): release_approval(task, host.timeline(request['task']), request) return check_task(task, request) def release(host, request): validate_request(request) path = host.receipt_path existing = read_json(path) if path.exists() else None if request['action'] == 'writeback': require(existing and existing['state'] in ('deployed_hifox_pending', 'published'), 'writeback requires a verified deployment receipt') require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch') manifest = verify_bundle(host.bundle_path(request), request, archives=False) host.approve(request) host.verify(manifest) return writeback(host, request, existing) require(existing is None, 'existing release receipt: use writeback or inspect interrupted/failed deployment') host.approve(request) manifest = verify_bundle(host.bundle_path(request), request) host.check_ci(request) accepted = accepted_task(host, request) before = host.baseline() host.pre_switch(before, manifest) receipt = {k: request[k] for k in ('sha', 'task', 'bundle', 'run_id')} receipt.update(state='preparing', target=TARGET, started_at=stamp(), previous=before) record(path, receipt) try: host.load(host.bundle_path(request), manifest) host.pre_switch(before, manifest) host.approve(request) require(accepted_task(host, request) == accepted, 'business acceptance changed before switch') except (CDError, OSError, ValueError, KeyError) as exc: receipt.update(state='failed_before_cutover') record(path, receipt) raise CDError(f'failed_before_cutover: {exc}' if isinstance(exc, CDError) else 'failed_before_cutover') from exc receipt.update(state='cutover_started') record(path, receipt) try: host.switch(before, manifest) evidence = host.verify(manifest) except (CDError, OSError, ValueError, KeyError) as exc: status = 'rollback_failed' try: host.rollback(before) status = 'rolled_back' except (CDError, OSError, ValueError, KeyError): pass receipt.update(state=status, finished_at=stamp(), failure=str(exc) if isinstance(exc, CDError) else 'operation failed') record(path, receipt) raise CDError(f'{status}: deployment verification failed') from exc receipt.update(state='deployed_hifox_pending', deployed_at=stamp(), evidence=evidence) receipt['hifox'] = {**evidence, 'target': TARGET, 'bundle': request['bundle'], 'deployed_at': receipt['deployed_at'], 'login_url': LOGIN_URL, 'run_url': f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'} record(path, receipt) return writeback(host, request, receipt) def writeback(host, request, receipt): try: task = host.task(request['task']) check_task(task, request, receipt) host.callback(request['task'], task, receipt.get('hifox')) receipt.update(state='published') record(host.receipt_path, receipt) except (CDError, OSError, ValueError, KeyError) as exc: raise CDError('DEPLOYED_HIFOX_PENDING: retry writeback for this receipt') from exc return receipt def with_images(config, images): result = copy.deepcopy(config) require(set(result['services']) == set(ROLES), 'unexpected Compose service set') for role in ROLES: result['services'][role]['image'] = images[role] return result def run(argv, purpose, *, cwd=None, input_data=None, timeout=30, allowed=(0,), env=None): try: result = subprocess.run(argv, cwd=cwd, input=input_data, capture_output=True, timeout=timeout, env=env) except (OSError, subprocess.TimeoutExpired) as exc: raise CDError(f'{purpose} unavailable or timed out') from exc require(result.returncode in allowed, f'{purpose} failed (exit {result.returncode})') return result.stdout class NoRedirect(HTTPRedirectHandler): def redirect_request(self, *_args, **_kwargs): raise CDError('unexpected HTTP redirect') def http(url, *, direct=False, limit=2**20): handlers = [NoRedirect()] if direct: handlers.append(ProxyHandler({})) try: with build_opener(*handlers).open(url, timeout=10) as response: data = response.read(limit + 1) require(len(data) <= limit, 'HTTP response exceeds limit') return data except (URLError, TimeoutError) as exc: raise CDError('HTTP read failed') from exc class Assets(HTMLParser): def __init__(self): super().__init__() self.paths = [] def handle_starttag(self, tag, attrs): attrs = dict(attrs) path = attrs.get('src' if tag == 'script' else 'href', '') if path and re.fullmatch(r'/assets/[A-Za-z0-9_.-]+\.(?:js|css)', path): self.paths.append(path) def disk_assets(html): parser = Assets() parser.feed(html.decode()) require(any(p.endswith('.js') for p in parser.paths), 'web index has no compiled entry point') return sorted(set(parser.paths)) def image_inspect(image): return json.loads(run(['docker', 'image', 'inspect', image], 'read image identity'))[0] def inspect(name): obj = json.loads(run(['docker', 'container', 'inspect', name], 'read running container'))[0] require(obj['State']['Status'] == 'running' and matches(IMAGE, obj.get('Image')), 'expected container is not running') return obj def image_label(image): return image_inspect(image)['Config']['Labels'].get(REVISION, '') def api_identity(container): raw = run(['docker', 'exec', container, '/app/api', '--version-record-status'], 'read API build identity', allowed=(0, 1)) obj = json.loads(raw) require(matches(SHA, obj.get('commit_sha')), 'API binary has no full build identity') return obj['commit_sha'] def migration_files(git, sha): paths = git('ls-tree', '-r', '--name-only', sha, 'internal/platform/postgres/migrations').decode().splitlines() return sorted(int(Path(line).name.split('_', 1)[0]) for line in paths if line.endswith('.up.sql')) def ledger(container): probe = Path(__file__).with_name('cd_migration_probe.py').read_text() data = run(['docker', 'exec', container, 'python3', '-c', probe], 'read migration ledger') result = json.loads(data) require(isinstance(result, list) and result, 'unexpected migration ledger result') return result def verify_archive(archive, expected): # docker save by ID omits mutable RepoTags. Refuse archives that could retag another app. with tarfile.open(archive, 'r:') as t: entry = t.getmember('manifest.json') require(entry.isfile() and entry.size < 65536, 'unexpected Docker archive manifest') containers = json.load(t.extractfile(entry)) require(len(containers) == 1 and not containers[0].get('RepoTags'), 'archive must contain one untagged image') config = t.getmember(containers[0]['Config']) require(config.isfile() and config.size < 2**20, 'unexpected Docker image config') require('sha256:' + hashlib.sha256(t.extractfile(config).read()).hexdigest() == expected, 'archive config does not match approved image ID') for member in t: require((member.isdir() or member.isfile()) and not member.name.startswith('/') and '..' not in Path(member.name).parts, 'unexpected Docker archive path') if 'index.json' in t.getnames(): index = t.getmember('index.json') require(index.isfile() and index.size < 65536, 'unexpected OCI archive index') oci = json.load(t.extractfile(index)) require(len(oci.get('manifests', [])) == 1 and all( key not in descriptor.get('annotations', {}) for descriptor in oci['manifests'] for key in ('org.opencontainers.image.ref.name', 'io.containerd.image.name')), 'archive OCI index contains a mutable name') class LiveHost: def __init__(self, request): self.receipt_path = DATA / 'receipts' / (request['bundle'] + '.json') self.compose = APP / 'compose.json' self.git_path = DATA / 'source.git' self.containers = {role: 'kg-cloud-' + role for role in ROLES} self.health_url = 'http://127.0.0.1:8088' self.request = request def bundle_path(self, request): return DATA / 'bundles' / request['bundle'] def git(self, *args): return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90) def approve(self, request): repo_url = f'{FORGE}/{REPO}' run_id = request['run_id'] run_api = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{run_id}' dispatch = json.loads(http(run_api)) control_sha = dispatch.get('head_sha') trigger_actor = dispatch.get('trigger_actor') or dispatch.get('actor') or {} require(str(dispatch.get('id')) == run_id and dispatch.get('event') == 'workflow_dispatch' and dispatch.get('path') == 'deploy-production.yml@refs/heads/main' and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running') and (dispatch.get('actor') or {}).get('id') == 1 and trigger_actor.get('id') == 1 and matches(SHA, control_sha), 'Gitea run is not an active deployment dispatched by the release owner') # Repo secrets can be read by other workflows. Only a completed guard from # the installed workflow and controller may authorize the exact inputs. try: self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main') except CDError: self.fetch_source() self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main') trusted = {'scripts/production_cd.py': Path(__file__).resolve(), '.gitea/workflows/deploy-production.yml': DATA / 'deploy-production.yml'} for name, local in trusted.items(): require(self.git('show', f'{control_sha}:{name}') == immutable_file(local).read_bytes(), 'Gitea approval workflow differs from installed control files') jobs = json.loads(http(run_api + '/jobs')).get('jobs') or [] guards = [job for job in jobs if job.get('name') == 'guard' and job.get('conclusion') == 'success' and str(job.get('run_id')) == run_id and job.get('head_sha') == control_sha and matches(re.compile('[1-9][0-9]{0,14}'), str(job.get('id')))] require(len(guards) == 1, 'Gitea approval guard has not succeeded') guard_id = guards[0]['id'] verify_guard_log(http(f'{repo_url}/actions/runs/{run_id}/jobs/{guard_id}/logs'), request, control_sha) def fetch_source(self): self.git('fetch', '--no-tags', '--prune', f'{FORGE}/{REPO}.git', '+refs/heads/*:refs/remotes/origin/*') def check_ci(self, request): self.fetch_source() refs = self.git('for-each-ref', '--format=%(refname)', '--contains', request['sha'], 'refs/remotes/origin/') require(refs.strip(), 'candidate commit is not reachable from a repository branch') contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines() verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')), [line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')]) def hifox(self, *args): cli = DATA / 'bin/hifox' env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(DATA / 'hifox-home'), 'CI': '1'} raw = run([str(cli), '--organization-id', ORG, *args, '--output', 'json'], 'HiFox operation', env=env) result = json.JSONDecoder().raw_decode(raw.decode().strip())[0] require(isinstance(result, (dict, list)) and (not isinstance(result, dict) or (result.get('success') is not False and not result.get('error'))), 'HiFox did not return a successful result') return result def task(self, task): return self.hifox('task', 'view', task) def timeline(self, task): return self.hifox('api', 'get', f'/rest/v1/tasks/{task}/timeline') def comment_once(self, task, key, text): marker = f'KG-CD {key}' body = f'{text}\n\n`{marker}`' def present(): return any(row.get('event') == 'commented' and row.get('content') == body and (row.get('actor') or {}).get('id') == RELEASE_OWNER for row in self.timeline(task)) if not present(): self.hifox('task', 'comment', task, '--body', body) require(present(), 'HiFox release comment readback mismatch') def report_failure(self, request): if not self.receipt_path.exists(): return receipt = read_json(self.receipt_path) if any(receipt.get(key) != request[key] for key in ('task', 'sha', 'bundle', 'run_id')): return messages = { 'failed_before_cutover': '发布检查失败,未切换主站。', 'rolled_back': '发布核验失败,已恢复原版本并复验。', 'rollback_failed': '发布核验失败,回滚未通过复验,请检查主站。', 'deployed_hifox_pending': '主站已部署并核验,HiFox 回写待完成。请重试回写。', } state = receipt['state'] if state in messages: run_url = f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}' self.comment_once(request['task'], f'{state} #{request["run_id"]}', f'{messages[state]}\n\n候选 `{request["sha"]}`\n\n[发布运行]({run_url})') def callback(self, task_id, task, expected): if metadata(task, 'cd_release') != expected: self.hifox('task', 'metadata', 'set', task_id, 'cd_release', json.dumps(json.dumps(expected))) fresh = self.task(task_id) check_task(fresh, self.request, {'hifox': expected}) require(metadata(fresh, 'cd_release') == expected, 'HiFox release receipt readback mismatch') images = '\n'.join(f'- {role} `{expected["images"][role]}`' for role in ROLES) self.comment_once(task_id, f'published {expected["bundle"]}', f'云治理主站发布完成,健康检查通过。\n\n' f'运行版本 `{expected.get("version") or expected["sha"]}`\n\n' f'提交 `{expected["sha"]}`\n\n{images}\n\n' f'[登录主站]({LOGIN_URL}) · [发布运行]({expected["run_url"]})') fresh = self.task(task_id) check_task(fresh, self.request, {'hifox': expected}) require(metadata(fresh, 'cd_release') == expected, 'HiFox release changed during comment writeback') if fresh['status']['id'] != PUBLISHED: self.hifox('task', 'edit', task_id, '--status', PUBLISHED) final = self.task(task_id) final_status = (final.get('status') or {}).get('id') require(final_status == PUBLISHED and metadata(final, 'cd_release') == expected, 'HiFox published readback mismatch') def compose_cmd(self, *args): return ['docker', 'compose', '-p', PROJECT, '-f', str(self.compose), *args] def rendered_config(self): raw = run(self.compose_cmd('config', '--format', 'json'), 'render Compose configuration', cwd=APP) return json.loads(raw)['services'] def runtime(self): rendered = self.rendered_config() require(set(rendered) == set(ROLES), 'unexpected Compose service set') result = {} for role, name in self.containers.items(): c = inspect(name) labels = c['Config']['Labels'] require(labels.get('com.docker.compose.project') == PROJECT and labels.get('com.docker.compose.service') == role and labels.get('com.docker.compose.project.working_dir') == str(APP) and labels.get('com.docker.compose.project.config_files') == str(self.compose), f'{role} does not belong to the approved Compose chain') config = rendered[role] image = image_inspect(c['Image']) require(image_inspect(config['image'])['Id'] == c['Image'], f'{role} Compose image has drifted') env = lambda entries: dict(item.split('=', 1) for item in entries or []) expected_env = env(image['Config'].get('Env')) expected_env.update({key: str(value) for key, value in (config.get('environment') or {}).items() if value is not None}) require(env(c['Config'].get('Env')) == expected_env, f'{role} running environment has drifted') expected_mounts = config.get('volumes') or [] require(all(mount['type'] == 'bind' for mount in expected_mounts), 'unexpected non-bind Compose volume') mounts = {(v['Source'], v['Destination'], not v['RW']) for v in c.get('Mounts', []) if v['Type'] == 'bind'} require(mounts == {(v['source'], v['target'], v.get('read_only', False)) for v in expected_mounts} and len(mounts) == len(c.get('Mounts', [])), f'{role} running mounts have drifted') require(not config.get('network_mode') or config['network_mode'] == c['HostConfig']['NetworkMode'], f'{role} running network has drifted') result[role] = c['Image'] return result def baseline(self): images = self.runtime() self.probe_web() return {'compose_sha256': hash_file(self.compose), 'images': images, 'revisions': {role: image_label(image) for role, image in images.items()}} def pre_switch(self, previous, manifest): require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'], 'target baseline changed during deployment') for revision in set(previous['revisions'].values()): require(matches(SHA, revision), 'running image has no source revision') self.git('merge-base', '--is-ancestor', revision, manifest['sha']) require(migration_files(self.git, manifest['sha']) == manifest['migrations'], 'source migration set differs from bundle') for role in ('api', 'worker'): require(ledger(self.containers[role]) == manifest['migrations'], f'{role} migration ledger mismatch; verify schema changes outside CD') def load(self, directory, manifest): archives = {role: directory / (role + '.tar') for role in ROLES} for role, archive in archives.items(): verify_archive(archive, manifest['images'][role]['id']) for role, archive in archives.items(): image = manifest['images'][role]['id'] run(['docker', 'image', 'load', '-i', str(archive)], f'load {role} image', timeout=600) require(image_inspect(image)['Id'] == image and image_label(image) == manifest['sha'], f'{role} loaded image differs from approved build') @property def backup(self): return self.receipt_path.with_suffix('.compose.backup') def switch(self, previous, manifest): current = self.compose.read_bytes() require(hashlib.sha256(current).hexdigest() == previous['compose_sha256'], 'Compose changed before switch') images = {role: manifest['images'][role]['id'] for role in ROLES} changed = with_images(json.loads(current), images) for role, image in previous['images'].items(): run(['docker', 'image', 'tag', image, f'knowledge-governance-cd-rollback:{self.request["bundle"]}-{role}'], 'retain rollback image') atomic_write(self.backup, current) atomic_write(self.compose, (json.dumps(changed, indent=2) + '\n').encode(), self.compose.stat().st_mode & 0o777) self.recreate() def recreate(self): script = Path(__file__).with_name('deploy-api-web.sh') run(['bash', str(script), '--verified-compose', str(APP), PROJECT, str(self.compose), 'api', 'web', 'worker'], 'switch governance Compose services', timeout=120) def probe_web(self): origin = self.health_url require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'web proxy health probe failed') require(json.loads(http('http://127.0.0.1:18098/readyz', direct=True)).get('status') in ('ok', 'ready'), 'API readiness probe failed') index = http(origin + '/', direct=True) return index, {path: hashlib.sha256(http(origin + path, direct=True, limit=32 * 2**20)).hexdigest() for path in disk_assets(index)} def verify_web_image(self): origin_index, assets = self.probe_web() name = self.containers['web'] def read_asset(path): return run(['docker', 'exec', name, 'cat', '/usr/share/nginx/html' + path], 'read running web asset') require(origin_index == read_asset('/index.html'), 'served index differs from running web image') for path, served_digest in assets.items(): require(hashlib.sha256(read_asset(path)).hexdigest() == served_digest, 'served asset differs from running web image') return assets def verify(self, manifest): images = {role: manifest['images'][role]['id'] for role in ROLES} def check(): require(self.runtime() == images, 'running image IDs differ from approved bundle') require(api_identity(self.containers['api']) == manifest['sha'], 'running API SHA mismatches approval') assets = self.verify_web_image() version = image_inspect(images['api'])['Config']['Labels'].get('org.opencontainers.image.version') if not isinstance(version, str) or not re.fullmatch(r'[0-9A-Za-z.+_-]{1,128}', version): version = manifest['sha'] return {'sha': manifest['sha'], 'version': version, 'images': images, 'health': 'ok', 'asset_sha256': assets} return wait_ready(check) def rollback(self, previous): if not self.backup.exists(): require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'], 'rollback baseline cannot be established') self.verify_web_image() return original = self.backup.read_bytes() require(hashlib.sha256(original).hexdigest() == previous['compose_sha256'], 'rollback backup checksum mismatch') source = json.loads(original) for role, image in previous['images'].items(): reference = source['services'][role]['image'] require(image_inspect(reference)['Id'] == image, 'rollback source tag has drifted') atomic_write(self.compose, original, self.compose.stat().st_mode & 0o777) self.recreate() def check(): require(self.runtime() == previous['images'], 'rollback running images mismatch') require(api_identity(self.containers['api']) == previous['revisions']['api'], 'rollback API SHA mismatch') self.verify_web_image() wait_ready(check) def wait_ready(check, seconds=90): deadline = time.monotonic() + seconds while True: try: return check() except (CDError, OSError, KeyError, ValueError): if time.monotonic() >= deadline: raise time.sleep(3) def event_request(event, env): require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main' and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required') inputs = event.get('inputs') or {} request = {name: inputs.get(name) or '' for name in ('action', 'sha', 'task', 'bundle', 'confirm', 'approval_event', 'acceptance_digest')} request['run_id'] = env.get('GITHUB_RUN_ID', '') validate_request(request) return request def pack(args): require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID') require(not args.output.exists(), 'bundle output already exists') source = lambda *argv: run(['git', *argv], 'read candidate source') require(source('rev-parse', args.sha + '^{commit}').decode().strip() == args.sha, 'source commit not found') images = {} versions = migration_files(source, args.sha) manifest = dict(schema=1, target=TARGET, sha=args.sha, task=args.task, images=images, migrations=versions) with tempfile.TemporaryDirectory(dir=args.output.parent) as directory: build = Path(directory) / "sealed" build.mkdir(mode=0o700) for role in ROLES: image = image_inspect(getattr(args, role).removeprefix('sha256:')) image_id = image['Id'] require(matches(IMAGE, image_id) and image_label(image_id) == args.sha, f'{role} source label mismatch') archive = build / (role + '.tar') run(['docker', 'image', 'save', '-o', str(archive), image_id], f'archive {role} image', timeout=900) verify_archive(archive, image_id) archive.chmod(0o600) images[role] = {'id': image_id, 'sha256': hash_file(archive)} record(build / 'release.json', manifest) for item in build.iterdir(): with item.open("rb") as artifact: os.fsync(artifact.fileno()) sync_dir(build) build.rename(args.output) sync_dir(args.output.parent) print(hash_file(args.output / 'release.json')) def attest(args): request = {'sha': args.sha, 'task': args.task, 'bundle': hash_file(args.directory / 'release.json')} require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID') manifest = verify_bundle(args.directory, request) before = {role: inspect(getattr(args, role)) for role in ROLES} for role, obj in before.items(): require(obj['Image'] == manifest['images'][role]['id'] and obj['Name'] != '/kg-cloud-' + role, 'isolated test runtime image differs from bundle') require(api_identity(args.api) == args.sha and ledger(args.api) == manifest['migrations'] and ledger(args.worker) == manifest['migrations'], 'test runtime SHA or schema differs') base = urlsplit(args.test_url) require(base.scheme in ('http', 'https') and base.hostname and not (base.username or base.password or base.query or base.fragment) and base.path in ('', '/') and base.port != 8088, 'provide isolated test endpoint') origin = args.test_url.rstrip('/') require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'isolated test health probe failed') index = http(origin + '/', direct=True) require(index == run(['docker', 'exec', args.web, 'cat', '/usr/share/nginx/html/index.html'], 'read test web index'), 'test endpoint is serving another web image') for role, obj in before.items(): require(inspect(getattr(args, role))['Id'] == obj['Id'], 'test instance changed during readback') print(json.dumps({'sha': args.sha, 'bundle': request['bundle'], 'test_url': args.test_url, 'images': {role: obj['Image'] for role, obj in before.items()}}, sort_keys=True)) def cli(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) for command in ('guard', 'send', 'serve', 'pack', 'attest'): sub = commands.add_parser(command) if command in ('guard', 'send'): sub.add_argument('--event', type=Path, required=True) if command in ('pack', 'attest'): sub.add_argument('--sha', required=True) sub.add_argument('--task', required=True) if command in ('pack', 'attest'): for role in ROLES: sub.add_argument('--' + role, required=True) if command == 'pack': sub.add_argument('--output', type=Path, required=True) if command == 'attest': sub.add_argument('--directory', type=Path, required=True) sub.add_argument('--test-url', required=True) args = parser.parse_args() if args.command in ('guard', 'send'): request = event_request(read_json(args.event), os.environ) if args.command == 'guard': print(guard_proof(request, os.environ.get('GITHUB_SHA'))) else: send(request) elif args.command == 'pack': pack(args) elif args.command == 'attest': attest(args) else: raw = sys.stdin.buffer.read(8193) require(len(raw) <= 8192, 'request exceeds size limit') request = json.loads(raw) validate_request(request) with deploy_lock(DATA / 'release.lock'): host = LiveHost(request) for item in (DATA / 'receipts').glob('*.json'): if read_json(item)['state'] in ('cutover_started', 'rollback_failed'): raise CDError('previous deployment outcome requires operator reconciliation') try: result = release(host, request) except (CDError, OSError, ValueError, KeyError): try: host.report_failure(request) except (CDError, OSError, ValueError, KeyError): print('CD: HiFox failure comment pending', file=sys.stderr) raise print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False)) def send(request): key = os.environ.get('KG_CD_SSH_KEY', '') known_hosts = os.environ.get('KG_CD_KNOWN_HOSTS', '') host = os.environ.get('KG_CD_SSH_HOST', '') port = os.environ.get('KG_CD_SSH_PORT', '22') or '22' require(key.strip() and known_hosts.strip(), 'missing KG_CD_SSH_KEY or KG_CD_KNOWN_HOSTS') require(host in ('100.85.111.121', '::1') and port.isdecimal() and 1 <= int(port) <= 65535, 'configure the direct or approved tunnel address for the target release endpoint') with tempfile.TemporaryDirectory() as directory: temp = Path(directory) for filename, value in (('key', key), ('known_hosts', known_hosts)): atomic_write(temp / filename, (value.strip() + '\n').encode()) argv = ['ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'IdentitiesOnly=yes', '-o', 'ClearAllForwardings=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'HostKeyAlias=kg-governance-cd', '-o', 'GlobalKnownHostsFile=/dev/null', '-o', f'UserKnownHostsFile={temp / "known_hosts"}', '-i', str(temp / 'key'), '-p', port, '-T', 'root@' + host, 'kg-cd-v1'] try: result = subprocess.run(argv, input=json.dumps(request).encode(), capture_output=True, timeout=2100) except (OSError, subprocess.TimeoutExpired) as exc: raise CDError('target outcome unknown; inspect durable receipt before retry') from exc if result.returncode: error = result.stderr.decode(errors='replace') for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:', 'target preflight rejected', 'deployment lock busy', 'writeback requires a verified deployment receipt', 'Gitea approval guard has not succeeded', 'Gitea guard dispatch proof missing or different'): if marker in error: raise CDError(error.strip().splitlines()[-1]) raise CDError('target rejected request or outcome unknown; inspect durable receipt') response = json.loads(result.stdout) require(response.get('state') == 'published' and all(response.get(k) == request[k] for k in ('sha', 'task', 'bundle')), 'invalid or unverified deployment receipt') print(json.dumps(response, ensure_ascii=False)) if __name__ == '__main__': try: cli() except CDError as error: print(f'CD: {error}', file=sys.stderr) sys.exit(1) except (OSError, KeyError, ValueError, TypeError, tarfile.TarError) as error: print(f'CD: operation could not be verified ({type(error).__name__})', file=sys.stderr) sys.exit(1)