Files
knowledge-governance/scripts/production_cd.py
T
姚章浩 084ee1b176
build / build (push) Successful in 1m29s
fix(deploy): 兼容 HiFox 确认事件跨秒写入
任务更新时间与状态事件分别写入,允许小幅时间偏差。
仍以最近事件验证操作者,并要求验收证据早于两条时间记录。
2026-11-23 03:01:42 +08:00

858 lines
42 KiB
Python

#!/usr/bin/env python3
"""Governance application CD. The installed host copy owns the release boundary."""
import argparse
from contextlib import contextmanager
import copy
from datetime import datetime, timezone
import fcntl
import hashlib
from html.parser import HTMLParser
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
import tarfile
import tempfile
import time
from urllib.error import URLError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, ProxyHandler, build_opener
REPO = 'Weknora/knowledge-governance'
FORGE = 'https://gitea.zheromiku.top'
TARGET = 'cloud-test-governance-main'
ORG = '169e6f7f-e508-4a1f-9a9b-63d240439a3d'
SPACE = '106e05d1-4091-40b9-8c57-f76dc94f7825'
READY = '3e15556f-ca19-417c-8bfd-666634ea5d29'
PUBLISHED = '16185166-b8e9-4761-9926-845a1f41c96d'
RELEASE_OWNER = '39ae59fb-876c-4587-ab5d-a12a841c6c09'
LOGIN_URL = 'http://100.85.111.121:8088/'
ROLES = ('api', 'web', 'worker')
REVISION = 'org.opencontainers.image.revision'
DATA = Path('/opt/knowledge-governance/cd')
APP = DATA.parent
PROJECT = 'knowledge-governance-cloud-test'
SHA = re.compile('[0-9a-f]{40}')
HASH = re.compile('[0-9a-f]{64}')
IMAGE = re.compile('sha256:[0-9a-f]{64}')
UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}')
GUARD_PREFIX = 'KG_CD_CONFIRM_V1 '
class CDError(RuntimeError):
pass
def require(ok, message):
if not ok:
raise CDError(message)
def stamp():
return datetime.now(timezone.utc).isoformat()
def read_json(path):
return json.loads(path.read_bytes())
def sync_dir(directory):
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)
def atomic_write(path, data, mode=0o600):
fd, temp = tempfile.mkstemp(dir=path.parent)
try:
with os.fdopen(fd, 'wb') as out:
os.fchmod(out.fileno(), mode)
out.write(data)
out.flush()
os.fsync(out.fileno())
os.replace(temp, path)
sync_dir(path.parent)
finally:
Path(temp).unlink(missing_ok=True)
def record(path, value):
atomic_write(path, (json.dumps(value, ensure_ascii=False, sort_keys=True) + '\n').encode())
@contextmanager
def deploy_lock(path):
fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise CDError('deployment lock busy') from exc
yield
def matches(pattern, text):
return isinstance(text, str) and pattern.fullmatch(text) is not None
def validate_request(request):
actions = {'deploy': 'deploy-production', 'writeback': 'retry-writeback'}
action = request.get('action')
require(action in actions and request.get('confirm') == actions[action], 'explicit confirmation required')
require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task'))
and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID')
require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID')
event, digest = request.get('approval_event', ''), request.get('acceptance_digest', '')
require((event == '' and digest == '') or
(action == 'deploy' and matches(UUID, event) and matches(HASH, digest)),
'invalid HiFox approval binding')
def guard_proof(request, control_sha):
require(matches(SHA, control_sha), 'invalid workflow source SHA')
binding = {key: request[key] for key in ('run_id', 'action', 'confirm', 'sha', 'bundle', 'task')}
if request.get('approval_event'):
binding.update(approval_event=request['approval_event'], acceptance_digest=request['acceptance_digest'])
binding.update(repository=REPO, target=TARGET, workflow_sha=control_sha)
serialized = json.dumps(binding, sort_keys=True, separators=(',', ':')).encode()
return GUARD_PREFIX + hashlib.sha256(serialized).hexdigest()
def verify_guard_log(log, request, control_sha):
expected = guard_proof(request, control_sha)
pattern = re.compile(r'\d{4}-\d{2}-\d{2}T[0-9:.]+Z (' + GUARD_PREFIX + r'[0-9a-f]{64})')
proofs = [match.group(1) for line in log.decode().splitlines() if (match := pattern.fullmatch(line))]
require(proofs == [expected], 'Gitea guard dispatch proof missing or different')
def hash_file(path):
with path.open('rb') as src:
return hashlib.file_digest(src, 'sha256').hexdigest()
def immutable_file(path):
info = path.lstat()
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and not info.st_mode & 0o022,
'bundle requires regular files owned by the deployment account')
return path
def verify_bundle(directory, request, archives=True):
raw = immutable_file(directory / 'release.json').read_bytes()
require(hashlib.sha256(raw).hexdigest() == request['bundle'], 'release manifest digest mismatch')
manifest = json.loads(raw)
require(manifest.get('schema') == 1 and manifest.get('target') == TARGET
and manifest.get('sha') == request['sha'] and manifest.get('task') == request['task'],
'bundle commit, task or target mismatch')
require(set(manifest.get('images', {})) == set(ROLES), 'bundle must include API, web and worker')
for role, image in manifest['images'].items():
require(matches(IMAGE, image.get('id')) and matches(HASH, image.get('sha256')), 'invalid image record')
if archives:
archive = immutable_file(directory / (role + '.tar'))
require(hash_file(archive) == image['sha256'], f'{role} archive checksum mismatch')
versions = manifest.get('migrations')
require(isinstance(versions, list) and versions and all(type(n) is int and n > 0 for n in versions)
and versions == sorted(set(versions)), 'invalid migration ledger in manifest')
return manifest
def verify_contexts(result, contexts):
latest = {}
for item in result.get('statuses') or []:
latest.setdefault(item.get('context'), item.get('status'))
require(bool(contexts) and all(latest.get(context) == 'success' for context in contexts),
'required CI status is missing or unsuccessful')
def metadata(task, key):
value = (task.get('metadata') or {}).get(key)
return json.loads(value) if isinstance(value, str) else None
def check_task(task, request, receipt=None):
require(task.get('id') == request['task'] and task.get('spaceId') == SPACE, 'HiFox task or space mismatch')
status = (task.get('status') or {}).get('id')
same_publication = receipt and metadata(task, 'cd_release') == receipt.get('hifox')
require(status == READY or (status == PUBLISHED and same_publication), 'HiFox task is not ready for release')
evidence = metadata(task, 'cd_acceptance') or {}
url = urlsplit(evidence.get('evidence', ''))
require(evidence.get('verdict') == 'passed' and evidence.get('sha') == request['sha']
and evidence.get('bundle') == request['bundle'] and url.scheme == 'https' and url.hostname
and not (url.username or url.password or url.query)
and (not url.fragment or (url.netloc == 'app.hifox.com' and url.fragment.startswith('comment-')
and matches(UUID, url.fragment.removeprefix('comment-')))),
'missing or changed business acceptance for this commit and bundle')
return evidence
def acceptance_digest(acceptance):
return hashlib.sha256(json.dumps(acceptance, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
def release_approval(task, timeline, request):
evidence = check_task(task, request)
changes = [row for row in timeline if row.get('event') == 'status_changed']
require(bool(changes), 'missing HiFox status history')
approval = changes[-1]
approver = approval.get('actor') or {}
event_at = approval.get('createdAt') or ''
status_at = task.get('statusChangedAt') or ''
try:
difference = abs((datetime.fromisoformat(event_at.replace('Z', '+00:00'))
- datetime.fromisoformat(status_at.replace('Z', '+00:00'))).total_seconds())
except (ValueError, TypeError):
difference = float('inf')
# HiFox logs the status change after saving the task, sometimes across a second boundary.
require((approval.get('metadata') or {}).get('to') == '待发布'
and difference <= 2
and approver.get('id') == RELEASE_OWNER and approver.get('type') == 'human'
and matches(UUID, approval.get('id')), 'the latest transition must be an approval by the release owner')
task_url = task.get('url', '')
prefix = task_url + '#comment-'
require(task_url.startswith('https://app.hifox.com/') and evidence['evidence'].startswith(prefix),
'automated release requires the same-task acceptance report')
comment_id = evidence['evidence'].removeprefix(prefix)
require(matches(UUID, comment_id), 'invalid acceptance report ID')
reports = [row for row in timeline if row.get('event') == 'commented' and row.get('id') == comment_id]
require(len(reports) == 1, 'acceptance report missing from task history')
report = reports[0]
body = report.get('content') or ''
require(report.get('createdAt') and (report.get('updatedAt') or report['createdAt']) < min(event_at, status_at)
and request['sha'] in body and request['bundle'] in body,
'the unchanged report must identify the candidate and predate approval')
binding = {'approval_event': approval['id'], 'acceptance_digest': acceptance_digest(evidence)}
require(all(request.get(key, value) == value for key, value in binding.items()),
'HiFox approval or acceptance changed after dispatch')
return binding, min(event_at, status_at)
def accepted_task(host, request):
task = host.task(request['task'])
if request.get('approval_event'):
release_approval(task, host.timeline(request['task']), request)
return check_task(task, request)
def release(host, request):
validate_request(request)
path = host.receipt_path
existing = read_json(path) if path.exists() else None
if request['action'] == 'writeback':
require(existing and existing['state'] in ('deployed_hifox_pending', 'published'),
'writeback requires a verified deployment receipt')
require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch')
manifest = verify_bundle(host.bundle_path(request), request, archives=False)
host.approve(request)
host.verify(manifest)
return writeback(host, request, existing)
require(existing is None, 'existing release receipt: use writeback or inspect interrupted/failed deployment')
host.approve(request)
manifest = verify_bundle(host.bundle_path(request), request)
host.check_ci(request)
accepted = accepted_task(host, request)
before = host.baseline()
host.pre_switch(before, manifest)
receipt = {k: request[k] for k in ('sha', 'task', 'bundle', 'run_id')}
receipt.update(state='preparing', target=TARGET, started_at=stamp(), previous=before)
record(path, receipt)
try:
host.load(host.bundle_path(request), manifest)
host.pre_switch(before, manifest)
host.approve(request)
require(accepted_task(host, request) == accepted, 'business acceptance changed before switch')
except (CDError, OSError, ValueError, KeyError) as exc:
receipt.update(state='failed_before_cutover')
record(path, receipt)
raise CDError(f'failed_before_cutover: {exc}' if isinstance(exc, CDError) else 'failed_before_cutover') from exc
receipt.update(state='cutover_started')
record(path, receipt)
try:
host.switch(before, manifest)
evidence = host.verify(manifest)
except (CDError, OSError, ValueError, KeyError) as exc:
status = 'rollback_failed'
try:
host.rollback(before)
status = 'rolled_back'
except (CDError, OSError, ValueError, KeyError):
pass
receipt.update(state=status, finished_at=stamp(), failure=str(exc) if isinstance(exc, CDError) else 'operation failed')
record(path, receipt)
raise CDError(f'{status}: deployment verification failed') from exc
receipt.update(state='deployed_hifox_pending', deployed_at=stamp(), evidence=evidence)
receipt['hifox'] = {**evidence, 'target': TARGET, 'bundle': request['bundle'], 'deployed_at': receipt['deployed_at'],
'login_url': LOGIN_URL,
'run_url': f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'}
record(path, receipt)
return writeback(host, request, receipt)
def writeback(host, request, receipt):
try:
task = host.task(request['task'])
check_task(task, request, receipt)
host.callback(request['task'], task, receipt.get('hifox'))
receipt.update(state='published')
record(host.receipt_path, receipt)
except (CDError, OSError, ValueError, KeyError) as exc:
raise CDError('DEPLOYED_HIFOX_PENDING: retry writeback for this receipt') from exc
return receipt
def with_images(config, images):
result = copy.deepcopy(config)
require(set(result['services']) == set(ROLES), 'unexpected Compose service set')
for role in ROLES:
result['services'][role]['image'] = images[role]
return result
def run(argv, purpose, *, cwd=None, input_data=None, timeout=30, allowed=(0,), env=None):
try:
result = subprocess.run(argv, cwd=cwd, input=input_data, capture_output=True, timeout=timeout, env=env)
except (OSError, subprocess.TimeoutExpired) as exc:
raise CDError(f'{purpose} unavailable or timed out') from exc
require(result.returncode in allowed, f'{purpose} failed (exit {result.returncode})')
return result.stdout
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *_args, **_kwargs):
raise CDError('unexpected HTTP redirect')
def http(url, *, direct=False, limit=2**20):
handlers = [NoRedirect()]
if direct:
handlers.append(ProxyHandler({}))
try:
with build_opener(*handlers).open(url, timeout=10) as response:
data = response.read(limit + 1)
require(len(data) <= limit, 'HTTP response exceeds limit')
return data
except (URLError, TimeoutError) as exc:
raise CDError('HTTP read failed') from exc
class Assets(HTMLParser):
def __init__(self):
super().__init__()
self.paths = []
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
path = attrs.get('src' if tag == 'script' else 'href', '')
if path and re.fullmatch(r'/assets/[A-Za-z0-9_.-]+\.(?:js|css)', path):
self.paths.append(path)
def disk_assets(html):
parser = Assets()
parser.feed(html.decode())
require(any(p.endswith('.js') for p in parser.paths), 'web index has no compiled entry point')
return sorted(set(parser.paths))
def image_inspect(image):
return json.loads(run(['docker', 'image', 'inspect', image], 'read image identity'))[0]
def inspect(name):
obj = json.loads(run(['docker', 'container', 'inspect', name], 'read running container'))[0]
require(obj['State']['Status'] == 'running' and matches(IMAGE, obj.get('Image')), 'expected container is not running')
return obj
def image_label(image):
return image_inspect(image)['Config']['Labels'].get(REVISION, '')
def api_identity(container):
raw = run(['docker', 'exec', container, '/app/api', '--version-record-status'], 'read API build identity', allowed=(0, 1))
obj = json.loads(raw)
require(matches(SHA, obj.get('commit_sha')), 'API binary has no full build identity')
return obj['commit_sha']
def migration_files(git, sha):
paths = git('ls-tree', '-r', '--name-only', sha, 'internal/platform/postgres/migrations').decode().splitlines()
return sorted(int(Path(line).name.split('_', 1)[0]) for line in paths if line.endswith('.up.sql'))
def ledger(container):
probe = Path(__file__).with_name('cd_migration_probe.py').read_text()
data = run(['docker', 'exec', container, 'python3', '-c', probe], 'read migration ledger')
result = json.loads(data)
require(isinstance(result, list) and result, 'unexpected migration ledger result')
return result
def verify_archive(archive, expected):
# docker save by ID omits mutable RepoTags. Refuse archives that could retag another app.
with tarfile.open(archive, 'r:') as t:
entry = t.getmember('manifest.json')
require(entry.isfile() and entry.size < 65536, 'unexpected Docker archive manifest')
containers = json.load(t.extractfile(entry))
require(len(containers) == 1 and not containers[0].get('RepoTags'), 'archive must contain one untagged image')
config = t.getmember(containers[0]['Config'])
require(config.isfile() and config.size < 2**20, 'unexpected Docker image config')
require('sha256:' + hashlib.sha256(t.extractfile(config).read()).hexdigest() == expected,
'archive config does not match approved image ID')
for member in t:
require((member.isdir() or member.isfile()) and not member.name.startswith('/')
and '..' not in Path(member.name).parts, 'unexpected Docker archive path')
if 'index.json' in t.getnames():
index = t.getmember('index.json')
require(index.isfile() and index.size < 65536, 'unexpected OCI archive index')
oci = json.load(t.extractfile(index))
require(len(oci.get('manifests', [])) == 1 and all(
key not in descriptor.get('annotations', {}) for descriptor in oci['manifests']
for key in ('org.opencontainers.image.ref.name', 'io.containerd.image.name')),
'archive OCI index contains a mutable name')
class LiveHost:
def __init__(self, request):
self.receipt_path = DATA / 'receipts' / (request['bundle'] + '.json')
self.compose = APP / 'compose.json'
self.git_path = DATA / 'source.git'
self.containers = {role: 'kg-cloud-' + role for role in ROLES}
self.health_url = 'http://127.0.0.1:8088'
self.request = request
def bundle_path(self, request):
return DATA / 'bundles' / request['bundle']
def git(self, *args):
return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90)
def approve(self, request):
repo_url = f'{FORGE}/{REPO}'
run_id = request['run_id']
run_api = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{run_id}'
dispatch = json.loads(http(run_api))
control_sha = dispatch.get('head_sha')
trigger_actor = dispatch.get('trigger_actor') or dispatch.get('actor') or {}
require(str(dispatch.get('id')) == run_id and dispatch.get('event') == 'workflow_dispatch'
and dispatch.get('path') == 'deploy-production.yml@refs/heads/main'
and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running')
and (dispatch.get('actor') or {}).get('id') == 1 and trigger_actor.get('id') == 1
and matches(SHA, control_sha), 'Gitea run is not an active deployment dispatched by the release owner')
# Repo secrets can be read by other workflows. Only a completed guard from
# the installed workflow and controller may authorize the exact inputs.
try:
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
except CDError:
self.fetch_source()
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
trusted = {'scripts/production_cd.py': Path(__file__).resolve(),
'.gitea/workflows/deploy-production.yml': DATA / 'deploy-production.yml'}
for name, local in trusted.items():
require(self.git('show', f'{control_sha}:{name}') == immutable_file(local).read_bytes(),
'Gitea approval workflow differs from installed control files')
jobs = json.loads(http(run_api + '/jobs')).get('jobs') or []
guards = [job for job in jobs if job.get('name') == 'guard' and job.get('conclusion') == 'success'
and str(job.get('run_id')) == run_id and job.get('head_sha') == control_sha
and matches(re.compile('[1-9][0-9]{0,14}'), str(job.get('id')))]
require(len(guards) == 1, 'Gitea approval guard has not succeeded')
guard_id = guards[0]['id']
verify_guard_log(http(f'{repo_url}/actions/runs/{run_id}/jobs/{guard_id}/logs'), request, control_sha)
def fetch_source(self):
self.git('fetch', '--no-tags', '--prune', f'{FORGE}/{REPO}.git', '+refs/heads/*:refs/remotes/origin/*')
def check_ci(self, request):
self.fetch_source()
refs = self.git('for-each-ref', '--format=%(refname)', '--contains', request['sha'], 'refs/remotes/origin/')
require(refs.strip(), 'candidate commit is not reachable from a repository branch')
contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines()
verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')),
[line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')])
def hifox(self, *args):
cli = DATA / 'bin/hifox'
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(DATA / 'hifox-home'), 'CI': '1'}
raw = run([str(cli), '--organization-id', ORG, *args, '--output', 'json'], 'HiFox operation', env=env)
result = json.JSONDecoder().raw_decode(raw.decode().strip())[0]
require(isinstance(result, (dict, list)) and (not isinstance(result, dict)
or (result.get('success') is not False and not result.get('error'))),
'HiFox did not return a successful result')
return result
def task(self, task):
return self.hifox('task', 'view', task)
def timeline(self, task):
return self.hifox('api', 'get', f'/rest/v1/tasks/{task}/timeline')
def comment_once(self, task, key, text):
marker = f'KG-CD {key}'
body = f'{text}\n\n`{marker}`'
def present():
return any(row.get('event') == 'commented' and row.get('content') == body
and (row.get('actor') or {}).get('id') == RELEASE_OWNER for row in self.timeline(task))
if not present():
self.hifox('task', 'comment', task, '--body', body)
require(present(), 'HiFox release comment readback mismatch')
def report_failure(self, request):
if not self.receipt_path.exists():
return
receipt = read_json(self.receipt_path)
if any(receipt.get(key) != request[key] for key in ('task', 'sha', 'bundle', 'run_id')):
return
messages = {
'failed_before_cutover': '发布检查失败,未切换主站。',
'rolled_back': '发布核验失败,已恢复原版本并复验。',
'rollback_failed': '发布核验失败,回滚未通过复验,请检查主站。',
'deployed_hifox_pending': '主站已部署并核验,HiFox 回写待完成。请重试回写。',
}
state = receipt['state']
if state in messages:
run_url = f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'
self.comment_once(request['task'], f'{state} #{request["run_id"]}',
f'{messages[state]}\n\n候选 `{request["sha"]}`\n\n[发布运行]({run_url})')
def callback(self, task_id, task, expected):
if metadata(task, 'cd_release') != expected:
self.hifox('task', 'metadata', 'set', task_id, 'cd_release', json.dumps(json.dumps(expected)))
fresh = self.task(task_id)
check_task(fresh, self.request, {'hifox': expected})
require(metadata(fresh, 'cd_release') == expected, 'HiFox release receipt readback mismatch')
images = '\n'.join(f'- {role} `{expected["images"][role]}`' for role in ROLES)
self.comment_once(task_id, f'published {expected["bundle"]}',
f'云治理主站发布完成,健康检查通过。\n\n'
f'运行版本 `{expected.get("version") or expected["sha"]}`\n\n'
f'提交 `{expected["sha"]}`\n\n{images}\n\n'
f'[登录主站]({LOGIN_URL}) · [发布运行]({expected["run_url"]})')
fresh = self.task(task_id)
check_task(fresh, self.request, {'hifox': expected})
require(metadata(fresh, 'cd_release') == expected, 'HiFox release changed during comment writeback')
if fresh['status']['id'] != PUBLISHED:
self.hifox('task', 'edit', task_id, '--status', PUBLISHED)
final = self.task(task_id)
final_status = (final.get('status') or {}).get('id')
require(final_status == PUBLISHED and metadata(final, 'cd_release') == expected, 'HiFox published readback mismatch')
def compose_cmd(self, *args):
return ['docker', 'compose', '-p', PROJECT, '-f', str(self.compose), *args]
def rendered_config(self):
raw = run(self.compose_cmd('config', '--format', 'json'), 'render Compose configuration', cwd=APP)
return json.loads(raw)['services']
def runtime(self):
rendered = self.rendered_config()
require(set(rendered) == set(ROLES), 'unexpected Compose service set')
result = {}
for role, name in self.containers.items():
c = inspect(name)
labels = c['Config']['Labels']
require(labels.get('com.docker.compose.project') == PROJECT
and labels.get('com.docker.compose.service') == role
and labels.get('com.docker.compose.project.working_dir') == str(APP)
and labels.get('com.docker.compose.project.config_files') == str(self.compose),
f'{role} does not belong to the approved Compose chain')
config = rendered[role]
image = image_inspect(c['Image'])
require(image_inspect(config['image'])['Id'] == c['Image'], f'{role} Compose image has drifted')
env = lambda entries: dict(item.split('=', 1) for item in entries or [])
expected_env = env(image['Config'].get('Env'))
expected_env.update({key: str(value) for key, value in (config.get('environment') or {}).items()
if value is not None})
require(env(c['Config'].get('Env')) == expected_env, f'{role} running environment has drifted')
expected_mounts = config.get('volumes') or []
require(all(mount['type'] == 'bind' for mount in expected_mounts), 'unexpected non-bind Compose volume')
mounts = {(v['Source'], v['Destination'], not v['RW']) for v in c.get('Mounts', []) if v['Type'] == 'bind'}
require(mounts == {(v['source'], v['target'], v.get('read_only', False)) for v in expected_mounts}
and len(mounts) == len(c.get('Mounts', [])), f'{role} running mounts have drifted')
require(not config.get('network_mode') or config['network_mode'] == c['HostConfig']['NetworkMode'],
f'{role} running network has drifted')
result[role] = c['Image']
return result
def baseline(self):
images = self.runtime()
self.probe_web()
return {'compose_sha256': hash_file(self.compose), 'images': images,
'revisions': {role: image_label(image) for role, image in images.items()}}
def pre_switch(self, previous, manifest):
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
'target baseline changed during deployment')
for revision in set(previous['revisions'].values()):
require(matches(SHA, revision), 'running image has no source revision')
self.git('merge-base', '--is-ancestor', revision, manifest['sha'])
require(migration_files(self.git, manifest['sha']) == manifest['migrations'], 'source migration set differs from bundle')
for role in ('api', 'worker'):
require(ledger(self.containers[role]) == manifest['migrations'],
f'{role} migration ledger mismatch; verify schema changes outside CD')
def load(self, directory, manifest):
archives = {role: directory / (role + '.tar') for role in ROLES}
for role, archive in archives.items():
verify_archive(archive, manifest['images'][role]['id'])
for role, archive in archives.items():
image = manifest['images'][role]['id']
run(['docker', 'image', 'load', '-i', str(archive)], f'load {role} image', timeout=600)
require(image_inspect(image)['Id'] == image and image_label(image) == manifest['sha'],
f'{role} loaded image differs from approved build')
@property
def backup(self):
return self.receipt_path.with_suffix('.compose.backup')
def switch(self, previous, manifest):
current = self.compose.read_bytes()
require(hashlib.sha256(current).hexdigest() == previous['compose_sha256'], 'Compose changed before switch')
images = {role: manifest['images'][role]['id'] for role in ROLES}
changed = with_images(json.loads(current), images)
for role, image in previous['images'].items():
run(['docker', 'image', 'tag', image, f'knowledge-governance-cd-rollback:{self.request["bundle"]}-{role}'],
'retain rollback image')
atomic_write(self.backup, current)
atomic_write(self.compose, (json.dumps(changed, indent=2) + '\n').encode(), self.compose.stat().st_mode & 0o777)
self.recreate()
def recreate(self):
script = Path(__file__).with_name('deploy-api-web.sh')
run(['bash', str(script), '--verified-compose', str(APP), PROJECT, str(self.compose), 'api', 'web', 'worker'],
'switch governance Compose services', timeout=120)
def probe_web(self):
origin = self.health_url
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'web proxy health probe failed')
require(json.loads(http('http://127.0.0.1:18098/readyz', direct=True)).get('status') in ('ok', 'ready'),
'API readiness probe failed')
index = http(origin + '/', direct=True)
return index, {path: hashlib.sha256(http(origin + path, direct=True, limit=32 * 2**20)).hexdigest()
for path in disk_assets(index)}
def verify_web_image(self):
origin_index, assets = self.probe_web()
name = self.containers['web']
def read_asset(path):
return run(['docker', 'exec', name, 'cat', '/usr/share/nginx/html' + path], 'read running web asset')
require(origin_index == read_asset('/index.html'), 'served index differs from running web image')
for path, served_digest in assets.items():
require(hashlib.sha256(read_asset(path)).hexdigest() == served_digest, 'served asset differs from running web image')
return assets
def verify(self, manifest):
images = {role: manifest['images'][role]['id'] for role in ROLES}
def check():
require(self.runtime() == images, 'running image IDs differ from approved bundle')
require(api_identity(self.containers['api']) == manifest['sha'], 'running API SHA mismatches approval')
assets = self.verify_web_image()
version = image_inspect(images['api'])['Config']['Labels'].get('org.opencontainers.image.version')
if not isinstance(version, str) or not re.fullmatch(r'[0-9A-Za-z.+_-]{1,128}', version):
version = manifest['sha']
return {'sha': manifest['sha'], 'version': version, 'images': images, 'health': 'ok', 'asset_sha256': assets}
return wait_ready(check)
def rollback(self, previous):
if not self.backup.exists():
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
'rollback baseline cannot be established')
self.verify_web_image()
return
original = self.backup.read_bytes()
require(hashlib.sha256(original).hexdigest() == previous['compose_sha256'], 'rollback backup checksum mismatch')
source = json.loads(original)
for role, image in previous['images'].items():
reference = source['services'][role]['image']
require(image_inspect(reference)['Id'] == image, 'rollback source tag has drifted')
atomic_write(self.compose, original, self.compose.stat().st_mode & 0o777)
self.recreate()
def check():
require(self.runtime() == previous['images'], 'rollback running images mismatch')
require(api_identity(self.containers['api']) == previous['revisions']['api'], 'rollback API SHA mismatch')
self.verify_web_image()
wait_ready(check)
def wait_ready(check, seconds=90):
deadline = time.monotonic() + seconds
while True:
try:
return check()
except (CDError, OSError, KeyError, ValueError):
if time.monotonic() >= deadline:
raise
time.sleep(3)
def event_request(event, env):
require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main'
and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required')
inputs = event.get('inputs') or {}
request = {name: inputs.get(name) or '' for name in
('action', 'sha', 'task', 'bundle', 'confirm', 'approval_event', 'acceptance_digest')}
request['run_id'] = env.get('GITHUB_RUN_ID', '')
validate_request(request)
return request
def pack(args):
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
require(not args.output.exists(), 'bundle output already exists')
source = lambda *argv: run(['git', *argv], 'read candidate source')
require(source('rev-parse', args.sha + '^{commit}').decode().strip() == args.sha, 'source commit not found')
images = {}
versions = migration_files(source, args.sha)
manifest = dict(schema=1, target=TARGET, sha=args.sha, task=args.task, images=images, migrations=versions)
with tempfile.TemporaryDirectory(dir=args.output.parent) as directory:
build = Path(directory) / "sealed"
build.mkdir(mode=0o700)
for role in ROLES:
image = image_inspect(getattr(args, role).removeprefix('sha256:'))
image_id = image['Id']
require(matches(IMAGE, image_id) and image_label(image_id) == args.sha, f'{role} source label mismatch')
archive = build / (role + '.tar')
run(['docker', 'image', 'save', '-o', str(archive), image_id], f'archive {role} image', timeout=900)
verify_archive(archive, image_id)
archive.chmod(0o600)
images[role] = {'id': image_id, 'sha256': hash_file(archive)}
record(build / 'release.json', manifest)
for item in build.iterdir():
with item.open("rb") as artifact:
os.fsync(artifact.fileno())
sync_dir(build)
build.rename(args.output)
sync_dir(args.output.parent)
print(hash_file(args.output / 'release.json'))
def attest(args):
request = {'sha': args.sha, 'task': args.task, 'bundle': hash_file(args.directory / 'release.json')}
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
manifest = verify_bundle(args.directory, request)
before = {role: inspect(getattr(args, role)) for role in ROLES}
for role, obj in before.items():
require(obj['Image'] == manifest['images'][role]['id'] and obj['Name'] != '/kg-cloud-' + role,
'isolated test runtime image differs from bundle')
require(api_identity(args.api) == args.sha and ledger(args.api) == manifest['migrations']
and ledger(args.worker) == manifest['migrations'], 'test runtime SHA or schema differs')
base = urlsplit(args.test_url)
require(base.scheme in ('http', 'https') and base.hostname and not (base.username or base.password or base.query or base.fragment)
and base.path in ('', '/') and base.port != 8088, 'provide isolated test endpoint')
origin = args.test_url.rstrip('/')
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'isolated test health probe failed')
index = http(origin + '/', direct=True)
require(index == run(['docker', 'exec', args.web, 'cat', '/usr/share/nginx/html/index.html'], 'read test web index'),
'test endpoint is serving another web image')
for role, obj in before.items():
require(inspect(getattr(args, role))['Id'] == obj['Id'], 'test instance changed during readback')
print(json.dumps({'sha': args.sha, 'bundle': request['bundle'], 'test_url': args.test_url,
'images': {role: obj['Image'] for role, obj in before.items()}}, sort_keys=True))
def cli():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
for command in ('guard', 'send', 'serve', 'pack', 'attest'):
sub = commands.add_parser(command)
if command in ('guard', 'send'):
sub.add_argument('--event', type=Path, required=True)
if command in ('pack', 'attest'):
sub.add_argument('--sha', required=True)
sub.add_argument('--task', required=True)
if command in ('pack', 'attest'):
for role in ROLES:
sub.add_argument('--' + role, required=True)
if command == 'pack':
sub.add_argument('--output', type=Path, required=True)
if command == 'attest':
sub.add_argument('--directory', type=Path, required=True)
sub.add_argument('--test-url', required=True)
args = parser.parse_args()
if args.command in ('guard', 'send'):
request = event_request(read_json(args.event), os.environ)
if args.command == 'guard':
print(guard_proof(request, os.environ.get('GITHUB_SHA')))
else:
send(request)
elif args.command == 'pack':
pack(args)
elif args.command == 'attest':
attest(args)
else:
raw = sys.stdin.buffer.read(8193)
require(len(raw) <= 8192, 'request exceeds size limit')
request = json.loads(raw)
validate_request(request)
with deploy_lock(DATA / 'release.lock'):
host = LiveHost(request)
for item in (DATA / 'receipts').glob('*.json'):
if read_json(item)['state'] in ('cutover_started', 'rollback_failed'):
raise CDError('previous deployment outcome requires operator reconciliation')
try:
result = release(host, request)
except (CDError, OSError, ValueError, KeyError):
try:
host.report_failure(request)
except (CDError, OSError, ValueError, KeyError):
print('CD: HiFox failure comment pending', file=sys.stderr)
raise
print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False))
def send(request):
key = os.environ.get('KG_CD_SSH_KEY', '')
known_hosts = os.environ.get('KG_CD_KNOWN_HOSTS', '')
host = os.environ.get('KG_CD_SSH_HOST', '')
port = os.environ.get('KG_CD_SSH_PORT', '22') or '22'
require(key.strip() and known_hosts.strip(), 'missing KG_CD_SSH_KEY or KG_CD_KNOWN_HOSTS')
require(host in ('100.85.111.121', '::1') and port.isdecimal() and 1 <= int(port) <= 65535,
'configure the direct or approved tunnel address for the target release endpoint')
with tempfile.TemporaryDirectory() as directory:
temp = Path(directory)
for filename, value in (('key', key), ('known_hosts', known_hosts)):
atomic_write(temp / filename, (value.strip() + '\n').encode())
argv = ['ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'IdentitiesOnly=yes',
'-o', 'ClearAllForwardings=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'HostKeyAlias=kg-governance-cd',
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', f'UserKnownHostsFile={temp / "known_hosts"}',
'-i', str(temp / 'key'), '-p', port, '-T', 'root@' + host, 'kg-cd-v1']
try:
result = subprocess.run(argv, input=json.dumps(request).encode(), capture_output=True, timeout=2100)
except (OSError, subprocess.TimeoutExpired) as exc:
raise CDError('target outcome unknown; inspect durable receipt before retry') from exc
if result.returncode:
error = result.stderr.decode(errors='replace')
for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:',
'target preflight rejected', 'deployment lock busy', 'writeback requires a verified deployment receipt',
'Gitea approval guard has not succeeded', 'Gitea guard dispatch proof missing or different'):
if marker in error:
raise CDError(error.strip().splitlines()[-1])
raise CDError('target rejected request or outcome unknown; inspect durable receipt')
response = json.loads(result.stdout)
require(response.get('state') == 'published' and all(response.get(k) == request[k] for k in ('sha', 'task', 'bundle')),
'invalid or unverified deployment receipt')
print(json.dumps(response, ensure_ascii=False))
if __name__ == '__main__':
try:
cli()
except CDError as error:
print(f'CD: {error}', file=sys.stderr)
sys.exit(1)
except (OSError, KeyError, ValueError, TypeError, tarfile.TarError) as error:
print(f'CD: operation could not be verified ({type(error).__name__})', file=sys.stderr)
sys.exit(1)