47428a8b58
build / build (push) Successful in 1m27s
将发布人的状态变更与业务验收报告绑定,复用 Gitea 发布及回滚链路。 从部署回执写入运行身份、镜像及主站登录入口,避免重复调度和回写。
850 lines
41 KiB
Python
850 lines
41 KiB
Python
#!/usr/bin/env python3
|
|
"""Governance application CD. The installed host copy owns the release boundary."""
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import copy
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
from html.parser import HTMLParser
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
from urllib.error import URLError
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import HTTPRedirectHandler, ProxyHandler, build_opener
|
|
|
|
REPO = 'Weknora/knowledge-governance'
|
|
FORGE = 'https://gitea.zheromiku.top'
|
|
TARGET = 'cloud-test-governance-main'
|
|
ORG = '169e6f7f-e508-4a1f-9a9b-63d240439a3d'
|
|
SPACE = '106e05d1-4091-40b9-8c57-f76dc94f7825'
|
|
READY = '3e15556f-ca19-417c-8bfd-666634ea5d29'
|
|
PUBLISHED = '16185166-b8e9-4761-9926-845a1f41c96d'
|
|
RELEASE_OWNER = '39ae59fb-876c-4587-ab5d-a12a841c6c09'
|
|
LOGIN_URL = 'http://100.85.111.121:8088/'
|
|
ROLES = ('api', 'web', 'worker')
|
|
REVISION = 'org.opencontainers.image.revision'
|
|
DATA = Path('/opt/knowledge-governance/cd')
|
|
APP = DATA.parent
|
|
PROJECT = 'knowledge-governance-cloud-test'
|
|
SHA = re.compile('[0-9a-f]{40}')
|
|
HASH = re.compile('[0-9a-f]{64}')
|
|
IMAGE = re.compile('sha256:[0-9a-f]{64}')
|
|
UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}')
|
|
GUARD_PREFIX = 'KG_CD_CONFIRM_V1 '
|
|
|
|
|
|
class CDError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise CDError(message)
|
|
|
|
|
|
def stamp():
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def read_json(path):
|
|
return json.loads(path.read_bytes())
|
|
|
|
|
|
def sync_dir(directory):
|
|
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def atomic_write(path, data, mode=0o600):
|
|
fd, temp = tempfile.mkstemp(dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'wb') as out:
|
|
os.fchmod(out.fileno(), mode)
|
|
out.write(data)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
os.replace(temp, path)
|
|
sync_dir(path.parent)
|
|
finally:
|
|
Path(temp).unlink(missing_ok=True)
|
|
|
|
|
|
def record(path, value):
|
|
atomic_write(path, (json.dumps(value, ensure_ascii=False, sort_keys=True) + '\n').encode())
|
|
|
|
|
|
@contextmanager
|
|
def deploy_lock(path):
|
|
fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'r+') as handle:
|
|
try:
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise CDError('deployment lock busy') from exc
|
|
yield
|
|
|
|
|
|
def matches(pattern, text):
|
|
return isinstance(text, str) and pattern.fullmatch(text) is not None
|
|
|
|
|
|
def validate_request(request):
|
|
actions = {'deploy': 'deploy-production', 'writeback': 'retry-writeback'}
|
|
action = request.get('action')
|
|
require(action in actions and request.get('confirm') == actions[action], 'explicit confirmation required')
|
|
require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task'))
|
|
and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID')
|
|
require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID')
|
|
event, digest = request.get('approval_event', ''), request.get('acceptance_digest', '')
|
|
require((event == '' and digest == '') or
|
|
(action == 'deploy' and matches(UUID, event) and matches(HASH, digest)),
|
|
'invalid HiFox approval binding')
|
|
|
|
|
|
def guard_proof(request, control_sha):
|
|
require(matches(SHA, control_sha), 'invalid workflow source SHA')
|
|
binding = {key: request[key] for key in ('run_id', 'action', 'confirm', 'sha', 'bundle', 'task')}
|
|
if request.get('approval_event'):
|
|
binding.update(approval_event=request['approval_event'], acceptance_digest=request['acceptance_digest'])
|
|
binding.update(repository=REPO, target=TARGET, workflow_sha=control_sha)
|
|
serialized = json.dumps(binding, sort_keys=True, separators=(',', ':')).encode()
|
|
return GUARD_PREFIX + hashlib.sha256(serialized).hexdigest()
|
|
|
|
|
|
def verify_guard_log(log, request, control_sha):
|
|
expected = guard_proof(request, control_sha)
|
|
pattern = re.compile(r'\d{4}-\d{2}-\d{2}T[0-9:.]+Z (' + GUARD_PREFIX + r'[0-9a-f]{64})')
|
|
proofs = [match.group(1) for line in log.decode().splitlines() if (match := pattern.fullmatch(line))]
|
|
require(proofs == [expected], 'Gitea guard dispatch proof missing or different')
|
|
|
|
|
|
def hash_file(path):
|
|
with path.open('rb') as src:
|
|
return hashlib.file_digest(src, 'sha256').hexdigest()
|
|
|
|
|
|
def immutable_file(path):
|
|
info = path.lstat()
|
|
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and not info.st_mode & 0o022,
|
|
'bundle requires regular files owned by the deployment account')
|
|
return path
|
|
|
|
|
|
def verify_bundle(directory, request, archives=True):
|
|
raw = immutable_file(directory / 'release.json').read_bytes()
|
|
require(hashlib.sha256(raw).hexdigest() == request['bundle'], 'release manifest digest mismatch')
|
|
manifest = json.loads(raw)
|
|
require(manifest.get('schema') == 1 and manifest.get('target') == TARGET
|
|
and manifest.get('sha') == request['sha'] and manifest.get('task') == request['task'],
|
|
'bundle commit, task or target mismatch')
|
|
require(set(manifest.get('images', {})) == set(ROLES), 'bundle must include API, web and worker')
|
|
for role, image in manifest['images'].items():
|
|
require(matches(IMAGE, image.get('id')) and matches(HASH, image.get('sha256')), 'invalid image record')
|
|
if archives:
|
|
archive = immutable_file(directory / (role + '.tar'))
|
|
require(hash_file(archive) == image['sha256'], f'{role} archive checksum mismatch')
|
|
versions = manifest.get('migrations')
|
|
require(isinstance(versions, list) and versions and all(type(n) is int and n > 0 for n in versions)
|
|
and versions == sorted(set(versions)), 'invalid migration ledger in manifest')
|
|
return manifest
|
|
|
|
|
|
def verify_contexts(result, contexts):
|
|
latest = {}
|
|
for item in result.get('statuses') or []:
|
|
latest.setdefault(item.get('context'), item.get('status'))
|
|
require(bool(contexts) and all(latest.get(context) == 'success' for context in contexts),
|
|
'required CI status is missing or unsuccessful')
|
|
|
|
|
|
def metadata(task, key):
|
|
value = (task.get('metadata') or {}).get(key)
|
|
return json.loads(value) if isinstance(value, str) else None
|
|
|
|
|
|
def check_task(task, request, receipt=None):
|
|
require(task.get('id') == request['task'] and task.get('spaceId') == SPACE, 'HiFox task or space mismatch')
|
|
status = (task.get('status') or {}).get('id')
|
|
same_publication = receipt and metadata(task, 'cd_release') == receipt.get('hifox')
|
|
require(status == READY or (status == PUBLISHED and same_publication), 'HiFox task is not ready for release')
|
|
evidence = metadata(task, 'cd_acceptance') or {}
|
|
url = urlsplit(evidence.get('evidence', ''))
|
|
require(evidence.get('verdict') == 'passed' and evidence.get('sha') == request['sha']
|
|
and evidence.get('bundle') == request['bundle'] and url.scheme == 'https' and url.hostname
|
|
and not (url.username or url.password or url.query)
|
|
and (not url.fragment or (url.netloc == 'app.hifox.com' and url.fragment.startswith('comment-')
|
|
and matches(UUID, url.fragment.removeprefix('comment-')))),
|
|
'missing or changed business acceptance for this commit and bundle')
|
|
return evidence
|
|
|
|
|
|
def acceptance_digest(acceptance):
|
|
return hashlib.sha256(json.dumps(acceptance, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
|
|
|
|
|
def release_approval(task, timeline, request):
|
|
evidence = check_task(task, request)
|
|
changes = [row for row in timeline if row.get('event') == 'status_changed']
|
|
require(bool(changes), 'missing HiFox status history')
|
|
approval = changes[-1]
|
|
approver = approval.get('actor') or {}
|
|
require((approval.get('metadata') or {}).get('to') == '待发布'
|
|
and approval.get('createdAt') and approval['createdAt'] == task.get('statusChangedAt')
|
|
and approver.get('id') == RELEASE_OWNER and approver.get('type') == 'human'
|
|
and matches(UUID, approval.get('id')), 'the latest transition must be an approval by the release owner')
|
|
task_url = task.get('url', '')
|
|
prefix = task_url + '#comment-'
|
|
require(task_url.startswith('https://app.hifox.com/') and evidence['evidence'].startswith(prefix),
|
|
'automated release requires the same-task acceptance report')
|
|
comment_id = evidence['evidence'].removeprefix(prefix)
|
|
require(matches(UUID, comment_id), 'invalid acceptance report ID')
|
|
reports = [row for row in timeline if row.get('event') == 'commented' and row.get('id') == comment_id]
|
|
require(len(reports) == 1, 'acceptance report missing from task history')
|
|
report = reports[0]
|
|
body = report.get('content') or ''
|
|
require(report.get('createdAt') and (report.get('updatedAt') or report['createdAt']) < approval['createdAt']
|
|
and request['sha'] in body and request['bundle'] in body,
|
|
'the unchanged report must identify the candidate and predate approval')
|
|
binding = {'approval_event': approval['id'], 'acceptance_digest': acceptance_digest(evidence)}
|
|
require(all(request.get(key, value) == value for key, value in binding.items()),
|
|
'HiFox approval or acceptance changed after dispatch')
|
|
return binding, approval['createdAt']
|
|
|
|
|
|
def accepted_task(host, request):
|
|
task = host.task(request['task'])
|
|
if request.get('approval_event'):
|
|
release_approval(task, host.timeline(request['task']), request)
|
|
return check_task(task, request)
|
|
|
|
|
|
def release(host, request):
|
|
validate_request(request)
|
|
path = host.receipt_path
|
|
existing = read_json(path) if path.exists() else None
|
|
if request['action'] == 'writeback':
|
|
require(existing and existing['state'] in ('deployed_hifox_pending', 'published'),
|
|
'writeback requires a verified deployment receipt')
|
|
require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch')
|
|
manifest = verify_bundle(host.bundle_path(request), request, archives=False)
|
|
host.approve(request)
|
|
host.verify(manifest)
|
|
return writeback(host, request, existing)
|
|
|
|
require(existing is None, 'existing release receipt: use writeback or inspect interrupted/failed deployment')
|
|
host.approve(request)
|
|
manifest = verify_bundle(host.bundle_path(request), request)
|
|
host.check_ci(request)
|
|
accepted = accepted_task(host, request)
|
|
before = host.baseline()
|
|
host.pre_switch(before, manifest)
|
|
receipt = {k: request[k] for k in ('sha', 'task', 'bundle', 'run_id')}
|
|
receipt.update(state='preparing', target=TARGET, started_at=stamp(), previous=before)
|
|
record(path, receipt)
|
|
try:
|
|
host.load(host.bundle_path(request), manifest)
|
|
host.pre_switch(before, manifest)
|
|
host.approve(request)
|
|
require(accepted_task(host, request) == accepted, 'business acceptance changed before switch')
|
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
|
receipt.update(state='failed_before_cutover')
|
|
record(path, receipt)
|
|
raise CDError(f'failed_before_cutover: {exc}' if isinstance(exc, CDError) else 'failed_before_cutover') from exc
|
|
|
|
receipt.update(state='cutover_started')
|
|
record(path, receipt)
|
|
try:
|
|
host.switch(before, manifest)
|
|
evidence = host.verify(manifest)
|
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
|
status = 'rollback_failed'
|
|
try:
|
|
host.rollback(before)
|
|
status = 'rolled_back'
|
|
except (CDError, OSError, ValueError, KeyError):
|
|
pass
|
|
receipt.update(state=status, finished_at=stamp(), failure=str(exc) if isinstance(exc, CDError) else 'operation failed')
|
|
record(path, receipt)
|
|
raise CDError(f'{status}: deployment verification failed') from exc
|
|
|
|
receipt.update(state='deployed_hifox_pending', deployed_at=stamp(), evidence=evidence)
|
|
receipt['hifox'] = {**evidence, 'target': TARGET, 'bundle': request['bundle'], 'deployed_at': receipt['deployed_at'],
|
|
'login_url': LOGIN_URL,
|
|
'run_url': f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'}
|
|
record(path, receipt)
|
|
return writeback(host, request, receipt)
|
|
|
|
|
|
def writeback(host, request, receipt):
|
|
try:
|
|
task = host.task(request['task'])
|
|
check_task(task, request, receipt)
|
|
host.callback(request['task'], task, receipt.get('hifox'))
|
|
receipt.update(state='published')
|
|
record(host.receipt_path, receipt)
|
|
except (CDError, OSError, ValueError, KeyError) as exc:
|
|
raise CDError('DEPLOYED_HIFOX_PENDING: retry writeback for this receipt') from exc
|
|
return receipt
|
|
|
|
|
|
def with_images(config, images):
|
|
result = copy.deepcopy(config)
|
|
require(set(result['services']) == set(ROLES), 'unexpected Compose service set')
|
|
for role in ROLES:
|
|
result['services'][role]['image'] = images[role]
|
|
return result
|
|
|
|
|
|
def run(argv, purpose, *, cwd=None, input_data=None, timeout=30, allowed=(0,), env=None):
|
|
try:
|
|
result = subprocess.run(argv, cwd=cwd, input=input_data, capture_output=True, timeout=timeout, env=env)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise CDError(f'{purpose} unavailable or timed out') from exc
|
|
require(result.returncode in allowed, f'{purpose} failed (exit {result.returncode})')
|
|
return result.stdout
|
|
|
|
|
|
class NoRedirect(HTTPRedirectHandler):
|
|
def redirect_request(self, *_args, **_kwargs):
|
|
raise CDError('unexpected HTTP redirect')
|
|
|
|
|
|
def http(url, *, direct=False, limit=2**20):
|
|
handlers = [NoRedirect()]
|
|
if direct:
|
|
handlers.append(ProxyHandler({}))
|
|
try:
|
|
with build_opener(*handlers).open(url, timeout=10) as response:
|
|
data = response.read(limit + 1)
|
|
require(len(data) <= limit, 'HTTP response exceeds limit')
|
|
return data
|
|
except (URLError, TimeoutError) as exc:
|
|
raise CDError('HTTP read failed') from exc
|
|
|
|
|
|
class Assets(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.paths = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = dict(attrs)
|
|
path = attrs.get('src' if tag == 'script' else 'href', '')
|
|
if path and re.fullmatch(r'/assets/[A-Za-z0-9_.-]+\.(?:js|css)', path):
|
|
self.paths.append(path)
|
|
|
|
|
|
def disk_assets(html):
|
|
parser = Assets()
|
|
parser.feed(html.decode())
|
|
require(any(p.endswith('.js') for p in parser.paths), 'web index has no compiled entry point')
|
|
return sorted(set(parser.paths))
|
|
|
|
|
|
def image_inspect(image):
|
|
return json.loads(run(['docker', 'image', 'inspect', image], 'read image identity'))[0]
|
|
|
|
|
|
def inspect(name):
|
|
obj = json.loads(run(['docker', 'container', 'inspect', name], 'read running container'))[0]
|
|
require(obj['State']['Status'] == 'running' and matches(IMAGE, obj.get('Image')), 'expected container is not running')
|
|
return obj
|
|
|
|
|
|
def image_label(image):
|
|
return image_inspect(image)['Config']['Labels'].get(REVISION, '')
|
|
|
|
|
|
def api_identity(container):
|
|
raw = run(['docker', 'exec', container, '/app/api', '--version-record-status'], 'read API build identity', allowed=(0, 1))
|
|
obj = json.loads(raw)
|
|
require(matches(SHA, obj.get('commit_sha')), 'API binary has no full build identity')
|
|
return obj['commit_sha']
|
|
|
|
|
|
def migration_files(git, sha):
|
|
paths = git('ls-tree', '-r', '--name-only', sha, 'internal/platform/postgres/migrations').decode().splitlines()
|
|
return sorted(int(Path(line).name.split('_', 1)[0]) for line in paths if line.endswith('.up.sql'))
|
|
|
|
|
|
def ledger(container):
|
|
probe = Path(__file__).with_name('cd_migration_probe.py').read_text()
|
|
data = run(['docker', 'exec', container, 'python3', '-c', probe], 'read migration ledger')
|
|
result = json.loads(data)
|
|
require(isinstance(result, list) and result, 'unexpected migration ledger result')
|
|
return result
|
|
|
|
|
|
def verify_archive(archive, expected):
|
|
# docker save by ID omits mutable RepoTags. Refuse archives that could retag another app.
|
|
with tarfile.open(archive, 'r:') as t:
|
|
entry = t.getmember('manifest.json')
|
|
require(entry.isfile() and entry.size < 65536, 'unexpected Docker archive manifest')
|
|
containers = json.load(t.extractfile(entry))
|
|
require(len(containers) == 1 and not containers[0].get('RepoTags'), 'archive must contain one untagged image')
|
|
config = t.getmember(containers[0]['Config'])
|
|
require(config.isfile() and config.size < 2**20, 'unexpected Docker image config')
|
|
require('sha256:' + hashlib.sha256(t.extractfile(config).read()).hexdigest() == expected,
|
|
'archive config does not match approved image ID')
|
|
for member in t:
|
|
require((member.isdir() or member.isfile()) and not member.name.startswith('/')
|
|
and '..' not in Path(member.name).parts, 'unexpected Docker archive path')
|
|
if 'index.json' in t.getnames():
|
|
index = t.getmember('index.json')
|
|
require(index.isfile() and index.size < 65536, 'unexpected OCI archive index')
|
|
oci = json.load(t.extractfile(index))
|
|
require(len(oci.get('manifests', [])) == 1 and all(
|
|
key not in descriptor.get('annotations', {}) for descriptor in oci['manifests']
|
|
for key in ('org.opencontainers.image.ref.name', 'io.containerd.image.name')),
|
|
'archive OCI index contains a mutable name')
|
|
|
|
|
|
class LiveHost:
|
|
def __init__(self, request):
|
|
self.receipt_path = DATA / 'receipts' / (request['bundle'] + '.json')
|
|
self.compose = APP / 'compose.json'
|
|
self.git_path = DATA / 'source.git'
|
|
self.containers = {role: 'kg-cloud-' + role for role in ROLES}
|
|
self.health_url = 'http://127.0.0.1:8088'
|
|
self.request = request
|
|
|
|
def bundle_path(self, request):
|
|
return DATA / 'bundles' / request['bundle']
|
|
|
|
def git(self, *args):
|
|
return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90)
|
|
|
|
def approve(self, request):
|
|
repo_url = f'{FORGE}/{REPO}'
|
|
run_id = request['run_id']
|
|
run_api = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{run_id}'
|
|
dispatch = json.loads(http(run_api))
|
|
control_sha = dispatch.get('head_sha')
|
|
trigger_actor = dispatch.get('trigger_actor') or dispatch.get('actor') or {}
|
|
require(str(dispatch.get('id')) == run_id and dispatch.get('event') == 'workflow_dispatch'
|
|
and dispatch.get('path') == 'deploy-production.yml@refs/heads/main'
|
|
and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running')
|
|
and (dispatch.get('actor') or {}).get('id') == 1 and trigger_actor.get('id') == 1
|
|
and matches(SHA, control_sha), 'Gitea run is not an active deployment dispatched by the release owner')
|
|
# Repo secrets can be read by other workflows. Only a completed guard from
|
|
# the installed workflow and controller may authorize the exact inputs.
|
|
try:
|
|
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
|
|
except CDError:
|
|
self.fetch_source()
|
|
self.git('merge-base', '--is-ancestor', control_sha, 'refs/remotes/origin/main')
|
|
trusted = {'scripts/production_cd.py': Path(__file__).resolve(),
|
|
'.gitea/workflows/deploy-production.yml': DATA / 'deploy-production.yml'}
|
|
for name, local in trusted.items():
|
|
require(self.git('show', f'{control_sha}:{name}') == immutable_file(local).read_bytes(),
|
|
'Gitea approval workflow differs from installed control files')
|
|
jobs = json.loads(http(run_api + '/jobs')).get('jobs') or []
|
|
guards = [job for job in jobs if job.get('name') == 'guard' and job.get('conclusion') == 'success'
|
|
and str(job.get('run_id')) == run_id and job.get('head_sha') == control_sha
|
|
and matches(re.compile('[1-9][0-9]{0,14}'), str(job.get('id')))]
|
|
require(len(guards) == 1, 'Gitea approval guard has not succeeded')
|
|
guard_id = guards[0]['id']
|
|
verify_guard_log(http(f'{repo_url}/actions/runs/{run_id}/jobs/{guard_id}/logs'), request, control_sha)
|
|
|
|
def fetch_source(self):
|
|
self.git('fetch', '--no-tags', '--prune', f'{FORGE}/{REPO}.git', '+refs/heads/*:refs/remotes/origin/*')
|
|
|
|
def check_ci(self, request):
|
|
self.fetch_source()
|
|
refs = self.git('for-each-ref', '--format=%(refname)', '--contains', request['sha'], 'refs/remotes/origin/')
|
|
require(refs.strip(), 'candidate commit is not reachable from a repository branch')
|
|
contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines()
|
|
verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')),
|
|
[line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')])
|
|
|
|
def hifox(self, *args):
|
|
cli = DATA / 'bin/hifox'
|
|
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(DATA / 'hifox-home'), 'CI': '1'}
|
|
raw = run([str(cli), '--organization-id', ORG, *args, '--output', 'json'], 'HiFox operation', env=env)
|
|
result = json.JSONDecoder().raw_decode(raw.decode().strip())[0]
|
|
require(isinstance(result, (dict, list)) and (not isinstance(result, dict)
|
|
or (result.get('success') is not False and not result.get('error'))),
|
|
'HiFox did not return a successful result')
|
|
return result
|
|
|
|
def task(self, task):
|
|
return self.hifox('task', 'view', task)
|
|
|
|
def timeline(self, task):
|
|
return self.hifox('api', 'get', f'/rest/v1/tasks/{task}/timeline')
|
|
|
|
def comment_once(self, task, key, text):
|
|
marker = f'KG-CD {key}'
|
|
body = f'{text}\n\n`{marker}`'
|
|
|
|
def present():
|
|
return any(row.get('event') == 'commented' and row.get('content') == body
|
|
and (row.get('actor') or {}).get('id') == RELEASE_OWNER for row in self.timeline(task))
|
|
|
|
if not present():
|
|
self.hifox('task', 'comment', task, '--body', body)
|
|
require(present(), 'HiFox release comment readback mismatch')
|
|
|
|
def report_failure(self, request):
|
|
if not self.receipt_path.exists():
|
|
return
|
|
receipt = read_json(self.receipt_path)
|
|
if any(receipt.get(key) != request[key] for key in ('task', 'sha', 'bundle', 'run_id')):
|
|
return
|
|
messages = {
|
|
'failed_before_cutover': '发布检查失败,未切换主站。',
|
|
'rolled_back': '发布核验失败,已恢复原版本并复验。',
|
|
'rollback_failed': '发布核验失败,回滚未通过复验,请检查主站。',
|
|
'deployed_hifox_pending': '主站已部署并核验,HiFox 回写待完成。请重试回写。',
|
|
}
|
|
state = receipt['state']
|
|
if state in messages:
|
|
run_url = f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'
|
|
self.comment_once(request['task'], f'{state} #{request["run_id"]}',
|
|
f'{messages[state]}\n\n候选 `{request["sha"]}`\n\n[发布运行]({run_url})')
|
|
|
|
def callback(self, task_id, task, expected):
|
|
if metadata(task, 'cd_release') != expected:
|
|
self.hifox('task', 'metadata', 'set', task_id, 'cd_release', json.dumps(json.dumps(expected)))
|
|
fresh = self.task(task_id)
|
|
check_task(fresh, self.request, {'hifox': expected})
|
|
require(metadata(fresh, 'cd_release') == expected, 'HiFox release receipt readback mismatch')
|
|
images = '\n'.join(f'- {role} `{expected["images"][role]}`' for role in ROLES)
|
|
self.comment_once(task_id, f'published {expected["bundle"]}',
|
|
f'云治理主站发布完成,健康检查通过。\n\n'
|
|
f'运行版本 `{expected.get("version") or expected["sha"]}`\n\n'
|
|
f'提交 `{expected["sha"]}`\n\n{images}\n\n'
|
|
f'[登录主站]({LOGIN_URL}) · [发布运行]({expected["run_url"]})')
|
|
fresh = self.task(task_id)
|
|
check_task(fresh, self.request, {'hifox': expected})
|
|
require(metadata(fresh, 'cd_release') == expected, 'HiFox release changed during comment writeback')
|
|
if fresh['status']['id'] != PUBLISHED:
|
|
self.hifox('task', 'edit', task_id, '--status', PUBLISHED)
|
|
final = self.task(task_id)
|
|
final_status = (final.get('status') or {}).get('id')
|
|
require(final_status == PUBLISHED and metadata(final, 'cd_release') == expected, 'HiFox published readback mismatch')
|
|
|
|
def compose_cmd(self, *args):
|
|
return ['docker', 'compose', '-p', PROJECT, '-f', str(self.compose), *args]
|
|
|
|
def rendered_config(self):
|
|
raw = run(self.compose_cmd('config', '--format', 'json'), 'render Compose configuration', cwd=APP)
|
|
return json.loads(raw)['services']
|
|
|
|
def runtime(self):
|
|
rendered = self.rendered_config()
|
|
require(set(rendered) == set(ROLES), 'unexpected Compose service set')
|
|
result = {}
|
|
for role, name in self.containers.items():
|
|
c = inspect(name)
|
|
labels = c['Config']['Labels']
|
|
require(labels.get('com.docker.compose.project') == PROJECT
|
|
and labels.get('com.docker.compose.service') == role
|
|
and labels.get('com.docker.compose.project.working_dir') == str(APP)
|
|
and labels.get('com.docker.compose.project.config_files') == str(self.compose),
|
|
f'{role} does not belong to the approved Compose chain')
|
|
config = rendered[role]
|
|
image = image_inspect(c['Image'])
|
|
require(image_inspect(config['image'])['Id'] == c['Image'], f'{role} Compose image has drifted')
|
|
env = lambda entries: dict(item.split('=', 1) for item in entries or [])
|
|
expected_env = env(image['Config'].get('Env'))
|
|
expected_env.update({key: str(value) for key, value in (config.get('environment') or {}).items()
|
|
if value is not None})
|
|
require(env(c['Config'].get('Env')) == expected_env, f'{role} running environment has drifted')
|
|
expected_mounts = config.get('volumes') or []
|
|
require(all(mount['type'] == 'bind' for mount in expected_mounts), 'unexpected non-bind Compose volume')
|
|
mounts = {(v['Source'], v['Destination'], not v['RW']) for v in c.get('Mounts', []) if v['Type'] == 'bind'}
|
|
require(mounts == {(v['source'], v['target'], v.get('read_only', False)) for v in expected_mounts}
|
|
and len(mounts) == len(c.get('Mounts', [])), f'{role} running mounts have drifted')
|
|
require(not config.get('network_mode') or config['network_mode'] == c['HostConfig']['NetworkMode'],
|
|
f'{role} running network has drifted')
|
|
result[role] = c['Image']
|
|
return result
|
|
|
|
def baseline(self):
|
|
images = self.runtime()
|
|
self.probe_web()
|
|
return {'compose_sha256': hash_file(self.compose), 'images': images,
|
|
'revisions': {role: image_label(image) for role, image in images.items()}}
|
|
|
|
def pre_switch(self, previous, manifest):
|
|
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
|
|
'target baseline changed during deployment')
|
|
for revision in set(previous['revisions'].values()):
|
|
require(matches(SHA, revision), 'running image has no source revision')
|
|
self.git('merge-base', '--is-ancestor', revision, manifest['sha'])
|
|
require(migration_files(self.git, manifest['sha']) == manifest['migrations'], 'source migration set differs from bundle')
|
|
for role in ('api', 'worker'):
|
|
require(ledger(self.containers[role]) == manifest['migrations'],
|
|
f'{role} migration ledger mismatch; verify schema changes outside CD')
|
|
|
|
def load(self, directory, manifest):
|
|
archives = {role: directory / (role + '.tar') for role in ROLES}
|
|
for role, archive in archives.items():
|
|
verify_archive(archive, manifest['images'][role]['id'])
|
|
for role, archive in archives.items():
|
|
image = manifest['images'][role]['id']
|
|
run(['docker', 'image', 'load', '-i', str(archive)], f'load {role} image', timeout=600)
|
|
require(image_inspect(image)['Id'] == image and image_label(image) == manifest['sha'],
|
|
f'{role} loaded image differs from approved build')
|
|
|
|
@property
|
|
def backup(self):
|
|
return self.receipt_path.with_suffix('.compose.backup')
|
|
|
|
def switch(self, previous, manifest):
|
|
current = self.compose.read_bytes()
|
|
require(hashlib.sha256(current).hexdigest() == previous['compose_sha256'], 'Compose changed before switch')
|
|
images = {role: manifest['images'][role]['id'] for role in ROLES}
|
|
changed = with_images(json.loads(current), images)
|
|
for role, image in previous['images'].items():
|
|
run(['docker', 'image', 'tag', image, f'knowledge-governance-cd-rollback:{self.request["bundle"]}-{role}'],
|
|
'retain rollback image')
|
|
atomic_write(self.backup, current)
|
|
atomic_write(self.compose, (json.dumps(changed, indent=2) + '\n').encode(), self.compose.stat().st_mode & 0o777)
|
|
self.recreate()
|
|
|
|
def recreate(self):
|
|
script = Path(__file__).with_name('deploy-api-web.sh')
|
|
run(['bash', str(script), '--verified-compose', str(APP), PROJECT, str(self.compose), 'api', 'web', 'worker'],
|
|
'switch governance Compose services', timeout=120)
|
|
|
|
def probe_web(self):
|
|
origin = self.health_url
|
|
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'web proxy health probe failed')
|
|
require(json.loads(http('http://127.0.0.1:18098/readyz', direct=True)).get('status') in ('ok', 'ready'),
|
|
'API readiness probe failed')
|
|
index = http(origin + '/', direct=True)
|
|
return index, {path: hashlib.sha256(http(origin + path, direct=True, limit=32 * 2**20)).hexdigest()
|
|
for path in disk_assets(index)}
|
|
|
|
def verify_web_image(self):
|
|
origin_index, assets = self.probe_web()
|
|
name = self.containers['web']
|
|
def read_asset(path):
|
|
return run(['docker', 'exec', name, 'cat', '/usr/share/nginx/html' + path], 'read running web asset')
|
|
|
|
require(origin_index == read_asset('/index.html'), 'served index differs from running web image')
|
|
for path, served_digest in assets.items():
|
|
require(hashlib.sha256(read_asset(path)).hexdigest() == served_digest, 'served asset differs from running web image')
|
|
return assets
|
|
|
|
def verify(self, manifest):
|
|
images = {role: manifest['images'][role]['id'] for role in ROLES}
|
|
def check():
|
|
require(self.runtime() == images, 'running image IDs differ from approved bundle')
|
|
require(api_identity(self.containers['api']) == manifest['sha'], 'running API SHA mismatches approval')
|
|
assets = self.verify_web_image()
|
|
version = image_inspect(images['api'])['Config']['Labels'].get('org.opencontainers.image.version')
|
|
if not isinstance(version, str) or not re.fullmatch(r'[0-9A-Za-z.+_-]{1,128}', version):
|
|
version = manifest['sha']
|
|
return {'sha': manifest['sha'], 'version': version, 'images': images, 'health': 'ok', 'asset_sha256': assets}
|
|
|
|
return wait_ready(check)
|
|
|
|
def rollback(self, previous):
|
|
if not self.backup.exists():
|
|
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
|
|
'rollback baseline cannot be established')
|
|
self.verify_web_image()
|
|
return
|
|
original = self.backup.read_bytes()
|
|
require(hashlib.sha256(original).hexdigest() == previous['compose_sha256'], 'rollback backup checksum mismatch')
|
|
source = json.loads(original)
|
|
for role, image in previous['images'].items():
|
|
reference = source['services'][role]['image']
|
|
require(image_inspect(reference)['Id'] == image, 'rollback source tag has drifted')
|
|
atomic_write(self.compose, original, self.compose.stat().st_mode & 0o777)
|
|
self.recreate()
|
|
|
|
def check():
|
|
require(self.runtime() == previous['images'], 'rollback running images mismatch')
|
|
require(api_identity(self.containers['api']) == previous['revisions']['api'], 'rollback API SHA mismatch')
|
|
self.verify_web_image()
|
|
|
|
wait_ready(check)
|
|
|
|
|
|
def wait_ready(check, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
try:
|
|
return check()
|
|
except (CDError, OSError, KeyError, ValueError):
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
time.sleep(3)
|
|
|
|
|
|
def event_request(event, env):
|
|
require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main'
|
|
and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required')
|
|
inputs = event.get('inputs') or {}
|
|
request = {name: inputs.get(name) or '' for name in
|
|
('action', 'sha', 'task', 'bundle', 'confirm', 'approval_event', 'acceptance_digest')}
|
|
request['run_id'] = env.get('GITHUB_RUN_ID', '')
|
|
validate_request(request)
|
|
return request
|
|
|
|
|
|
def pack(args):
|
|
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
|
|
require(not args.output.exists(), 'bundle output already exists')
|
|
source = lambda *argv: run(['git', *argv], 'read candidate source')
|
|
require(source('rev-parse', args.sha + '^{commit}').decode().strip() == args.sha, 'source commit not found')
|
|
images = {}
|
|
versions = migration_files(source, args.sha)
|
|
manifest = dict(schema=1, target=TARGET, sha=args.sha, task=args.task, images=images, migrations=versions)
|
|
with tempfile.TemporaryDirectory(dir=args.output.parent) as directory:
|
|
build = Path(directory) / "sealed"
|
|
build.mkdir(mode=0o700)
|
|
for role in ROLES:
|
|
image = image_inspect(getattr(args, role).removeprefix('sha256:'))
|
|
image_id = image['Id']
|
|
require(matches(IMAGE, image_id) and image_label(image_id) == args.sha, f'{role} source label mismatch')
|
|
archive = build / (role + '.tar')
|
|
run(['docker', 'image', 'save', '-o', str(archive), image_id], f'archive {role} image', timeout=900)
|
|
verify_archive(archive, image_id)
|
|
archive.chmod(0o600)
|
|
images[role] = {'id': image_id, 'sha256': hash_file(archive)}
|
|
record(build / 'release.json', manifest)
|
|
for item in build.iterdir():
|
|
with item.open("rb") as artifact:
|
|
os.fsync(artifact.fileno())
|
|
sync_dir(build)
|
|
build.rename(args.output)
|
|
sync_dir(args.output.parent)
|
|
print(hash_file(args.output / 'release.json'))
|
|
|
|
|
|
def attest(args):
|
|
request = {'sha': args.sha, 'task': args.task, 'bundle': hash_file(args.directory / 'release.json')}
|
|
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
|
|
manifest = verify_bundle(args.directory, request)
|
|
before = {role: inspect(getattr(args, role)) for role in ROLES}
|
|
for role, obj in before.items():
|
|
require(obj['Image'] == manifest['images'][role]['id'] and obj['Name'] != '/kg-cloud-' + role,
|
|
'isolated test runtime image differs from bundle')
|
|
require(api_identity(args.api) == args.sha and ledger(args.api) == manifest['migrations']
|
|
and ledger(args.worker) == manifest['migrations'], 'test runtime SHA or schema differs')
|
|
base = urlsplit(args.test_url)
|
|
require(base.scheme in ('http', 'https') and base.hostname and not (base.username or base.password or base.query or base.fragment)
|
|
and base.path in ('', '/') and base.port != 8088, 'provide isolated test endpoint')
|
|
origin = args.test_url.rstrip('/')
|
|
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'isolated test health probe failed')
|
|
index = http(origin + '/', direct=True)
|
|
require(index == run(['docker', 'exec', args.web, 'cat', '/usr/share/nginx/html/index.html'], 'read test web index'),
|
|
'test endpoint is serving another web image')
|
|
for role, obj in before.items():
|
|
require(inspect(getattr(args, role))['Id'] == obj['Id'], 'test instance changed during readback')
|
|
print(json.dumps({'sha': args.sha, 'bundle': request['bundle'], 'test_url': args.test_url,
|
|
'images': {role: obj['Image'] for role, obj in before.items()}}, sort_keys=True))
|
|
|
|
|
|
def cli():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest='command', required=True)
|
|
for command in ('guard', 'send', 'serve', 'pack', 'attest'):
|
|
sub = commands.add_parser(command)
|
|
if command in ('guard', 'send'):
|
|
sub.add_argument('--event', type=Path, required=True)
|
|
if command in ('pack', 'attest'):
|
|
sub.add_argument('--sha', required=True)
|
|
sub.add_argument('--task', required=True)
|
|
if command in ('pack', 'attest'):
|
|
for role in ROLES:
|
|
sub.add_argument('--' + role, required=True)
|
|
if command == 'pack':
|
|
sub.add_argument('--output', type=Path, required=True)
|
|
if command == 'attest':
|
|
sub.add_argument('--directory', type=Path, required=True)
|
|
sub.add_argument('--test-url', required=True)
|
|
args = parser.parse_args()
|
|
if args.command in ('guard', 'send'):
|
|
request = event_request(read_json(args.event), os.environ)
|
|
if args.command == 'guard':
|
|
print(guard_proof(request, os.environ.get('GITHUB_SHA')))
|
|
else:
|
|
send(request)
|
|
elif args.command == 'pack':
|
|
pack(args)
|
|
elif args.command == 'attest':
|
|
attest(args)
|
|
else:
|
|
raw = sys.stdin.buffer.read(8193)
|
|
require(len(raw) <= 8192, 'request exceeds size limit')
|
|
request = json.loads(raw)
|
|
validate_request(request)
|
|
with deploy_lock(DATA / 'release.lock'):
|
|
host = LiveHost(request)
|
|
for item in (DATA / 'receipts').glob('*.json'):
|
|
if read_json(item)['state'] in ('cutover_started', 'rollback_failed'):
|
|
raise CDError('previous deployment outcome requires operator reconciliation')
|
|
try:
|
|
result = release(host, request)
|
|
except (CDError, OSError, ValueError, KeyError):
|
|
try:
|
|
host.report_failure(request)
|
|
except (CDError, OSError, ValueError, KeyError):
|
|
print('CD: HiFox failure comment pending', file=sys.stderr)
|
|
raise
|
|
print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False))
|
|
|
|
|
|
def send(request):
|
|
key = os.environ.get('KG_CD_SSH_KEY', '')
|
|
known_hosts = os.environ.get('KG_CD_KNOWN_HOSTS', '')
|
|
host = os.environ.get('KG_CD_SSH_HOST', '')
|
|
port = os.environ.get('KG_CD_SSH_PORT', '22') or '22'
|
|
require(key.strip() and known_hosts.strip(), 'missing KG_CD_SSH_KEY or KG_CD_KNOWN_HOSTS')
|
|
require(host in ('100.85.111.121', '::1') and port.isdecimal() and 1 <= int(port) <= 65535,
|
|
'configure the direct or approved tunnel address for the target release endpoint')
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
temp = Path(directory)
|
|
for filename, value in (('key', key), ('known_hosts', known_hosts)):
|
|
atomic_write(temp / filename, (value.strip() + '\n').encode())
|
|
argv = ['ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'IdentitiesOnly=yes',
|
|
'-o', 'ClearAllForwardings=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'HostKeyAlias=kg-governance-cd',
|
|
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', f'UserKnownHostsFile={temp / "known_hosts"}',
|
|
'-i', str(temp / 'key'), '-p', port, '-T', 'root@' + host, 'kg-cd-v1']
|
|
try:
|
|
result = subprocess.run(argv, input=json.dumps(request).encode(), capture_output=True, timeout=2100)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise CDError('target outcome unknown; inspect durable receipt before retry') from exc
|
|
if result.returncode:
|
|
error = result.stderr.decode(errors='replace')
|
|
for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:',
|
|
'target preflight rejected', 'deployment lock busy', 'writeback requires a verified deployment receipt',
|
|
'Gitea approval guard has not succeeded', 'Gitea guard dispatch proof missing or different'):
|
|
if marker in error:
|
|
raise CDError(error.strip().splitlines()[-1])
|
|
raise CDError('target rejected request or outcome unknown; inspect durable receipt')
|
|
response = json.loads(result.stdout)
|
|
require(response.get('state') == 'published' and all(response.get(k) == request[k] for k in ('sha', 'task', 'bundle')),
|
|
'invalid or unverified deployment receipt')
|
|
print(json.dumps(response, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
cli()
|
|
except CDError as error:
|
|
print(f'CD: {error}', file=sys.stderr)
|
|
sys.exit(1)
|
|
except (OSError, KeyError, ValueError, TypeError, tarfile.TarError) as error:
|
|
print(f'CD: operation could not be verified ({type(error).__name__})', file=sys.stderr)
|
|
sys.exit(1)
|