feat(deploy): 接通云治理主站的人工确认发布流水线
build / build (push) Successful in 2m0s

This commit is contained in:
姚章浩
2026-11-23 01:35:18 +08:00
parent d7d636792d
commit 0b5de6ef7e
10 changed files with 1092 additions and 2 deletions
+56
View File
@@ -0,0 +1,56 @@
"""Run inside a governance container. Read the real application DB ledger only."""
import json
import os
import subprocess
from urllib.parse import parse_qsl, unquote, urlsplit
def connection_env(env):
# Keep only the connection properties actually shared by libpq and pgx.
# Unknown DSN options can change the database identity or schema search path.
u = urlsplit(env['DATABASE' + '_URL'])
if u.scheme not in ('postgres', 'postgresql') or not u.hostname or not u.username or not u.path[1:] or u.fragment:
raise ValueError('invalid database URL')
query = parse_qsl(u.query, strict_parsing=True, keep_blank_values=True)
supported = {'sslmode': 'PGSSLMODE', 'sslrootcert': 'PGSSLROOTCERT', 'sslcert': 'PGSSLCERT', 'sslkey': 'PGSSLKEY'}
if len(dict(query)) != len(query) or any(key not in supported or not val for key, val in query):
raise ValueError('unknown database connection options')
safe = {'PATH': '/usr/bin:/bin', 'HOME': env.get('HOME', '/nonexistent'), 'LC_ALL': 'C',
'PGHOST': u.hostname, 'PGPORT': str(u.port or 5432), 'PGDATABASE': unquote(u.path[1:]),
'PGUSER': unquote(u.username), 'PGCONNECT_TIMEOUT': '5',
'PGOPTIONS': '-c default_transaction_read_only=on -c statement_timeout=5000'}
for name in ('PGPASSFILE', 'PGSSLMODE', 'PGSSLROOTCERT', 'PGSSLCERT', 'PGSSLKEY'):
if env.get(name):
safe[name] = env[name]
for name in env:
if name.startswith('PG') and name not in safe:
raise ValueError('unknown inherited database connection options')
safe.update({supported[key]: value for key, value in query})
if u.password:
safe['PGPASSWORD'] = unquote(u.password)
for name in ('PGDATABASE', 'PGUSER'):
if any(char in safe[name] for char in '\r\n\x00='):
raise ValueError('unsafe database identifier')
return safe
def main():
env = connection_env(os.environ)
sql = """SELECT json_build_object('database', current_database(), 'user', current_user,
'read_only', current_setting('transaction_read_only'),
'versions', (SELECT json_agg(version ORDER BY version) FROM schema_migrations));"""
p = subprocess.run(['psql', '-XAtw', '-v', 'ON_ERROR_STOP=1', '-c', sql],
env=env, capture_output=True, timeout=12)
if p.returncode:
raise ValueError('query failed')
data = json.loads(p.stdout)
if data['read_only'] != 'on' or data['database'] != env['PGDATABASE'] or data['user'] != env['PGUSER']:
raise ValueError('database identity mismatch')
print(json.dumps(data['versions']))
if __name__ == '__main__':
try:
main()
except (OSError, ValueError, KeyError, subprocess.TimeoutExpired):
raise SystemExit('read-only migration probe failed') from None
+4
View File
@@ -360,6 +360,10 @@ def cmd_internal_contracts(go: str) -> int:
cwd=ROOT).returncode
if qa_rc:
return qa_rc
cd_rc = subprocess.run(py("-m", "unittest", "discover", "-s", "scripts", "-p", "test_production_cd.py", "-v"),
cwd=ROOT).returncode
if cd_rc:
return cd_rc
snapshot_rc = subprocess.run(py("scripts/development_snapshot.py","--self-test"),cwd=ROOT).returncode
if snapshot_rc:
return 1
+17 -1
View File
@@ -1,6 +1,20 @@
#!/usr/bin/env bash
# 重建并上线 governance api + web(本机 docker build 需要宿主代理 + --network host)
set -u
# Share the image-only Compose switch with CD without entering the legacy build path.
compose_up() {
"${compose[@]}" up -d --no-build --no-deps "$@"
}
if [[ "${1:-}" == --verified-compose ]]; then
[[ "$#" == 7 && "$5" == api && "$6" == web && "$7" == worker && "$4" == /* ]] || exit 2
cd -- "$2" || exit 1
compose=(docker compose -p "$3" -f "$4")
compose_up --pull never "$5" "$6" "$7"
exit $?
fi
cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1
[ -z "$(git status --porcelain --untracked-files=normal)" ] || { printf "%s\n" "REFUSED_DIRTY_SOURCE"; exit 1; }
TS=$(date +%Y%m%d-%H%M%S)
@@ -103,7 +117,9 @@ if [ "${GATES_PREV}" = "true" ] && [[ "${COMPOSE_FILES_EFFECTIVE}" != *compose.r
echo " 确实要关闸:ALLOW_GATE_OFF=1 bash scripts/deploy-api-web.sh"
exit 1
fi
docker compose ${COMPOSE_FILES_EFFECTIVE} up -d --no-build --no-deps api web || { echo "UP_FAILED"; exit 1; }
read -r -a compose_file_args <<< "$COMPOSE_FILES_EFFECTIVE"
compose=(docker compose "${compose_file_args[@]}")
compose_up api web || { echo "UP_FAILED"; exit 1; }
echo "=== 5. 状态"
docker ps --filter name=knowledge-governance-p1 --format '{{.Names}} {{.Status}} {{.Image}}'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
umask 077
[ "${SSH_ORIGINAL_COMMAND:-}" = kg-cd-v1 ] || exit 1
exec /usr/bin/env -i HOME=/opt/knowledge-governance/cd PATH=/usr/bin:/bin LANG=C.UTF-8 \
GIT_TERMINAL_PROMPT=0 /usr/bin/python3 -I /opt/knowledge-governance/cd/bin/production_cd.py serve
+726
View File
@@ -0,0 +1,726 @@
#!/usr/bin/env python3
"""Governance application CD. The installed host copy owns the release boundary."""
import argparse
import base64
import binascii
from contextlib import contextmanager
import copy
from datetime import datetime, timezone
import fcntl
import hashlib
from html.parser import HTMLParser
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
import tarfile
import tempfile
import time
from urllib.error import URLError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, ProxyHandler, build_opener
REPO = 'Weknora/knowledge-governance'
FORGE = 'https://gitea.zheromiku.top'
TARGET = 'cloud-test-governance-main'
ORG = '169e6f7f-e508-4a1f-9a9b-63d240439a3d'
SPACE = '106e05d1-4091-40b9-8c57-f76dc94f7825'
READY = '3e15556f-ca19-417c-8bfd-666634ea5d29'
PUBLISHED = '16185166-b8e9-4761-9926-845a1f41c96d'
ROLES = ('api', 'web', 'worker')
REVISION = 'org.opencontainers.image.revision'
DATA = Path('/opt/knowledge-governance/cd')
APP = DATA.parent
PROJECT = 'knowledge-governance-cloud-test'
SHA = re.compile('[0-9a-f]{40}')
HASH = re.compile('[0-9a-f]{64}')
IMAGE = re.compile('sha256:[0-9a-f]{64}')
UUID = re.compile('[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}')
NAMESPACE = 'knowledge-governance-release-v1'
class CDError(RuntimeError):
pass
def require(ok, message):
if not ok:
raise CDError(message)
def stamp():
return datetime.now(timezone.utc).isoformat()
def read_json(path):
return json.loads(path.read_bytes())
def sync_dir(directory):
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)
def atomic_write(path, data, mode=0o600):
fd, temp = tempfile.mkstemp(dir=path.parent)
try:
with os.fdopen(fd, 'wb') as out:
os.fchmod(out.fileno(), mode)
out.write(data)
out.flush()
os.fsync(out.fileno())
os.replace(temp, path)
sync_dir(path.parent)
finally:
Path(temp).unlink(missing_ok=True)
def record(path, value):
atomic_write(path, (json.dumps(value, ensure_ascii=False, sort_keys=True) + '\n').encode())
@contextmanager
def deploy_lock(path):
fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise CDError('deployment lock busy') from exc
yield
def matches(pattern, text):
return isinstance(text, str) and pattern.fullmatch(text) is not None
def validate_request(request):
actions = {'deploy': 'deploy-production', 'writeback': 'retry-writeback'}
action = request.get('action')
require(action in actions and request.get('confirm') == actions[action], 'explicit confirmation required')
require(matches(SHA, request.get('sha')) and matches(UUID, request.get('task'))
and matches(HASH, request.get('bundle')), 'require full SHA, bundle SHA256 and task UUID')
require(matches(re.compile('[1-9][0-9]{0,14}'), request.get('run_id')), 'invalid Gitea run ID')
approval = request.get('approval')
require(isinstance(approval, str) and len(approval) <= 4096 and (action == 'writeback' or approval),
'missing or invalid release approval')
def approval_message(request):
return (f"{NAMESPACE}\n{REPO}\n{TARGET}\n{request['sha']}\n{request['bundle']}\n{request['task']}\n").encode()
def hash_file(path):
with path.open('rb') as src:
return hashlib.file_digest(src, 'sha256').hexdigest()
def immutable_file(path):
info = path.lstat()
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and not info.st_mode & 0o022,
'bundle requires regular files owned by the deployment account')
return path
def verify_bundle(directory, request, archives=True):
raw = immutable_file(directory / 'release.json').read_bytes()
require(hashlib.sha256(raw).hexdigest() == request['bundle'], 'release manifest digest mismatch')
manifest = json.loads(raw)
require(manifest.get('schema') == 1 and manifest.get('target') == TARGET
and manifest.get('sha') == request['sha'] and manifest.get('task') == request['task'],
'bundle commit, task or target mismatch')
require(set(manifest.get('images', {})) == set(ROLES), 'bundle must include API, web and worker')
for role, image in manifest['images'].items():
require(matches(IMAGE, image.get('id')) and matches(HASH, image.get('sha256')), 'invalid image record')
if archives:
archive = immutable_file(directory / (role + '.tar'))
require(hash_file(archive) == image['sha256'], f'{role} archive checksum mismatch')
versions = manifest.get('migrations')
require(isinstance(versions, list) and versions and all(type(n) is int and n > 0 for n in versions)
and versions == sorted(set(versions)), 'invalid migration ledger in manifest')
return manifest
def verify_contexts(result, contexts):
latest = {}
for item in result.get('statuses') or []:
latest.setdefault(item.get('context'), item.get('status'))
require(bool(contexts) and all(latest.get(context) == 'success' for context in contexts),
'required CI status is missing or unsuccessful')
def metadata(task, key):
value = (task.get('metadata') or {}).get(key)
return json.loads(value) if isinstance(value, str) else None
def check_task(task, request, receipt=None):
require(task.get('id') == request['task'] and task.get('spaceId') == SPACE, 'HiFox task or space mismatch')
status = (task.get('status') or {}).get('id')
same_publication = receipt and metadata(task, 'cd_release') == receipt.get('hifox')
require(status == READY or (status == PUBLISHED and same_publication), 'HiFox task is not ready for release')
evidence = metadata(task, 'cd_acceptance') or {}
url = urlsplit(evidence.get('evidence', ''))
require(evidence.get('verdict') == 'passed' and evidence.get('sha') == request['sha']
and evidence.get('bundle') == request['bundle'] and url.scheme == 'https' and url.hostname
and not (url.username or url.password or url.query or url.fragment),
'missing or changed business acceptance for this commit and bundle')
return evidence
def release(host, request):
validate_request(request)
path = host.receipt_path
existing = read_json(path) if path.exists() else None
if request['action'] == 'writeback':
require(existing and existing['state'] in ('deployed_hifox_pending', 'published'),
'writeback requires a verified deployment receipt')
require(all(existing[k] == request[k] for k in ('sha', 'task', 'bundle')), 'receipt binding mismatch')
manifest = verify_bundle(host.bundle_path(request), request, archives=False)
host.verify(manifest)
return writeback(host, request, existing)
require(existing is None, 'existing release receipt: use writeback or inspect interrupted/failed deployment')
host.approve(request)
manifest = verify_bundle(host.bundle_path(request), request)
host.check_ci(request)
accepted = check_task(host.task(request['task']), request)
before = host.baseline()
host.pre_switch(before, manifest)
receipt = {k: request[k] for k in ('sha', 'task', 'bundle', 'run_id')}
receipt.update(state='preparing', target=TARGET, started_at=stamp(), previous=before)
record(path, receipt)
try:
host.load(host.bundle_path(request), manifest)
host.pre_switch(before, manifest)
host.approve(request)
require(check_task(host.task(request['task']), request) == accepted, 'business acceptance changed before switch')
except (CDError, OSError, ValueError, KeyError) as exc:
receipt.update(state='failed_before_cutover')
record(path, receipt)
raise CDError(f'failed_before_cutover: {exc}' if isinstance(exc, CDError) else 'failed_before_cutover') from exc
receipt.update(state='cutover_started')
record(path, receipt)
try:
host.switch(before, manifest)
evidence = host.verify(manifest)
except (CDError, OSError, ValueError, KeyError) as exc:
status = 'rollback_failed'
try:
host.rollback(before)
status = 'rolled_back'
except (CDError, OSError, ValueError, KeyError):
pass
receipt.update(state=status, finished_at=stamp(), failure=str(exc) if isinstance(exc, CDError) else 'operation failed')
record(path, receipt)
raise CDError(f'{status}: deployment verification failed') from exc
receipt.update(state='deployed_hifox_pending', deployed_at=stamp(), evidence=evidence)
receipt['hifox'] = {**evidence, 'target': TARGET, 'bundle': request['bundle'], 'deployed_at': receipt['deployed_at'],
'run_url': f'{FORGE}/{REPO}/actions/runs/{request["run_id"]}'}
record(path, receipt)
return writeback(host, request, receipt)
def writeback(host, request, receipt):
try:
task = host.task(request['task'])
check_task(task, request, receipt)
host.callback(request['task'], task, receipt.get('hifox'))
receipt.update(state='published')
record(host.receipt_path, receipt)
except (CDError, OSError, ValueError, KeyError) as exc:
raise CDError('DEPLOYED_HIFOX_PENDING: retry writeback for this receipt') from exc
return receipt
def with_images(config, images):
result = copy.deepcopy(config)
require(set(result['services']) == set(ROLES), 'unexpected Compose service set')
for role in ROLES:
result['services'][role]['image'] = images[role]
return result
def run(argv, purpose, *, cwd=None, input_data=None, timeout=30, allowed=(0,), env=None):
try:
result = subprocess.run(argv, cwd=cwd, input=input_data, capture_output=True, timeout=timeout, env=env)
except (OSError, subprocess.TimeoutExpired) as exc:
raise CDError(f'{purpose} unavailable or timed out') from exc
require(result.returncode in allowed, f'{purpose} failed (exit {result.returncode})')
return result.stdout
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *_args, **_kwargs):
raise CDError('unexpected HTTP redirect')
def http(url, *, direct=False, limit=2**20):
handlers = [NoRedirect()]
if direct:
handlers.append(ProxyHandler({}))
try:
with build_opener(*handlers).open(url, timeout=10) as response:
data = response.read(limit + 1)
require(len(data) <= limit, 'HTTP response exceeds limit')
return data
except (URLError, TimeoutError) as exc:
raise CDError('HTTP read failed') from exc
class Assets(HTMLParser):
def __init__(self):
super().__init__()
self.paths = []
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
path = attrs.get('src' if tag == 'script' else 'href', '')
if path and re.fullmatch(r'/assets/[A-Za-z0-9_.-]+\.(?:js|css)', path):
self.paths.append(path)
def disk_assets(html):
parser = Assets()
parser.feed(html.decode())
require(any(p.endswith('.js') for p in parser.paths), 'web index has no compiled entry point')
return sorted(set(parser.paths))
def image_inspect(image):
return json.loads(run(['docker', 'image', 'inspect', image], 'read image identity'))[0]
def inspect(name):
obj = json.loads(run(['docker', 'container', 'inspect', name], 'read running container'))[0]
require(obj['State']['Status'] == 'running' and matches(IMAGE, obj.get('Image')), 'expected container is not running')
return obj
def image_label(image):
return image_inspect(image)['Config']['Labels'].get(REVISION, '')
def api_identity(container):
raw = run(['docker', 'exec', container, '/app/api', '--version-record-status'], 'read API build identity', allowed=(0, 1))
obj = json.loads(raw)
require(matches(SHA, obj.get('commit_sha')), 'API binary has no full build identity')
return obj['commit_sha']
def migration_files(git, sha):
paths = git('ls-tree', '-r', '--name-only', sha, 'internal/platform/postgres/migrations').decode().splitlines()
return sorted(int(Path(line).name.split('_', 1)[0]) for line in paths if line.endswith('.up.sql'))
def ledger(container):
probe = Path(__file__).with_name('cd_migration_probe.py').read_text()
data = run(['docker', 'exec', container, 'python3', '-c', probe], 'read migration ledger')
result = json.loads(data)
require(isinstance(result, list) and result, 'unexpected migration ledger result')
return result
def verify_archive(archive, expected):
# docker save by ID omits mutable RepoTags. Refuse archives that could retag another app.
with tarfile.open(archive, 'r:') as t:
entry = t.getmember('manifest.json')
require(entry.isfile() and entry.size < 65536, 'unexpected Docker archive manifest')
containers = json.load(t.extractfile(entry))
require(len(containers) == 1 and not containers[0].get('RepoTags'), 'archive must contain one untagged image')
config = t.getmember(containers[0]['Config'])
require(config.isfile() and config.size < 2**20, 'unexpected Docker image config')
require('sha256:' + hashlib.sha256(t.extractfile(config).read()).hexdigest() == expected,
'archive config does not match approved image ID')
for member in t:
require((member.isdir() or member.isfile()) and not member.name.startswith('/')
and '..' not in Path(member.name).parts, 'unexpected Docker archive path')
if 'index.json' in t.getnames():
index = t.getmember('index.json')
require(index.isfile() and index.size < 65536, 'unexpected OCI archive index')
oci = json.load(t.extractfile(index))
require(len(oci.get('manifests', [])) == 1 and all(
key not in descriptor.get('annotations', {}) for descriptor in oci['manifests']
for key in ('org.opencontainers.image.ref.name', 'io.containerd.image.name')),
'archive OCI index contains a mutable name')
class LiveHost:
def __init__(self, request):
self.receipt_path = DATA / 'receipts' / (request['bundle'] + '.json')
self.compose = APP / 'compose.json'
self.git_path = DATA / 'source.git'
self.containers = {role: 'kg-cloud-' + role for role in ROLES}
self.health_url = 'http://127.0.0.1:8088'
self.request = request
def bundle_path(self, request):
return DATA / 'bundles' / request['bundle']
def git(self, *args):
return run(['git', '-C', str(self.git_path), *args], 'release source check', timeout=90)
def approve(self, request):
try:
signature = base64.b64decode(request['approval'], validate=True)
except (binascii.Error, ValueError) as exc:
raise CDError('approval signature is invalid') from exc
with tempfile.TemporaryDirectory(dir=DATA) as directory:
signature_file = Path(directory) / 'approval.sig'
signature_file.write_bytes(signature)
run(['ssh-keygen', '-Y', 'verify', '-f', str(DATA / 'allowed_signers'), '-I', 'release-owner',
'-n', NAMESPACE, '-s', str(signature_file)], 'release approval', input_data=approval_message(request))
def check_dispatch_run(self, request):
path = f'{FORGE}/api/v1/repos/{REPO}/actions/runs/{request["run_id"]}'
dispatch = json.loads(http(path))
require(str(dispatch.get('id')) == request['run_id'] and dispatch.get('event') == 'workflow_dispatch'
and dispatch.get('path') == 'deploy-production.yml@refs/heads/main'
and dispatch.get('head_branch') == 'main' and dispatch.get('status') in ('in_progress', 'running')
and (dispatch.get('actor') or {}).get('id') == 1,
'Gitea run is not an active deployment dispatched by the release owner')
def check_ci(self, request):
self.git('fetch', '--no-tags', f'{FORGE}/{REPO}.git', '+refs/heads/main:refs/remotes/origin/main')
self.git('merge-base', '--is-ancestor', request['sha'], 'refs/remotes/origin/main')
contexts = immutable_file(DATA / 'required-contexts.txt').read_text().splitlines()
verify_contexts(json.loads(http(f'{FORGE}/api/v1/repos/{REPO}/commits/{request["sha"]}/status')),
[line.strip() for line in contexts if line.strip() and not line.lstrip().startswith('#')])
def hifox(self, *args):
cli = DATA / 'bin/hifox'
env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'HOME': str(DATA / 'hifox-home'), 'CI': '1'}
raw = run([str(cli), '--organization-id', ORG, *args, '--output', 'json'], 'HiFox operation', env=env)
result = json.loads(raw)
require(isinstance(result, dict) and result.get('success') is not False and not result.get('error'),
'HiFox did not return a successful object')
return result
def task(self, task):
return self.hifox('task', 'view', task)
def callback(self, task_id, task, expected):
if metadata(task, 'cd_release') != expected:
self.hifox('task', 'metadata', 'set', task_id, 'cd_release', json.dumps(json.dumps(expected)))
fresh = self.task(task_id)
check_task(fresh, self.request, {'hifox': expected})
require(metadata(fresh, 'cd_release') == expected, 'HiFox release receipt readback mismatch')
if fresh['status']['id'] != PUBLISHED:
self.hifox('task', 'edit', task_id, '--status', PUBLISHED)
final = self.task(task_id)
final_status = (final.get('status') or {}).get('id')
require(final_status == PUBLISHED and metadata(final, 'cd_release') == expected, 'HiFox published readback mismatch')
def compose_cmd(self, *args):
return ['docker', 'compose', '-p', PROJECT, '-f', str(self.compose), *args]
def rendered_config(self):
raw = run(self.compose_cmd('config', '--format', 'json'), 'render Compose configuration', cwd=APP)
return json.loads(raw)['services']
def runtime(self):
rendered = self.rendered_config()
require(set(rendered) == set(ROLES), 'unexpected Compose service set')
result = {}
for role, name in self.containers.items():
c = inspect(name)
labels = c['Config']['Labels']
require(labels.get('com.docker.compose.project') == PROJECT
and labels.get('com.docker.compose.service') == role
and labels.get('com.docker.compose.project.working_dir') == str(APP)
and labels.get('com.docker.compose.project.config_files') == str(self.compose),
f'{role} does not belong to the approved Compose chain')
config = rendered[role]
image = image_inspect(c['Image'])
require(image_inspect(config['image'])['Id'] == c['Image'], f'{role} Compose image has drifted')
env = lambda entries: dict(item.split('=', 1) for item in entries or [])
expected_env = env(image['Config'].get('Env'))
expected_env.update({key: str(value) for key, value in (config.get('environment') or {}).items()
if value is not None})
require(env(c['Config'].get('Env')) == expected_env, f'{role} running environment has drifted')
expected_mounts = config.get('volumes') or []
require(all(mount['type'] == 'bind' for mount in expected_mounts), 'unexpected non-bind Compose volume')
mounts = {(v['Source'], v['Destination'], not v['RW']) for v in c.get('Mounts', []) if v['Type'] == 'bind'}
require(mounts == {(v['source'], v['target'], v.get('read_only', False)) for v in expected_mounts}
and len(mounts) == len(c.get('Mounts', [])), f'{role} running mounts have drifted')
require(not config.get('network_mode') or config['network_mode'] == c['HostConfig']['NetworkMode'],
f'{role} running network has drifted')
result[role] = c['Image']
return result
def baseline(self):
images = self.runtime()
self.probe_web()
return {'compose_sha256': hash_file(self.compose), 'images': images,
'revisions': {role: image_label(image) for role, image in images.items()}}
def pre_switch(self, previous, manifest):
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
'target baseline changed during deployment')
for revision in set(previous['revisions'].values()):
require(matches(SHA, revision), 'running image has no source revision')
self.git('merge-base', '--is-ancestor', revision, manifest['sha'])
require(migration_files(self.git, manifest['sha']) == manifest['migrations'], 'source migration set differs from bundle')
for role in ('api', 'worker'):
require(ledger(self.containers[role]) == manifest['migrations'],
f'{role} migration ledger mismatch; verify schema changes outside CD')
def load(self, directory, manifest):
archives = {role: directory / (role + '.tar') for role in ROLES}
for role, archive in archives.items():
verify_archive(archive, manifest['images'][role]['id'])
for role, archive in archives.items():
image = manifest['images'][role]['id']
run(['docker', 'image', 'load', '-i', str(archive)], f'load {role} image', timeout=600)
require(image_inspect(image)['Id'] == image and image_label(image) == manifest['sha'],
f'{role} loaded image differs from approved build')
@property
def backup(self):
return self.receipt_path.with_suffix('.compose.backup')
def switch(self, previous, manifest):
current = self.compose.read_bytes()
require(hashlib.sha256(current).hexdigest() == previous['compose_sha256'], 'Compose changed before switch')
images = {role: manifest['images'][role]['id'] for role in ROLES}
changed = with_images(json.loads(current), images)
for role, image in previous['images'].items():
run(['docker', 'image', 'tag', image, f'knowledge-governance-cd-rollback:{self.request["bundle"]}-{role}'],
'retain rollback image')
atomic_write(self.backup, current)
atomic_write(self.compose, (json.dumps(changed, indent=2) + '\n').encode(), self.compose.stat().st_mode & 0o777)
self.recreate()
def recreate(self):
script = Path(__file__).with_name('deploy-api-web.sh')
run(['bash', str(script), '--verified-compose', str(APP), PROJECT, str(self.compose), 'api', 'web', 'worker'],
'switch governance Compose services', timeout=120)
def probe_web(self):
origin = self.health_url
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'web proxy health probe failed')
require(json.loads(http('http://127.0.0.1:18098/readyz', direct=True)).get('status') in ('ok', 'ready'),
'API readiness probe failed')
index = http(origin + '/', direct=True)
return index, {path: hashlib.sha256(http(origin + path, direct=True, limit=32 * 2**20)).hexdigest()
for path in disk_assets(index)}
def verify_web_image(self):
origin_index, assets = self.probe_web()
name = self.containers['web']
def read_asset(path):
return run(['docker', 'exec', name, 'cat', '/usr/share/nginx/html' + path], 'read running web asset')
require(origin_index == read_asset('/index.html'), 'served index differs from running web image')
for path, served_digest in assets.items():
require(hashlib.sha256(read_asset(path)).hexdigest() == served_digest, 'served asset differs from running web image')
return assets
def verify(self, manifest):
images = {role: manifest['images'][role]['id'] for role in ROLES}
def check():
require(self.runtime() == images, 'running image IDs differ from approved bundle')
require(api_identity(self.containers['api']) == manifest['sha'], 'running API SHA mismatches approval')
assets = self.verify_web_image()
return {'sha': manifest['sha'], 'images': images, 'health': 'ok', 'asset_sha256': assets}
return wait_ready(check)
def rollback(self, previous):
if not self.backup.exists():
require(hash_file(self.compose) == previous['compose_sha256'] and self.runtime() == previous['images'],
'rollback baseline cannot be established')
self.verify_web_image()
return
original = self.backup.read_bytes()
require(hashlib.sha256(original).hexdigest() == previous['compose_sha256'], 'rollback backup checksum mismatch')
source = json.loads(original)
for role, image in previous['images'].items():
reference = source['services'][role]['image']
require(image_inspect(reference)['Id'] == image, 'rollback source tag has drifted')
atomic_write(self.compose, original, self.compose.stat().st_mode & 0o777)
self.recreate()
def check():
require(self.runtime() == previous['images'], 'rollback running images mismatch')
require(api_identity(self.containers['api']) == previous['revisions']['api'], 'rollback API SHA mismatch')
self.verify_web_image()
wait_ready(check)
def wait_ready(check, seconds=90):
deadline = time.monotonic() + seconds
while True:
try:
return check()
except (CDError, OSError, KeyError, ValueError):
if time.monotonic() >= deadline:
raise
time.sleep(3)
def event_request(event, env):
require(env.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' and env.get('GITHUB_REF') == 'refs/heads/main'
and env.get('GITHUB_REPOSITORY') == REPO, 'manual dispatch from main required')
inputs = event.get('inputs') or {}
request = {name: inputs.get(name, '') for name in ('action', 'sha', 'task', 'bundle', 'confirm', 'approval')}
request['run_id'] = env.get('GITHUB_RUN_ID', '')
validate_request(request)
return request
def pack(args):
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
require(not args.output.exists(), 'bundle output already exists')
source = lambda *argv: run(['git', *argv], 'read candidate source')
require(source('rev-parse', args.sha + '^{commit}').decode().strip() == args.sha, 'source commit not found')
images = {}
versions = migration_files(source, args.sha)
manifest = dict(schema=1, target=TARGET, sha=args.sha, task=args.task, images=images, migrations=versions)
with tempfile.TemporaryDirectory(dir=args.output.parent) as directory:
build = Path(directory) / "sealed"
build.mkdir(mode=0o700)
for role in ROLES:
image = image_inspect(getattr(args, role).removeprefix('sha256:'))
image_id = image['Id']
require(matches(IMAGE, image_id) and image_label(image_id) == args.sha, f'{role} source label mismatch')
archive = build / (role + '.tar')
run(['docker', 'image', 'save', '-o', str(archive), image_id], f'archive {role} image', timeout=900)
verify_archive(archive, image_id)
archive.chmod(0o600)
images[role] = {'id': image_id, 'sha256': hash_file(archive)}
record(build / 'release.json', manifest)
for item in build.iterdir():
with item.open("rb") as artifact:
os.fsync(artifact.fileno())
sync_dir(build)
build.rename(args.output)
sync_dir(args.output.parent)
print(hash_file(args.output / 'release.json'))
def attest(args):
request = {'sha': args.sha, 'task': args.task, 'bundle': hash_file(args.directory / 'release.json')}
require(matches(SHA, args.sha) and matches(UUID, args.task), 'require full SHA and task UUID')
manifest = verify_bundle(args.directory, request)
before = {role: inspect(getattr(args, role)) for role in ROLES}
for role, obj in before.items():
require(obj['Image'] == manifest['images'][role]['id'] and obj['Name'] != '/kg-cloud-' + role,
'isolated test runtime image differs from bundle')
require(api_identity(args.api) == args.sha and ledger(args.api) == manifest['migrations']
and ledger(args.worker) == manifest['migrations'], 'test runtime SHA or schema differs')
base = urlsplit(args.test_url)
require(base.scheme in ('http', 'https') and base.hostname and not (base.username or base.password or base.query or base.fragment)
and base.path in ('', '/') and base.port != 8088, 'provide isolated test endpoint')
origin = args.test_url.rstrip('/')
require(json.loads(http(origin + '/health', direct=True)).get('status') == 'ok', 'isolated test health probe failed')
index = http(origin + '/', direct=True)
require(index == run(['docker', 'exec', args.web, 'cat', '/usr/share/nginx/html/index.html'], 'read test web index'),
'test endpoint is serving another web image')
for role, obj in before.items():
require(inspect(getattr(args, role))['Id'] == obj['Id'], 'test instance changed during readback')
print(json.dumps({'sha': args.sha, 'bundle': request['bundle'], 'test_url': args.test_url,
'images': {role: obj['Image'] for role, obj in before.items()}}, sort_keys=True))
def cli():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
for command in ('guard', 'send', 'approval-message', 'serve', 'pack', 'attest'):
sub = commands.add_parser(command)
if command in ('guard', 'send'):
sub.add_argument('--event', type=Path, required=True)
if command in ('approval-message', 'pack', 'attest'):
sub.add_argument('--sha', required=True)
sub.add_argument('--task', required=True)
if command == 'approval-message':
sub.add_argument('--bundle', required=True)
if command in ('pack', 'attest'):
for role in ROLES:
sub.add_argument('--' + role, required=True)
if command == 'pack':
sub.add_argument('--output', type=Path, required=True)
if command == 'attest':
sub.add_argument('--directory', type=Path, required=True)
sub.add_argument('--test-url', required=True)
args = parser.parse_args()
if args.command in ('guard', 'send'):
request = event_request(read_json(args.event), os.environ)
if args.command == 'guard':
print('dispatch parameters verified')
else:
send(request)
elif args.command == 'approval-message':
require(matches(SHA, args.sha) and matches(UUID, args.task) and matches(HASH, args.bundle), 'invalid approval binding')
sys.stdout.buffer.write(approval_message(vars(args)))
elif args.command == 'pack':
pack(args)
elif args.command == 'attest':
attest(args)
else:
raw = sys.stdin.buffer.read(8193)
require(len(raw) <= 8192, 'request exceeds size limit')
request = json.loads(raw)
validate_request(request)
with deploy_lock(DATA / 'release.lock'):
host = LiveHost(request)
for item in (DATA / 'receipts').glob('*.json'):
if read_json(item)['state'] in ('cutover_started', 'rollback_failed'):
raise CDError('previous deployment outcome requires operator reconciliation')
host.check_dispatch_run(request)
result = release(host, request)
print(json.dumps({k: result[k] for k in ('state', 'sha', 'task', 'bundle', 'hifox')}, ensure_ascii=False))
def send(request):
key = os.environ.get('KG_CD_SSH_KEY', '')
known_hosts = os.environ.get('KG_CD_KNOWN_HOSTS', '')
host = os.environ.get('KG_CD_SSH_HOST', '')
port = os.environ.get('KG_CD_SSH_PORT', '22') or '22'
require(key.strip() and known_hosts.strip(), 'missing KG_CD_SSH_KEY or KG_CD_KNOWN_HOSTS')
require(host in ('100.85.111.121', '::1') and port.isdecimal() and 1 <= int(port) <= 65535,
'configure the direct or approved tunnel address for the target release endpoint')
with tempfile.TemporaryDirectory() as directory:
temp = Path(directory)
for filename, value in (('key', key), ('known_hosts', known_hosts)):
atomic_write(temp / filename, (value.strip() + '\n').encode())
argv = ['ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'IdentitiesOnly=yes',
'-o', 'ClearAllForwardings=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'HostKeyAlias=kg-governance-cd',
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', f'UserKnownHostsFile={temp / "known_hosts"}',
'-i', str(temp / 'key'), '-p', port, '-T', 'root@' + host, 'kg-cd-v1']
try:
result = subprocess.run(argv, input=json.dumps(request).encode(), capture_output=True, timeout=2100)
except (OSError, subprocess.TimeoutExpired) as exc:
raise CDError('target outcome unknown; inspect durable receipt before retry') from exc
if result.returncode:
error = result.stderr.decode(errors='replace')
for marker in ('DEPLOYED_HIFOX_PENDING', 'rollback_failed:', 'rolled_back:', 'failed_before_cutover:',
'target preflight rejected', 'deployment lock busy'):
if marker in error:
raise CDError(error.strip().splitlines()[-1])
raise CDError('target rejected request or outcome unknown; inspect durable receipt')
response = json.loads(result.stdout)
require(response.get('state') == 'published' and all(response.get(k) == request[k] for k in ('sha', 'task', 'bundle')),
'invalid or unverified deployment receipt')
print(json.dumps(response, ensure_ascii=False))
if __name__ == '__main__':
try:
cli()
except CDError as error:
print(f'CD: {error}', file=sys.stderr)
sys.exit(1)
except (OSError, KeyError, ValueError, TypeError, tarfile.TarError) as error:
print(f'CD: operation could not be verified ({type(error).__name__})', file=sys.stderr)
sys.exit(1)
+145
View File
@@ -0,0 +1,145 @@
import hashlib
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock
import production_cd as cd
SHA = 'a' * 40
TASK = 'd5abdf29-d0b5-4992-8073-0cb8fffd84ed'
IMAGES = {role: 'sha256:' + str(i) * 64 for i, role in enumerate(('api', 'web', 'worker'), 1)}
CTX = 'build / build (push)'
class ProductionCD(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.request = dict(sha=SHA, task=TASK, bundle='b' * 64, confirm='deploy-production',
action='deploy', approval='signed', run_id='42')
self.manifest = dict(schema=1, target=cd.TARGET, sha=SHA, task=TASK, migrations=[1, 2], images={})
for role, image in IMAGES.items():
archive = self.root / (role + '.tar')
archive.write_text(role)
self.manifest['images'][role] = dict(id=image, sha256=hashlib.sha256(role.encode()).hexdigest())
data = json.dumps(self.manifest).encode()
(self.root / 'release.json').write_bytes(data)
for path in self.root.iterdir():
path.chmod(0o600)
self.request['bundle'] = hashlib.sha256(data).hexdigest()
self.acceptance = {'verdict': 'passed', 'sha': SHA, 'bundle': self.request['bundle'],
'evidence': 'https://example.test/result/42'}
self.host = Mock()
self.host.task.return_value = {'id': TASK, 'spaceId': cd.SPACE, 'status': {'id': cd.READY},
'metadata': {'cd_acceptance': json.dumps(self.acceptance)}}
self.host.baseline.return_value = {'before': 'baseline'}
self.host.pre_switch.return_value = None
self.host.verify.return_value = {'sha': SHA, 'images': IMAGES, 'health': 'ok'}
self.host.receipt_path = self.root / 'receipt.json'
self.host.bundle_path.return_value = self.root
def execute(self):
return cd.release(self.host, self.request)
def assert_no_deploy(self):
self.host.load.assert_not_called()
self.host.switch.assert_not_called()
self.host.callback.assert_not_called()
def test_no_confirmation_has_zero_side_effects(self):
self.request['confirm'] = ''
with self.assertRaisesRegex(cd.CDError, 'confirmation'):
self.execute()
self.host.assert_not_called()
self.assert_no_deploy()
def test_preconditions_prevent_loading_and_switching(self):
for mode in ('approval', 'ci', 'test', 'changed_bundle', 'tampered_archive', 'migration'):
with self.subTest(mode=mode):
self.setUp()
if mode in ('approval', 'ci', 'migration'):
step = {'approval': self.host.approve, 'ci': self.host.check_ci,
'migration': self.host.pre_switch}[mode]
step.side_effect = cd.CDError(mode)
elif mode == 'test':
self.host.task.return_value['metadata'].clear()
elif mode == 'changed_bundle':
self.request['bundle'] = 'f' * 64
else:
(self.root / 'web.tar').write_bytes(b'other image')
with self.assertRaises(cd.CDError):
self.execute()
self.assert_no_deploy()
def test_bare_ci_success_cannot_replace_required_context(self):
for statuses in ([], [{'context': 'different', 'status': 'success'}],
[{'context': CTX, 'status': 'failure'}, {'context': CTX, 'status': 'success'}]):
with self.assertRaises(cd.CDError):
cd.verify_contexts({'state': 'success', 'statuses': statuses}, [CTX])
cd.verify_contexts({'state': 'success', 'statuses': [{'context': CTX, 'status': 'success'}]}, [CTX])
def test_approval_and_acceptance_rechecked_before_switch(self):
self.host.approve.side_effect = [None, cd.CDError('revoked')]
with self.assertRaisesRegex(cd.CDError, 'revoked'):
self.execute()
self.host.switch.assert_not_called()
self.host.callback.assert_not_called()
def test_success_and_writeback_retry_do_not_switch_twice(self):
self.host.callback.side_effect = [cd.CDError('HiFox timeout'), None]
with self.assertRaisesRegex(cd.CDError, 'DEPLOYED_HIFOX_PENDING'):
self.execute()
self.assertEqual(cd.read_json(self.host.receipt_path)['state'], 'deployed_hifox_pending')
self.request.update(action='writeback', confirm='retry-writeback', approval='')
self.assertEqual(self.execute()['state'], 'published')
self.host.switch.assert_called_once()
self.host.load.assert_called_once()
self.assertEqual(self.host.verify.call_count, 2)
def test_writeback_without_receipt_refuses(self):
self.request.update(action='writeback', confirm='retry-writeback', approval='')
with self.assertRaisesRegex(cd.CDError, 'receipt'):
self.execute()
self.assert_no_deploy()
def test_failed_cutover_rolls_back_and_never_marks_published(self):
for result in ('rolled_back', 'rollback_failed'):
with self.subTest(result=result):
self.setUp()
self.host.verify.side_effect = cd.CDError('new unhealthy')
if result == 'rollback_failed':
self.host.rollback.side_effect = cd.CDError('old unhealthy')
with self.assertRaisesRegex(cd.CDError, result):
self.execute()
self.assertEqual(cd.read_json(self.host.receipt_path)['state'], result)
self.host.rollback.assert_called_once()
self.host.callback.assert_not_called()
with self.assertRaisesRegex(cd.CDError, 'receipt'):
self.execute()
self.host.switch.assert_called_once()
def test_target_lock_is_exclusive(self):
path = self.root / 'lock'
with cd.deploy_lock(path):
with self.assertRaisesRegex(cd.CDError, 'deployment lock'):
with cd.deploy_lock(path):
self.fail('lock was acquired twice')
def test_persisted_compose_changes_only_images(self):
original = {'services': {role: {'image': role + ':old', 'env_file': [{'path': '/kept', 'format': 'raw'}],
'volumes': ['/kept:/run/key:ro'], 'environment': {'GATE': 'true'}}
for role in IMAGES}, 'name': 'governance'}
changed = cd.with_images(original, IMAGES)
for role, new in changed['services'].items():
self.assertEqual(new.pop('image'), IMAGES[role])
old = original['services'][role].copy()
old.pop('image')
self.assertEqual(new, old)
if __name__ == '__main__':
unittest.main()