#!/usr/bin/env python3 """Prepare and exercise an isolated Knowledge Governance QA run.""" import argparse from contextlib import ExitStack import fcntl import http.client import json import os from pathlib import Path import re import shutil import signal import socket import stat import subprocess import sys import tempfile import time ROOT = Path(__file__).resolve().parents[1] CONFIG = Path.home() / '.config/knowledge-governance/qa.json' API_PORT = 8088 WEB_PORT = 5173 RUNTIME_RESPONSE = 'KG_QA_RUNTIME_OK' class QAError(RuntimeError): pass def report(stage, detail): print(f'[qa:{stage}] {detail}', flush=True) def capture(argv, stage, timeout=10, env=None): try: return subprocess.run(argv, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, stdin=subprocess.DEVNULL) except FileNotFoundError as exc: raise QAError(f'{stage} 缺少命令 {argv[0]}') from exc except subprocess.TimeoutExpired as exc: raise QAError(f'{stage} 超过 {timeout} 秒') from exc def private_file(path): try: info = path.lstat() except FileNotFoundError as exc: raise QAError(f'缺少测试配置文件 {path}') from exc if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600: raise QAError(f'{path} 必须是当前用户所有的 0600 普通文件') def database_config(path): private_file(path) try: config = json.loads(path.read_text()) except (UnicodeError, json.JSONDecodeError) as exc: raise QAError(f'{path} 不是有效 JSON') from exc if not isinstance(config, dict) or config.keys() != {'host', 'port', 'database', 'user'}: raise QAError('qa.json 只接受 host、port、database、user 四个字段') for field in ('database', 'user'): if not re.fullmatch(r'kg_qa_[a-z0-9_]{1,56}', str(config[field])): raise QAError(f'RDS {field} 必须以 kg_qa_ 开头并使用小写字母、数字或下划线') host = config['host'] if not isinstance(host, str) or not re.fullmatch(r'[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+', host): raise QAError('RDS host 必须填写证书对应的 DNS 主机名') if type(config['port']) is not int or not 1 <= config['port'] <= 65535: raise QAError('RDS port 必须是 1–65535 的整数') return config def tool_env(): allowed = ('HOME', 'PATH', 'LANG', 'TMPDIR', 'GOPROXY', 'GOSUMDB', 'GOPRIVATE', 'GONOSUMDB', 'GOMODCACHE', 'GOCACHE', 'npm_config_cache', 'npm_config_registry', 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy') env = {key: os.environ[key] for key in allowed if key in os.environ} env['GOFLAGS'] = '-mod=readonly' return env def database_env(config, directory): # Keep inherited PG variables and integration switches out of application children. env = tool_env() env.update(PGHOST=config['host'], PGPORT=str(config['port']), PGDATABASE=config['database'], PGUSER=config['user'], PGSSLMODE='verify-full', PGCONNECT_TIMEOUT='5', PGPASSFILE=str(directory / 'qa.pgpass'), PGSSLROOTCERT=str(directory / 'rds-ca.pem'), PGOPTIONS='-c search_path=public', LC_ALL='C', TEST_WEKNORA_ENABLED='false', PP_SCAN_ENABLED='false', RELEASE_WORKER_ENABLED='false', PRODUCTION_RELEASE_REQUESTS_ENABLED='false', DOBOTAI_OPS_READ_ENABLED='false', DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED='false') env['APP_ADDR'] = f'127.0.0.1:{API_PORT}' env['DATA' + 'BASE_URL'] = ( f"postgresql://{config['user']}@{config['host']}:{config['port']}/{config['database']}" '?sslmode=verify-full&connect_timeout=5&search_path=public' ) return env def psql(env, sql): result = capture(['psql', '-XAtw', '-v', 'ON_ERROR_STOP=1', '-c', sql], 'RDS 探针', env=env) if result.returncode == 0: return result.stdout.strip() detail = result.stderr.lower() hints = ( ('password', '口令认证失败,检查 qa.pgpass 和登录角色'), ('certificate', 'TLS 校验失败,检查 RDS 主机名和 CA'), ('does not exist', '数据库或角色不存在,联系管理员确认专用 QA 库'), ('permission denied', 'RDS 权限不足,检查专用 QA 角色的 CONNECT、schema USAGE/CREATE 权限'), ('could not translate host name', 'RDS 主机名无法解析,检查私网 DNS'), ('timeout', '连接 RDS 超时,检查 QA 虚拟机到 RDS 的私网和安全组'), ('connection refused', 'RDS 拒绝 TCP 连接,检查主机名、端口和私网接入'), ) hint = next((message for marker, message in hints if marker in detail), '连接或 SQL 探针失败,检查 RDS 接入和权限') raise QAError(f'RDS {hint}(psql 退出 {result.returncode})') def database_state(env): identity = psql(env, """ SELECT concat_ws('|', current_database(), current_user, COALESCE((SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()), false), (SELECT NOT (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = current_user), (has_schema_privilege(current_user, 'public', 'USAGE') AND has_schema_privilege(current_user, 'public', 'CREATE')), to_regclass('public.schema_migrations') IS NOT NULL, EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname = 'public' AND c.relname <> 'schema_migrations') OR EXISTS (SELECT FROM pg_namespace WHERE nspname NOT IN ('public', 'information_schema') AND nspname !~ '^pg_') OR EXISTS (SELECT FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname = 'public')); """) fields = identity.split('|') if len(fields) != 7 or fields[:4] != [env['PGDATABASE'], env['PGUSER'], 't', 't']: raise QAError('RDS 实际库名、角色、TLS 或角色权限与 QA 配置不符') if fields[4] != 't': raise QAError('RDS 角色需要 public schema 的 USAGE 和 CREATE 权限') rows = psql(env, 'SELECT version FROM public.schema_migrations ORDER BY version;') if fields[5] == 't' else '' if any(not re.fullmatch(r'[0-9]+', row) for row in rows.splitlines()): raise QAError('RDS 迁移记录包含无效版本') return [int(row) for row in rows.splitlines()], fields[6] == 't' def expected_migrations(): paths = (ROOT / 'internal/platform/postgres/migrations').glob('*.up.sql') try: versions = sorted(int(path.name.split('_', 1)[0]) for path in paths) except ValueError as exc: raise QAError('仓库存在无效的迁移文件名') from exc if not versions or len(versions) != len(set(versions)): raise QAError('仓库迁移文件缺失或版本重复') return versions def check_migrations(expected, applied, nonempty, *, complete): if not applied and nonempty: raise QAError('RDS 非空且缺少有效迁移记录,停止初始化') if expected[:len(applied)] != applied: raise QAError('RDS 迁移序列与当前提交不兼容,使用专用空库或匹配的测试库') if complete and applied != expected: raise QAError(f'RDS 迁移未完成,缺少版本 {expected[len(applied)]} 至 {expected[-1]}') def check_hifox(output): try: state = json.loads(output) online = state['connected'] is True and state['remoteLiveness']['status'] == 'online' except (ValueError, KeyError, TypeError) as exc: raise QAError('HiFox 未返回有效的在线状态,检查本机注册和接收服务') from exc if not online: raise QAError('HiFox 未确认测试机在线,检查本机接收服务和网络') def check_checkout(expected): if not re.fullmatch(r'[0-9a-f]{40}', expected): raise QAError('设置 KG_QA_SHA 为提测的完整 40 位 Git SHA') actual = capture(['git', 'rev-parse', 'HEAD'], 'Git 检出') if actual.returncode or actual.stdout.strip() != expected: raise QAError('Git 检出与提测提交不一致') state = capture(['git', 'status', '--porcelain'], 'Git 状态') if state.returncode or state.stdout.strip(): raise QAError('QA 需要干净的独立检出,请检查 Git 状态') def preflight(env): for cmd in ('git', 'make', 'go', 'node', 'npm', 'psql', 'hifox', 'omp'): if not shutil.which(cmd, path=env.get('PATH')): raise QAError(f'工具链缺少 {cmd},检查 QA 机器的 PATH') go = capture(['go', 'env', 'GOROOT', 'GOVERSION'], 'Go 工具链', env=env) fields = go.stdout.splitlines() if go.returncode or len(fields) != 2 or not fields[1].startswith('go1.26.'): raise QAError('QA 需要 Go 1.26 工具链') root = Path(fields[0]) / 'bin' if not os.access(root / 'gofmt', os.X_OK): raise QAError('Go 工具链缺少 gofmt,请安装完整的 Go 1.26') env['PATH'] = f"{root}:{env['PATH']}" node = capture(['node', '--version'], 'Node.js 工具链', env=env) if node.returncode or not node.stdout.startswith('v22.'): raise QAError('QA 需要 Node.js 22 工具链') report('toolchain', f'{fields[1]} / Node.js {node.stdout.strip()}') hifox = capture(['hifox', 'computer', 'status', '--output', 'json'], 'HiFox 在线检查', timeout=15) if hifox.returncode: raise QAError(f'HiFox 状态命令失败(退出 {hifox.returncode})') check_hifox(hifox.stdout) report('machine', f'{socket.gethostname()} / HiFox online') prompt = f'Reply with exactly {RUNTIME_RESPONSE}. Do not use tools.' omp = capture(['omp', '--no-session', '-p', prompt], 'OMP Runtime 探针', timeout=35) if omp.returncode or RUNTIME_RESPONSE not in {line.strip() for line in omp.stdout.splitlines()}: raise QAError('OMP Runtime 没有返回探针结果,检查 OMP 模型配置和 Magpie 网关') report('runtime', 'OMP 模型请求通过') def check_ports(): for port in (API_PORT, WEB_PORT): with socket.socket() as listener: listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: listener.bind(('127.0.0.1', port)) except OSError as exc: raise QAError(f'QA 端口 {port} 无法绑定,检查已有监听进程') from exc def run_step(argv, stage, env, timeout): report(stage, ' '.join(argv[:2])) process = subprocess.Popen(argv, cwd=ROOT, env=env, stdin=subprocess.DEVNULL, start_new_session=True) try: code = process.wait(timeout=timeout) except subprocess.TimeoutExpired as exc: raise QAError(f'{stage} 超过 {timeout} 秒') from exc finally: stop_process(process) if code: raise QAError(f'{stage} 失败(退出 {code}),查看上方命令输出') def stop_process(process): try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: pass try: process.wait(timeout=5) except subprocess.TimeoutExpired: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass process.wait(timeout=5) def start_service(stack, argv, env, log): output = stack.enter_context(log.open('w', encoding='utf-8')) process = subprocess.Popen(argv, cwd=ROOT, env=env, stdin=subprocess.DEVNULL, stdout=output, stderr=subprocess.STDOUT, start_new_session=True) stack.callback(stop_process, process) return process def request(port, path): connection = http.client.HTTPConnection('127.0.0.1', port, timeout=2) try: connection.request('GET', path) response = connection.getresponse() return response.status, response.getheader('Content-Type', ''), response.read(32768) finally: connection.close() def probe(port, path, expected): try: code, content_type, body = request(port, path) if code != 200: return False if expected == 'html': return 'text/html' in content_type and b'id="root"' in body return 'application/json' in content_type and json.loads(body)['status'] == expected except (OSError, http.client.HTTPException, ValueError, KeyError, TypeError): return False def wait_service(process, port, path, expected, log): deadline = time.monotonic() + 45 while time.monotonic() < deadline: if process.poll() is not None: raise QAError(f'服务端口 {port} 启动进程提前退出(退出 {process.returncode}),日志 {log}') if probe(port, path, expected): return time.sleep(0.25) raise QAError(f'服务探针超时 127.0.0.1:{port}{path},日志 {log}') def qa_run(commit, env, db_env, versions, state): with tempfile.TemporaryDirectory(prefix='run-', dir=state) as directory, ExitStack() as services: binary = Path(directory) / 'api' run_step(['make', 'deps-install'], 'deps', env, 300) ldflags = f'-X knowledge-governance/internal/version.CommitSHA={commit}' run_step(['go', 'build', '-ldflags', ldflags, '-o', str(binary), './cmd/api'], 'build-api', env, 300) check_ports() log_dir = Path(tempfile.mkdtemp(prefix=f'{commit[:12]}-', dir=state)) report('logs', str(log_dir)) app_env = db_env | {'PATH': env['PATH'], 'GOVERNANCE_SPOOL_ROOT': str(Path(directory) / 'spool')} api_log = log_dir / 'api.log' api = start_service(services, [str(binary)], app_env, api_log) wait_service(api, API_PORT, '/readyz', 'ready', api_log) check_migrations(versions, *database_state(db_env), complete=True) report('migration', f'已核对全部 {len(versions)} 条迁移,最新版本 {versions[-1]}') web_log = log_dir / 'web.log' web = start_service(services, ['npm', '--prefix', 'web', 'run', 'dev', '--', '--host', '127.0.0.1', '--port', str(WEB_PORT), '--strictPort'], env, web_log) wait_service(web, WEB_PORT, '/', 'html', web_log) wait_service(web, WEB_PORT, '/health', 'ok', web_log) report('services', 'API /readyz、Web 首页与 Vite /health 代理通过') run_step(['make', 'tests-go', 'tests-web'], 'tests', env, 300) check_checkout(commit) if api.poll() is not None or web.poll() is not None or not probe(WEB_PORT, '/health', 'ok'): raise QAError(f'服务在测试期间失去健康状态,日志目录 {log_dir}') report('result', f'PASS {commit},已停止本次 API 和 Web 进程') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('doctor', 'run')) args = parser.parse_args() env = tool_env() try: commit = os.environ.get('KG_QA_SHA', '') if args.action == 'run': check_checkout(commit) check_ports() preflight(env) config_path = Path(os.environ.get('KG_QA_CONFIG', CONFIG)).expanduser().absolute() config = database_config(config_path) private_file(config_path.parent / 'qa.pgpass') if not (config_path.parent / 'rds-ca.pem').is_file(): raise QAError(f'缺少 RDS CA 文件 {config_path.parent / "rds-ca.pem"}') db_env = database_env(config, config_path.parent) versions = expected_migrations() if args.action == 'doctor': applied, nonempty = database_state(db_env) check_migrations(versions, applied, nonempty, complete=False) report('database', f'连接与身份正常,已应用 {len(applied)}/{len(versions)} 条迁移') return 0 state = Path.home() / '.local/state/knowledge-governance/qa' state.mkdir(mode=0o700, parents=True, exist_ok=True) with (state / 'run.lock').open('a') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: raise QAError('本机已有 QA 任务,等待当前任务退出') from exc check_ports() check_migrations(versions, *database_state(db_env), complete=False) qa_run(commit, env, db_env, versions, state) return 0 except (QAError, OSError, ValueError) as exc: print(f'[qa:FAIL] {exc}', file=sys.stderr) return 1 except KeyboardInterrupt: print('[qa:FAIL] QA 任务中断', file=sys.stderr) return 130 if __name__ == '__main__': def interrupt(_signal, _frame): raise KeyboardInterrupt signal.signal(signal.SIGTERM, interrupt) sys.exit(main())