姚章浩
79c154f9af
对着试点仓落地时暴露的问题做的口径与模板修正（本轮不推、不打 tag、不部署）： - 条目形状：四个自定义必填节 → **固定四类** Added/Changed/Fixed/Security （分类可省略、不许空栏目）；过程记录进 docs/archive/，未合并事项进 docs/BACKLOG.md。 - 破坏性变化/迁移：固定写在 Changed 里、以 `**破坏性**` 开头并**必须链接发布文档**， 仓内 `docs/….md` 链接**对账**（不存在即红）。 - 构建身份：release_version / build_kind / commit_sha / display_version / built_at； 开发构建显示 X.Y.Z-dev+g<短SHA>（无基线 dev+g<短SHA>），**不许只显示上一正式版本号、 不许自称正式版**；release 只认显式发布上下文。 - **还没有正式版本一律放行**：不判红；首个正式版本号**由人指定**（预检把它列成阻塞项）。 - **没核对远端就不许说校验成功**：CI/预检用 `--require-remote`（本地模式明确写「未核对远端」）。 - 修正"CI 取不到 tag 所以不用 tag"的过时口径：正式版本**就是**用注释 tag， 只是**构建时不要用 git describe 取版本号**；**测试 tag 只在临时测试仓库里创建**。 - 模板同步：templates/check-changelog.py、templates/release-preflight.py、 新增 templates/version.py；ci-gate.py 的 version 项在 CI 下强制核对远端； new-project.sh 落 scripts/version.py 并按四类重写 CHANGELOG 骨架； README/07/模板 README 同步；04 增加 §I（V-1~V-4 四条，含负例在哪）。 验证：templates 两个检查器 --self-test 全过；new-project.sh bash -n 通过； 生成的新工程里 check_changelog --tags 绿、--identity 给出 dev+g<短SHA>、 CI=true 下 version 项因"未核对远端"判红（预期）；试点仓反向夹具 80/0。
800 lines
39 KiB
Python
Executable File
800 lines
39 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""通用权威门禁（模板版）：生成器把它落到新工程的 `scripts/ci_checks.py`。
|
||
|
||
make verify # = python3 scripts/ci_checks.py
|
||
python3 scripts/ci_checks.py --list
|
||
python3 scripts/ci_checks.py --self-test
|
||
|
||
它和 CI 的关系：**.gitea/workflows/build.yml 只调用这一条命令**，不再自己写一套步骤。
|
||
|
||
判据来自正本 §4.8（必需检查清单与结果口径）：
|
||
1. 失败即停，但汇总照做：剩下的记 `not_run` 并如实写进报告；
|
||
2. `not_run` / `skipped` 都不算过；不适用要在清单里声明 `required_in`；
|
||
3. 工具缺失 / 依赖没按锁装 / 输出解析不了 ⇒ **失败**，不当成"检查通过"。
|
||
|
||
⚠️ 三条"不许假检查"的具体落实（都踩过）：
|
||
· **每一门语言都必须真的执行**：不支持的语言 / 工具缺失 ⇒ 明确失败，
|
||
绝不允许"这个语言我没写检查所以返回成功"（Python 曾经就是这样空跑成功的）。
|
||
· **子进程退出码也算判据**：测试报告写 pass，但进程退出 1 ⇒ 失败。
|
||
· **前端测试框架按声明来**：不是所有项目都用 vitest；用什么、能不能报出用例数，
|
||
都读 `scripts/test-scope.json` 的声明；认不出来 ⇒ 明确失败。
|
||
|
||
⚠️ 本文件是**模板**：改它等于改以后所有新工程的默认行为。改完请跑 `--self-test`。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import contextlib
|
||
import datetime as dt
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
POLICY_REVISION = "ci-checks/generic-3"
|
||
SUCCESS, FAILURE, SKIPPED, NOT_RUN = "success", "failure", "skipped", "not_run"
|
||
FULL_SHA = re.compile(r"^[0-9a-f]{40}$")
|
||
GOVULNCHECK = "golang.org/x/vuln/cmd/govulncheck@v1.1.4" # 锁版本：不静默升级工具
|
||
MANIFESTS = ["go.mod", "go.sum", "package.json", "package-lock.json", "pnpm-lock.yaml",
|
||
"yarn.lock", "requirements.txt", "pyproject.toml", "uv.lock"]
|
||
SCOPE_FILE = ROOT / "scripts" / "test-scope.json"
|
||
|
||
# 这些检查**与语言无关**：认不出语言时它们照样要跑（版本记录、清单改写核对）
|
||
LANG_FREE = {"version", "manifests-locked"}
|
||
|
||
# 必需检查清单的声明（id → 在哪些档案下必需）。汇总拿它对账，不遍历"跑过的项"。
|
||
CHECK_MANIFEST: dict[str, tuple[str, ...]] = {
|
||
"identity": ("ci",),
|
||
"deps-install": ("local", "ci"),
|
||
"format": ("local", "ci"),
|
||
"static": ("local", "ci"),
|
||
"build": ("local", "ci"),
|
||
"tests": ("local", "ci"),
|
||
"version": ("local", "ci"),
|
||
"security": ("local", "ci"),
|
||
"manifests-locked": ("local", "ci"),
|
||
}
|
||
|
||
|
||
def which(name: str, *extra: str) -> str:
|
||
for cand in (shutil.which(name), *extra):
|
||
if cand and Path(cand).exists():
|
||
return cand
|
||
raise FileNotFoundError(name)
|
||
|
||
|
||
def need(name: str, why: str) -> str:
|
||
"""工具缺失 ⇒ 抛错（调用方转成 FAILURE）。不允许"没有工具所以跳过"。"""
|
||
try:
|
||
return which(name, f"/opt/citools/bin/{name}", f"/opt/citools/go/bin/{name}")
|
||
except FileNotFoundError:
|
||
raise FileNotFoundError(f"{name}（{why}）")
|
||
|
||
|
||
def go_bin() -> str:
|
||
return need("go", "Go 检查需要它；缺失就是红，不跳过")
|
||
|
||
|
||
def gofmt_bin() -> str:
|
||
return need("gofmt", "Go 格式检查需要它；缺失就是红，不跳过")
|
||
|
||
|
||
def go_env() -> dict:
|
||
env = dict(os.environ)
|
||
env["GOFLAGS"] = "-mod=readonly" # 禁止隐式改写 go.mod / go.sum
|
||
env.setdefault("GOPROXY", "https://goproxy.cn,direct")
|
||
env.setdefault("GOTOOLCHAIN", "local")
|
||
cache = Path("/opt/citools-cache")
|
||
if cache.is_dir() and os.access(cache, os.W_OK):
|
||
for key, sub in (("GOCACHE", "gocache"), ("GOMODCACHE", "gomod"), ("GOPATH", "gopath")):
|
||
p = cache / sub
|
||
try:
|
||
p.mkdir(parents=True, exist_ok=True)
|
||
except OSError:
|
||
continue
|
||
env.setdefault(key, str(p))
|
||
return env
|
||
|
||
|
||
def python_lock_plan() -> tuple[bool, list[str], str]:
|
||
"""Python 依赖怎么装：**必须按锁文件、装进项目内隔离环境**。
|
||
|
||
· 有 `uv.lock` ⇒ `uv sync --frozen`：冻结锁文件 + 项目内 `.venv`（不碰系统解释器）；
|
||
· 只有 `requirements.txt` ⇒ **失败**并让人先建锁文件 —— 不接受无约束解析，
|
||
也不用 `uv pip install --system` 装进系统环境（那会让"本地与 CI 一致"变成空话）；
|
||
· 两个都没有 ⇒ 失败（不许"没有依赖入口所以跳过"）。
|
||
"""
|
||
if (ROOT / "uv.lock").is_file():
|
||
note = "uv sync --frozen（锁文件锁定 + 项目内 .venv 隔离）"
|
||
if (ROOT / "requirements.txt").is_file():
|
||
note += "；⚠️ 同时存在 requirements.txt，但**不参与安装**（以 uv.lock 为准）"
|
||
return True, ["sync", "--frozen"], note
|
||
if (ROOT / "requirements.txt").is_file():
|
||
return False, [], ("只有 requirements.txt、没有 uv.lock：不接受无约束安装，也不会用 --system "
|
||
"装进系统环境。请先 `uv lock`（或 `uv add -r requirements.txt`）生成锁文件后重跑")
|
||
return False, [], "既没有 uv.lock 也没有 requirements.txt：Python 依赖装不了，不许空跑成功"
|
||
|
||
|
||
def digest(p: Path) -> str:
|
||
return hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else "missing"
|
||
|
||
|
||
def detect():
|
||
"""识别本仓的语言入口。识别不到 ⇒ 调用方判红（不许"跳过编译然后成功"）。"""
|
||
langs = []
|
||
if (ROOT / "go.mod").is_file():
|
||
langs.append("go")
|
||
if any((ROOT / f).is_file() for f in ("pyproject.toml", "requirements.txt", "setup.py")):
|
||
langs.append("python")
|
||
web = next((d for d in (ROOT / "web", ROOT, ROOT / "frontend", ROOT / "ui")
|
||
if (d / "package.json").is_file()), None)
|
||
if web is not None:
|
||
langs.append("node")
|
||
return langs, web
|
||
|
||
|
||
def load_scope() -> tuple[dict, list[str]]:
|
||
"""读 `scripts/test-scope.json`：前端测试框架声明 + 允许跳过的用例。
|
||
|
||
严格 schema（"不能用空对象逃逸"）：
|
||
· optional 条目必须给出 package 或 test **至少一个**，否则通配任意跳过；
|
||
· 必须有 reason 与 owner；package 级条目还要给 expected_skips（跳过数不能悄悄涨）；
|
||
· review_by 过期 ⇒ 整份声明失效（红）。
|
||
"""
|
||
if not SCOPE_FILE.is_file():
|
||
return {}, [f"找不到 {SCOPE_FILE.relative_to(ROOT)}：测试范围/前端框架声明不出来"]
|
||
try:
|
||
raw = json.loads(SCOPE_FILE.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError as e:
|
||
return {}, [f"{SCOPE_FILE.name} 解析失败：{e}"]
|
||
if not isinstance(raw, dict):
|
||
return {}, [f"{SCOPE_FILE.name} 顶层必须是对象"]
|
||
problems: list[str] = []
|
||
today = dt.date.today()
|
||
entries = raw.get("optional") or []
|
||
if not isinstance(entries, list):
|
||
problems.append("optional 必须是数组")
|
||
entries = []
|
||
for i, e in enumerate(entries):
|
||
if not isinstance(e, dict):
|
||
problems.append(f"optional[{i}] 不是对象")
|
||
continue
|
||
if not e.get("package") and not e.get("test"):
|
||
problems.append(f"optional[{i}] 既没有 package 也没有 test —— 它会通配任意跳过（不许这样）")
|
||
for f in ("reason", "owner"):
|
||
if not str(e.get(f) or "").strip():
|
||
problems.append(f"optional[{i}] 缺 {f}")
|
||
if e.get("package") and "expected_skips" not in e:
|
||
problems.append(f"optional[{i}]（package 级）必须给 expected_skips：跳过数不许无声明地涨")
|
||
rb = str(e.get("review_by") or "").strip()
|
||
if not rb:
|
||
problems.append(f"optional[{i}] 缺 review_by（到期日）—— 永久豁免不允许")
|
||
else:
|
||
try:
|
||
if dt.date.fromisoformat(rb) < today:
|
||
problems.append(f"optional[{i}] 的 review_by {rb} 已过期，重新走一遍评审")
|
||
except ValueError:
|
||
problems.append(f"optional[{i}] 的 review_by 不是 YYYY-MM-DD")
|
||
return raw, problems
|
||
|
||
|
||
def run(argv, cwd=None, env=None):
|
||
p = subprocess.run(argv, cwd=cwd or ROOT, env={**os.environ, **(env or {})},
|
||
capture_output=True, text=True)
|
||
return p.returncode, p.stdout, p.stderr
|
||
|
||
|
||
# ── 判定纯函数：--self-test 直接打它们 ──────────────────────────────────
|
||
def judge_frontend(rc: int, report: dict | None, declared: str | None,
|
||
evidence: str | None, scope_problems: list[str]) -> tuple[str, str]:
|
||
"""前端测试的判据。两件事必须都想清楚：
|
||
|
||
· **子进程退出码也是判据**：报告写 pass 但进程退出非零 ⇒ 失败（实测会漏）；
|
||
· **框架不写死 vitest**：用声明/依赖里认出来的框架；认不出来 ⇒ 明确失败
|
||
（"认不出用什么测"不等于"测过了"）。
|
||
"""
|
||
if scope_problems:
|
||
return FAILURE, "；".join(scope_problems)
|
||
if declared and declared not in ("vitest", "jest"):
|
||
return FAILURE, f"test-scope.json 声明的 frontend_runner={declared!r} 不在支持列表（vitest/jest）"
|
||
if rc != 0:
|
||
return FAILURE, f"前端测试进程退出码 {rc}（报告就算写 pass 也不算过）"
|
||
if evidence == "exitcode":
|
||
return FAILURE, ("声明 frontend_evidence=exitcode 不足以判定："
|
||
"空套件与用例数都不再可知。请换成能报用例数的框架，"
|
||
"或由人批准后在清单里显式声明这条缺口。")
|
||
if not report:
|
||
return FAILURE, "前端测试没有可解析的用例报告（既不认框架也没有 JSON）—— 不允许只凭退出码"
|
||
total = int(report.get("numTotalTests") or 0)
|
||
failed = int(report.get("numFailedTests") or 0)
|
||
pend = int(report.get("numPendingTests") or 0) + int(report.get("numTodoTests") or 0)
|
||
npass = int(report.get("numPassedTests") or 0)
|
||
detail = f"通过 {npass} / 失败 {failed} / 待跑 {pend} / 共 {total}"
|
||
if failed:
|
||
return FAILURE, f"前端用例失败：{detail}"
|
||
if total == 0:
|
||
return FAILURE, f"前端一个用例都没执行（空套件不算通过）：{detail}"
|
||
if pend:
|
||
return FAILURE, f"前端有未执行用例（skip/todo）：{detail}"
|
||
return SUCCESS, detail
|
||
|
||
|
||
def judge_python_tests(rc: int, out: str) -> tuple[str, str]:
|
||
"""pytest 退出码有语义：0 全过 / 1 有用例失败 / 2 中断 / 5 一个用例都没收集到。"""
|
||
if rc == 5:
|
||
return FAILURE, "pytest 一个用例都没收集到（空套件不能算通过）"
|
||
if rc == 0:
|
||
tail = [l for l in out.splitlines() if l.strip()][-1:] or [""]
|
||
return SUCCESS, f"pytest 退出 0｜{tail[0][:120]}"
|
||
return FAILURE, f"pytest 退出码 {rc}（1=有用例失败，2=中断，3=内部错误，4=用法错误）"
|
||
|
||
|
||
def manifest_mismatch(declared: dict, actual: list[tuple[str, tuple[str, ...]]]) -> list[str]:
|
||
problems, seen = [], {cid: tuple(req) for cid, req in actual}
|
||
for cid in declared:
|
||
if cid not in seen:
|
||
problems.append(f"清单声明了 {cid}，实现里没有")
|
||
for cid, req in seen.items():
|
||
if cid not in declared:
|
||
problems.append(f"实现里有 {cid}，清单没声明")
|
||
elif tuple(declared[cid]) != req:
|
||
problems.append(f"{cid} 档案不一致：清单 {declared[cid]} vs 实现 {req}")
|
||
return problems
|
||
|
||
|
||
def aggregate(results: list[dict], profile: str,
|
||
declared: dict[str, tuple[str, ...]] | None = None) -> tuple[str, list[str]]:
|
||
declared = declared if declared is not None else CHECK_MANIFEST
|
||
problems, counts = [], {}
|
||
for r in results:
|
||
cid = str(r.get("id") or "")
|
||
counts[cid] = counts.get(cid, 0) + 1
|
||
if cid not in declared:
|
||
problems.append(f"报告里有清单之外的检查项 {cid}")
|
||
continue
|
||
if profile not in set(declared[cid]):
|
||
if r["status"] in (FAILURE, "unknown"):
|
||
problems.append(f"{cid} 在 {profile} 档案下不要求，但状态是 {r['status']}")
|
||
continue
|
||
if r["status"] != SUCCESS:
|
||
problems.append(f"{cid} 状态 {r['status']}｜{str(r.get('reason') or '')[:160]}")
|
||
for cid, n in counts.items():
|
||
if n > 1:
|
||
problems.append(f"{cid} 重复出现 {n} 次")
|
||
for cid, req in declared.items():
|
||
if profile in set(req) and cid not in counts:
|
||
problems.append(f"清单里的必需项 {cid} 在报告里缺失")
|
||
return (SUCCESS if not problems else FAILURE), problems
|
||
|
||
|
||
def self_test() -> int:
|
||
problems: list[str] = []
|
||
cases = [
|
||
("全绿", [{"id": "a", "status": SUCCESS, "reason": ""}], "local", 0),
|
||
("失败", [{"id": "a", "status": FAILURE, "reason": ""}], "local", 1),
|
||
("缺失", [], "local", 1),
|
||
("重复", [{"id": "a", "status": SUCCESS}, {"id": "a", "status": SUCCESS}], "local", 1),
|
||
("未知项", [{"id": "a", "status": SUCCESS}, {"id": "zz", "status": SUCCESS}], "local", 1),
|
||
]
|
||
for name, results, profile, want in cases:
|
||
_, got = aggregate(results, profile, {"a": ("local", "ci")})
|
||
if len(got) != want:
|
||
problems.append(f"汇总 {name}：期望 {want} 个问题，实际 {len(got)}：{got}")
|
||
|
||
ok_rep = {"numTotalTests": 3, "numPassedTests": 3, "numFailedTests": 0,
|
||
"numPendingTests": 0, "numTodoTests": 0}
|
||
fe = [
|
||
("rc=0 + 报告全过 ⇒ 绿", (0, ok_rep, "vitest", None, []), True),
|
||
("rc=1 + 报告写 pass ⇒ 红", (1, ok_rep, "vitest", None, []), False),
|
||
("空套件 ⇒ 红", (0, {"numTotalTests": 0}, "vitest", None, []), False),
|
||
("有 todo ⇒ 红", (0, {**ok_rep, "numTodoTests": 1}, "vitest", None, []), False),
|
||
("没有报告且未声明 ⇒ 红", (0, None, "vitest", None, []), False),
|
||
("声明只用退出码 ⇒ 仍然红（不足以判定）", (0, None, None, "exitcode", []), False),
|
||
("声明了不支持的框架 ⇒ 红", (0, ok_rep, "mocha", None, []), False),
|
||
("范围声明有问题 ⇒ 红", (0, ok_rep, "vitest", None, ["optional[0] 缺 owner"]), False),
|
||
]
|
||
for name, cargs, want in fe:
|
||
status, why = judge_frontend(*cargs)
|
||
if (status == SUCCESS) != want:
|
||
problems.append(f"前端 {name}：期望 {'绿' if want else '红'}，实际 {status}（{why}）")
|
||
|
||
for name, (rc, want) in {"0 ⇒ 绿": (0, True), "1 ⇒ 红": (1, False),
|
||
"5（没收集到用例）⇒ 红": (5, False)}.items():
|
||
status, _ = judge_python_tests(rc, "1 passed")
|
||
if (status == SUCCESS) != want:
|
||
problems.append(f"pytest {name}：实际 {status}")
|
||
|
||
# Python 锁定/隔离：缺锁必须失败，且**从不**出现 --system
|
||
import tempfile as _tf
|
||
with _tf.TemporaryDirectory() as _d:
|
||
_root = ROOT
|
||
try:
|
||
globals()["ROOT"] = Path(_d)
|
||
ok, argv, _why = python_lock_plan()
|
||
if ok:
|
||
problems.append("既没有 uv.lock 也没有 requirements.txt 时竟然返回可安装")
|
||
(Path(_d) / "requirements.txt").write_text("pytest\n", encoding="utf-8")
|
||
ok, argv, _why = python_lock_plan()
|
||
if ok or "--system" in " ".join(argv):
|
||
problems.append("只有 requirements.txt 时没有明确失败（或用了 --system）")
|
||
(Path(_d) / "uv.lock").write_text("version = 1\n", encoding="utf-8")
|
||
ok, argv, _why = python_lock_plan()
|
||
if not ok or argv != ["sync", "--frozen"]:
|
||
problems.append(f"有 uv.lock 时没有走 --frozen：{ok} {argv}")
|
||
finally:
|
||
globals()["ROOT"] = _root
|
||
|
||
if not FULL_SHA.match("a" * 40) or FULL_SHA.match("a" * 12):
|
||
problems.append("完整 SHA 形状判定不对")
|
||
if not manifest_mismatch({"a": ("local",)}, [("a", ("local",)), ("b", ("ci",))]):
|
||
problems.append("清单比对没报出实现比清单多一项")
|
||
if problems:
|
||
print("❌ 门禁自检失败：", file=sys.stderr)
|
||
for p in problems:
|
||
print(f" · {p}", file=sys.stderr)
|
||
return 1
|
||
print(f"✅ 门禁自检通过：汇总 {len(cases)} 分支 + 前端 {len(fe)} 判据"
|
||
f"（含「进程非零必红」「不写死 vitest」）+ pytest 退出码语义 + 完整 SHA + 清单比对"
|
||
f" + Python 锁定/隔离（缺锁失败、从不用 --system） + 版本记录检查")
|
||
return 0
|
||
|
||
|
||
# ── 各检查项 ─────────────────────────────────────────────────────────────
|
||
def c_identity() -> tuple[str, str]:
|
||
want = (os.environ.get("GITHUB_SHA") or "").strip()
|
||
if not want:
|
||
return SKIPPED, "本地运行没有事件负载（GITHUB_SHA 未设置）；CI 上这一项是必需项"
|
||
if not FULL_SHA.match(want):
|
||
return FAILURE, f"事件 SHA 不是完整 40 位：{want[:20]!r}"
|
||
rc, out, _ = run(["git", "rev-parse", "--verify", "HEAD"])
|
||
got = out.strip() if rc == 0 else ""
|
||
if not FULL_SHA.match(got):
|
||
return FAILURE, f"取不到完整 HEAD：{got[:20]!r}"
|
||
if got != want:
|
||
return FAILURE, f"实际 HEAD {got[:12]} != 事件 SHA {want[:12]}"
|
||
return SUCCESS, f"检出 {got} 与事件完整 SHA 逐字符相同"
|
||
|
||
|
||
def c_deps() -> tuple[str, str]:
|
||
langs, web = detect()
|
||
if "node" in langs:
|
||
assert web is not None
|
||
pm = which("npm")
|
||
if not (web / "package-lock.json").is_file() and not (web / "pnpm-lock.yaml").is_file():
|
||
return FAILURE, f"{web.name}/ 下没有锁文件：不接受无约束安装（先提交锁文件）"
|
||
cmd = ([pm, "ci", "--prefix", str(web)] if (web / "package-lock.json").is_file()
|
||
else [which("pnpm"), "install", "--frozen-lockfile", "--dir", str(web)])
|
||
rc, out, err = run(cmd)
|
||
if rc:
|
||
return FAILURE, f"依赖安装失败（锁与清单不一致）：{(err or out).strip()[-200:]}"
|
||
if "go" in langs:
|
||
rc, out, err = run([go_bin(), "mod", "download"], env=go_env())
|
||
if rc:
|
||
return FAILURE, f"go mod download 失败：{(err or out).strip()[-200:]}"
|
||
if "python" in langs:
|
||
ok, argv, why = python_lock_plan()
|
||
if not ok:
|
||
return FAILURE, why
|
||
uv = need("uv", "Python 依赖按锁文件装需要 uv；没有就明确失败，不空跑")
|
||
rc, out, err = run([uv, *argv])
|
||
if rc:
|
||
return FAILURE, f"Python 依赖安装失败（{why}）：{(err or out).strip()[-200:]}"
|
||
# 隔离断言：环境必须是项目内 .venv，**不是**系统解释器（--system 那种）
|
||
rc2, out2, err2 = run([uv, "run", "--frozen", "python", "-c",
|
||
"import sys; assert sys.prefix != sys.base_prefix, 'not isolated'"])
|
||
if rc2:
|
||
return FAILURE, (f"Python 环境没有隔离（uv run 用的是系统解释器，"
|
||
f"不是项目内 .venv）：{(out2 or err2).strip()[-200:]}")
|
||
if not langs:
|
||
return FAILURE, "识别不到任何依赖入口（go.mod / pyproject.toml / package.json 都没有）"
|
||
return SUCCESS, f"依赖按锁文件安装完成（{'/'.join(langs)}）"
|
||
|
||
|
||
def c_format() -> tuple[str, str]:
|
||
langs, web = detect()
|
||
problems = []
|
||
if "go" in langs:
|
||
rc, out, _ = run(["git", "ls-files", "*.go"])
|
||
files = out.split()
|
||
rc, listed, err = run([gofmt_bin(), "-l", *files]) if files else (0, "", "")
|
||
# ⚠️ gofmt -l 列出文件时**退出码仍是 0**：必须判输出
|
||
bad = [l for l in listed.splitlines() if l.strip()]
|
||
if rc != 0:
|
||
problems.append(f"gofmt 执行失败：{err[:120]}")
|
||
elif bad:
|
||
problems.append(f"{len(bad)} 个 Go 文件未格式化（例：{bad[0]}）")
|
||
if "python" in langs:
|
||
# 工具要**从项目环境里跑**（uv run）：锁文件装了哪个版本就用哪个版本，
|
||
# 不依赖 PATH 上碰巧存在的 ruff（那会让本地与 CI 跑出不同结果）。
|
||
uv = need("uv", "Python 格式检查要经 uv 在项目环境里跑（--frozen）；没有就明确失败")
|
||
rc, out, err = run([uv, "run", "--frozen", "ruff", "format", "--check", ".", *["--exclude", "scripts/ci_checks.py", "--exclude", "scripts/check_secrets.py"]])
|
||
if rc:
|
||
problems.append(f"ruff format --check 非零：{(out or err).strip()[-160:]}")
|
||
if "node" in langs and web is not None:
|
||
pm = web / "node_modules" / ".bin" / "prettier"
|
||
if pm.exists():
|
||
rc, out, _ = run([str(pm), "--check", "."], cwd=web)
|
||
if rc != 0:
|
||
problems.append("前端未格式化（prettier --check 非零）")
|
||
else:
|
||
problems.append("package.json 已声明 prettier 但 node_modules 里没有它（先 npm ci）")
|
||
if not problems:
|
||
return SUCCESS, "格式检查通过"
|
||
return FAILURE, "；".join(problems)
|
||
|
||
|
||
def c_static() -> tuple[str, str]:
|
||
langs, web = detect()
|
||
problems = []
|
||
if "go" in langs:
|
||
rc, out, err = run([go_bin(), "vet", "./..."], env=go_env())
|
||
if rc:
|
||
problems.append(f"go vet 非零：{(err or out).strip()[-200:]}")
|
||
if "python" in langs:
|
||
uv = need("uv", "Python 静态检查要经 uv 在项目环境里跑（--frozen）")
|
||
rc, out, err = run([uv, "run", "--frozen", "ruff", "check", ".", *["--exclude", "scripts/ci_checks.py", "--exclude", "scripts/check_secrets.py"]])
|
||
if rc:
|
||
problems.append(f"ruff check 非零：{(out or err).strip()[-160:]}")
|
||
if "node" in langs and web is not None:
|
||
pkg = json.loads((web / "package.json").read_text(encoding="utf-8"))
|
||
if "typecheck" in (pkg.get("scripts") or {}):
|
||
rc, out, err = run([which("npm"), "run", "typecheck", "--prefix", str(web)])
|
||
if rc:
|
||
problems.append(f"前端类型检查非零：{(err or out).strip()[-200:]}")
|
||
else:
|
||
problems.append("package.json 里没有 typecheck 脚本 —— 前端静态检查没有可执行入口")
|
||
if not problems:
|
||
return SUCCESS, "静态检查通过"
|
||
return FAILURE, "；".join(problems)
|
||
|
||
|
||
def c_build() -> tuple[str, str]:
|
||
langs, web = detect()
|
||
problems = []
|
||
if "go" in langs:
|
||
out_dir = Path(os.environ.get("CI_BUILD_OUT") or (ROOT / "bin"))
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
rc, out, err = run([go_bin(), "build", "-trimpath", "-o", f"{out_dir}/", "./cmd/..."], env=go_env())
|
||
if rc:
|
||
problems.append(f"go build 非零：{(err or out).strip()[-200:]}")
|
||
if "python" in langs:
|
||
# Python 没有"通用产物"：这里做**真实**的字节码编译校验（不是空跑）
|
||
skip = {"node_modules", "web", "docs", "scripts", ".git", ".venv", "tmp", "build", "dist"}
|
||
targets = [p.name for p in sorted(ROOT.iterdir()) if p.is_dir() and p.name not in skip]
|
||
rc, out, err = run([sys.executable or "python3", "-m", "compileall", "-q", *(targets or ["."])])
|
||
if rc:
|
||
problems.append(f"Python 编译校验（compileall）非零：{(out or err).strip()[-200:]}")
|
||
if "node" in langs and web is not None:
|
||
pkg = json.loads((web / "package.json").read_text(encoding="utf-8"))
|
||
if "build" in (pkg.get("scripts") or {}):
|
||
rc, out, err = run([which("npm"), "run", "build", "--prefix", str(web)])
|
||
if rc:
|
||
problems.append(f"前端构建非零：{(err or out).strip()[-200:]}")
|
||
else:
|
||
problems.append("package.json 里没有 build 脚本 —— 构建没有可执行入口")
|
||
if not problems:
|
||
return SUCCESS, "构建/编译校验通过（" + "/".join(langs) + "）"
|
||
return FAILURE, "；".join(problems)
|
||
|
||
|
||
def c_tests() -> tuple[str, str]:
|
||
langs, web = detect()
|
||
scope, scope_problems = load_scope()
|
||
problems, notes = [], []
|
||
optional = scope.get("optional") or []
|
||
|
||
def optional_hit(pkg: str, test: str) -> dict | None:
|
||
for e in optional:
|
||
if e.get("package") and e["package"] != pkg:
|
||
continue
|
||
if e.get("test") and e["test"] != test:
|
||
continue
|
||
return e
|
||
return None
|
||
|
||
if "go" in langs:
|
||
rc, out, err = run([go_bin(), "test", "-count=1", "-json", "./..."], env=go_env())
|
||
passed = failed = 0
|
||
required_skips: list[str] = []
|
||
opt_counts: dict[str, int] = {}
|
||
reasons: dict[str, str] = {}
|
||
for line in out.splitlines():
|
||
line = line.strip()
|
||
if not line.startswith("{"):
|
||
continue
|
||
try:
|
||
ev = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
pkg, test = ev.get("Package") or "", ev.get("Test") or ""
|
||
m = re.match(r"\s*\S+_test\.go:\d+:\s*(.+)$", ev.get("Output") or "")
|
||
if m and test:
|
||
reasons[test] = m.group(1).strip()
|
||
if not test:
|
||
continue
|
||
if ev.get("Action") == "pass":
|
||
passed += 1
|
||
elif ev.get("Action") == "fail":
|
||
failed += 1
|
||
elif ev.get("Action") == "skip":
|
||
if optional_hit(pkg, test):
|
||
opt_counts[pkg] = opt_counts.get(pkg, 0) + 1
|
||
else:
|
||
required_skips.append(f"{pkg}::{test}（{reasons.get(test, '未取到原因')}）")
|
||
if failed or rc != 0:
|
||
problems.append(f"go 测试失败 {failed} 个：{(err or out).strip()[-200:]}")
|
||
elif passed == 0:
|
||
problems.append("go 测试一个用例都没执行 —— 无测试不能算全绿")
|
||
elif required_skips:
|
||
problems.append("必需用例被跳过（未在 scripts/test-scope.json 里声明）："
|
||
+ "；".join(required_skips[:3]))
|
||
for e in optional:
|
||
if e.get("package") and "expected_skips" in e:
|
||
got = opt_counts.get(e["package"], 0)
|
||
if got > int(e["expected_skips"]):
|
||
problems.append(f"{e['package']} 的跳过数 {got} > 声明的 expected_skips "
|
||
f"{e['expected_skips']}（涨了就要重新声明）")
|
||
notes.append(f"go 用例 通过 {passed} / 失败 {failed} / 已声明可选未运行 {sum(opt_counts.values())}")
|
||
|
||
if "python" in langs:
|
||
uv = need("uv", "Python 测试要按锁文件跑（uv run --frozen pytest）；没有 uv 就明确失败")
|
||
rc, out, err = run([uv, "run", "--frozen", "pytest", "-q"])
|
||
status, why = judge_python_tests(rc, out or err)
|
||
notes.append(f"pytest 退出码 {rc}")
|
||
if status != SUCCESS:
|
||
problems.append(why)
|
||
|
||
if "node" in langs and web is not None:
|
||
pkg = json.loads((web / "package.json").read_text(encoding="utf-8"))
|
||
dev = {**(pkg.get("devDependencies") or {}), **(pkg.get("dependencies") or {})}
|
||
declared = scope.get("frontend_runner")
|
||
evidence = scope.get("frontend_evidence")
|
||
runner = declared or ("vitest" if "vitest" in dev else ("jest" if "jest" in dev else None))
|
||
rep = None
|
||
rc = 1
|
||
with contextlib.ExitStack() as stack:
|
||
if runner == "vitest":
|
||
d = stack.enter_context(tempfile.TemporaryDirectory())
|
||
out_file = Path(d) / "vitest.json"
|
||
rc, out, err = run([which("npm"), "run", "test", "--prefix", str(web), "--",
|
||
"--reporter=json", f"--outputFile={out_file}"])
|
||
if out_file.is_file():
|
||
try:
|
||
rep = json.loads(out_file.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError as e:
|
||
problems.append(f"vitest 报告解析失败：{e}")
|
||
elif runner == "jest":
|
||
d = stack.enter_context(tempfile.TemporaryDirectory())
|
||
out_file = Path(d) / "jest.json"
|
||
rc, out, err = run([which("npm"), "run", "test", "--prefix", str(web), "--",
|
||
"--json", f"--outputFile={out_file}"])
|
||
if out_file.is_file():
|
||
try:
|
||
j = json.loads(out_file.read_text(encoding="utf-8"))
|
||
rep = {"numTotalTests": j.get("numTotalTests"),
|
||
"numPassedTests": j.get("numPassedTests"),
|
||
"numFailedTests": j.get("numFailedTests"),
|
||
"numPendingTests": j.get("numPendingTests")}
|
||
except json.JSONDecodeError as e:
|
||
problems.append(f"jest 报告解析失败：{e}")
|
||
else:
|
||
rc, out, err = run([which("npm"), "run", "test", "--prefix", str(web)])
|
||
status, why = judge_frontend(rc, rep, runner, evidence, scope_problems)
|
||
notes.append(f"前端框架 {runner or '未声明/未识别'}｜退出码 {rc}")
|
||
if status != SUCCESS:
|
||
problems.append(why)
|
||
|
||
if not problems:
|
||
return SUCCESS, "测试通过｜" + "｜".join(notes)
|
||
return FAILURE, "；".join(problems) + "｜" + "｜".join(notes)
|
||
|
||
|
||
def c_version() -> tuple[str, str]:
|
||
"""CHANGELOG 与版本记录（正本 §6.6）：四类小节 / 无空栏目 / 正式 tag 与提交内版本一致。
|
||
|
||
四条要点：**没有 tag 不算失败**（开发版本不打 tag 是常态）；检查器缺失即红；
|
||
项目没声明版本来源（默认就是 CHANGELOG 顶部）也判红 —— 不许"没有来源所以跳过"；
|
||
**拿不到远端 tag 清单也要红**（`--require-remote`）—— 没核对远端就不许声称"校验成功"。
|
||
"""
|
||
script = ROOT / "scripts" / "check_changelog.py"
|
||
if not script.is_file():
|
||
return FAILURE, "找不到 scripts/check_changelog.py（版本记录检查器）：缺失即红，不跳过"
|
||
if not (ROOT / "CHANGELOG.md").is_file():
|
||
return FAILURE, ("没有 CHANGELOG.md —— 每个项目必须声明版本号权威来源"
|
||
"（默认是 CHANGELOG 顶部那条已发布版本，正本 §6.1）")
|
||
# ⚠️ CI 里必须真的核对远端 tag（检出步骤会写 `.ci-remote-tags`）；查不到远端清单 ⇒ 红，
|
||
# 而不是打一句 ✅ 然后放过去。
|
||
# 本地（没有 CI=true）不加这个开关：一个还没有 origin 的仓不该因为"没法问远端"而在本地假红
|
||
# —— 本地模式下检查器会明确打印「未核对远端」，不会声称"校验成功"。
|
||
strict = ["--require-remote"] if os.environ.get("CI") == "true" else []
|
||
rc, out, err = run([sys.executable or "python3", str(script), "--tags", *strict])
|
||
if rc:
|
||
return FAILURE, f"版本记录检查未通过：{(out or err).strip()[-300:]}"
|
||
tail = [l for l in (out or "").strip().splitlines() if l.strip()][-1:]
|
||
return SUCCESS, f"版本记录合规｜{(tail[0] if tail else '')[:90]}"
|
||
|
||
|
||
def c_security() -> tuple[str, str]:
|
||
"""密钥扫描 + 依赖漏洞。工具缺失 / 库不可达 ⇒ 失败（不许伪装成"没有漏洞"）。"""
|
||
langs, web = detect()
|
||
problems, notes = [], []
|
||
scanner = ROOT / "scripts" / "check_secrets.py"
|
||
if not scanner.is_file():
|
||
problems.append("找不到 scripts/check_secrets.py（密钥扫描器）：缺失即红，不跳过")
|
||
else:
|
||
rc, out, err = run([sys.executable or "python3", str(scanner)])
|
||
if rc == 0:
|
||
notes.append("密钥扫描通过")
|
||
else:
|
||
problems.append(f"密钥扫描未通过：{(out or err).strip()[-200:]}")
|
||
if "go" in langs:
|
||
rc, out, err = run([go_bin(), "run", GOVULNCHECK, "./..."], env=go_env())
|
||
text = out + err
|
||
if "No vulnerabilities found" in text:
|
||
notes.append("govulncheck：无命中")
|
||
elif rc == 0 and "Vulnerability" not in text:
|
||
notes.append("govulncheck：无命中")
|
||
else:
|
||
problems.append(f"govulncheck 有命中或未完成：{text.strip()[-200:]}")
|
||
if "node" in langs and web is not None:
|
||
if not (web / "package-lock.json").is_file():
|
||
problems.append("前端没有锁文件 ⇒ npm audit 无法按锁判定")
|
||
else:
|
||
rc, out, err = run([which("npm"), "audit", "--omit=dev", "--json"], cwd=web)
|
||
try:
|
||
data = json.loads(out or "{}")
|
||
v = ((data.get("metadata") or {}).get("vulnerabilities") or {})
|
||
high = int(v.get("high") or 0) + int(v.get("critical") or 0)
|
||
if high:
|
||
problems.append(f"生产依赖有 {high} 个 high/critical 漏洞")
|
||
else:
|
||
notes.append(f"npm audit（生产依赖）：0 high/critical")
|
||
except json.JSONDecodeError:
|
||
problems.append(f"npm audit 输出解析不了（退出码 {rc}）—— 未完成不算过")
|
||
if "python" in langs:
|
||
problems.append("Python 依赖漏洞扫描尚未接入（pip-audit 需另行审批）—— 显式缺口，不默认为通过")
|
||
if not problems:
|
||
return SUCCESS, "安全门禁通过｜" + "｜".join(notes)
|
||
return FAILURE, "；".join(problems) + "｜" + "｜".join(notes)
|
||
|
||
|
||
def c_manifests(before: dict) -> tuple[str, str]:
|
||
after = {f: digest(ROOT / f) for f in MANIFESTS}
|
||
drift = [f for f in MANIFESTS if before.get(f) != after.get(f)]
|
||
if drift:
|
||
return FAILURE, f"依赖清单在门禁过程中被改写了：{drift}（改写必须由人显式提交）"
|
||
n = len([f for f in MANIFESTS if (ROOT / f).is_file()])
|
||
return SUCCESS, f"依赖清单未被改写（{n} 个文件）"
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
ap = argparse.ArgumentParser(add_help=True)
|
||
ap.add_argument("--list", action="store_true")
|
||
ap.add_argument("--only", default="", help="只跑指定检查（调试用；**不算全绿**）")
|
||
ap.add_argument("--self-test", action="store_true")
|
||
ap.add_argument("--report", default=str(ROOT / os.environ.get("CI_REPORT_NAME", "ci-report.json")))
|
||
ap.add_argument("--profile", default="ci" if os.environ.get("GITHUB_ACTIONS") else "local",
|
||
choices=["local", "ci"])
|
||
args = ap.parse_args(argv)
|
||
if args.self_test:
|
||
return self_test()
|
||
|
||
langs, web = detect()
|
||
if args.list:
|
||
print(f"── 必需检查清单（策略 {POLICY_REVISION}）──")
|
||
for cid, req in CHECK_MANIFEST.items():
|
||
print(f" {cid:<18} {'/'.join(req)}")
|
||
print(f" 识别到的语言入口：{langs or '（无 —— 会判红）'}")
|
||
_, scope_problems = load_scope()
|
||
for p in scope_problems:
|
||
print(f" ⚠️ 范围声明：{p}")
|
||
return 0
|
||
|
||
before = {f: digest(ROOT / f) for f in MANIFESTS}
|
||
results: list[dict] = []
|
||
stopped = False
|
||
started = time.time()
|
||
|
||
def record(cid: str, status: str, reason: str, dur: float = 0.0) -> None:
|
||
results.append({"id": cid, "status": status, "reason": reason,
|
||
"required_in": list(CHECK_MANIFEST[cid]), "duration_s": round(dur, 2)})
|
||
|
||
steps = [("identity", None), ("deps-install", c_deps), ("format", c_format),
|
||
("static", c_static), ("build", c_build), ("tests", c_tests),
|
||
("version", c_version), ("security", c_security)]
|
||
want = [x for x in args.only.split(",") if x] if args.only else None
|
||
if want:
|
||
unknown = [x for x in want if x not in CHECK_MANIFEST]
|
||
if unknown:
|
||
print(f"❌ 没有这些检查 id：{unknown}", file=sys.stderr)
|
||
return 2
|
||
steps = [s for s in steps if s[0] in want]
|
||
print(f"⚠️ --only {args.only}：只跑了子集，**不算全绿**（报告里 overall_status 会写 not_run_only）")
|
||
for cid, fn in steps:
|
||
if stopped:
|
||
record(cid, NOT_RUN, "前一项失败即停，未运行")
|
||
continue
|
||
print(f"── [{cid}] ──")
|
||
if fn is None:
|
||
status, reason = c_identity()
|
||
record(cid, status, reason)
|
||
stopped = status == FAILURE
|
||
print(f" {reason}")
|
||
continue
|
||
if not langs and cid not in LANG_FREE:
|
||
# 与语言无关的检查（版本记录、清单改写核对）**不因为"认不出语言"而停** ——
|
||
# 它们查的是仓库级的记录，跟有没有 go.mod 无关。
|
||
status, reason = FAILURE, "识别不到任何语言入口（go.mod / pyproject.toml / package.json）"
|
||
record(cid, status, reason)
|
||
stopped = True
|
||
print(f" ❌ {reason}")
|
||
continue
|
||
t0 = time.time()
|
||
try:
|
||
status, reason = fn()
|
||
except FileNotFoundError as e:
|
||
status, reason = FAILURE, f"工具缺失：{e}"
|
||
record(cid, status, reason, time.time() - t0)
|
||
stopped = status != SUCCESS
|
||
print(f" {'✅' if status == SUCCESS else '❌'} {reason}")
|
||
|
||
if want is None:
|
||
run_manifests = True
|
||
else:
|
||
run_manifests = "manifests-locked" in want
|
||
if run_manifests:
|
||
if stopped:
|
||
record("manifests-locked", NOT_RUN, "前一项失败即停，未做改写核对")
|
||
else:
|
||
status, reason = c_manifests(before)
|
||
record("manifests-locked", status, reason)
|
||
|
||
# `--only` 时"必需清单"就是被选中的那几项，否则汇总会把没选的项报成"缺失"（噪声）
|
||
selected = {s[0] for s in steps} | ({"manifests-locked"} if run_manifests else set())
|
||
declared = ({cid: CHECK_MANIFEST[cid] for cid in selected} if args.only else CHECK_MANIFEST)
|
||
overall, problems = aggregate(results, args.profile, declared)
|
||
extra = []
|
||
if os.environ.get("GITHUB_ACTIONS") and args.profile != "ci":
|
||
extra.append("运行环境是 CI 但档案是 local —— identity 会被合法跳过，等于关掉身份核对")
|
||
sha = (os.environ.get("GITHUB_SHA") or "").strip()
|
||
if not sha:
|
||
rc, out, _ = run(["git", "rev-parse", "--verify", "HEAD"])
|
||
sha = out.strip() if rc == 0 else ""
|
||
if not FULL_SHA.match(sha or ""):
|
||
extra.append(f"commit_sha 不是完整 40 位（{sha[:20]!r}）—— 仓库是不是还没有提交？")
|
||
problems += extra
|
||
if args.only:
|
||
overall = "not_run_only" if overall == SUCCESS and not problems else FAILURE
|
||
elif problems and overall == SUCCESS:
|
||
overall = FAILURE
|
||
|
||
report = {
|
||
"schema": "ci-report/1", "policy_revision": POLICY_REVISION,
|
||
"repository": os.environ.get("GITHUB_REPOSITORY", ""),
|
||
"commit_sha": sha, "event_sha": (os.environ.get("GITHUB_SHA") or "").strip(),
|
||
"branch": os.environ.get("GITHUB_REF_NAME", ""), "event": os.environ.get("GITHUB_EVENT_NAME", "local"),
|
||
"run_id": os.environ.get("GITHUB_RUN_ID", ""), "profile": args.profile, "run_mode": "full",
|
||
"duration_s": round(time.time() - started, 2), "checks": results,
|
||
"manifest": {cid: list(req) for cid, req in CHECK_MANIFEST.items()},
|
||
"overall_status": overall, "problems": problems,
|
||
"notification_status": "not_attempted", "deployed": False,
|
||
}
|
||
Path(args.report).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
print("\n════════ 汇总 ════════")
|
||
print(" " + "｜".join(f"{r['id']}={r['status']}" for r in results))
|
||
print(f" 总体：{overall}（档案 {args.profile}） 报告：{args.report}")
|
||
for p in problems:
|
||
print(f" ❌ {p}", file=sys.stderr)
|
||
return 1 if problems else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:])) |