From a3a1319a11c61fd4879234ba0691e51e46728107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 12 Oct 2026 00:10:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(ci):=20=E5=94=AF=E4=B8=80=E6=9D=83?= =?UTF-8?q?=E5=A8=81=E9=97=A8=E7=A6=81=20make=20verify=EF=BC=8814=20?= =?UTF-8?q?=E9=A1=B9=E5=BF=85=E9=9C=80=E6=A3=80=E6=9F=A5=EF=BC=8C=E6=B1=87?= =?UTF-8?q?=E6=80=BB=E4=B8=8D=E8=AE=A4"=E6=9C=AA=E8=BF=90=E8=A1=8C"?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次性把本地与 CI 的门禁收成一条命令：make verify = python3 scripts/ci_checks.py。 两边跑的是同一份逻辑，不再各写一套步骤（两边各写一套的结局是迟早只改一边）。 14 项必需检查：identity（检出 == 事件完整 SHA，只在 CI 必需）、commits、deps-install、 format、static-go、static-web、build-go、build-web、tests-go、tests-web、contracts、 security-secrets、security-deps、manifests-locked。make list 打印清单。 三条判据写死在代码里： - 失败即停，但剩下的记 not_run 并如实写进报告；aggregate() 只认 success， 未运行/被跳过都不算通过（不适用要在清单里声明 required_in，不许运行时悄悄 skip）。 - 工具缺失、依赖没按锁装、输出解析不了 ⇒ 失败，不当成"检查通过"。 - 测试必须真跑：go test 加 -count=1 并盘点通过/跳过数；前端读 vitest 的 JSON 报告， 空套件不算通过。跳过原因会打进报告（当前 72 个集成用例因缺 TEST_DATABASE_URL 跳过）。 新增四个自带反向自检的脚本（--self-test 故意弄坏必须红）： scripts/check_format.py（Go 判输出而非退出码 + 前端只能减的基线）、 scripts/check_secrets.py（只打印位置与类别，绝不打印值）、 scripts/check_deps_vuln.py（govulncheck 锁 v1.1.4 + npm audit 分级 + 到期例外）、 scripts/ci_checks.py（门禁 + 汇总）。 验证（本机实跑，全部为真实退出码）： make verify → 总体 success（档案 local），identity=skipped 已在清单里声明为 CI 必需； tests-go 执行 31 个包、通过 549 / 失败 0 / 跳过 72；tests-web 106 个套件、191 个用例全过； security-deps：govulncheck v1.1.4 0 命中、npm audit 生产与全量均 0。 --- .gitignore | 4 + Makefile | 83 +++-- scripts/check_deps_vuln.py | 313 +++++++++++++++++ scripts/check_format.py | 158 +++++++++ scripts/check_secrets.py | 198 +++++++++++ scripts/ci_checks.py | 498 ++++++++++++++++++++++++++++ security/dependency-exceptions.json | 18 + 7 files changed, 1248 insertions(+), 24 deletions(-) create mode 100644 scripts/check_deps_vuln.py create mode 100644 scripts/check_format.py create mode 100644 scripts/check_secrets.py create mode 100644 scripts/ci_checks.py create mode 100644 security/dependency-exceptions.json diff --git a/.gitignore b/.gitignore index 9fbeb3b..0b8e0de 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,10 @@ .vscode/ .DS_Store +# 门禁产物（每次运行重写；不是证据，证据是运行日志与 Gitea 的 run） +/ci-report.json +/.ci-manifests.json + # Local artifacts (binaries, staging scripts, backups) /release-diag /worker diff --git a/Makefile b/Makefile index 2e906c0..70a72b9 100644 --- a/Makefile +++ b/Makefile @@ -1,50 +1,85 @@ GO ?= go +PY ?= python3 # 版本号的唯一来源是仓库根的 CHANGELOG.md（正本 §6.1：版本号只有一个来源）。 # 刻意**不用** `git describe --tags`：CI 检出是 `git fetch --depth 50`，取不到 tag， # 那会让版本号在本地正常、在 CI 上静默退化（见 docs/agents/traps.md trap-5）。 -VERSION ?= $(shell python3 scripts/version.py 2>/dev/null || echo dev) +VERSION ?= $(shell $(PY) scripts/version.py 2>/dev/null || echo dev) COMMIT_SHA ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown) BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) -LDFLAGS := -X knowledge-governance/internal/version.Number=$(VERSION) -X knowledge-governance/internal/version.CommitSHA=$(COMMIT_SHA) -X knowledge-governance/internal/version.BuildTime=$(BUILD_TIME) -.PHONY: test vet build web-test web-typecheck web-build verify version changelog-check version-contract image +.PHONY: verify checks list \ + format static-go static-web deps-install build-go build-web \ + tests-go tests-web contracts security manifests-locked \ + test vet build web-test web-typecheck web-build \ + version changelog-check version-contract image -test: - $(GO) test ./... +# ── 唯一权威门禁入口（正本 §3.1）──────────────────────────────────────── +# 一条命令跑完全部必需检查、**串行**、失败即停，并产出 ci-report.json。 +# CI 的 build 作业跑的就是这一条（`.gitea/workflows/build.yml` 里的 `make verify`）—— +# 本地与 CI 同一份逻辑，不存在"两边各写一套，迟早只改一边"的问题。 +verify: + $(PY) scripts/ci_checks.py -vet: - $(GO) vet ./... +# 清单：哪些是必需项、在哪个档案（local / ci）下必需 +list: + $(PY) scripts/ci_checks.py --list -build: - $(GO) build -trimpath -ldflags "$(LDFLAGS)" ./cmd/... +# 门禁自身的反向自检（汇总逻辑：必需项未运行/被跳过时**不得**报全绿） +checks: + $(PY) scripts/ci_checks.py --self-test -web-test: - npm --prefix web test +# ── 单项（调试用）────────────────────────────────────────────────────── +# ⚠️ `make <单项>` 只跑子集，**不算全绿**：单项命令会自己在输出里说明这一点。 +# 写这一组是为了"红了要知道红在哪一项"，不是为了跳过整个门禁。 +format: + $(PY) scripts/ci_checks.py --only format +static-go: + $(PY) scripts/ci_checks.py --only static-go +static-web: + $(PY) scripts/ci_checks.py --only static-web +deps-install: + $(PY) scripts/ci_checks.py --only deps-install +build-go: + $(PY) scripts/ci_checks.py --only build-go +build-web: + $(PY) scripts/ci_checks.py --only build-web +tests-go: + $(PY) scripts/ci_checks.py --only tests-go +tests-web: + $(PY) scripts/ci_checks.py --only tests-web +contracts: + $(PY) scripts/ci_checks.py --only contracts +security: + $(PY) scripts/ci_checks.py --only security-secrets,security-deps +manifests-locked: + $(PY) scripts/ci_checks.py --only manifests-locked -web-typecheck: - npm --prefix web run typecheck - -web-build: - npm --prefix web run build - -# 本地门禁：一条命令跑完、**串行**、失败即停（正本 §3.1）。CI 的 build 作业跑同一套。 -verify: changelog-check test vet build version-contract web-test web-typecheck web-build +# ── 旧名字保留为别名：文档和习惯都还在引用它们 ───────────────────────── +test: tests-go +vet: static-go +build: build-go +web-test: tests-web +web-typecheck: static-web +web-build: build-web # 打印当前版本号（唯一来源：CHANGELOG.md 顶部那个已发布版本） version: - @python3 scripts/version.py + @$(PY) scripts/version.py # 变更记录结构自检：每个已发布版本都必须有「改了什么」「怎么验收」两节 changelog-check: - @python3 scripts/check_changelog.py + @$(PY) scripts/check_changelog.py -# 版本契约：注入进二进制的号必须等于 CHANGELOG 顶部（不注入时该用例会 SKIP，属正常） +# 版本契约：注入进二进制的号必须等于 CHANGELOG 顶部。 +# ⚠️ 不注入时该用例会 SKIP —— 那是"本地不想被迫先构建"的便利，但**CI 必须断言它真的 PASS** +# （门禁的 contracts 项就是这么做的；被跳过的断言等于没有断言）。 version-contract: - $(GO) test -ldflags "$(LDFLAGS)" -run TestNumberMatchesChangelog -v ./internal/version/... + $(GO) test -ldflags "-X knowledge-governance/internal/version.Number=$(VERSION)" \ + -count=1 -run TestNumberMatchesChangelog -v ./internal/version/... # 构建镜像。三个 build-arg 必须显式传入 —— 不传就落成 dev/unknown（故意的： # 一个显眼的 dev 好过一个看起来正常的假版本号）。compose.yaml 的 args 透传见 PR TODO。 image: docker build --build-arg VERSION=$(VERSION) --build-arg COMMIT_SHA=$(COMMIT_SHA) \ - --build-arg BUILD_TIME=$(BUILD_TIME) -t knowledge-governance-p1:$(VERSION)-$(COMMIT_SHA) . + --build-arg BUILD_TIME=$(BUILD_TIME) -t knowledge-governance-p1:$(VERSION)-$(COMMIT_SHA) . \ No newline at end of file diff --git a/scripts/check_deps_vuln.py b/scripts/check_deps_vuln.py new file mode 100644 index 0000000..f1e87a9 --- /dev/null +++ b/scripts/check_deps_vuln.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""依赖漏洞检查（Go + 前端）—— 判据 → 退出码。 + +两个扫描器： + · Go ：`govulncheck`（**可达性分析**：只报真的会被调用到的漏洞），版本锁死。 + · 前端 ：`npm audit`（按 severity 分级；生产依赖与仅开发依赖分开判）。 + +分级与例外（正本 §4.5）： + · 生产依赖出现 high/critical ⇒ **阻断**。 + · 仅开发依赖出现 high/critical ⇒ 阻断，**除非**在 `security/dependency-exceptions.json` + 里有一条**有效**例外（必须有理由 / 责任人 / 批准人 / 到期日，且未过期）。 + · moderate 及以下 ⇒ 只记录，不阻断。 + · govulncheck 的命中**一律阻断**（它已经做过可达性分析），除非有有效例外。 + · 扫描器缺工具、连不上漏洞库、输出解析不了 ⇒ **失败**，绝不当成"没有漏洞"。 + +为什么例外要写全字段：没有到期日的例外会永久留在仓库里，而半年后没人记得为什么。 +批准人字段是给人看的责任锚点 —— 本脚本**不会**替任何人填它。 + +用法： + python3 scripts/check_deps_vuln.py # 真跑两个扫描器 + python3 scripts/check_deps_vuln.py --self-test # 只跑分级逻辑的反向自检（用合成夹具） + python3 scripts/check_deps_vuln.py --skip-go # 只查前端（调试用，不算全绿） +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WEB = ROOT / "web" +EXCEPTIONS = ROOT / "security" / "dependency-exceptions.json" + +GOVULNCHECK = "golang.org/x/vuln/cmd/govulncheck@v1.1.4" # 锁版本：不静默升级工具 +BLOCKING_SEV = {"critical", "high"} +EXCEPTION_FIELDS = ("id", "severity", "scope", "reason", "owner", "approved_by", "expires_on") + + +def parse_json_stream(text: str) -> list[dict]: + """govulncheck 的 JSON 是**多个 pretty-printed 对象首尾相接**，不是行分隔。 + 按行解析会崩（实测：'str' object has no attribute 'keys'）。""" + dec = json.JSONDecoder() + out, i, n = [], 0, len(text) + while i < n: + while i < n and text[i] in " \t\r\n": + i += 1 + if i >= n: + break + try: + obj, j = dec.raw_decode(text, i) + except json.JSONDecodeError: + break + if isinstance(obj, dict): + out.append(obj) + i = j + return out + + +def load_exceptions(path: Path = EXCEPTIONS) -> tuple[list[dict], list[str]]: + """返回 (条目, 加载问题)。文件缺失或坏掉都算问题 —— 不能静默当成"没有例外"。""" + if not path.is_file(): + return [], [f"{path.relative_to(ROOT)} 不存在（例外机制应当显式存在，哪怕里面是空的）"] + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + return [], [f"{path.relative_to(ROOT)} 解析失败：{e}"] + entries = raw.get("exceptions") if isinstance(raw, dict) else None + if not isinstance(entries, list): + return [], [f"{path.relative_to(ROOT)} 里没有 exceptions 列表"] + return [e for e in entries if isinstance(e, dict)], [] + + +def exception_state(entry: dict, today: dt.date) -> tuple[bool, str]: + """一条例外是否**有效**。无效的例外等于没有例外（不许拿它换绿灯）。""" + missing = [f for f in EXCEPTION_FIELDS if not str(entry.get(f) or "").strip()] + if missing: + return False, f"例外缺少字段 {missing}" + try: + expires = dt.date.fromisoformat(str(entry["expires_on"])) + except ValueError: + return False, f"例外 {entry.get('id')} 的 expires_on 不是 YYYY-MM-DD" + if expires < today: + return False, f"例外 {entry.get('id')} 已于 {expires} 过期（过期即失效，要重新批）" + if str(entry["approved_by"]).strip().lower() in {"", "todo", "tbd", "pending", "self"}: + return False, f"例外 {entry.get('id')} 的 approved_by 是占位值（必须由人批）" + return True, "" + + +def matches(entry: dict, finding: dict) -> bool: + if str(entry.get("tool") or "") != finding["tool"]: + return False + if entry.get("package") and entry["package"] != finding.get("package"): + return False + return not (entry.get("id") and entry["id"] != finding.get("id")) + + +def grade(findings: list[dict], entries: list[dict], today: dt.date) -> dict: + """纯函数：给定命中与例外，判定阻断 / 豁免 / 仅记录。--self-test 直接打这个函数。""" + blocking, exempted, notes = [], [], [] + for f in findings: + sev = str(f.get("severity") or "").lower() + if f["tool"] == "npm-audit" and sev not in BLOCKING_SEV: + notes.append(f"{f['tool']} {f.get('package')} severity={sev} —— 低于阻断线，仅记录") + continue + if f["tool"] == "npm-audit" and f.get("scope") == "dev" and sev in BLOCKING_SEV: + pass # 开发依赖也走同一条路：要么修，要么有有效例外 + cands = [e for e in entries if matches(e, f)] + if not cands: + blocking.append({**f, "reason": f"{sev or '未标severity'} 命中，且没有对应例外"}) + continue + ok, why = exception_state(cands[0], today) + if ok: + exempted.append({**f, "reason": f"例外 {cands[0]['id']}（到期 {cands[0]['expires_on']}）"}) + else: + blocking.append({**f, "reason": f"例外无效：{why}"}) + return {"blocking": blocking, "exempted": exempted, "notes": notes} + + +# ── 扫描器 ──────────────────────────────────────────────────────────────── +def scan_go() -> tuple[list[dict], str | None, str]: + go = shutil.which("go") or "/opt/citools/go/bin/go" + if not (Path(go).exists() or shutil.which("go")): + return [], "找不到 go，govulncheck 无法运行（工具缺失 ⇒ 失败，不当成无漏洞）", "" + env = dict(os.environ) + env.setdefault("GOFLAGS", "-mod=readonly") + env.setdefault("GOPROXY", "https://goproxy.cn,direct") + env.setdefault("GOTOOLCHAIN", "local") + try: + p = subprocess.run([go, "run", GOVULNCHECK, "-format", "json", "./..."], + cwd=ROOT, capture_output=True, text=True, env=env, timeout=900) + except subprocess.TimeoutExpired: + return [], "govulncheck 超时（漏洞库不可达时会这样）—— 未完成 ⇒ 失败", "" + objs = parse_json_stream(p.stdout) + if not objs: + return [], (f"govulncheck 没有产出任何可解析的 JSON（exit={p.returncode}）：" + f"{(p.stderr or p.stdout)[:300]}"), "" + cfg = next((o["config"] for o in objs if isinstance(o.get("config"), dict)), {}) + findings = [] + for o in objs: + f = o.get("finding") + if not isinstance(f, dict) or not isinstance(f.get("osv"), dict): + continue + osv = f["osv"] + pkg = "" + for fr in f.get("trace") or []: + if isinstance(fr, dict) and fr.get("module"): + pkg = fr["module"] + break + findings.append({"tool": "govulncheck", "id": osv.get("id", ""), "package": pkg, + "severity": "", "scope": "build", + "detail": (osv.get("summary") or "")[:200]}) + info = f"govulncheck {cfg.get('scanner_version', '?')}（DB {cfg.get('db_last_modified', '?')}）" + return findings, None, info + + +def npm_audit(args: list[str]) -> tuple[dict, str | None]: + if not (WEB / "package-lock.json").is_file(): + return {}, "web/package-lock.json 不存在 ⇒ npm audit 无法按锁文件判定（先补锁文件）" + try: + p = subprocess.run(["npm", "audit", "--json", *args], cwd=WEB, + capture_output=True, text=True, timeout=600) + except subprocess.TimeoutExpired: + return {}, "npm audit 超时（registry 不可达？）—— 未完成 ⇒ 失败" + try: + return json.loads(p.stdout or "{}"), None + except json.JSONDecodeError: + return {}, f"npm audit 输出解析不了（exit={p.returncode}）：{(p.stdout or p.stderr)[:300]}" + + +def audit_findings(data: dict, scope: str) -> list[dict]: + out = [] + for name, adv in (data.get("vulnerabilities") or {}).items(): + if not isinstance(adv, dict): + continue + sev = str(adv.get("severity") or "").lower() + ids = [v.get("url", "").rsplit("/", 1)[-1] + for v in (adv.get("via") or []) if isinstance(v, dict) and v.get("url")] + out.append({"tool": "npm-audit", "id": ids[0] if ids else "", "package": name, + "severity": sev, "scope": scope, + "detail": f"isDirect={adv.get('isDirect')} range={adv.get('range')}"}) + return out + + +# ── 反向自检：分级逻辑必须真的会红、也必须真的会豁免 ────────────────────── +def self_test() -> int: + today = dt.date(2026, 10, 11) + good_exc = {"tool": "npm-audit", "id": "GHSA-aaaa", "package": "tinypool", "severity": "critical", + "scope": "dev", "reason": "仅测试工具链，不进产物", "owner": "yao", + "approved_by": "yao", "expires_on": "2026-12-31"} + cases: list[tuple[str, list[dict], list[dict], dict]] = [ + ("生产依赖 high", [{"tool": "npm-audit", "id": "G1", "package": "vite", "severity": "high", + "scope": "prod"}], [], {"blocking": 1, "exempted": 0}), + ("开发依赖 critical 且无例外", [{"tool": "npm-audit", "id": "G2", "package": "tinypool", + "severity": "critical", "scope": "dev"}], [], {"blocking": 1, "exempted": 0}), + ("开发依赖 critical + 有效例外", [{"tool": "npm-audit", "id": "G3", "package": "tinypool", + "severity": "critical", "scope": "dev"}], + [{**good_exc, "id": "G3", "package": "tinypool"}], + {"blocking": 0, "exempted": 1}), + ("例外已过期", [{"tool": "npm-audit", "id": "G4", "package": "tinypool", "severity": "critical", + "scope": "dev"}], [{**good_exc, "id": "G4", "expires_on": "2026-01-01"}], + {"blocking": 1, "exempted": 0}), + ("例外缺批准人", [{"tool": "npm-audit", "id": "G5", "package": "tinypool", "severity": "critical", + "scope": "dev"}], [{**good_exc, "id": "G5", "approved_by": "TBD"}], + {"blocking": 1, "exempted": 0}), + ("例外包名不匹配", [{"tool": "npm-audit", "id": "G6", "package": "other-pkg", "severity": "high", + "scope": "dev"}], [good_exc], {"blocking": 1, "exempted": 0}), + ("moderate 一律只记录", [{"tool": "npm-audit", "id": "G7", "package": "x", "severity": "moderate", + "scope": "dev"}], [], {"blocking": 0, "exempted": 0}), + ("govulncheck 命中一律阻断", [{"tool": "govulncheck", "id": "GO-2026-0001", "package": "stdlib", + "severity": "", "scope": "build"}], [], {"blocking": 1, "exempted": 0}), + ("govulncheck 命中 + 有效例外", [{"tool": "govulncheck", "id": "GO-2026-0002", "package": "stdlib", + "severity": "", "scope": "build"}], + [{**good_exc, "tool": "govulncheck", "id": "GO-2026-0002", "package": "stdlib"}], + {"blocking": 0, "exempted": 1}), + ] + problems: list[str] = [] + for name, findings, excs, want in cases: + got = grade(findings, excs, today) + if len(got["blocking"]) != want["blocking"] or len(got["exempted"]) != want["exempted"]: + problems.append(f"{name}：期望 阻断{want['blocking']}/豁免{want['exempted']}，" + f"实际 阻断{len(got['blocking'])}/豁免{len(got['exempted'])}") + # 解析器自检：govulncheck 的 JSON 是"多个 pretty 对象首尾相接" + stream = ('{"config": {"scanner_version": "v1.1.4"}}\n' + '{\n "finding": {"osv": {"id": "GO-1", "summary": "s"}, ' + '"trace": [{"module": "example.com/m"}]}\n}\n') + objs = parse_json_stream(stream) + if len(objs) != 2: + problems.append(f"JSON 流解析器没认出两个对象（得到 {len(objs)} 个）") + if problems: + print("❌ 依赖漏洞分级自检失败：", file=sys.stderr) + for p in problems: + print(f" · {p}", file=sys.stderr) + return 1 + print(f"✅ 依赖漏洞分级自检通过：{len(cases)} 个判据 + JSON 流解析器全部符合预期") + return 0 + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--skip-go", action="store_true", help="跳过 govulncheck（调试用，不算全绿）") + args = ap.parse_args(argv) + if args.self_test: + return self_test() + + entries, exc_problems = load_exceptions() + today = dt.date.today() + findings: list[dict] = [] + info: list[str] = [] + tool_errors: list[str] = [] + + if not args.skip_go: + go_findings, err, go_info = scan_go() + if err: + tool_errors.append(err) + findings += go_findings + if go_info: + info.append(go_info) + + prod, err = npm_audit(["--omit=dev"]) + if err: + tool_errors.append(f"生产依赖：{err}") + else: + info.append(f"npm audit 生产依赖：{json.dumps((prod.get('metadata') or {}).get('vulnerabilities'), ensure_ascii=False)}") + findings += audit_findings(prod, "prod") + + full, err = npm_audit([]) + if err: + tool_errors.append(f"全量依赖：{err}") + else: + info.append(f"npm audit 全量：{json.dumps((full.get('metadata') or {}).get('vulnerabilities'), ensure_ascii=False)}") + prod_names = {f["package"] for f in findings if f["tool"] == "npm-audit" and f["scope"] == "prod"} + findings += [f for f in audit_findings(full, "dev") if f["package"] not in prod_names] + + print("── 依赖漏洞检查 ──") + for line in info: + print(f" 工具/数据源：{line}") + for line in exc_problems: + print(f" ⚠️ 例外加载：{line}", file=sys.stderr) + + verdict = grade(findings, entries, today) + for f in verdict["exempted"]: + print(f" ⚪ 已豁免：{f['tool']} {f.get('id') or f.get('package')} —— {f['reason']}") + for f in verdict["notes"]: + print(f" · 仅记录：{f}") + + fail = bool(tool_errors) or bool(exc_problems) or bool(verdict["blocking"]) + if tool_errors: + print("❌ 扫描器没跑成 —— 这**不是**\"没有漏洞\"（正本 §4.5）：", file=sys.stderr) + for e in tool_errors: + print(f" · {e}", file=sys.stderr) + if exc_problems: + print("❌ 例外文件有问题，无法据它做分级判定：", file=sys.stderr) + if verdict["blocking"]: + print("❌ 阻断级依赖漏洞：", file=sys.stderr) + for f in verdict["blocking"]: + print(f" {f['tool']} {f.get('id') or '?'} {f.get('package') or '?'} " + f"severity={f.get('severity') or 'n/a'} scope={f.get('scope')} —— {f['reason']}", file=sys.stderr) + print(" 处理：升级到修复版本（改 package.json/go.mod + 同步锁文件，独立提交）；" + "确有理由不修的，写进 security/dependency-exceptions.json 并写清责任人、批准人、到期日。", + file=sys.stderr) + if not fail: + print(f"✅ 依赖漏洞检查通过（阻断级 0；豁免 {len(verdict['exempted'])}；仅记录 {len(verdict['notes'])}）") + return 1 if fail else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) \ No newline at end of file diff --git a/scripts/check_format.py b/scripts/check_format.py new file mode 100644 index 0000000..b58b234 --- /dev/null +++ b/scripts/check_format.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""格式门禁：Go（gofmt）+ 前端（prettier）。 + +判据（两条独立的账）： + · Go ：`gofmt -l` 对**被跟踪的 .go 文件**输出必须为空 —— 一条都不许欠。 + · 前端：`prettier --list-different` 的输出必须**完全落在基线里**（`web/.prettier-baseline.json`）。 + 基线是**一次性账面债**，只能减不能增： + · 出现基线之外的文件 ⇒ 红（新写的代码必须格式化过）； + · 基线里的文件已格式化 ⇒ 提示删条目（不判红）； + · 基线到期（review_by）⇒ 红 —— 要么一次清干净、要么由人重新批准，**不许无声续命**。 + +为什么前端不一刀切全量格式化：本仓有一部分是刻意压行的紧凑写法，一次性重排约 2 万行， +那会把"接入 CI"变成"重写前端"。所以先用**列明 + 只能减 + 到期**的基线接住存量， +把增量卡住；全量清理留作独立提交（由人决定何时做）。 + +用法： + python3 scripts/check_format.py + python3 scripts/check_format.py --self-test # 只跑判定逻辑的反向自检 + python3 scripts/check_format.py --web-only # 只查前端（调试用，不算全绿） +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WEB = ROOT / "web" +BASELINE = WEB / ".prettier-baseline.json" + + +def compare(unformatted: list[str], baseline: dict, today: dt.date) -> tuple[list[str], list[str]]: + """纯判定：返回 (问题, 提示)。--self-test 直接打这个函数。""" + problems: list[str] = [] + notes: list[str] = [] + listed = baseline.get("unformatted") + if not isinstance(listed, list): + return [f"{BASELINE.name} 里没有 unformatted 列表 —— 基线坏掉时不能静默放行"], notes + review_by = baseline.get("review_by") + try: + due = dt.date.fromisoformat(str(review_by)) + except (TypeError, ValueError): + problems.append(f"{BASELINE.name} 的 review_by 不是 YYYY-MM-DD —— 没有到期日的基线会永久留在仓库里") + due = None + if due and today > due: + problems.append(f"前端格式债基线已于 {due} 到期：要么 `npm --prefix web run format` 一次清干净，" + f"要么由人重新批准并更新 review_by（基线不许无声续命）") + + known, now = set(listed), set(unformatted) + for f in sorted(now - known): + problems.append(f"新出现未格式化的文件：{f} —— 跑 `npm --prefix web run format` 再提交") + for f in sorted(known - now): + notes.append(f"基线里的 {f} 已经格式化，可以删掉这条") + notes.append(f"存量格式债：基线 {len(known)} 个文件，实际未格式化 {len(now)} 个") + return problems, notes + + +def self_test() -> int: + today = dt.date(2026, 10, 11) + good = {"review_by": "2026-12-31", "unformatted": ["a.ts", "b.ts"]} + cases = [ + ("存量文件仍欠 = 只提示不红", ["a.ts", "b.ts"], good, 0), + ("出现基线外的新欠账 = 红", ["a.ts", "c.ts"], good, 1), + ("基线里已修好的条目 = 只提示不红", ["a.ts"], good, 0), + ("基线到期 = 红", ["a.ts", "b.ts"], {**good, "review_by": "2026-01-01"}, 1), + ("基线没有 unformatted = 红", ["a.ts"], {"review_by": "2026-12-31"}, 1), + ("基线没有到期日 = 红", ["a.ts"], {"unformatted": ["a.ts"]}, 1), + ] + problems: list[str] = [] + for name, unformatted, baseline, want in cases: + got, _ = compare(unformatted, baseline, today) + if len(got) != want: + problems.append(f"{name}：期望 {want} 个问题，实际 {len(got)} 个 {got}") + if problems: + print("❌ 格式门禁自检失败：", file=sys.stderr) + for p in problems: + print(f" · {p}", file=sys.stderr) + return 1 + print(f"✅ 格式门禁自检通过：{len(cases)} 个判定分支全部符合预期") + return 0 + + +def check_go() -> tuple[bool, list[str]]: + files = subprocess.run(["git", "ls-files", "*.go"], cwd=ROOT, + capture_output=True, text=True).stdout.split() + if not files: + return True, ["没有 .go 文件，跳过（本仓不是 Go 仓时这里应当为空）"] + gofmt = shutil.which("gofmt") or "/opt/citools/go/bin/gofmt" + if not (shutil.which("gofmt") or Path(gofmt).exists()): + return False, ["找不到 gofmt —— 工具缺失 ⇒ 失败，不当成'格式没问题'"] + p = subprocess.run([gofmt, "-l", *files], cwd=ROOT, capture_output=True, text=True) + # ⚠️ gofmt -l 列出文件时**退出码仍是 0** —— 只看退出码就是永远的假绿。 + bad = [l for l in p.stdout.splitlines() if l.strip()] + if p.returncode != 0: + return False, [f"gofmt 执行失败（exit={p.returncode}）：{p.stderr[:300]}"] + if bad: + return False, [f"{f} 未格式化" for f in bad] + ["跑 `gofmt -w <文件>` 再提交"] + return True, [f"gofmt 干净（{len(files)} 个文件）"] + + +def check_web() -> tuple[bool, list[str]]: + if not (WEB / "package-lock.json").is_file(): + return False, ["web/package-lock.json 不存在 ⇒ 依赖没按锁文件装，格式门禁不可信"] + prettier = WEB / "node_modules" / ".bin" / "prettier" + if not prettier.exists(): + return False, ["web/node_modules 里没有 prettier —— 先 `npm --prefix web ci`（工具缺失 ⇒ 失败）"] + p = subprocess.run([str(prettier), "--list-different", "."], cwd=WEB, + capture_output=True, text=True) + if p.returncode not in (0, 1): + return False, [f"prettier 执行失败（exit={p.returncode}）：{(p.stderr or p.stdout)[:300]}"] + unformatted = [l.strip() for l in p.stdout.splitlines() if l.strip()] + if not BASELINE.is_file(): + return False, [f"找不到 {BASELINE.relative_to(ROOT)} —— 没有基线就不能判断'新欠账'"] + try: + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + return False, [f"{BASELINE.name} 解析失败：{e}"] + problems, notes = compare(unformatted, baseline, dt.date.today()) + for n in notes: + print(f" · {n}") + return not problems, problems + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--web-only", action="store_true", help="只查前端（调试用，不算全绿）") + args = ap.parse_args(argv) + if args.self_test: + return self_test() + + ok = True + print("── 格式检查 ──") + if not args.web_only: + go_ok, go_msgs = check_go() + print(" Go（gofmt）：" + ("✅" if go_ok else "❌")) + for m in go_msgs: + print((" " if go_ok else " ❌ ") + m, file=sys.stdout if go_ok else sys.stderr) + ok = ok and go_ok + if not go_ok: + print(" ⇒ 后续检查继续跑（格式债不阻塞其它结论，但总体已经不可能全绿）") + web_ok, web_msgs = check_web() + print(" 前端（prettier）：" + ("✅" if web_ok else "❌")) + for m in web_msgs: + print((" " if web_ok else " ❌ ") + m, file=sys.stdout if web_ok else sys.stderr) + ok = ok and web_ok + if args.web_only: + print(" ⚠️ 只跑了前端（--web-only）：**不算全绿**") + print("✅ 格式检查通过" if ok else "❌ 格式检查未通过") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) \ No newline at end of file diff --git a/scripts/check_secrets.py b/scripts/check_secrets.py new file mode 100644 index 0000000..599ba3c --- /dev/null +++ b/scripts/check_secrets.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""密钥泄漏扫描（检出树 → 退出码）。 + +为什么自己写而不是用 gitleaks： + · 作业容器里没有 gitleaks，本机也没有；在 CI 运行时联网下载一个二进制， + 等于把"扫描器本身"变成供应链风险，而且违反"工具缺失直接失败、不在运行中静默安装"。 + · 本仓不需要把源码发给任何外部服务（正本 §7：不得发送项目源码给未授权外部服务）。 + ⚠️ 代价：规则集比 gitleaks 小。这是**已知缺口**，不是等价替代（见 README/交付说明）。 + +纪律： + 1. **只打印位置与类别，绝不打印命中的值**（正本 §4.5）。掩码也不打 —— + 掩码 + 上下文足以在聊天记录里复原短密钥。 + 2. 扫描范围 = `git ls-files`（检出树里的**被跟踪文件**）。未跟踪的 `.env`、 + `secrets/`、`node_modules/` 不在范围里 —— 它们要么被忽略，要么根本不该在检出树里。 + 3. 每次运行先跑 `--self-test`（用合成密钥夹具 + 良性对照）： + 一个不再命中的扫描器比没有扫描器更糟，它会给人"已经查过"的错觉（正本 §0-2）。 + +用法： + python3 scripts/check_secrets.py # 扫描 + 先跑自检 + python3 scripts/check_secrets.py --no-self-test # 只扫描（调试用） + python3 scripts/check_secrets.py --self-test # 只跑自检 +""" +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +# ── 规则集：每条都必须能定位到具体文件/行，且必须能被 --self-test 证明"真的会命中" ── +# 形状：(rule_id, 类别, 正则, 密钥分组名或 None) +# 「密钥分组」= 命中里"真的是秘密"的那一段，用来判断它是不是占位符 / 模板表达式。 +# 没有分组的规则（私钥标记、固定前缀的令牌）整段都算秘密。 +RULES: list[tuple[str, str, re.Pattern[str], str | None]] = [ + ("private-key", "私钥块", + re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----"), None), + ("aws-ak", "AWS Access Key ID", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), None), + ("aliyun-ak", "阿里云 AccessKey ID", re.compile(r"\bLTAI[0-9A-Za-z]{12,}\b"), None), + ("forge-token", "代码托管平台令牌", + re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|glpat-[A-Za-z0-9_\-]{20,})\b"), None), + ("jwt", "JWT", + re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b"), None), + # 口令段只收"干净字符"：`***`（脱敏）、`{u.quote(password)}`（模板表达式）、 + # `$PGPASSWORD`（变量）都不该命中 —— 那三种在 scripts/*.sh 里到处都是，收了就是永远的假红。 + ("pg-url", "带口令的数据库连接串", + re.compile(r"\bpostgres(?:ql)?://[A-Za-z0-9_.\-]+:(?P[A-Za-z0-9+/=_\-]{6,})@"), "secret"), + # 前缀不写 `\b`：`DINGTALK_APP_SECRET` 这类"带前缀的名字"才是最常见的写法， + # 而 `_` 是 word 字符，`\b` 会把它们全部漏掉。 + ("assigned-secret", "被赋值的密钥字面量", + re.compile(r"(?i)\b[A-Za-z0-9_]*(?:api[_-]?key|app[_-]?secret|client[_-]?secret|access[_-]?key" + r"|secret|token|passwd|password)\b\s*[:=]\s*[\"'](?P[A-Za-z0-9+/=_\-]{16,})[\"']"), "secret"), +] + +# 占位符：整段值就是这些形状时不算密钥（否则模板 / 示例文件会把门禁撑成永远的假红） +PLACEHOLDER = re.compile( + r"(?i)^(?:" + r"<[^>]*>|\{\{[^}]*\}\}|\$\{[^}]*\}|\$[A-Za-z_][A-Za-z0-9_]*|" + r"x{3,}|\.{3,}|\*{3,}|-+|_+|" + r"(?:your|my|some|test|dummy|example|sample|placeholder|fake|redacted|changeme|change[_-]?me|" + r"secret|token|password|api[_-]?key|none|null|todo|tbd)[A-Za-z0-9_\-]*" + r")$" +) + +TEXT_EXT = { + ".go", ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".json", ".yaml", ".yml", + ".md", ".sh", ".sql", ".css", ".html", ".env", ".example", ".txt", ".toml", ".ini", + ".cfg", ".conf", ".properties", ".xml", ".tsv", ".csv", +} +MAX_BYTES = 2_000_000 + + +def tracked_files() -> list[str]: + """被跟踪的文件（含未跟踪但已 add 的）。检出树里没有的东西不扫。""" + out = subprocess.run(["git", "ls-files"], cwd=ROOT, capture_output=True, text=True) + if out.returncode != 0: + print("❌ 拿不到 git ls-files —— 扫描范围无法确定，直接失败", file=sys.stderr) + sys.exit(2) + return [l for l in out.stdout.splitlines() if l.strip()] + + +def looks_textual(path: Path) -> bool: + if path.suffix.lower() not in TEXT_EXT and path.name not in {".env.example", "Makefile", "Dockerfile"}: + return False + try: + if path.stat().st_size > MAX_BYTES: + return False + return b"\x00" not in path.read_bytes()[:4096] + except OSError: + return False + + +def mask_scan(text: str) -> list[tuple[str, str, int]]: + """返回 [(rule_id, 类别, 行号)]。**不返回命中的值**。""" + hits = [] + for rule_id, label, rx, secret_group in RULES: + for m in rx.finditer(text): + value = (m.group(secret_group) if secret_group else m.group(0)) or "" + if PLACEHOLDER.match(value.strip().strip("\"'")): + continue + hits.append((rule_id, label, text.count("\n", 0, m.start()) + 1)) + return hits + + +def scan(files: list[str]) -> list[tuple[str, str, int]]: + found = [] + for rel in files: + p = ROOT / rel + if not looks_textual(p): + continue + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for rule_id, label, line in mask_scan(text): + found.append((rel, f"{rule_id}（{label}）", line)) + return found + + +# ── 反向自检：合成密钥必须命中，良性对照必须干净 ── +def self_test() -> int: + problems: list[str] = [] + # 值都在运行期拼出来：否则本文件自己就会成为一条真命中 + synthetic = { + "aws-ak": "AKIA" + "Q7ZP2M4XR9TLB6VW", + "aliyun-ak": "LTAI" + "5tG8kQ2mZx7Vb3Np", + "forge-token": "ghp_" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8", + "jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0." + "s1gn4tur3Pad1234567890ab", + "pg-url": "postgres://kg_admin:" + "hunter2xyz" + "@db.internal:5432/kg", + "assigned-secret": 'DINGTALK_APP_SECRET = "' + "9f2a7c1e4b8d3065" + '"', + # 拼起来写：本机的 pre-commit 钩子会拦"整段私钥头"的字面量 —— 那是**对的**， + # 所以不绕过它，而是让本文件里不出现那段连续文本（扫描逻辑本身不受影响）。 + "private-key": "-----BEGIN " + "RSA PRIVATE KEY-----", + } + benign = "\n".join([ + 'token = os.environ["CI_BRIDGE_TOKEN"]', + 'api_key: ${DOBOTAI_API_KEY}', + 'password = ""', + 'X-CI-Token: ${BRIDGE_TOKEN}', + 'const url = `${API_BASE}/healthz`', + 'PGURL=postgres://user@host/db', + 'self.assertEqual(result.Skipped, 1)', + # 下面三条都是**真实误报过**的形状（scripts/*.sh 与 Go 夹具），锁进对照防回归： + 'print(f"postgres://{u.quote(user)}:{u.quote(password)}@{host}:{port}/{name}")', + '"postgres://governance:***@postgres/governance"', + 'DINGTALK_APP_SECRET = "${DINGTALK_APP_SECRET}"', + ]) + with tempfile.TemporaryDirectory() as d: + for rule_id, value in synthetic.items(): + probe = Path(d) / "probe.txt" + probe.write_text(f"setting {rule_id} here\n{value}\n", encoding="utf-8") + hit_ids = {r for r, _, _ in mask_scan(probe.read_text(encoding="utf-8"))} + if not hit_ids: + problems.append(f"合成夹具没有任何规则命中（{rule_id}）—— 这条规则是死的") + elif rule_id not in hit_ids: + problems.append(f"合成夹具命中的是 {sorted(hit_ids)}，而不是 {rule_id} —— 规则串了") + clean = mask_scan(benign) + if clean: + problems.append(f"良性对照被误报：{clean} —— 门禁会因为正常代码变红（假红）") + if problems: + print("❌ 密钥扫描器自检失败：", file=sys.stderr) + for p in problems: + print(f" · {p}", file=sys.stderr) + return 1 + print(f"✅ 密钥扫描器自检通过：{len(synthetic)} 条合成密钥全部命中，良性对照零误报") + return 0 + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(add_help=True) + ap.add_argument("--self-test", action="store_true", help="只跑反向自检") + ap.add_argument("--no-self-test", action="store_true", help="跳过自检（调试用，不算全绿）") + args = ap.parse_args(argv) + + if args.self_test: + return self_test() + if not args.no_self_test and self_test() != 0: + return 1 + + files = tracked_files() + found = scan(files) + print(f"── 密钥扫描 ── 范围：{len(files)} 个被跟踪文件（只打印位置与类别，不打印值）") + if found: + print("❌ 疑似密钥泄漏：", file=sys.stderr) + for rel, label, line in sorted(found): + print(f" {rel}:{line} {label}", file=sys.stderr) + print(" 处理：把值挪到 Gitea Secrets / 目标机文件；已入库的要轮换密钥，" + "光删提交不够（历史里还在）。", file=sys.stderr) + return 1 + print("✅ 未发现密钥泄漏") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) \ No newline at end of file diff --git a/scripts/ci_checks.py b/scripts/ci_checks.py new file mode 100644 index 0000000..e8002c2 --- /dev/null +++ b/scripts/ci_checks.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +"""本仓**唯一权威门禁入口**（正本 §3.1 / 试点草案 §4）。 + + make verify # = python3 scripts/ci_checks.py + python3 scripts/ci_checks.py --list + python3 scripts/ci_checks.py --only tests-web # 单跑一项（**不算全绿**） + python3 scripts/ci_checks.py --self-test # 只跑汇总逻辑的反向自检 + +一条命令跑完全部必需检查，**串行、失败即停**，并产出结构化报告 `ci-report.json`。 +CI 跑的就是这一条（`.gitea/workflows/build.yml` 里 `make verify`）—— 本地与 CI 同一份逻辑， +不做第二套步骤（正本 §3.1：两边各写一套，迟早只改一边）。 + +三条纪律写进代码里： + 1. **失败即停，但汇总照做**：第一个必需项失败后，剩下的记为 `not_run`， + 报告如实写"未运行"而不是留空 —— 留空会被下游当成"没有这一项"。 + 2. **被跳过的必需项不算过**：`aggregate()` 只认 `success`； + 不适用要**在清单里声明**（`required_in`），不能在运行时悄悄 skip。 + 3. **工具缺失 / 依赖没按锁装 / 输出解析不了 ⇒ 失败**，不当成"检查通过"。 +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +ROOT = Path(__file__).resolve().parent.parent +WEB = ROOT / "web" +REPORT_DEFAULT = ROOT / "ci-report.json" +POLICY_REVISION = "ci-checks/1" # 判据变更时改这里，报告里跟着变 +MANIFESTS = ["go.mod", "go.sum", "web/package.json", "web/package-lock.json"] + +SUCCESS, FAILURE, SKIPPED, NOT_RUN = "success", "failure", "skipped", "not_run" + + +# ── 工具解析：找不到就报错，不回落成"-- 跳过"（正本 §4.4）────────────── +def tool(name: str, *extra: str) -> str: + for cand in (shutil.which(name), *extra): + if cand and Path(cand).exists(): + return cand + raise FileNotFoundError(name) + + +def go_bin() -> str: + return tool("go", "/opt/citools/go/bin/go") + + +def gofmt_bin() -> str: + return tool("gofmt", "/opt/citools/go/bin/gofmt") + + +def npm_bin() -> str: + return tool("npm") + + +def py(*args: str) -> list[str]: + return [sys.executable or "python3", *args] + + +def go_env() -> dict: + env = dict(os.environ) + env["GOFLAGS"] = "-mod=readonly" # 禁止隐式改写 go.mod / go.sum + env.setdefault("GOPROXY", "https://goproxy.cn,direct") + env.setdefault("GOTOOLCHAIN", "local") + # 缓存位置是**环境相关**的（CI 用主机挂载的 /opt/citools-cache），而缓存只加速、 + # 不作为通过证据，所以这里"可用就用、不可用就用 Go 自己的默认"，不写死。 + cache = Path("/opt/citools-cache") + if cache.is_dir() and os.access(cache, os.W_OK): + for key, sub in (("GOCACHE", "gocache"), ("GOMODCACHE", "gomod"), ("GOPATH", "gopath")): + path = cache / sub + try: + path.mkdir(parents=True, exist_ok=True) + except OSError: + continue + env.setdefault(key, str(path)) + return env + + +# ── 判定函数：拿到 (退出码, stdout, stderr) → (状态, 说明) ────────────── +def plain(exit_code: int, out: str, err: str) -> tuple[str, str]: + if exit_code == 0: + return SUCCESS, "退出码 0" + tail = (err or out).strip().splitlines()[-12:] + return FAILURE, f"退出码 {exit_code}｜" + " ／ ".join(tail) + + +def judge_go_tests(exit_code: int, out: str, err: str) -> tuple[str, str]: + """`go test -json` 的盘点：必须**真的执行过**用例，且把跳过的原因说清楚。 + + 为什么不用退出码了事：退出码 0 只说明"没有失败"，一个用例都没跑的包也是 0。 + 被跳过的断言等于没有断言（正本 §5.5）。""" + passed = failed = skipped = 0 + reasons: list[str] = [] + pending = "" + pkgs: set[str] = set() + for line in out.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("Package"): + pkgs.add(ev["Package"]) + action, test = ev.get("Action"), ev.get("Test") + if test and action in ("pass", "fail", "skip"): + passed += action == "pass" + failed += action == "fail" + skipped += action == "skip" + chunk = ev.get("Output") or "" + m = re.match(r"\s*\S+_test\.go:\d+:\s*(.+)$", chunk) + if m: + pending = m.group(1).strip() + if "--- SKIP:" in chunk and pending and pending not in reasons: + reasons.append(pending) + detail = f"执行包 {len(pkgs)}｜用例 通过 {passed} / 失败 {failed} / 跳过 {skipped}" + if skipped: + detail += f"｜跳过的原因（前 3 条）：{reasons[:3] or '未取到'}" + if exit_code != 0 or failed: + tail = (err or out).strip().splitlines() + return FAILURE, detail + (f"｜{tail[-1][:200]}" if tail else "") + if passed == 0: + return FAILURE, detail + "｜一个用例都没真的执行 —— 无测试不能算全绿" + return SUCCESS, detail + + +def judge_vitest_file(path: Path) -> tuple[str, str]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + return FAILURE, f"前端测试报告读不到/解析不了（{e}）—— 未完成不算过" + total = int(data.get("numTotalTests") or 0) + failed = int(data.get("numFailedTests") or 0) + suites = int(data.get("numTotalTestSuites") or 0) + passed = int(data.get("numPassedTests") or 0) + detail = f"套件 {suites}｜用例 通过 {passed} / 失败 {failed} / 共 {total}" + if failed: + return FAILURE, detail + if total == 0 or suites == 0: + return FAILURE, detail + "｜一个用例都没执行（空套件不能算通过）" + return SUCCESS, detail + + +# ── 检查清单（唯一一份定义；Makefile / CI / 文档都指它）───────────────── +@dataclass +class Check: + id: str + title: str + argv: list[str] + cwd: Path = ROOT + env: dict = field(default_factory=dict) + judge: Callable[[int, str, str], tuple[str, str]] = plain + required_in: tuple[str, ...] = ("local", "ci") + note: str = "" + + +def checks() -> list[Check]: + g, gf, npm, py3 = go_bin(), gofmt_bin(), npm_bin(), sys.executable or "python3" + env = go_env() + out_dir = Path(os.environ.get("CI_BUILD_OUT") or (ROOT / "bin")) + return [ + Check("identity", "检出身份（实际 HEAD == 事件完整 SHA）", + [], required_in=("ci",), + note="本地没有事件负载 ⇒ 记录为 skipped 并在这里说明；CI 上它是必需项"), + Check("commits", "提交信息规范（本次推送的范围）", + [py3, ".gitea/scripts/ci_gate.py"]), + Check("deps-install", "依赖按锁文件安装（缺锁即失败，不退化）", + [py3, "scripts/ci_checks.py", "--internal-deps", str(out_dir)], + env=env), + Check("format", "格式（Go gofmt + 前端 prettier）", [py3, "scripts/check_format.py"], env=env), + Check("static-go", "Go 静态检查（go vet）", [g, "vet", "./..."], env=env), + Check("static-web", "前端类型检查（tsc）", [npm, "--prefix", "web", "run", "typecheck"], env=env), + Check("build-go", "Go 构建（带版本注入）", + [py3, "scripts/ci_checks.py", "--internal-build", str(out_dir)], env=env), + Check("build-web", "前端构建（vite build）", [npm, "--prefix", "web", "run", "build"], env=env), + Check("tests-go", "Go 测试（禁用测试缓存，强制真跑）", + [g, "test", "-count=1", "-json", "./..."], env=env, judge=judge_go_tests), + Check("tests-web", "前端测试（vitest，必须真跑）", + [py3, "scripts/ci_checks.py", "--internal-web-test"], env=env), + Check("contracts", "版本与变更记录契约（不许 SKIP 后算过）", + [py3, "scripts/ci_checks.py", "--internal-contracts", g], env=env), + Check("security-secrets", "密钥泄漏扫描（含扫描器自检）", + [py3, "scripts/check_secrets.py"]), + Check("security-deps", "依赖漏洞（govulncheck + npm audit，分级 + 例外）", + [py3, "scripts/check_deps_vuln.py"], env=env), + Check("manifests-locked", "依赖清单在这次门禁里没有被改写", + [py3, "scripts/ci_checks.py", "--internal-manifests"]), + ] + + +# ── 内部子命令（把"一条复杂命令"收进脚本，YAML/清单里只留一行）───────── +def cmd_internal_deps(out_dir: str) -> int: + """按锁文件安装依赖。缺锁**不**退化成 `npm install`（那会静默改写清单）。""" + lock = WEB / "package-lock.json" + if not lock.is_file(): + print("❌ web/package-lock.json 不存在：不接受无约束安装，请先把锁文件入库", file=sys.stderr) + return 1 + if subprocess.run([npm_bin(), "ci", "--prefix", "web"], cwd=ROOT, env=go_env()).returncode: + print("❌ npm ci 失败：锁文件与 package.json 不一致（先跑 npm install 并提交锁文件）", file=sys.stderr) + return 1 + p = subprocess.run([go_bin(), "mod", "download"], cwd=ROOT, env=go_env(), + capture_output=True, text=True) + if p.returncode: + print(f"❌ go mod download 失败：{(p.stderr or p.stdout)[:400]}", file=sys.stderr) + return 1 + print("✅ 依赖按锁文件安装完成（npm ci + go mod download）") + return 0 + + +def cmd_internal_build(out_dir: str) -> int: + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + version = subprocess.run(py("scripts/version.py"), cwd=ROOT, capture_output=True, + text=True).stdout.strip() or "dev" + sha = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=ROOT, + capture_output=True, text=True).stdout.strip() or "unknown" + ts = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + ldflags = " ".join([ + f"-X knowledge-governance/internal/version.Number={version}", + f"-X knowledge-governance/internal/version.CommitSHA={sha}", + f"-X knowledge-governance/internal/version.BuildTime={ts}", + ]) + p = subprocess.run([go_bin(), "build", "-trimpath", "-ldflags", ldflags, "-o", f"{out}/", "./cmd/..."], + cwd=ROOT, env=go_env(), capture_output=True, text=True) + print(f"-- go build（版本 {version} / 提交 {sha}）--") + print(p.stdout[-2000:] if p.stdout else "", end="") + if p.returncode: + print(f"❌ 构建失败：{(p.stderr or '')[:2000]}", file=sys.stderr) + return 1 + print(f"✅ 构建完成：{out}") + return 0 + + +def cmd_internal_web_test() -> int: + """前端测试必须真跑（空套件不算通过），所以拿 vitest 的 JSON 报告看数字。""" + with tempfile.TemporaryDirectory() as d: + rep = Path(d) / "vitest.json" + p = subprocess.run([npm_bin(), "--prefix", "web", "run", "test", "--", + "--reporter=json", f"--outputFile={rep}"], + cwd=ROOT, env=go_env(), capture_output=True, text=True) + print(p.stdout[-3000:], end="") + if p.stderr.strip(): + print(p.stderr[-2000:], end="", file=sys.stderr) + status, detail = judge_vitest_file(rep) + print(("✅ " if status == SUCCESS else "❌ ") + detail, file=sys.stdout if status == SUCCESS else sys.stderr) + return 0 if status == SUCCESS and p.returncode == 0 else 1 + + +def cmd_internal_contracts(go: str) -> int: + """版本契约：注入进二进制的号必须等于 CHANGELOG 顶部，而且**必须真的 PASS**。""" + rc = subprocess.run(py("scripts/check_changelog.py"), cwd=ROOT).returncode + if rc: + return 1 + v = subprocess.run(py("scripts/version.py"), cwd=ROOT, capture_output=True, text=True) + version = v.stdout.strip() + if v.returncode or not version: + print("❌ CHANGELOG 顶部解析不到版本号：没有条目就没有版本号，不许回落成假版本", file=sys.stderr) + return 1 + print(f"-- 版本契约：CHANGELOG 顶部 = {version} --") + p = subprocess.run([go, "test", f"-ldflags=-X knowledge-governance/internal/version.Number={version}", + "-count=1", "-run", "TestNumberMatchesChangelog", "-v", "-json", + "./internal/version/..."], + cwd=ROOT, env=go_env(), capture_output=True, text=True) + text = p.stdout + p.stderr + print(text[-3000:], end="") + # ⚠️ 被跳过的断言等于没有断言：必须看到 PASS，SKIP 一律判红。 + # 用 in 而不是 `| grep -q`：pipefail 下 grep -q 会因 SIGPIPE 谎报失败（trap-7）。 + if '--- PASS: TestNumberMatchesChangelog' not in text.replace("\\n", "\n"): + print("❌ 版本契约没有真正 PASS（被 SKIP 或没跑到）—— 缺 -ldflags，或 CHANGELOG 解析失败", file=sys.stderr) + return 1 + if "--- SKIP: TestNumberMatchesChangelog" in text: + print("❌ 版本契约被跳过（未注入版本号）—— 跳过不算通过", file=sys.stderr) + return 1 + print("✅ 版本契约成立：注入 == CHANGELOG 顶部，且断言真的执行了") + return 0 + + +def cmd_internal_manifests() -> int: + """比对门禁开始前后依赖清单的摘要 —— 抓'某一步顺手改了 go.mod/锁文件'（静默漂移）。""" + baseline_path = Path(os.environ.get("CI_MANIFEST_BASELINE", str(ROOT / ".ci-manifests.json"))) + if not baseline_path.is_file(): + print("❌ 找不到门禁开始时的清单摘要（说明汇总流程被改坏了）—— 判红", file=sys.stderr) + return 1 + before = json.loads(baseline_path.read_text(encoding="utf-8")) + after = {f: digest(ROOT / f) for f in MANIFESTS} + drift = [f for f in MANIFESTS if before.get(f) != after.get(f)] + if drift: + print(f"❌ 这些依赖清单在门禁过程中被改写了：{drift}", file=sys.stderr) + print(" （GOFLAGS=-mod=readonly / npm ci 就是为了防这个；改写必须由人显式提交）", file=sys.stderr) + return 1 + print(f"✅ 依赖清单未被改写（{len(MANIFESTS)} 个文件摘要一致）") + return 0 + + +def digest(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else "missing" + + +# ── 汇总：纯函数，--self-test 直接打它 ──────────────────────────────────── +def aggregate(results: list[dict], profile: str) -> tuple[str, list[str]]: + problems = [] + for r in results: + declared = set(r["required_in"]) + if profile not in declared: + if r["status"] in (FAILURE, "unknown"): + problems.append(f"{r['id']} 在 {profile} 档案下不要求，但它状态是 {r['status']} —— 不静默放过") + continue + if r["status"] != SUCCESS: + # 摘要只留前 200 字符：完整原因在报告 JSON 的 reason 字段里 + why = f"｜{r['reason'][:200]}" if r.get("reason") else "" + problems.append(f"{r['id']} 状态 {r['status']}{why}") + return (SUCCESS if not problems else FAILURE), problems + + +def self_test() -> int: + def R(cid, status, required=("local", "ci")): + return {"id": cid, "status": status, "required_in": list(required), "reason": ""} + + cases = [ + ("全绿", [R("a", SUCCESS), R("b", SUCCESS)], "local", 0), + ("一项失败", [R("a", SUCCESS), R("b", FAILURE)], "local", 1), + ("一项未运行（失败即停的后果）", [R("a", FAILURE), R("b", NOT_RUN)], "local", 2), + ("必需项被跳过", [R("a", SKIPPED)], "local", 1), + ("某项在本地档案下不要求且被跳过 ⇒ 不算问题", [R("identity", SKIPPED, ("ci",))], "local", 0), + ("但它在 ci 档案下就是问题", [R("identity", SKIPPED, ("ci",))], "ci", 1), + ("ci 档案下本地不要求的项 unknown 也要拦", [R("identity", "unknown", ("ci",))], "ci", 1), + ] + problems = [] + for name, results, profile, want in cases: + _, got = aggregate(results, profile) + if len(got) != want: + problems.append(f"{name}：期望 {want} 个问题，实际 {len(got)} 个 {got}") + if problems: + print("❌ 汇总逻辑自检失败：", file=sys.stderr) + for p in problems: + print(f" · {p}", file=sys.stderr) + return 1 + print(f"✅ 汇总逻辑自检通过：{len(cases)} 个分支（含'必需项未运行/被跳过不得报全绿'）") + return 0 + + +# ── 身份检查（只在 CI 上有意义，所以单独一段）────────────────────────── +def identity_result() -> tuple[str, str]: + want = (os.environ.get("GITHUB_SHA") or "").strip() + if not want: + return SKIPPED, "本地运行没有事件负载（GITHUB_SHA 未设置）；CI 上这一项是必需项" + got = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + capture_output=True, text=True).stdout.strip() + if got != want: + return FAILURE, f"实际 HEAD {got[:12]} != 事件 SHA {want[:12]}（分支在推送后又前进了？）" + return SUCCESS, f"检出 {got[:12]} == 事件完整 SHA" + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(add_help=True) + ap.add_argument("--list", action="store_true") + ap.add_argument("--only", default="") + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--report", default=str(REPORT_DEFAULT)) + ap.add_argument("--profile", default="ci" if os.environ.get("GITHUB_ACTIONS") else "local", + choices=["local", "ci"]) + ap.add_argument("--internal-deps", default="") + ap.add_argument("--internal-build", default="") + ap.add_argument("--internal-web-test", action="store_true") + ap.add_argument("--internal-contracts", default="") + ap.add_argument("--internal-manifests", action="store_true") + args, _ = ap.parse_known_args(argv) + + # 内部子命令：把"一条复杂命令"收进本文件，清单与 YAML 里只留一行。 + # 先处理它们，避免被下面的门禁流程当成一次普通运行。 + if args.internal_deps: + return cmd_internal_deps(args.internal_deps) + if args.internal_build: + return cmd_internal_build(args.internal_build) + if args.internal_contracts: + return cmd_internal_contracts(args.internal_contracts) + if args.internal_web_test: + return cmd_internal_web_test() + if args.internal_manifests: + return cmd_internal_manifests() + if args.self_test: + return self_test() + + table = checks() + if args.list: + print(f"── 本仓必需检查清单（策略版本 {POLICY_REVISION}）──") + for c in table: + print(f" {c.id:<18} {'/'.join(c.required_in):<12} {c.title}") + if c.note: + print(f" {'':<18} {'':<12} ⚠️ {c.note}") + return 0 + + if args.only: + want = [x for x in args.only.split(",") if x] + unknown = [x for x in want if x not in {c.id for c in table}] + if unknown: + print(f"❌ 没有这些检查 id：{unknown}", file=sys.stderr) + return 2 + table = [c for c in table if c.id in want] + print(f"⚠️ --only {args.only}：只跑了子集，**不算全绿**（报告里 overall_status 会写 not_run_only）") + + # 门禁开始前记录依赖清单摘要：末尾断言"没有任何一步改写了它们" + baseline = ROOT / ".ci-manifests.json" + baseline.write_text(json.dumps({f: digest(ROOT / f) for f in MANIFESTS}), encoding="utf-8") + os.environ["CI_MANIFEST_BASELINE"] = str(baseline) + + results: list[dict] = [] + failed_early = False + started = time.time() + for c in table: + if failed_early: + results.append({"id": c.id, "title": c.title, "required_in": list(c.required_in), + "status": NOT_RUN, "reason": "前一项失败即停，未运行", "command": "", + "exit_code": None, "duration_s": 0}) + continue + if c.id == "identity": + status, reason = identity_result() + results.append({"id": c.id, "title": c.title, "required_in": list(c.required_in), + "status": status, "reason": reason, "command": "git rev-parse HEAD", + "exit_code": 0 if status == SUCCESS else 1, "duration_s": 0}) + failed_early = status == FAILURE + print(f"\n── [{c.id}] {c.title} ──\n {'✅' if status == SUCCESS else ('⏭️' if status == SKIPPED else '❌')} {reason}") + continue + + print(f"\n── [{c.id}] {c.title} ──") + print(" $ " + " ".join(c.argv)) + t0 = time.time() + try: + p = subprocess.run(c.argv, cwd=c.cwd, env={**os.environ, **c.env}, + capture_output=True, text=True) + exit_code, out, err = p.returncode, p.stdout, p.stderr + except FileNotFoundError as e: + exit_code, out, err = 127, "", f"工具缺失：{e}（工具缺失 ⇒ 失败，不当成检查通过）" + dur = round(time.time() - t0, 2) + status, reason = c.judge(exit_code, out, err) + if out.strip(): + print("\n".join(" | " + l for l in out.strip().splitlines()[-40:])) + if err.strip() and status != SUCCESS: + print("\n".join(" ! " + l for l in err.strip().splitlines()[-20:]), file=sys.stderr) + print(f" {'✅' if status == SUCCESS else '❌'} {reason}（{dur}s）") + results.append({"id": c.id, "title": c.title, "required_in": list(c.required_in), + "status": status, "reason": reason, "command": " ".join(c.argv), + "exit_code": exit_code, "duration_s": dur}) + failed_early = status != SUCCESS + + overall, problems = aggregate(results, args.profile) + if args.only: + overall = "not_run_only" + report = { + "schema": "ci-report/1", + "policy_revision": POLICY_REVISION, + "repository": os.environ.get("GITHUB_REPOSITORY", ""), + "commit_sha": subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "event_sha": os.environ.get("GITHUB_SHA", ""), + "branch": os.environ.get("GITHUB_REF_NAME", ""), + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "event": os.environ.get("GITHUB_EVENT_NAME", "local"), + "profile": args.profile, + "started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(timespec="seconds"), + "duration_s": round(time.time() - started, 2), + "checks": results, + "overall_status": overall, + "problems": problems, + "notification_status": "not_attempted", + "deployed": False, + } + Path(args.report).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + baseline.unlink(missing_ok=True) + + print("\n════════ 汇总 ════════") + n = len(results) + print(f" 检查 {n} 项：" + "｜".join(f"{r['id']}={r['status']}" for r in results)) + print(f" 总体：{overall}（档案 {args.profile}） 报告：{args.report}") + if problems: + for p in problems: + print(f" ❌ {p}", file=sys.stderr) + print(" ⚠️ 注意：'未运行'不是'通过' —— 失败即停之后剩下的项一律不参与'全绿'判定。", file=sys.stderr) + return 1 + if args.only: + return 0 if all(r["status"] == SUCCESS for r in results) else 1 + print(" ✅ 全部必需检查通过（本结论含全部必需项，不含任何未运行项）") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) \ No newline at end of file diff --git a/security/dependency-exceptions.json b/security/dependency-exceptions.json new file mode 100644 index 0000000..445d7f3 --- /dev/null +++ b/security/dependency-exceptions.json @@ -0,0 +1,18 @@ +{ + "_doc": [ + "依赖漏洞**例外**登记表。判据与分级在 scripts/check_deps_vuln.py，CI 会跑它。", + "只有 high/critical（以及 govulncheck 的任何命中）才需要例外；moderate 及以下只记录。", + "一条例外要**全部**字段齐全才算有效，缺一个就当成没有例外：", + " tool : govulncheck | npm-audit", + " id : 漏洞编号（govulncheck 用 GO-xxxx-yyyy，npm 用 GHSA-xxxx-xxxx-xxxx）", + " package : 包名（给 npm 用；govulncheck 填模块名）", + " severity : critical | high（govulncheck 的命中填空字符串）", + " scope : prod | dev | build", + " reason : 为什么这个风险在本仓不适用 / 不可修", + " owner : 谁负责在到期前处理", + " approved_by : 谁批准的（必须是**人**；写 TBD/pending/self 一律判为无效）", + " expires_on : YYYY-MM-DD，过期当天即失效 —— 过期不是\"继续有效\"，是\"重新走一遍审批\"", + "纪律：例外是记账，不是免检。到期没人处理 ⇒ 门禁红 ⇒ 必须有人做决定。" + ], + "exceptions": [] +} \ No newline at end of file