feat(artifact): 钉钉取件优先走 dws(DingTalk Workspace CLI),老路径保留为回退

- artifact_fetch.py: 新增 dingtalk dws 取件器(adoc→doc export、axls→sheet 转换、其余→drive 直下),
  ARTIFACT_DINGTALK_FETCHER=dws|auto|legacy 三级开关,auto 仅对可重试错误回落老路径;
  平台侧判定(pick_suffix/SLIDES/magic/大小上限/sha256/spool/空壳检查)全部保留;
  结果 JSON 字段不变(Go runner 用 DisallowUnknownFields 解析 stdout),取件器来源只写 stderr。
- axls_to_xlsx.py: 纯标准库 axls→xlsx(文本写入 sharedStrings 以通过空壳判定)。
- dws_error_map.py: dws 错误→平台 {ok,code,retryable,missing};未分类错误默认不重试。
- Dockerfile: alpine 增加 gcompat/libc6-compat;dws v1.0.63 固定版本+sha256 校验;构建期 dws version 自检。
- compose.yaml: worker 增加 dws 环境与凭据挂载(DWS_HOME 方案,XDG_DATA_HOME 不被 dws 识别)。
- 测试: artifact_fetch_test 新增 11 项(路由/回落语义/契约字段锁定);dws_error_map 17 断言。
- 实测: 真容器(--read-only --tmpfs --user 1000:1000) 端到端 adoc/axls/pdf 3/3 成功,docx sha 与平台基准一致。
This commit is contained in:
姚章浩
2026-10-09 10:57:11 +08:00
parent 8bb5cd9a62
commit f4e1ecc5f1
8 changed files with 872 additions and 6 deletions
+152 -4
View File
@@ -34,6 +34,24 @@ import urllib.parse
import urllib.request
# Sibling modules shipped into /app/scripts next to this file.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import axls_to_xlsx # noqa: E402
import dws_error_map # noqa: E402
DWS_BIN = os.environ.get("DWS_BIN") or "/usr/local/bin/dws"
# dws 在文件型 keychain 模式下把凭据放在 $HOME/.local/share/dws-cli(XDG_DATA_HOME 不被识别),
# 所以给 dws 子进程单独指定一个 HOME,避免动整个 worker 的 HOME。
DWS_HOME = os.environ.get("DWS_HOME") or ""
# dws -> only the dws CLI; auto -> dws first, fall back to the legacy path on
# retryable failures; legacy -> only the captured-header web path.
DINGTALK_FETCHER = (os.environ.get("ARTIFACT_DINGTALK_FETCHER") or "auto").strip().lower()
DINGTALK_FETCHER_MODES = ("dws", "auto", "legacy")
if DINGTALK_FETCHER not in DINGTALK_FETCHER_MODES:
DINGTALK_FETCHER = "auto"
SUPPORTED = {
"pdf": "pdf", "docx": "docx", "xlsx": "xlsx", "md": "md", "txt": "txt",
"csv": "csv", "json": "json",
@@ -380,6 +398,118 @@ def _space_key(roots, name):
return ""
def _dws_binary():
if os.path.exists(DWS_BIN) and os.access(DWS_BIN, os.X_OK):
return DWS_BIN
return shutil.which("dws") or ""
def dws_run(args, timeout=300, cwd=None):
"""Run the dws CLI and translate its result into the platform's semantics."""
binary = _dws_binary()
if not binary:
# Infrastructure missing: auto mode must be able to fall back, so this is
# marked retryable rather than terminal.
return None, {"ok": False, "code": "dws_missing", "retryable": True, "missing": False,
"error": "dws binary not found (DWS_BIN=%s)" % DWS_BIN}
try:
env = dict(os.environ)
if DWS_HOME:
env["HOME"] = DWS_HOME
completed = subprocess.run([binary, *args], capture_output=True, text=True,
timeout=timeout, cwd=cwd, env=env)
except subprocess.TimeoutExpired:
return None, {"ok": False, "code": "export_timeout", "retryable": True, "missing": False,
"error": "dws call timed out: %s" % " ".join(args[:3])}
payload = None
if completed.stdout.strip():
try:
payload = json.loads(completed.stdout)
except ValueError:
payload = None
verdict = dws_error_map.classify(payload, stderr=completed.stderr, exit_code=completed.returncode)
return payload, verdict
def _dws_error(verdict):
if verdict.get("missing"):
return MissingSource(verdict.get("error") or "DingTalk node missing on the source side")
return FetchError(verdict.get("error") or "dws fetch failed",
retryable=bool(verdict.get("retryable")),
code=verdict.get("code") or "unknown")
def _dws_inspect_extension(raw_key):
"""Last-resort extension lookup (only when the manifest carries no usable suffix)."""
payload, verdict = dws_run(["drive", "+inspect", "--node", raw_key], timeout=60)
if not verdict["ok"] or not isinstance(payload, dict):
return ""
found = []
def walk(node, depth=0):
if depth > 5 or found:
return
if isinstance(node, dict):
for key, value in node.items():
if key == "extension" and isinstance(value, str) and value:
found.append(value.lower())
return
walk(value, depth + 1)
elif isinstance(node, list):
for value in node[:10]:
walk(value, depth + 1)
walk(payload.get("data") if "data" in payload else payload)
return found[0] if found else ""
def dingtalk_dws_download(manifest, max_bytes):
"""Fetch one DingTalk node through dws: adoc -> docx export, axls -> converted
xlsx, everything else -> raw drive download. Returns (path, size, digest, matched_local)."""
catalog_size = manifest.get("size_bytes")
if isinstance(catalog_size, int) and catalog_size > max_bytes:
# The catalog already knows the size: refuse before burning a download.
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
raw_key, _dentry_type = dingtalk_raw_keys(manifest)
# The node type comes from the RESOLVED name suffix first (that is what the legacy
# path keys on); declared_type is an optional hint and dingtalk_raw_keys returns a
# dentry type ("file"), never a file extension.
raw_suffix = pathlib.PurePosixPath(manifest["name"]).suffix.lower().lstrip(".")
extension = raw_suffix
if not extension or extension in ("file", "folder"):
declared = (manifest.get("declared_type") or "").lower()
extension = declared if declared not in ("", "file", "folder") else _dws_inspect_extension(raw_key)
extension = (extension or "").lower().lstrip(".")
if extension in ("dlink", "hlink"):
raise MissingSource("DingTalk link node has no bytes (fileSize=0)")
if extension in ("amind", "appt"):
raise FetchError("dws has no content-export entry for .%s nodes" % extension,
retryable=False, code="unsupported_format")
workdir = pathlib.Path(tempfile.mkdtemp(prefix="dws-fetch-"))
if extension == "adoc":
target = "out.docx"
_, verdict = dws_run(["doc", "export", "--node", raw_key, "--export-format", "docx",
"--output", target], cwd=str(workdir))
if not verdict["ok"]:
raise _dws_error(verdict)
elif extension == "axls":
target = "out.xlsx"
axls_to_xlsx.build_xlsx(axls_to_xlsx.fetch_sheets(raw_key), str(workdir / target))
else:
target = "out." + (extension or "bin")
_, verdict = dws_run(["drive", "+download", "--node", raw_key, "--output", target],
cwd=str(workdir), timeout=900)
if not verdict["ok"]:
raise _dws_error(verdict)
exported = workdir / target
if not exported.is_file() or exported.stat().st_size == 0:
raise FetchError("dws produced no usable bytes", retryable=True, code="dws_empty_output")
size = exported.stat().st_size
if size > max_bytes:
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
return str(exported), size, sha256_file(str(exported)), False
def dingtalk_download(manifest, max_bytes):
spreadsheet = pathlib.PurePosixPath(manifest["name"]).suffix.lower() == ".axls"
@@ -390,23 +520,38 @@ def dingtalk_download(manifest, max_bytes):
retryable=False, code="empty_document")
return local_path
# dws first (this is the configured preference); the captured-header path stays
# as the fallback for auto mode and as the only path in legacy mode.
if DINGTALK_FETCHER in ("dws", "auto"):
try:
path, size, digest, matched = dingtalk_dws_download(manifest, max_bytes)
return guarded(path), size, digest, matched, "dws"
except MissingSource:
raise
except FetchError as error:
if DINGTALK_FETCHER == "dws" or not error.retryable:
raise
# auto: only retryable failures fall back, terminal ones must surface as-is
if DINGTALK_FETCHER == "dws":
raise FetchError("dws fetcher requested but unavailable", retryable=False, code="dws_missing")
export = dingtalk_local_export(manifest)
modified = str(manifest.get("modified_at") or "")
if export:
path, fresh = export
size = os.path.getsize(path)
if size > 0 and (fresh or not modified):
return guarded(path), size, sha256_file(path), True
return guarded(path), size, sha256_file(path), True, "legacy"
# stale local export -> fall through to a live export when available
raw_key, declared_type = dingtalk_raw_keys(manifest)
headers = load_capture_headers()
live_path = live_export_dingtalk_document(raw_key, headers, manifest, max_bytes)
if export and not live_path:
path, fresh = export
return guarded(path), os.path.getsize(path), sha256_file(path), True # keep old bytes; staleness shown in UI
return guarded(path), os.path.getsize(path), sha256_file(path), True, "legacy" # keep old bytes; staleness shown in UI
if not live_path:
raise MissingSource("DingTalk file is not exported and cannot be exported automatically")
return live_path
return live_path[0], live_path[1], live_path[2], live_path[3], "legacy"
def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None):
@@ -602,7 +747,10 @@ def main():
elif manifest["source_type"] == "pp":
path, size, digest, matched_local = pp_download(manifest, max_bytes)
elif manifest["source_type"] == "dingtalk":
path, size, digest, matched_local = dingtalk_download(manifest, max_bytes)
path, size, digest, matched_local, fetcher = dingtalk_download(manifest, max_bytes)
# Only stderr: the Go runner decodes stdout with DisallowUnknownFields, so the
# result JSON shape must stay exactly as before.
print("[artifact_fetch] dingtalk fetcher=%s node=%s" % (fetcher, manifest.get("node_id")), file=sys.stderr)
else:
raise FetchError("unknown source type", retryable=False, code="bad_manifest")
# Type is decided on the RESOLVED bytes, not the catalog title: PP ledger