feat(artifact): 钉钉取件优先走 dws(DingTalk Workspace CLI),老路径保留为回退
- artifact_fetch.py: 新增 dingtalk dws 取件器(adoc→doc export、axls→sheet 转换、其余→drive 直下),
ARTIFACT_DINGTALK_FETCHER=dws|auto|legacy 三级开关,auto 仅对可重试错误回落老路径;
平台侧判定(pick_suffix/SLIDES/magic/大小上限/sha256/spool/空壳检查)全部保留;
结果 JSON 字段不变(Go runner 用 DisallowUnknownFields 解析 stdout),取件器来源只写 stderr。
- axls_to_xlsx.py: 纯标准库 axls→xlsx(文本写入 sharedStrings 以通过空壳判定)。
- dws_error_map.py: dws 错误→平台 {ok,code,retryable,missing};未分类错误默认不重试。
- Dockerfile: alpine 增加 gcompat/libc6-compat;dws v1.0.63 固定版本+sha256 校验;构建期 dws version 自检。
- compose.yaml: worker 增加 dws 环境与凭据挂载(DWS_HOME 方案,XDG_DATA_HOME 不被 dws 识别)。
- 测试: artifact_fetch_test 新增 11 项(路由/回落语义/契约字段锁定);dws_error_map 17 断言。
- 实测: 真容器(--read-only --tmpfs --user 1000:1000) 端到端 adoc/axls/pdf 3/3 成功,docx sha 与平台基准一致。
This commit is contained in:
+152
-4
@@ -34,6 +34,24 @@ import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
# Sibling modules shipped into /app/scripts next to this file.
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import axls_to_xlsx # noqa: E402
|
||||
import dws_error_map # noqa: E402
|
||||
|
||||
|
||||
DWS_BIN = os.environ.get("DWS_BIN") or "/usr/local/bin/dws"
|
||||
# dws 在文件型 keychain 模式下把凭据放在 $HOME/.local/share/dws-cli(XDG_DATA_HOME 不被识别),
|
||||
# 所以给 dws 子进程单独指定一个 HOME,避免动整个 worker 的 HOME。
|
||||
DWS_HOME = os.environ.get("DWS_HOME") or ""
|
||||
# dws -> only the dws CLI; auto -> dws first, fall back to the legacy path on
|
||||
# retryable failures; legacy -> only the captured-header web path.
|
||||
DINGTALK_FETCHER = (os.environ.get("ARTIFACT_DINGTALK_FETCHER") or "auto").strip().lower()
|
||||
DINGTALK_FETCHER_MODES = ("dws", "auto", "legacy")
|
||||
if DINGTALK_FETCHER not in DINGTALK_FETCHER_MODES:
|
||||
DINGTALK_FETCHER = "auto"
|
||||
|
||||
|
||||
SUPPORTED = {
|
||||
"pdf": "pdf", "docx": "docx", "xlsx": "xlsx", "md": "md", "txt": "txt",
|
||||
"csv": "csv", "json": "json",
|
||||
@@ -380,6 +398,118 @@ def _space_key(roots, name):
|
||||
return ""
|
||||
|
||||
|
||||
def _dws_binary():
|
||||
if os.path.exists(DWS_BIN) and os.access(DWS_BIN, os.X_OK):
|
||||
return DWS_BIN
|
||||
return shutil.which("dws") or ""
|
||||
|
||||
|
||||
def dws_run(args, timeout=300, cwd=None):
|
||||
"""Run the dws CLI and translate its result into the platform's semantics."""
|
||||
binary = _dws_binary()
|
||||
if not binary:
|
||||
# Infrastructure missing: auto mode must be able to fall back, so this is
|
||||
# marked retryable rather than terminal.
|
||||
return None, {"ok": False, "code": "dws_missing", "retryable": True, "missing": False,
|
||||
"error": "dws binary not found (DWS_BIN=%s)" % DWS_BIN}
|
||||
try:
|
||||
env = dict(os.environ)
|
||||
if DWS_HOME:
|
||||
env["HOME"] = DWS_HOME
|
||||
completed = subprocess.run([binary, *args], capture_output=True, text=True,
|
||||
timeout=timeout, cwd=cwd, env=env)
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, {"ok": False, "code": "export_timeout", "retryable": True, "missing": False,
|
||||
"error": "dws call timed out: %s" % " ".join(args[:3])}
|
||||
payload = None
|
||||
if completed.stdout.strip():
|
||||
try:
|
||||
payload = json.loads(completed.stdout)
|
||||
except ValueError:
|
||||
payload = None
|
||||
verdict = dws_error_map.classify(payload, stderr=completed.stderr, exit_code=completed.returncode)
|
||||
return payload, verdict
|
||||
|
||||
|
||||
def _dws_error(verdict):
|
||||
if verdict.get("missing"):
|
||||
return MissingSource(verdict.get("error") or "DingTalk node missing on the source side")
|
||||
return FetchError(verdict.get("error") or "dws fetch failed",
|
||||
retryable=bool(verdict.get("retryable")),
|
||||
code=verdict.get("code") or "unknown")
|
||||
|
||||
|
||||
def _dws_inspect_extension(raw_key):
|
||||
"""Last-resort extension lookup (only when the manifest carries no usable suffix)."""
|
||||
payload, verdict = dws_run(["drive", "+inspect", "--node", raw_key], timeout=60)
|
||||
if not verdict["ok"] or not isinstance(payload, dict):
|
||||
return ""
|
||||
found = []
|
||||
|
||||
def walk(node, depth=0):
|
||||
if depth > 5 or found:
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key == "extension" and isinstance(value, str) and value:
|
||||
found.append(value.lower())
|
||||
return
|
||||
walk(value, depth + 1)
|
||||
elif isinstance(node, list):
|
||||
for value in node[:10]:
|
||||
walk(value, depth + 1)
|
||||
|
||||
walk(payload.get("data") if "data" in payload else payload)
|
||||
return found[0] if found else ""
|
||||
|
||||
|
||||
def dingtalk_dws_download(manifest, max_bytes):
|
||||
"""Fetch one DingTalk node through dws: adoc -> docx export, axls -> converted
|
||||
xlsx, everything else -> raw drive download. Returns (path, size, digest, matched_local)."""
|
||||
catalog_size = manifest.get("size_bytes")
|
||||
if isinstance(catalog_size, int) and catalog_size > max_bytes:
|
||||
# The catalog already knows the size: refuse before burning a download.
|
||||
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
raw_key, _dentry_type = dingtalk_raw_keys(manifest)
|
||||
# The node type comes from the RESOLVED name suffix first (that is what the legacy
|
||||
# path keys on); declared_type is an optional hint and dingtalk_raw_keys returns a
|
||||
# dentry type ("file"), never a file extension.
|
||||
raw_suffix = pathlib.PurePosixPath(manifest["name"]).suffix.lower().lstrip(".")
|
||||
extension = raw_suffix
|
||||
if not extension or extension in ("file", "folder"):
|
||||
declared = (manifest.get("declared_type") or "").lower()
|
||||
extension = declared if declared not in ("", "file", "folder") else _dws_inspect_extension(raw_key)
|
||||
extension = (extension or "").lower().lstrip(".")
|
||||
if extension in ("dlink", "hlink"):
|
||||
raise MissingSource("DingTalk link node has no bytes (fileSize=0)")
|
||||
if extension in ("amind", "appt"):
|
||||
raise FetchError("dws has no content-export entry for .%s nodes" % extension,
|
||||
retryable=False, code="unsupported_format")
|
||||
workdir = pathlib.Path(tempfile.mkdtemp(prefix="dws-fetch-"))
|
||||
if extension == "adoc":
|
||||
target = "out.docx"
|
||||
_, verdict = dws_run(["doc", "export", "--node", raw_key, "--export-format", "docx",
|
||||
"--output", target], cwd=str(workdir))
|
||||
if not verdict["ok"]:
|
||||
raise _dws_error(verdict)
|
||||
elif extension == "axls":
|
||||
target = "out.xlsx"
|
||||
axls_to_xlsx.build_xlsx(axls_to_xlsx.fetch_sheets(raw_key), str(workdir / target))
|
||||
else:
|
||||
target = "out." + (extension or "bin")
|
||||
_, verdict = dws_run(["drive", "+download", "--node", raw_key, "--output", target],
|
||||
cwd=str(workdir), timeout=900)
|
||||
if not verdict["ok"]:
|
||||
raise _dws_error(verdict)
|
||||
exported = workdir / target
|
||||
if not exported.is_file() or exported.stat().st_size == 0:
|
||||
raise FetchError("dws produced no usable bytes", retryable=True, code="dws_empty_output")
|
||||
size = exported.stat().st_size
|
||||
if size > max_bytes:
|
||||
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
return str(exported), size, sha256_file(str(exported)), False
|
||||
|
||||
|
||||
def dingtalk_download(manifest, max_bytes):
|
||||
spreadsheet = pathlib.PurePosixPath(manifest["name"]).suffix.lower() == ".axls"
|
||||
|
||||
@@ -390,23 +520,38 @@ def dingtalk_download(manifest, max_bytes):
|
||||
retryable=False, code="empty_document")
|
||||
return local_path
|
||||
|
||||
# dws first (this is the configured preference); the captured-header path stays
|
||||
# as the fallback for auto mode and as the only path in legacy mode.
|
||||
if DINGTALK_FETCHER in ("dws", "auto"):
|
||||
try:
|
||||
path, size, digest, matched = dingtalk_dws_download(manifest, max_bytes)
|
||||
return guarded(path), size, digest, matched, "dws"
|
||||
except MissingSource:
|
||||
raise
|
||||
except FetchError as error:
|
||||
if DINGTALK_FETCHER == "dws" or not error.retryable:
|
||||
raise
|
||||
# auto: only retryable failures fall back, terminal ones must surface as-is
|
||||
if DINGTALK_FETCHER == "dws":
|
||||
raise FetchError("dws fetcher requested but unavailable", retryable=False, code="dws_missing")
|
||||
|
||||
export = dingtalk_local_export(manifest)
|
||||
modified = str(manifest.get("modified_at") or "")
|
||||
if export:
|
||||
path, fresh = export
|
||||
size = os.path.getsize(path)
|
||||
if size > 0 and (fresh or not modified):
|
||||
return guarded(path), size, sha256_file(path), True
|
||||
return guarded(path), size, sha256_file(path), True, "legacy"
|
||||
# stale local export -> fall through to a live export when available
|
||||
raw_key, declared_type = dingtalk_raw_keys(manifest)
|
||||
headers = load_capture_headers()
|
||||
live_path = live_export_dingtalk_document(raw_key, headers, manifest, max_bytes)
|
||||
if export and not live_path:
|
||||
path, fresh = export
|
||||
return guarded(path), os.path.getsize(path), sha256_file(path), True # keep old bytes; staleness shown in UI
|
||||
return guarded(path), os.path.getsize(path), sha256_file(path), True, "legacy" # keep old bytes; staleness shown in UI
|
||||
if not live_path:
|
||||
raise MissingSource("DingTalk file is not exported and cannot be exported automatically")
|
||||
return live_path
|
||||
return live_path[0], live_path[1], live_path[2], live_path[3], "legacy"
|
||||
|
||||
|
||||
def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None):
|
||||
@@ -602,7 +747,10 @@ def main():
|
||||
elif manifest["source_type"] == "pp":
|
||||
path, size, digest, matched_local = pp_download(manifest, max_bytes)
|
||||
elif manifest["source_type"] == "dingtalk":
|
||||
path, size, digest, matched_local = dingtalk_download(manifest, max_bytes)
|
||||
path, size, digest, matched_local, fetcher = dingtalk_download(manifest, max_bytes)
|
||||
# Only stderr: the Go runner decodes stdout with DisallowUnknownFields, so the
|
||||
# result JSON shape must stay exactly as before.
|
||||
print("[artifact_fetch] dingtalk fetcher=%s node=%s" % (fetcher, manifest.get("node_id")), file=sys.stderr)
|
||||
else:
|
||||
raise FetchError("unknown source type", retryable=False, code="bad_manifest")
|
||||
# Type is decided on the RESOLVED bytes, not the catalog title: PP ledger
|
||||
|
||||
Reference in New Issue
Block a user