姚章浩
f4e1ecc5f1
- artifact_fetch.py: 新增 dingtalk dws 取件器（adoc→doc export、axls→sheet 转换、其余→drive 直下），
ARTIFACT_DINGTALK_FETCHER=dws|auto|legacy 三级开关，auto 仅对可重试错误回落老路径；
平台侧判定（pick_suffix/SLIDES/magic/大小上限/sha256/spool/空壳检查）全部保留；
结果 JSON 字段不变（Go runner 用 DisallowUnknownFields 解析 stdout），取件器来源只写 stderr。
- axls_to_xlsx.py: 纯标准库 axls→xlsx（文本写入 sharedStrings 以通过空壳判定）。
- dws_error_map.py: dws 错误→平台 {ok,code,retryable,missing}；未分类错误默认不重试。
- Dockerfile: alpine 增加 gcompat/libc6-compat；dws v1.0.63 固定版本+sha256 校验；构建期 dws version 自检。
- compose.yaml: worker 增加 dws 环境与凭据挂载（DWS_HOME 方案，XDG_DATA_HOME 不被 dws 识别）。
- 测试: artifact_fetch_test 新增 11 项（路由/回落语义/契约字段锁定）；dws_error_map 17 断言。
- 实测: 真容器(--read-only --tmpfs --user 1000:1000) 端到端 adoc/axls/pdf 3/3 成功，docx sha 与平台基准一致。
126 lines
6.1 KiB
Python
126 lines
6.1 KiB
Python
#!/usr/bin/env python3
|
||
"""dws 错误 → 平台错误码/retryable 映射（阶段 B-③ 交付物）。
|
||
|
||
平台契约（internal/workflows/artifactfetch/handler.go）需要：
|
||
{ok, missing, retryable, code, error}
|
||
本模块把 dws 的两种错误体（严格命令的 ok/error 包裹、普通子命令的 error 对象）
|
||
以及本地错误、TLS/网络文本，统一映射成平台语义。
|
||
|
||
用法：
|
||
from dws_error_map import classify
|
||
classify(dws_json, stderr_text) -> {"ok":bool,"code":str,"retryable":bool,"missing":bool,"error":str}
|
||
|
||
原则：
|
||
- 只有"明确瞬时"的信号才 retryable=True（429/5xx/超时/连接层失败）
|
||
- 未知错误默认 retryable=False + code=unknown —— 让它显式暴露，而不是被无限重试掩盖
|
||
- missing=True 表示源侧对象已不存在/被删（平台据此走 missing 分支而不是 failed）
|
||
"""
|
||
import json
|
||
import re
|
||
|
||
# (匹配正则, code, retryable, missing, 说明)
|
||
RULES = [
|
||
# ---- 凭据 ----
|
||
(r"请重新运行 dws auth login|resolve access token|invalidParameter\.authCode\.notFound|未登录|authenticated\"?\s*:\s*false|旧版登录态",
|
||
"credentials", False, False, "登录态失效/被顶掉，需重新设备流授权"),
|
||
# ---- TLS 信任库 ----
|
||
(r"x509: certificate signed by unknown authority|tls: failed to verify certificate",
|
||
"tls_ca", False, False, "容器缺少 CA 信任库（装 ca-certificates 或挂载 bundle/SSL_CERT_FILE）"),
|
||
# ---- 权限 ----
|
||
(r"forbidden\.accessDenied|Access Denied|accessdenied|HTTP 403|\"code\":\s*403",
|
||
"forbidden", False, False, "该账号对目标节点无权限"),
|
||
# ---- 限流 ----
|
||
(r"HTTP 429|\b429\b|rate.?limit|Too Many Requests|请求过于频繁",
|
||
"http_429", True, False, "被限流，退避后重试"),
|
||
# ---- 本地文件冲突 / 大小限制 ----
|
||
(r"LOCAL_FILE_EXISTS|目标文件已存在", "file_exists", False, False, "本地已有同名产物，换路径或用 drive download --overwrite"),
|
||
(r"LOCAL_DOWNLOAD_TOO_LARGE|超过上限 \d+ 字节|exceeds ingest size limit|超过入库大小限制",
|
||
"over_limit", False, False, "超过大小上限（严格版 512MB / 平台 200MB 配置）"),
|
||
# ---- 类型不支持 ----
|
||
(r"不是钉钉在线文档|不是钉钉表格|notSupported\.docType|NODE_TYPE_MISMATCH|不是 a standalone \.adraw|无法使用文档相关工具|file type .* is not supported|在线表格不支持直接下载|不支持直接下载|请使用 getRange",
|
||
"unsupported_format", False, False, "节点类型与该入口不匹配（走对应产品命令）"),
|
||
# ---- 源对象不存在 ----
|
||
(r"workspace node not found|not found in current inventory|节点不存在|dentry not resolvable|nodeId 指向的节点已",
|
||
"missing", False, True, "源侧节点已删除/移动"),
|
||
# ---- 空文档（与平台既有判定同义） ----
|
||
(r"content is empty|正文为空|content\s+is\s+empty", "empty_document", False, False, "文档正文为空，无需入库"),
|
||
# ---- 服务端导出失败/超时 ----
|
||
(r"导出任务失败|export job failed|submitExportJob", "export_failed", False, False, "钉钉导出任务失败"),
|
||
(r"导出任务超时|export job timed out|timed out|timeout|超时", "export_timeout", True, False, "导出/请求超时，可重试"),
|
||
# ---- 连接层 ----
|
||
(r"connection refused|no such host|network is unreachable|i/o timeout|EOF|connection reset|request_failed|502|503|504|HTTP 5\d\d",
|
||
"source_unavailable", True, False, "源端不可达，可重试"),
|
||
]
|
||
|
||
_JSON_ERR_KEYS = ("upstream_code", "server_error_code", "code", "category", "subtype", "message", "cause", "hint", "error")
|
||
|
||
|
||
def _flatten(payload):
|
||
"""把 dws 错误体压成一段可正则匹配的文本。"""
|
||
if payload is None:
|
||
return ""
|
||
if isinstance(payload, str):
|
||
return payload
|
||
parts = []
|
||
|
||
def walk(node, depth=0):
|
||
if depth > 6:
|
||
return
|
||
if isinstance(node, dict):
|
||
for k, v in node.items():
|
||
if k in _JSON_ERR_KEYS or isinstance(v, (dict, list)):
|
||
parts.append(str(k))
|
||
walk(v, depth + 1)
|
||
elif isinstance(node, list):
|
||
for v in node[:20]:
|
||
walk(v, depth + 1)
|
||
else:
|
||
parts.append(str(node))
|
||
|
||
walk(payload)
|
||
return " | ".join(parts)
|
||
|
||
|
||
def classify(dws_json=None, stderr=None, exit_code=None):
|
||
"""dws 结果 → 平台语义。dws_json 可为 dict / JSON 字符串 / None。"""
|
||
payload = dws_json
|
||
if isinstance(payload, str):
|
||
try:
|
||
payload = json.loads(payload)
|
||
except Exception:
|
||
payload = {"error": {"message": payload}}
|
||
text = _flatten(payload) + " || " + _flatten(stderr)
|
||
|
||
ok = bool(payload) and isinstance(payload, dict) and (
|
||
payload.get("ok") is True or payload.get("success") is True
|
||
or payload.get("outcome") == "success" or payload.get("status") == "SUCCESS")
|
||
if ok:
|
||
return {"ok": True, "code": "", "retryable": False, "missing": False, "error": ""}
|
||
|
||
# 先判 missing 关键词（missing 优先于其它分类）
|
||
for pattern, code, retryable, missing, why in RULES:
|
||
if re.search(pattern, text, re.I):
|
||
msg = _short_message(payload) or why
|
||
return {"ok": False, "code": code, "retryable": retryable, "missing": missing,
|
||
"error": f"{why}: {msg}"[:400]}
|
||
|
||
return {"ok": False, "code": "unknown", "retryable": False, "missing": False,
|
||
"error": (_short_message(payload) or str(stderr or "") or "dws 未分类错误")[:400]}
|
||
|
||
|
||
def _short_message(payload):
|
||
if isinstance(payload, dict):
|
||
err = payload.get("error") or payload
|
||
if isinstance(err, dict):
|
||
return str(err.get("message") or err.get("cause") or err.get("reason") or "")
|
||
return str(err)[:200]
|
||
return ""
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
raw = sys.stdin.read()
|
||
try:
|
||
print(json.dumps(classify(raw), ensure_ascii=False, indent=1))
|
||
except Exception as e:
|
||
print(json.dumps({"ok": False, "code": "unknown", "error": str(e)}, ensure_ascii=False)) |