feat(artifact): 钉钉取件优先走 dws(DingTalk Workspace CLI),老路径保留为回退

- artifact_fetch.py: 新增 dingtalk dws 取件器(adoc→doc export、axls→sheet 转换、其余→drive 直下),
  ARTIFACT_DINGTALK_FETCHER=dws|auto|legacy 三级开关,auto 仅对可重试错误回落老路径;
  平台侧判定(pick_suffix/SLIDES/magic/大小上限/sha256/spool/空壳检查)全部保留;
  结果 JSON 字段不变(Go runner 用 DisallowUnknownFields 解析 stdout),取件器来源只写 stderr。
- axls_to_xlsx.py: 纯标准库 axls→xlsx(文本写入 sharedStrings 以通过空壳判定)。
- dws_error_map.py: dws 错误→平台 {ok,code,retryable,missing};未分类错误默认不重试。
- Dockerfile: alpine 增加 gcompat/libc6-compat;dws v1.0.63 固定版本+sha256 校验;构建期 dws version 自检。
- compose.yaml: worker 增加 dws 环境与凭据挂载(DWS_HOME 方案,XDG_DATA_HOME 不被 dws 识别)。
- 测试: artifact_fetch_test 新增 11 项(路由/回落语义/契约字段锁定);dws_error_map 17 断言。
- 实测: 真容器(--read-only --tmpfs --user 1000:1000) 端到端 adoc/axls/pdf 3/3 成功,docx sha 与平台基准一致。
This commit is contained in:
姚章浩
2026-10-09 10:57:11 +08:00
parent 8bb5cd9a62
commit f4e1ecc5f1
8 changed files with 872 additions and 6 deletions
+192
View File
@@ -1,5 +1,8 @@
import importlib.util
import json
import os
import pathlib
import sys
import unittest
from unittest import mock
@@ -66,5 +69,194 @@ class DingTalkOnlineDocumentExportTest(unittest.TestCase):
download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000)
class DingTalkDwsFetcherTest(unittest.TestCase):
"""dws 取件器:路由、回落语义、以及"结果 JSON 形状不得变化"的契约锁定。"""
def setUp(self):
self.manifest = {
"spec_id": "spec-1", "connection_id": "conn-1", "node_id": "sn-test",
"source_type": "dingtalk", "source_name": "公司级技术支持知识库",
"name": "说明.adoc", "relative_path": "公司级技术支持知识库/说明.adoc",
"declared_type": "adoc", "size_bytes": 1000, "modified_at": None, "locator_hash": "loc",
}
self.ok_verdict = {"ok": True, "code": "", "retryable": False, "missing": False, "error": ""}
def _fail(self, code, retryable, missing=False):
return {"ok": False, "code": code, "retryable": retryable, "missing": missing, "error": code}
def test_dws_adoc_export_is_preferred_and_reported(self):
def fake_run(args, timeout=300, cwd=None):
self.assertEqual(args[:2], ["doc", "export"])
self.assertIn("--export-format", args)
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"x" * 100)
return {"success": True}, self.ok_verdict
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-1", "adoc")), \
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
self.assertEqual(fetcher, "dws")
self.assertFalse(matched)
self.assertEqual(size, 104)
self.assertEqual(digest, artifact_fetch.sha256_file(path))
def test_axls_is_routed_through_the_converter(self):
built = {}
def build(sheets, out):
built["sheets"] = sheets
pathlib.Path(out).write_bytes(b"PK\x03\x04" + b"y" * 60)
return {"sheets": ["S1"]}
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-2", "axls")), \
mock.patch.object(artifact_fetch.axls_to_xlsx, "fetch_sheets", return_value=[{"name": "S1", "cells": []}]), \
mock.patch.object(artifact_fetch.axls_to_xlsx, "build_xlsx", side_effect=build), \
mock.patch.object(artifact_fetch, "dws_run") as dws_run, \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(
{**self.manifest, "name": "汇总.axls", "declared_type": "axls"}, 1_000_000)
self.assertEqual(fetcher, "dws")
self.assertTrue(path.endswith("out.xlsx"))
self.assertFalse(dws_run.called) # 转换器内部自己调 dws,取件器不再直接调
self.assertEqual(built["sheets"], [{"name": "S1", "cells": []}])
def test_auto_falls_back_to_legacy_on_retryable_failure(self):
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "auto"), \
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
side_effect=artifact_fetch.FetchError("dws missing", retryable=True, code="dws_missing")), \
mock.patch.object(artifact_fetch, "dingtalk_local_export", return_value=("/tmp/legacy.docx", True)), \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False), \
mock.patch.object(artifact_fetch, "sha256_file", return_value="b" * 64), \
mock.patch.object(artifact_fetch.os.path, "getsize", return_value=42):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
self.assertEqual((path, size, matched, fetcher), ("/tmp/legacy.docx", 42, True, "legacy"))
def test_auto_does_not_fall_back_on_terminal_failure(self):
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "auto"), \
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
side_effect=artifact_fetch.FetchError("nope", retryable=False, code="forbidden")), \
mock.patch.object(artifact_fetch, "dingtalk_local_export") as legacy:
with self.assertRaises(artifact_fetch.FetchError) as raised:
artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
self.assertEqual(raised.exception.code, "forbidden")
self.assertFalse(legacy.called) # 终态错误不得回落(否则掩盖真因)
def test_dws_mode_never_falls_back(self):
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
side_effect=artifact_fetch.FetchError("dws missing", retryable=True, code="dws_missing")), \
mock.patch.object(artifact_fetch, "dingtalk_local_export") as legacy:
with self.assertRaises(artifact_fetch.FetchError) as raised:
artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
self.assertEqual(raised.exception.code, "dws_missing")
self.assertFalse(legacy.called)
def test_legacy_mode_skips_dws_entirely(self):
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "legacy"), \
mock.patch.object(artifact_fetch, "dingtalk_dws_download") as dws_download, \
mock.patch.object(artifact_fetch, "dingtalk_local_export", return_value=("/tmp/legacy.docx", True)), \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False), \
mock.patch.object(artifact_fetch, "sha256_file", return_value="c" * 64), \
mock.patch.object(artifact_fetch.os.path, "getsize", return_value=7):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
self.assertEqual(fetcher, "legacy")
self.assertFalse(dws_download.called)
def test_catalog_size_over_limit_never_starts_a_download(self):
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys") as raw_keys, \
mock.patch.object(artifact_fetch, "dws_run") as dws_run:
with self.assertRaises(artifact_fetch.FetchError) as raised:
artifact_fetch.dingtalk_download({**self.manifest, "size_bytes": 500_000_000}, 1_000_000)
self.assertEqual(raised.exception.code, "over_limit")
self.assertFalse(raw_keys.called)
self.assertFalse(dws_run.called)
def test_link_and_mind_nodes_map_to_missing_and_unsupported(self):
cases = (("dlink", "链接节点.dlink", "missing", True),
("amind", "培训大纲.amind", "unsupported_format", False))
for declared, name, expected, expected_missing in cases:
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-3", "file")), \
mock.patch.object(artifact_fetch, "dws_run") as dws_run:
with self.assertRaises((artifact_fetch.FetchError, artifact_fetch.MissingSource)) as raised:
artifact_fetch.dingtalk_download(
{**self.manifest, "name": name, "declared_type": declared}, 1_000_000)
if expected_missing:
self.assertIsInstance(raised.exception, artifact_fetch.MissingSource)
else:
self.assertEqual(raised.exception.code, expected)
self.assertFalse(dws_run.called)
def test_routing_uses_name_suffix_without_declared_type(self):
# 生产 manifest 里没有 declared_type 字段;必须靠文件名后缀路由(与老路径一致)
def fake_run(args, timeout=300, cwd=None):
self.assertEqual(args[:2], ["doc", "export"])
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"q" * 20)
return {"success": True}, self.ok_verdict
manifest = {k: v for k, v in self.manifest.items() if k != "declared_type"}
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-9", "file")), \
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(manifest, 1_000_000)
self.assertEqual(fetcher, "dws")
self.assertTrue(path.endswith("out.docx"))
def test_routing_falls_back_to_dws_inspect_when_name_has_no_suffix(self):
calls = []
def fake_run(args, timeout=300, cwd=None):
calls.append(args)
if args[:2] == ["drive", "+inspect"]:
return {"ok": True, "data": {"data": {"file": {"extension": "adoc", "name": "无后缀节点"}}}}, self.ok_verdict
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"w" * 30)
return {"success": True}, self.ok_verdict
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-10", "file")), \
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(
{k: v for k, v in self.manifest.items() if k != "declared_type"} | {"name": "无后缀节点"}, 1_000_000)
self.assertEqual(calls[0][:2], ["drive", "+inspect"])
self.assertEqual(calls[1][:2], ["doc", "export"])
self.assertEqual(fetcher, "dws")
def test_result_json_shape_is_unchanged(self):
# 平台 Go runner 用 DisallowUnknownFields 解析 stdout:多一个字段就会让所有取件失败。
captured = []
temp = pathlib.Path("/tmp/kg-artifact-fetch-shape.docx")
temp.write_bytes(b"PK\x03\x04" + b"z" * 200)
try:
with mock.patch.object(artifact_fetch, "emit", side_effect=captured.append), \
mock.patch.object(artifact_fetch, "dingtalk_download",
return_value=(str(temp), temp.stat().st_size,
artifact_fetch.sha256_file(str(temp)), False, "dws")), \
mock.patch.object(sys, "argv", ["artifact_fetch.py", json.dumps(self.manifest)]), \
mock.patch.dict(os.environ, {"INGESTION_SPOOL_ROOT": "/tmp/kg-shape-spool",
"ARTIFACT_FETCH_MAX_BYTES": "100000"}):
artifact_fetch.main()
finally:
temp.unlink(missing_ok=True)
self.assertEqual(len(captured), 1)
self.assertEqual(set(captured[0].keys()), {
"ok", "spec_id", "node_id", "connection_id", "sha256", "size_bytes",
"spool_ref", "validated_format", "matched_local"})
if __name__ == "__main__":
unittest.main()