feat(artifact): 钉钉取件优先走 dws(DingTalk Workspace CLI),老路径保留为回退
- artifact_fetch.py: 新增 dingtalk dws 取件器(adoc→doc export、axls→sheet 转换、其余→drive 直下),
ARTIFACT_DINGTALK_FETCHER=dws|auto|legacy 三级开关,auto 仅对可重试错误回落老路径;
平台侧判定(pick_suffix/SLIDES/magic/大小上限/sha256/spool/空壳检查)全部保留;
结果 JSON 字段不变(Go runner 用 DisallowUnknownFields 解析 stdout),取件器来源只写 stderr。
- axls_to_xlsx.py: 纯标准库 axls→xlsx(文本写入 sharedStrings 以通过空壳判定)。
- dws_error_map.py: dws 错误→平台 {ok,code,retryable,missing};未分类错误默认不重试。
- Dockerfile: alpine 增加 gcompat/libc6-compat;dws v1.0.63 固定版本+sha256 校验;构建期 dws version 自检。
- compose.yaml: worker 增加 dws 环境与凭据挂载(DWS_HOME 方案,XDG_DATA_HOME 不被 dws 识别)。
- 测试: artifact_fetch_test 新增 11 项(路由/回落语义/契约字段锁定);dws_error_map 17 断言。
- 实测: 真容器(--read-only --tmpfs --user 1000:1000) 端到端 adoc/axls/pdf 3/3 成功,docx sha 与平台基准一致。
This commit is contained in:
@@ -1,5 +1,8 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
@@ -66,5 +69,194 @@ class DingTalkOnlineDocumentExportTest(unittest.TestCase):
|
||||
download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000)
|
||||
|
||||
|
||||
class DingTalkDwsFetcherTest(unittest.TestCase):
|
||||
"""dws 取件器:路由、回落语义、以及"结果 JSON 形状不得变化"的契约锁定。"""
|
||||
|
||||
def setUp(self):
|
||||
self.manifest = {
|
||||
"spec_id": "spec-1", "connection_id": "conn-1", "node_id": "sn-test",
|
||||
"source_type": "dingtalk", "source_name": "公司级技术支持知识库",
|
||||
"name": "说明.adoc", "relative_path": "公司级技术支持知识库/说明.adoc",
|
||||
"declared_type": "adoc", "size_bytes": 1000, "modified_at": None, "locator_hash": "loc",
|
||||
}
|
||||
self.ok_verdict = {"ok": True, "code": "", "retryable": False, "missing": False, "error": ""}
|
||||
|
||||
def _fail(self, code, retryable, missing=False):
|
||||
return {"ok": False, "code": code, "retryable": retryable, "missing": missing, "error": code}
|
||||
|
||||
def test_dws_adoc_export_is_preferred_and_reported(self):
|
||||
def fake_run(args, timeout=300, cwd=None):
|
||||
self.assertEqual(args[:2], ["doc", "export"])
|
||||
self.assertIn("--export-format", args)
|
||||
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"x" * 100)
|
||||
return {"success": True}, self.ok_verdict
|
||||
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-1", "adoc")), \
|
||||
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
|
||||
|
||||
self.assertEqual(fetcher, "dws")
|
||||
self.assertFalse(matched)
|
||||
self.assertEqual(size, 104)
|
||||
self.assertEqual(digest, artifact_fetch.sha256_file(path))
|
||||
|
||||
def test_axls_is_routed_through_the_converter(self):
|
||||
built = {}
|
||||
|
||||
def build(sheets, out):
|
||||
built["sheets"] = sheets
|
||||
pathlib.Path(out).write_bytes(b"PK\x03\x04" + b"y" * 60)
|
||||
return {"sheets": ["S1"]}
|
||||
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-2", "axls")), \
|
||||
mock.patch.object(artifact_fetch.axls_to_xlsx, "fetch_sheets", return_value=[{"name": "S1", "cells": []}]), \
|
||||
mock.patch.object(artifact_fetch.axls_to_xlsx, "build_xlsx", side_effect=build), \
|
||||
mock.patch.object(artifact_fetch, "dws_run") as dws_run, \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(
|
||||
{**self.manifest, "name": "汇总.axls", "declared_type": "axls"}, 1_000_000)
|
||||
|
||||
self.assertEqual(fetcher, "dws")
|
||||
self.assertTrue(path.endswith("out.xlsx"))
|
||||
self.assertFalse(dws_run.called) # 转换器内部自己调 dws,取件器不再直接调
|
||||
self.assertEqual(built["sheets"], [{"name": "S1", "cells": []}])
|
||||
|
||||
def test_auto_falls_back_to_legacy_on_retryable_failure(self):
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "auto"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
|
||||
side_effect=artifact_fetch.FetchError("dws missing", retryable=True, code="dws_missing")), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_local_export", return_value=("/tmp/legacy.docx", True)), \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False), \
|
||||
mock.patch.object(artifact_fetch, "sha256_file", return_value="b" * 64), \
|
||||
mock.patch.object(artifact_fetch.os.path, "getsize", return_value=42):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
|
||||
|
||||
self.assertEqual((path, size, matched, fetcher), ("/tmp/legacy.docx", 42, True, "legacy"))
|
||||
|
||||
def test_auto_does_not_fall_back_on_terminal_failure(self):
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "auto"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
|
||||
side_effect=artifact_fetch.FetchError("nope", retryable=False, code="forbidden")), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_local_export") as legacy:
|
||||
with self.assertRaises(artifact_fetch.FetchError) as raised:
|
||||
artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
|
||||
|
||||
self.assertEqual(raised.exception.code, "forbidden")
|
||||
self.assertFalse(legacy.called) # 终态错误不得回落(否则掩盖真因)
|
||||
|
||||
def test_dws_mode_never_falls_back(self):
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_dws_download",
|
||||
side_effect=artifact_fetch.FetchError("dws missing", retryable=True, code="dws_missing")), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_local_export") as legacy:
|
||||
with self.assertRaises(artifact_fetch.FetchError) as raised:
|
||||
artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
|
||||
|
||||
self.assertEqual(raised.exception.code, "dws_missing")
|
||||
self.assertFalse(legacy.called)
|
||||
|
||||
def test_legacy_mode_skips_dws_entirely(self):
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "legacy"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_dws_download") as dws_download, \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_local_export", return_value=("/tmp/legacy.docx", True)), \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False), \
|
||||
mock.patch.object(artifact_fetch, "sha256_file", return_value="c" * 64), \
|
||||
mock.patch.object(artifact_fetch.os.path, "getsize", return_value=7):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(self.manifest, 1_000_000)
|
||||
|
||||
self.assertEqual(fetcher, "legacy")
|
||||
self.assertFalse(dws_download.called)
|
||||
|
||||
def test_catalog_size_over_limit_never_starts_a_download(self):
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys") as raw_keys, \
|
||||
mock.patch.object(artifact_fetch, "dws_run") as dws_run:
|
||||
with self.assertRaises(artifact_fetch.FetchError) as raised:
|
||||
artifact_fetch.dingtalk_download({**self.manifest, "size_bytes": 500_000_000}, 1_000_000)
|
||||
|
||||
self.assertEqual(raised.exception.code, "over_limit")
|
||||
self.assertFalse(raw_keys.called)
|
||||
self.assertFalse(dws_run.called)
|
||||
|
||||
def test_link_and_mind_nodes_map_to_missing_and_unsupported(self):
|
||||
cases = (("dlink", "链接节点.dlink", "missing", True),
|
||||
("amind", "培训大纲.amind", "unsupported_format", False))
|
||||
for declared, name, expected, expected_missing in cases:
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-3", "file")), \
|
||||
mock.patch.object(artifact_fetch, "dws_run") as dws_run:
|
||||
with self.assertRaises((artifact_fetch.FetchError, artifact_fetch.MissingSource)) as raised:
|
||||
artifact_fetch.dingtalk_download(
|
||||
{**self.manifest, "name": name, "declared_type": declared}, 1_000_000)
|
||||
if expected_missing:
|
||||
self.assertIsInstance(raised.exception, artifact_fetch.MissingSource)
|
||||
else:
|
||||
self.assertEqual(raised.exception.code, expected)
|
||||
self.assertFalse(dws_run.called)
|
||||
|
||||
def test_routing_uses_name_suffix_without_declared_type(self):
|
||||
# 生产 manifest 里没有 declared_type 字段;必须靠文件名后缀路由(与老路径一致)
|
||||
def fake_run(args, timeout=300, cwd=None):
|
||||
self.assertEqual(args[:2], ["doc", "export"])
|
||||
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"q" * 20)
|
||||
return {"success": True}, self.ok_verdict
|
||||
|
||||
manifest = {k: v for k, v in self.manifest.items() if k != "declared_type"}
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-9", "file")), \
|
||||
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(manifest, 1_000_000)
|
||||
|
||||
self.assertEqual(fetcher, "dws")
|
||||
self.assertTrue(path.endswith("out.docx"))
|
||||
|
||||
def test_routing_falls_back_to_dws_inspect_when_name_has_no_suffix(self):
|
||||
calls = []
|
||||
|
||||
def fake_run(args, timeout=300, cwd=None):
|
||||
calls.append(args)
|
||||
if args[:2] == ["drive", "+inspect"]:
|
||||
return {"ok": True, "data": {"data": {"file": {"extension": "adoc", "name": "无后缀节点"}}}}, self.ok_verdict
|
||||
pathlib.Path(cwd, "out.docx").write_bytes(b"PK\x03\x04" + b"w" * 30)
|
||||
return {"success": True}, self.ok_verdict
|
||||
|
||||
with mock.patch.object(artifact_fetch, "DINGTALK_FETCHER", "dws"), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_raw_keys", return_value=("raw-10", "file")), \
|
||||
mock.patch.object(artifact_fetch, "dws_run", side_effect=fake_run), \
|
||||
mock.patch.object(artifact_fetch, "exported_doc_is_blank", return_value=False):
|
||||
path, size, digest, matched, fetcher = artifact_fetch.dingtalk_download(
|
||||
{k: v for k, v in self.manifest.items() if k != "declared_type"} | {"name": "无后缀节点"}, 1_000_000)
|
||||
|
||||
self.assertEqual(calls[0][:2], ["drive", "+inspect"])
|
||||
self.assertEqual(calls[1][:2], ["doc", "export"])
|
||||
self.assertEqual(fetcher, "dws")
|
||||
|
||||
def test_result_json_shape_is_unchanged(self):
|
||||
# 平台 Go runner 用 DisallowUnknownFields 解析 stdout:多一个字段就会让所有取件失败。
|
||||
captured = []
|
||||
temp = pathlib.Path("/tmp/kg-artifact-fetch-shape.docx")
|
||||
temp.write_bytes(b"PK\x03\x04" + b"z" * 200)
|
||||
try:
|
||||
with mock.patch.object(artifact_fetch, "emit", side_effect=captured.append), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_download",
|
||||
return_value=(str(temp), temp.stat().st_size,
|
||||
artifact_fetch.sha256_file(str(temp)), False, "dws")), \
|
||||
mock.patch.object(sys, "argv", ["artifact_fetch.py", json.dumps(self.manifest)]), \
|
||||
mock.patch.dict(os.environ, {"INGESTION_SPOOL_ROOT": "/tmp/kg-shape-spool",
|
||||
"ARTIFACT_FETCH_MAX_BYTES": "100000"}):
|
||||
artifact_fetch.main()
|
||||
finally:
|
||||
temp.unlink(missing_ok=True)
|
||||
|
||||
self.assertEqual(len(captured), 1)
|
||||
self.assertEqual(set(captured[0].keys()), {
|
||||
"ok", "spec_id", "node_id", "connection_id", "sha256", "size_bytes",
|
||||
"spool_ref", "validated_format", "matched_local"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user