feat(ingest): 源与目录文件名过滤及钉钉文档导出
- filename_filter_rules table (000035) with source-root and per-directory rows - additive inheritance: source default + every ancestor directory contributes - filename_is_filtered() SQL predicate applied to catalog browsing, subtree counts, change-set diffing, routing fan-out and artifact fetch manifests - fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch jobs, retires planned publications and blocks items when rules change - GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized extensions) + Settings and Catalog filter editors - failJobSQL no longer swallows the claimed row when the failure projection upserts (fixes spurious 'lease lost' retries) - DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403 from file/download fixed), header-signature downloads supported
This commit is contained in:
+96
-16
@@ -392,32 +392,112 @@ def dingtalk_download(manifest, max_bytes):
|
||||
return live_path
|
||||
|
||||
|
||||
def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None):
|
||||
request_headers = {
|
||||
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
|
||||
"cookie": headers.get("cookie", ""), "content-type": "application/json",
|
||||
"accept": "application/json", "accept-language": "zh-CN",
|
||||
"origin": "https://alidocs.dingtalk.com", "referer": headers["referer"],
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
}
|
||||
request_headers.update(extra_headers or {})
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
|
||||
request = urllib.request.Request(url, headers=request_headers, data=body, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as error:
|
||||
raise FetchError("DingTalk web API returned HTTP %d" % error.code,
|
||||
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
|
||||
except (urllib.error.URLError, TimeoutError) as error:
|
||||
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
|
||||
|
||||
|
||||
def dingtalk_put_bytes(url, payload):
|
||||
request = urllib.request.Request(url, data=payload, method="PUT", headers={
|
||||
"content-length": str(len(payload)), "content-type": "",
|
||||
"referer": "https://alidocs.dingtalk.com/", "user-agent": "Mozilla/5.0",
|
||||
})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=90) as response:
|
||||
if response.status not in (200, 201, 204):
|
||||
raise FetchError("DingTalk export upload returned HTTP %d" % response.status,
|
||||
retryable=response.status >= 500, code="export_upload")
|
||||
except urllib.error.HTTPError as error:
|
||||
raise FetchError("DingTalk export upload returned HTTP %d" % error.code,
|
||||
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
|
||||
|
||||
|
||||
def dingtalk_export_adoc(info, headers, manifest, max_bytes, spreadsheet=False):
|
||||
doc_key = str(info.get("docKey") or "")
|
||||
dentry_key = str(info.get("dentryKey") or "")
|
||||
if not doc_key:
|
||||
raise MissingSource("DingTalk online document has no docKey")
|
||||
source_app = "spreadsheet" if spreadsheet else "doc"
|
||||
doc_type = "sheet" if spreadsheet else "doc"
|
||||
referer = ("https://alidocs.dingtalk.com/note/edit?dt_editor_toolbar=true&biz_ver=10"
|
||||
"&docType=%s&docId=%s&dontjump=true" % (doc_type, urllib.parse.quote(doc_key)))
|
||||
extra = {"referer": referer, "source_doc_app": source_app,
|
||||
"a-doc-key": doc_key, "a-dentry-key": dentry_key}
|
||||
document = dingtalk_json_api("https://alidocs.dingtalk.com/api/document/data", headers,
|
||||
{"docKey": doc_key}, extra_headers=extra, method="POST")
|
||||
checkpoint = (((document.get("data") or {}).get("documentContent") or {}).get("checkpoint") or {})
|
||||
content = checkpoint.get("content") or ""
|
||||
if not isinstance(content, str) or not content:
|
||||
raise FetchError("DingTalk online document content is empty", retryable=False, code="empty_document")
|
||||
content_bytes = content.encode("utf-8")
|
||||
if len(content_bytes) > max_bytes:
|
||||
raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
upload = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/resources/9/upload_info", headers,
|
||||
{"size": len(content_bytes), "resourceName": manifest["name"], "contentType": ""},
|
||||
extra_headers={**extra, "a-host-doc-key": ""}, method="POST")
|
||||
upload_info = upload.get("data") or {}
|
||||
storage_path, upload_url = upload_info.get("storagePath"), upload_info.get("uploadUrl")
|
||||
if not storage_path or not upload_url:
|
||||
raise FetchError("DingTalk export upload slot is missing", retryable=True, code="export_slot")
|
||||
dingtalk_put_bytes(upload_url, content_bytes)
|
||||
submit_body = {"storagePath": storage_path,
|
||||
"exportType": "dingTalksheetToxlsx" if spreadsheet else "dingTalkdocTodocx"}
|
||||
if spreadsheet:
|
||||
submit_body["extra"] = {"imgCnt": 0}
|
||||
submitted = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/submitExportJob", headers,
|
||||
submit_body, extra_headers=extra, method="POST")
|
||||
job_id = str((submitted.get("data") or {}).get("jobId") or "")
|
||||
if not job_id:
|
||||
raise FetchError("DingTalk export job was not created", retryable=True, code="export_submit")
|
||||
deadline = time.time() + 120
|
||||
while time.time() < deadline:
|
||||
status = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/queryExportJobInfo?jobId=" + urllib.parse.quote(job_id),
|
||||
headers, extra_headers=extra)
|
||||
detail = status.get("data") or {}
|
||||
if detail.get("status") == "success" and detail.get("ossUrl"):
|
||||
return http_get(detail["ossUrl"], {}, 180, max_bytes) + (False,)
|
||||
if detail.get("status") in ("fail", "failed", "error"):
|
||||
raise FetchError("DingTalk export job failed", retryable=False, code="export_failed")
|
||||
time.sleep(3)
|
||||
raise FetchError("DingTalk export job timed out", retryable=True, code="export_timeout")
|
||||
|
||||
|
||||
def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes):
|
||||
opener = urllib.request.build_opener()
|
||||
|
||||
def api(url, payload=None):
|
||||
request = urllib.request.Request(url, headers={
|
||||
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
|
||||
"cookie": headers.get("cookie", ""), "content-type": "application/json",
|
||||
"referer": headers["referer"], "user-agent": "Mozilla/5.0",
|
||||
}, data=json.dumps(payload).encode() if payload is not None else None)
|
||||
try:
|
||||
with opener.open(request, timeout=60) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
|
||||
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
|
||||
|
||||
info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key))
|
||||
info = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key), headers)
|
||||
data = info.get("data") or {}
|
||||
extension = str(data.get("extension") or pathlib.PurePosixPath(manifest["name"]).suffix.lstrip(".")).lower()
|
||||
if extension in ("adoc", "axls"):
|
||||
return dingtalk_export_adoc(data, headers, manifest, max_bytes, spreadsheet=extension == "axls")
|
||||
space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "")
|
||||
if not space_id or not dentry_id:
|
||||
raise MissingSource("DingTalk dentry not resolvable")
|
||||
download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id))
|
||||
download = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id), headers)
|
||||
payload = download.get("data") or {}
|
||||
if payload.get("downloadType") == "URL_PRE_SIGNATURE":
|
||||
urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or []
|
||||
if urls:
|
||||
return http_get(urls[0], {}, 180, max_bytes) + (False,)
|
||||
if payload.get("downloadType") == "HEADER_SIGNATURE":
|
||||
signature = payload.get("headerSignatureInfo") or {}
|
||||
urls = signature.get("resourceUrl") or []
|
||||
if urls:
|
||||
return http_get(urls[0], signature.get("headers") or {}, 180, max_bytes) + (False,)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import importlib.util
|
||||
import pathlib
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
MODULE_PATH = pathlib.Path(__file__).with_name("artifact_fetch.py")
|
||||
spec = importlib.util.spec_from_file_location("artifact_fetch", MODULE_PATH)
|
||||
assert spec is not None and spec.loader is not None
|
||||
artifact_fetch = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(artifact_fetch)
|
||||
|
||||
|
||||
class DingTalkOnlineDocumentExportTest(unittest.TestCase):
|
||||
def test_adoc_uses_export_job_instead_of_ordinary_file_download(self):
|
||||
calls = []
|
||||
|
||||
def api(url, headers, payload=None, extra_headers=None, method=None):
|
||||
calls.append((url, payload))
|
||||
if "/dentry/info" in url:
|
||||
return {"data": {"docKey": "doc-key", "dentryKey": "dentry-key", "extension": "adoc"}}
|
||||
if "/api/document/data" in url:
|
||||
return {"data": {"documentContent": {"checkpoint": {"content": "document-json"}}}}
|
||||
if "/upload_info" in url:
|
||||
return {"data": {"storagePath": "tmp/path", "uploadUrl": "https://oss.example/upload"}}
|
||||
if "/submitExportJob" in url:
|
||||
return {"data": {"jobId": "job-1"}}
|
||||
if "/queryExportJobInfo" in url:
|
||||
return {"data": {"status": "success", "ossUrl": "https://oss.example/result.docx"}}
|
||||
self.fail("unexpected API " + url)
|
||||
|
||||
with mock.patch.object(artifact_fetch, "dingtalk_json_api", side_effect=api), \
|
||||
mock.patch.object(artifact_fetch, "dingtalk_put_bytes") as put, \
|
||||
mock.patch.object(artifact_fetch, "http_get", return_value=("/tmp/result", 123, "a" * 64)) as download:
|
||||
result = artifact_fetch.live_export_dingtalk_document(
|
||||
"raw-node", {"a-token": "x", "x-xsrf-token": "y", "cookie": "z", "referer": "https://alidocs.dingtalk.com/"},
|
||||
{"name": "说明.adoc", "relative_path": "说明.adoc"}, 1_000_000,
|
||||
)
|
||||
|
||||
self.assertEqual(result, ("/tmp/result", 123, "a" * 64, False))
|
||||
self.assertTrue(any("submitExportJob" in url for url, _ in calls))
|
||||
self.assertFalse(any("/file/download" in url for url, _ in calls))
|
||||
put.assert_called_once()
|
||||
download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user