feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
+96 -16
View File
@@ -392,32 +392,112 @@ def dingtalk_download(manifest, max_bytes):
return live_path
def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None):
request_headers = {
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
"cookie": headers.get("cookie", ""), "content-type": "application/json",
"accept": "application/json", "accept-language": "zh-CN",
"origin": "https://alidocs.dingtalk.com", "referer": headers["referer"],
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}
request_headers.update(extra_headers or {})
body = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
request = urllib.request.Request(url, headers=request_headers, data=body, method=method)
try:
with urllib.request.urlopen(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as error:
raise FetchError("DingTalk web API returned HTTP %d" % error.code,
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
except (urllib.error.URLError, TimeoutError) as error:
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
def dingtalk_put_bytes(url, payload):
request = urllib.request.Request(url, data=payload, method="PUT", headers={
"content-length": str(len(payload)), "content-type": "",
"referer": "https://alidocs.dingtalk.com/", "user-agent": "Mozilla/5.0",
})
try:
with urllib.request.urlopen(request, timeout=90) as response:
if response.status not in (200, 201, 204):
raise FetchError("DingTalk export upload returned HTTP %d" % response.status,
retryable=response.status >= 500, code="export_upload")
except urllib.error.HTTPError as error:
raise FetchError("DingTalk export upload returned HTTP %d" % error.code,
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
def dingtalk_export_adoc(info, headers, manifest, max_bytes, spreadsheet=False):
doc_key = str(info.get("docKey") or "")
dentry_key = str(info.get("dentryKey") or "")
if not doc_key:
raise MissingSource("DingTalk online document has no docKey")
source_app = "spreadsheet" if spreadsheet else "doc"
doc_type = "sheet" if spreadsheet else "doc"
referer = ("https://alidocs.dingtalk.com/note/edit?dt_editor_toolbar=true&biz_ver=10"
"&docType=%s&docId=%s&dontjump=true" % (doc_type, urllib.parse.quote(doc_key)))
extra = {"referer": referer, "source_doc_app": source_app,
"a-doc-key": doc_key, "a-dentry-key": dentry_key}
document = dingtalk_json_api("https://alidocs.dingtalk.com/api/document/data", headers,
{"docKey": doc_key}, extra_headers=extra, method="POST")
checkpoint = (((document.get("data") or {}).get("documentContent") or {}).get("checkpoint") or {})
content = checkpoint.get("content") or ""
if not isinstance(content, str) or not content:
raise FetchError("DingTalk online document content is empty", retryable=False, code="empty_document")
content_bytes = content.encode("utf-8")
if len(content_bytes) > max_bytes:
raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit")
upload = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/resources/9/upload_info", headers,
{"size": len(content_bytes), "resourceName": manifest["name"], "contentType": ""},
extra_headers={**extra, "a-host-doc-key": ""}, method="POST")
upload_info = upload.get("data") or {}
storage_path, upload_url = upload_info.get("storagePath"), upload_info.get("uploadUrl")
if not storage_path or not upload_url:
raise FetchError("DingTalk export upload slot is missing", retryable=True, code="export_slot")
dingtalk_put_bytes(upload_url, content_bytes)
submit_body = {"storagePath": storage_path,
"exportType": "dingTalksheetToxlsx" if spreadsheet else "dingTalkdocTodocx"}
if spreadsheet:
submit_body["extra"] = {"imgCnt": 0}
submitted = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/submitExportJob", headers,
submit_body, extra_headers=extra, method="POST")
job_id = str((submitted.get("data") or {}).get("jobId") or "")
if not job_id:
raise FetchError("DingTalk export job was not created", retryable=True, code="export_submit")
deadline = time.time() + 120
while time.time() < deadline:
status = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/queryExportJobInfo?jobId=" + urllib.parse.quote(job_id),
headers, extra_headers=extra)
detail = status.get("data") or {}
if detail.get("status") == "success" and detail.get("ossUrl"):
return http_get(detail["ossUrl"], {}, 180, max_bytes) + (False,)
if detail.get("status") in ("fail", "failed", "error"):
raise FetchError("DingTalk export job failed", retryable=False, code="export_failed")
time.sleep(3)
raise FetchError("DingTalk export job timed out", retryable=True, code="export_timeout")
def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes):
opener = urllib.request.build_opener()
def api(url, payload=None):
request = urllib.request.Request(url, headers={
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
"cookie": headers.get("cookie", ""), "content-type": "application/json",
"referer": headers["referer"], "user-agent": "Mozilla/5.0",
}, data=json.dumps(payload).encode() if payload is not None else None)
try:
with opener.open(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key))
info = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key), headers)
data = info.get("data") or {}
extension = str(data.get("extension") or pathlib.PurePosixPath(manifest["name"]).suffix.lstrip(".")).lower()
if extension in ("adoc", "axls"):
return dingtalk_export_adoc(data, headers, manifest, max_bytes, spreadsheet=extension == "axls")
space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "")
if not space_id or not dentry_id:
raise MissingSource("DingTalk dentry not resolvable")
download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id))
download = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id), headers)
payload = download.get("data") or {}
if payload.get("downloadType") == "URL_PRE_SIGNATURE":
urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or []
if urls:
return http_get(urls[0], {}, 180, max_bytes) + (False,)
if payload.get("downloadType") == "HEADER_SIGNATURE":
signature = payload.get("headerSignatureInfo") or {}
urls = signature.get("resourceUrl") or []
if urls:
return http_get(urls[0], signature.get("headers") or {}, 180, max_bytes) + (False,)
return None
+47
View File
@@ -0,0 +1,47 @@
import importlib.util
import pathlib
import unittest
from unittest import mock
MODULE_PATH = pathlib.Path(__file__).with_name("artifact_fetch.py")
spec = importlib.util.spec_from_file_location("artifact_fetch", MODULE_PATH)
assert spec is not None and spec.loader is not None
artifact_fetch = importlib.util.module_from_spec(spec)
spec.loader.exec_module(artifact_fetch)
class DingTalkOnlineDocumentExportTest(unittest.TestCase):
def test_adoc_uses_export_job_instead_of_ordinary_file_download(self):
calls = []
def api(url, headers, payload=None, extra_headers=None, method=None):
calls.append((url, payload))
if "/dentry/info" in url:
return {"data": {"docKey": "doc-key", "dentryKey": "dentry-key", "extension": "adoc"}}
if "/api/document/data" in url:
return {"data": {"documentContent": {"checkpoint": {"content": "document-json"}}}}
if "/upload_info" in url:
return {"data": {"storagePath": "tmp/path", "uploadUrl": "https://oss.example/upload"}}
if "/submitExportJob" in url:
return {"data": {"jobId": "job-1"}}
if "/queryExportJobInfo" in url:
return {"data": {"status": "success", "ossUrl": "https://oss.example/result.docx"}}
self.fail("unexpected API " + url)
with mock.patch.object(artifact_fetch, "dingtalk_json_api", side_effect=api), \
mock.patch.object(artifact_fetch, "dingtalk_put_bytes") as put, \
mock.patch.object(artifact_fetch, "http_get", return_value=("/tmp/result", 123, "a" * 64)) as download:
result = artifact_fetch.live_export_dingtalk_document(
"raw-node", {"a-token": "x", "x-xsrf-token": "y", "cookie": "z", "referer": "https://alidocs.dingtalk.com/"},
{"name": "说明.adoc", "relative_path": "说明.adoc"}, 1_000_000,
)
self.assertEqual(result, ("/tmp/result", 123, "a" * 64, False))
self.assertTrue(any("submitExportJob" in url for url, _ in calls))
self.assertFalse(any("/file/download" in url for url, _ in calls))
put.assert_called_once()
download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000)
if __name__ == "__main__":
unittest.main()