姚章浩
140 lines
5.4 KiB
Python
140 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Read a complete DingTalk space through shared dws credentials.
|
|
|
|
Only a complete traversal produces an inventory. Source keys are retained so
|
|
refreshing an existing source never changes directory annotation identities.
|
|
"""
|
|
import argparse
|
|
from collections import deque
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def dws(args, timeout=180):
|
|
env = {k: v for k, v in os.environ.items() if k in (
|
|
'PATH', 'HOME', 'DWS_HOME', 'DWS_CONFIG_DIR', 'DWS_DISABLE_KEYCHAIN',
|
|
'SSL_CERT_FILE', 'SSL_CERT_DIR', 'LANG')}
|
|
if env.get('DWS_HOME'):
|
|
env['HOME'] = env['DWS_HOME']
|
|
process = subprocess.run([os.environ.get('DWS_BIN', 'dws')] + args,
|
|
env=env, capture_output=True, text=True, timeout=timeout)
|
|
# Never include upstream stderr/URLs/credentials in the job's public message.
|
|
if process.returncode != 0:
|
|
raise RuntimeError('dws read rejected; check shared authorization and source access')
|
|
payload = json.loads(process.stdout)
|
|
if payload.get('ok') is not True or not isinstance(payload.get('data'), dict):
|
|
raise RuntimeError('dws returned an invalid response')
|
|
return payload['data']
|
|
|
|
|
|
def pages(args, key):
|
|
items, cursors, ids = [], set(), set()
|
|
cursor = ''
|
|
while True:
|
|
data = dws(args + (['--cursor', cursor] if cursor else []))
|
|
batch = data.get(key)
|
|
if not isinstance(batch, list) or not isinstance(data.get('hasMore'), bool):
|
|
raise RuntimeError('dws directory response is incomplete')
|
|
for item in batch:
|
|
if not isinstance(item, dict) or not item.get('nodeId'):
|
|
raise RuntimeError('dws directory contains an invalid node')
|
|
node_id = item['nodeId']
|
|
if node_id in ids:
|
|
raise RuntimeError('dws directory contains duplicate nodes')
|
|
ids.add(node_id)
|
|
items.append(item)
|
|
if not data['hasMore']:
|
|
return items
|
|
cursor = data.get('nextCursor')
|
|
if not isinstance(cursor, str) or not cursor or cursor in cursors:
|
|
raise RuntimeError('dws pagination did not advance')
|
|
cursors.add(cursor)
|
|
if len(cursors) >= 10000:
|
|
raise RuntimeError('dws pagination safety limit reached')
|
|
|
|
|
|
def list_wiki(workspace, folder):
|
|
args = ['wiki', '+node-list', '--workspace', workspace, '--limit', '50']
|
|
if folder:
|
|
args += ['--folder', folder]
|
|
return pages(args, 'nodes')
|
|
|
|
|
|
def list_drive(space, folder):
|
|
args = ['drive', '+list', '--space-id', space, '--limit', '50']
|
|
if folder:
|
|
args += ['--folder', folder]
|
|
return pages(args, 'files')
|
|
|
|
|
|
def normalise(item, parent):
|
|
node_id = item.get('nodeId')
|
|
kind = str(item.get('nodeType') or item.get('type') or '').lower()
|
|
name = str(item.get('name') or '')
|
|
if not node_id or not name or kind not in ('folder', 'file'):
|
|
raise RuntimeError('dws node identity or type is missing')
|
|
is_folder = kind == 'folder'
|
|
extension = str(item.get('extension') or '').lstrip('.')
|
|
if not is_folder and extension and not name.lower().endswith('.' + extension.lower()):
|
|
name += '.' + extension
|
|
size = item.get('file_size') or item.get('fileSize') or item.get('sizeBytes') or item.get('size')
|
|
updated = item.get('content_updated_time') or item.get('updateTime') or item.get('modifiedTime') or item.get('createTime')
|
|
return node_id, {
|
|
'name': name, 'parent_dentry_uuid': parent,
|
|
'dentry_type': 'folder' if is_folder else 'file',
|
|
'file_size': size, 'content_updated_time': updated,
|
|
}, is_folder
|
|
|
|
|
|
def walk(space_id, space_kind, root_id, emit):
|
|
nodes, seen, queue = {}, set(), deque([root_id])
|
|
files = 0
|
|
while queue:
|
|
current = queue.popleft()
|
|
if current in seen:
|
|
raise RuntimeError('dws directory cycle')
|
|
seen.add(current)
|
|
folder = current if current != root_id else ''
|
|
items = list_wiki(space_id, folder) if space_kind == 'wiki' else list_drive(space_id, folder)
|
|
for item in items:
|
|
node_id, node, is_folder = normalise(item, current)
|
|
if node_id == root_id or node_id in nodes:
|
|
raise RuntimeError('dws node appears under multiple parents')
|
|
nodes[node_id] = node
|
|
if is_folder:
|
|
queue.append(node_id)
|
|
else:
|
|
files += 1
|
|
emit(len(seen), len(queue), files)
|
|
return nodes
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--root-key', required=True)
|
|
parser.add_argument('--name', required=True)
|
|
parser.add_argument('--space-id', required=True)
|
|
parser.add_argument('--root-id', required=True)
|
|
parser.add_argument('--space-kind', choices=('wiki', 'drive'), required=True)
|
|
parser.add_argument('--output', required=True)
|
|
args = parser.parse_args()
|
|
|
|
def emit(done, pending, files):
|
|
print(f'progress root={args.root_id} folders={done} pending={pending} files={files}', flush=True)
|
|
|
|
nodes = walk(args.space_id, args.space_kind, args.root_id, emit)
|
|
payload = {'roots': {args.root_key: {
|
|
'configured_root_id': args.root_id, 'live_space_name': args.name, 'nodes': nodes}}}
|
|
pathlib.Path(args.output).write_text(json.dumps(payload, ensure_ascii=False), encoding='utf-8')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception:
|
|
print('dws directory scan failed; no snapshot was produced', file=sys.stderr)
|
|
sys.exit(1)
|