姚章浩
9155bd3170
build / build (push) Successful in 23s
压测/取件实测暴露的三类问题（均在真实 worker 容器里复现并已验证修复）： - 名为 .axls 实为 docx 的节点（清单扩展名与真实类型不符）：dws 明确回报真实类型 → 回退 drive download 取原始字节（实测该节点 243,949B docx 可正常取到）， 不再直接失败 - server model operate block 判可重试：同一节点 5 连测 3/5 成功（另一支 timeout）， 原判终态会把可成功节点永久失败 - axls 大表偶发返回全空单元格（12 次取件出现 1 次）：全空时重读一次再上报， 避免被平台空壳判定当「空文档」终态 新增 3 个回归测试（18 tests / 19 断言全绿）
217 lines
9.7 KiB
Python
217 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
||
"""dws axls → xlsx 转换器（纯标准库，无 openpyxl）。
|
||
|
||
用途：dws 没有 axls 直出接口，只能 sheet range read 拿单元格；本脚本把单元格写成
|
||
合法的 .xlsx，供平台 artifact_fetch 的 dingtalk dws 取件器落盘入库。
|
||
|
||
兼容平台既有检查：
|
||
- `exported_doc_is_blank()` 对 spreadsheet 只读 `xl/sharedStrings.xml` → 本脚本**把文本写进 sharedStrings**，
|
||
否则会被判定为"空壳文档"而拒收。
|
||
- 必须 PK magic、能被子解析器识别为 xlsx。
|
||
|
||
归一化约定（与平台既有 xlsx 产物对齐）：
|
||
- 单元格换行 CRLF/CR → LF
|
||
- 日期保持 dws 返回的显示文本（平台旧产物是 Excel 序列号，转换后语义一致）
|
||
|
||
dws 可执行文件解析顺序：$DWS_BIN → PATH 里的 dws。
|
||
"""
|
||
import argparse, json, os, re, shutil, subprocess, sys, zipfile
|
||
from xml.sax.saxutils import escape
|
||
|
||
NUM_RE = re.compile(r"^-?\d+(\.\d+)?$")
|
||
|
||
|
||
class DwsCallError(RuntimeError):
|
||
"""dws 调用失败（供取件器映射成平台错误码，而不是让进程直接退出）。"""
|
||
|
||
|
||
def dws_binary():
|
||
return os.environ.get("DWS_BIN") or shutil.which("dws") or "/usr/local/bin/dws"
|
||
|
||
|
||
def run_json(args, timeout=180):
|
||
# dws 的文件型 keychain 在 $HOME/.local/share/dws-cli，容器里用 DWS_HOME 单独指定
|
||
env = dict(os.environ)
|
||
dws_home = os.environ.get("DWS_HOME")
|
||
if dws_home:
|
||
env["HOME"] = dws_home
|
||
p = subprocess.run([dws_binary(), *args, "-f", "json"], capture_output=True, text=True,
|
||
timeout=timeout, env=env)
|
||
try:
|
||
payload = json.loads(p.stdout)
|
||
except Exception:
|
||
raise DwsCallError(f"dws 调用失败: {' '.join(args)} :: {(p.stdout or p.stderr)[:300]}")
|
||
failed = (p.returncode != 0 or payload.get("success") is False or payload.get("ok") is False
|
||
or payload.get("status") in ("failure", "failed"))
|
||
if failed:
|
||
message = ""
|
||
err = payload.get("error")
|
||
if isinstance(err, dict):
|
||
message = str(err.get("message") or err.get("cause") or "")
|
||
raise DwsCallError(message or f"dws 调用失败: {' '.join(args)}")
|
||
return payload
|
||
|
||
|
||
def fetch_sheets(node):
|
||
"""返回 [{'name':str,'cells':[[...]]}]"""
|
||
out = _read_sheets(node)
|
||
if all(not s.get("cells") for s in out):
|
||
# 实测：大表（220MB/15 表）偶发返回全空单元格，重读一次即正常；空白结果
|
||
# 一旦上报会被平台的空壳判定当作"空文档"终态处理，所以这里先重读一遍。
|
||
retry = _read_sheets(node)
|
||
if any(s.get("cells") for s in retry):
|
||
return retry
|
||
return out
|
||
|
||
|
||
def _read_sheets(node):
|
||
listing = run_json(["sheet", "list", "--node", node])
|
||
sheets = listing.get("sheets") or []
|
||
if not sheets:
|
||
raise DwsCallError(f"sheet list 无工作表: {json.dumps(listing, ensure_ascii=False)[:200]}")
|
||
out = []
|
||
for s in sheets:
|
||
sid = s.get("sheetId") or s.get("name")
|
||
r = run_json(["sheet", "range", "read", "--node", node, "--sheet-id", sid])
|
||
out.append({"name": s.get("name") or sid, "cells": r.get("cells") or []})
|
||
return out
|
||
|
||
|
||
def norm(v):
|
||
if v is None:
|
||
return None
|
||
s = str(v)
|
||
if "\r" in s:
|
||
s = s.replace("\r\n", "\n").replace("\r", "\n")
|
||
return s
|
||
|
||
|
||
def col_letter(idx):
|
||
"""0-based 列号 → A/B/.../AA"""
|
||
s = ""
|
||
idx += 1
|
||
while idx:
|
||
idx, rem = divmod(idx - 1, 26)
|
||
s = chr(65 + rem) + s
|
||
return s
|
||
|
||
|
||
def sheet_name(name, used):
|
||
n = re.sub(r"[\[\]:*?/\\]", "_", str(name or "Sheet")).strip()[:31] or "Sheet"
|
||
base, i = n, 1
|
||
while n.lower() in used:
|
||
suffix = f"_{i}"
|
||
n = base[: 31 - len(suffix)] + suffix
|
||
i += 1
|
||
used.add(n.lower())
|
||
return n
|
||
|
||
|
||
def build_xlsx(sheets, path):
|
||
# 1) 收集共享字符串 + 每表 XML
|
||
shared, shared_idx = [], {}
|
||
def sidx(text):
|
||
if text not in shared_idx:
|
||
shared_idx[text] = len(shared)
|
||
shared.append(text)
|
||
return shared_idx[text]
|
||
|
||
used, sheet_xml, names = set(), [], []
|
||
for s in sheets:
|
||
sname = sheet_name(s.get("name"), used)
|
||
names.append(sname)
|
||
rows = s.get("cells") or []
|
||
chunks = []
|
||
for ri, row in enumerate(rows, start=1):
|
||
cells = []
|
||
for ci, cell in enumerate(row):
|
||
v = cell.get("value") if isinstance(cell, dict) else cell
|
||
v = norm(v)
|
||
if v is None or v == "":
|
||
continue
|
||
ref = f"{col_letter(ci)}{ri}"
|
||
if NUM_RE.match(v):
|
||
cells.append(f'<c r="{ref}"><v>{v}</v></c>')
|
||
else:
|
||
cells.append(f'<c r="{ref}" t="s"><v>{sidx(v)}</v></c>')
|
||
if cells:
|
||
chunks.append(f'<row r="{ri}">{"".join(cells)}</row>')
|
||
sheet_xml.append(
|
||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
|
||
f'<sheetData>{"".join(chunks)}</sheetData></worksheet>')
|
||
|
||
n = len(names)
|
||
ct = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
|
||
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
|
||
'<Default Extension="xml" ContentType="application/xml"/>'
|
||
'<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
|
||
'<Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>'
|
||
'<Override PartName="/xl/sharedStrings.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml"/>'
|
||
+ "".join(f'<Override PartName="/xl/worksheets/sheet{i+1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>' for i in range(n))
|
||
+ '</Types>')
|
||
rels = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
|
||
'<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>'
|
||
'</Relationships>')
|
||
wb = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
|
||
'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>'
|
||
+ "".join(f'<sheet name="{escape(names[i])}" sheetId="{i+1}" r:id="rId{i+1}"/>' for i in range(n))
|
||
+ '</sheets></workbook>')
|
||
wb_rels = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
|
||
+ "".join(f'<Relationship Id="rId{i+1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{i+1}.xml"/>' for i in range(n))
|
||
+ f'<Relationship Id="rId{n+1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>'
|
||
+ f'<Relationship Id="rId{n+2}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="sharedStrings.xml"/>'
|
||
+ '</Relationships>')
|
||
styles = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
'<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
|
||
'<fonts count="1"><font><sz val="11"/><name val="Calibri"/></font></fonts>'
|
||
'<fills count="1"><fill><patternFill patternType="none"/></fill></fills>'
|
||
'<borders count="1"><border/></borders>'
|
||
'<cellStyleXfs count="1"><xf/></cellStyleXfs>'
|
||
'<cellXfs count="1"><xf/></cellXfs></styleSheet>')
|
||
sst = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||
f'<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="{len(shared)}" uniqueCount="{len(shared)}">'
|
||
+ "".join(f"<si><t xml:space=\"preserve\">{escape(t)}</t></si>" for t in shared)
|
||
+ '</sst>')
|
||
|
||
with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z:
|
||
z.writestr("[Content_Types].xml", ct)
|
||
z.writestr("_rels/.rels", rels)
|
||
z.writestr("xl/workbook.xml", wb)
|
||
z.writestr("xl/_rels/workbook.xml.rels", wb_rels)
|
||
z.writestr("xl/styles.xml", styles)
|
||
z.writestr("xl/sharedStrings.xml", sst)
|
||
for i, xml in enumerate(sheet_xml):
|
||
z.writestr(f"xl/worksheets/sheet{i+1}.xml", xml)
|
||
return {"sheets": names, "shared_strings": len(shared)}
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--node")
|
||
ap.add_argument("--cells-json")
|
||
ap.add_argument("--out", required=True)
|
||
a = ap.parse_args()
|
||
if a.cells_json:
|
||
sheets = json.load(open(a.cells_json))["sheets"]
|
||
elif a.node:
|
||
sheets = fetch_sheets(a.node)
|
||
else:
|
||
raise SystemExit("需要 --node 或 --cells-json")
|
||
info = build_xlsx(sheets, a.out)
|
||
rows = sum(len(s.get("cells") or []) for s in sheets)
|
||
print(json.dumps({"success": True, "out": a.out, "sheets": info["sheets"],
|
||
"sheet_count": len(sheets), "rows": rows,
|
||
"shared_strings": info["shared_strings"]}, ensure_ascii=False))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
main()
|
||
except DwsCallError as error:
|
||
print(json.dumps({"success": False, "error": str(error)[:300]}, ensure_ascii=False), file=sys.stderr)
|
||
sys.exit(2) |