test(agent): add end-to-end DuckDB excel tests and fix sheet enumeration (#1007)

Catching up on a real-world check: when a multi-sheet .xlsx fixture is
fed through LoadFromExcel against an in-memory DuckDB with the spatial
and excel extensions, the first iteration's sheet enumeration quietly
fell back because st_read_meta doesn't expose a scalar 'layer_name'
column — it returns a LIST<STRUCT> column called 'layers'. We now
UNNEST(layers).name so enumeration actually works, and assert the full
data path with three DuckDB-backed tests:

  - multi-sheet workbook yields the sum of all rows and per-sheet
    breakdown via __sheet_name (including schema drift: columns only
    present in one sheet are NULL for the other)
  - single-sheet workbook still works and tags rows with its sheet name
  - sheet name containing a single quote (Q1'24) survives the SQL
    literal round-trip

excelize/v2 is added as a test-time helper to build the fixtures
deterministically. If DuckDB's spatial/excel extensions can't be
installed (offline CI), the tests skip rather than fail.

Refs: https://github.com/Tencent/WeKnora/issues/1007
This commit is contained in:
wizardchen
2026-04-24 19:37:39 +08:00
committed by lyingbug
parent 11811f9bf8
commit 002d98a640
4 changed files with 334 additions and 24 deletions
+13 -7
View File
@@ -56,6 +56,7 @@ require (
github.com/wailsapp/wails/v2 v2.12.0
github.com/weaviate/weaviate v1.33.0-rc.1
github.com/weaviate/weaviate-go-client/v5 v5.5.0
github.com/xuri/excelize/v2 v2.10.1
github.com/yanyiwu/gojieba v1.4.5
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
@@ -64,9 +65,9 @@ require (
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
go.uber.org/dig v1.18.1
golang.org/x/crypto v0.46.0
golang.org/x/mod v0.31.0
golang.org/x/net v0.48.0
golang.org/x/crypto v0.48.0
golang.org/x/mod v0.32.0
golang.org/x/net v0.50.0
golang.org/x/sync v0.19.0
golang.org/x/time v0.14.0
google.golang.org/api v0.259.0
@@ -243,6 +244,8 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/quic-go/quic-go v0.54.0 // indirect
github.com/richardlehane/mscfb v1.0.6 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/rs/xid v1.6.0 // indirect
@@ -260,6 +263,7 @@ require (
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tiendc/go-deepcopy v1.7.2 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/tkrajina/go-reflector v0.5.8 // indirect
@@ -275,6 +279,8 @@ require (
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
@@ -301,10 +307,10 @@ require (
golang.org/x/arch v0.20.0 // indirect
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/tools v0.40.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.41.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
+30 -16
View File
@@ -2338,6 +2338,10 @@ github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -2465,6 +2469,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw=
github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
@@ -2537,6 +2543,12 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.10.1 h1:V62UlqopMqha3kOpnlHy2CcRVw1V8E63jFoWUmMzxN0=
github.com/xuri/excelize/v2 v2.10.1/go.mod h1:iG5tARpgaEeIhTqt3/fgXCGoBRt4hNXgCp3tfXKoOIc=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
@@ -2725,8 +2737,8 @@ golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn5
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2759,6 +2771,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -2797,8 +2811,8 @@ golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2887,8 +2901,8 @@ golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -3084,12 +3098,12 @@ golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 h1:H52Mhyrc44wBgLTGzq6+0cmuVuF3LURCSXsLMOqfFos=
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523/go.mod h1:ArQvPJS723nJQietgilmZA+shuB3CZxH1n2iXq9VSfs=
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -3116,8 +3130,8 @@ golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -3144,8 +3158,8 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -3237,8 +3251,8 @@ golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+8 -1
View File
@@ -364,8 +364,15 @@ func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, t
// listExcelSheets returns the names of every sheet (layer) inside the given
// Excel workbook by querying DuckDB's spatial st_read_meta table function.
// The returned slice preserves the on-disk order of sheets.
//
// st_read_meta returns a single row whose `layers` column is a LIST of
// STRUCTs (one per layer / sheet). We UNNEST that list and project the
// struct's `name` field to get a flat list of sheet names.
func (t *DataAnalysisTool) listExcelSheets(ctx context.Context, filename string) ([]string, error) {
metaSQL := fmt.Sprintf("SELECT layer_name FROM st_read_meta('%s')", sqlSingleQuoteEscape(filename))
metaSQL := fmt.Sprintf(
"SELECT UNNEST(layers).name FROM st_read_meta('%s')",
sqlSingleQuoteEscape(filename),
)
rows, err := t.db.QueryContext(ctx, metaSQL)
if err != nil {
@@ -0,0 +1,283 @@
package tools
import (
"context"
"database/sql"
"fmt"
"path/filepath"
"testing"
_ "github.com/duckdb/duckdb-go/v2"
"github.com/xuri/excelize/v2"
)
// newTestDuckDB opens an in-memory DuckDB and loads the extensions the
// Data Analysis tool needs. If the extensions can't be installed (e.g. the
// test environment has no network), the test is skipped — we can't validate
// Excel handling without them.
func newTestDuckDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("duckdb", ":memory:")
if err != nil {
t.Fatalf("open duckdb: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
ctx := context.Background()
for _, ext := range []string{"spatial", "excel"} {
if _, err := db.ExecContext(ctx, fmt.Sprintf("INSTALL %s;", ext)); err != nil {
t.Skipf("cannot install duckdb %s extension (network required): %v", ext, err)
}
if _, err := db.ExecContext(ctx, fmt.Sprintf("LOAD %s;", ext)); err != nil {
t.Skipf("cannot load duckdb %s extension: %v", ext, err)
}
}
return db
}
// writeWorkbook builds a minimal .xlsx with the given sheet -> rows layout.
// Each inner row is a list of cell values; the first row of each sheet is
// treated as the header, matching what read_xlsx expects by default.
func writeWorkbook(t *testing.T, path string, sheets map[string][][]any, sheetOrder []string) {
t.Helper()
f := excelize.NewFile()
defer func() { _ = f.Close() }()
// Start from a clean slate: we'll add our own sheets in the order
// requested and delete the default Sheet1 at the end if it isn't used.
for idx, name := range sheetOrder {
if _, err := f.NewSheet(name); err != nil {
t.Fatalf("new sheet %q: %v", name, err)
}
rows := sheets[name]
for r, row := range rows {
for c, val := range row {
cell, err := excelize.CoordinatesToCellName(c+1, r+1)
if err != nil {
t.Fatalf("coord (%d,%d): %v", c, r, err)
}
if err := f.SetCellValue(name, cell, val); err != nil {
t.Fatalf("set cell %s on sheet %q: %v", cell, name, err)
}
}
}
if idx == 0 {
sheetIdx, err := f.GetSheetIndex(name)
if err != nil {
t.Fatalf("get sheet index %q: %v", name, err)
}
f.SetActiveSheet(sheetIdx)
}
}
// Delete the default "Sheet1" that excelize.NewFile() creates, unless
// the caller explicitly asked for one.
if _, explicit := sheets["Sheet1"]; !explicit {
if err := f.DeleteSheet("Sheet1"); err != nil {
t.Fatalf("delete default Sheet1: %v", err)
}
}
if err := f.SaveAs(path); err != nil {
t.Fatalf("save workbook: %v", err)
}
}
// TestLoadFromExcel_MultiSheet is the end-to-end guard for issue #1007:
// given a workbook with 2 sheets of different schemas, every row from every
// sheet must land in the DuckDB table, and the __sheet_name column must
// correctly identify each row's origin.
func TestLoadFromExcel_MultiSheet(t *testing.T) {
db := newTestDuckDB(t)
tmp := t.TempDir()
path := filepath.Join(tmp, "multi_sheet.xlsx")
writeWorkbook(t, path,
map[string][][]any{
"Sales": {
{"id", "amount"},
{1, 100},
{2, 250},
},
"Inventory": {
{"sku", "stock"},
{"A", 5},
{"B", 12},
{"C", 7},
},
},
[]string{"Sales", "Inventory"},
)
tool := &DataAnalysisTool{
BaseTool: dataAnalysisTool,
db: db,
sessionID: "test-multi-sheet",
}
ctx := context.Background()
schema, err := tool.LoadFromExcel(ctx, path, "t_multi_sheet")
if err != nil {
t.Fatalf("LoadFromExcel: %v", err)
}
t.Cleanup(func() { tool.Cleanup(ctx) })
// 2 sheets × (rows - header) = 2 + 3 = 5 rows total.
if schema.RowCount != 5 {
t.Fatalf("expected 5 rows total across both sheets, got %d", schema.RowCount)
}
// Columns from both sheets plus __sheet_name must all be present.
colSet := map[string]bool{}
for _, c := range schema.Columns {
colSet[c.Name] = true
}
for _, want := range []string{"id", "amount", "sku", "stock", excelSheetNameColumn} {
if !colSet[want] {
t.Errorf("expected column %q to be present, got columns=%v", want, schema.Columns)
}
}
// Each sheet must contribute its own rows, identifiable via
// __sheet_name.
countPerSheet := map[string]int{}
rows, err := db.QueryContext(ctx,
fmt.Sprintf("SELECT %s, COUNT(*) FROM \"%s\" GROUP BY %s",
excelSheetNameColumn, "t_multi_sheet", excelSheetNameColumn),
)
if err != nil {
t.Fatalf("group-by query: %v", err)
}
defer rows.Close()
for rows.Next() {
var name string
var cnt int
if err := rows.Scan(&name, &cnt); err != nil {
t.Fatalf("scan: %v", err)
}
countPerSheet[name] = cnt
}
if rows.Err() != nil {
t.Fatalf("rows err: %v", rows.Err())
}
if countPerSheet["Sales"] != 2 {
t.Errorf("Sales rows = %d, want 2 (per-sheet breakdown=%v)", countPerSheet["Sales"], countPerSheet)
}
if countPerSheet["Inventory"] != 3 {
t.Errorf("Inventory rows = %d, want 3 (per-sheet breakdown=%v)", countPerSheet["Inventory"], countPerSheet)
}
// Cross-sheet schema drift: a column that exists only in "Sales"
// (amount) must be NULL for rows coming from "Inventory".
var nullCount int
err = db.QueryRowContext(ctx,
fmt.Sprintf("SELECT COUNT(*) FROM \"%s\" WHERE %s = 'Inventory' AND amount IS NULL",
"t_multi_sheet", excelSheetNameColumn),
).Scan(&nullCount)
if err != nil {
t.Fatalf("null-amount query: %v", err)
}
if nullCount != 3 {
t.Errorf("Inventory rows with NULL amount = %d, want 3", nullCount)
}
}
// TestLoadFromExcel_SingleSheet regression-tests the common single-sheet
// case: no multi-sheet UNION wiring should accidentally break it.
func TestLoadFromExcel_SingleSheet(t *testing.T) {
db := newTestDuckDB(t)
tmp := t.TempDir()
path := filepath.Join(tmp, "single.xlsx")
writeWorkbook(t, path,
map[string][][]any{
"Data": {
{"name", "value"},
{"alpha", 1},
{"beta", 2},
},
},
[]string{"Data"},
)
tool := &DataAnalysisTool{
BaseTool: dataAnalysisTool,
db: db,
sessionID: "test-single-sheet",
}
ctx := context.Background()
schema, err := tool.LoadFromExcel(ctx, path, "t_single")
if err != nil {
t.Fatalf("LoadFromExcel: %v", err)
}
t.Cleanup(func() { tool.Cleanup(ctx) })
if schema.RowCount != 2 {
t.Fatalf("expected 2 data rows, got %d", schema.RowCount)
}
var sheetVal string
if err := db.QueryRowContext(ctx,
fmt.Sprintf("SELECT DISTINCT %s FROM \"%s\"", excelSheetNameColumn, "t_single"),
).Scan(&sheetVal); err != nil {
t.Fatalf("sheet-name query: %v", err)
}
if sheetVal != "Data" {
t.Errorf("expected all rows tagged with sheet 'Data', got %q", sheetVal)
}
}
// TestLoadFromExcel_QuotedSheetName makes sure sheet names with single
// quotes survive the SQL round-trip (guards the quote-escaping path).
func TestLoadFromExcel_QuotedSheetName(t *testing.T) {
db := newTestDuckDB(t)
tmp := t.TempDir()
path := filepath.Join(tmp, "quoted.xlsx")
// Excel disallows some characters in sheet names (e.g. []:*?/) but
// single quotes are actually permitted; this is the most likely quote
// bomb for SQL-literal construction.
sheet := "Q1'24"
writeWorkbook(t, path,
map[string][][]any{
sheet: {
{"metric", "value"},
{"revenue", 42},
},
},
[]string{sheet},
)
tool := &DataAnalysisTool{
BaseTool: dataAnalysisTool,
db: db,
sessionID: "test-quoted",
}
ctx := context.Background()
schema, err := tool.LoadFromExcel(ctx, path, "t_quoted")
if err != nil {
t.Fatalf("LoadFromExcel: %v", err)
}
t.Cleanup(func() { tool.Cleanup(ctx) })
if schema.RowCount != 1 {
t.Fatalf("expected 1 data row, got %d", schema.RowCount)
}
var got string
if err := db.QueryRowContext(ctx,
fmt.Sprintf("SELECT %s FROM \"%s\" LIMIT 1", excelSheetNameColumn, "t_quoted"),
).Scan(&got); err != nil {
t.Fatalf("sheet-name query: %v", err)
}
if got != sheet {
t.Errorf("sheet-name roundtrip mismatch: got %q, want %q", got, sheet)
}
}