test(agent): add end-to-end DuckDB excel tests and fix sheet enumeration (#1007)
Catching up on a real-world check: when a multi-sheet .xlsx fixture is
fed through LoadFromExcel against an in-memory DuckDB with the spatial
and excel extensions, the first iteration's sheet enumeration quietly
fell back because st_read_meta doesn't expose a scalar 'layer_name'
column — it returns a LIST<STRUCT> column called 'layers'. We now
UNNEST(layers).name so enumeration actually works, and assert the full
data path with three DuckDB-backed tests:
- multi-sheet workbook yields the sum of all rows and per-sheet
breakdown via __sheet_name (including schema drift: columns only
present in one sheet are NULL for the other)
- single-sheet workbook still works and tags rows with its sheet name
- sheet name containing a single quote (Q1'24) survives the SQL
literal round-trip
excelize/v2 is added as a test-time helper to build the fixtures
deterministically. If DuckDB's spatial/excel extensions can't be
installed (offline CI), the tests skip rather than fail.
Refs: https://github.com/Tencent/WeKnora/issues/1007
This commit is contained in:
@@ -56,6 +56,7 @@ require (
|
||||
github.com/wailsapp/wails/v2 v2.12.0
|
||||
github.com/weaviate/weaviate v1.33.0-rc.1
|
||||
github.com/weaviate/weaviate-go-client/v5 v5.5.0
|
||||
github.com/xuri/excelize/v2 v2.10.1
|
||||
github.com/yanyiwu/gojieba v1.4.5
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
|
||||
@@ -64,9 +65,9 @@ require (
|
||||
go.opentelemetry.io/otel/sdk v1.38.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
go.uber.org/dig v1.18.1
|
||||
golang.org/x/crypto v0.46.0
|
||||
golang.org/x/mod v0.31.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/mod v0.32.0
|
||||
golang.org/x/net v0.50.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/api v0.259.0
|
||||
@@ -243,6 +244,8 @@ require (
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.54.0 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.6 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.6 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
@@ -260,6 +263,7 @@ require (
|
||||
github.com/tidwall/gjson v1.17.1 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tiendc/go-deepcopy v1.7.2 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/tkrajina/go-reflector v0.5.8 // indirect
|
||||
@@ -275,6 +279,8 @@ require (
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
github.com/xuri/efp v0.0.1 // indirect
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
@@ -301,10 +307,10 @@ require (
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
|
||||
@@ -2338,6 +2338,10 @@ github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7
|
||||
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
|
||||
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
|
||||
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
|
||||
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -2465,6 +2469,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
|
||||
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
|
||||
github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw=
|
||||
github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
@@ -2537,6 +2543,12 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
|
||||
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
|
||||
github.com/xuri/excelize/v2 v2.10.1 h1:V62UlqopMqha3kOpnlHy2CcRVw1V8E63jFoWUmMzxN0=
|
||||
github.com/xuri/excelize/v2 v2.10.1/go.mod h1:iG5tARpgaEeIhTqt3/fgXCGoBRt4hNXgCp3tfXKoOIc=
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
|
||||
@@ -2725,8 +2737,8 @@ golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn5
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2759,6 +2771,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
|
||||
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2797,8 +2811,8 @@ golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
|
||||
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
|
||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2887,8 +2901,8 @@ golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -3084,12 +3098,12 @@ golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
|
||||
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 h1:H52Mhyrc44wBgLTGzq6+0cmuVuF3LURCSXsLMOqfFos=
|
||||
golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523/go.mod h1:ArQvPJS723nJQietgilmZA+shuB3CZxH1n2iXq9VSfs=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -3116,8 +3130,8 @@ golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
|
||||
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
|
||||
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -3144,8 +3158,8 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -3237,8 +3251,8 @@ golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg
|
||||
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
|
||||
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
|
||||
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -364,8 +364,15 @@ func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, t
|
||||
// listExcelSheets returns the names of every sheet (layer) inside the given
|
||||
// Excel workbook by querying DuckDB's spatial st_read_meta table function.
|
||||
// The returned slice preserves the on-disk order of sheets.
|
||||
//
|
||||
// st_read_meta returns a single row whose `layers` column is a LIST of
|
||||
// STRUCTs (one per layer / sheet). We UNNEST that list and project the
|
||||
// struct's `name` field to get a flat list of sheet names.
|
||||
func (t *DataAnalysisTool) listExcelSheets(ctx context.Context, filename string) ([]string, error) {
|
||||
metaSQL := fmt.Sprintf("SELECT layer_name FROM st_read_meta('%s')", sqlSingleQuoteEscape(filename))
|
||||
metaSQL := fmt.Sprintf(
|
||||
"SELECT UNNEST(layers).name FROM st_read_meta('%s')",
|
||||
sqlSingleQuoteEscape(filename),
|
||||
)
|
||||
|
||||
rows, err := t.db.QueryContext(ctx, metaSQL)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
// newTestDuckDB opens an in-memory DuckDB and loads the extensions the
|
||||
// Data Analysis tool needs. If the extensions can't be installed (e.g. the
|
||||
// test environment has no network), the test is skipped — we can't validate
|
||||
// Excel handling without them.
|
||||
func newTestDuckDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("duckdb", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open duckdb: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
ctx := context.Background()
|
||||
for _, ext := range []string{"spatial", "excel"} {
|
||||
if _, err := db.ExecContext(ctx, fmt.Sprintf("INSTALL %s;", ext)); err != nil {
|
||||
t.Skipf("cannot install duckdb %s extension (network required): %v", ext, err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, fmt.Sprintf("LOAD %s;", ext)); err != nil {
|
||||
t.Skipf("cannot load duckdb %s extension: %v", ext, err)
|
||||
}
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// writeWorkbook builds a minimal .xlsx with the given sheet -> rows layout.
|
||||
// Each inner row is a list of cell values; the first row of each sheet is
|
||||
// treated as the header, matching what read_xlsx expects by default.
|
||||
func writeWorkbook(t *testing.T, path string, sheets map[string][][]any, sheetOrder []string) {
|
||||
t.Helper()
|
||||
f := excelize.NewFile()
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
// Start from a clean slate: we'll add our own sheets in the order
|
||||
// requested and delete the default Sheet1 at the end if it isn't used.
|
||||
for idx, name := range sheetOrder {
|
||||
if _, err := f.NewSheet(name); err != nil {
|
||||
t.Fatalf("new sheet %q: %v", name, err)
|
||||
}
|
||||
rows := sheets[name]
|
||||
for r, row := range rows {
|
||||
for c, val := range row {
|
||||
cell, err := excelize.CoordinatesToCellName(c+1, r+1)
|
||||
if err != nil {
|
||||
t.Fatalf("coord (%d,%d): %v", c, r, err)
|
||||
}
|
||||
if err := f.SetCellValue(name, cell, val); err != nil {
|
||||
t.Fatalf("set cell %s on sheet %q: %v", cell, name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx == 0 {
|
||||
sheetIdx, err := f.GetSheetIndex(name)
|
||||
if err != nil {
|
||||
t.Fatalf("get sheet index %q: %v", name, err)
|
||||
}
|
||||
f.SetActiveSheet(sheetIdx)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the default "Sheet1" that excelize.NewFile() creates, unless
|
||||
// the caller explicitly asked for one.
|
||||
if _, explicit := sheets["Sheet1"]; !explicit {
|
||||
if err := f.DeleteSheet("Sheet1"); err != nil {
|
||||
t.Fatalf("delete default Sheet1: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := f.SaveAs(path); err != nil {
|
||||
t.Fatalf("save workbook: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadFromExcel_MultiSheet is the end-to-end guard for issue #1007:
|
||||
// given a workbook with 2 sheets of different schemas, every row from every
|
||||
// sheet must land in the DuckDB table, and the __sheet_name column must
|
||||
// correctly identify each row's origin.
|
||||
func TestLoadFromExcel_MultiSheet(t *testing.T) {
|
||||
db := newTestDuckDB(t)
|
||||
|
||||
tmp := t.TempDir()
|
||||
path := filepath.Join(tmp, "multi_sheet.xlsx")
|
||||
|
||||
writeWorkbook(t, path,
|
||||
map[string][][]any{
|
||||
"Sales": {
|
||||
{"id", "amount"},
|
||||
{1, 100},
|
||||
{2, 250},
|
||||
},
|
||||
"Inventory": {
|
||||
{"sku", "stock"},
|
||||
{"A", 5},
|
||||
{"B", 12},
|
||||
{"C", 7},
|
||||
},
|
||||
},
|
||||
[]string{"Sales", "Inventory"},
|
||||
)
|
||||
|
||||
tool := &DataAnalysisTool{
|
||||
BaseTool: dataAnalysisTool,
|
||||
db: db,
|
||||
sessionID: "test-multi-sheet",
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
schema, err := tool.LoadFromExcel(ctx, path, "t_multi_sheet")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromExcel: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { tool.Cleanup(ctx) })
|
||||
|
||||
// 2 sheets × (rows - header) = 2 + 3 = 5 rows total.
|
||||
if schema.RowCount != 5 {
|
||||
t.Fatalf("expected 5 rows total across both sheets, got %d", schema.RowCount)
|
||||
}
|
||||
|
||||
// Columns from both sheets plus __sheet_name must all be present.
|
||||
colSet := map[string]bool{}
|
||||
for _, c := range schema.Columns {
|
||||
colSet[c.Name] = true
|
||||
}
|
||||
for _, want := range []string{"id", "amount", "sku", "stock", excelSheetNameColumn} {
|
||||
if !colSet[want] {
|
||||
t.Errorf("expected column %q to be present, got columns=%v", want, schema.Columns)
|
||||
}
|
||||
}
|
||||
|
||||
// Each sheet must contribute its own rows, identifiable via
|
||||
// __sheet_name.
|
||||
countPerSheet := map[string]int{}
|
||||
rows, err := db.QueryContext(ctx,
|
||||
fmt.Sprintf("SELECT %s, COUNT(*) FROM \"%s\" GROUP BY %s",
|
||||
excelSheetNameColumn, "t_multi_sheet", excelSheetNameColumn),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("group-by query: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var cnt int
|
||||
if err := rows.Scan(&name, &cnt); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
countPerSheet[name] = cnt
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
t.Fatalf("rows err: %v", rows.Err())
|
||||
}
|
||||
|
||||
if countPerSheet["Sales"] != 2 {
|
||||
t.Errorf("Sales rows = %d, want 2 (per-sheet breakdown=%v)", countPerSheet["Sales"], countPerSheet)
|
||||
}
|
||||
if countPerSheet["Inventory"] != 3 {
|
||||
t.Errorf("Inventory rows = %d, want 3 (per-sheet breakdown=%v)", countPerSheet["Inventory"], countPerSheet)
|
||||
}
|
||||
|
||||
// Cross-sheet schema drift: a column that exists only in "Sales"
|
||||
// (amount) must be NULL for rows coming from "Inventory".
|
||||
var nullCount int
|
||||
err = db.QueryRowContext(ctx,
|
||||
fmt.Sprintf("SELECT COUNT(*) FROM \"%s\" WHERE %s = 'Inventory' AND amount IS NULL",
|
||||
"t_multi_sheet", excelSheetNameColumn),
|
||||
).Scan(&nullCount)
|
||||
if err != nil {
|
||||
t.Fatalf("null-amount query: %v", err)
|
||||
}
|
||||
if nullCount != 3 {
|
||||
t.Errorf("Inventory rows with NULL amount = %d, want 3", nullCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadFromExcel_SingleSheet regression-tests the common single-sheet
|
||||
// case: no multi-sheet UNION wiring should accidentally break it.
|
||||
func TestLoadFromExcel_SingleSheet(t *testing.T) {
|
||||
db := newTestDuckDB(t)
|
||||
|
||||
tmp := t.TempDir()
|
||||
path := filepath.Join(tmp, "single.xlsx")
|
||||
|
||||
writeWorkbook(t, path,
|
||||
map[string][][]any{
|
||||
"Data": {
|
||||
{"name", "value"},
|
||||
{"alpha", 1},
|
||||
{"beta", 2},
|
||||
},
|
||||
},
|
||||
[]string{"Data"},
|
||||
)
|
||||
|
||||
tool := &DataAnalysisTool{
|
||||
BaseTool: dataAnalysisTool,
|
||||
db: db,
|
||||
sessionID: "test-single-sheet",
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
schema, err := tool.LoadFromExcel(ctx, path, "t_single")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromExcel: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { tool.Cleanup(ctx) })
|
||||
|
||||
if schema.RowCount != 2 {
|
||||
t.Fatalf("expected 2 data rows, got %d", schema.RowCount)
|
||||
}
|
||||
|
||||
var sheetVal string
|
||||
if err := db.QueryRowContext(ctx,
|
||||
fmt.Sprintf("SELECT DISTINCT %s FROM \"%s\"", excelSheetNameColumn, "t_single"),
|
||||
).Scan(&sheetVal); err != nil {
|
||||
t.Fatalf("sheet-name query: %v", err)
|
||||
}
|
||||
if sheetVal != "Data" {
|
||||
t.Errorf("expected all rows tagged with sheet 'Data', got %q", sheetVal)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadFromExcel_QuotedSheetName makes sure sheet names with single
|
||||
// quotes survive the SQL round-trip (guards the quote-escaping path).
|
||||
func TestLoadFromExcel_QuotedSheetName(t *testing.T) {
|
||||
db := newTestDuckDB(t)
|
||||
|
||||
tmp := t.TempDir()
|
||||
path := filepath.Join(tmp, "quoted.xlsx")
|
||||
|
||||
// Excel disallows some characters in sheet names (e.g. []:*?/) but
|
||||
// single quotes are actually permitted; this is the most likely quote
|
||||
// bomb for SQL-literal construction.
|
||||
sheet := "Q1'24"
|
||||
|
||||
writeWorkbook(t, path,
|
||||
map[string][][]any{
|
||||
sheet: {
|
||||
{"metric", "value"},
|
||||
{"revenue", 42},
|
||||
},
|
||||
},
|
||||
[]string{sheet},
|
||||
)
|
||||
|
||||
tool := &DataAnalysisTool{
|
||||
BaseTool: dataAnalysisTool,
|
||||
db: db,
|
||||
sessionID: "test-quoted",
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
schema, err := tool.LoadFromExcel(ctx, path, "t_quoted")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromExcel: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { tool.Cleanup(ctx) })
|
||||
|
||||
if schema.RowCount != 1 {
|
||||
t.Fatalf("expected 1 data row, got %d", schema.RowCount)
|
||||
}
|
||||
|
||||
var got string
|
||||
if err := db.QueryRowContext(ctx,
|
||||
fmt.Sprintf("SELECT %s FROM \"%s\" LIMIT 1", excelSheetNameColumn, "t_quoted"),
|
||||
).Scan(&got); err != nil {
|
||||
t.Fatalf("sheet-name query: %v", err)
|
||||
}
|
||||
if got != sheet {
|
||||
t.Errorf("sheet-name roundtrip mismatch: got %q, want %q", got, sheet)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user