fix(agent): materialize knowledge files to temp path for DuckDB (#1007)

Dev-mode repro from the issue reporter's logs:

  [Tool][DataAnalysis] Failed to create table from Excel: IO Error:
  GDAL Error (4): Failed to open file
  'local://10000/.../1777030910246871000.xlsx': No such file or directory

The local file service returns a custom 'local://' URL from GetFileURL,
which the duckdb spatial/excel extensions can't resolve (they expect
plain paths or http(s)/s3:// style URIs). Presigned HTTPS URLs from
cloud backends worked by accident; the 'local://' dev path has always
been broken for the Data Analysis tool.

Instead of per-scheme adapters, stream the file through
FileService.GetFile into a temp file and hand DuckDB the resulting
filesystem path. This works uniformly across every backend (local,
OSS, S3, MinIO, COS) and survives future scheme changes without any
changes to the Data Analysis tool.

- Preserve the original file extension on the temp file so DuckDB's
  format auto-detection (csv / xlsx / xls) still kicks in.
- Clean up the temp file when LoadFromKnowledge returns, including on
  every error branch. Cleanup is idempotent (double-invoke safe).
- New tests stub FileService to lock in that (a) we never leak a
  provider:// URL to DuckDB, (b) GetFile errors propagate, (c) the
  extension survives case normalization.

Refs: https://github.com/Tencent/WeKnora/issues/1007
This commit is contained in:
wizardchen
2026-04-24 19:49:11 +08:00
committed by lyingbug
parent 002d98a640
commit 8103398fcf
2 changed files with 229 additions and 6 deletions
+68 -6
View File
@@ -5,6 +5,8 @@ import (
"database/sql"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
@@ -439,8 +441,14 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s
)
}
// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema
// It automatically determines the file type and calls the appropriate loading method
// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema.
// It automatically determines the file type and calls the appropriate loading method.
//
// The source file is first materialized to a local temp file via FileService.GetFile
// so DuckDB's st_read / read_xlsx / read_csv_auto can open it directly. This
// side-steps provider-specific URL schemes (e.g. the local:// URL returned by
// the local file service) that DuckDB's extensions cannot resolve on their own.
//
// Parameters:
// - ctx: context for cancellation and timeout
// - knowledge: the Knowledge entity containing file information
@@ -460,16 +468,17 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ
logger.Infof(ctx, "[Tool][DataAnalysis] Loading knowledge '%s' (type: %s) into table '%s' for session %s",
knowledge.ID, fileType, tableName, t.sessionID)
fileURL, err := t.fileService.GetFileURL(ctx, knowledge.FilePath)
localPath, cleanup, err := t.materializeKnowledgeFile(ctx, knowledge)
if err != nil {
return nil, fmt.Errorf("failed to get file URL for knowledge '%s': %w", knowledge.ID, err)
return nil, fmt.Errorf("failed to materialize knowledge '%s' for DuckDB: %w", knowledge.ID, err)
}
defer cleanup()
switch fileType {
case "csv":
return t.LoadFromCSV(ctx, fileURL, tableName)
return t.LoadFromCSV(ctx, localPath, tableName)
case "xlsx", "xls":
return t.LoadFromExcel(ctx, fileURL, tableName)
return t.LoadFromExcel(ctx, localPath, tableName)
default:
logger.Warnf(ctx, "[Tool][DataAnalysis] Unsupported file type '%s' for knowledge '%s' in session %s",
fileType, knowledge.ID, t.sessionID)
@@ -477,6 +486,59 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ
}
}
// materializeKnowledgeFile copies the knowledge's backing blob into a fresh
// temp file on the local filesystem so DuckDB can open it with ordinary path
// semantics. It returns the temp path and a cleanup closure that removes the
// temp file; the closure is always safe to call and is a no-op on failure.
//
// This hides storage-backend-specific URL schemes (local://, oss://, s3://,
// minio://, cos://, …) behind the FileService.GetFile abstraction, so the
// Data Analysis tool works identically across all deployments.
func (t *DataAnalysisTool) materializeKnowledgeFile(ctx context.Context, knowledge *types.Knowledge) (string, func(), error) {
noop := func() {}
reader, err := t.fileService.GetFile(ctx, knowledge.FilePath)
if err != nil {
return "", noop, fmt.Errorf("failed to open file for knowledge '%s': %w", knowledge.ID, err)
}
defer reader.Close()
// Preserve the file extension so DuckDB's format auto-detection still
// works (e.g. the CSV reader expects .csv, xlsx reader expects .xlsx).
suffix := ""
if ext := strings.ToLower(strings.TrimSpace(knowledge.FileType)); ext != "" {
suffix = "." + ext
}
tmp, err := os.CreateTemp("", "weknora-data-analysis-*"+suffix)
if err != nil {
return "", noop, fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmp.Name()
cleanup := func() {
// Best-effort cleanup; a missing file is fine, any other error is
// only logged to avoid masking the original operation's result.
if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) {
logger.Warnf(ctx, "[Tool][DataAnalysis] Failed to remove temp file %s: %v", tmpPath, err)
}
}
if _, err := io.Copy(tmp, reader); err != nil {
_ = tmp.Close()
cleanup()
return "", noop, fmt.Errorf("failed to copy knowledge '%s' to temp file: %w", knowledge.ID, err)
}
if err := tmp.Close(); err != nil {
cleanup()
return "", noop, fmt.Errorf("failed to finalize temp file for knowledge '%s': %w", knowledge.ID, err)
}
logger.Infof(ctx, "[Tool][DataAnalysis] Materialized knowledge '%s' to temp file %s for session %s",
knowledge.ID, tmpPath, t.sessionID)
return tmpPath, cleanup, nil
}
// LoadFromKnowledgeID loads data from a Knowledge ID into a DuckDB table and returns the table schema
// Parameters:
// - ctx: context for cancellation and timeout
@@ -0,0 +1,161 @@
package tools
import (
"bytes"
"context"
"errors"
"io"
"mime/multipart"
"os"
"path/filepath"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// fakeFileService implements interfaces.FileService just enough to drive
// materializeKnowledgeFile. Every method we don't care about simply errors
// out so accidental usage is loud.
type fakeFileService struct {
readers map[string]func() (io.ReadCloser, error)
}
func (f *fakeFileService) CheckConnectivity(ctx context.Context) error { return nil }
func (f *fakeFileService) SaveFile(ctx context.Context, _ *multipart.FileHeader, _ uint64, _ string) (string, error) {
return "", errors.New("not implemented in fake")
}
func (f *fakeFileService) SaveBytes(ctx context.Context, _ []byte, _ uint64, _ string, _ bool) (string, error) {
return "", errors.New("not implemented in fake")
}
func (f *fakeFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
fn, ok := f.readers[filePath]
if !ok {
return nil, errors.New("unknown path: " + filePath)
}
return fn()
}
func (f *fakeFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// Return a URL that DuckDB would NOT be able to open on its own; the
// production code must *not* pass this through to DuckDB.
return "local://" + strings.TrimPrefix(filePath, "/"), nil
}
func (f *fakeFileService) DeleteFile(ctx context.Context, _ string) error { return nil }
// TestMaterializeKnowledgeFile_HandlesLocalScheme is the regression guard
// for the dev-mode failure where DuckDB was handed a local:// URL it can't
// resolve. The tool must pull bytes via FileService.GetFile and hand DuckDB
// a concrete filesystem path with the right extension.
func TestMaterializeKnowledgeFile_HandlesLocalScheme(t *testing.T) {
payload := []byte("col1,col2\n1,2\n3,4\n")
fs := &fakeFileService{
readers: map[string]func() (io.ReadCloser, error){
"tenants/42/data.csv": func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(payload)), nil
},
},
}
tool := &DataAnalysisTool{
fileService: fs,
sessionID: "test-materialize",
}
k := &types.Knowledge{
ID: "k-abc",
FileType: "csv",
FilePath: "tenants/42/data.csv",
}
path, cleanup, err := tool.materializeKnowledgeFile(context.Background(), k)
if err != nil {
t.Fatalf("materialize: %v", err)
}
t.Cleanup(cleanup)
// Path must be a real filesystem path (no provider scheme) so DuckDB
// can st_read / read_xlsx / read_csv_auto it directly.
if strings.Contains(path, "://") {
t.Fatalf("expected bare filesystem path, got scheme URL: %q", path)
}
if filepath.Ext(path) != ".csv" {
t.Errorf("expected .csv suffix preserved for DuckDB format detection, got %q", path)
}
// File must actually exist and have the exact bytes we fed in.
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read temp file: %v", err)
}
if !bytes.Equal(got, payload) {
t.Errorf("payload roundtrip mismatch: got %q, want %q", got, payload)
}
// cleanup() must remove the temp file without error.
cleanup()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Errorf("expected temp file to be removed after cleanup, stat err=%v", err)
}
// Double-cleanup must be safe (test harness may invoke it again).
cleanup()
}
func TestMaterializeKnowledgeFile_PropagatesGetFileError(t *testing.T) {
fs := &fakeFileService{
readers: map[string]func() (io.ReadCloser, error){
"bad/path": func() (io.ReadCloser, error) {
return nil, errors.New("boom")
},
},
}
tool := &DataAnalysisTool{fileService: fs, sessionID: "test-error"}
_, cleanup, err := tool.materializeKnowledgeFile(
context.Background(),
&types.Knowledge{ID: "k-err", FileType: "xlsx", FilePath: "bad/path"},
)
if err == nil {
defer cleanup()
t.Fatal("expected error from underlying GetFile, got nil")
}
if !strings.Contains(err.Error(), "boom") {
t.Errorf("expected wrapped 'boom' error, got: %v", err)
}
}
func TestMaterializeKnowledgeFile_PreservesExtension(t *testing.T) {
cases := []struct {
fileType string
wantExt string
}{
{"xlsx", ".xlsx"},
{"XLSX", ".xlsx"}, // normalized lowercase
{"csv", ".csv"},
{"xls", ".xls"},
}
for _, c := range cases {
t.Run(c.fileType, func(t *testing.T) {
fs := &fakeFileService{
readers: map[string]func() (io.ReadCloser, error){
"p": func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader([]byte("x"))), nil
},
},
}
tool := &DataAnalysisTool{fileService: fs, sessionID: "ext"}
path, cleanup, err := tool.materializeKnowledgeFile(
context.Background(),
&types.Knowledge{ID: "k", FileType: c.fileType, FilePath: "p"},
)
if err != nil {
t.Fatalf("materialize: %v", err)
}
defer cleanup()
if filepath.Ext(path) != c.wantExt {
t.Errorf("fileType=%q: extension mismatch got %q want %q", c.fileType, filepath.Ext(path), c.wantExt)
}
})
}
}