From 8103398fcf48397f46214832f67d686c4ed4b251 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 24 Apr 2026 19:49:11 +0800 Subject: [PATCH] fix(agent): materialize knowledge files to temp path for DuckDB (#1007) Dev-mode repro from the issue reporter's logs: [Tool][DataAnalysis] Failed to create table from Excel: IO Error: GDAL Error (4): Failed to open file 'local://10000/.../1777030910246871000.xlsx': No such file or directory The local file service returns a custom 'local://' URL from GetFileURL, which the duckdb spatial/excel extensions can't resolve (they expect plain paths or http(s)/s3:// style URIs). Presigned HTTPS URLs from cloud backends worked by accident; the 'local://' dev path has always been broken for the Data Analysis tool. Instead of per-scheme adapters, stream the file through FileService.GetFile into a temp file and hand DuckDB the resulting filesystem path. This works uniformly across every backend (local, OSS, S3, MinIO, COS) and survives future scheme changes without any changes to the Data Analysis tool. - Preserve the original file extension on the temp file so DuckDB's format auto-detection (csv / xlsx / xls) still kicks in. - Clean up the temp file when LoadFromKnowledge returns, including on every error branch. Cleanup is idempotent (double-invoke safe). - New tests stub FileService to lock in that (a) we never leak a provider:// URL to DuckDB, (b) GetFile errors propagate, (c) the extension survives case normalization. Refs: https://github.com/Tencent/WeKnora/issues/1007 --- internal/agent/tools/data_analysis.go | 74 +++++++- .../tools/data_analysis_materialize_test.go | 161 ++++++++++++++++++ 2 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 internal/agent/tools/data_analysis_materialize_test.go diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go index 973c9de5..cda1dc2f 100644 --- a/internal/agent/tools/data_analysis.go +++ b/internal/agent/tools/data_analysis.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/json" "fmt" + "io" + "os" "strings" "github.com/Tencent/WeKnora/internal/logger" @@ -439,8 +441,14 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s ) } -// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema -// It automatically determines the file type and calls the appropriate loading method +// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema. +// It automatically determines the file type and calls the appropriate loading method. +// +// The source file is first materialized to a local temp file via FileService.GetFile +// so DuckDB's st_read / read_xlsx / read_csv_auto can open it directly. This +// side-steps provider-specific URL schemes (e.g. the local:// URL returned by +// the local file service) that DuckDB's extensions cannot resolve on their own. +// // Parameters: // - ctx: context for cancellation and timeout // - knowledge: the Knowledge entity containing file information @@ -460,16 +468,17 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ logger.Infof(ctx, "[Tool][DataAnalysis] Loading knowledge '%s' (type: %s) into table '%s' for session %s", knowledge.ID, fileType, tableName, t.sessionID) - fileURL, err := t.fileService.GetFileURL(ctx, knowledge.FilePath) + localPath, cleanup, err := t.materializeKnowledgeFile(ctx, knowledge) if err != nil { - return nil, fmt.Errorf("failed to get file URL for knowledge '%s': %w", knowledge.ID, err) + return nil, fmt.Errorf("failed to materialize knowledge '%s' for DuckDB: %w", knowledge.ID, err) } + defer cleanup() switch fileType { case "csv": - return t.LoadFromCSV(ctx, fileURL, tableName) + return t.LoadFromCSV(ctx, localPath, tableName) case "xlsx", "xls": - return t.LoadFromExcel(ctx, fileURL, tableName) + return t.LoadFromExcel(ctx, localPath, tableName) default: logger.Warnf(ctx, "[Tool][DataAnalysis] Unsupported file type '%s' for knowledge '%s' in session %s", fileType, knowledge.ID, t.sessionID) @@ -477,6 +486,59 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ } } +// materializeKnowledgeFile copies the knowledge's backing blob into a fresh +// temp file on the local filesystem so DuckDB can open it with ordinary path +// semantics. It returns the temp path and a cleanup closure that removes the +// temp file; the closure is always safe to call and is a no-op on failure. +// +// This hides storage-backend-specific URL schemes (local://, oss://, s3://, +// minio://, cos://, …) behind the FileService.GetFile abstraction, so the +// Data Analysis tool works identically across all deployments. +func (t *DataAnalysisTool) materializeKnowledgeFile(ctx context.Context, knowledge *types.Knowledge) (string, func(), error) { + noop := func() {} + + reader, err := t.fileService.GetFile(ctx, knowledge.FilePath) + if err != nil { + return "", noop, fmt.Errorf("failed to open file for knowledge '%s': %w", knowledge.ID, err) + } + defer reader.Close() + + // Preserve the file extension so DuckDB's format auto-detection still + // works (e.g. the CSV reader expects .csv, xlsx reader expects .xlsx). + suffix := "" + if ext := strings.ToLower(strings.TrimSpace(knowledge.FileType)); ext != "" { + suffix = "." + ext + } + + tmp, err := os.CreateTemp("", "weknora-data-analysis-*"+suffix) + if err != nil { + return "", noop, fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := tmp.Name() + cleanup := func() { + // Best-effort cleanup; a missing file is fine, any other error is + // only logged to avoid masking the original operation's result. + if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { + logger.Warnf(ctx, "[Tool][DataAnalysis] Failed to remove temp file %s: %v", tmpPath, err) + } + } + + if _, err := io.Copy(tmp, reader); err != nil { + _ = tmp.Close() + cleanup() + return "", noop, fmt.Errorf("failed to copy knowledge '%s' to temp file: %w", knowledge.ID, err) + } + if err := tmp.Close(); err != nil { + cleanup() + return "", noop, fmt.Errorf("failed to finalize temp file for knowledge '%s': %w", knowledge.ID, err) + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Materialized knowledge '%s' to temp file %s for session %s", + knowledge.ID, tmpPath, t.sessionID) + + return tmpPath, cleanup, nil +} + // LoadFromKnowledgeID loads data from a Knowledge ID into a DuckDB table and returns the table schema // Parameters: // - ctx: context for cancellation and timeout diff --git a/internal/agent/tools/data_analysis_materialize_test.go b/internal/agent/tools/data_analysis_materialize_test.go new file mode 100644 index 00000000..552fd87d --- /dev/null +++ b/internal/agent/tools/data_analysis_materialize_test.go @@ -0,0 +1,161 @@ +package tools + +import ( + "bytes" + "context" + "errors" + "io" + "mime/multipart" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// fakeFileService implements interfaces.FileService just enough to drive +// materializeKnowledgeFile. Every method we don't care about simply errors +// out so accidental usage is loud. +type fakeFileService struct { + readers map[string]func() (io.ReadCloser, error) +} + +func (f *fakeFileService) CheckConnectivity(ctx context.Context) error { return nil } +func (f *fakeFileService) SaveFile(ctx context.Context, _ *multipart.FileHeader, _ uint64, _ string) (string, error) { + return "", errors.New("not implemented in fake") +} +func (f *fakeFileService) SaveBytes(ctx context.Context, _ []byte, _ uint64, _ string, _ bool) (string, error) { + return "", errors.New("not implemented in fake") +} +func (f *fakeFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + fn, ok := f.readers[filePath] + if !ok { + return nil, errors.New("unknown path: " + filePath) + } + return fn() +} +func (f *fakeFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + // Return a URL that DuckDB would NOT be able to open on its own; the + // production code must *not* pass this through to DuckDB. + return "local://" + strings.TrimPrefix(filePath, "/"), nil +} +func (f *fakeFileService) DeleteFile(ctx context.Context, _ string) error { return nil } + +// TestMaterializeKnowledgeFile_HandlesLocalScheme is the regression guard +// for the dev-mode failure where DuckDB was handed a local:// URL it can't +// resolve. The tool must pull bytes via FileService.GetFile and hand DuckDB +// a concrete filesystem path with the right extension. +func TestMaterializeKnowledgeFile_HandlesLocalScheme(t *testing.T) { + payload := []byte("col1,col2\n1,2\n3,4\n") + + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "tenants/42/data.csv": func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(payload)), nil + }, + }, + } + + tool := &DataAnalysisTool{ + fileService: fs, + sessionID: "test-materialize", + } + + k := &types.Knowledge{ + ID: "k-abc", + FileType: "csv", + FilePath: "tenants/42/data.csv", + } + + path, cleanup, err := tool.materializeKnowledgeFile(context.Background(), k) + if err != nil { + t.Fatalf("materialize: %v", err) + } + t.Cleanup(cleanup) + + // Path must be a real filesystem path (no provider scheme) so DuckDB + // can st_read / read_xlsx / read_csv_auto it directly. + if strings.Contains(path, "://") { + t.Fatalf("expected bare filesystem path, got scheme URL: %q", path) + } + if filepath.Ext(path) != ".csv" { + t.Errorf("expected .csv suffix preserved for DuckDB format detection, got %q", path) + } + + // File must actually exist and have the exact bytes we fed in. + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read temp file: %v", err) + } + if !bytes.Equal(got, payload) { + t.Errorf("payload roundtrip mismatch: got %q, want %q", got, payload) + } + + // cleanup() must remove the temp file without error. + cleanup() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("expected temp file to be removed after cleanup, stat err=%v", err) + } + // Double-cleanup must be safe (test harness may invoke it again). + cleanup() +} + +func TestMaterializeKnowledgeFile_PropagatesGetFileError(t *testing.T) { + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "bad/path": func() (io.ReadCloser, error) { + return nil, errors.New("boom") + }, + }, + } + + tool := &DataAnalysisTool{fileService: fs, sessionID: "test-error"} + + _, cleanup, err := tool.materializeKnowledgeFile( + context.Background(), + &types.Knowledge{ID: "k-err", FileType: "xlsx", FilePath: "bad/path"}, + ) + if err == nil { + defer cleanup() + t.Fatal("expected error from underlying GetFile, got nil") + } + if !strings.Contains(err.Error(), "boom") { + t.Errorf("expected wrapped 'boom' error, got: %v", err) + } +} + +func TestMaterializeKnowledgeFile_PreservesExtension(t *testing.T) { + cases := []struct { + fileType string + wantExt string + }{ + {"xlsx", ".xlsx"}, + {"XLSX", ".xlsx"}, // normalized lowercase + {"csv", ".csv"}, + {"xls", ".xls"}, + } + for _, c := range cases { + t.Run(c.fileType, func(t *testing.T) { + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "p": func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte("x"))), nil + }, + }, + } + tool := &DataAnalysisTool{fileService: fs, sessionID: "ext"} + path, cleanup, err := tool.materializeKnowledgeFile( + context.Background(), + &types.Knowledge{ID: "k", FileType: c.fileType, FilePath: "p"}, + ) + if err != nil { + t.Fatalf("materialize: %v", err) + } + defer cleanup() + if filepath.Ext(path) != c.wantExt { + t.Errorf("fileType=%q: extension mismatch got %q want %q", c.fileType, filepath.Ext(path), c.wantExt) + } + }) + } +}