diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go index 973c9de5..cda1dc2f 100644 --- a/internal/agent/tools/data_analysis.go +++ b/internal/agent/tools/data_analysis.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/json" "fmt" + "io" + "os" "strings" "github.com/Tencent/WeKnora/internal/logger" @@ -439,8 +441,14 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s ) } -// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema -// It automatically determines the file type and calls the appropriate loading method +// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema. +// It automatically determines the file type and calls the appropriate loading method. +// +// The source file is first materialized to a local temp file via FileService.GetFile +// so DuckDB's st_read / read_xlsx / read_csv_auto can open it directly. This +// side-steps provider-specific URL schemes (e.g. the local:// URL returned by +// the local file service) that DuckDB's extensions cannot resolve on their own. +// // Parameters: // - ctx: context for cancellation and timeout // - knowledge: the Knowledge entity containing file information @@ -460,16 +468,17 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ logger.Infof(ctx, "[Tool][DataAnalysis] Loading knowledge '%s' (type: %s) into table '%s' for session %s", knowledge.ID, fileType, tableName, t.sessionID) - fileURL, err := t.fileService.GetFileURL(ctx, knowledge.FilePath) + localPath, cleanup, err := t.materializeKnowledgeFile(ctx, knowledge) if err != nil { - return nil, fmt.Errorf("failed to get file URL for knowledge '%s': %w", knowledge.ID, err) + return nil, fmt.Errorf("failed to materialize knowledge '%s' for DuckDB: %w", knowledge.ID, err) } + defer cleanup() switch fileType { case "csv": - return t.LoadFromCSV(ctx, fileURL, tableName) + return t.LoadFromCSV(ctx, localPath, tableName) case "xlsx", "xls": - return t.LoadFromExcel(ctx, fileURL, tableName) + return t.LoadFromExcel(ctx, localPath, tableName) default: logger.Warnf(ctx, "[Tool][DataAnalysis] Unsupported file type '%s' for knowledge '%s' in session %s", fileType, knowledge.ID, t.sessionID) @@ -477,6 +486,59 @@ func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *typ } } +// materializeKnowledgeFile copies the knowledge's backing blob into a fresh +// temp file on the local filesystem so DuckDB can open it with ordinary path +// semantics. It returns the temp path and a cleanup closure that removes the +// temp file; the closure is always safe to call and is a no-op on failure. +// +// This hides storage-backend-specific URL schemes (local://, oss://, s3://, +// minio://, cos://, …) behind the FileService.GetFile abstraction, so the +// Data Analysis tool works identically across all deployments. +func (t *DataAnalysisTool) materializeKnowledgeFile(ctx context.Context, knowledge *types.Knowledge) (string, func(), error) { + noop := func() {} + + reader, err := t.fileService.GetFile(ctx, knowledge.FilePath) + if err != nil { + return "", noop, fmt.Errorf("failed to open file for knowledge '%s': %w", knowledge.ID, err) + } + defer reader.Close() + + // Preserve the file extension so DuckDB's format auto-detection still + // works (e.g. the CSV reader expects .csv, xlsx reader expects .xlsx). + suffix := "" + if ext := strings.ToLower(strings.TrimSpace(knowledge.FileType)); ext != "" { + suffix = "." + ext + } + + tmp, err := os.CreateTemp("", "weknora-data-analysis-*"+suffix) + if err != nil { + return "", noop, fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := tmp.Name() + cleanup := func() { + // Best-effort cleanup; a missing file is fine, any other error is + // only logged to avoid masking the original operation's result. + if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { + logger.Warnf(ctx, "[Tool][DataAnalysis] Failed to remove temp file %s: %v", tmpPath, err) + } + } + + if _, err := io.Copy(tmp, reader); err != nil { + _ = tmp.Close() + cleanup() + return "", noop, fmt.Errorf("failed to copy knowledge '%s' to temp file: %w", knowledge.ID, err) + } + if err := tmp.Close(); err != nil { + cleanup() + return "", noop, fmt.Errorf("failed to finalize temp file for knowledge '%s': %w", knowledge.ID, err) + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Materialized knowledge '%s' to temp file %s for session %s", + knowledge.ID, tmpPath, t.sessionID) + + return tmpPath, cleanup, nil +} + // LoadFromKnowledgeID loads data from a Knowledge ID into a DuckDB table and returns the table schema // Parameters: // - ctx: context for cancellation and timeout diff --git a/internal/agent/tools/data_analysis_materialize_test.go b/internal/agent/tools/data_analysis_materialize_test.go new file mode 100644 index 00000000..552fd87d --- /dev/null +++ b/internal/agent/tools/data_analysis_materialize_test.go @@ -0,0 +1,161 @@ +package tools + +import ( + "bytes" + "context" + "errors" + "io" + "mime/multipart" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// fakeFileService implements interfaces.FileService just enough to drive +// materializeKnowledgeFile. Every method we don't care about simply errors +// out so accidental usage is loud. +type fakeFileService struct { + readers map[string]func() (io.ReadCloser, error) +} + +func (f *fakeFileService) CheckConnectivity(ctx context.Context) error { return nil } +func (f *fakeFileService) SaveFile(ctx context.Context, _ *multipart.FileHeader, _ uint64, _ string) (string, error) { + return "", errors.New("not implemented in fake") +} +func (f *fakeFileService) SaveBytes(ctx context.Context, _ []byte, _ uint64, _ string, _ bool) (string, error) { + return "", errors.New("not implemented in fake") +} +func (f *fakeFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + fn, ok := f.readers[filePath] + if !ok { + return nil, errors.New("unknown path: " + filePath) + } + return fn() +} +func (f *fakeFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + // Return a URL that DuckDB would NOT be able to open on its own; the + // production code must *not* pass this through to DuckDB. + return "local://" + strings.TrimPrefix(filePath, "/"), nil +} +func (f *fakeFileService) DeleteFile(ctx context.Context, _ string) error { return nil } + +// TestMaterializeKnowledgeFile_HandlesLocalScheme is the regression guard +// for the dev-mode failure where DuckDB was handed a local:// URL it can't +// resolve. The tool must pull bytes via FileService.GetFile and hand DuckDB +// a concrete filesystem path with the right extension. +func TestMaterializeKnowledgeFile_HandlesLocalScheme(t *testing.T) { + payload := []byte("col1,col2\n1,2\n3,4\n") + + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "tenants/42/data.csv": func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(payload)), nil + }, + }, + } + + tool := &DataAnalysisTool{ + fileService: fs, + sessionID: "test-materialize", + } + + k := &types.Knowledge{ + ID: "k-abc", + FileType: "csv", + FilePath: "tenants/42/data.csv", + } + + path, cleanup, err := tool.materializeKnowledgeFile(context.Background(), k) + if err != nil { + t.Fatalf("materialize: %v", err) + } + t.Cleanup(cleanup) + + // Path must be a real filesystem path (no provider scheme) so DuckDB + // can st_read / read_xlsx / read_csv_auto it directly. + if strings.Contains(path, "://") { + t.Fatalf("expected bare filesystem path, got scheme URL: %q", path) + } + if filepath.Ext(path) != ".csv" { + t.Errorf("expected .csv suffix preserved for DuckDB format detection, got %q", path) + } + + // File must actually exist and have the exact bytes we fed in. + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read temp file: %v", err) + } + if !bytes.Equal(got, payload) { + t.Errorf("payload roundtrip mismatch: got %q, want %q", got, payload) + } + + // cleanup() must remove the temp file without error. + cleanup() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("expected temp file to be removed after cleanup, stat err=%v", err) + } + // Double-cleanup must be safe (test harness may invoke it again). + cleanup() +} + +func TestMaterializeKnowledgeFile_PropagatesGetFileError(t *testing.T) { + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "bad/path": func() (io.ReadCloser, error) { + return nil, errors.New("boom") + }, + }, + } + + tool := &DataAnalysisTool{fileService: fs, sessionID: "test-error"} + + _, cleanup, err := tool.materializeKnowledgeFile( + context.Background(), + &types.Knowledge{ID: "k-err", FileType: "xlsx", FilePath: "bad/path"}, + ) + if err == nil { + defer cleanup() + t.Fatal("expected error from underlying GetFile, got nil") + } + if !strings.Contains(err.Error(), "boom") { + t.Errorf("expected wrapped 'boom' error, got: %v", err) + } +} + +func TestMaterializeKnowledgeFile_PreservesExtension(t *testing.T) { + cases := []struct { + fileType string + wantExt string + }{ + {"xlsx", ".xlsx"}, + {"XLSX", ".xlsx"}, // normalized lowercase + {"csv", ".csv"}, + {"xls", ".xls"}, + } + for _, c := range cases { + t.Run(c.fileType, func(t *testing.T) { + fs := &fakeFileService{ + readers: map[string]func() (io.ReadCloser, error){ + "p": func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte("x"))), nil + }, + }, + } + tool := &DataAnalysisTool{fileService: fs, sessionID: "ext"} + path, cleanup, err := tool.materializeKnowledgeFile( + context.Background(), + &types.Knowledge{ID: "k", FileType: c.fileType, FilePath: "p"}, + ) + if err != nil { + t.Fatalf("materialize: %v", err) + } + defer cleanup() + if filepath.Ext(path) != c.wantExt { + t.Errorf("fileType=%q: extension mismatch got %q want %q", c.fileType, filepath.Ext(path), c.wantExt) + } + }) + } +}