From 535aec87e3b017b9e0213dfec4d05d114bdb5f1a Mon Sep 17 00:00:00 2001 From: hjz <1141520944@qq.com> Date: Mon, 27 Apr 2026 16:05:01 +0800 Subject: [PATCH] feat: data_analysis sql validation & type processing --- internal/agent/tools/data_analysis.go | 91 +++++++++++++++++++++- internal/agent/tools/data_analysis_test.go | 6 +- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go index 58cffd99..0ca1b2ea 100644 --- a/internal/agent/tools/data_analysis.go +++ b/internal/agent/tools/data_analysis.go @@ -8,6 +8,7 @@ import ( filesvc "github.com/Tencent/WeKnora/internal/application/service/file" "io" "os" + "regexp" "strings" "github.com/Tencent/WeKnora/internal/logger" @@ -34,6 +35,73 @@ func sqlSingleQuoteEscape(s string) string { return strings.ReplaceAll(s, "'", "''") } +func normalizeIdentifierForMatch(s string) string { + normalized := strings.ToLower(strings.TrimSpace(s)) + normalized = strings.ReplaceAll(normalized, " ", "") + normalized = strings.ReplaceAll(normalized, "\u3000", "") + return normalized +} + +func reconcileSQLColumnsWithSchema(sqlText string, schema *TableSchema) (string, []string) { + if schema == nil || len(schema.Columns) == 0 { + return sqlText, nil + } + + normalizedToCanonical := make(map[string]string, len(schema.Columns)) + for _, col := range schema.Columns { + key := normalizeIdentifierForMatch(col.Name) + if key == "" { + continue + } + if _, exists := normalizedToCanonical[key]; !exists { + normalizedToCanonical[key] = col.Name + } + } + + quotedIdentifierPattern := regexp.MustCompile(`"([^"]+)"`) + fixes := make([]string, 0) + rewritten := quotedIdentifierPattern.ReplaceAllStringFunc(sqlText, func(token string) string { + name := strings.Trim(token, "\"") + canonical, ok := normalizedToCanonical[normalizeIdentifierForMatch(name)] + if !ok || canonical == name { + return token + } + fixes = append(fixes, fmt.Sprintf("%q -> %q", name, canonical)) + return fmt.Sprintf(`"%s"`, canonical) + }) + + return rewritten, fixes +} + +func buildMissingColumnSuggestion(sqlErr error, schema *TableSchema) string { + if sqlErr == nil || schema == nil { + return "" + } + msg := sqlErr.Error() + if !strings.Contains(msg, `Referenced column "`) || !strings.Contains(msg, `not found`) { + return "" + } + + matches := regexp.MustCompile(`Referenced column "([^"]+)" not found`).FindStringSubmatch(msg) + if len(matches) < 2 { + return "" + } + + missing := matches[1] + normalizedMissing := normalizeIdentifierForMatch(missing) + if normalizedMissing == "" { + return "" + } + + for _, col := range schema.Columns { + if normalizeIdentifierForMatch(col.Name) == normalizedMissing { + return fmt.Sprintf("Column %q does not exist. Did you mean %q? Please use the exact column name from schema.", missing, col.Name) + } + } + + return "" +} + type DataAnalysisInput struct { KnowledgeID string `json:"knowledge_id" jsonschema:"id of the knowledge to query"` Sql string `json:"sql" jsonschema:"SQL to be executed on knowledge"` @@ -127,6 +195,10 @@ func (t *DataAnalysisTool) Execute(ctx context.Context, args json.RawMessage) (* // Replace knowledge ID with table name input.Sql = strings.ReplaceAll(input.Sql, input.KnowledgeID, schema.TableName) + if rewrittenSQL, fixes := reconcileSQLColumnsWithSchema(input.Sql, schema); len(fixes) > 0 { + logger.Infof(ctx, "[Tool][DataAnalysis] Auto-rewrote SQL identifiers for session %s: %v", t.sessionID, fixes) + input.Sql = rewrittenSQL + } // Check if this is a read-only query normalizedSQL := strings.TrimSpace(strings.ToLower(input.Sql)) @@ -164,6 +236,12 @@ func (t *DataAnalysisTool) Execute(ctx context.Context, args json.RawMessage) (* // Execute single query and get results results, err := t.executeSingleQuery(ctx, input.Sql) if err != nil { + if suggestion := buildMissingColumnSuggestion(err, schema); suggestion != "" { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Query execution failed: %v. %s", err, suggestion), + }, err + } return &types.ToolResult{ Success: false, Error: fmt.Sprintf("Query execution failed: %v", err), @@ -305,8 +383,13 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab // Record the created table for cleanup. If already exists, skip creation if t.recordCreatedTable(tableName) { // Create table from CSV using DuckDB's read_csv_auto function + // with explicit header detection and VARCHAR coercion to align with + // Excel loading behavior. // Table will be created in the session schema - createTableSQL := fmt.Sprintf("CREATE TABLE \"%s\" AS SELECT * FROM read_csv_auto('%s')", tableName, filename) + createTableSQL := fmt.Sprintf( + "CREATE TABLE \"%s\" AS SELECT * FROM read_csv_auto('%s', header=true, all_varchar=true)", + tableName, sqlSingleQuoteEscape(filename), + ) _, err := t.db.ExecContext(ctx, createTableSQL) if err != nil { @@ -415,7 +498,7 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s // No sheet info (enumeration failed or empty): read the first sheet only. if len(sheetNames) == 0 { return fmt.Sprintf( - "CREATE TABLE \"%s\" AS SELECT * FROM read_xlsx('%s')", + "CREATE TABLE \"%s\" AS SELECT * FROM read_xlsx('%s', header=true, all_varchar=true)", tableName, escFile, ) } @@ -425,7 +508,7 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s if len(sheetNames) == 1 { escSheet := sqlSingleQuoteEscape(sheetNames[0]) return fmt.Sprintf( - "CREATE TABLE \"%s\" AS SELECT *, '%s' AS %s FROM read_xlsx('%s', sheet = '%s')", + "CREATE TABLE \"%s\" AS SELECT *, '%s' AS %s FROM read_xlsx('%s', sheet = '%s', header=true, all_varchar=true)", tableName, escSheet, excelSheetNameColumn, escFile, escSheet, ) } @@ -437,7 +520,7 @@ func buildExcelCreateTableSQL(tableName, filename string, sheetNames []string) s for _, sheet := range sheetNames { escSheet := sqlSingleQuoteEscape(sheet) parts = append(parts, fmt.Sprintf( - "SELECT *, '%s' AS %s FROM read_xlsx('%s', sheet = '%s')", + "SELECT *, '%s' AS %s FROM read_xlsx('%s', sheet = '%s', header=true, all_varchar=true)", escSheet, excelSheetNameColumn, escFile, escSheet, )) } diff --git a/internal/agent/tools/data_analysis_test.go b/internal/agent/tools/data_analysis_test.go index 8285bdb8..3d46b37a 100644 --- a/internal/agent/tools/data_analysis_test.go +++ b/internal/agent/tools/data_analysis_test.go @@ -7,7 +7,7 @@ import ( func TestBuildExcelCreateTableSQL_NoSheets(t *testing.T) { got := buildExcelCreateTableSQL("tbl", "/tmp/data.xlsx", nil) - want := `CREATE TABLE "tbl" AS SELECT * FROM read_xlsx('/tmp/data.xlsx')` + want := `CREATE TABLE "tbl" AS SELECT * FROM read_xlsx('/tmp/data.xlsx', header=true, all_varchar=true)` if got != want { t.Fatalf("mismatch.\n got: %s\nwant: %s", got, want) } @@ -17,7 +17,7 @@ func TestBuildExcelCreateTableSQL_SingleSheetTagsSource(t *testing.T) { got := buildExcelCreateTableSQL("tbl", "/tmp/data.xlsx", []string{"Sheet1"}) // Must use read_xlsx (excel extension) with explicit sheet param. - if !strings.Contains(got, "FROM read_xlsx('/tmp/data.xlsx', sheet = 'Sheet1')") { + if !strings.Contains(got, "FROM read_xlsx('/tmp/data.xlsx', sheet = 'Sheet1', header=true, all_varchar=true)") { t.Fatalf("expected read_xlsx with sheet param, got: %s", got) } // Must tag the source sheet name via the synthetic column so downstream @@ -33,7 +33,7 @@ func TestBuildExcelCreateTableSQL_MultiSheetUsesUnionAllByName(t *testing.T) { // Each sheet must appear as a SELECT reading that specific sheet, and // the __sheet_name column must carry its name for per-sheet filtering. for _, sheet := range []string{"Sheet1", "Sheet2", "报表"} { - needleRead := "FROM read_xlsx('/tmp/data.xlsx', sheet = '" + sheet + "')" + needleRead := "FROM read_xlsx('/tmp/data.xlsx', sheet = '" + sheet + "', header=true, all_varchar=true)" needleTag := "'" + sheet + "' AS " + excelSheetNameColumn if !strings.Contains(got, needleRead) { t.Fatalf("missing read_xlsx for sheet %q in:\n%s", sheet, got)