From 58783ec96c12afb84127021228d67c19c3ed04db Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 3 Mar 2026 22:10:46 +0800 Subject: [PATCH] feat: implement automatic soft-delete filtering in database queries - Added automatic filtering for `deleted_at IS NULL` in database queries to ensure only active records are returned. - Updated the `database_query.go` documentation to reflect the new soft-delete feature and its implications for query construction. - Enhanced the SQL validation process to include soft-delete conditions, improving data integrity and security. These changes enhance the robustness of database interactions by enforcing soft-delete logic across relevant queries. --- internal/agent/tools/database_query.go | 9 ++- internal/utils/inject.go | 86 +++++++++++++++++++++----- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/internal/agent/tools/database_query.go b/internal/agent/tools/database_query.go index 90ec73f2..3264e51b 100644 --- a/internal/agent/tools/database_query.go +++ b/internal/agent/tools/database_query.go @@ -18,6 +18,7 @@ var databaseQueryTool = BaseTool{ ## Security Features - Automatic tenant_id injection: All queries are automatically filtered by the logged-in user's tenant_id +- Automatic soft-delete filtering: All queries are automatically filtered to include only records with deleted_at IS NULL - Read-only queries: Only SELECT statements are allowed - Safe tables: Only allow queries on authorized tables (knowledge_bases, knowledges, chunks) @@ -30,7 +31,7 @@ var databaseQueryTool = BaseTool{ - tenant_id (INTEGER): Owner tenant ID - embedding_model_id, summary_model_id, rerank_model_id (VARCHAR): Model IDs - vlm_config (JSON): Includes VLM settings such as enabled flag and model_id -- created_at, updated_at (TIMESTAMP) +- created_at, updated_at, deleted_at (TIMESTAMP) ### knowledges (documents) - id (VARCHAR): Document ID @@ -44,7 +45,7 @@ var databaseQueryTool = BaseTool{ - enable_status (VARCHAR): Enable status (enabled/disabled) - file_name, file_type (VARCHAR): File information - file_size, storage_size (BIGINT): Size in bytes -- created_at, updated_at, processed_at (TIMESTAMP) +- created_at, updated_at, processed_at, deleted_at (TIMESTAMP) @@ -57,7 +58,7 @@ var databaseQueryTool = BaseTool{ - chunk_index (INTEGER): Index in document - is_enabled (BOOLEAN): Enable status - chunk_type (VARCHAR): Type (text/image/table) -- created_at, updated_at (TIMESTAMP) +- created_at, updated_at, deleted_at (TIMESTAMP) ## Usage Examples @@ -88,6 +89,7 @@ Join knowledge bases and documents: ## Important Notes - DO NOT include tenant_id in WHERE clause - it's automatically added +- DO NOT include deleted_at filtering manually unless needed - default query already enforces deleted_at IS NULL - Only SELECT queries are allowed - Limit results with LIMIT clause for better performance - Use appropriate JOINs when querying across tables @@ -257,6 +259,7 @@ func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint6 securedSQL, validationResult, err := utils.ValidateAndSecureSQL( sqlQuery, utils.WithSecurityDefaults(tenantID), + utils.WithSoftDeleteFilter("knowledge_bases", "knowledges", "chunks"), utils.WithInjectionRiskCheck(), ) if err != nil { diff --git a/internal/utils/inject.go b/internal/utils/inject.go index b106049d..2e53e726 100644 --- a/internal/utils/inject.go +++ b/internal/utils/inject.go @@ -132,6 +132,10 @@ type sqlValidator struct { enableTenantInjection bool tenantID uint64 tablesWithTenantID map[string]bool + + // Soft delete filtering + enableSoftDeleteInjection bool + tablesWithDeletedAt map[string]bool } // ParseSQL parses a SQL statement using pg_query_go and extracts table names, select fields, and where fields @@ -574,6 +578,26 @@ func WithTenantIsolation(tenantID uint64, tables ...string) SQLValidationOption } } +// WithSoftDeleteFilter enables automatic deleted_at IS NULL injection. +func WithSoftDeleteFilter(tables ...string) SQLValidationOption { + return func(v *sqlValidator) { + v.enableSoftDeleteInjection = true + v.tablesWithDeletedAt = make(map[string]bool) + if len(tables) == 0 { + // Default tables with soft-delete support. + v.tablesWithDeletedAt = map[string]bool{ + "knowledge_bases": true, + "knowledges": true, + "chunks": true, + } + } else { + for _, table := range tables { + v.tablesWithDeletedAt[strings.ToLower(table)] = true + } + } + } +} + // WithSecurityDefaults applies a comprehensive set of security validations func WithSecurityDefaults(tenantID uint64) SQLValidationOption { return func(v *sqlValidator) { @@ -608,6 +632,7 @@ func ValidateSQL(sql string, opts ...SQLValidationOption) (*SQLParseResult, *SQL allowedTables: make(map[string]bool), allowedFunctions: make(map[string]bool), tablesWithTenantID: make(map[string]bool), + tablesWithDeletedAt: make(map[string]bool), minLength: 6, maxLength: 4096, } @@ -772,13 +797,14 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL // Find validator config to check if tenant injection is enabled validator := &sqlValidator{ tablesWithTenantID: make(map[string]bool), + tablesWithDeletedAt: make(map[string]bool), } for _, opt := range opts { opt(validator) } - // If tenant injection is not enabled, return original SQL - if !validator.enableTenantInjection { + // If no SQL rewriting is enabled, return original SQL + if !validator.enableTenantInjection && !validator.enableSoftDeleteInjection { return sql, validationResult, nil } @@ -802,10 +828,37 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL // Inject tenant conditions securedSQL := validator.injectTenantConditions(normalizedSQL, tablesInQuery) + // Inject deleted_at IS NULL conditions + securedSQL = validator.injectSoftDeleteConditions(securedSQL, tablesInQuery) return securedSQL, validationResult, nil } +// InjectAndConditions injects filter conditions into a SQL statement using AND semantics. +// If WHERE exists, the original WHERE predicates will be wrapped in parentheses. +func InjectAndConditions(sql, filter string) string { + filter = strings.TrimSpace(filter) + if filter == "" { + return sql + } + + // Check if WHERE clause exists + wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`) + if wherePattern.MatchString(sql) { + // Add filter and wrap existing conditions in parentheses to prevent OR precedence issues + return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", filter)) + ")" + } + + // Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc. + clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`) + if loc := clausePattern.FindStringIndex(sql); loc != nil { + return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", filter) + sql[loc[0]:] + } + + // Add WHERE clause at the end + return fmt.Sprintf("%s WHERE %s", sql, filter) +} + // injectTenantConditions adds tenant_id filtering to the query func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[string]string) string { if !v.enableTenantInjection { @@ -829,24 +882,27 @@ func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[stri } tenantFilter := strings.Join(conditions, " AND ") + return InjectAndConditions(sql, tenantFilter) +} - // Check if WHERE clause exists - wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`) - if wherePattern.MatchString(sql) { - // Add tenant filter and wrap existing conditions in parentheses to prevent OR injection - // This ensures: WHERE tenant_filter AND (original_conditions) - // Instead of: WHERE tenant_filter AND original_conditions OR malicious_condition - return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", tenantFilter)) + ")" +// injectSoftDeleteConditions adds deleted_at IS NULL filtering to the query. +func (v *sqlValidator) injectSoftDeleteConditions(sql string, tablesInQuery map[string]string) string { + if !v.enableSoftDeleteInjection { + return sql } - // Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc. - clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`) - if loc := clausePattern.FindStringIndex(sql); loc != nil { - return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", tenantFilter) + sql[loc[0]:] + var conditions []string + for tableName, alias := range tablesInQuery { + if v.tablesWithDeletedAt[tableName] { + conditions = append(conditions, fmt.Sprintf("%s.deleted_at IS NULL", alias)) + } } - // Add WHERE clause at the end - return fmt.Sprintf("%s WHERE %s", sql, tenantFilter) + if len(conditions) == 0 { + return sql + } + + return InjectAndConditions(sql, strings.Join(conditions, " AND ")) } // checkSQLInjectionRisks checks for common SQL injection patterns in WHERE clause