feat: implement automatic soft-delete filtering in database queries
- Added automatic filtering for `deleted_at IS NULL` in database queries to ensure only active records are returned. - Updated the `database_query.go` documentation to reflect the new soft-delete feature and its implications for query construction. - Enhanced the SQL validation process to include soft-delete conditions, improving data integrity and security. These changes enhance the robustness of database interactions by enforcing soft-delete logic across relevant queries.
This commit is contained in:
@@ -18,6 +18,7 @@ var databaseQueryTool = BaseTool{
|
||||
|
||||
## Security Features
|
||||
- Automatic tenant_id injection: All queries are automatically filtered by the logged-in user's tenant_id
|
||||
- Automatic soft-delete filtering: All queries are automatically filtered to include only records with deleted_at IS NULL
|
||||
- Read-only queries: Only SELECT statements are allowed
|
||||
- Safe tables: Only allow queries on authorized tables (knowledge_bases, knowledges, chunks)
|
||||
|
||||
@@ -30,7 +31,7 @@ var databaseQueryTool = BaseTool{
|
||||
- tenant_id (INTEGER): Owner tenant ID
|
||||
- embedding_model_id, summary_model_id, rerank_model_id (VARCHAR): Model IDs
|
||||
- vlm_config (JSON): Includes VLM settings such as enabled flag and model_id
|
||||
- created_at, updated_at (TIMESTAMP)
|
||||
- created_at, updated_at, deleted_at (TIMESTAMP)
|
||||
|
||||
### knowledges (documents)
|
||||
- id (VARCHAR): Document ID
|
||||
@@ -44,7 +45,7 @@ var databaseQueryTool = BaseTool{
|
||||
- enable_status (VARCHAR): Enable status (enabled/disabled)
|
||||
- file_name, file_type (VARCHAR): File information
|
||||
- file_size, storage_size (BIGINT): Size in bytes
|
||||
- created_at, updated_at, processed_at (TIMESTAMP)
|
||||
- created_at, updated_at, processed_at, deleted_at (TIMESTAMP)
|
||||
|
||||
|
||||
|
||||
@@ -57,7 +58,7 @@ var databaseQueryTool = BaseTool{
|
||||
- chunk_index (INTEGER): Index in document
|
||||
- is_enabled (BOOLEAN): Enable status
|
||||
- chunk_type (VARCHAR): Type (text/image/table)
|
||||
- created_at, updated_at (TIMESTAMP)
|
||||
- created_at, updated_at, deleted_at (TIMESTAMP)
|
||||
|
||||
## Usage Examples
|
||||
|
||||
@@ -88,6 +89,7 @@ Join knowledge bases and documents:
|
||||
|
||||
## Important Notes
|
||||
- DO NOT include tenant_id in WHERE clause - it's automatically added
|
||||
- DO NOT include deleted_at filtering manually unless needed - default query already enforces deleted_at IS NULL
|
||||
- Only SELECT queries are allowed
|
||||
- Limit results with LIMIT clause for better performance
|
||||
- Use appropriate JOINs when querying across tables
|
||||
@@ -257,6 +259,7 @@ func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint6
|
||||
securedSQL, validationResult, err := utils.ValidateAndSecureSQL(
|
||||
sqlQuery,
|
||||
utils.WithSecurityDefaults(tenantID),
|
||||
utils.WithSoftDeleteFilter("knowledge_bases", "knowledges", "chunks"),
|
||||
utils.WithInjectionRiskCheck(),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
+71
-15
@@ -132,6 +132,10 @@ type sqlValidator struct {
|
||||
enableTenantInjection bool
|
||||
tenantID uint64
|
||||
tablesWithTenantID map[string]bool
|
||||
|
||||
// Soft delete filtering
|
||||
enableSoftDeleteInjection bool
|
||||
tablesWithDeletedAt map[string]bool
|
||||
}
|
||||
|
||||
// ParseSQL parses a SQL statement using pg_query_go and extracts table names, select fields, and where fields
|
||||
@@ -574,6 +578,26 @@ func WithTenantIsolation(tenantID uint64, tables ...string) SQLValidationOption
|
||||
}
|
||||
}
|
||||
|
||||
// WithSoftDeleteFilter enables automatic deleted_at IS NULL injection.
|
||||
func WithSoftDeleteFilter(tables ...string) SQLValidationOption {
|
||||
return func(v *sqlValidator) {
|
||||
v.enableSoftDeleteInjection = true
|
||||
v.tablesWithDeletedAt = make(map[string]bool)
|
||||
if len(tables) == 0 {
|
||||
// Default tables with soft-delete support.
|
||||
v.tablesWithDeletedAt = map[string]bool{
|
||||
"knowledge_bases": true,
|
||||
"knowledges": true,
|
||||
"chunks": true,
|
||||
}
|
||||
} else {
|
||||
for _, table := range tables {
|
||||
v.tablesWithDeletedAt[strings.ToLower(table)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithSecurityDefaults applies a comprehensive set of security validations
|
||||
func WithSecurityDefaults(tenantID uint64) SQLValidationOption {
|
||||
return func(v *sqlValidator) {
|
||||
@@ -608,6 +632,7 @@ func ValidateSQL(sql string, opts ...SQLValidationOption) (*SQLParseResult, *SQL
|
||||
allowedTables: make(map[string]bool),
|
||||
allowedFunctions: make(map[string]bool),
|
||||
tablesWithTenantID: make(map[string]bool),
|
||||
tablesWithDeletedAt: make(map[string]bool),
|
||||
minLength: 6,
|
||||
maxLength: 4096,
|
||||
}
|
||||
@@ -772,13 +797,14 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
|
||||
// Find validator config to check if tenant injection is enabled
|
||||
validator := &sqlValidator{
|
||||
tablesWithTenantID: make(map[string]bool),
|
||||
tablesWithDeletedAt: make(map[string]bool),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(validator)
|
||||
}
|
||||
|
||||
// If tenant injection is not enabled, return original SQL
|
||||
if !validator.enableTenantInjection {
|
||||
// If no SQL rewriting is enabled, return original SQL
|
||||
if !validator.enableTenantInjection && !validator.enableSoftDeleteInjection {
|
||||
return sql, validationResult, nil
|
||||
}
|
||||
|
||||
@@ -802,10 +828,37 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
|
||||
|
||||
// Inject tenant conditions
|
||||
securedSQL := validator.injectTenantConditions(normalizedSQL, tablesInQuery)
|
||||
// Inject deleted_at IS NULL conditions
|
||||
securedSQL = validator.injectSoftDeleteConditions(securedSQL, tablesInQuery)
|
||||
|
||||
return securedSQL, validationResult, nil
|
||||
}
|
||||
|
||||
// InjectAndConditions injects filter conditions into a SQL statement using AND semantics.
|
||||
// If WHERE exists, the original WHERE predicates will be wrapped in parentheses.
|
||||
func InjectAndConditions(sql, filter string) string {
|
||||
filter = strings.TrimSpace(filter)
|
||||
if filter == "" {
|
||||
return sql
|
||||
}
|
||||
|
||||
// Check if WHERE clause exists
|
||||
wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`)
|
||||
if wherePattern.MatchString(sql) {
|
||||
// Add filter and wrap existing conditions in parentheses to prevent OR precedence issues
|
||||
return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", filter)) + ")"
|
||||
}
|
||||
|
||||
// Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc.
|
||||
clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`)
|
||||
if loc := clausePattern.FindStringIndex(sql); loc != nil {
|
||||
return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", filter) + sql[loc[0]:]
|
||||
}
|
||||
|
||||
// Add WHERE clause at the end
|
||||
return fmt.Sprintf("%s WHERE %s", sql, filter)
|
||||
}
|
||||
|
||||
// injectTenantConditions adds tenant_id filtering to the query
|
||||
func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[string]string) string {
|
||||
if !v.enableTenantInjection {
|
||||
@@ -829,24 +882,27 @@ func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[stri
|
||||
}
|
||||
|
||||
tenantFilter := strings.Join(conditions, " AND ")
|
||||
return InjectAndConditions(sql, tenantFilter)
|
||||
}
|
||||
|
||||
// Check if WHERE clause exists
|
||||
wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`)
|
||||
if wherePattern.MatchString(sql) {
|
||||
// Add tenant filter and wrap existing conditions in parentheses to prevent OR injection
|
||||
// This ensures: WHERE tenant_filter AND (original_conditions)
|
||||
// Instead of: WHERE tenant_filter AND original_conditions OR malicious_condition
|
||||
return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", tenantFilter)) + ")"
|
||||
// injectSoftDeleteConditions adds deleted_at IS NULL filtering to the query.
|
||||
func (v *sqlValidator) injectSoftDeleteConditions(sql string, tablesInQuery map[string]string) string {
|
||||
if !v.enableSoftDeleteInjection {
|
||||
return sql
|
||||
}
|
||||
|
||||
// Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc.
|
||||
clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`)
|
||||
if loc := clausePattern.FindStringIndex(sql); loc != nil {
|
||||
return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", tenantFilter) + sql[loc[0]:]
|
||||
var conditions []string
|
||||
for tableName, alias := range tablesInQuery {
|
||||
if v.tablesWithDeletedAt[tableName] {
|
||||
conditions = append(conditions, fmt.Sprintf("%s.deleted_at IS NULL", alias))
|
||||
}
|
||||
}
|
||||
|
||||
// Add WHERE clause at the end
|
||||
return fmt.Sprintf("%s WHERE %s", sql, tenantFilter)
|
||||
if len(conditions) == 0 {
|
||||
return sql
|
||||
}
|
||||
|
||||
return InjectAndConditions(sql, strings.Join(conditions, " AND "))
|
||||
}
|
||||
|
||||
// checkSQLInjectionRisks checks for common SQL injection patterns in WHERE clause
|
||||
|
||||
Reference in New Issue
Block a user