feat: implement automatic soft-delete filtering in database queries

- Added automatic filtering for `deleted_at IS NULL` in database queries to ensure only active records are returned.
- Updated the `database_query.go` documentation to reflect the new soft-delete feature and its implications for query construction.
- Enhanced the SQL validation process to include soft-delete conditions, improving data integrity and security.

These changes enhance the robustness of database interactions by enforcing soft-delete logic across relevant queries.
This commit is contained in:
wizardchen
2026-03-03 22:10:46 +08:00
committed by lyingbug
parent e2234e4960
commit 58783ec96c
2 changed files with 77 additions and 18 deletions
+6 -3
View File
@@ -18,6 +18,7 @@ var databaseQueryTool = BaseTool{
## Security Features
- Automatic tenant_id injection: All queries are automatically filtered by the logged-in user's tenant_id
- Automatic soft-delete filtering: All queries are automatically filtered to include only records with deleted_at IS NULL
- Read-only queries: Only SELECT statements are allowed
- Safe tables: Only allow queries on authorized tables (knowledge_bases, knowledges, chunks)
@@ -30,7 +31,7 @@ var databaseQueryTool = BaseTool{
- tenant_id (INTEGER): Owner tenant ID
- embedding_model_id, summary_model_id, rerank_model_id (VARCHAR): Model IDs
- vlm_config (JSON): Includes VLM settings such as enabled flag and model_id
- created_at, updated_at (TIMESTAMP)
- created_at, updated_at, deleted_at (TIMESTAMP)
### knowledges (documents)
- id (VARCHAR): Document ID
@@ -44,7 +45,7 @@ var databaseQueryTool = BaseTool{
- enable_status (VARCHAR): Enable status (enabled/disabled)
- file_name, file_type (VARCHAR): File information
- file_size, storage_size (BIGINT): Size in bytes
- created_at, updated_at, processed_at (TIMESTAMP)
- created_at, updated_at, processed_at, deleted_at (TIMESTAMP)
@@ -57,7 +58,7 @@ var databaseQueryTool = BaseTool{
- chunk_index (INTEGER): Index in document
- is_enabled (BOOLEAN): Enable status
- chunk_type (VARCHAR): Type (text/image/table)
- created_at, updated_at (TIMESTAMP)
- created_at, updated_at, deleted_at (TIMESTAMP)
## Usage Examples
@@ -88,6 +89,7 @@ Join knowledge bases and documents:
## Important Notes
- DO NOT include tenant_id in WHERE clause - it's automatically added
- DO NOT include deleted_at filtering manually unless needed - default query already enforces deleted_at IS NULL
- Only SELECT queries are allowed
- Limit results with LIMIT clause for better performance
- Use appropriate JOINs when querying across tables
@@ -257,6 +259,7 @@ func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint6
securedSQL, validationResult, err := utils.ValidateAndSecureSQL(
sqlQuery,
utils.WithSecurityDefaults(tenantID),
utils.WithSoftDeleteFilter("knowledge_bases", "knowledges", "chunks"),
utils.WithInjectionRiskCheck(),
)
if err != nil {
+71 -15
View File
@@ -132,6 +132,10 @@ type sqlValidator struct {
enableTenantInjection bool
tenantID uint64
tablesWithTenantID map[string]bool
// Soft delete filtering
enableSoftDeleteInjection bool
tablesWithDeletedAt map[string]bool
}
// ParseSQL parses a SQL statement using pg_query_go and extracts table names, select fields, and where fields
@@ -574,6 +578,26 @@ func WithTenantIsolation(tenantID uint64, tables ...string) SQLValidationOption
}
}
// WithSoftDeleteFilter enables automatic deleted_at IS NULL injection.
func WithSoftDeleteFilter(tables ...string) SQLValidationOption {
return func(v *sqlValidator) {
v.enableSoftDeleteInjection = true
v.tablesWithDeletedAt = make(map[string]bool)
if len(tables) == 0 {
// Default tables with soft-delete support.
v.tablesWithDeletedAt = map[string]bool{
"knowledge_bases": true,
"knowledges": true,
"chunks": true,
}
} else {
for _, table := range tables {
v.tablesWithDeletedAt[strings.ToLower(table)] = true
}
}
}
}
// WithSecurityDefaults applies a comprehensive set of security validations
func WithSecurityDefaults(tenantID uint64) SQLValidationOption {
return func(v *sqlValidator) {
@@ -608,6 +632,7 @@ func ValidateSQL(sql string, opts ...SQLValidationOption) (*SQLParseResult, *SQL
allowedTables: make(map[string]bool),
allowedFunctions: make(map[string]bool),
tablesWithTenantID: make(map[string]bool),
tablesWithDeletedAt: make(map[string]bool),
minLength: 6,
maxLength: 4096,
}
@@ -772,13 +797,14 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
// Find validator config to check if tenant injection is enabled
validator := &sqlValidator{
tablesWithTenantID: make(map[string]bool),
tablesWithDeletedAt: make(map[string]bool),
}
for _, opt := range opts {
opt(validator)
}
// If tenant injection is not enabled, return original SQL
if !validator.enableTenantInjection {
// If no SQL rewriting is enabled, return original SQL
if !validator.enableTenantInjection && !validator.enableSoftDeleteInjection {
return sql, validationResult, nil
}
@@ -802,10 +828,37 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
// Inject tenant conditions
securedSQL := validator.injectTenantConditions(normalizedSQL, tablesInQuery)
// Inject deleted_at IS NULL conditions
securedSQL = validator.injectSoftDeleteConditions(securedSQL, tablesInQuery)
return securedSQL, validationResult, nil
}
// InjectAndConditions injects filter conditions into a SQL statement using AND semantics.
// If WHERE exists, the original WHERE predicates will be wrapped in parentheses.
func InjectAndConditions(sql, filter string) string {
filter = strings.TrimSpace(filter)
if filter == "" {
return sql
}
// Check if WHERE clause exists
wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`)
if wherePattern.MatchString(sql) {
// Add filter and wrap existing conditions in parentheses to prevent OR precedence issues
return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", filter)) + ")"
}
// Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc.
clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`)
if loc := clausePattern.FindStringIndex(sql); loc != nil {
return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", filter) + sql[loc[0]:]
}
// Add WHERE clause at the end
return fmt.Sprintf("%s WHERE %s", sql, filter)
}
// injectTenantConditions adds tenant_id filtering to the query
func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[string]string) string {
if !v.enableTenantInjection {
@@ -829,24 +882,27 @@ func (v *sqlValidator) injectTenantConditions(sql string, tablesInQuery map[stri
}
tenantFilter := strings.Join(conditions, " AND ")
return InjectAndConditions(sql, tenantFilter)
}
// Check if WHERE clause exists
wherePattern := regexp.MustCompile(`(?i)\bWHERE\b`)
if wherePattern.MatchString(sql) {
// Add tenant filter and wrap existing conditions in parentheses to prevent OR injection
// This ensures: WHERE tenant_filter AND (original_conditions)
// Instead of: WHERE tenant_filter AND original_conditions OR malicious_condition
return wherePattern.ReplaceAllString(sql, fmt.Sprintf("WHERE %s AND (", tenantFilter)) + ")"
// injectSoftDeleteConditions adds deleted_at IS NULL filtering to the query.
func (v *sqlValidator) injectSoftDeleteConditions(sql string, tablesInQuery map[string]string) string {
if !v.enableSoftDeleteInjection {
return sql
}
// Add new WHERE clause before ORDER BY, GROUP BY, LIMIT, etc.
clausePattern := regexp.MustCompile(`(?i)\b(GROUP BY|ORDER BY|LIMIT|OFFSET|HAVING|FETCH)\b`)
if loc := clausePattern.FindStringIndex(sql); loc != nil {
return sql[:loc[0]] + fmt.Sprintf(" WHERE %s ", tenantFilter) + sql[loc[0]:]
var conditions []string
for tableName, alias := range tablesInQuery {
if v.tablesWithDeletedAt[tableName] {
conditions = append(conditions, fmt.Sprintf("%s.deleted_at IS NULL", alias))
}
}
// Add WHERE clause at the end
return fmt.Sprintf("%s WHERE %s", sql, tenantFilter)
if len(conditions) == 0 {
return sql
}
return InjectAndConditions(sql, strings.Join(conditions, " AND "))
}
// checkSQLInjectionRisks checks for common SQL injection patterns in WHERE clause