feat(database): enhance DatabaseQueryTool with search scope filtering
- Updated DatabaseQueryTool to accept search targets during initialization. - Implemented search scope filtering in SQL validation to restrict queries to specified knowledge bases and documents. - Enhanced the SQL validation process to include new filtering options for improved security and query accuracy.
This commit is contained in:
@@ -104,14 +104,16 @@ type DatabaseQueryInput struct {
|
||||
// DatabaseQueryTool allows AI to query the database with auto-injected tenant_id for security
|
||||
type DatabaseQueryTool struct {
|
||||
BaseTool
|
||||
db *gorm.DB
|
||||
db *gorm.DB
|
||||
searchTargets types.SearchTargets
|
||||
}
|
||||
|
||||
// NewDatabaseQueryTool creates a new database query tool
|
||||
func NewDatabaseQueryTool(db *gorm.DB) *DatabaseQueryTool {
|
||||
func NewDatabaseQueryTool(db *gorm.DB, searchTargets types.SearchTargets) *DatabaseQueryTool {
|
||||
return &DatabaseQueryTool{
|
||||
BaseTool: databaseQueryTool,
|
||||
db: db,
|
||||
BaseTool: databaseQueryTool,
|
||||
db: db,
|
||||
searchTargets: searchTargets,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,12 +255,21 @@ func (t *DatabaseQueryTool) Execute(ctx context.Context, args json.RawMessage) (
|
||||
|
||||
// validateAndSecureSQL validates the SQL query and injects tenant_id conditions
|
||||
func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint64) (string, error) {
|
||||
kbIDs := t.searchTargets.GetAllKnowledgeBaseIDs()
|
||||
var knowledgeIDs []string
|
||||
for _, target := range t.searchTargets {
|
||||
if target.Type == types.SearchTargetTypeKnowledge && len(target.KnowledgeIDs) > 0 {
|
||||
knowledgeIDs = append(knowledgeIDs, target.KnowledgeIDs...)
|
||||
}
|
||||
}
|
||||
|
||||
securedSQL, validationResult, err := utils.ValidateAndSecureSQL(
|
||||
sqlQuery,
|
||||
utils.WithSecurityDefaults(tenantID),
|
||||
utils.WithSoftDeleteFilter("knowledge_bases", "knowledges", "chunks"),
|
||||
utils.WithHiddenKBFilter(),
|
||||
utils.WithInjectionRiskCheck(),
|
||||
utils.WithSearchScopeFilter(kbIDs, knowledgeIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -395,7 +395,7 @@ func (s *agentService) registerTools(
|
||||
case tools.ToolGetDocumentInfo:
|
||||
toolToRegister = tools.NewGetDocumentInfoTool(s.knowledgeService, s.chunkService, config.SearchTargets)
|
||||
case tools.ToolDatabaseQuery:
|
||||
toolToRegister = tools.NewDatabaseQueryTool(s.db)
|
||||
toolToRegister = tools.NewDatabaseQueryTool(s.db, config.SearchTargets)
|
||||
case tools.ToolWebSearch:
|
||||
toolToRegister = tools.NewWebSearchTool(
|
||||
s.webSearchService,
|
||||
|
||||
@@ -139,6 +139,11 @@ type sqlValidator struct {
|
||||
|
||||
// Hidden knowledge base filtering (is_temporary = false)
|
||||
enableHiddenKBFilter bool
|
||||
|
||||
// Search scope filtering (restrict to specific KBs and knowledges)
|
||||
enableSearchScopeFilter bool
|
||||
searchScopeKBIDs []string
|
||||
searchScopeKnowledgeIDs []string
|
||||
}
|
||||
|
||||
// ParseSQL parses a SQL statement using pg_query_go and extracts table names, select fields, and where fields
|
||||
@@ -610,6 +615,21 @@ func WithHiddenKBFilter() SQLValidationOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithSearchScopeFilter restricts queries to the specified knowledge bases and
|
||||
// (optionally) specific knowledge documents. For the knowledge_bases table it
|
||||
// filters by id; for knowledges it filters by knowledge_base_id (and id when
|
||||
// knowledgeIDs is non-empty); for chunks it filters by knowledge_base_id (and
|
||||
// knowledge_id when knowledgeIDs is non-empty).
|
||||
func WithSearchScopeFilter(kbIDs []string, knowledgeIDs []string) SQLValidationOption {
|
||||
return func(v *sqlValidator) {
|
||||
if len(kbIDs) > 0 {
|
||||
v.enableSearchScopeFilter = true
|
||||
v.searchScopeKBIDs = kbIDs
|
||||
v.searchScopeKnowledgeIDs = knowledgeIDs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithSecurityDefaults applies a comprehensive set of security validations
|
||||
func WithSecurityDefaults(tenantID uint64) SQLValidationOption {
|
||||
return func(v *sqlValidator) {
|
||||
@@ -816,7 +836,7 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
|
||||
}
|
||||
|
||||
// If no SQL rewriting is enabled, return original SQL
|
||||
if !validator.enableTenantInjection && !validator.enableSoftDeleteInjection && !validator.enableHiddenKBFilter {
|
||||
if !validator.enableTenantInjection && !validator.enableSoftDeleteInjection && !validator.enableHiddenKBFilter && !validator.enableSearchScopeFilter {
|
||||
return sql, validationResult, nil
|
||||
}
|
||||
|
||||
@@ -841,6 +861,8 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL
|
||||
securedSQL = validator.injectSoftDeleteConditions(securedSQL, tablesInQuery)
|
||||
// Inject hidden KB filter (exclude is_temporary = true knowledge bases)
|
||||
securedSQL = validator.injectHiddenKBFilter(securedSQL, tablesInQuery)
|
||||
// Inject search scope filter (restrict to allowed KBs and knowledges)
|
||||
securedSQL = validator.injectSearchScopeConditions(securedSQL, tablesInQuery)
|
||||
|
||||
return securedSQL, validationResult, nil
|
||||
}
|
||||
@@ -984,6 +1006,54 @@ func (v *sqlValidator) injectHiddenKBFilter(sql string, tablesInQuery map[string
|
||||
return InjectAndConditions(sql, fmt.Sprintf("%s.is_temporary = false", alias))
|
||||
}
|
||||
|
||||
// injectSearchScopeConditions restricts queries to the allowed knowledge bases
|
||||
// and (optionally) specific knowledge documents.
|
||||
func (v *sqlValidator) injectSearchScopeConditions(sql string, tablesInQuery map[string]string) string {
|
||||
if !v.enableSearchScopeFilter || len(v.searchScopeKBIDs) == 0 {
|
||||
return sql
|
||||
}
|
||||
|
||||
quotedKBIDs := quoteStringSlice(v.searchScopeKBIDs)
|
||||
kbList := strings.Join(quotedKBIDs, ", ")
|
||||
|
||||
var conditions []string
|
||||
|
||||
if alias, ok := tablesInQuery["knowledge_bases"]; ok {
|
||||
conditions = append(conditions, fmt.Sprintf("%s.id IN (%s)", alias, kbList))
|
||||
}
|
||||
|
||||
if alias, ok := tablesInQuery["knowledges"]; ok {
|
||||
conditions = append(conditions, fmt.Sprintf("%s.knowledge_base_id IN (%s)", alias, kbList))
|
||||
if len(v.searchScopeKnowledgeIDs) > 0 {
|
||||
quotedKIDs := quoteStringSlice(v.searchScopeKnowledgeIDs)
|
||||
conditions = append(conditions, fmt.Sprintf("%s.id IN (%s)", alias, strings.Join(quotedKIDs, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
if alias, ok := tablesInQuery["chunks"]; ok {
|
||||
conditions = append(conditions, fmt.Sprintf("%s.knowledge_base_id IN (%s)", alias, kbList))
|
||||
if len(v.searchScopeKnowledgeIDs) > 0 {
|
||||
quotedKIDs := quoteStringSlice(v.searchScopeKnowledgeIDs)
|
||||
conditions = append(conditions, fmt.Sprintf("%s.knowledge_id IN (%s)", alias, strings.Join(quotedKIDs, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
if len(conditions) == 0 {
|
||||
return sql
|
||||
}
|
||||
|
||||
return InjectAndConditions(sql, strings.Join(conditions, " AND "))
|
||||
}
|
||||
|
||||
func quoteStringSlice(ss []string) []string {
|
||||
quoted := make([]string, len(ss))
|
||||
for i, s := range ss {
|
||||
escaped := strings.ReplaceAll(s, "'", "''")
|
||||
quoted[i] = fmt.Sprintf("'%s'", escaped)
|
||||
}
|
||||
return quoted
|
||||
}
|
||||
|
||||
// checkSQLInjectionRisks checks for common SQL injection patterns in WHERE clause
|
||||
func checkSQLInjectionRisks(whereClause string) []SQLValidationError {
|
||||
errors := make([]SQLValidationError, 0)
|
||||
|
||||
Reference in New Issue
Block a user