Merge pull request #555 from lyingbug/main

FAQ Update
This commit is contained in:
lyingbug
2026-01-15 20:09:20 +08:00
committed by GitHub
58 changed files with 3453 additions and 1178 deletions
+33 -16
View File
@@ -44,6 +44,18 @@ func (r *chunkRepository) GetChunkByID(ctx context.Context, tenantID uint64, id
return &chunk, nil
}
// GetChunkBySeqID retrieves a chunk by its seq_id and tenant ID
func (r *chunkRepository) GetChunkBySeqID(ctx context.Context, tenantID uint64, seqID int64) (*types.Chunk, error) {
var chunk types.Chunk
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND seq_id = ?", tenantID, seqID).First(&chunk).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New("chunk not found")
}
return nil, err
}
return &chunk, nil
}
// ListChunksByID retrieves multiple chunks by their IDs
func (r *chunkRepository) ListChunksByID(
ctx context.Context, tenantID uint64, ids []string,
@@ -57,6 +69,22 @@ func (r *chunkRepository) ListChunksByID(
return chunks, nil
}
// ListChunksBySeqID retrieves multiple chunks by their seq_ids
func (r *chunkRepository) ListChunksBySeqID(
ctx context.Context, tenantID uint64, seqIDs []int64,
) ([]*types.Chunk, error) {
if len(seqIDs) == 0 {
return []*types.Chunk{}, nil
}
var chunks []*types.Chunk
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND seq_id IN ?", tenantID, seqIDs).
Find(&chunks).Error; err != nil {
return nil, err
}
return chunks, nil
}
// ListChunksByKnowledgeID lists all chunks for a knowledge ID
func (r *chunkRepository) ListChunksByKnowledgeID(
ctx context.Context, tenantID uint64, knowledgeID string,
@@ -91,10 +119,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
baseFilter := func(db *gorm.DB) *gorm.DB {
db = db.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
tenantID, knowledgeID, chunkType, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)})
if tagID == types.UntaggedTagID {
// Special value to filter entries without a tag
db = db.Where("tag_id = ''")
} else if tagID != "" {
if tagID != "" {
db = db.Where("tag_id = ?", tagID)
}
if keyword != "" {
@@ -640,9 +665,7 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
Select("id").
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
tenantID, kbID, types.ChunkTypeFAQ)
if tagID == "" || tagID == types.UntaggedTagID {
query = query.Where("(tag_id = '' OR tag_id IS NULL)")
} else {
if tagID != "" {
query = query.Where("tag_id = ?", tagID)
}
@@ -669,22 +692,16 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
updates["is_enabled"] = *isEnabled
}
// Handle newTagID update (__untagged__ is stored as empty string)
// Handle newTagID update
if newTagID != nil {
if *newTagID == types.UntaggedTagID || *newTagID == "" {
updates["tag_id"] = ""
} else {
updates["tag_id"] = *newTagID
}
updates["tag_id"] = *newTagID
}
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
tenantID, kbID, types.ChunkTypeFAQ)
if tagID == "" || tagID == types.UntaggedTagID {
query = query.Where("(tag_id = '' OR tag_id IS NULL)")
} else {
if tagID != "" {
query = query.Where("tag_id = ?", tagID)
}
+15 -12
View File
@@ -75,12 +75,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID)
if tagID != "" {
if tagID == types.UntaggedTagID {
// Special value to filter entries without a tag
query = query.Where("tag_id = '' OR tag_id IS NULL")
} else {
query = query.Where("tag_id = ?", tagID)
}
query = query.Where("tag_id = ?", tagID)
}
if keyword != "" {
query = query.Where("file_name LIKE ?", "%"+keyword+"%")
@@ -104,12 +99,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
dataQuery := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID)
if tagID != "" {
if tagID == types.UntaggedTagID {
// Special value to filter entries without a tag
dataQuery = dataQuery.Where("tag_id = '' OR tag_id IS NULL")
} else {
dataQuery = dataQuery.Where("tag_id = ?", tagID)
}
dataQuery = dataQuery.Where("tag_id = ?", tagID)
}
if keyword != "" {
dataQuery = dataQuery.Where("file_name LIKE ?", "%"+keyword+"%")
@@ -411,3 +401,16 @@ func (r *knowledgeRepository) SearchKnowledge(
}
return knowledges, hasMore, nil
}
// ListIDsByTagID returns all knowledge IDs that have the specified tag ID
func (r *knowledgeRepository) ListIDsByTagID(
ctx context.Context,
tenantID uint64,
kbID, tagID string,
) ([]string, error) {
var ids []string
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
Pluck("id", &ids).Error
return ids, err
}
+29 -25
View File
@@ -54,6 +54,31 @@ func (r *knowledgeTagRepository) GetByIDs(ctx context.Context, tenantID uint64,
return tags, nil
}
// GetBySeqID retrieves a tag by its seq_id
func (r *knowledgeTagRepository) GetBySeqID(ctx context.Context, tenantID uint64, seqID int64) (*types.KnowledgeTag, error) {
var tag types.KnowledgeTag
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND seq_id = ?", tenantID, seqID).
First(&tag).Error; err != nil {
return nil, err
}
return &tag, nil
}
// GetBySeqIDs retrieves multiple tags by their seq_ids in a single query
func (r *knowledgeTagRepository) GetBySeqIDs(ctx context.Context, tenantID uint64, seqIDs []int64) ([]*types.KnowledgeTag, error) {
if len(seqIDs) == 0 {
return []*types.KnowledgeTag{}, nil
}
var tags []*types.KnowledgeTag
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND seq_id IN (?)", tenantID, seqIDs).
Find(&tags).Error; err != nil {
return nil, err
}
return tags, nil
}
// GetByName gets a knowledge tag by name
func (r *knowledgeTagRepository) GetByName(ctx context.Context, tenantID uint64, kbID string, name string) (*types.KnowledgeTag, error) {
var tag types.KnowledgeTag
@@ -97,7 +122,7 @@ func (r *knowledgeTagRepository) ListByKB(
var tags []*types.KnowledgeTag
if err := dataQuery.
Order("sort_order ASC, created_at ASC").
Order("sort_order ASC, created_at DESC").
Offset(page.Offset()).
Limit(page.Limit()).
Find(&tags).Error; err != nil {
@@ -136,35 +161,14 @@ func (r *knowledgeTagRepository) CountReferences(
return
}
// CountUntaggedReferences returns the number of knowledges and chunks without a tag
func (r *knowledgeTagRepository) CountUntaggedReferences(
ctx context.Context,
tenantID uint64,
kbID string,
) (knowledgeCount int64, chunkCount int64, err error) {
if err = r.db.WithContext(ctx).
Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND (tag_id = '' OR tag_id IS NULL)", tenantID, kbID).
Count(&knowledgeCount).Error; err != nil {
return
}
if err = r.db.WithContext(ctx).
Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND (tag_id = '' OR tag_id IS NULL)", tenantID, kbID).
Count(&chunkCount).Error; err != nil {
return
}
return
}
// DeleteUnusedTags deletes tags that are not referenced by any knowledge or chunk.
// Returns the number of deleted tags.
func (r *knowledgeTagRepository) DeleteUnusedTags(ctx context.Context, tenantID uint64, kbID string) (int64, error) {
// Delete tags that have no references in both knowledges and chunks tables
// Delete tags that have no references in both knowledges and chunks tables (excluding soft-deleted records)
result := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
Where("id NOT IN (SELECT DISTINCT tag_id FROM knowledges WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '')", tenantID, kbID).
Where("id NOT IN (SELECT DISTINCT tag_id FROM chunks WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '')", tenantID, kbID).
Where("id NOT IN (SELECT DISTINCT tag_id FROM knowledges WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '' AND deleted_at IS NULL)", tenantID, kbID).
Where("id NOT IN (SELECT DISTINCT tag_id FROM chunks WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '' AND deleted_at IS NULL)", tenantID, kbID).
Delete(&types.KnowledgeTag{})
return result.RowsAffected, result.Error
}
+8 -6
View File
@@ -59,13 +59,15 @@ func (r *tenantRepository) SearchTenants(ctx context.Context, keyword string, te
query := r.db.WithContext(ctx).Model(&types.Tenant{})
// Filter by tenant ID if provided
if tenantID > 0 {
// Build search conditions
if tenantID > 0 && keyword != "" {
// When both tenantID and keyword are provided, use OR to match either
query = query.Where("id = ? OR name LIKE ? OR description LIKE ?", tenantID, "%"+keyword+"%", "%"+keyword+"%")
} else if tenantID > 0 {
// Filter by tenant ID only
query = query.Where("id = ?", tenantID)
}
// Filter by keyword if provided (search in name and description)
if keyword != "" {
} else if keyword != "" {
// Filter by keyword only (search in name and description)
query = query.Where("name LIKE ? OR description LIKE ?", "%"+keyword+"%", "%"+keyword+"%")
}
@@ -375,7 +375,7 @@ func (s *agentService) getKnowledgeBaseInfos(ctx context.Context, kbIDs []string
pageResult, err := s.knowledgeService.ListFAQEntries(ctx, kbID, &types.Pagination{
Page: 1,
PageSize: 10,
}, "", "", "", "")
}, 0, "", "", "")
if err == nil && pageResult != nil {
docCount = int(pageResult.Total)
if entries, ok := pageResult.Data.([]*types.FAQEntry); ok {
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/Tencent/WeKnora/internal/utils"
"golang.org/x/sync/errgroup"
)
@@ -253,7 +253,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
// Create evaluation task with unique ID
logger.Info(ctx, "Creating evaluation task")
taskID := uuid.New().String()
taskID := utils.GenerateTaskID("evaluation", tenantID, datasetID)
logger.Infof(ctx, "Generated task ID: %s", taskID)
// Prepare evaluation detail with all parameters
+87 -3
View File
@@ -1,6 +1,7 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
@@ -9,6 +10,7 @@ import (
"net/url"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -20,11 +22,19 @@ type cosFileService struct {
client *cos.Client
bucketURL string
cosPathPrefix string
// 临时桶配置(用于存放自动过期的临时文件)
tempClient *cos.Client
tempBucketURL string
}
// NewCosFileService creates a new COS file service instance
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
}
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
u, err := url.Parse(bucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
@@ -36,11 +46,35 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
SecretKey: secretKey,
},
})
return &cosFileService{
svc := &cosFileService{
client: client,
bucketURL: bucketURL,
cosPathPrefix: cosPathPrefix,
}, nil
}
// 如果配置了临时桶,初始化临时桶客户端
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region // 默认使用相同的 region
}
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
tempU, err := url.Parse(tempBucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
}
tempB := &cos.BaseURL{BucketURL: tempU}
tempClient := cos.NewClient(tempB, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.tempClient = tempClient
svc.tempBucketURL = tempBucketURL
}
return svc, nil
}
// SaveFile saves a file to COS storage
@@ -81,3 +115,53 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
}
return nil
}
// SaveBytes saves bytes data to COS
// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration)
// Otherwise saves to main bucket
func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
ext := filepath.Ext(fileName)
reader := bytes.NewReader(data)
// 如果请求写入临时桶且临时桶已配置
if temp && s.tempClient != nil {
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
if err != nil {
return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err)
}
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
}
// 写入主桶
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
_, err := s.client.Object.Put(ctx, objectName, reader, nil)
if err != nil {
return "", fmt.Errorf("failed to upload bytes to COS: %w", err)
}
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
}
// GetFileURL returns a presigned download URL for the file
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// 判断文件属于哪个桶
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err)
}
return presignedURL.String(), nil
}
objectName := strings.TrimPrefix(filePath, s.bucketURL)
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
}
return presignedURL.String(), nil
}
@@ -36,3 +36,13 @@ func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.Rea
func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error {
return nil
}
// SaveBytes pretends to save bytes but just returns a random UUID
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
return uuid.New().String(), nil
}
// GetFileURL returns the file path as URL (dummy implementation)
func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
return filePath, nil
}
@@ -109,3 +109,38 @@ func (s *localFileService) DeleteFile(ctx context.Context, filePath string) erro
logger.Info(ctx, "File deleted successfully")
return nil
}
// SaveBytes saves bytes data to a file and returns the file path
// temp parameter is ignored for local storage (no auto-expiration support)
func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp)
// Create storage directory with tenant ID
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports")
if err := os.MkdirAll(dir, 0o755); err != nil {
logger.Errorf(ctx, "Failed to create directory: %v", err)
return "", fmt.Errorf("failed to create directory: %w", err)
}
// Generate unique filename using timestamp
ext := filepath.Ext(fileName)
baseName := fileName[:len(fileName)-len(ext)]
uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext)
filePath := filepath.Join(dir, uniqueFileName)
// Write data to file
if err := os.WriteFile(filePath, data, 0o644); err != nil {
logger.Errorf(ctx, "Failed to write file: %v", err)
return "", fmt.Errorf("failed to write file: %w", err)
}
logger.Infof(ctx, "Bytes data saved successfully: %s", filePath)
return filePath, nil
}
// GetFileURL returns a download URL for the file
// For local storage, returns the file path itself (no URL support)
func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// Local storage doesn't support URLs, return the path
return filePath, nil
}
@@ -1,11 +1,13 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -125,3 +127,43 @@ func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) erro
return nil
}
// SaveBytes saves bytes data to MinIO and returns the file path
// temp parameter is ignored for MinIO (no auto-expiration support in this implementation)
func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
ext := filepath.Ext(fileName)
objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext)
// Upload bytes to MinIO
reader := bytes.NewReader(data)
_, err := s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{
ContentType: "text/csv; charset=utf-8",
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err)
}
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
}
// GetFileURL returns a presigned download URL for the file
func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// Parse MinIO path
if len(filePath) < 9 || filePath[:8] != "minio://" {
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
}
// Extract object name
objectName := filePath[9+len(s.bucketName):]
if objectName[0] == '/' {
objectName = objectName[1:]
}
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
}
return presignedURL.String(), nil
}
File diff suppressed because it is too large Load Diff
+6 -33
View File
@@ -31,20 +31,9 @@ func NewMCPServiceService(
// CreateMCPService creates a new MCP service
func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types.MCPService) error {
// Security validation for stdio transport type
// Stdio transport is disabled for security reasons
if service.TransportType == types.MCPTransportStdio {
if service.StdioConfig == nil {
return fmt.Errorf("stdio_config is required for stdio transport")
}
// Validate stdio configuration to prevent command injection (CWE-78)
if err := secutils.ValidateStdioConfig(
service.StdioConfig.Command,
service.StdioConfig.Args,
service.EnvVars,
); err != nil {
logger.GetLogger(ctx).Warnf("MCP service creation blocked due to security validation: %v", err)
return fmt.Errorf("security validation failed: %w", err)
}
return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead")
}
// Set default advanced config if not provided
@@ -129,31 +118,15 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
return fmt.Errorf("MCP service not found")
}
// Security validation for stdio transport type when updating stdio config
// Determine the final transport type and stdio config after merge
// Determine the final transport type after merge
finalTransportType := existing.TransportType
if service.TransportType != "" {
finalTransportType = service.TransportType
}
finalStdioConfig := existing.StdioConfig
if service.StdioConfig != nil {
finalStdioConfig = service.StdioConfig
}
finalEnvVars := existing.EnvVars
if service.EnvVars != nil {
finalEnvVars = service.EnvVars
}
// Validate if the final configuration uses stdio transport
if finalTransportType == types.MCPTransportStdio && finalStdioConfig != nil {
if err := secutils.ValidateStdioConfig(
finalStdioConfig.Command,
finalStdioConfig.Args,
finalEnvVars,
); err != nil {
logger.GetLogger(ctx).Warnf("MCP service update blocked due to security validation: %v", err)
return fmt.Errorf("security validation failed: %w", err)
}
// Stdio transport is disabled for security reasons
if finalTransportType == types.MCPTransportStdio {
return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead")
}
// Store old enabled state BEFORE any updates
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"golang.org/x/sync/errgroup"
)
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
@@ -58,7 +59,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
}
// BatchIndex creates embeddings for multiple content items and saves them to the repository
// in batches for efficiency
// in batches for efficiency. Uses concurrent batch saving to improve performance.
func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Context,
embedder embedding.Embedder, indexInfoList []*types.IndexInfo, retrieverTypes []types.RetrieverType,
) error {
@@ -85,30 +86,123 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
if err != nil {
return err
}
batchSize := 40
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
chunks := utils.ChunkSlice(indexInfoList, batchSize)
// Use concurrent batch saving for better performance
// Limit concurrency to avoid overwhelming the backend
const maxConcurrency = 5
if len(chunks) <= maxConcurrency {
// For small number of batches, use simple concurrency
return v.concurrentBatchSave(ctx, chunks, embeddings, batchSize)
}
// For large number of batches, use bounded concurrency
return v.boundedConcurrentBatchSave(ctx, chunks, embeddings, batchSize, maxConcurrency)
}
// For non-vector retrieval, use concurrent batch saving as well
chunks := utils.ChunkSlice(indexInfoList, 10)
const maxConcurrency = 5
if len(chunks) <= maxConcurrency {
return v.concurrentBatchSaveNoEmbedding(ctx, chunks)
}
return v.boundedConcurrentBatchSaveNoEmbedding(ctx, chunks, maxConcurrency)
}
// concurrentBatchSave saves all batches concurrently without concurrency limit
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSave(
ctx context.Context,
chunks [][]*types.IndexInfo,
embeddings [][]float32,
batchSize int,
) error {
g, ctx := errgroup.WithContext(ctx)
for i, indexChunk := range chunks {
g.Go(func() error {
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
}
params["embedding"] = embeddingMap
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
if err != nil {
return err
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return g.Wait()
}
// boundedConcurrentBatchSave saves batches with bounded concurrency using semaphore pattern
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSave(
ctx context.Context,
chunks [][]*types.IndexInfo,
embeddings [][]float32,
batchSize int,
maxConcurrency int,
) error {
g, ctx := errgroup.WithContext(ctx)
sem := make(chan struct{}, maxConcurrency)
for i, indexChunk := range chunks {
g.Go(func() error {
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
return ctx.Err()
}
}
return nil
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
}
params["embedding"] = embeddingMap
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
var err error
for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) {
params := make(map[string]any)
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
if err != nil {
return err
}
return g.Wait()
}
// concurrentBatchSaveNoEmbedding saves all batches concurrently without embeddings
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSaveNoEmbedding(
ctx context.Context,
chunks [][]*types.IndexInfo,
) error {
g, ctx := errgroup.WithContext(ctx)
for _, indexChunk := range chunks {
g.Go(func() error {
params := make(map[string]any)
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return nil
return g.Wait()
}
// boundedConcurrentBatchSaveNoEmbedding saves batches with bounded concurrency without embeddings
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSaveNoEmbedding(
ctx context.Context,
chunks [][]*types.IndexInfo,
maxConcurrency int,
) error {
g, ctx := errgroup.WithContext(ctx)
sem := make(chan struct{}, maxConcurrency)
for _, indexChunk := range chunks {
g.Go(func() error {
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
return ctx.Err()
}
params := make(map[string]any)
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return g.Wait()
}
// DeleteByChunkIDList deletes vectors by their chunk IDs
+64 -32
View File
@@ -21,6 +21,7 @@ import (
type knowledgeTagService struct {
kbService interfaces.KnowledgeBaseService
repo interfaces.KnowledgeTagRepository
knowledgeRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
modelService interfaces.ModelService
@@ -31,6 +32,7 @@ type knowledgeTagService struct {
func NewKnowledgeTagService(
kbService interfaces.KnowledgeBaseService,
repo interfaces.KnowledgeTagRepository,
knowledgeRepo interfaces.KnowledgeRepository,
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
modelService interfaces.ModelService,
@@ -39,6 +41,7 @@ func NewKnowledgeTagService(
return &knowledgeTagService{
kbService: kbService,
repo: repo,
knowledgeRepo: knowledgeRepo,
chunkRepo: chunkRepo,
retrieveEngine: retrieveEngine,
modelService: modelService,
@@ -72,28 +75,7 @@ func (s *knowledgeTagService) ListTags(
return nil, err
}
results := make([]*types.KnowledgeTagWithStats, 0, len(tags)+1)
// Add untagged pseudo-tag at the beginning (only on first page and when no keyword filter)
if page.Page <= 1 && keyword == "" {
untaggedKCount, untaggedCCount, err := s.repo.CountUntaggedReferences(ctx, tenantID, kbID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"kb_id": kbID,
})
return nil, err
}
results = append(results, &types.KnowledgeTagWithStats{
KnowledgeTag: types.KnowledgeTag{
ID: types.UntaggedTagID,
TenantID: tenantID,
KnowledgeBaseID: kbID,
Name: "未分类",
},
KnowledgeCount: untaggedKCount,
ChunkCount: untaggedCCount,
})
}
results := make([]*types.KnowledgeTagWithStats, 0, len(tags))
for _, tag := range tags {
if tag == nil {
@@ -114,11 +96,6 @@ func (s *knowledgeTagService) ListTags(
})
}
// Adjust total to include the untagged pseudo-tag
if keyword == "" {
total++
}
return types.NewPageResult(total, page, results), nil
}
@@ -149,6 +126,10 @@ func (s *knowledgeTagService) CreateTag(
}
now := time.Now()
// "未分类" tag should have the lowest sort order to appear first
if name == types.UntaggedTagName {
sortOrder = -1
}
tag := &types.KnowledgeTag{
ID: uuid.New().String(),
TenantID: kb.TenantID,
@@ -203,6 +184,7 @@ func (s *knowledgeTagService) UpdateTag(
}
// DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag.
// For document-type knowledge bases, also deletes all knowledge files under this tag.
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error {
if id == "" {
@@ -246,9 +228,49 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
return nil
}
// Helper function to enqueue knowledge list delete task for document-type knowledge bases
enqueueKnowledgeDeleteTask := func() error {
if kb.Type != types.KnowledgeBaseTypeDocument {
return nil
}
// Get all knowledge IDs under this tag
knowledgeIDs, err := s.knowledgeRepo.ListIDsByTagID(ctx, tenantID, kb.ID, tag.ID)
if err != nil {
logger.Errorf(ctx, "Failed to list knowledge IDs by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("获取标签下的文档失败")
}
if len(knowledgeIDs) == 0 {
return nil
}
// Enqueue async task to delete knowledge files
payload := types.KnowledgeListDeletePayload{
TenantID: tenantID,
KnowledgeIDs: knowledgeIDs,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal knowledge list delete payload: %v", err)
return werrors.NewInternalServerError("删除标签下的文档失败")
}
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge list delete task: %v", err)
return werrors.NewInternalServerError("删除标签下的文档失败")
}
logger.Infof(ctx, "Enqueued knowledge list delete task %s for %d knowledge files under tag %s", info.ID, len(knowledgeIDs), tag.ID)
return nil
}
// contentOnly mode: only delete content, keep the tag
if contentOnly {
if cCount > 0 {
// For document-type KB, delete knowledge files first (which will also delete chunks)
if kb.Type == types.KnowledgeBaseTypeDocument && kCount > 0 {
if err := enqueueKnowledgeDeleteTask(); err != nil {
return err
}
} else if cCount > 0 {
// For FAQ-type KB, only delete chunks
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
@@ -259,12 +281,22 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
if !force && (kCount > 0 || cCount > 0) {
return werrors.NewBadRequestError("标签仍有知识或FAQ条目引用,无法删除")
}
// When force=true, delete all chunks under this tag first
if force && cCount > 0 {
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
// When force=true, delete all content under this tag first
if force {
// For document-type KB, delete knowledge files first (which will also delete chunks)
if kb.Type == types.KnowledgeBaseTypeDocument && kCount > 0 {
if err := enqueueKnowledgeDeleteTask(); err != nil {
return err
}
} else if cCount > 0 {
// For FAQ-type KB, only delete chunks
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
}
}
// If there are excludeIDs, we cannot delete the tag itself as it still has content
if len(excludeIDs) > 0 {
return nil