@@ -44,6 +44,18 @@ func (r *chunkRepository) GetChunkByID(ctx context.Context, tenantID uint64, id
|
||||
return &chunk, nil
|
||||
}
|
||||
|
||||
// GetChunkBySeqID retrieves a chunk by its seq_id and tenant ID
|
||||
func (r *chunkRepository) GetChunkBySeqID(ctx context.Context, tenantID uint64, seqID int64) (*types.Chunk, error) {
|
||||
var chunk types.Chunk
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND seq_id = ?", tenantID, seqID).First(&chunk).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New("chunk not found")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &chunk, nil
|
||||
}
|
||||
|
||||
// ListChunksByID retrieves multiple chunks by their IDs
|
||||
func (r *chunkRepository) ListChunksByID(
|
||||
ctx context.Context, tenantID uint64, ids []string,
|
||||
@@ -57,6 +69,22 @@ func (r *chunkRepository) ListChunksByID(
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// ListChunksBySeqID retrieves multiple chunks by their seq_ids
|
||||
func (r *chunkRepository) ListChunksBySeqID(
|
||||
ctx context.Context, tenantID uint64, seqIDs []int64,
|
||||
) ([]*types.Chunk, error) {
|
||||
if len(seqIDs) == 0 {
|
||||
return []*types.Chunk{}, nil
|
||||
}
|
||||
var chunks []*types.Chunk
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND seq_id IN ?", tenantID, seqIDs).
|
||||
Find(&chunks).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// ListChunksByKnowledgeID lists all chunks for a knowledge ID
|
||||
func (r *chunkRepository) ListChunksByKnowledgeID(
|
||||
ctx context.Context, tenantID uint64, knowledgeID string,
|
||||
@@ -91,10 +119,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
|
||||
baseFilter := func(db *gorm.DB) *gorm.DB {
|
||||
db = db.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
|
||||
tenantID, knowledgeID, chunkType, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)})
|
||||
if tagID == types.UntaggedTagID {
|
||||
// Special value to filter entries without a tag
|
||||
db = db.Where("tag_id = ''")
|
||||
} else if tagID != "" {
|
||||
if tagID != "" {
|
||||
db = db.Where("tag_id = ?", tagID)
|
||||
}
|
||||
if keyword != "" {
|
||||
@@ -640,9 +665,7 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
|
||||
Select("id").
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
|
||||
tenantID, kbID, types.ChunkTypeFAQ)
|
||||
if tagID == "" || tagID == types.UntaggedTagID {
|
||||
query = query.Where("(tag_id = '' OR tag_id IS NULL)")
|
||||
} else {
|
||||
if tagID != "" {
|
||||
query = query.Where("tag_id = ?", tagID)
|
||||
}
|
||||
|
||||
@@ -669,22 +692,16 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
|
||||
updates["is_enabled"] = *isEnabled
|
||||
}
|
||||
|
||||
// Handle newTagID update (__untagged__ is stored as empty string)
|
||||
// Handle newTagID update
|
||||
if newTagID != nil {
|
||||
if *newTagID == types.UntaggedTagID || *newTagID == "" {
|
||||
updates["tag_id"] = ""
|
||||
} else {
|
||||
updates["tag_id"] = *newTagID
|
||||
}
|
||||
updates["tag_id"] = *newTagID
|
||||
}
|
||||
|
||||
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
|
||||
tenantID, kbID, types.ChunkTypeFAQ)
|
||||
|
||||
if tagID == "" || tagID == types.UntaggedTagID {
|
||||
query = query.Where("(tag_id = '' OR tag_id IS NULL)")
|
||||
} else {
|
||||
if tagID != "" {
|
||||
query = query.Where("tag_id = ?", tagID)
|
||||
}
|
||||
|
||||
|
||||
@@ -75,12 +75,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
|
||||
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID)
|
||||
if tagID != "" {
|
||||
if tagID == types.UntaggedTagID {
|
||||
// Special value to filter entries without a tag
|
||||
query = query.Where("tag_id = '' OR tag_id IS NULL")
|
||||
} else {
|
||||
query = query.Where("tag_id = ?", tagID)
|
||||
}
|
||||
query = query.Where("tag_id = ?", tagID)
|
||||
}
|
||||
if keyword != "" {
|
||||
query = query.Where("file_name LIKE ?", "%"+keyword+"%")
|
||||
@@ -104,12 +99,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
|
||||
dataQuery := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID)
|
||||
if tagID != "" {
|
||||
if tagID == types.UntaggedTagID {
|
||||
// Special value to filter entries without a tag
|
||||
dataQuery = dataQuery.Where("tag_id = '' OR tag_id IS NULL")
|
||||
} else {
|
||||
dataQuery = dataQuery.Where("tag_id = ?", tagID)
|
||||
}
|
||||
dataQuery = dataQuery.Where("tag_id = ?", tagID)
|
||||
}
|
||||
if keyword != "" {
|
||||
dataQuery = dataQuery.Where("file_name LIKE ?", "%"+keyword+"%")
|
||||
@@ -411,3 +401,16 @@ func (r *knowledgeRepository) SearchKnowledge(
|
||||
}
|
||||
return knowledges, hasMore, nil
|
||||
}
|
||||
|
||||
// ListIDsByTagID returns all knowledge IDs that have the specified tag ID
|
||||
func (r *knowledgeRepository) ListIDsByTagID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID, tagID string,
|
||||
) ([]string, error) {
|
||||
var ids []string
|
||||
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
|
||||
Pluck("id", &ids).Error
|
||||
return ids, err
|
||||
}
|
||||
|
||||
@@ -54,6 +54,31 @@ func (r *knowledgeTagRepository) GetByIDs(ctx context.Context, tenantID uint64,
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// GetBySeqID retrieves a tag by its seq_id
|
||||
func (r *knowledgeTagRepository) GetBySeqID(ctx context.Context, tenantID uint64, seqID int64) (*types.KnowledgeTag, error) {
|
||||
var tag types.KnowledgeTag
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND seq_id = ?", tenantID, seqID).
|
||||
First(&tag).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tag, nil
|
||||
}
|
||||
|
||||
// GetBySeqIDs retrieves multiple tags by their seq_ids in a single query
|
||||
func (r *knowledgeTagRepository) GetBySeqIDs(ctx context.Context, tenantID uint64, seqIDs []int64) ([]*types.KnowledgeTag, error) {
|
||||
if len(seqIDs) == 0 {
|
||||
return []*types.KnowledgeTag{}, nil
|
||||
}
|
||||
var tags []*types.KnowledgeTag
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND seq_id IN (?)", tenantID, seqIDs).
|
||||
Find(&tags).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// GetByName gets a knowledge tag by name
|
||||
func (r *knowledgeTagRepository) GetByName(ctx context.Context, tenantID uint64, kbID string, name string) (*types.KnowledgeTag, error) {
|
||||
var tag types.KnowledgeTag
|
||||
@@ -97,7 +122,7 @@ func (r *knowledgeTagRepository) ListByKB(
|
||||
|
||||
var tags []*types.KnowledgeTag
|
||||
if err := dataQuery.
|
||||
Order("sort_order ASC, created_at ASC").
|
||||
Order("sort_order ASC, created_at DESC").
|
||||
Offset(page.Offset()).
|
||||
Limit(page.Limit()).
|
||||
Find(&tags).Error; err != nil {
|
||||
@@ -136,35 +161,14 @@ func (r *knowledgeTagRepository) CountReferences(
|
||||
return
|
||||
}
|
||||
|
||||
// CountUntaggedReferences returns the number of knowledges and chunks without a tag
|
||||
func (r *knowledgeTagRepository) CountUntaggedReferences(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) (knowledgeCount int64, chunkCount int64, err error) {
|
||||
if err = r.db.WithContext(ctx).
|
||||
Model(&types.Knowledge{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND (tag_id = '' OR tag_id IS NULL)", tenantID, kbID).
|
||||
Count(&knowledgeCount).Error; err != nil {
|
||||
return
|
||||
}
|
||||
if err = r.db.WithContext(ctx).
|
||||
Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND (tag_id = '' OR tag_id IS NULL)", tenantID, kbID).
|
||||
Count(&chunkCount).Error; err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteUnusedTags deletes tags that are not referenced by any knowledge or chunk.
|
||||
// Returns the number of deleted tags.
|
||||
func (r *knowledgeTagRepository) DeleteUnusedTags(ctx context.Context, tenantID uint64, kbID string) (int64, error) {
|
||||
// Delete tags that have no references in both knowledges and chunks tables
|
||||
// Delete tags that have no references in both knowledges and chunks tables (excluding soft-deleted records)
|
||||
result := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
||||
Where("id NOT IN (SELECT DISTINCT tag_id FROM knowledges WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '')", tenantID, kbID).
|
||||
Where("id NOT IN (SELECT DISTINCT tag_id FROM chunks WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '')", tenantID, kbID).
|
||||
Where("id NOT IN (SELECT DISTINCT tag_id FROM knowledges WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '' AND deleted_at IS NULL)", tenantID, kbID).
|
||||
Where("id NOT IN (SELECT DISTINCT tag_id FROM chunks WHERE tenant_id = ? AND knowledge_base_id = ? AND tag_id IS NOT NULL AND tag_id != '' AND deleted_at IS NULL)", tenantID, kbID).
|
||||
Delete(&types.KnowledgeTag{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
@@ -59,13 +59,15 @@ func (r *tenantRepository) SearchTenants(ctx context.Context, keyword string, te
|
||||
|
||||
query := r.db.WithContext(ctx).Model(&types.Tenant{})
|
||||
|
||||
// Filter by tenant ID if provided
|
||||
if tenantID > 0 {
|
||||
// Build search conditions
|
||||
if tenantID > 0 && keyword != "" {
|
||||
// When both tenantID and keyword are provided, use OR to match either
|
||||
query = query.Where("id = ? OR name LIKE ? OR description LIKE ?", tenantID, "%"+keyword+"%", "%"+keyword+"%")
|
||||
} else if tenantID > 0 {
|
||||
// Filter by tenant ID only
|
||||
query = query.Where("id = ?", tenantID)
|
||||
}
|
||||
|
||||
// Filter by keyword if provided (search in name and description)
|
||||
if keyword != "" {
|
||||
} else if keyword != "" {
|
||||
// Filter by keyword only (search in name and description)
|
||||
query = query.Where("name LIKE ? OR description LIKE ?", "%"+keyword+"%", "%"+keyword+"%")
|
||||
}
|
||||
|
||||
|
||||
@@ -375,7 +375,7 @@ func (s *agentService) getKnowledgeBaseInfos(ctx context.Context, kbIDs []string
|
||||
pageResult, err := s.knowledgeService.ListFAQEntries(ctx, kbID, &types.Pagination{
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
}, "", "", "", "")
|
||||
}, 0, "", "", "")
|
||||
if err == nil && pageResult != nil {
|
||||
docCount = int(pageResult.Total)
|
||||
if entries, ok := pageResult.Data.([]*types.FAQEntry); ok {
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -253,7 +253,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
|
||||
// Create evaluation task with unique ID
|
||||
logger.Info(ctx, "Creating evaluation task")
|
||||
taskID := uuid.New().String()
|
||||
taskID := utils.GenerateTaskID("evaluation", tenantID, datasetID)
|
||||
logger.Infof(ctx, "Generated task ID: %s", taskID)
|
||||
|
||||
// Prepare evaluation detail with all parameters
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
@@ -20,11 +22,19 @@ type cosFileService struct {
|
||||
client *cos.Client
|
||||
bucketURL string
|
||||
cosPathPrefix string
|
||||
// 临时桶配置(用于存放自动过期的临时文件)
|
||||
tempClient *cos.Client
|
||||
tempBucketURL string
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
|
||||
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
@@ -36,11 +46,35 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
return &cosFileService{
|
||||
|
||||
svc := &cosFileService{
|
||||
client: client,
|
||||
bucketURL: bucketURL,
|
||||
cosPathPrefix: cosPathPrefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 如果配置了临时桶,初始化临时桶客户端
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region // 默认使用相同的 region
|
||||
}
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
|
||||
tempU, err := url.Parse(tempBucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
tempB := &cos.BaseURL{BucketURL: tempU}
|
||||
tempClient := cos.NewClient(tempB, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.tempClient = tempClient
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to COS storage
|
||||
@@ -81,3 +115,53 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to COS
|
||||
// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration)
|
||||
// Otherwise saves to main bucket
|
||||
func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
// 如果请求写入临时桶且临时桶已配置
|
||||
if temp && s.tempClient != nil {
|
||||
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// 写入主桶
|
||||
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
|
||||
_, err := s.client.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// 判断文件属于哪个桶
|
||||
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
|
||||
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err)
|
||||
}
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
objectName := strings.TrimPrefix(filePath, s.bucketURL)
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
@@ -36,3 +36,13 @@ func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.Rea
|
||||
func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes pretends to save bytes but just returns a random UUID
|
||||
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
return uuid.New().String(), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns the file path as URL (dummy implementation)
|
||||
func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
@@ -109,3 +109,38 @@ func (s *localFileService) DeleteFile(ctx context.Context, filePath string) erro
|
||||
logger.Info(ctx, "File deleted successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to a file and returns the file path
|
||||
// temp parameter is ignored for local storage (no auto-expiration support)
|
||||
func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp)
|
||||
|
||||
// Create storage directory with tenant ID
|
||||
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create directory: %v", err)
|
||||
return "", fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate unique filename using timestamp
|
||||
ext := filepath.Ext(fileName)
|
||||
baseName := fileName[:len(fileName)-len(ext)]
|
||||
uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext)
|
||||
filePath := filepath.Join(dir, uniqueFileName)
|
||||
|
||||
// Write data to file
|
||||
if err := os.WriteFile(filePath, data, 0o644); err != nil {
|
||||
logger.Errorf(ctx, "Failed to write file: %v", err)
|
||||
return "", fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Bytes data saved successfully: %s", filePath)
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a download URL for the file
|
||||
// For local storage, returns the file path itself (no URL support)
|
||||
func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// Local storage doesn't support URLs, return the path
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
@@ -125,3 +127,43 @@ func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) erro
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to MinIO and returns the file path
|
||||
// temp parameter is ignored for MinIO (no auto-expiration support in this implementation)
|
||||
func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext)
|
||||
|
||||
// Upload bytes to MinIO
|
||||
reader := bytes.NewReader(data)
|
||||
_, err := s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{
|
||||
ContentType: "text/csv; charset=utf-8",
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// Parse MinIO path
|
||||
if len(filePath) < 9 || filePath[:8] != "minio://" {
|
||||
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
|
||||
}
|
||||
|
||||
// Extract object name
|
||||
objectName := filePath[9+len(s.bucketName):]
|
||||
if objectName[0] == '/' {
|
||||
objectName = objectName[1:]
|
||||
}
|
||||
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,20 +31,9 @@ func NewMCPServiceService(
|
||||
|
||||
// CreateMCPService creates a new MCP service
|
||||
func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types.MCPService) error {
|
||||
// Security validation for stdio transport type
|
||||
// Stdio transport is disabled for security reasons
|
||||
if service.TransportType == types.MCPTransportStdio {
|
||||
if service.StdioConfig == nil {
|
||||
return fmt.Errorf("stdio_config is required for stdio transport")
|
||||
}
|
||||
// Validate stdio configuration to prevent command injection (CWE-78)
|
||||
if err := secutils.ValidateStdioConfig(
|
||||
service.StdioConfig.Command,
|
||||
service.StdioConfig.Args,
|
||||
service.EnvVars,
|
||||
); err != nil {
|
||||
logger.GetLogger(ctx).Warnf("MCP service creation blocked due to security validation: %v", err)
|
||||
return fmt.Errorf("security validation failed: %w", err)
|
||||
}
|
||||
return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead")
|
||||
}
|
||||
|
||||
// Set default advanced config if not provided
|
||||
@@ -129,31 +118,15 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
|
||||
return fmt.Errorf("MCP service not found")
|
||||
}
|
||||
|
||||
// Security validation for stdio transport type when updating stdio config
|
||||
// Determine the final transport type and stdio config after merge
|
||||
// Determine the final transport type after merge
|
||||
finalTransportType := existing.TransportType
|
||||
if service.TransportType != "" {
|
||||
finalTransportType = service.TransportType
|
||||
}
|
||||
finalStdioConfig := existing.StdioConfig
|
||||
if service.StdioConfig != nil {
|
||||
finalStdioConfig = service.StdioConfig
|
||||
}
|
||||
finalEnvVars := existing.EnvVars
|
||||
if service.EnvVars != nil {
|
||||
finalEnvVars = service.EnvVars
|
||||
}
|
||||
|
||||
// Validate if the final configuration uses stdio transport
|
||||
if finalTransportType == types.MCPTransportStdio && finalStdioConfig != nil {
|
||||
if err := secutils.ValidateStdioConfig(
|
||||
finalStdioConfig.Command,
|
||||
finalStdioConfig.Args,
|
||||
finalEnvVars,
|
||||
); err != nil {
|
||||
logger.GetLogger(ctx).Warnf("MCP service update blocked due to security validation: %v", err)
|
||||
return fmt.Errorf("security validation failed: %w", err)
|
||||
}
|
||||
// Stdio transport is disabled for security reasons
|
||||
if finalTransportType == types.MCPTransportStdio {
|
||||
return fmt.Errorf("stdio transport is disabled for security reasons; please use SSE or HTTP Streamable transport instead")
|
||||
}
|
||||
|
||||
// Store old enabled state BEFORE any updates
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
|
||||
@@ -58,7 +59,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
|
||||
}
|
||||
|
||||
// BatchIndex creates embeddings for multiple content items and saves them to the repository
|
||||
// in batches for efficiency
|
||||
// in batches for efficiency. Uses concurrent batch saving to improve performance.
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Context,
|
||||
embedder embedding.Embedder, indexInfoList []*types.IndexInfo, retrieverTypes []types.RetrieverType,
|
||||
) error {
|
||||
@@ -85,30 +86,123 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batchSize := 40
|
||||
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
|
||||
chunks := utils.ChunkSlice(indexInfoList, batchSize)
|
||||
|
||||
// Use concurrent batch saving for better performance
|
||||
// Limit concurrency to avoid overwhelming the backend
|
||||
const maxConcurrency = 5
|
||||
if len(chunks) <= maxConcurrency {
|
||||
// For small number of batches, use simple concurrency
|
||||
return v.concurrentBatchSave(ctx, chunks, embeddings, batchSize)
|
||||
}
|
||||
|
||||
// For large number of batches, use bounded concurrency
|
||||
return v.boundedConcurrentBatchSave(ctx, chunks, embeddings, batchSize, maxConcurrency)
|
||||
}
|
||||
|
||||
// For non-vector retrieval, use concurrent batch saving as well
|
||||
chunks := utils.ChunkSlice(indexInfoList, 10)
|
||||
const maxConcurrency = 5
|
||||
if len(chunks) <= maxConcurrency {
|
||||
return v.concurrentBatchSaveNoEmbedding(ctx, chunks)
|
||||
}
|
||||
return v.boundedConcurrentBatchSaveNoEmbedding(ctx, chunks, maxConcurrency)
|
||||
}
|
||||
|
||||
// concurrentBatchSave saves all batches concurrently without concurrency limit
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSave(
|
||||
ctx context.Context,
|
||||
chunks [][]*types.IndexInfo,
|
||||
embeddings [][]float32,
|
||||
batchSize int,
|
||||
) error {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
for i, indexChunk := range chunks {
|
||||
g.Go(func() error {
|
||||
params := make(map[string]any)
|
||||
embeddingMap := make(map[string][]float32)
|
||||
for j, indexInfo := range indexChunk {
|
||||
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
|
||||
}
|
||||
params["embedding"] = embeddingMap
|
||||
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
if err != nil {
|
||||
return err
|
||||
return v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
})
|
||||
}
|
||||
return g.Wait()
|
||||
}
|
||||
|
||||
// boundedConcurrentBatchSave saves batches with bounded concurrency using semaphore pattern
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSave(
|
||||
ctx context.Context,
|
||||
chunks [][]*types.IndexInfo,
|
||||
embeddings [][]float32,
|
||||
batchSize int,
|
||||
maxConcurrency int,
|
||||
) error {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
sem := make(chan struct{}, maxConcurrency)
|
||||
|
||||
for i, indexChunk := range chunks {
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
params := make(map[string]any)
|
||||
embeddingMap := make(map[string][]float32)
|
||||
for j, indexInfo := range indexChunk {
|
||||
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
|
||||
}
|
||||
params["embedding"] = embeddingMap
|
||||
return v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
})
|
||||
}
|
||||
var err error
|
||||
for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) {
|
||||
params := make(map[string]any)
|
||||
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return g.Wait()
|
||||
}
|
||||
|
||||
// concurrentBatchSaveNoEmbedding saves all batches concurrently without embeddings
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSaveNoEmbedding(
|
||||
ctx context.Context,
|
||||
chunks [][]*types.IndexInfo,
|
||||
) error {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
for _, indexChunk := range chunks {
|
||||
g.Go(func() error {
|
||||
params := make(map[string]any)
|
||||
return v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
})
|
||||
}
|
||||
return nil
|
||||
return g.Wait()
|
||||
}
|
||||
|
||||
// boundedConcurrentBatchSaveNoEmbedding saves batches with bounded concurrency without embeddings
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSaveNoEmbedding(
|
||||
ctx context.Context,
|
||||
chunks [][]*types.IndexInfo,
|
||||
maxConcurrency int,
|
||||
) error {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
sem := make(chan struct{}, maxConcurrency)
|
||||
|
||||
for _, indexChunk := range chunks {
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
params := make(map[string]any)
|
||||
return v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
})
|
||||
}
|
||||
return g.Wait()
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList deletes vectors by their chunk IDs
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
type knowledgeTagService struct {
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
repo interfaces.KnowledgeTagRepository
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
modelService interfaces.ModelService
|
||||
@@ -31,6 +32,7 @@ type knowledgeTagService struct {
|
||||
func NewKnowledgeTagService(
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
repo interfaces.KnowledgeTagRepository,
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
modelService interfaces.ModelService,
|
||||
@@ -39,6 +41,7 @@ func NewKnowledgeTagService(
|
||||
return &knowledgeTagService{
|
||||
kbService: kbService,
|
||||
repo: repo,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
modelService: modelService,
|
||||
@@ -72,28 +75,7 @@ func (s *knowledgeTagService) ListTags(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results := make([]*types.KnowledgeTagWithStats, 0, len(tags)+1)
|
||||
|
||||
// Add untagged pseudo-tag at the beginning (only on first page and when no keyword filter)
|
||||
if page.Page <= 1 && keyword == "" {
|
||||
untaggedKCount, untaggedCCount, err := s.repo.CountUntaggedReferences(ctx, tenantID, kbID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"kb_id": kbID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, &types.KnowledgeTagWithStats{
|
||||
KnowledgeTag: types.KnowledgeTag{
|
||||
ID: types.UntaggedTagID,
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Name: "未分类",
|
||||
},
|
||||
KnowledgeCount: untaggedKCount,
|
||||
ChunkCount: untaggedCCount,
|
||||
})
|
||||
}
|
||||
results := make([]*types.KnowledgeTagWithStats, 0, len(tags))
|
||||
|
||||
for _, tag := range tags {
|
||||
if tag == nil {
|
||||
@@ -114,11 +96,6 @@ func (s *knowledgeTagService) ListTags(
|
||||
})
|
||||
}
|
||||
|
||||
// Adjust total to include the untagged pseudo-tag
|
||||
if keyword == "" {
|
||||
total++
|
||||
}
|
||||
|
||||
return types.NewPageResult(total, page, results), nil
|
||||
}
|
||||
|
||||
@@ -149,6 +126,10 @@ func (s *knowledgeTagService) CreateTag(
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
// "未分类" tag should have the lowest sort order to appear first
|
||||
if name == types.UntaggedTagName {
|
||||
sortOrder = -1
|
||||
}
|
||||
tag := &types.KnowledgeTag{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: kb.TenantID,
|
||||
@@ -203,6 +184,7 @@ func (s *knowledgeTagService) UpdateTag(
|
||||
}
|
||||
|
||||
// DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag.
|
||||
// For document-type knowledge bases, also deletes all knowledge files under this tag.
|
||||
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
|
||||
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error {
|
||||
if id == "" {
|
||||
@@ -246,9 +228,49 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper function to enqueue knowledge list delete task for document-type knowledge bases
|
||||
enqueueKnowledgeDeleteTask := func() error {
|
||||
if kb.Type != types.KnowledgeBaseTypeDocument {
|
||||
return nil
|
||||
}
|
||||
// Get all knowledge IDs under this tag
|
||||
knowledgeIDs, err := s.knowledgeRepo.ListIDsByTagID(ctx, tenantID, kb.ID, tag.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to list knowledge IDs by tag ID %s: %v", tag.ID, err)
|
||||
return werrors.NewInternalServerError("获取标签下的文档失败")
|
||||
}
|
||||
if len(knowledgeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Enqueue async task to delete knowledge files
|
||||
payload := types.KnowledgeListDeletePayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeIDs: knowledgeIDs,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal knowledge list delete payload: %v", err)
|
||||
return werrors.NewInternalServerError("删除标签下的文档失败")
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue knowledge list delete task: %v", err)
|
||||
return werrors.NewInternalServerError("删除标签下的文档失败")
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued knowledge list delete task %s for %d knowledge files under tag %s", info.ID, len(knowledgeIDs), tag.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentOnly mode: only delete content, keep the tag
|
||||
if contentOnly {
|
||||
if cCount > 0 {
|
||||
// For document-type KB, delete knowledge files first (which will also delete chunks)
|
||||
if kb.Type == types.KnowledgeBaseTypeDocument && kCount > 0 {
|
||||
if err := enqueueKnowledgeDeleteTask(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if cCount > 0 {
|
||||
// For FAQ-type KB, only delete chunks
|
||||
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -259,12 +281,22 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
if !force && (kCount > 0 || cCount > 0) {
|
||||
return werrors.NewBadRequestError("标签仍有知识或FAQ条目引用,无法删除")
|
||||
}
|
||||
// When force=true, delete all chunks under this tag first
|
||||
if force && cCount > 0 {
|
||||
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
|
||||
return err
|
||||
|
||||
// When force=true, delete all content under this tag first
|
||||
if force {
|
||||
// For document-type KB, delete knowledge files first (which will also delete chunks)
|
||||
if kb.Type == types.KnowledgeBaseTypeDocument && kCount > 0 {
|
||||
if err := enqueueKnowledgeDeleteTask(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if cCount > 0 {
|
||||
// For FAQ-type KB, only delete chunks
|
||||
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there are excludeIDs, we cannot delete the tag itself as it still has content
|
||||
if len(excludeIDs) > 0 {
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user