package service import ( "context" "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "io" "mime/multipart" "regexp" "runtime" "slices" "sort" "strings" "time" "github.com/Tencent/WeKnora/docreader/client" "github.com/Tencent/WeKnora/docreader/proto" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/config" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/models/utils" "github.com/Tencent/WeKnora/internal/tracing" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" "github.com/hibiken/asynq" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" ) // Error definitions for knowledge service operations var ( // ErrInvalidFileType is returned when an unsupported file type is provided ErrInvalidFileType = errors.New("unsupported file type") // ErrInvalidURL is returned when an invalid URL is provided ErrInvalidURL = errors.New("invalid URL") // ErrChunkNotFound is returned when a requested chunk cannot be found ErrChunkNotFound = errors.New("chunk not found") // ErrDuplicateFile is returned when trying to add a file that already exists ErrDuplicateFile = errors.New("file already exists") // ErrDuplicateURL is returned when trying to add a URL that already exists ErrDuplicateURL = errors.New("URL already exists") // ErrImageNotParse is returned when trying to update image information without enabling multimodel ErrImageNotParse = errors.New("image not parse without enable multimodel") ) // knowledgeService implements the knowledge service interface // service 实现知识服务接口 type knowledgeService struct { config *config.Config retrieveEngine interfaces.RetrieveEngineRegistry repo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService tenantRepo interfaces.TenantRepository docReaderClient *client.Client chunkService interfaces.ChunkService chunkRepo interfaces.ChunkRepository tagRepo interfaces.KnowledgeTagRepository fileSvc interfaces.FileService modelService interfaces.ModelService task *asynq.Client graphEngine interfaces.RetrieveGraphRepository } const ( manualContentMaxLength = 200000 manualFileExtension = ".md" faqImportBatchSize = 50 // 每批处理的FAQ条目数 ) // NewKnowledgeService creates a new knowledge service instance func NewKnowledgeService( config *config.Config, repo interfaces.KnowledgeRepository, docReaderClient *client.Client, kbService interfaces.KnowledgeBaseService, tenantRepo interfaces.TenantRepository, chunkService interfaces.ChunkService, chunkRepo interfaces.ChunkRepository, tagRepo interfaces.KnowledgeTagRepository, fileSvc interfaces.FileService, modelService interfaces.ModelService, task *asynq.Client, graphEngine interfaces.RetrieveGraphRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ) (interfaces.KnowledgeService, error) { return &knowledgeService{ config: config, repo: repo, kbService: kbService, tenantRepo: tenantRepo, docReaderClient: docReaderClient, chunkService: chunkService, chunkRepo: chunkRepo, tagRepo: tagRepo, fileSvc: fileSvc, modelService: modelService, task: task, graphEngine: graphEngine, retrieveEngine: retrieveEngine, }, nil } // GetRepository gets the knowledge repository // Parameters: // - ctx: Context with authentication and request information // // Returns: // - interfaces.KnowledgeRepository: Knowledge repository func (s *knowledgeService) GetRepository() interfaces.KnowledgeRepository { return s.repo } // CreateKnowledgeFromFile creates a knowledge entry from an uploaded file func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from file") // Use custom filename if provided, otherwise use original filename fileName := file.Filename if customFileName != "" { fileName = customFileName logger.Infof(ctx, "Using custom filename: %s (original: %s)", customFileName, file.Filename) } logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName) if metadata != nil { logger.Infof(ctx, "Received metadata: %v", metadata) } // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // 检查多模态配置完整性 - 只在图片文件时校验 // 检查是否为图片文件 if !IsImageType(getFileType(fileName)) { logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation") } else { // 检查COS配置 switch kb.StorageConfig.Provider { case "cos": if kb.StorageConfig.SecretID == "" || kb.StorageConfig.SecretKey == "" || kb.StorageConfig.Region == "" || kb.StorageConfig.BucketName == "" || kb.StorageConfig.AppID == "" { logger.Error(ctx, "COS configuration incomplete for image multimodal processing") return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") } case "minio": if kb.StorageConfig.BucketName == "" { logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing") return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") } } // 检查VLM配置 if !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" { logger.Error(ctx, "VLM model is not configured") return nil, werrors.NewBadRequestError("上传图片文件需要设置VLM模型") } logger.Info(ctx, "Image multimodal configuration validation passed") } // Validate file type logger.Infof(ctx, "Checking file type: %s", fileName) if !isValidFileType(fileName) { logger.Error(ctx, "Invalid file type") return nil, ErrInvalidFileType } // Calculate file hash for deduplication logger.Info(ctx, "Calculating file hash") hash, err := calculateFileHash(file) if err != nil { logger.Errorf(ctx, "Failed to calculate file hash: %v", err) return nil, err } // Check if file already exists tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "file", FileName: fileName, FileSize: file.Size, FileHash: hash, }) if err != nil { logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) return nil, err } if exists { logger.Infof(ctx, "File already exists: %s", fileName) // Update creation time for existing knowledge if err := s.repo.UpdateKnowledgeColumn(ctx, existingKnowledge.ID, "created_at", time.Now()); err != nil { logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) return nil, err } return existingKnowledge, types.NewDuplicateFileError(existingKnowledge) } // Check storage quota tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { logger.Error(ctx, "Storage quota exceeded") return nil, types.NewStorageQuotaExceededError() } // Convert metadata to JSON format if provided var metadataJSON types.JSON if metadata != nil { metadataBytes, err := json.Marshal(metadata) if err != nil { logger.Errorf(ctx, "Failed to marshal metadata: %v", err) return nil, err } metadataJSON = types.JSON(metadataBytes) } // 验证文件名安全性 safeFilename, isValid := secutils.ValidateInput(fileName) if !isValid { logger.Errorf(ctx, "Invalid filename: %s", fileName) return nil, werrors.NewValidationError("文件名包含非法字符") } // Create knowledge record logger.Info(ctx, "Creating knowledge record") knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, Type: "file", Title: safeFilename, FileName: safeFilename, FileType: getFileType(safeFilename), FileSize: file.Size, FileHash: hash, ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, Metadata: metadataJSON, } // Save knowledge record to database logger.Info(ctx, "Saving knowledge record to database") if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record, ID: %s, error: %v", knowledge.ID, err) return nil, err } // Save the file to storage logger.Infof(ctx, "Saving file, knowledge ID: %s", knowledge.ID) filePath, err := s.fileSvc.SaveFile(ctx, file, knowledge.TenantID, knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to save file, knowledge ID: %s, error: %v", knowledge.ID, err) return nil, err } knowledge.FilePath = filePath // Update knowledge record with file path logger.Info(ctx, "Updating knowledge record with file path") if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to update knowledge with file path, ID: %s, error: %v", knowledge.ID, err) return nil, err } // Enqueue document processing task to Asynq logger.Info(ctx, "Enqueuing document processing task to Asynq") enableMultimodelValue := false if enableMultimodel != nil { enableMultimodelValue = *enableMultimodel } else { enableMultimodelValue = kb.VLMConfig.Enabled } taskPayload := types.DocumentProcessPayload{ TenantID: tenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: kbID, FilePath: filePath, FileName: safeFilename, FileType: getFileType(safeFilename), EnableMultimodel: enableMultimodelValue, } payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal document process task payload: %v", err) // 即使入队失败,也返回knowledge,因为文件已保存 return knowledge, nil } task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default")) info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue document process task: %v", err) // 即使入队失败,也返回knowledge,因为文件已保存 return knowledge, nil } logger.Infof(ctx, "Enqueued document process task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, knowledge.ID) logger.Infof(ctx, "Knowledge from file created successfully, ID: %s", knowledge.ID) return knowledge, nil } // CreateKnowledgeFromURL creates a knowledge entry from a URL source func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from URL") logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // Validate URL format and security logger.Info(ctx, "Validating URL") if !isValidURL(url) || !secutils.IsValidURL(url) { logger.Error(ctx, "Invalid or unsafe URL format") return nil, ErrInvalidURL } // Check if URL already exists in the knowledge base tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Checking if URL exists, tenant ID: %d", tenantID) fileHash := calculateStr(url) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "url", URL: url, FileHash: fileHash, }) if err != nil { logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) return nil, err } if exists { logger.Infof(ctx, "URL already exists: %s", url) // Update creation time for existing knowledge existingKnowledge.CreatedAt = time.Now() existingKnowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, existingKnowledge); err != nil { logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) return nil, err } return existingKnowledge, types.NewDuplicateURLError(existingKnowledge) } // Check storage quota tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { logger.Error(ctx, "Storage quota exceeded") return nil, types.NewStorageQuotaExceededError() } // Create knowledge record logger.Info(ctx, "Creating knowledge record") knowledge := &types.Knowledge{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeBaseID: kbID, Type: "url", Source: url, FileHash: fileHash, ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, } // Save knowledge record logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record: %v", err) return nil, err } // Enqueue URL processing task to Asynq logger.Info(ctx, "Enqueuing URL processing task to Asynq") enableMultimodelValue := false if enableMultimodel != nil { enableMultimodelValue = *enableMultimodel } else { enableMultimodelValue = kb.VLMConfig.Enabled } taskPayload := types.DocumentProcessPayload{ TenantID: tenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: kbID, URL: url, EnableMultimodel: enableMultimodelValue, } payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal URL process task payload: %v", err) return knowledge, nil } task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default")) info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue URL process task: %v", err) return knowledge, nil } logger.Infof(ctx, "Enqueued URL process task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, knowledge.ID) logger.Infof(ctx, "Knowledge from URL created successfully, ID: %s", knowledge.ID) return knowledge, nil } // CreateKnowledgeFromPassage creates a knowledge entry from text passages func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string, ) (*types.Knowledge, error) { return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, false) } // CreateKnowledgeFromPassageSync creates a knowledge entry from text passages and waits for indexing to complete. func (s *knowledgeService) CreateKnowledgeFromPassageSync(ctx context.Context, kbID string, passage []string, ) (*types.Knowledge, error) { return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, true) } // CreateKnowledgeFromManual creates or saves manual Markdown knowledge content. func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context, kbID string, payload *types.ManualKnowledgePayload, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating manual knowledge entry") if payload == nil { return nil, werrors.NewBadRequestError("请求内容不能为空") } cleanContent := secutils.CleanMarkdown(payload.Content) if strings.TrimSpace(cleanContent) == "" { return nil, werrors.NewValidationError("内容不能为空") } if len([]rune(cleanContent)) > manualContentMaxLength { return nil, werrors.NewValidationError(fmt.Sprintf("内容长度超出限制(最多%d个字符)", manualContentMaxLength)) } safeTitle, ok := secutils.ValidateInput(payload.Title) if !ok { return nil, werrors.NewValidationError("标题包含非法字符或超出长度限制") } status := strings.ToLower(strings.TrimSpace(payload.Status)) if status == "" { status = types.ManualKnowledgeStatusDraft } if status != types.ManualKnowledgeStatusDraft && status != types.ManualKnowledgeStatusPublish { return nil, werrors.NewValidationError("状态仅支持 draft 或 publish") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) now := time.Now() title := safeTitle if title == "" { title = fmt.Sprintf("Knowledge-%s", now.Format("20060102-150405")) } fileName := ensureManualFileName(title) meta := types.NewManualKnowledgeMetadata(cleanContent, status, 1) knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, Type: types.KnowledgeTypeManual, Title: title, Description: "", Source: types.KnowledgeTypeManual, ParseStatus: types.ManualKnowledgeStatusDraft, EnableStatus: "disabled", CreatedAt: now, UpdatedAt: now, EmbeddingModelID: kb.EmbeddingModelID, FileName: fileName, FileType: types.KnowledgeTypeManual, } if err := knowledge.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata: %v", err) return nil, err } knowledge.EnsureManualDefaults() if status == types.ManualKnowledgeStatusPublish { knowledge.ParseStatus = "pending" } if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create manual knowledge record: %v", err) return nil, err } if status == types.ManualKnowledgeStatusPublish { logger.Infof(ctx, "Manual knowledge created, scheduling indexing, ID: %s", knowledge.ID) s.triggerManualProcessing(ctx, kb, knowledge, cleanContent, false) } return knowledge, nil } // createKnowledgeFromPassageInternal consolidates the common logic for creating knowledge from passages. // When syncMode is true, chunk processing is performed synchronously; otherwise, it's processed asynchronously. func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Context, kbID string, passage []string, syncMode bool, ) (*types.Knowledge, error) { if syncMode { logger.Info(ctx, "Start creating knowledge from passage (sync)") } else { logger.Info(ctx, "Start creating knowledge from passage") } logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage)) // 验证段落内容安全性 safePassages := make([]string, 0, len(passage)) for i, p := range passage { safePassage, isValid := secutils.ValidateInput(p) if !isValid { logger.Errorf(ctx, "Invalid passage content at index %d", i) return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1)) } safePassages = append(safePassages, safePassage) } // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // Create knowledge record if syncMode { logger.Info(ctx, "Creating knowledge record (sync)") } else { logger.Info(ctx, "Creating knowledge record") } knowledge := &types.Knowledge{ ID: uuid.New().String(), TenantID: ctx.Value(types.TenantIDContextKey).(uint), KnowledgeBaseID: kbID, Type: "passage", ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, } // Save knowledge record logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record: %v", err) return nil, err } // Process passages if syncMode { logger.Info(ctx, "Processing passage synchronously") s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) logger.Infof(ctx, "Knowledge from passage created successfully (sync), ID: %s", knowledge.ID) } else { // Enqueue passage processing task to Asynq logger.Info(ctx, "Enqueuing passage processing task to Asynq") tenantID := ctx.Value(types.TenantIDContextKey).(uint) taskPayload := types.DocumentProcessPayload{ TenantID: tenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: kbID, Passages: safePassages, EnableMultimodel: false, // 文本段落不支持多模态 } payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal passage process task payload: %v", err) // 即使入队失败,也返回knowledge return knowledge, nil } task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default")) info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue passage process task: %v", err) return knowledge, nil } logger.Infof(ctx, "Enqueued passage process task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, knowledge.ID) logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID) } return knowledge, nil } // GetKnowledgeByID retrieves a knowledge entry by its ID func (s *knowledgeService) GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) { logger.Info(ctx, "Start getting knowledge by ID") logger.Infof(ctx, "Knowledge ID: %s", id) tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Tenant ID: %d", tenantID) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "knowledge_id": id, "tenant_id": tenantID, }) return nil, err } logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, type: %s", knowledge.ID, knowledge.Type) return knowledge, nil } // ListKnowledgeByKnowledgeBaseID returns all knowledge entries in a knowledge base func (s *knowledgeService) ListKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, ) ([]*types.Knowledge, error) { return s.repo.ListKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint), kbID) } // ListPagedKnowledgeByKnowledgeBaseID returns paginated knowledge entries in a knowledge base func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, page *types.Pagination, tagID string, ) (*types.PageResult, error) { knowledges, total, err := s.repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint), kbID, page, tagID) if err != nil { return nil, err } return types.NewPageResult(total, page, knowledges), nil } // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error { // Get the knowledge entry knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) if err != nil { return err } wg := errgroup.Group{} // Delete knowledge embeddings from vector store wg.Go(func() error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } return nil }) // Delete all chunks associated with this knowledge wg.Go(func() error { if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") return err } return nil }) // Delete the physical file if it exists wg.Go(func() error { if knowledge.FilePath != "" { if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") } } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) tenantInfo.StorageUsed -= knowledge.StorageSize if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") } return nil }) // Delete the knowledge graph wg.Go(func() error { namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") return err } return nil }) if err = wg.Wait(); err != nil { return err } // Delete the knowledge entry itself from the database return s.repo.DeleteKnowledge(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) } // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error { if len(ids) == 0 { return nil } // 1. Get the knowledge entry tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantInfo.ID, ids) if err != nil { return err } wg := errgroup.Group{} // 2. Delete knowledge embeddings from vector store wg.Go(func() error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } group := map[string][]string{} for _, knowledge := range knowledgeList { group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID) } for embeddingModelID, knowledgeList := range group { embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed") return err } if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } } return nil }) // 3. Delete all chunks associated with this knowledge wg.Go(func() error { if err := s.chunkService.DeleteByKnowledgeList(ctx, ids); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") return err } return nil }) // 4. Delete the physical file if it exists wg.Go(func() error { storageAdjust := int64(0) for _, knowledge := range knowledgeList { if knowledge.FilePath != "" { if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") } } storageAdjust -= knowledge.StorageSize } tenantInfo.StorageUsed += storageAdjust if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") } return nil }) // Delete the knowledge graph wg.Go(func() error { namespaces := []types.NameSpace{} for _, knowledge := range knowledgeList { namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}) } if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") return err } return nil }) if err = wg.Wait(); err != nil { return err } // 5. Delete the knowledge entry itself from the database return s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids) } func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowledge, targetKB *types.KnowledgeBase) (err error) { if src.ParseStatus != "completed" { logger.GetLogger(ctx).WithField("knowledge_id", src.ID).Errorf("MoveKnowledge parse status is not completed") return nil } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) dst := &types.Knowledge{ ID: uuid.New().String(), TenantID: targetKB.TenantID, KnowledgeBaseID: targetKB.ID, Type: src.Type, Title: src.Title, Description: src.Description, Source: src.Source, ParseStatus: "processing", EnableStatus: "disabled", EmbeddingModelID: targetKB.EmbeddingModelID, FileName: src.FileName, FileType: src.FileType, FileSize: src.FileSize, FileHash: src.FileHash, FilePath: src.FilePath, StorageSize: src.StorageSize, Metadata: src.Metadata, } defer func() { if err != nil { dst.ParseStatus = "failed" dst.ErrorMessage = err.Error() _ = s.repo.UpdateKnowledge(ctx, dst) logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge failed to move knowledge") } else { dst.ParseStatus = "completed" dst.EnableStatus = "enabled" _ = s.repo.UpdateKnowledge(ctx, dst) logger.GetLogger(ctx).WithField("knowledge_id", dst.ID).Infof("MoveKnowledge move knowledge successfully") } }() if err = s.repo.CreateKnowledge(ctx, dst); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge create knowledge failed") return } tenantInfo.StorageUsed += dst.StorageSize if err = s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, dst.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge update tenant storage used failed") return } if err = s.CloneChunk(ctx, src, dst); err != nil { logger.GetLogger(ctx).WithField("knowledge_id", dst.ID). WithField("error", err).Errorf("MoveKnowledge move chunks failed") return } return } // processDocument handles asynchronous processing of document files func (s *knowledgeService) processDocument(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool, ) { logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel) ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument") defer span.End() span.SetAttributes( attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)), attribute.String("knowledge_base_id", kb.ID), attribute.Int("tenant_id", int(kb.TenantID)), attribute.String("knowledge_id", knowledge.ID), attribute.String("file_name", knowledge.FileName), attribute.String("file_type", knowledge.FileType), attribute.String("file_path", knowledge.FilePath), attribute.Int64("file_size", knowledge.FileSize), attribute.String("embedding_model", knowledge.EmbeddingModelID), attribute.Bool("enable_multimodal", enableMultimodel), ) logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String()) if !enableMultimodel && IsImageType(knowledge.FileType) { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = ErrImageNotParse.Error() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(ErrImageNotParse) return } // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { span.RecordError(err) return } // Read and chunk the document f, err := file.Open() if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument open file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } defer f.Close() span.AddEvent("start read file") contentBytes, err := io.ReadAll(f) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Split file into chunks using document reader service logger.GetLogger(ctx).Infof("processDocument split file content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v", len(contentBytes), knowledge.FileName, knowledge.FileType, kb.ChunkingConfig.Separators, enableMultimodel) var vlmConfig *proto.VLMConfig if enableMultimodel { vlmConfig, err = s.getVLMProtoConfig(ctx, kb) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument build VLM config failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } if vlmConfig == nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). Error("processDocument enable multimodal but VLM config missing") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "VLM 配置缺失" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(errors.New("vlm config missing")) return } } resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ FileContent: contentBytes, FileName: knowledge.FileName, FileType: knowledge.FileType, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: vlmConfig, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument read file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Process and store chunks span.AddEvent("start process chunks") s.processChunks(ctx, kb, knowledge, resp.Chunks) } // processDocumentFromURL handles asynchronous processing of URL content func (s *knowledgeService) processDocumentFromURL(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool, ) { // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { return } logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel) // Fetch and chunk content from URL var vlmConfig *proto.VLMConfig if enableMultimodel { cfg, cfgErr := s.getVLMProtoConfig(ctx, kb) if cfgErr != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", cfgErr).Errorf("processDocumentFromURL build VLM config failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = cfgErr.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } if cfg == nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). Error("processDocumentFromURL enable multimodal but VLM config missing") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "VLM 配置缺失" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } vlmConfig = cfg } resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{ Url: url, Title: knowledge.Title, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: vlmConfig, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } // Process and store chunks s.processChunks(ctx, kb, knowledge, resp.Chunks) } // processDocumentFromPassage handles asynchronous processing of text passages func (s *knowledgeService) processDocumentFromPassage(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string, ) { // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { return } // Convert passages to chunks chunks := make([]*proto.Chunk, 0, len(passage)) start, end := 0, 0 for i, p := range passage { if p == "" { continue } end += len([]rune(p)) chunk := &proto.Chunk{ Content: p, Seq: int32(i), Start: int32(start), End: int32(end), } start = end chunks = append(chunks, chunk) } // Process and store chunks s.processChunks(ctx, kb, knowledge, chunks) } // processChunks processes chunks and creates embeddings for knowledge content func (s *knowledgeService) processChunks(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*proto.Chunk, ) { ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processChunks") defer span.End() span.SetAttributes( attribute.Int("tenant_id", int(knowledge.TenantID)), attribute.String("knowledge_base_id", knowledge.KnowledgeBaseID), attribute.String("knowledge_id", knowledge.ID), attribute.String("embedding_model_id", kb.EmbeddingModelID), attribute.Int("chunk_count", len(chunks)), ) // Get embedding model for vectorization embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed") span.RecordError(err) return } // 幂等性处理:清理旧的chunks和索引数据,避免重复数据 logger.Infof(ctx, "Cleaning up existing chunks and index data for knowledge: %s", knowledge.ID) // 删除旧的chunks if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.Warnf(ctx, "Failed to delete existing chunks (may not exist): %v", err) // 不返回错误,继续处理(可能没有旧数据) } // 删除旧的索引数据 tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err == nil { if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err) // 不返回错误,继续处理(可能没有旧数据) } else { logger.Infof(ctx, "Successfully deleted existing index data for knowledge: %s", knowledge.ID) } } // 删除知识图谱数据(如果存在) namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { logger.Warnf(ctx, "Failed to delete existing graph data (may not exist): %v", err) // 不返回错误,继续处理 } logger.Infof(ctx, "Cleanup completed, starting to process new chunks") // Generate document summary - 只使用文本类型的 Chunk chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get summary model failed") span.RecordError(err) return } // Create chunk objects from proto chunks maxSeq := 0 // 统计图片相关的子Chunk数量,用于扩展insertChunks的容量 imageChunkCount := 0 for _, chunkData := range chunks { if len(chunkData.Images) > 0 { // 为每个图片的OCR和Caption分别创建一个Chunk imageChunkCount += len(chunkData.Images) * 2 } if int(chunkData.Seq) > maxSeq { maxSeq = int(chunkData.Seq) } } // 重新分配容量,考虑图片相关的Chunk insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount) for _, chunkData := range chunks { if strings.TrimSpace(chunkData.Content) == "" { continue } // 创建主文本Chunk textChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: chunkData.Content, ChunkIndex: int(chunkData.Seq), IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(chunkData.Start), EndAt: int(chunkData.End), ChunkType: types.ChunkTypeText, } var chunkImages []types.ImageInfo insertChunks = append(insertChunks, textChunk) // 处理图片信息 if len(chunkData.Images) > 0 { logger.GetLogger(ctx).Infof("Processing %d images in chunk #%d", len(chunkData.Images), chunkData.Seq) for i, img := range chunkData.Images { // 保存图片信息到文本Chunk imageInfo := types.ImageInfo{ URL: img.Url, OriginalURL: img.OriginalUrl, StartPos: int(img.Start), EndPos: int(img.End), OCRText: img.OcrText, Caption: img.Caption, } chunkImages = append(chunkImages, imageInfo) // 将ImageInfo序列化为JSON imageInfoJSON, err := json.Marshal([]types.ImageInfo{imageInfo}) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") continue } // 如果有OCR文本,创建OCR Chunk if img.OcrText != "" { ocrChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: img.OcrText, ChunkIndex: maxSeq + i*100 + 1, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(img.Start), EndAt: int(img.End), ChunkType: types.ChunkTypeImageOCR, ParentChunkID: textChunk.ID, ImageInfo: string(imageInfoJSON), } insertChunks = append(insertChunks, ocrChunk) logger.GetLogger(ctx).Infof("Created OCR chunk for image %d in chunk #%d", i, chunkData.Seq) } // 如果有图片描述,创建Caption Chunk if img.Caption != "" { captionChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: img.Caption, ChunkIndex: maxSeq + i*100 + 2, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(img.Start), EndAt: int(img.End), ChunkType: types.ChunkTypeImageCaption, ParentChunkID: textChunk.ID, ImageInfo: string(imageInfoJSON), } insertChunks = append(insertChunks, captionChunk) logger.GetLogger(ctx).Infof("Created caption chunk for image %d in chunk #%d", i, chunkData.Seq) } } imageInfoJSON, err := json.Marshal(chunkImages) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") continue } textChunk.ImageInfo = string(imageInfoJSON) } } // Sort chunks by index for proper ordering sort.Slice(insertChunks, func(i, j int) bool { return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex }) // 仅为文本类型的Chunk设置前后关系 textChunks := make([]*types.Chunk, 0, len(chunks)) for _, chunk := range insertChunks { if chunk.ChunkType == types.ChunkTypeText { textChunks = append(textChunks, chunk) } } // 设置文本Chunk之间的前后关系 for i, chunk := range textChunks { if i > 0 { textChunks[i-1].NextChunkID = chunk.ID } if i < len(textChunks)-1 { textChunks[i+1].PreChunkID = chunk.ID } } span.AddEvent("extract summary") summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processChunks get summary failed, use first chunk as description") if len(textChunks) > 0 { knowledge.Description = textChunks[0].Content } } else { knowledge.Description = summary } span.SetAttributes(attribute.String("summary", knowledge.Description)) // 批量索引 if strings.TrimSpace(knowledge.Description) != "" && len(textChunks) > 0 { sChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, knowledge.Description), ChunkIndex: maxSeq + 3, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: 0, EndAt: 0, ChunkType: types.ChunkTypeSummary, ParentChunkID: textChunks[0].ID, } logger.GetLogger(ctx).Infof("Created summary chunk for %s with index %d", sChunk.ParentChunkID, sChunk.ChunkIndex) insertChunks = append(insertChunks, sChunk) } // Create index information for each chunk indexInfoList := utils.MapSlice(insertChunks, func(chunk *types.Chunk) *types.IndexInfo { return &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, } }) // Initialize retrieval engine // Calculate storage size required for embeddings span.AddEvent("estimate storage size") totalStorageSize := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfoList) if tenantInfo.StorageQuota > 0 { // Re-fetch tenant storage information tenantInfo, err = s.tenantRepo.GetTenantByID(ctx, tenantInfo.ID) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Check if there's enough storage quota available if tenantInfo.StorageUsed+totalStorageSize > tenantInfo.StorageQuota { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "存储空间不足" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(errors.New("storage quota exceeded")) return } } // Save chunks to database span.AddEvent("create chunks") if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } span.AddEvent("batch index") err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) // delete failed chunks if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.Errorf(ctx, "Delete chunks failed: %v", err) } // delete index if err := retrieveEngine.DeleteByKnowledgeIDList( ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), ); err != nil { logger.Errorf(ctx, "Delete index failed: %v", err) } span.RecordError(err) return } logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList)) logger.Infof(ctx, "processChunks create relationship rag task") if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled { for _, chunk := range textChunks { err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed") span.RecordError(err) } } } // Update knowledge status to completed knowledge.ParseStatus = "completed" knowledge.EnableStatus = "enabled" knowledge.StorageSize = totalStorageSize now := time.Now() knowledge.ProcessedAt = &now knowledge.UpdatedAt = now if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed") } // Update tenant's storage usage tenantInfo.StorageUsed += totalStorageSize if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update tenant storage used failed") } logger.GetLogger(ctx).Infof("processChunks successfully") } // GetSummary generates a summary for knowledge content using an AI model func (s *knowledgeService) getSummary(ctx context.Context, summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk, ) (string, error) { // Get knowledge info from the first chunk if len(chunks) == 0 { return "", fmt.Errorf("no chunks provided for summary generation") } // concat chunk contents chunkContents := "" allImageInfos := make([]*types.ImageInfo, 0) // then, sort chunks by StartAt sortedChunks := make([]*types.Chunk, len(chunks)) copy(sortedChunks, chunks) sort.Slice(sortedChunks, func(i, j int) bool { return sortedChunks[i].StartAt < sortedChunks[j].StartAt }) // concat chunk contents and collect image infos for _, chunk := range sortedChunks { if chunk.EndAt > 4096 { break } chunkContents = string([]rune(chunkContents)[:chunk.StartAt]) + chunk.Content if chunk.ImageInfo != "" { var images []*types.ImageInfo if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err == nil { allImageInfos = append(allImageInfos, images...) } } } // remove markdown image syntax re := regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) chunkContents = re.ReplaceAllString(chunkContents, "") // collect all image infos if len(allImageInfos) > 0 { // add image infos to chunk contents var imageAnnotations string for _, img := range allImageInfos { if img.Caption != "" { imageAnnotations += fmt.Sprintf("\n[图片描述: %s]", img.Caption) } if img.OCRText != "" { imageAnnotations += fmt.Sprintf("\n[图片文字: %s]", img.OCRText) } } // concat chunk contents and image annotations chunkContents = chunkContents + imageAnnotations } if len(chunkContents) < 300 { return chunkContents, nil } // Prepare content with metadata for summary generation contentWithMetadata := chunkContents // Add knowledge metadata if available if knowledge != nil { metadataIntro := fmt.Sprintf("文档类型: %s\n文件名称: %s\n", knowledge.FileType, knowledge.FileName) // Add additional metadata if available if knowledge.Type != "" { metadataIntro += fmt.Sprintf("知识类型: %s\n", knowledge.Type) } // Prepend metadata to content contentWithMetadata = metadataIntro + "\n内容:\n" + contentWithMetadata } // Generate summary using AI model thinking := false summary, err := summaryModel.Chat(ctx, []chat.Message{ { Role: "system", Content: s.config.Conversation.GenerateSummaryPrompt, }, { Role: "user", Content: contentWithMetadata, }, }, &chat.ChatOptions{ Temperature: 0.3, MaxTokens: 1024, Thinking: &thinking, }) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("GetSummary failed") return "", err } logger.GetLogger(ctx).WithField("summary", summary.Content).Infof("GetSummary success") return summary.Content, nil } // GetKnowledgeFile retrieves the physical file associated with a knowledge entry func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { // Get knowledge record knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) if err != nil { return nil, "", err } // Get the file from storage file, err := s.fileSvc.GetFile(ctx, knowledge.FilePath) if err != nil { return nil, "", err } return file, knowledge.FileName, nil } func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { record, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge record: %v", err) return err } // if need other fields update, please add here if knowledge.Title != "" { record.Title = knowledge.Title } // Update knowledge record in the repository if err := s.repo.UpdateKnowledge(ctx, record); err != nil { logger.Errorf(ctx, "Failed to update knowledge: %v", err) return err } logger.Infof(ctx, "Knowledge updated successfully, ID: %s", knowledge.ID) return nil } // UpdateManualKnowledge updates manual Markdown knowledge content. func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context, knowledgeID string, payload *types.ManualKnowledgePayload, ) (*types.Knowledge, error) { logger.Info(ctx, "Start updating manual knowledge entry") if payload == nil { return nil, werrors.NewBadRequestError("请求内容不能为空") } cleanContent := secutils.CleanMarkdown(payload.Content) if strings.TrimSpace(cleanContent) == "" { return nil, werrors.NewValidationError("内容不能为空") } if len([]rune(cleanContent)) > manualContentMaxLength { return nil, werrors.NewValidationError(fmt.Sprintf("内容长度超出限制(最多%d个字符)", manualContentMaxLength)) } safeTitle, ok := secutils.ValidateInput(payload.Title) if !ok { return nil, werrors.NewValidationError("标题包含非法字符或超出长度限制") } status := strings.ToLower(strings.TrimSpace(payload.Status)) if status == "" { status = types.ManualKnowledgeStatusDraft } if status != types.ManualKnowledgeStatusDraft && status != types.ManualKnowledgeStatusPublish { return nil, werrors.NewValidationError("状态仅支持 draft 或 publish") } tenantID := ctx.Value(types.TenantIDContextKey).(uint) existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to load knowledge: %v", err) return nil, err } if !existing.IsManual() { return nil, werrors.NewBadRequestError("仅支持手工知识的在线编辑") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base for manual update: %v", err) return nil, err } var version int if meta, err := existing.ManualMetadata(); err == nil && meta != nil { version = meta.Version + 1 } else { version = 1 } meta := types.NewManualKnowledgeMetadata(cleanContent, status, version) if err := existing.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata during update: %v", err) return nil, err } if safeTitle != "" { existing.Title = safeTitle } else if existing.Title == "" { existing.Title = fmt.Sprintf("手工知识-%s", time.Now().Format("20060102-150405")) } existing.FileName = ensureManualFileName(existing.Title) existing.FileType = types.KnowledgeTypeManual existing.Type = types.KnowledgeTypeManual existing.Source = types.KnowledgeTypeManual existing.EnableStatus = "disabled" existing.UpdatedAt = time.Now() if err := s.cleanupKnowledgeResources(ctx, existing); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "knowledge_id": knowledgeID, }) return nil, err } existing.EmbeddingModelID = kb.EmbeddingModelID if status == types.ManualKnowledgeStatusDraft { existing.ParseStatus = types.ManualKnowledgeStatusDraft existing.Description = "" existing.ProcessedAt = nil if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { logger.Errorf(ctx, "Failed to persist manual draft: %v", err) return nil, err } return existing, nil } existing.ParseStatus = "pending" existing.Description = "" existing.ProcessedAt = nil if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { logger.Errorf(ctx, "Failed to persist manual knowledge before indexing: %v", err) return nil, err } logger.Infof(ctx, "Manual knowledge updated, scheduling indexing, ID: %s", existing.ID) s.triggerManualProcessing(ctx, kb, existing, cleanContent, false) return existing, nil } // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls": return true default: return false } } // getFileType extracts the file extension from a filename func getFileType(filename string) string { ext := strings.Split(filename, ".") if len(ext) < 2 { return "unknown" } return ext[len(ext)-1] } // isValidURL verifies if a URL is valid // isValidURL 检查URL是否有效 func isValidURL(url string) bool { if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { return true } return false } // GetKnowledgeBatch retrieves multiple knowledge entries by their IDs func (s *knowledgeService) GetKnowledgeBatch(ctx context.Context, tenantID uint, ids []string, ) ([]*types.Knowledge, error) { if len(ids) == 0 { return nil, nil } return s.repo.GetKnowledgeBatch(ctx, tenantID, ids) } // calculateFileHash calculates MD5 hash of a file func calculateFileHash(file *multipart.FileHeader) (string, error) { f, err := file.Open() if err != nil { return "", err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return "", err } // Reset file pointer for subsequent operations if _, err := f.Seek(0, 0); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } func calculateStr(strList ...string) string { h := md5.New() input := strings.Join(strList, "") h.Write([]byte(input)) return hex.EncodeToString(h.Sum(nil)) } func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error { srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID) if err != nil { logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) return err } addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) batch := 10 g, gctx := errgroup.WithContext(ctx) for ids := range slices.Chunk(delKnowledge, batch) { ids = ids g.Go(func() error { err := s.DeleteKnowledgeList(gctx, ids) if err != nil { logger.Errorf(gctx, "delete partial knowledge %v: %w", ids, err) return err } return nil }) } err = g.Wait() if err != nil { logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) return err } // Copy context out of auto-stop task g, gctx = errgroup.WithContext(ctx) g.SetLimit(batch) for _, knowledge := range addKnowledge { knowledge = knowledge g.Go(func() error { srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) if err != nil { logger.Errorf(gctx, "get knowledge %s: %w", knowledge, err) return err } err = s.cloneKnowledge(gctx, srcKn, dstKB) if err != nil { logger.Errorf(gctx, "clone knowledge %s: %w", knowledge, err) return err } return nil }) } err = g.Wait() if err != nil { logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) return err } return nil } func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, chunks []*types.Chunk) error { // Get embedding model from knowledge base sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, sourceKB.EmbeddingModelID) if err != nil { return err } // Initialize composite retrieve engine from tenant configuration indexInfo := make([]*types.IndexInfo, 0, len(chunks)) ids := make([]string, 0, len(chunks)) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kbID { logger.Warnf(ctx, "Knowledge base ID mismatch: %s != %s", chunk.KnowledgeBaseID, kbID) continue } indexInfo = append(indexInfo, &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) ids = append(ids, chunk.ID) } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } // Delete old vector representation of the chunk err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions()) if err != nil { return err } // Index updated chunk content with new vector representation err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo) if err != nil { return err } return nil } func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID string, chunkID string, imageInfo string) error { var images []*types.ImageInfo if err := json.Unmarshal([]byte(imageInfo), &images); err != nil { logger.Errorf(ctx, "Failed to unmarshal image info: %v", err) return err } if len(images) != 1 { logger.Warnf(ctx, "Expected exactly one image info, got %d", len(images)) return nil } image := images[0] // Retrieve all chunks with the given parent chunk ID chunk, err := s.chunkService.GetChunkByID(ctx, chunkID) if err != nil { logger.Errorf(ctx, "Failed to get chunk: %v", err) return err } chunk.ImageInfo = imageInfo tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunkChildren, err := s.chunkService.ListChunkByParentID(ctx, tenantID, chunkID) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "parent_chunk_id": chunkID, "tenant_id": tenantID, }) return err } logger.Infof(ctx, "Found %d chunks with parent chunk ID: %s", len(chunkChildren), chunkID) // Iterate through each chunk and update its content based on the image information updateChunk := []*types.Chunk{chunk} var addChunk []*types.Chunk // Track whether we've found OCR and caption child chunks for this image hasOCRChunk := false hasCaptionChunk := false for i, child := range chunkChildren { // Skip chunks that are not image types var cImageInfo []*types.ImageInfo err = json.Unmarshal([]byte(child.ImageInfo), &cImageInfo) if err != nil { logger.Warnf(ctx, "Failed to unmarshal image %s info: %v", child.ID, err) continue } if len(cImageInfo) == 0 { continue } if cImageInfo[0].OriginalURL != image.OriginalURL { logger.Warnf(ctx, "Skipping chunk ID: %s, image URL mismatch: %s != %s", child.ID, cImageInfo[0].OriginalURL, image.OriginalURL) continue } // Mark that we've found chunks for this image if child.ChunkType == types.ChunkTypeImageCaption { hasCaptionChunk = true // Update caption if it has changed if image.Caption != cImageInfo[0].Caption { child.Content = image.Caption child.ImageInfo = imageInfo updateChunk = append(updateChunk, chunkChildren[i]) } } else if child.ChunkType == types.ChunkTypeImageOCR { hasOCRChunk = true // Update OCR if it has changed if image.OCRText != cImageInfo[0].OCRText { child.Content = image.OCRText child.ImageInfo = imageInfo updateChunk = append(updateChunk, chunkChildren[i]) } } } // Create a new caption chunk if it doesn't exist and we have caption data if !hasCaptionChunk && image.Caption != "" { captionChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, Content: image.Caption, ChunkType: types.ChunkTypeImageCaption, ParentChunkID: chunk.ID, ImageInfo: imageInfo, } addChunk = append(addChunk, captionChunk) logger.Infof(ctx, "Created new caption chunk ID: %s for image URL: %s", captionChunk.ID, image.OriginalURL) } // Create a new OCR chunk if it doesn't exist and we have OCR data if !hasOCRChunk && image.OCRText != "" { ocrChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, Content: image.OCRText, ChunkType: types.ChunkTypeImageOCR, ParentChunkID: chunk.ID, ImageInfo: imageInfo, } addChunk = append(addChunk, ocrChunk) logger.Infof(ctx, "Created new OCR chunk ID: %s for image URL: %s", ocrChunk.ID, image.OriginalURL) } logger.Infof(ctx, "Updated %d chunks out of %d total chunks", len(updateChunk), len(chunkChildren)+1) if len(addChunk) > 0 { err := s.chunkService.CreateChunks(ctx, addChunk) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "add_chunk_size": len(addChunk), }) return err } } // Update the chunks for _, c := range updateChunk { err := s.chunkService.UpdateChunk(ctx, c) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": c.ID, "knowledge_id": c.KnowledgeID, }) return err } } // Update the chunk vector err = s.updateChunkVector(ctx, chunk.KnowledgeBaseID, append(updateChunk, addChunk...)) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": chunk.ID, "knowledge_id": chunk.KnowledgeID, }) return err } // Update the knowledge file hash knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } fileHash := calculateStr(knowledgeID, knowledge.FileHash, imageInfo) knowledge.FileHash = fileHash err = s.repo.UpdateKnowledge(ctx, knowledge) if err != nil { logger.Warnf(ctx, "Failed to update knowledge file hash: %v", err) } logger.Infof(ctx, "Updated chunk successfully, chunk ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID) return nil } // CloneChunk clone chunks from one knowledge to another // This method transfers a chunk from a source knowledge document to a target knowledge document // It handles the creation of new chunks in the target knowledge and updates the vector database accordingly // Parameters: // - ctx: Context with authentication and request information // - src: Source knowledge document containing the chunk to move // - dst: Target knowledge document where the chunk will be moved // // Returns: // - error: Any error encountered during the move operation // // This method handles the chunk transfer logic, including creating new chunks in the target knowledge // and updating the vector database representation of the moved chunks. // It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained // by mapping the source chunk IDs to the new target chunk IDs. func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) error { chunkPage := 1 chunkPageSize := 100 srcTodst := map[string]string{} targetChunks := make([]*types.Chunk, 0, 10) chunkType := []types.ChunkType{ types.ChunkTypeText, types.ChunkTypeSummary, types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, } for { sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx, src.TenantID, src.ID, &types.Pagination{ Page: chunkPage, PageSize: chunkPageSize, }, chunkType, "", ) chunkPage++ if err != nil { return err } if len(sourceChunks) == 0 { break } for _, sourceChunk := range sourceChunks { targetChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: dst.TenantID, KnowledgeID: dst.ID, KnowledgeBaseID: dst.KnowledgeBaseID, Content: sourceChunk.Content, ChunkIndex: sourceChunk.ChunkIndex, IsEnabled: sourceChunk.IsEnabled, StartAt: sourceChunk.StartAt, EndAt: sourceChunk.EndAt, PreChunkID: sourceChunk.PreChunkID, NextChunkID: sourceChunk.NextChunkID, ChunkType: sourceChunk.ChunkType, ParentChunkID: sourceChunk.ParentChunkID, ImageInfo: sourceChunk.ImageInfo, } targetChunks = append(targetChunks, targetChunk) srcTodst[sourceChunk.ID] = targetChunk.ID } } for _, targetChunk := range targetChunks { if val, ok := srcTodst[targetChunk.PreChunkID]; ok { targetChunk.PreChunkID = val } else { targetChunk.PreChunkID = "" } if val, ok := srcTodst[targetChunk.NextChunkID]; ok { targetChunk.NextChunkID = val } else { targetChunk.NextChunkID = "" } if val, ok := srcTodst[targetChunk.ParentChunkID]; ok { targetChunk.ParentChunkID = val } else { targetChunk.ParentChunkID = "" } } for chunks := range slices.Chunk(targetChunks, chunkPageSize) { err := s.chunkRepo.CreateChunks(ctx, chunks) if err != nil { return err } } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID) if err != nil { return err } if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID, map[string]string{src.ID: dst.ID}, srcTodst, embeddingModel.GetDimensions(), ); err != nil { return err } return nil } // ListFAQEntries lists FAQ entries under a FAQ knowledge base. func (s *knowledgeService) ListFAQEntries(ctx context.Context, kbID string, page *types.Pagination, tagID string, ) (*types.PageResult, error) { if page == nil { page = &types.Pagination{} } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) if err != nil { return nil, err } if faqKnowledge == nil { return types.NewPageResult(0, page, []*types.FAQEntry{}), nil } chunkType := []types.ChunkType{types.ChunkTypeFAQ} chunks, total, err := s.chunkRepo.ListPagedChunksByKnowledgeID( ctx, tenantID, faqKnowledge.ID, page, chunkType, tagID, ) if err != nil { return nil, err } kb.EnsureDefaults() entries := make([]*types.FAQEntry, 0, len(chunks)) for _, chunk := range chunks { entry, err := s.chunkToFAQEntry(chunk, kb) if err != nil { return nil, err } entries = append(entries, entry) } return types.NewPageResult(total, page, entries), nil } // UpsertFAQEntries imports or appends FAQ entries asynchronously. // Returns task ID for tracking import progress. func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload, ) (string, error) { if payload == nil || len(payload.Entries) == 0 { return "", werrors.NewBadRequestError("FAQ 条目不能为空") } if payload.Mode == "" { payload.Mode = types.FAQBatchModeAppend } if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace { return "", werrors.NewBadRequestError("模式仅支持 append 或 replace") } // 验证知识库是否存在且有效 kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return "", err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // 检查是否有正在进行的导入任务 runningKnowledge, err := s.getRunningFAQImportTask(ctx, kbID, tenantID) if err != nil { logger.Errorf(ctx, "Failed to check running import task: %v", err) // 检查失败不影响导入,继续执行 } else if runningKnowledge != nil { logger.Warnf(ctx, "Import task already running for KB %s: %s (status: %s)", kbID, runningKnowledge.ID, runningKnowledge.ParseStatus) return "", werrors.NewBadRequestError(fmt.Sprintf("该知识库已有导入任务正在进行中(任务ID: %s),请等待完成后再试", runningKnowledge.ID)) } faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb) if err != nil { return "", fmt.Errorf("failed to ensure FAQ knowledge: %w", err) } // 初始化导入任务状态到Knowledge表 taskID := faqKnowledge.ID // 使用Knowledge ID作为taskID // 从Knowledge Metadata中读取当前的NextChunkIndex(自增计数器) // 如果不存在,则初始化为1 existingMeta, _ := types.ParseFAQImportMetadata(faqKnowledge) nextChunkIndex := 1 if existingMeta != nil && existingMeta.NextChunkIndex > 0 { nextChunkIndex = existingMeta.NextChunkIndex } startChunkIndex := nextChunkIndex endChunkIndex := startChunkIndex + len(payload.Entries) - 1 // 更新NextChunkIndex为下一个要分配的值 newNextChunkIndex := nextChunkIndex + len(payload.Entries) // 更新Metadata,包含区间信息和新的NextChunkIndex importMeta := &types.FAQImportMetadata{ ImportProgress: 0, ImportTotal: len(payload.Entries), ImportProcessed: 0, NextChunkIndex: newNextChunkIndex, } if err := s.updateFAQImportStatusWithRanges(ctx, taskID, types.FAQImportStatusPending, 0, len(payload.Entries), 0, "", importMeta.NextChunkIndex); err != nil { logger.Errorf(ctx, "Failed to initialize FAQ import task status: %v", err) return "", fmt.Errorf("failed to initialize task: %w", err) } logger.Infof(ctx, "Allocated ChunkIndex range [%d, %d] for FAQ import task %s, next ChunkIndex will be %d", startChunkIndex, endChunkIndex, taskID, newNextChunkIndex) // Enqueue FAQ import task to Asynq logger.Info(ctx, "Enqueuing FAQ import task to Asynq") taskPayload := types.FAQImportPayload{ TenantID: tenantID, TaskID: taskID, KBID: kbID, KnowledgeID: faqKnowledge.ID, Entries: payload.Entries, Mode: payload.Mode, StartChunkIndex: startChunkIndex, EndChunkIndex: endChunkIndex, } payloadBytes, err := json.Marshal(taskPayload) if err != nil { logger.Errorf(ctx, "Failed to marshal FAQ import task payload: %v", err) return "", fmt.Errorf("failed to marshal task payload: %w", err) } task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(5)) info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err) return "", fmt.Errorf("failed to enqueue task: %w", err) } logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s", info.ID, info.Queue, taskID) return taskID, nil } // executeFAQImport 执行实际的FAQ导入逻辑 func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string, payload *types.FAQBatchUpsertPayload, tenantID uint, startChunkIndex int, processed int, totalEntries int) (err error) { // 保存知识库和embedding模型信息,用于清理索引 var kb *types.KnowledgeBase var embeddingModel embedding.Embedder // Recovery机制:如果发生任何错误或panic,回滚所有已创建的chunks和索引数据 defer func() { // 捕获panic if r := recover(); r != nil { buf := make([]byte, 8192) n := runtime.Stack(buf, false) stack := string(buf[:n]) logger.Errorf(ctx, "FAQ import task %s panicked: %v\n%s", taskID, r, stack) err = fmt.Errorf("panic during FAQ import: %v", r) } }() kb, err = s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } kb.EnsureDefaults() // 获取embedding模型,用于后续清理索引 embeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { return fmt.Errorf("failed to get embedding model: %w", err) } faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb) if err != nil { return err } // 如果模式为replace,则清理知识库 if payload.Mode == types.FAQBatchModeReplace { if err := s.cleanupFAQKnowledge(ctx, faqKnowledge); err != nil { return err } } // 获取索引模式 indexMode := types.FAQIndexModeQuestionOnly if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } // 分批处理 remainingEntries := len(payload.Entries) totalStartTime := time.Now() // 保存初始的processed值,用于计算ChunkIndex(i+idx是剩余条目中的索引,加上初始processed得到原始位置) initialProcessed := processed logger.Infof(ctx, "FAQ import task %s: starting batch processing, remaining entries: %d, total entries: %d, batch size: %d", taskID, remainingEntries, totalEntries, faqImportBatchSize) for i := 0; i < remainingEntries; i += faqImportBatchSize { batchStartTime := time.Now() end := i + faqImportBatchSize if end > remainingEntries { end = remainingEntries } batch := payload.Entries[i:end] logger.Infof(ctx, "FAQ import task %s: processing batch %d-%d (%d entries)", taskID, i+1, end, len(batch)) // 构建chunks buildStartTime := time.Now() chunks := make([]*types.Chunk, 0, len(batch)) chunkIds := make([]string, 0, len(batch)) for idx, entry := range batch { meta, err := sanitizeFAQEntryPayload(&entry) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "entry": entry, "task_id": taskID, }) return fmt.Errorf("failed to sanitize entry at index %d: %w", i+idx, err) } isEnabled := true if entry.IsEnabled != nil { isEnabled = *entry.IsEnabled } // ChunkIndex计算:startChunkIndex + (i+idx) + initialProcessed // i+idx是剩余条目中的索引,加上initialProcessed得到原始条目中的位置 chunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: faqKnowledge.ID, KnowledgeBaseID: kb.ID, Content: buildFAQChunkContent(meta, indexMode), ChunkIndex: startChunkIndex + i + idx + initialProcessed, IsEnabled: isEnabled, ChunkType: types.ChunkTypeFAQ, TagID: entry.TagID, } if err := chunk.SetFAQMetadata(meta); err != nil { return fmt.Errorf("failed to set FAQ metadata: %w", err) } chunks = append(chunks, chunk) chunkIds = append(chunkIds, chunk.ID) } buildDuration := time.Since(buildStartTime) logger.Debugf(ctx, "FAQ import task %s: batch %d-%d built %d chunks in %v, chunk IDs: %v", taskID, i+1, end, len(chunks), buildDuration, chunkIds) // 创建chunks createStartTime := time.Now() if err := s.chunkService.CreateChunks(ctx, chunks); err != nil { return fmt.Errorf("failed to create chunks: %w", err) } createDuration := time.Since(createStartTime) logger.Infof(ctx, "FAQ import task %s: batch %d-%d created %d chunks in %v", taskID, i+1, end, len(chunks), createDuration) // 索引chunks indexStartTime := time.Now() // 注意:如果索引失败,defer中的recovery机制会自动回滚已创建的chunks和索引数据 if err := s.indexFAQChunks(ctx, kb, faqKnowledge, chunks, embeddingModel, true, false); err != nil { return fmt.Errorf("failed to index chunks: %w", err) } indexDuration := time.Since(indexStartTime) logger.Infof(ctx, "FAQ import task %s: batch %d-%d indexed %d chunks in %v", taskID, i+1, end, len(chunks), indexDuration) processed = processed + len(batch) // 更新任务进度 progress := int(float64(processed) / float64(totalEntries) * 100) if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, processed, ""); err != nil { logger.Errorf(ctx, "Failed to update task progress: %v", err) } batchDuration := time.Since(batchStartTime) logger.Infof(ctx, "FAQ import task %s: batch %d-%d completed in %v (build: %v, create: %v, index: %v), total progress: %d/%d (%d%%)", taskID, i+1, end, batchDuration, buildDuration, createDuration, indexDuration, processed, totalEntries, progress) } totalDuration := time.Since(totalStartTime) logger.Infof(ctx, "FAQ import task %s: all batches completed, processed: %d entries in %v, avg: %v per entry", taskID, remainingEntries, totalDuration, totalDuration/time.Duration(remainingEntries)) return nil } // UpdateFAQEntry updates a single FAQ entry. func (s *knowledgeService) UpdateFAQEntry(ctx context.Context, kbID string, entryID string, payload *types.FAQEntryPayload, ) error { if payload == nil { return werrors.NewBadRequestError("请求体不能为空") } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } kb.EnsureDefaults() tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID { return werrors.NewForbiddenError("无权操作该 FAQ 条目") } if chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目") } meta, err := sanitizeFAQEntryPayload(payload) if err != nil { return err } if existing, err := chunk.FAQMetadata(); err == nil && existing != nil { meta.Version = existing.Version + 1 } if err := chunk.SetFAQMetadata(meta); err != nil { return err } // 获取索引模式 indexMode := types.FAQIndexModeQuestionOnly if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } chunk.Content = buildFAQChunkContent(meta, indexMode) chunk.TagID = payload.TagID isEnabledUpdated := false if payload.IsEnabled != nil { oldEnabled := chunk.IsEnabled chunk.IsEnabled = *payload.IsEnabled isEnabledUpdated = (oldEnabled != chunk.IsEnabled) } chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } // Sync is_enabled status to retriever engines if it was updated if isEnabledUpdated { chunkStatusMap := map[string]bool{chunk.ID: chunk.IsEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } } faqKnowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) if err != nil { return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { return err } return s.indexFAQChunks(ctx, kb, faqKnowledge, []*types.Chunk{chunk}, embeddingModel, false, true) } // UpdateFAQEntryStatus updates enable status for a FAQ entry. func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context, kbID string, entryID string, isEnabled bool, ) error { kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目") } if chunk.IsEnabled == isEnabled { return nil } chunk.IsEnabled = isEnabled chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } // Sync update to retriever engines chunkStatusMap := map[string]bool{chunk.ID: isEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } return nil } // UpdateFAQEntryStatusBatch updates enable status for FAQ entries in batch. func (s *knowledgeService) UpdateFAQEntryStatusBatch(ctx context.Context, kbID string, updates map[string]bool, ) error { if len(updates) == 0 { return nil } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all chunks in batch entryIDs := make([]string, 0, len(updates)) for entryID := range updates { entryIDs = append(entryIDs, entryID) } chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, entryIDs) if err != nil { return err } // Group chunks by enabled status for batch update chunkStatusMap := make(map[string]bool) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { continue } isEnabled, exists := updates[chunk.ID] if !exists { continue } if chunk.IsEnabled == isEnabled { continue } chunk.IsEnabled = isEnabled chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } chunkStatusMap[chunk.ID] = isEnabled } // Sync update to retriever engines if len(chunkStatusMap) > 0 { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } } return nil } // UpdateKnowledgeTag updates the tag assigned to a knowledge document. func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID string, tagID *string) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { return err } var resolvedTagID string if tagID != nil && *tagID != "" { tag, err := s.tagRepo.GetByID(ctx, tenantID, *tagID) if err != nil { return err } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError("标签不属于当前知识库") } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID return s.repo.UpdateKnowledge(ctx, knowledge) } // UpdateKnowledgeTagBatch updates tags for document knowledge items in batch. func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, updates map[string]*string) error { if len(updates) == 0 { return nil } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all knowledge items in batch knowledgeIDs := make([]string, 0, len(updates)) for knowledgeID := range updates { knowledgeIDs = append(knowledgeIDs, knowledgeID) } knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs) if err != nil { return err } // Build tag ID map for validation tagIDSet := make(map[string]bool) for _, tagID := range updates { if tagID != nil && *tagID != "" { tagIDSet[*tagID] = true } } // Validate all tags in batch tagMap := make(map[string]*types.KnowledgeTag) if len(tagIDSet) > 0 { tagIDs := make([]string, 0, len(tagIDSet)) for tagID := range tagIDSet { tagIDs = append(tagIDs, tagID) } for _, tagID := range tagIDs { tag, err := s.tagRepo.GetByID(ctx, tenantID, tagID) if err != nil { return err } tagMap[tagID] = tag } } // Update knowledge items knowledgeToUpdate := make([]*types.Knowledge, 0) for _, knowledge := range knowledgeList { tagID, exists := updates[knowledge.ID] if !exists { continue } var resolvedTagID string if tagID != nil && *tagID != "" { tag, ok := tagMap[*tagID] if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", *tagID)) } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于知识库 %s", *tagID, knowledge.KnowledgeBaseID)) } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID knowledgeToUpdate = append(knowledgeToUpdate, knowledge) } if len(knowledgeToUpdate) > 0 { return s.repo.UpdateKnowledgeBatch(ctx, knowledgeToUpdate) } return nil } // UpdateFAQEntryTag updates the tag assigned to an FAQ entry. func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, entryID string, tagID *string) error { kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目标签") } var resolvedTagID string if tagID != nil && *tagID != "" { tag, err := s.tagRepo.GetByID(ctx, tenantID, *tagID) if err != nil { return err } if tag.KnowledgeBaseID != kb.ID { return werrors.NewBadRequestError("标签不属于当前知识库") } resolvedTagID = tag.ID } chunk.TagID = resolvedTagID chunk.UpdatedAt = time.Now() return s.chunkRepo.UpdateChunk(ctx, chunk) } // UpdateFAQEntryTagBatch updates tags for FAQ entries in batch. func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID string, updates map[string]*string) error { if len(updates) == 0 { return nil } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all chunks in batch entryIDs := make([]string, 0, len(updates)) for entryID := range updates { entryIDs = append(entryIDs, entryID) } chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, entryIDs) if err != nil { return err } // Build tag ID map for validation tagIDSet := make(map[string]bool) for _, tagID := range updates { if tagID != nil && *tagID != "" { tagIDSet[*tagID] = true } } // Validate all tags in batch tagMap := make(map[string]*types.KnowledgeTag) if len(tagIDSet) > 0 { tagIDs := make([]string, 0, len(tagIDSet)) for tagID := range tagIDSet { tagIDs = append(tagIDs, tagID) } for _, tagID := range tagIDs { tag, err := s.tagRepo.GetByID(ctx, tenantID, tagID) if err != nil { return err } if tag.KnowledgeBaseID != kb.ID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于当前知识库", tagID)) } tagMap[tagID] = tag } } // Update chunks chunksToUpdate := make([]*types.Chunk, 0) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { continue } tagID, exists := updates[chunk.ID] if !exists { continue } var resolvedTagID string if tagID != nil && *tagID != "" { tag, ok := tagMap[*tagID] if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", *tagID)) } resolvedTagID = tag.ID } chunk.TagID = resolvedTagID chunk.UpdatedAt = time.Now() chunksToUpdate = append(chunksToUpdate, chunk) } if len(chunksToUpdate) > 0 { return s.chunkRepo.UpdateChunks(ctx, chunksToUpdate) } return nil } // SearchFAQEntries searches FAQ entries using hybrid search. func (s *knowledgeService) SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest, ) ([]*types.FAQEntry, error) { // Validate FAQ knowledge base kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } // Set default values if req.VectorThreshold <= 0 { req.VectorThreshold = 0.7 } if req.MatchCount <= 0 { req.MatchCount = 10 } if req.MatchCount > 50 { req.MatchCount = 50 } // Prepare search parameters searchParams := types.SearchParams{ QueryText: req.QueryText, VectorThreshold: req.VectorThreshold, MatchCount: req.MatchCount, DisableKeywordsMatch: true, } // Call HybridSearch searchResults, err := s.kbService.HybridSearch(ctx, kbID, searchParams) if err != nil { return nil, err } if len(searchResults) == 0 { return []*types.FAQEntry{}, nil } // Extract chunk IDs and build score/match type maps chunkIDs := make([]string, 0, len(searchResults)) chunkScores := make(map[string]float64) chunkMatchTypes := make(map[string]types.MatchType) for _, result := range searchResults { // SearchResult.ID is the chunk ID chunkID := result.ID chunkIDs = append(chunkIDs, chunkID) chunkScores[chunkID] = result.Score chunkMatchTypes[chunkID] = result.MatchType } // Batch fetch chunks tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs) if err != nil { return nil, err } // Filter FAQ chunks and convert to FAQEntry kb.EnsureDefaults() entries := make([]*types.FAQEntry, 0, len(chunks)) for _, chunk := range chunks { // Only process FAQ type chunks if chunk.ChunkType != types.ChunkTypeFAQ { continue } if !chunk.IsEnabled { continue } entry, err := s.chunkToFAQEntry(chunk, kb) if err != nil { logger.Warnf(ctx, "Failed to convert chunk to FAQ entry: %v", err) continue } // Preserve score and match type from search results // Note: Negative question filtering is now handled in HybridSearch if score, ok := chunkScores[chunk.ID]; ok { entry.Score = score } if matchType, ok := chunkMatchTypes[chunk.ID]; ok { entry.MatchType = matchType } entries = append(entries, entry) } slices.SortFunc(entries, func(a, b *types.FAQEntry) int { return int(b.Score - a.Score) }) return entries, nil } // DeleteFAQEntries deletes FAQ entries in batch. func (s *knowledgeService) DeleteFAQEntries(ctx context.Context, kbID string, entryIDs []string, ) error { if len(entryIDs) == 0 { return werrors.NewBadRequestError("请选择需要删除的 FAQ 条目") } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) var faqKnowledge *types.Knowledge chunksToRemove := make([]*types.Chunk, 0, len(entryIDs)) for _, id := range entryIDs { if id == "" { continue } chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, id) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("包含无效的 FAQ 条目") } if err := s.chunkService.DeleteChunk(ctx, id); err != nil { return err } if faqKnowledge == nil { faqKnowledge, err = s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) if err != nil { return err } } chunksToRemove = append(chunksToRemove, chunk) } if len(chunksToRemove) > 0 && faqKnowledge != nil { if err := s.deleteFAQChunkVectors(ctx, kb, faqKnowledge, chunksToRemove); err != nil { return err } } return nil } func (s *knowledgeService) validateFAQKnowledgeBase(ctx context.Context, kbID string) (*types.KnowledgeBase, error) { if kbID == "" { return nil, werrors.NewBadRequestError("知识库 ID 不能为空") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { return nil, err } kb.EnsureDefaults() if kb.Type != types.KnowledgeBaseTypeFAQ { return nil, werrors.NewBadRequestError("仅 FAQ 知识库支持该操作") } return kb, nil } func (s *knowledgeService) findFAQKnowledge(ctx context.Context, tenantID uint, kbID string) (*types.Knowledge, error) { knowledges, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID) if err != nil { return nil, err } for _, knowledge := range knowledges { if knowledge.Type == types.KnowledgeTypeFAQ { return knowledge, nil } } return nil, nil } func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint, kb *types.KnowledgeBase) (*types.Knowledge, error) { existing, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) if err != nil { return nil, err } if existing != nil { return existing, nil } knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kb.ID, Type: types.KnowledgeTypeFAQ, Title: fmt.Sprintf("%s - FAQ", kb.Name), Description: "FAQ 条目容器", Source: types.KnowledgeTypeFAQ, ParseStatus: "completed", EnableStatus: "enabled", EmbeddingModelID: kb.EmbeddingModelID, CreatedAt: time.Now(), UpdatedAt: time.Now(), } if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { return nil, err } return knowledge, nil } func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, progress, total, processed int, errorMsg string) error { return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg, 0) } // updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, progress, total, processed int, errorMsg string, nextChunkIndex int) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { return err } // 更新ParseStatus:将FAQImportTaskStatus映射到ParseStatus knowledge.ParseStatus = string(status) knowledge.UpdatedAt = time.Now() meta, err := types.ParseFAQImportMetadata(knowledge) if err != nil || meta == nil { meta = &types.FAQImportMetadata{} } // 更新ErrorMessage knowledge.ErrorMessage = errorMsg if status == types.FAQImportStatusCompleted { knowledge.ErrorMessage = "" } // 更新Metadata中的导入进度信息,保留已有的ChunkIndexRanges和NextChunkIndex meta.ImportProgress = progress meta.ImportTotal = total meta.ImportProcessed = processed if nextChunkIndex > 0 { meta.NextChunkIndex = nextChunkIndex } metaJSON, err := meta.ToJSON() if err != nil { return fmt.Errorf("failed to marshal import metadata: %w", err) } knowledge.Metadata = metaJSON return s.repo.UpdateKnowledge(ctx, knowledge) } // getRunningFAQImportTask 获取指定知识库的进行中导入任务 func (s *knowledgeService) getRunningFAQImportTask(ctx context.Context, kbID string, tenantID uint) (*types.Knowledge, error) { faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kbID) if err != nil { return nil, err } if faqKnowledge == nil { return nil, errors.New("FAQ knowledge not found") } // 检查ParseStatus是否为pending或processing(进行中状态) if faqKnowledge.ParseStatus == "pending" || faqKnowledge.ParseStatus == "processing" { return faqKnowledge, nil } return nil, nil } func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase) (*types.FAQEntry, error) { meta, err := chunk.FAQMetadata() if err != nil { return nil, err } if meta == nil { meta = &types.FAQChunkMetadata{StandardQuestion: chunk.Content} } entry := &types.FAQEntry{ ID: chunk.ID, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, TagID: chunk.TagID, IsEnabled: chunk.IsEnabled, StandardQuestion: meta.StandardQuestion, SimilarQuestions: meta.SimilarQuestions, NegativeQuestions: meta.NegativeQuestions, Answers: meta.Answers, IndexMode: kb.FAQConfig.IndexMode, UpdatedAt: chunk.UpdatedAt, CreatedAt: chunk.CreatedAt, ChunkType: chunk.ChunkType, } return entry, nil } func buildFAQChunkContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode) string { var builder strings.Builder builder.WriteString(fmt.Sprintf("Q: %s\n", meta.StandardQuestion)) if len(meta.SimilarQuestions) > 0 { builder.WriteString("Similar Questions:\n") for _, q := range meta.SimilarQuestions { builder.WriteString(fmt.Sprintf("- %s\n", q)) } } // 负例不应该包含在 Content 中,因为它们不应该被索引 // 答案根据索引模式决定是否包含 if mode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { builder.WriteString("Answers:\n") for _, ans := range meta.Answers { builder.WriteString(fmt.Sprintf("- %s\n", ans)) } } return builder.String() } func sanitizeFAQEntryPayload(payload *types.FAQEntryPayload) (*types.FAQChunkMetadata, error) { meta := &types.FAQChunkMetadata{ StandardQuestion: strings.TrimSpace(payload.StandardQuestion), SimilarQuestions: payload.SimilarQuestions, NegativeQuestions: payload.NegativeQuestions, Answers: payload.Answers, Version: 1, Source: "faq", } meta.Normalize() if meta.StandardQuestion == "" { return nil, werrors.NewBadRequestError("标准问不能为空") } if len(meta.Answers) == 0 { return nil, werrors.NewBadRequestError("至少提供一个答案") } return meta, nil } func buildFAQIndexContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode) string { var builder strings.Builder builder.WriteString(meta.StandardQuestion) for _, q := range meta.SimilarQuestions { builder.WriteString("\n") builder.WriteString(q) } if mode == types.FAQIndexModeQuestionAnswer { for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } } return builder.String() } // buildFAQIndexInfoList 构建FAQ索引信息列表,支持分别索引模式 func (s *knowledgeService) buildFAQIndexInfoList(ctx context.Context, kb *types.KnowledgeBase, chunk *types.Chunk) ([]*types.IndexInfo, error) { indexMode := types.FAQIndexModeQuestionAnswer questionIndexMode := types.FAQQuestionIndexModeCombined if kb.FAQConfig != nil { if kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } if kb.FAQConfig.QuestionIndexMode != "" { questionIndexMode = kb.FAQConfig.QuestionIndexMode } } meta, err := chunk.FAQMetadata() if err != nil { return nil, err } if meta == nil { meta = &types.FAQChunkMetadata{StandardQuestion: chunk.Content} } // 如果是一起索引模式,使用原有逻辑 if questionIndexMode == types.FAQQuestionIndexModeCombined { content := buildFAQIndexContent(meta, indexMode) return []*types.IndexInfo{ { Content: content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }, }, nil } // 分别索引模式:为每个问题创建独立的索引项 indexInfoList := make([]*types.IndexInfo, 0) // 标准问索引项 standardContent := meta.StandardQuestion if indexMode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { var builder strings.Builder builder.WriteString(meta.StandardQuestion) for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } standardContent = builder.String() } indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: standardContent, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) // 每个相似问创建一个索引项 for i, similarQ := range meta.SimilarQuestions { similarContent := similarQ if indexMode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { var builder strings.Builder builder.WriteString(similarQ) for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } similarContent = builder.String() } sourceID := fmt.Sprintf("%s-%d", chunk.ID, i) indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: similarContent, SourceID: sourceID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) } return indexInfoList, nil } func (s *knowledgeService) indexFAQChunks(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*types.Chunk, embeddingModel embedding.Embedder, adjustStorage bool, needDelete bool, ) error { if len(chunks) == 0 { return nil } indexStartTime := time.Now() logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks)) tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } // 构建索引信息 buildIndexInfoStartTime := time.Now() indexInfo := make([]*types.IndexInfo, 0) chunkIDs := make([]string, 0, len(chunks)) for _, chunk := range chunks { infoList, err := s.buildFAQIndexInfoList(ctx, kb, chunk) if err != nil { return err } indexInfo = append(indexInfo, infoList...) chunkIDs = append(chunkIDs, chunk.ID) } buildIndexInfoDuration := time.Since(buildIndexInfoStartTime) logger.Debugf(ctx, "indexFAQChunks: built %d index info entries for %d chunks in %v", len(indexInfo), len(chunks), buildIndexInfoDuration) var size int64 if adjustStorage { estimateStartTime := time.Now() size = retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo) estimateDuration := time.Since(estimateStartTime) logger.Debugf(ctx, "indexFAQChunks: estimated storage size %d bytes in %v", size, estimateDuration) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed+size > tenantInfo.StorageQuota { return types.NewStorageQuotaExceededError() } } // 删除旧向量 var deleteDuration time.Duration if needDelete { deleteStartTime := time.Now() if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err) } deleteDuration = time.Since(deleteStartTime) if deleteDuration > 100*time.Millisecond { logger.Debugf(ctx, "indexFAQChunks: deleted old vectors for %d chunks in %v", len(chunkIDs), deleteDuration) } } // 批量索引(这里可能是性能瓶颈) batchIndexStartTime := time.Now() if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil { return err } batchIndexDuration := time.Since(batchIndexStartTime) logger.Debugf(ctx, "indexFAQChunks: batch indexed %d index info entries in %v (avg: %v per entry)", len(indexInfo), batchIndexDuration, batchIndexDuration/time.Duration(len(indexInfo))) if adjustStorage && size > 0 { adjustStartTime := time.Now() if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, size); err == nil { tenantInfo.StorageUsed += size } knowledge.StorageSize += size adjustDuration := time.Since(adjustStartTime) if adjustDuration > 50*time.Millisecond { logger.Debugf(ctx, "indexFAQChunks: adjusted storage in %v", adjustDuration) } } updateStartTime := time.Now() now := time.Now() knowledge.UpdatedAt = now knowledge.ProcessedAt = &now err = s.repo.UpdateKnowledge(ctx, knowledge) updateDuration := time.Since(updateStartTime) if updateDuration > 50*time.Millisecond { logger.Debugf(ctx, "indexFAQChunks: updated knowledge in %v", updateDuration) } totalDuration := time.Since(indexStartTime) logger.Debugf(ctx, "indexFAQChunks: completed indexing %d chunks in %v (build: %v, delete: %v, batchIndex: %v, update: %v)", len(chunks), totalDuration, buildIndexInfoDuration, deleteDuration, batchIndexDuration, updateDuration) return err } func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*types.Chunk, ) error { if len(chunks) == 0 { return nil } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { return err } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } indexInfo := make([]*types.IndexInfo, 0) chunkIDs := make([]string, 0, len(chunks)) for _, chunk := range chunks { infoList, err := s.buildFAQIndexInfoList(ctx, kb, chunk) if err != nil { return err } indexInfo = append(indexInfo, infoList...) chunkIDs = append(chunkIDs, chunk.ID) } size := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo) if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { return err } if size > 0 { if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -size); err == nil { tenantInfo.StorageUsed -= size if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } } if knowledge.StorageSize >= size { knowledge.StorageSize -= size } else { knowledge.StorageSize = 0 } } knowledge.UpdatedAt = time.Now() return s.repo.UpdateKnowledge(ctx, knowledge) } func (s *knowledgeService) cleanupFAQKnowledge(ctx context.Context, knowledge *types.Knowledge) error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err == nil { if embeddingModel, modelErr := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID); modelErr == nil { _ = retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()) } } } if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { return err } if knowledge.StorageSize > 0 { if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err == nil { tenantInfo.StorageUsed -= knowledge.StorageSize if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } } knowledge.StorageSize = 0 } knowledge.UpdatedAt = time.Now() knowledge.ProcessedAt = nil return s.repo.UpdateKnowledge(ctx, knowledge) } func ensureManualFileName(title string) string { if title == "" { return fmt.Sprintf("manual-%s%s", time.Now().Format("20060102-150405"), manualFileExtension) } trimmed := strings.TrimSpace(title) if strings.HasSuffix(strings.ToLower(trimmed), manualFileExtension) { return trimmed } return trimmed + manualFileExtension } func (s *knowledgeService) triggerManualProcessing(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, content string, sync bool, ) { clean := strings.TrimSpace(content) if clean == "" { return } // 使用 docreader 按照 MD 格式处理,并使用知识库配置的分隔符 contentBytes := []byte(clean) fileName := ensureManualFileName(knowledge.Title) fileType := "md" // 检查是否需要启用多模态(对于手动内容通常不需要,但保持一致性) enableMultimodel := kb.VLMConfig.Enabled && kb.StorageConfig.Provider != "" logger.GetLogger(ctx).Infof("triggerManualProcessing split manual content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v", len(contentBytes), fileName, fileType, kb.ChunkingConfig.Separators, enableMultimodel) var vlmConfig *proto.VLMConfig if enableMultimodel { cfg, cfgErr := s.getVLMProtoConfig(ctx, kb) if cfgErr != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", cfgErr).Errorf("triggerManualProcessing build VLM config failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = cfgErr.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } if cfg == nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). Error("triggerManualProcessing enable multimodal but VLM config missing") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "VLM 配置缺失" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } vlmConfig = cfg } // 调用 docreader 解析 markdown 内容 resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ FileContent: contentBytes, FileName: fileName, FileType: fileType, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: vlmConfig, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("triggerManualProcessing read file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } if sync { s.processChunks(ctx, kb, knowledge, resp.Chunks) return } newCtx := logger.CloneContext(ctx) go s.processChunks(newCtx, kb, knowledge, resp.Chunks) } func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowledge *types.Knowledge) error { logger.GetLogger(ctx).Infof("Cleaning knowledge resources before manual update, knowledge ID: %s", knowledge.ID) var cleanupErr error if knowledge.ParseStatus == types.ManualKnowledgeStatusDraft && knowledge.StorageSize == 0 { // Draft without indexed data, skip cleanup. return nil } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup") cleanupErr = errors.Join(cleanupErr, err) } else { embeddingModel, modelErr := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) if modelErr != nil { logger.GetLogger(ctx).WithField("error", modelErr).Error("Failed to get embedding model during cleanup") cleanupErr = errors.Join(cleanupErr, modelErr) } else { if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge index") cleanupErr = errors.Join(cleanupErr, err) } } } } if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge chunks") cleanupErr = errors.Join(cleanupErr, err) } namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge graph data") cleanupErr = errors.Join(cleanupErr, err) } if knowledge.StorageSize > 0 { tenantInfo.StorageUsed -= knowledge.StorageSize if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to adjust storage usage during manual cleanup") cleanupErr = errors.Join(cleanupErr, err) } knowledge.StorageSize = 0 } return cleanupErr } func (s *knowledgeService) getVLMProtoConfig(ctx context.Context, kb *types.KnowledgeBase) (*proto.VLMConfig, error) { if kb == nil || !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" { return nil, nil } model, err := s.modelService.GetModelByID(ctx, kb.VLMConfig.ModelID) if err != nil { return nil, err } interfaceType := model.Parameters.InterfaceType if interfaceType == "" { interfaceType = "openai" } return &proto.VLMConfig{ ModelName: model.Name, BaseUrl: model.Parameters.BaseURL, ApiKey: model.Parameters.APIKey, InterfaceType: interfaceType, }, nil } func IsImageType(fileType string) bool { switch fileType { case "jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "tiff": return true default: return false } } // ProcessDocument handles Asynq document processing tasks func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) error { var payload types.DocumentProcessPayload if err := json.Unmarshal(t.Payload(), &payload); err != nil { logger.Errorf(ctx, "failed to unmarshal document process task payload: %v", err) return nil } ctx = logger.WithRequestID(ctx, payload.RequestId) ctx = logger.WithField(ctx, "document_process", payload.KnowledgeID) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Errorf(ctx, "failed to get tenant: %v", err) return nil } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) logger.Infof(ctx, "Processing document task: knowledge_id=%s, file_path=%s", payload.KnowledgeID, payload.FilePath) // 幂等性检查:获取knowledge记录 knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) if err != nil { logger.Errorf(ctx, "failed to get knowledge: %v", err) return nil } if knowledge == nil { return nil } // 检查任务状态 - 幂等性处理 if knowledge.ParseStatus == "completed" { logger.Infof(ctx, "Document already completed, skipping: %s", payload.KnowledgeID) return nil // 幂等:已完成的任务直接返回 } if knowledge.ParseStatus == "failed" { // 检查是否可恢复(例如:超时、临时错误等) // 对于不可恢复的错误,直接返回 logger.Warnf(ctx, "Document processing previously failed: %s, error: %s", payload.KnowledgeID, knowledge.ErrorMessage) // 这里可以根据错误类型判断是否可恢复,暂时允许重试 } // 检查是否有部分处理(有chunks但状态不是completed) if knowledge.ParseStatus != "completed" && knowledge.ParseStatus != "pending" && knowledge.ParseStatus != "processing" { // 状态异常,记录日志但继续处理 logger.Warnf(ctx, "Unexpected parse status: %s for knowledge: %s", knowledge.ParseStatus, payload.KnowledgeID) } // 获取知识库信息 kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID) if err != nil { logger.Errorf(ctx, "failed to get knowledge base: %v", err) knowledge.ParseStatus = "failed" knowledge.ErrorMessage = fmt.Sprintf("failed to get knowledge base: %v", err) knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return nil } knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "failed to update knowledge status to processing: %v", err) return nil } // 构建VLM配置(如果需要) var vlmConfig *proto.VLMConfig if payload.EnableMultimodel { vlmConfig, err = s.getVLMProtoConfig(ctx, kb) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument build VLM config failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return nil } if vlmConfig == nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). Error("processDocument enable multimodal but VLM config missing") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "VLM 配置缺失" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return nil } } // 检查多模态配置(仅对文件导入) if payload.FilePath != "" && !payload.EnableMultimodel && IsImageType(payload.FileType) { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = ErrImageNotParse.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return nil } // 处理不同类型的导入:文件、URL、文本段落 var chunks []*proto.Chunk if payload.URL != "" { // URL导入 urlResp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{ Url: payload.URL, Title: knowledge.Title, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: payload.EnableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: vlmConfig, }, RequestId: payload.RequestId, }) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return fmt.Errorf("failed to read from URL: %w", err) } chunks = urlResp.Chunks } else if len(payload.Passages) > 0 { // 文本段落导入 chunks := make([]*proto.Chunk, 0, len(payload.Passages)) start, end := 0, 0 for i, p := range payload.Passages { if p == "" { continue } end += len([]rune(p)) chunk := &proto.Chunk{ Content: p, Seq: int32(i), Start: int32(start), End: int32(end), } start = end chunks = append(chunks, chunk) } // 直接处理chunks,不需要调用docReader s.processChunks(ctx, kb, knowledge, chunks) return nil } else { // 文件导入 fileReader, err := s.fileSvc.GetFile(ctx, payload.FilePath) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument get file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return fmt.Errorf("failed to get file: %w", err) } defer fileReader.Close() // 读取文件内容 contentBytes, err := io.ReadAll(fileReader) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return fmt.Errorf("failed to read file: %w", err) } // 调用docReader处理文件 fileResp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ FileContent: contentBytes, FileName: payload.FileName, FileType: payload.FileType, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: payload.EnableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: vlmConfig, }, RequestId: payload.RequestId, }) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument read file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return fmt.Errorf("failed to read file from docreader: %w", err) } chunks = fileResp.Chunks } // 处理chunks(这会更新状态为completed) s.processChunks(ctx, kb, knowledge, chunks) return nil } // ProcessFAQImport handles Asynq FAQ import tasks func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) error { var payload types.FAQImportPayload if err := json.Unmarshal(t.Payload(), &payload); err != nil { logger.Errorf(ctx, "failed to unmarshal FAQ import task payload: %v", err) return fmt.Errorf("failed to unmarshal task payload: %w", err) } ctx = logger.WithRequestID(ctx, uuid.New().String()) ctx = logger.WithField(ctx, "faq_import", payload.TaskID) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Errorf(ctx, "failed to get tenant: %v", err) return nil } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d", payload.TaskID, payload.KBID, len(payload.Entries)) // 幂等性检查:获取knowledge记录(FAQ任务使用knowledge ID作为taskID) knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.TaskID) if err != nil { logger.Errorf(ctx, "failed to get FAQ knowledge: %v", err) return nil } if knowledge == nil { return nil } // 检查任务状态 - 幂等性处理 if knowledge.ParseStatus == "completed" { logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID) return nil // 幂等:已完成的任务直接返回 } // 检查已处理进度 importMeta, _ := types.ParseFAQImportMetadata(knowledge) var processedCount int if importMeta != nil { processedCount = importMeta.ImportProcessed logger.Infof(ctx, "Resuming FAQ import from progress: %d/%d", processedCount, len(payload.Entries)) } // 保存原始总数量(在截断payload.Entries之前) originalTotalEntries := len(payload.Entries) // 如果已经处理了一部分,需要从该位置继续 if processedCount < originalTotalEntries { // 幂等性处理:清理processedCount之后可能已部分处理的chunks和索引数据 // 因为任务可能在处理过程中中断,导致部分chunks已入库或已写入索引 logger.Infof(ctx, "Cleaning up potentially partially processed chunks after entry %d", processedCount) // 计算需要清理的ChunkIndex范围 // 已处理的chunks的ChunkIndex范围是 [payload.StartChunkIndex, payload.StartChunkIndex + processedCount - 1] // 需要清理的chunks是从已处理位置之后开始,到分配的结束位置 minChunkIndex := payload.StartChunkIndex + processedCount maxChunkIndex := payload.EndChunkIndex logger.Infof(ctx, "Cleaning chunks with ChunkIndex in range [%d, %d] (allocated range: [%d, %d], processedCount=%d), knowledge_id=%s, kb_id=%s", minChunkIndex, maxChunkIndex, payload.StartChunkIndex, payload.EndChunkIndex, processedCount, payload.KnowledgeID, payload.KBID) // 查询并删除需要清理的chunks(使用ChunkIndex范围查询) // 注意:只清理processedCount之后的部分,避免删除已成功处理的chunks chunksToDelete, err := s.chunkRepo.DeleteChunksByChunkIndexRange(ctx, payload.TenantID, payload.KnowledgeID, minChunkIndex, maxChunkIndex, ) if err != nil { logger.Errorf(ctx, "Failed to query chunks for cleanup: %v", err) return fmt.Errorf("failed to query chunks: %w", err) } logger.Infof(ctx, "Cleaned %d chunks, from knowledge: %s, range: [%d, %d]", len(chunksToDelete), payload.KnowledgeID, minChunkIndex, maxChunkIndex) // 获取KB信息以删除索引数据 kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KBID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return fmt.Errorf("failed to get knowledge base: %w", err) } // 删除索引数据 embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err == nil { retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err == nil { chunkIDs := make([]string, 0, len(chunksToDelete)) for _, chunk := range chunksToDelete { chunkIDs = append(chunkIDs, chunk.ID) } if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Failed to delete index data for chunks (may not exist): %v", err) } else { logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksToDelete)) } } } // 从已处理的位置继续 payload.Entries = payload.Entries[processedCount:] logger.Infof(ctx, "Continuing FAQ import from entry %d, remaining: %d entries", processedCount, len(payload.Entries)) } // 更新任务状态为运行中 if err := s.updateFAQImportStatusWithRanges(ctx, payload.TaskID, types.FAQImportStatusProcessing, 0, originalTotalEntries, processedCount, "", importMeta.NextChunkIndex); err != nil { logger.Errorf(ctx, "Failed to update task status to running: %v", err) } // 构建FAQBatchUpsertPayload faqPayload := &types.FAQBatchUpsertPayload{ Entries: payload.Entries, Mode: payload.Mode, } // 执行FAQ导入(传入原始总数量) if err := s.executeFAQImport(ctx, payload.TaskID, payload.KBID, faqPayload, payload.TenantID, payload.StartChunkIndex, processedCount, originalTotalEntries); err != nil { logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err) return fmt.Errorf("FAQ import failed: %w", err) } // 任务成功完成 logger.Infof(ctx, "FAQ import task completed: %s", payload.TaskID) if err := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, 100, originalTotalEntries, originalTotalEntries, ""); err != nil { logger.Errorf(ctx, "Failed to update task status to success: %v", err) } return nil }