package service import ( "context" "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "io" "mime/multipart" "os" "regexp" "slices" "sort" "strings" "time" "github.com/Tencent/WeKnora/docreader/client" "github.com/Tencent/WeKnora/docreader/proto" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/config" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/utils" "github.com/Tencent/WeKnora/internal/tracing" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" "github.com/hibiken/asynq" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" ) // Error definitions for knowledge service operations var ( // ErrInvalidFileType is returned when an unsupported file type is provided ErrInvalidFileType = errors.New("unsupported file type") // ErrInvalidURL is returned when an invalid URL is provided ErrInvalidURL = errors.New("invalid URL") // ErrChunkNotFound is returned when a requested chunk cannot be found ErrChunkNotFound = errors.New("chunk not found") // ErrDuplicateFile is returned when trying to add a file that already exists ErrDuplicateFile = errors.New("file already exists") // ErrDuplicateURL is returned when trying to add a URL that already exists ErrDuplicateURL = errors.New("URL already exists") // ErrImageNotParse is returned when trying to update image information without enabling multimodel ErrImageNotParse = errors.New("image not parse without enable multimodel") ) // knowledgeService implements the knowledge service interface // service 实现知识服务接口 type knowledgeService struct { config *config.Config retrieveEngine interfaces.RetrieveEngineRegistry repo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService tenantRepo interfaces.TenantRepository docReaderClient *client.Client chunkService interfaces.ChunkService chunkRepo interfaces.ChunkRepository tagRepo interfaces.KnowledgeTagRepository fileSvc interfaces.FileService modelService interfaces.ModelService task *asynq.Client graphEngine interfaces.RetrieveGraphRepository } const ( manualContentMaxLength = 200000 manualFileExtension = ".md" ) // NewKnowledgeService creates a new knowledge service instance func NewKnowledgeService( config *config.Config, repo interfaces.KnowledgeRepository, docReaderClient *client.Client, kbService interfaces.KnowledgeBaseService, tenantRepo interfaces.TenantRepository, chunkService interfaces.ChunkService, chunkRepo interfaces.ChunkRepository, tagRepo interfaces.KnowledgeTagRepository, fileSvc interfaces.FileService, modelService interfaces.ModelService, task *asynq.Client, graphEngine interfaces.RetrieveGraphRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ) (interfaces.KnowledgeService, error) { return &knowledgeService{ config: config, repo: repo, kbService: kbService, tenantRepo: tenantRepo, docReaderClient: docReaderClient, chunkService: chunkService, chunkRepo: chunkRepo, tagRepo: tagRepo, fileSvc: fileSvc, modelService: modelService, task: task, graphEngine: graphEngine, retrieveEngine: retrieveEngine, }, nil } // GetRepository gets the knowledge repository // Parameters: // - ctx: Context with authentication and request information // // Returns: // - interfaces.KnowledgeRepository: Knowledge repository func (s *knowledgeService) GetRepository() interfaces.KnowledgeRepository { return s.repo } // CreateKnowledgeFromFile creates a knowledge entry from an uploaded file func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from file") logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, file.Filename) if metadata != nil { logger.Infof(ctx, "Received metadata: %v", metadata) } // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // 检查多模态配置完整性 - 只在图片文件时校验 // 检查是否为图片文件 if !IsImageType(getFileType(file.Filename)) { logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation") } else { // 检查COS配置 switch kb.StorageConfig.Provider { case "cos": if kb.StorageConfig.SecretID == "" || kb.StorageConfig.SecretKey == "" || kb.StorageConfig.Region == "" || kb.StorageConfig.BucketName == "" || kb.StorageConfig.AppID == "" { logger.Error(ctx, "COS configuration incomplete for image multimodal processing") return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") } case "minio": if kb.StorageConfig.BucketName == "" { logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing") return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") } } // 检查VLM配置 if kb.VLMConfig.ModelName == "" || kb.VLMConfig.BaseURL == "" { logger.Error(ctx, "VLM configuration incomplete for image multimodal processing") return nil, werrors.NewBadRequestError("上传图片文件需要完整的VLM配置信息, 请前往系统设置页面进行补全") } logger.Info(ctx, "Image multimodal configuration validation passed") } // Validate file type logger.Infof(ctx, "Checking file type: %s", file.Filename) if !isValidFileType(file.Filename) { logger.Error(ctx, "Invalid file type") return nil, ErrInvalidFileType } // Calculate file hash for deduplication logger.Info(ctx, "Calculating file hash") hash, err := calculateFileHash(file) if err != nil { logger.Errorf(ctx, "Failed to calculate file hash: %v", err) return nil, err } // Check if file already exists tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "file", FileName: file.Filename, FileSize: file.Size, FileHash: hash, }) if err != nil { logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) return nil, err } if exists { logger.Infof(ctx, "File already exists: %s", file.Filename) // Update creation time for existing knowledge if err := s.repo.UpdateKnowledgeColumn(ctx, existingKnowledge.ID, "created_at", time.Now()); err != nil { logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) return nil, err } return existingKnowledge, types.NewDuplicateFileError(existingKnowledge) } // Check storage quota tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { logger.Error(ctx, "Storage quota exceeded") return nil, types.NewStorageQuotaExceededError() } // Convert metadata to JSON format if provided var metadataJSON types.JSON if metadata != nil { metadataBytes, err := json.Marshal(metadata) if err != nil { logger.Errorf(ctx, "Failed to marshal metadata: %v", err) return nil, err } metadataJSON = types.JSON(metadataBytes) } // 验证文件名安全性 safeFilename, isValid := secutils.ValidateInput(file.Filename) if !isValid { logger.Errorf(ctx, "Invalid filename: %s", file.Filename) return nil, werrors.NewValidationError("文件名包含非法字符") } // Create knowledge record logger.Info(ctx, "Creating knowledge record") knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, Type: "file", Title: safeFilename, FileName: safeFilename, FileType: getFileType(safeFilename), FileSize: file.Size, FileHash: hash, ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, Metadata: metadataJSON, } // Save knowledge record to database logger.Info(ctx, "Saving knowledge record to database") if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record, ID: %s, error: %v", knowledge.ID, err) return nil, err } // Save the file to storage logger.Infof(ctx, "Saving file, knowledge ID: %s", knowledge.ID) filePath, err := s.fileSvc.SaveFile(ctx, file, knowledge.TenantID, knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to save file, knowledge ID: %s, error: %v", knowledge.ID, err) return nil, err } knowledge.FilePath = filePath // Update knowledge record with file path logger.Info(ctx, "Updating knowledge record with file path") if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to update knowledge with file path, ID: %s, error: %v", knowledge.ID, err) return nil, err } // Process document asynchronously logger.Info(ctx, "Starting asynchronous document processing") newCtx := logger.CloneContext(ctx) if enableMultimodel == nil { enableMultimodel = &kb.ChunkingConfig.EnableMultimodal } go s.processDocument(newCtx, kb, knowledge, file, *enableMultimodel) logger.Infof(ctx, "Knowledge from file created successfully, ID: %s", knowledge.ID) return knowledge, nil } // CreateKnowledgeFromURL creates a knowledge entry from a URL source func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from URL") logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // Validate URL format and security logger.Info(ctx, "Validating URL") if !isValidURL(url) || !secutils.IsValidURL(url) { logger.Error(ctx, "Invalid or unsafe URL format") return nil, ErrInvalidURL } // Check if URL already exists in the knowledge base tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Checking if URL exists, tenant ID: %d", tenantID) fileHash := calculateStr(url) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "url", URL: url, FileHash: fileHash, }) if err != nil { logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) return nil, err } if exists { logger.Infof(ctx, "URL already exists: %s", url) // Update creation time for existing knowledge existingKnowledge.CreatedAt = time.Now() existingKnowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, existingKnowledge); err != nil { logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) return nil, err } return existingKnowledge, types.NewDuplicateURLError(existingKnowledge) } // Check storage quota tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { logger.Error(ctx, "Storage quota exceeded") return nil, types.NewStorageQuotaExceededError() } // Create knowledge record logger.Info(ctx, "Creating knowledge record") knowledge := &types.Knowledge{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeBaseID: kbID, Type: "url", Source: url, FileHash: fileHash, ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, } // Save knowledge record logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record: %v", err) return nil, err } // Process URL asynchronously logger.Info(ctx, "Starting asynchronous URL processing") if enableMultimodel == nil { enableMultimodel = &kb.ChunkingConfig.EnableMultimodal } newCtx := logger.CloneContext(ctx) go s.processDocumentFromURL(newCtx, kb, knowledge, url, *enableMultimodel) logger.Infof(ctx, "Knowledge from URL created successfully, ID: %s", knowledge.ID) return knowledge, nil } // CreateKnowledgeFromPassage creates a knowledge entry from text passages func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string, ) (*types.Knowledge, error) { return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, false) } // CreateKnowledgeFromPassageSync creates a knowledge entry from text passages and waits for indexing to complete. func (s *knowledgeService) CreateKnowledgeFromPassageSync(ctx context.Context, kbID string, passage []string, ) (*types.Knowledge, error) { return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, true) } // CreateKnowledgeFromManual creates or saves manual Markdown knowledge content. func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context, kbID string, payload *types.ManualKnowledgePayload, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating manual knowledge entry") if payload == nil { return nil, werrors.NewBadRequestError("请求内容不能为空") } cleanContent := secutils.CleanMarkdown(payload.Content) if strings.TrimSpace(cleanContent) == "" { return nil, werrors.NewValidationError("内容不能为空") } if len([]rune(cleanContent)) > manualContentMaxLength { return nil, werrors.NewValidationError(fmt.Sprintf("内容长度超出限制(最多%d个字符)", manualContentMaxLength)) } safeTitle, ok := secutils.ValidateInput(payload.Title) if !ok { return nil, werrors.NewValidationError("标题包含非法字符或超出长度限制") } status := strings.ToLower(strings.TrimSpace(payload.Status)) if status == "" { status = types.ManualKnowledgeStatusDraft } if status != types.ManualKnowledgeStatusDraft && status != types.ManualKnowledgeStatusPublish { return nil, werrors.NewValidationError("状态仅支持 draft 或 publish") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) now := time.Now() title := safeTitle if title == "" { title = fmt.Sprintf("Knowledge-%s", now.Format("20060102-150405")) } fileName := ensureManualFileName(title) meta := types.NewManualKnowledgeMetadata(cleanContent, status, 1) knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, Type: types.KnowledgeTypeManual, Title: title, Description: "", Source: types.KnowledgeTypeManual, ParseStatus: types.ManualKnowledgeStatusDraft, EnableStatus: "disabled", CreatedAt: now, UpdatedAt: now, EmbeddingModelID: kb.EmbeddingModelID, FileName: fileName, FileType: types.KnowledgeTypeManual, } if err := knowledge.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata: %v", err) return nil, err } knowledge.EnsureManualDefaults() if status == types.ManualKnowledgeStatusPublish { knowledge.ParseStatus = "pending" } if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create manual knowledge record: %v", err) return nil, err } if status == types.ManualKnowledgeStatusPublish { logger.Infof(ctx, "Manual knowledge created, scheduling indexing, ID: %s", knowledge.ID) s.triggerManualProcessing(ctx, kb, knowledge, cleanContent, false) } return knowledge, nil } // createKnowledgeFromPassageInternal consolidates the common logic for creating knowledge from passages. // When syncMode is true, chunk processing is performed synchronously; otherwise, it's processed asynchronously. func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Context, kbID string, passage []string, syncMode bool, ) (*types.Knowledge, error) { if syncMode { logger.Info(ctx, "Start creating knowledge from passage (sync)") } else { logger.Info(ctx, "Start creating knowledge from passage") } logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage)) // 验证段落内容安全性 safePassages := make([]string, 0, len(passage)) for i, p := range passage { safePassage, isValid := secutils.ValidateInput(p) if !isValid { logger.Errorf(ctx, "Invalid passage content at index %d", i) return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1)) } safePassages = append(safePassages, safePassage) } // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) return nil, err } // Create knowledge record if syncMode { logger.Info(ctx, "Creating knowledge record (sync)") } else { logger.Info(ctx, "Creating knowledge record") } knowledge := &types.Knowledge{ ID: uuid.New().String(), TenantID: ctx.Value(types.TenantIDContextKey).(uint), KnowledgeBaseID: kbID, Type: "passage", ParseStatus: "pending", EnableStatus: "disabled", CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, } // Save knowledge record logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { logger.Errorf(ctx, "Failed to create knowledge record: %v", err) return nil, err } // Process passages if syncMode { logger.Info(ctx, "Processing passage synchronously") s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) logger.Infof(ctx, "Knowledge from passage created successfully (sync), ID: %s", knowledge.ID) } else { logger.Info(ctx, "Starting asynchronous passage processing") go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID) } return knowledge, nil } // GetKnowledgeByID retrieves a knowledge entry by its ID func (s *knowledgeService) GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) { logger.Info(ctx, "Start getting knowledge by ID") logger.Infof(ctx, "Knowledge ID: %s", id) tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Tenant ID: %d", tenantID) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "knowledge_id": id, "tenant_id": tenantID, }) return nil, err } logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, type: %s", knowledge.ID, knowledge.Type) return knowledge, nil } // ListKnowledgeByKnowledgeBaseID returns all knowledge entries in a knowledge base func (s *knowledgeService) ListKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, ) ([]*types.Knowledge, error) { return s.repo.ListKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint), kbID) } // ListPagedKnowledgeByKnowledgeBaseID returns paginated knowledge entries in a knowledge base func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, page *types.Pagination, tagID string, ) (*types.PageResult, error) { knowledges, total, err := s.repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint), kbID, page, tagID) if err != nil { return nil, err } return types.NewPageResult(total, page, knowledges), nil } // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error { // Get the knowledge entry knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) if err != nil { return err } wg := errgroup.Group{} // Delete knowledge embeddings from vector store wg.Go(func() error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } return nil }) // Delete all chunks associated with this knowledge wg.Go(func() error { if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") return err } return nil }) // Delete the physical file if it exists wg.Go(func() error { if knowledge.FilePath != "" { if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") } } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) tenantInfo.StorageUsed -= knowledge.StorageSize if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") } return nil }) // Delete the knowledge graph wg.Go(func() error { namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") return err } return nil }) if err = wg.Wait(); err != nil { return err } // Delete the knowledge entry itself from the database return s.repo.DeleteKnowledge(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) } // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error { if len(ids) == 0 { return nil } // 1. Get the knowledge entry tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantInfo.ID, ids) if err != nil { return err } wg := errgroup.Group{} // 2. Delete knowledge embeddings from vector store wg.Go(func() error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } group := map[string][]string{} for _, knowledge := range knowledgeList { group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID) } for embeddingModelID, knowledgeList := range group { embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed") return err } if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } } return nil }) // 3. Delete all chunks associated with this knowledge wg.Go(func() error { if err := s.chunkService.DeleteByKnowledgeList(ctx, ids); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") return err } return nil }) // 4. Delete the physical file if it exists wg.Go(func() error { storageAdjust := int64(0) for _, knowledge := range knowledgeList { if knowledge.FilePath != "" { if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") } } storageAdjust -= knowledge.StorageSize } tenantInfo.StorageUsed += storageAdjust if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") } return nil }) // Delete the knowledge graph wg.Go(func() error { namespaces := []types.NameSpace{} for _, knowledge := range knowledgeList { namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}) } if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") return err } return nil }) if err = wg.Wait(); err != nil { return err } // 5. Delete the knowledge entry itself from the database return s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids) } func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowledge, targetKB *types.KnowledgeBase) (err error) { if src.ParseStatus != "completed" { logger.GetLogger(ctx).WithField("knowledge_id", src.ID).Errorf("MoveKnowledge parse status is not completed") return nil } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) dst := &types.Knowledge{ ID: uuid.New().String(), TenantID: targetKB.TenantID, KnowledgeBaseID: targetKB.ID, Type: src.Type, Title: src.Title, Description: src.Description, Source: src.Source, ParseStatus: "processing", EnableStatus: "disabled", EmbeddingModelID: targetKB.EmbeddingModelID, FileName: src.FileName, FileType: src.FileType, FileSize: src.FileSize, FileHash: src.FileHash, FilePath: src.FilePath, StorageSize: src.StorageSize, Metadata: src.Metadata, } defer func() { if err != nil { dst.ParseStatus = "failed" dst.ErrorMessage = err.Error() _ = s.repo.UpdateKnowledge(ctx, dst) logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge failed to move knowledge") } else { dst.ParseStatus = "completed" dst.EnableStatus = "enabled" _ = s.repo.UpdateKnowledge(ctx, dst) logger.GetLogger(ctx).WithField("knowledge_id", dst.ID).Infof("MoveKnowledge move knowledge successfully") } }() if err = s.repo.CreateKnowledge(ctx, dst); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge create knowledge failed") return } tenantInfo.StorageUsed += dst.StorageSize if err = s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, dst.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge update tenant storage used failed") return } if err = s.CloneChunk(ctx, src, dst); err != nil { logger.GetLogger(ctx).WithField("knowledge_id", dst.ID). WithField("error", err).Errorf("MoveKnowledge move chunks failed") return } return } // processDocument handles asynchronous processing of document files func (s *knowledgeService) processDocument(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool, ) { logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel) ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument") defer span.End() span.SetAttributes( attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)), attribute.String("knowledge_base_id", kb.ID), attribute.Int("tenant_id", int(kb.TenantID)), attribute.String("knowledge_id", knowledge.ID), attribute.String("file_name", knowledge.FileName), attribute.String("file_type", knowledge.FileType), attribute.String("file_path", knowledge.FilePath), attribute.Int64("file_size", knowledge.FileSize), attribute.String("embedding_model", knowledge.EmbeddingModelID), attribute.Bool("enable_multimodal", enableMultimodel), ) logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String()) if !enableMultimodel && IsImageType(knowledge.FileType) { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = ErrImageNotParse.Error() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(ErrImageNotParse) return } // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { span.RecordError(err) return } // Read and chunk the document f, err := file.Open() if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument open file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } defer f.Close() span.AddEvent("start read file") contentBytes, err := io.ReadAll(f) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Split file into chunks using document reader service logger.GetLogger(ctx).Infof("processDocument split file content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v", len(contentBytes), knowledge.FileName, knowledge.FileType, kb.ChunkingConfig.Separators, enableMultimodel) resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ FileContent: contentBytes, FileName: knowledge.FileName, FileType: knowledge.FileType, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: &proto.VLMConfig{ ModelName: kb.VLMConfig.ModelName, BaseUrl: kb.VLMConfig.BaseURL, ApiKey: kb.VLMConfig.APIKey, InterfaceType: kb.VLMConfig.InterfaceType, }, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument read file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Process and store chunks span.AddEvent("start process chunks") s.processChunks(ctx, kb, knowledge, resp.Chunks) } // processDocumentFromURL handles asynchronous processing of URL content func (s *knowledgeService) processDocumentFromURL(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool, ) { // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { return } logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel) // Fetch and chunk content from URL resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{ Url: url, Title: knowledge.Title, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: &proto.VLMConfig{ ModelName: kb.VLMConfig.ModelName, BaseUrl: kb.VLMConfig.BaseURL, ApiKey: kb.VLMConfig.APIKey, InterfaceType: kb.VLMConfig.InterfaceType, }, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } // Process and store chunks s.processChunks(ctx, kb, knowledge, resp.Chunks) } // processDocumentFromPassage handles asynchronous processing of text passages func (s *knowledgeService) processDocumentFromPassage(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string, ) { // Update status to processing knowledge.ParseStatus = "processing" knowledge.UpdatedAt = time.Now() if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { return } // Convert passages to chunks chunks := make([]*proto.Chunk, 0, len(passage)) start, end := 0, 0 for i, p := range passage { if p == "" { continue } end += len([]rune(p)) chunk := &proto.Chunk{ Content: p, Seq: int32(i), Start: int32(start), End: int32(end), } start = end chunks = append(chunks, chunk) } // Process and store chunks s.processChunks(ctx, kb, knowledge, chunks) } // processChunks processes chunks and creates embeddings for knowledge content func (s *knowledgeService) processChunks(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*proto.Chunk, ) { ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processChunks") defer span.End() span.SetAttributes( attribute.Int("tenant_id", int(knowledge.TenantID)), attribute.String("knowledge_base_id", knowledge.KnowledgeBaseID), attribute.String("knowledge_id", knowledge.ID), attribute.String("embedding_model_id", kb.EmbeddingModelID), attribute.Int("chunk_count", len(chunks)), ) // Get embedding model for vectorization embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed") span.RecordError(err) return } // Generate document summary - 只使用文本类型的 Chunk chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get summary model failed") span.RecordError(err) return } enableGraphRAG := os.Getenv("ENABLE_GRAPH_RAG") == "true" // Create chunk objects from proto chunks maxSeq := 0 // 统计图片相关的子Chunk数量,用于扩展insertChunks的容量 imageChunkCount := 0 for _, chunkData := range chunks { if len(chunkData.Images) > 0 { // 为每个图片的OCR和Caption分别创建一个Chunk imageChunkCount += len(chunkData.Images) * 2 } if int(chunkData.Seq) > maxSeq { maxSeq = int(chunkData.Seq) } } // 重新分配容量,考虑图片相关的Chunk insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount) for _, chunkData := range chunks { if strings.TrimSpace(chunkData.Content) == "" { continue } // 创建主文本Chunk textChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: chunkData.Content, ChunkIndex: int(chunkData.Seq), IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(chunkData.Start), EndAt: int(chunkData.End), ChunkType: types.ChunkTypeText, } var chunkImages []types.ImageInfo insertChunks = append(insertChunks, textChunk) // 处理图片信息 if len(chunkData.Images) > 0 { logger.GetLogger(ctx).Infof("Processing %d images in chunk #%d", len(chunkData.Images), chunkData.Seq) for i, img := range chunkData.Images { // 保存图片信息到文本Chunk imageInfo := types.ImageInfo{ URL: img.Url, OriginalURL: img.OriginalUrl, StartPos: int(img.Start), EndPos: int(img.End), OCRText: img.OcrText, Caption: img.Caption, } chunkImages = append(chunkImages, imageInfo) // 将ImageInfo序列化为JSON imageInfoJSON, err := json.Marshal([]types.ImageInfo{imageInfo}) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") continue } // 如果有OCR文本,创建OCR Chunk if img.OcrText != "" { ocrChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: img.OcrText, ChunkIndex: maxSeq + i*100 + 1, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(img.Start), EndAt: int(img.End), ChunkType: types.ChunkTypeImageOCR, ParentChunkID: textChunk.ID, ImageInfo: string(imageInfoJSON), } insertChunks = append(insertChunks, ocrChunk) logger.GetLogger(ctx).Infof("Created OCR chunk for image %d in chunk #%d", i, chunkData.Seq) } // 如果有图片描述,创建Caption Chunk if img.Caption != "" { captionChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: img.Caption, ChunkIndex: maxSeq + i*100 + 2, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: int(img.Start), EndAt: int(img.End), ChunkType: types.ChunkTypeImageCaption, ParentChunkID: textChunk.ID, ImageInfo: string(imageInfoJSON), } insertChunks = append(insertChunks, captionChunk) logger.GetLogger(ctx).Infof("Created caption chunk for image %d in chunk #%d", i, chunkData.Seq) } } imageInfoJSON, err := json.Marshal(chunkImages) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") continue } textChunk.ImageInfo = string(imageInfoJSON) } } // Sort chunks by index for proper ordering sort.Slice(insertChunks, func(i, j int) bool { return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex }) // 仅为文本类型的Chunk设置前后关系 textChunks := make([]*types.Chunk, 0, len(chunks)) for _, chunk := range insertChunks { if chunk.ChunkType == types.ChunkTypeText { textChunks = append(textChunks, chunk) } } // 设置文本Chunk之间的前后关系 for i, chunk := range textChunks { if i > 0 { textChunks[i-1].NextChunkID = chunk.ID } if i < len(textChunks)-1 { textChunks[i+1].PreChunkID = chunk.ID } } if enableGraphRAG { relationChunkSize := 5 indirectRelationChunkSize := 5 graphBuilder := NewGraphBuilder(s.config, chatModel) err = graphBuilder.BuildGraph(ctx, textChunks) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks build graph failed") span.RecordError(err) } else { for _, chunk := range textChunks { chunk.RelationChunks, _ = json.Marshal(graphBuilder.GetRelationChunks(chunk.ID, relationChunkSize)) chunk.IndirectRelationChunks, _ = json.Marshal(graphBuilder.GetIndirectRelationChunks(chunk.ID, indirectRelationChunkSize)) } for i, entity := range graphBuilder.GetAllEntities() { relationChunks, _ := json.Marshal(entity.ChunkIDs) entityChunk := &types.Chunk{ ID: entity.ID, TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: entity.Description, ChunkIndex: maxSeq + i*100 + 3, IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), ChunkType: types.ChunkTypeEntity, RelationChunks: types.JSON(relationChunks), } insertChunks = append(insertChunks, entityChunk) } for i, relationship := range graphBuilder.GetAllRelationships() { relationChunks, _ := json.Marshal(relationship.ChunkIDs) relationshipChunk := &types.Chunk{ ID: relationship.ID, TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: relationship.Description, ChunkIndex: maxSeq + i*100 + 4, IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), ChunkType: types.ChunkTypeRelationship, RelationChunks: types.JSON(relationChunks), } insertChunks = append(insertChunks, relationshipChunk) } } } span.AddEvent("extract summary") summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processChunks get summary failed, use first chunk as description") if len(textChunks) > 0 { knowledge.Description = textChunks[0].Content } } else { knowledge.Description = summary } span.SetAttributes(attribute.String("summary", knowledge.Description)) // 批量索引 if strings.TrimSpace(knowledge.Description) != "" && len(textChunks) > 0 { sChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, knowledge.Description), ChunkIndex: maxSeq + 3, // 使用不冲突的索引方式 IsEnabled: true, CreatedAt: time.Now(), UpdatedAt: time.Now(), StartAt: 0, EndAt: 0, ChunkType: types.ChunkTypeSummary, ParentChunkID: textChunks[0].ID, } logger.GetLogger(ctx).Infof("Created summary chunk for %s with index %d", sChunk.ParentChunkID, sChunk.ChunkIndex) insertChunks = append(insertChunks, sChunk) } // Create index information for each chunk indexInfoList := utils.MapSlice(insertChunks, func(chunk *types.Chunk) *types.IndexInfo { return &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, } }) // Initialize retrieval engine tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Calculate storage size required for embeddings span.AddEvent("estimate storage size") totalStorageSize := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfoList) if tenantInfo.StorageQuota > 0 { // Re-fetch tenant storage information tenantInfo, err = s.tenantRepo.GetTenantByID(ctx, tenantInfo.ID) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } // Check if there's enough storage quota available if tenantInfo.StorageUsed+totalStorageSize > tenantInfo.StorageQuota { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "存储空间不足" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(errors.New("storage quota exceeded")) return } } // Save chunks to database span.AddEvent("create chunks") if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) span.RecordError(err) return } span.AddEvent("batch index") err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList) if err != nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) // delete failed chunks if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.Errorf(ctx, "Delete chunks failed: %v", err) } // delete index if err := retrieveEngine.DeleteByKnowledgeIDList( ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), ); err != nil { logger.Errorf(ctx, "Delete index failed: %v", err) } span.RecordError(err) return } logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList)) logger.Infof(ctx, "processChunks create relationship rag task") for _, chunk := range textChunks { err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed") span.RecordError(err) } } // Update knowledge status to completed knowledge.ParseStatus = "completed" knowledge.EnableStatus = "enabled" knowledge.StorageSize = totalStorageSize now := time.Now() knowledge.ProcessedAt = &now knowledge.UpdatedAt = now if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed") } // Update tenant's storage usage tenantInfo.StorageUsed += totalStorageSize if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update tenant storage used failed") } logger.GetLogger(ctx).Infof("processChunks successfully") } // GetSummary generates a summary for knowledge content using an AI model func (s *knowledgeService) getSummary(ctx context.Context, summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk, ) (string, error) { // Get knowledge info from the first chunk if len(chunks) == 0 { return "", fmt.Errorf("no chunks provided for summary generation") } // concat chunk contents chunkContents := "" allImageInfos := make([]*types.ImageInfo, 0) // then, sort chunks by StartAt sortedChunks := make([]*types.Chunk, len(chunks)) copy(sortedChunks, chunks) sort.Slice(sortedChunks, func(i, j int) bool { return sortedChunks[i].StartAt < sortedChunks[j].StartAt }) // concat chunk contents and collect image infos for _, chunk := range sortedChunks { if chunk.EndAt > 4096 { break } chunkContents = string([]rune(chunkContents)[:chunk.StartAt]) + chunk.Content if chunk.ImageInfo != "" { var images []*types.ImageInfo if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err == nil { allImageInfos = append(allImageInfos, images...) } } } // remove markdown image syntax re := regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) chunkContents = re.ReplaceAllString(chunkContents, "") // collect all image infos if len(allImageInfos) > 0 { // add image infos to chunk contents var imageAnnotations string for _, img := range allImageInfos { if img.Caption != "" { imageAnnotations += fmt.Sprintf("\n[图片描述: %s]", img.Caption) } if img.OCRText != "" { imageAnnotations += fmt.Sprintf("\n[图片文字: %s]", img.OCRText) } } // concat chunk contents and image annotations chunkContents = chunkContents + imageAnnotations } if len(chunkContents) < 300 { return chunkContents, nil } // Prepare content with metadata for summary generation contentWithMetadata := chunkContents // Add knowledge metadata if available if knowledge != nil { metadataIntro := fmt.Sprintf("文档类型: %s\n文件名称: %s\n", knowledge.FileType, knowledge.FileName) // Add additional metadata if available if knowledge.Type != "" { metadataIntro += fmt.Sprintf("知识类型: %s\n", knowledge.Type) } // Prepend metadata to content contentWithMetadata = metadataIntro + "\n内容:\n" + contentWithMetadata } // Generate summary using AI model thinking := false summary, err := summaryModel.Chat(ctx, []chat.Message{ { Role: "system", Content: s.config.Conversation.GenerateSummaryPrompt, }, { Role: "user", Content: contentWithMetadata, }, }, &chat.ChatOptions{ Temperature: 0.3, MaxTokens: 1024, Thinking: &thinking, }) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("GetSummary failed") return "", err } logger.GetLogger(ctx).WithField("summary", summary.Content).Infof("GetSummary success") return summary.Content, nil } // GetKnowledgeFile retrieves the physical file associated with a knowledge entry func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { // Get knowledge record knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) if err != nil { return nil, "", err } // Get the file from storage file, err := s.fileSvc.GetFile(ctx, knowledge.FilePath) if err != nil { return nil, "", err } return file, knowledge.FileName, nil } func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { record, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge record: %v", err) return err } // if need other fields update, please add here if knowledge.Title != "" { record.Title = knowledge.Title } // Update knowledge record in the repository if err := s.repo.UpdateKnowledge(ctx, record); err != nil { logger.Errorf(ctx, "Failed to update knowledge: %v", err) return err } logger.Infof(ctx, "Knowledge updated successfully, ID: %s", knowledge.ID) return nil } // UpdateManualKnowledge updates manual Markdown knowledge content. func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context, knowledgeID string, payload *types.ManualKnowledgePayload, ) (*types.Knowledge, error) { logger.Info(ctx, "Start updating manual knowledge entry") if payload == nil { return nil, werrors.NewBadRequestError("请求内容不能为空") } cleanContent := secutils.CleanMarkdown(payload.Content) if strings.TrimSpace(cleanContent) == "" { return nil, werrors.NewValidationError("内容不能为空") } if len([]rune(cleanContent)) > manualContentMaxLength { return nil, werrors.NewValidationError(fmt.Sprintf("内容长度超出限制(最多%d个字符)", manualContentMaxLength)) } safeTitle, ok := secutils.ValidateInput(payload.Title) if !ok { return nil, werrors.NewValidationError("标题包含非法字符或超出长度限制") } status := strings.ToLower(strings.TrimSpace(payload.Status)) if status == "" { status = types.ManualKnowledgeStatusDraft } if status != types.ManualKnowledgeStatusDraft && status != types.ManualKnowledgeStatusPublish { return nil, werrors.NewValidationError("状态仅支持 draft 或 publish") } tenantID := ctx.Value(types.TenantIDContextKey).(uint) existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to load knowledge: %v", err) return nil, err } if !existing.IsManual() { return nil, werrors.NewBadRequestError("仅支持手工知识的在线编辑") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base for manual update: %v", err) return nil, err } var version int if meta, err := existing.ManualMetadata(); err == nil && meta != nil { version = meta.Version + 1 } else { version = 1 } meta := types.NewManualKnowledgeMetadata(cleanContent, status, version) if err := existing.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata during update: %v", err) return nil, err } if safeTitle != "" { existing.Title = safeTitle } else if existing.Title == "" { existing.Title = fmt.Sprintf("手工知识-%s", time.Now().Format("20060102-150405")) } existing.FileName = ensureManualFileName(existing.Title) existing.FileType = types.KnowledgeTypeManual existing.Type = types.KnowledgeTypeManual existing.Source = types.KnowledgeTypeManual existing.EnableStatus = "disabled" existing.UpdatedAt = time.Now() if err := s.cleanupKnowledgeResources(ctx, existing); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "knowledge_id": knowledgeID, }) return nil, err } existing.EmbeddingModelID = kb.EmbeddingModelID if status == types.ManualKnowledgeStatusDraft { existing.ParseStatus = types.ManualKnowledgeStatusDraft existing.Description = "" existing.ProcessedAt = nil if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { logger.Errorf(ctx, "Failed to persist manual draft: %v", err) return nil, err } return existing, nil } existing.ParseStatus = "pending" existing.Description = "" existing.ProcessedAt = nil if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { logger.Errorf(ctx, "Failed to persist manual knowledge before indexing: %v", err) return nil, err } logger.Infof(ctx, "Manual knowledge updated, scheduling indexing, ID: %s", existing.ID) s.triggerManualProcessing(ctx, kb, existing, cleanContent, false) return existing, nil } // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif": return true default: return false } } // getFileType extracts the file extension from a filename func getFileType(filename string) string { ext := strings.Split(filename, ".") if len(ext) < 2 { return "unknown" } return ext[len(ext)-1] } // isValidURL verifies if a URL is valid // isValidURL 检查URL是否有效 func isValidURL(url string) bool { if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { return true } return false } // GetKnowledgeBatch retrieves multiple knowledge entries by their IDs func (s *knowledgeService) GetKnowledgeBatch(ctx context.Context, tenantID uint, ids []string, ) ([]*types.Knowledge, error) { if len(ids) == 0 { return nil, nil } return s.repo.GetKnowledgeBatch(ctx, tenantID, ids) } // calculateFileHash calculates MD5 hash of a file func calculateFileHash(file *multipart.FileHeader) (string, error) { f, err := file.Open() if err != nil { return "", err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return "", err } // Reset file pointer for subsequent operations if _, err := f.Seek(0, 0); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } func calculateStr(strList ...string) string { h := md5.New() input := strings.Join(strList, "") h.Write([]byte(input)) return hex.EncodeToString(h.Sum(nil)) } func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error { srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID) if err != nil { logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) return err } addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) batch := 10 g, gctx := errgroup.WithContext(ctx) for ids := range slices.Chunk(delKnowledge, batch) { ids = ids g.Go(func() error { err := s.DeleteKnowledgeList(gctx, ids) if err != nil { logger.Errorf(gctx, "delete partial knowledge %v: %w", ids, err) return err } return nil }) } err = g.Wait() if err != nil { logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) return err } // Copy context out of auto-stop task g, gctx = errgroup.WithContext(ctx) g.SetLimit(batch) for _, knowledge := range addKnowledge { knowledge = knowledge g.Go(func() error { srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) if err != nil { logger.Errorf(gctx, "get knowledge %s: %w", knowledge, err) return err } err = s.cloneKnowledge(gctx, srcKn, dstKB) if err != nil { logger.Errorf(gctx, "clone knowledge %s: %w", knowledge, err) return err } return nil }) } err = g.Wait() if err != nil { logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) return err } return nil } func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, chunks []*types.Chunk) error { // Get embedding model from knowledge base sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, sourceKB.EmbeddingModelID) if err != nil { return err } // Initialize composite retrieve engine from tenant configuration indexInfo := make([]*types.IndexInfo, 0, len(chunks)) ids := make([]string, 0, len(chunks)) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kbID { logger.Warnf(ctx, "Knowledge base ID mismatch: %s != %s", chunk.KnowledgeBaseID, kbID) continue } indexInfo = append(indexInfo, &types.IndexInfo{ Content: chunk.Content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) ids = append(ids, chunk.ID) } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } // Delete old vector representation of the chunk err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions()) if err != nil { return err } // Index updated chunk content with new vector representation err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo) if err != nil { return err } return nil } func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID string, chunkID string, imageInfo string) error { var images []*types.ImageInfo if err := json.Unmarshal([]byte(imageInfo), &images); err != nil { logger.Errorf(ctx, "Failed to unmarshal image info: %v", err) return err } if len(images) != 1 { logger.Warnf(ctx, "Expected exactly one image info, got %d", len(images)) return nil } image := images[0] // Retrieve all chunks with the given parent chunk ID chunk, err := s.chunkService.GetChunkByID(ctx, chunkID) if err != nil { logger.Errorf(ctx, "Failed to get chunk: %v", err) return err } chunk.ImageInfo = imageInfo tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunkChildren, err := s.chunkService.ListChunkByParentID(ctx, tenantID, chunkID) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "parent_chunk_id": chunkID, "tenant_id": tenantID, }) return err } logger.Infof(ctx, "Found %d chunks with parent chunk ID: %s", len(chunkChildren), chunkID) // Iterate through each chunk and update its content based on the image information updateChunk := []*types.Chunk{chunk} var addChunk []*types.Chunk // Track whether we've found OCR and caption child chunks for this image hasOCRChunk := false hasCaptionChunk := false for i, child := range chunkChildren { // Skip chunks that are not image types var cImageInfo []*types.ImageInfo err = json.Unmarshal([]byte(child.ImageInfo), &cImageInfo) if err != nil { logger.Warnf(ctx, "Failed to unmarshal image %s info: %v", child.ID, err) continue } if len(cImageInfo) == 0 { continue } if cImageInfo[0].OriginalURL != image.OriginalURL { logger.Warnf(ctx, "Skipping chunk ID: %s, image URL mismatch: %s != %s", child.ID, cImageInfo[0].OriginalURL, image.OriginalURL) continue } // Mark that we've found chunks for this image if child.ChunkType == types.ChunkTypeImageCaption { hasCaptionChunk = true // Update caption if it has changed if image.Caption != cImageInfo[0].Caption { child.Content = image.Caption child.ImageInfo = imageInfo updateChunk = append(updateChunk, chunkChildren[i]) } } else if child.ChunkType == types.ChunkTypeImageOCR { hasOCRChunk = true // Update OCR if it has changed if image.OCRText != cImageInfo[0].OCRText { child.Content = image.OCRText child.ImageInfo = imageInfo updateChunk = append(updateChunk, chunkChildren[i]) } } } // Create a new caption chunk if it doesn't exist and we have caption data if !hasCaptionChunk && image.Caption != "" { captionChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, Content: image.Caption, ChunkType: types.ChunkTypeImageCaption, ParentChunkID: chunk.ID, ImageInfo: imageInfo, } addChunk = append(addChunk, captionChunk) logger.Infof(ctx, "Created new caption chunk ID: %s for image URL: %s", captionChunk.ID, image.OriginalURL) } // Create a new OCR chunk if it doesn't exist and we have OCR data if !hasOCRChunk && image.OCRText != "" { ocrChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, Content: image.OCRText, ChunkType: types.ChunkTypeImageOCR, ParentChunkID: chunk.ID, ImageInfo: imageInfo, } addChunk = append(addChunk, ocrChunk) logger.Infof(ctx, "Created new OCR chunk ID: %s for image URL: %s", ocrChunk.ID, image.OriginalURL) } logger.Infof(ctx, "Updated %d chunks out of %d total chunks", len(updateChunk), len(chunkChildren)+1) if len(addChunk) > 0 { err := s.chunkService.CreateChunks(ctx, addChunk) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "add_chunk_size": len(addChunk), }) return err } } // Update the chunks for _, c := range updateChunk { err := s.chunkService.UpdateChunk(ctx, c) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": c.ID, "knowledge_id": c.KnowledgeID, }) return err } } // Update the chunk vector err = s.updateChunkVector(ctx, chunk.KnowledgeBaseID, append(updateChunk, addChunk...)) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": chunk.ID, "knowledge_id": chunk.KnowledgeID, }) return err } // Update the knowledge file hash knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err } fileHash := calculateStr(knowledgeID, knowledge.FileHash, imageInfo) knowledge.FileHash = fileHash err = s.repo.UpdateKnowledge(ctx, knowledge) if err != nil { logger.Warnf(ctx, "Failed to update knowledge file hash: %v", err) } logger.Infof(ctx, "Updated chunk successfully, chunk ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID) return nil } // CloneChunk clone chunks from one knowledge to another // This method transfers a chunk from a source knowledge document to a target knowledge document // It handles the creation of new chunks in the target knowledge and updates the vector database accordingly // Parameters: // - ctx: Context with authentication and request information // - src: Source knowledge document containing the chunk to move // - dst: Target knowledge document where the chunk will be moved // // Returns: // - error: Any error encountered during the move operation // // This method handles the chunk transfer logic, including creating new chunks in the target knowledge // and updating the vector database representation of the moved chunks. // It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained // by mapping the source chunk IDs to the new target chunk IDs. func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) error { chunkPage := 1 chunkPageSize := 100 srcTodst := map[string]string{} targetChunks := make([]*types.Chunk, 0, 10) chunkType := []types.ChunkType{ types.ChunkTypeText, types.ChunkTypeSummary, types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, } for { sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx, src.TenantID, src.ID, &types.Pagination{ Page: chunkPage, PageSize: chunkPageSize, }, chunkType, "", ) chunkPage++ if err != nil { return err } if len(sourceChunks) == 0 { break } for _, sourceChunk := range sourceChunks { targetChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: dst.TenantID, KnowledgeID: dst.ID, KnowledgeBaseID: dst.KnowledgeBaseID, Content: sourceChunk.Content, ChunkIndex: sourceChunk.ChunkIndex, IsEnabled: sourceChunk.IsEnabled, StartAt: sourceChunk.StartAt, EndAt: sourceChunk.EndAt, PreChunkID: sourceChunk.PreChunkID, NextChunkID: sourceChunk.NextChunkID, ChunkType: sourceChunk.ChunkType, ParentChunkID: sourceChunk.ParentChunkID, ImageInfo: sourceChunk.ImageInfo, } targetChunks = append(targetChunks, targetChunk) srcTodst[sourceChunk.ID] = targetChunk.ID } } for _, targetChunk := range targetChunks { if val, ok := srcTodst[targetChunk.PreChunkID]; ok { targetChunk.PreChunkID = val } else { targetChunk.PreChunkID = "" } if val, ok := srcTodst[targetChunk.NextChunkID]; ok { targetChunk.NextChunkID = val } else { targetChunk.NextChunkID = "" } if val, ok := srcTodst[targetChunk.ParentChunkID]; ok { targetChunk.ParentChunkID = val } else { targetChunk.ParentChunkID = "" } } for chunks := range slices.Chunk(targetChunks, chunkPageSize) { err := s.chunkRepo.CreateChunks(ctx, chunks) if err != nil { return err } } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID) if err != nil { return err } if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID, map[string]string{src.ID: dst.ID}, srcTodst, embeddingModel.GetDimensions(), ); err != nil { return err } return nil } // ListFAQEntries lists FAQ entries under a FAQ knowledge base. func (s *knowledgeService) ListFAQEntries(ctx context.Context, kbID string, page *types.Pagination, tagID string, ) (*types.PageResult, error) { if page == nil { page = &types.Pagination{} } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) if err != nil { return nil, err } if faqKnowledge == nil { return types.NewPageResult(0, page, []*types.FAQEntry{}), nil } chunkType := []types.ChunkType{types.ChunkTypeFAQ} chunks, total, err := s.chunkRepo.ListPagedChunksByKnowledgeID( ctx, tenantID, faqKnowledge.ID, page, chunkType, tagID, ) if err != nil { return nil, err } kb.EnsureDefaults() entries := make([]*types.FAQEntry, 0, len(chunks)) for _, chunk := range chunks { entry, err := s.chunkToFAQEntry(chunk, kb) if err != nil { return nil, err } entries = append(entries, entry) } return types.NewPageResult(total, page, entries), nil } // UpsertFAQEntries imports or appends FAQ entries. func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload, ) error { if payload == nil || len(payload.Entries) == 0 { return werrors.NewBadRequestError("FAQ 条目不能为空") } if payload.Mode == "" { payload.Mode = types.FAQBatchModeAppend } if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace { return werrors.NewBadRequestError("模式仅支持 append 或 replace") } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } kb.EnsureDefaults() tenantID := ctx.Value(types.TenantIDContextKey).(uint) faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb) if err != nil { return err } if payload.Mode == types.FAQBatchModeReplace { if err := s.cleanupFAQKnowledge(ctx, faqKnowledge); err != nil { return err } } startIndex := 0 if payload.Mode == types.FAQBatchModeAppend { _, total, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx, tenantID, faqKnowledge.ID, &types.Pagination{Page: 1, PageSize: 1}, []types.ChunkType{types.ChunkTypeFAQ}, "", ) if err != nil { return err } startIndex = int(total) } // 获取索引模式 indexMode := types.FAQIndexModeQuestionOnly if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } chunks := make([]*types.Chunk, 0, len(payload.Entries)) for idx := range payload.Entries { entry := payload.Entries[idx] meta, err := sanitizeFAQEntryPayload(&entry) if err != nil { return err } isEnabled := true if entry.IsEnabled != nil { isEnabled = *entry.IsEnabled } chunk := &types.Chunk{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeID: faqKnowledge.ID, KnowledgeBaseID: kb.ID, Content: buildFAQChunkContent(meta, indexMode), ChunkIndex: startIndex + idx + 1, IsEnabled: isEnabled, ChunkType: types.ChunkTypeFAQ, TagID: entry.TagID, } if err := chunk.SetFAQMetadata(meta); err != nil { return err } chunks = append(chunks, chunk) } if err := s.chunkService.CreateChunks(ctx, chunks); err != nil { return err } if err := s.indexFAQChunks(ctx, kb, faqKnowledge, chunks, true); err != nil { for _, chunk := range chunks { _ = s.chunkService.DeleteChunk(ctx, chunk.ID) } return err } return nil } // UpdateFAQEntry updates a single FAQ entry. func (s *knowledgeService) UpdateFAQEntry(ctx context.Context, kbID string, entryID string, payload *types.FAQEntryPayload, ) error { if payload == nil { return werrors.NewBadRequestError("请求体不能为空") } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } kb.EnsureDefaults() tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID { return werrors.NewForbiddenError("无权操作该 FAQ 条目") } if chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目") } meta, err := sanitizeFAQEntryPayload(payload) if err != nil { return err } if existing, err := chunk.FAQMetadata(); err == nil && existing != nil { meta.Version = existing.Version + 1 } if err := chunk.SetFAQMetadata(meta); err != nil { return err } // 获取索引模式 indexMode := types.FAQIndexModeQuestionOnly if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } chunk.Content = buildFAQChunkContent(meta, indexMode) chunk.TagID = payload.TagID isEnabledUpdated := false if payload.IsEnabled != nil { oldEnabled := chunk.IsEnabled chunk.IsEnabled = *payload.IsEnabled isEnabledUpdated = (oldEnabled != chunk.IsEnabled) } chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } // Sync is_enabled status to retriever engines if it was updated if isEnabledUpdated { chunkStatusMap := map[string]bool{chunk.ID: chunk.IsEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } } faqKnowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) if err != nil { return err } return s.indexFAQChunks(ctx, kb, faqKnowledge, []*types.Chunk{chunk}, false) } // UpdateFAQEntryStatus updates enable status for a FAQ entry. func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context, kbID string, entryID string, isEnabled bool, ) error { kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目") } if chunk.IsEnabled == isEnabled { return nil } chunk.IsEnabled = isEnabled chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } // Sync update to retriever engines chunkStatusMap := map[string]bool{chunk.ID: isEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } return nil } // UpdateFAQEntryStatusBatch updates enable status for FAQ entries in batch. func (s *knowledgeService) UpdateFAQEntryStatusBatch(ctx context.Context, kbID string, updates map[string]bool, ) error { if len(updates) == 0 { return nil } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all chunks in batch entryIDs := make([]string, 0, len(updates)) for entryID := range updates { entryIDs = append(entryIDs, entryID) } chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, entryIDs) if err != nil { return err } // Group chunks by enabled status for batch update chunkStatusMap := make(map[string]bool) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { continue } isEnabled, exists := updates[chunk.ID] if !exists { continue } if chunk.IsEnabled == isEnabled { continue } chunk.IsEnabled = isEnabled chunk.UpdatedAt = time.Now() if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { return err } chunkStatusMap[chunk.ID] = isEnabled } // Sync update to retriever engines if len(chunkStatusMap) > 0 { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil { return err } } return nil } // UpdateKnowledgeTag updates the tag assigned to a knowledge document. func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID string, tagID *string) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { return err } var resolvedTagID string if tagID != nil && *tagID != "" { tag, err := s.tagRepo.GetByID(ctx, tenantID, *tagID) if err != nil { return err } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError("标签不属于当前知识库") } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID return s.repo.UpdateKnowledge(ctx, knowledge) } // UpdateKnowledgeTagBatch updates tags for document knowledge items in batch. func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, updates map[string]*string) error { if len(updates) == 0 { return nil } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all knowledge items in batch knowledgeIDs := make([]string, 0, len(updates)) for knowledgeID := range updates { knowledgeIDs = append(knowledgeIDs, knowledgeID) } knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs) if err != nil { return err } // Build tag ID map for validation tagIDSet := make(map[string]bool) for _, tagID := range updates { if tagID != nil && *tagID != "" { tagIDSet[*tagID] = true } } // Validate all tags in batch tagMap := make(map[string]*types.KnowledgeTag) if len(tagIDSet) > 0 { tagIDs := make([]string, 0, len(tagIDSet)) for tagID := range tagIDSet { tagIDs = append(tagIDs, tagID) } for _, tagID := range tagIDs { tag, err := s.tagRepo.GetByID(ctx, tenantID, tagID) if err != nil { return err } tagMap[tagID] = tag } } // Update knowledge items knowledgeToUpdate := make([]*types.Knowledge, 0) for _, knowledge := range knowledgeList { tagID, exists := updates[knowledge.ID] if !exists { continue } var resolvedTagID string if tagID != nil && *tagID != "" { tag, ok := tagMap[*tagID] if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", *tagID)) } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于知识库 %s", *tagID, knowledge.KnowledgeBaseID)) } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID knowledgeToUpdate = append(knowledgeToUpdate, knowledge) } if len(knowledgeToUpdate) > 0 { return s.repo.UpdateKnowledgeBatch(ctx, knowledgeToUpdate) } return nil } // UpdateFAQEntryTag updates the tag assigned to an FAQ entry. func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, entryID string, tagID *string) error { kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, entryID) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("仅支持更新 FAQ 条目标签") } var resolvedTagID string if tagID != nil && *tagID != "" { tag, err := s.tagRepo.GetByID(ctx, tenantID, *tagID) if err != nil { return err } if tag.KnowledgeBaseID != kb.ID { return werrors.NewBadRequestError("标签不属于当前知识库") } resolvedTagID = tag.ID } chunk.TagID = resolvedTagID chunk.UpdatedAt = time.Now() return s.chunkRepo.UpdateChunk(ctx, chunk) } // UpdateFAQEntryTagBatch updates tags for FAQ entries in batch. func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID string, updates map[string]*string) error { if len(updates) == 0 { return nil } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) // Get all chunks in batch entryIDs := make([]string, 0, len(updates)) for entryID := range updates { entryIDs = append(entryIDs, entryID) } chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, entryIDs) if err != nil { return err } // Build tag ID map for validation tagIDSet := make(map[string]bool) for _, tagID := range updates { if tagID != nil && *tagID != "" { tagIDSet[*tagID] = true } } // Validate all tags in batch tagMap := make(map[string]*types.KnowledgeTag) if len(tagIDSet) > 0 { tagIDs := make([]string, 0, len(tagIDSet)) for tagID := range tagIDSet { tagIDs = append(tagIDs, tagID) } for _, tagID := range tagIDs { tag, err := s.tagRepo.GetByID(ctx, tenantID, tagID) if err != nil { return err } if tag.KnowledgeBaseID != kb.ID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于当前知识库", tagID)) } tagMap[tagID] = tag } } // Update chunks chunksToUpdate := make([]*types.Chunk, 0) for _, chunk := range chunks { if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { continue } tagID, exists := updates[chunk.ID] if !exists { continue } var resolvedTagID string if tagID != nil && *tagID != "" { tag, ok := tagMap[*tagID] if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", *tagID)) } resolvedTagID = tag.ID } chunk.TagID = resolvedTagID chunk.UpdatedAt = time.Now() chunksToUpdate = append(chunksToUpdate, chunk) } if len(chunksToUpdate) > 0 { return s.chunkRepo.UpdateChunks(ctx, chunksToUpdate) } return nil } // SearchFAQEntries searches FAQ entries using hybrid search. func (s *knowledgeService) SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest, ) ([]*types.FAQEntry, error) { // Validate FAQ knowledge base kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } // Set default values if req.VectorThreshold <= 0 { req.VectorThreshold = 0.7 } if req.MatchCount <= 0 { req.MatchCount = 10 } if req.MatchCount > 50 { req.MatchCount = 50 } // Prepare search parameters searchParams := types.SearchParams{ QueryText: req.QueryText, VectorThreshold: req.VectorThreshold, MatchCount: req.MatchCount, DisableKeywordsMatch: true, } // Call HybridSearch searchResults, err := s.kbService.HybridSearch(ctx, kbID, searchParams) if err != nil { return nil, err } if len(searchResults) == 0 { return []*types.FAQEntry{}, nil } // Extract chunk IDs and build score/match type maps chunkIDs := make([]string, 0, len(searchResults)) chunkScores := make(map[string]float64) chunkMatchTypes := make(map[string]types.MatchType) for _, result := range searchResults { // SearchResult.ID is the chunk ID chunkID := result.ID chunkIDs = append(chunkIDs, chunkID) chunkScores[chunkID] = result.Score chunkMatchTypes[chunkID] = result.MatchType } // Batch fetch chunks tenantID := ctx.Value(types.TenantIDContextKey).(uint) chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs) if err != nil { return nil, err } // Filter FAQ chunks and convert to FAQEntry kb.EnsureDefaults() entries := make([]*types.FAQEntry, 0, len(chunks)) for _, chunk := range chunks { // Only process FAQ type chunks if chunk.ChunkType != types.ChunkTypeFAQ { continue } if !chunk.IsEnabled { continue } entry, err := s.chunkToFAQEntry(chunk, kb) if err != nil { logger.Warnf(ctx, "Failed to convert chunk to FAQ entry: %v", err) continue } // Preserve score and match type from search results // Note: Negative question filtering is now handled in HybridSearch if score, ok := chunkScores[chunk.ID]; ok { entry.Score = score } if matchType, ok := chunkMatchTypes[chunk.ID]; ok { entry.MatchType = matchType } entries = append(entries, entry) } slices.SortFunc(entries, func(a, b *types.FAQEntry) int { return int(b.Score - a.Score) }) return entries, nil } // DeleteFAQEntries deletes FAQ entries in batch. func (s *knowledgeService) DeleteFAQEntries(ctx context.Context, kbID string, entryIDs []string, ) error { if len(entryIDs) == 0 { return werrors.NewBadRequestError("请选择需要删除的 FAQ 条目") } kb, err := s.validateFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint) var faqKnowledge *types.Knowledge chunksToRemove := make([]*types.Chunk, 0, len(entryIDs)) for _, id := range entryIDs { if id == "" { continue } chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, id) if err != nil { return err } if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { return werrors.NewBadRequestError("包含无效的 FAQ 条目") } if err := s.chunkService.DeleteChunk(ctx, id); err != nil { return err } if faqKnowledge == nil { faqKnowledge, err = s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) if err != nil { return err } } chunksToRemove = append(chunksToRemove, chunk) } if len(chunksToRemove) > 0 && faqKnowledge != nil { if err := s.deleteFAQChunkVectors(ctx, kb, faqKnowledge, chunksToRemove); err != nil { return err } } return nil } func (s *knowledgeService) validateFAQKnowledgeBase(ctx context.Context, kbID string) (*types.KnowledgeBase, error) { if kbID == "" { return nil, werrors.NewBadRequestError("知识库 ID 不能为空") } kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { return nil, err } kb.EnsureDefaults() if kb.Type != types.KnowledgeBaseTypeFAQ { return nil, werrors.NewBadRequestError("仅 FAQ 知识库支持该操作") } return kb, nil } func (s *knowledgeService) findFAQKnowledge(ctx context.Context, tenantID uint, kbID string) (*types.Knowledge, error) { knowledges, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID) if err != nil { return nil, err } for _, knowledge := range knowledges { if knowledge.Type == types.KnowledgeTypeFAQ { return knowledge, nil } } return nil, nil } func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint, kb *types.KnowledgeBase) (*types.Knowledge, error) { existing, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) if err != nil { return nil, err } if existing != nil { return existing, nil } knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kb.ID, Type: types.KnowledgeTypeFAQ, Title: fmt.Sprintf("%s - FAQ", kb.Name), Description: "FAQ 条目容器", Source: types.KnowledgeTypeFAQ, ParseStatus: "completed", EnableStatus: "enabled", EmbeddingModelID: kb.EmbeddingModelID, CreatedAt: time.Now(), UpdatedAt: time.Now(), } if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { return nil, err } return knowledge, nil } func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase) (*types.FAQEntry, error) { meta, err := chunk.FAQMetadata() if err != nil { return nil, err } if meta == nil { meta = &types.FAQChunkMetadata{StandardQuestion: chunk.Content} } entry := &types.FAQEntry{ ID: chunk.ID, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, TagID: chunk.TagID, IsEnabled: chunk.IsEnabled, StandardQuestion: meta.StandardQuestion, SimilarQuestions: meta.SimilarQuestions, NegativeQuestions: meta.NegativeQuestions, Answers: meta.Answers, IndexMode: kb.FAQConfig.IndexMode, UpdatedAt: chunk.UpdatedAt, CreatedAt: chunk.CreatedAt, ChunkType: chunk.ChunkType, } return entry, nil } func buildFAQChunkContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode) string { var builder strings.Builder builder.WriteString(fmt.Sprintf("Q: %s\n", meta.StandardQuestion)) if len(meta.SimilarQuestions) > 0 { builder.WriteString("Similar Questions:\n") for _, q := range meta.SimilarQuestions { builder.WriteString(fmt.Sprintf("- %s\n", q)) } } // 负例不应该包含在 Content 中,因为它们不应该被索引 // 答案根据索引模式决定是否包含 if mode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { builder.WriteString("Answers:\n") for _, ans := range meta.Answers { builder.WriteString(fmt.Sprintf("- %s\n", ans)) } } return builder.String() } func sanitizeFAQEntryPayload(payload *types.FAQEntryPayload) (*types.FAQChunkMetadata, error) { meta := &types.FAQChunkMetadata{ StandardQuestion: strings.TrimSpace(payload.StandardQuestion), SimilarQuestions: payload.SimilarQuestions, NegativeQuestions: payload.NegativeQuestions, Answers: payload.Answers, Version: 1, Source: "faq", } meta.Normalize() if meta.StandardQuestion == "" { return nil, werrors.NewBadRequestError("标准问不能为空") } if len(meta.Answers) == 0 { return nil, werrors.NewBadRequestError("至少提供一个答案") } return meta, nil } func buildFAQIndexContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode) string { var builder strings.Builder builder.WriteString(meta.StandardQuestion) for _, q := range meta.SimilarQuestions { builder.WriteString("\n") builder.WriteString(q) } if mode == types.FAQIndexModeQuestionAnswer { for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } } return builder.String() } // buildFAQIndexInfoList 构建FAQ索引信息列表,支持分别索引模式 func (s *knowledgeService) buildFAQIndexInfoList(ctx context.Context, kb *types.KnowledgeBase, chunk *types.Chunk) ([]*types.IndexInfo, error) { indexMode := types.FAQIndexModeQuestionAnswer questionIndexMode := types.FAQQuestionIndexModeCombined if kb.FAQConfig != nil { if kb.FAQConfig.IndexMode != "" { indexMode = kb.FAQConfig.IndexMode } if kb.FAQConfig.QuestionIndexMode != "" { questionIndexMode = kb.FAQConfig.QuestionIndexMode } } logger.Infof(ctx, "buildFAQIndexInfoList: indexMode: %s, questionIndexMode: %s", indexMode, questionIndexMode) meta, err := chunk.FAQMetadata() if err != nil { return nil, err } if meta == nil { meta = &types.FAQChunkMetadata{StandardQuestion: chunk.Content} } // 如果是一起索引模式,使用原有逻辑 if questionIndexMode == types.FAQQuestionIndexModeCombined { content := buildFAQIndexContent(meta, indexMode) return []*types.IndexInfo{ { Content: content, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }, }, nil } // 分别索引模式:为每个问题创建独立的索引项 indexInfoList := make([]*types.IndexInfo, 0) // 标准问索引项 standardContent := meta.StandardQuestion if indexMode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { var builder strings.Builder builder.WriteString(meta.StandardQuestion) for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } standardContent = builder.String() } indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: standardContent, SourceID: chunk.ID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) // 每个相似问创建一个索引项 for i, similarQ := range meta.SimilarQuestions { similarContent := similarQ if indexMode == types.FAQIndexModeQuestionAnswer && len(meta.Answers) > 0 { var builder strings.Builder builder.WriteString(similarQ) for _, ans := range meta.Answers { builder.WriteString("\n") builder.WriteString(ans) } similarContent = builder.String() } sourceID := fmt.Sprintf("%s-%d", chunk.ID, i) indexInfoList = append(indexInfoList, &types.IndexInfo{ Content: similarContent, SourceID: sourceID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, }) } logger.Infof(ctx, "buildFAQIndexInfoList: indexInfoList: %v", indexInfoList) return indexInfoList, nil } func (s *knowledgeService) indexFAQChunks(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*types.Chunk, adjustStorage bool, ) error { if len(chunks) == 0 { return nil } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { return err } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } indexInfo := make([]*types.IndexInfo, 0) chunkIDs := make([]string, 0, len(chunks)) for _, chunk := range chunks { infoList, err := s.buildFAQIndexInfoList(ctx, kb, chunk) if err != nil { return err } indexInfo = append(indexInfo, infoList...) chunkIDs = append(chunkIDs, chunk.ID) } var size int64 if adjustStorage { size = retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo) if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed+size > tenantInfo.StorageQuota { return types.NewStorageQuotaExceededError() } } if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err) } if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil { return err } if adjustStorage && size > 0 { if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, size); err == nil { tenantInfo.StorageUsed += size } knowledge.StorageSize += size } now := time.Now() knowledge.UpdatedAt = now knowledge.ProcessedAt = &now return s.repo.UpdateKnowledge(ctx, knowledge) } func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*types.Chunk, ) error { if len(chunks) == 0 { return nil } embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { return err } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { return err } indexInfo := make([]*types.IndexInfo, 0) chunkIDs := make([]string, 0, len(chunks)) for _, chunk := range chunks { infoList, err := s.buildFAQIndexInfoList(ctx, kb, chunk) if err != nil { return err } indexInfo = append(indexInfo, infoList...) chunkIDs = append(chunkIDs, chunk.ID) } size := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo) if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { return err } if size > 0 { if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -size); err == nil { tenantInfo.StorageUsed -= size if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } } if knowledge.StorageSize >= size { knowledge.StorageSize -= size } else { knowledge.StorageSize = 0 } } knowledge.UpdatedAt = time.Now() return s.repo.UpdateKnowledge(ctx, knowledge) } func (s *knowledgeService) cleanupFAQKnowledge(ctx context.Context, knowledge *types.Knowledge) error { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err == nil { if embeddingModel, modelErr := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID); modelErr == nil { _ = retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()) } } } if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { return err } if knowledge.StorageSize > 0 { if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err == nil { tenantInfo.StorageUsed -= knowledge.StorageSize if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } } knowledge.StorageSize = 0 } knowledge.UpdatedAt = time.Now() knowledge.ProcessedAt = nil return s.repo.UpdateKnowledge(ctx, knowledge) } func ensureManualFileName(title string) string { if title == "" { return fmt.Sprintf("manual-%s%s", time.Now().Format("20060102-150405"), manualFileExtension) } trimmed := strings.TrimSpace(title) if strings.HasSuffix(strings.ToLower(trimmed), manualFileExtension) { return trimmed } return trimmed + manualFileExtension } func (s *knowledgeService) triggerManualProcessing(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, content string, sync bool, ) { clean := strings.TrimSpace(content) if clean == "" { return } // 使用 docreader 按照 MD 格式处理,并使用知识库配置的分隔符 contentBytes := []byte(clean) fileName := ensureManualFileName(knowledge.Title) fileType := "md" // 检查是否需要启用多模态(对于手动内容通常不需要,但保持一致性) enableMultimodel := kb.ChunkingConfig.EnableMultimodal && kb.StorageConfig.Provider != "" logger.GetLogger(ctx).Infof("triggerManualProcessing split manual content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v", len(contentBytes), fileName, fileType, kb.ChunkingConfig.Separators, enableMultimodel) // 调用 docreader 解析 markdown 内容 resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ FileContent: contentBytes, FileName: fileName, FileType: fileType, ReadConfig: &proto.ReadConfig{ ChunkSize: int32(kb.ChunkingConfig.ChunkSize), ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), Separators: kb.ChunkingConfig.Separators, EnableMultimodal: enableMultimodel, StorageConfig: &proto.StorageConfig{ Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), Region: kb.StorageConfig.Region, BucketName: kb.StorageConfig.BucketName, AccessKeyId: kb.StorageConfig.SecretID, SecretAccessKey: kb.StorageConfig.SecretKey, AppId: kb.StorageConfig.AppID, PathPrefix: kb.StorageConfig.PathPrefix, }, VlmConfig: &proto.VLMConfig{ ModelName: kb.VLMConfig.ModelName, BaseUrl: kb.VLMConfig.BaseURL, ApiKey: kb.VLMConfig.APIKey, InterfaceType: kb.VLMConfig.InterfaceType, }, }, RequestId: ctx.Value(types.RequestIDContextKey).(string), }) if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("triggerManualProcessing read file failed") knowledge.ParseStatus = "failed" knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) return } if sync { s.processChunks(ctx, kb, knowledge, resp.Chunks) return } newCtx := logger.CloneContext(ctx) go s.processChunks(newCtx, kb, knowledge, resp.Chunks) } func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowledge *types.Knowledge) error { logger.GetLogger(ctx).Infof("Cleaning knowledge resources before manual update, knowledge ID: %s", knowledge.ID) var cleanupErr error if knowledge.ParseStatus == types.ManualKnowledgeStatusDraft && knowledge.StorageSize == 0 { // Draft without indexed data, skip cleanup. return nil } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) if err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup") cleanupErr = errors.Join(cleanupErr, err) } else { embeddingModel, modelErr := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) if modelErr != nil { logger.GetLogger(ctx).WithField("error", modelErr).Error("Failed to get embedding model during cleanup") cleanupErr = errors.Join(cleanupErr, modelErr) } else { if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge index") cleanupErr = errors.Join(cleanupErr, err) } } } } if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge chunks") cleanupErr = errors.Join(cleanupErr, err) } namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge graph data") cleanupErr = errors.Join(cleanupErr, err) } if knowledge.StorageSize > 0 { tenantInfo.StorageUsed -= knowledge.StorageSize if tenantInfo.StorageUsed < 0 { tenantInfo.StorageUsed = 0 } if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to adjust storage usage during manual cleanup") cleanupErr = errors.Join(cleanupErr, err) } knowledge.StorageSize = 0 } return cleanupErr } func IsImageType(fileType string) bool { switch fileType { case "jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "tiff": return true default: return false } }