package service import ( "context" "errors" "fmt" "io" "strings" "sync" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/config" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/redis/go-redis/v9" ) // Error definitions for knowledge service operations var ( // ErrInvalidFileType is returned when an unsupported file type is provided ErrInvalidFileType = errors.New("unsupported file type") // ErrInvalidURL is returned when an invalid URL is provided ErrInvalidURL = errors.New("invalid URL") // ErrChunkNotFound is returned when a requested chunk cannot be found ErrChunkNotFound = errors.New("chunk not found") // ErrDuplicateFile is returned when trying to add a file that already exists ErrDuplicateFile = errors.New("file already exists") // ErrDuplicateURL is returned when trying to add a URL that already exists ErrDuplicateURL = errors.New("URL already exists") // ErrImageNotParse is returned when trying to update image information without enabling multimodel ErrImageNotParse = errors.New("image not parse without enable multimodel") ) // knowledgeService implements the knowledge service interface // service 实现知识服务接口 type knowledgeService struct { config *config.Config retrieveEngine interfaces.RetrieveEngineRegistry ownership retriever.TenantStoreOwnership repo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService tenantRepo interfaces.TenantRepository tenantService interfaces.TenantService documentReader interfaces.DocumentReader chunkService interfaces.ChunkService chunkRepo interfaces.ChunkRepository tagRepo interfaces.KnowledgeTagRepository tagService interfaces.KnowledgeTagService fileSvc interfaces.FileService modelService interfaces.ModelService task interfaces.TaskEnqueuer graphEngine interfaces.RetrieveGraphRepository redisClient *redis.Client kbShareService interfaces.KBShareService imageResolver *docparser.ImageResolver taskPendingRepo interfaces.TaskPendingOpsRepository // In-memory fallbacks for Lite mode (no Redis) memFAQProgress sync.Map // taskID -> *types.FAQImportProgress memFAQRunningImport sync.Map // kbID -> *runningFAQImportInfo wikiRepo interfaces.WikiPageRepository wikiService interfaces.WikiPageService } const ( manualContentMaxLength = 200000 manualFileExtension = ".md" faqImportBatchSize = 50 // 每批处理的FAQ条目数 ) // NewKnowledgeService creates a new knowledge service instance func NewKnowledgeService( config *config.Config, repo interfaces.KnowledgeRepository, documentReader interfaces.DocumentReader, kbService interfaces.KnowledgeBaseService, tenantRepo interfaces.TenantRepository, tenantService interfaces.TenantService, chunkService interfaces.ChunkService, chunkRepo interfaces.ChunkRepository, tagRepo interfaces.KnowledgeTagRepository, tagService interfaces.KnowledgeTagService, fileSvc interfaces.FileService, modelService interfaces.ModelService, task interfaces.TaskEnqueuer, graphEngine interfaces.RetrieveGraphRepository, retrieveEngine interfaces.RetrieveEngineRegistry, ownership retriever.TenantStoreOwnership, redisClient *redis.Client, kbShareService interfaces.KBShareService, imageResolver *docparser.ImageResolver, wikiRepo interfaces.WikiPageRepository, wikiService interfaces.WikiPageService, taskPendingRepo interfaces.TaskPendingOpsRepository, ) (interfaces.KnowledgeService, error) { return &knowledgeService{ config: config, repo: repo, kbService: kbService, tenantRepo: tenantRepo, tenantService: tenantService, documentReader: documentReader, chunkService: chunkService, chunkRepo: chunkRepo, tagRepo: tagRepo, tagService: tagService, fileSvc: fileSvc, modelService: modelService, task: task, graphEngine: graphEngine, retrieveEngine: retrieveEngine, ownership: ownership, redisClient: redisClient, kbShareService: kbShareService, imageResolver: imageResolver, wikiRepo: wikiRepo, wikiService: wikiService, taskPendingRepo: taskPendingRepo, }, nil } // getParserEngineOverridesFromContext returns parser engine overrides from tenant in context (e.g. MinerU endpoint, API key). // Used when building document ReadRequest so UI-configured values take precedence over env. func (s *knowledgeService) getParserEngineOverridesFromContext(ctx context.Context) map[string]string { if v := ctx.Value(types.TenantInfoContextKey); v != nil { if tenant, ok := v.(*types.Tenant); ok && tenant != nil { return tenant.ParserEngineConfig.ToOverridesMap() } } return nil } // GetRepository gets the knowledge repository // Parameters: // - ctx: Context with authentication and request information // // Returns: // - interfaces.KnowledgeRepository: Knowledge repository func (s *knowledgeService) GetRepository() interfaces.KnowledgeRepository { return s.repo } // isKnowledgeDeleting checks if a knowledge entry is being deleted. // This is used to prevent async tasks from conflicting with deletion operations. func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uint64, knowledgeID string) bool { knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { // If we can't find the knowledge, assume it's deleted logger.Warnf(ctx, "Failed to check knowledge deletion status (assuming deleted): %v", err) return true } if knowledge == nil { return true } return knowledge.ParseStatus == types.ParseStatusDeleting } // checkStorageEngineConfigured verifies that the knowledge base has a storage engine configured // (either at the KB level or via the tenant default). Returns an error if no storage engine is found. func checkStorageEngineConfigured(ctx context.Context, kb *types.KnowledgeBase) error { provider := kb.GetStorageProvider() if provider == "" { tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if tenant != nil && tenant.StorageEngineConfig != nil { provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider)) } } if provider == "" { return werrors.NewBadRequestError("请先为知识库选择存储引擎,再上传内容。请前往知识库设置页面进行配置。") } return nil } func defaultChannel(ch string) string { if ch == "" { return types.ChannelWeb } return ch } // GetKnowledgeByID retrieves a knowledge entry by its ID func (s *knowledgeService) GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "knowledge_id": id, "tenant_id": tenantID, }) return nil, err } logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, type: %s", knowledge.ID, knowledge.Type) return knowledge, nil } // GetKnowledgeByIDOnly retrieves knowledge by ID without tenant filter (for permission resolution). func (s *knowledgeService) GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) { return s.repo.GetKnowledgeByIDOnly(ctx, id) } // ListKnowledgeByKnowledgeBaseID returns all knowledge entries in a knowledge base func (s *knowledgeService) ListKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, ) ([]*types.Knowledge, error) { return s.repo.ListKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), kbID) } // ListPagedKnowledgeByKnowledgeBaseID returns paginated knowledge entries in a knowledge base func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string, page *types.Pagination, filter types.KnowledgeListFilter, ) (*types.PageResult, error) { knowledges, total, err := s.repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), kbID, page, filter) if err != nil { return nil, err } return types.NewPageResult(total, page, knowledges), nil } // GetKnowledgeFile retrieves the physical file associated with a knowledge entry func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { // Get knowledge record tenantID := ctx.Value(types.TenantIDContextKey).(uint64) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) if err != nil { return nil, "", err } // Manual knowledge stores content in Metadata — stream it directly as a .md file. if knowledge.IsManual() { meta, err := knowledge.ManualMetadata() if err != nil { return nil, "", err } // ManualMetadata returns (nil, nil) when Metadata column is empty; treat as empty content. content := "" if meta != nil { content = meta.Content } filename := sanitizeManualDownloadFilename(knowledge.Title) return io.NopCloser(strings.NewReader(content)), filename, nil } // Resolve KB-level file service with FilePath fallback protection kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) file, err := s.resolveFileServiceForPath(ctx, kb, knowledge.FilePath).GetFile(ctx, knowledge.FilePath) if err != nil { return nil, "", err } return file, knowledge.FileName, nil } func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { record, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge record: %v", err) return err } // if need other fields update, please add here if knowledge.Title != "" { record.Title = knowledge.Title } if knowledge.Description != "" { record.Description = knowledge.Description } // Update knowledge record in the repository if err := s.repo.UpdateKnowledge(ctx, record); err != nil { logger.Errorf(ctx, "Failed to update knowledge: %v", err) return err } logger.Infof(ctx, "Knowledge updated successfully, ID: %s", knowledge.ID) return nil } // GetKnowledgeBatch retrieves multiple knowledge entries by their IDs func (s *knowledgeService) GetKnowledgeBatch(ctx context.Context, tenantID uint64, ids []string, ) ([]*types.Knowledge, error) { if len(ids) == 0 { return nil, nil } return s.repo.GetKnowledgeBatch(ctx, tenantID, ids) } // GetKnowledgeBatchWithSharedAccess retrieves knowledge by IDs, including items from shared KBs the user has access to. // Used when building search targets so that @mentioned files from shared KBs are included. func (s *knowledgeService) GetKnowledgeBatchWithSharedAccess(ctx context.Context, tenantID uint64, ids []string, ) ([]*types.Knowledge, error) { if len(ids) == 0 { return nil, nil } ownList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, ids) if err != nil { return nil, err } foundSet := make(map[string]bool) for _, k := range ownList { if k != nil { foundSet[k.ID] = true } } userIDVal := ctx.Value(types.UserIDContextKey) if userIDVal == nil { return ownList, nil } userID, ok := userIDVal.(string) if !ok || userID == "" { return ownList, nil } for _, id := range ids { if foundSet[id] { continue } k, err := s.repo.GetKnowledgeByIDOnly(ctx, id) if err != nil || k == nil || k.KnowledgeBaseID == "" { continue } hasPermission, err := s.kbShareService.HasKBPermission(ctx, k.KnowledgeBaseID, userID, types.OrgRoleViewer) if err != nil || !hasPermission { continue } foundSet[k.ID] = true ownList = append(ownList, k) } return ownList, nil } // UpdateKnowledgeTag updates the tag assigned to a knowledge document. func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID string, tagID *string) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { return err } var resolvedTagID string if tagID != nil && *tagID != "" { tag, err := s.tagRepo.GetByID(ctx, tenantID, *tagID) if err != nil { return err } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError("标签不属于当前知识库") } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID return s.repo.UpdateKnowledge(ctx, knowledge) } // UpdateKnowledgeTagBatch updates tags for document knowledge items in batch. // authorizedKBID restricts all updates to knowledge items belonging to this KB; // pass empty string to skip the check (caller must ensure authorization by other means). func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, authorizedKBID string, updates map[string]*string) error { if len(updates) == 0 { return nil } tenantIDVal := ctx.Value(types.TenantIDContextKey) if tenantIDVal == nil { return werrors.NewUnauthorizedError("tenant ID not found in context") } tenantID, ok := tenantIDVal.(uint64) if !ok { return werrors.NewUnauthorizedError("invalid tenant ID in context") } // Get all knowledge items in batch knowledgeIDs := make([]string, 0, len(updates)) for knowledgeID := range updates { knowledgeIDs = append(knowledgeIDs, knowledgeID) } knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs) if err != nil { return err } // Validate all requested IDs were found and belong to the authorized KB if authorizedKBID != "" { if len(knowledgeList) != len(updates) { return werrors.NewForbiddenError("some knowledge IDs are not accessible in the authorized scope") } for _, k := range knowledgeList { if k.KnowledgeBaseID != authorizedKBID { return werrors.NewForbiddenError( fmt.Sprintf("knowledge %s does not belong to authorized knowledge base", k.ID)) } } } // Build tag ID map for validation tagIDSet := make(map[string]bool) for _, tagID := range updates { if tagID != nil && *tagID != "" { tagIDSet[*tagID] = true } } // Validate all tags in batch tagMap := make(map[string]*types.KnowledgeTag) if len(tagIDSet) > 0 { tagIDs := make([]string, 0, len(tagIDSet)) for tagID := range tagIDSet { tagIDs = append(tagIDs, tagID) } for _, tagID := range tagIDs { tag, err := s.tagRepo.GetByID(ctx, tenantID, tagID) if err != nil { return err } tagMap[tagID] = tag } } // Update knowledge items knowledgeToUpdate := make([]*types.Knowledge, 0) for _, knowledge := range knowledgeList { tagID, exists := updates[knowledge.ID] if !exists { continue } var resolvedTagID string if tagID != nil && *tagID != "" { tag, ok := tagMap[*tagID] if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", *tagID)) } if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于知识库 %s", *tagID, knowledge.KnowledgeBaseID)) } resolvedTagID = tag.ID } knowledge.TagID = resolvedTagID knowledgeToUpdate = append(knowledgeToUpdate, knowledge) } if len(knowledgeToUpdate) > 0 { return s.repo.UpdateKnowledgeBatch(ctx, knowledgeToUpdate) } return nil } // SearchKnowledge searches knowledge items by keyword across the tenant and shared knowledge bases. // fileTypes: optional list of file extensions to filter by (e.g., ["csv", "xlsx"]) func (s *knowledgeService) SearchKnowledge(ctx context.Context, keyword string, offset, limit int, fileTypes []string) ([]*types.Knowledge, bool, error) { tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) if !ok { return nil, false, werrors.NewUnauthorizedError("Tenant ID not found in context") } scopes := make([]types.KnowledgeSearchScope, 0) // Own tenant: document-type knowledge bases ownKBs, err := s.kbService.ListKnowledgeBases(ctx) if err == nil { for _, kb := range ownKBs { if kb != nil && kb.Type == types.KnowledgeBaseTypeDocument { scopes = append(scopes, types.KnowledgeSearchScope{TenantID: tenantID, KBID: kb.ID}) } } } // Shared knowledge bases (document type only) if userIDVal := ctx.Value(types.UserIDContextKey); userIDVal != nil { if userID, ok := userIDVal.(string); ok && userID != "" { sharedList, err := s.kbShareService.ListSharedKnowledgeBases(ctx, userID, tenantID) if err == nil { for _, info := range sharedList { if info != nil && info.KnowledgeBase != nil && info.KnowledgeBase.Type == types.KnowledgeBaseTypeDocument { scopes = append(scopes, types.KnowledgeSearchScope{ TenantID: info.SourceTenantID, KBID: info.KnowledgeBase.ID, }) } } } } } if len(scopes) == 0 { return nil, false, nil } return s.repo.SearchKnowledgeInScopes(ctx, scopes, keyword, offset, limit, fileTypes) } // SearchKnowledgeForScopes searches knowledge within the given scopes (e.g. for shared agent context). func (s *knowledgeService) SearchKnowledgeForScopes(ctx context.Context, scopes []types.KnowledgeSearchScope, keyword string, offset, limit int, fileTypes []string) ([]*types.Knowledge, bool, error) { if len(scopes) == 0 { return nil, false, nil } return s.repo.SearchKnowledgeInScopes(ctx, scopes, keyword, offset, limit, fileTypes) }