diff --git a/client/knowledgebase.go b/client/knowledgebase.go index 48808ca1..10983fc6 100644 --- a/client/knowledgebase.go +++ b/client/knowledgebase.go @@ -140,6 +140,29 @@ type CopyKnowledgeBaseRequest struct { TargetID string `json:"target_id"` } +// CopyKnowledgeBaseResponse represents the response from copy knowledge base API +type CopyKnowledgeBaseResponse struct { + TaskID string `json:"task_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` + Message string `json:"message"` +} + +// KBCloneProgress represents the progress of a knowledge base clone task +type KBCloneProgress struct { + TaskID string `json:"task_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` + Status string `json:"status"` // pending, processing, completed, failed + Progress int `json:"progress"` // 0-100 + Total int `json:"total"` // Total operations count + Processed int `json:"processed"` // Processed operations count + Message string `json:"message"` + Error string `json:"error,omitempty"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + // CreateKnowledgeBase creates a knowledge base func (c *Client) CreateKnowledgeBase(ctx context.Context, knowledgeBase *KnowledgeBase) (*KnowledgeBase, error) { resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-bases", knowledgeBase, nil) @@ -257,18 +280,44 @@ func (c *Client) HybridSearch(ctx context.Context, knowledgeBaseID string, param return response.Data, nil } -func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) error { +// CopyKnowledgeBase copies a knowledge base asynchronously and returns task info +func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) (*CopyKnowledgeBaseResponse, error) { path := "/api/v1/knowledge-bases/copy" resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil) if err != nil { - return err + return nil, err } var response struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` + Success bool `json:"success"` + Data CopyKnowledgeBaseResponse `json:"data"` } - return parseResponse(resp, &response) + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetKBCloneProgress gets the progress of a knowledge base clone task +func (c *Client) GetKBCloneProgress(ctx context.Context, taskID string) (*KBCloneProgress, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/copy/progress/%s", taskID) + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response struct { + Success bool `json:"success"` + Data KBCloneProgress `json:"data"` + } + + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil } diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index a6100c8e..03fa97cc 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -586,3 +586,43 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( return affectedIDs, nil } + +// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. +// Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst), +// +// chunksToDelete (IDs of chunks in dst whose content_hash is not in src) +func (r *chunkRepository) FAQChunkDiff( + ctx context.Context, + srcTenantID uint64, srcKBID string, + dstTenantID uint64, dstKBID string, +) (chunksToAdd []string, chunksToDelete []string, err error) { + // Get content_hash set from destination KB + dstHashSubQuery := r.db.Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). + Select("content_hash") + + // Find chunks in source that don't exist in destination (by content_hash) + err = r.db.WithContext(ctx).Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). + Where("content_hash NOT IN (?)", dstHashSubQuery). + Pluck("id", &chunksToAdd).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err) + } + + // Get content_hash set from source KB + srcHashSubQuery := r.db.Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). + Select("content_hash") + + // Find chunks in destination that don't exist in source (by content_hash) + err = r.db.WithContext(ctx).Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). + Where("content_hash NOT IN (?)", srcHashSubQuery). + Pluck("id", &chunksToDelete).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err) + } + + return chunksToAdd, chunksToDelete, nil +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index f34b8a16..5b21a751 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -30,6 +30,7 @@ import ( secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" ) @@ -67,6 +68,7 @@ type knowledgeService struct { modelService interfaces.ModelService task *asynq.Client graphEngine interfaces.RetrieveGraphRepository + redisClient *redis.Client } const ( @@ -91,6 +93,7 @@ func NewKnowledgeService( task *asynq.Client, graphEngine interfaces.RetrieveGraphRepository, retrieveEngine interfaces.RetrieveEngineRegistry, + redisClient *redis.Client, ) (interfaces.KnowledgeService, error) { return &knowledgeService{ config: config, @@ -107,6 +110,7 @@ func NewKnowledgeService( task: task, graphEngine: graphEngine, retrieveEngine: retrieveEngine, + redisClient: redisClient, }, nil } @@ -5174,3 +5178,430 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) return nil } + +const ( + kbCloneProgressKeyPrefix = "kb_clone_progress:" + kbCloneProgressTTL = 24 * time.Hour +) + +// getKBCloneProgressKey returns the Redis key for storing KB clone progress +func getKBCloneProgressKey(taskID string) string { + return kbCloneProgressKeyPrefix + taskID +} + +// ProcessKBClone handles Asynq knowledge base clone tasks +func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error { + var payload types.KBClonePayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + return fmt.Errorf("failed to unmarshal KB clone payload: %w", err) + } + + // Add tenant ID to context + ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + + // Check if this is the last retry + retryCount, _ := asynq.GetRetryCount(ctx) + maxRetry, _ := asynq.GetMaxRetry(ctx) + isLastRetry := retryCount >= maxRetry + + logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d", + payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry) + + // Helper function to handle errors - only mark as failed on last retry + handleError := func(progress *types.KBCloneProgress, err error, message string) { + if isLastRetry { + progress.Status = types.KBCloneStatusFailed + progress.Error = err.Error() + progress.Message = message + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + } + } + + // Update progress to processing + progress := &types.KBCloneProgress{ + TaskID: payload.TaskID, + SourceID: payload.SourceID, + TargetID: payload.TargetID, + Status: types.KBCloneStatusProcessing, + Progress: 0, + Message: "Starting knowledge base clone...", + UpdatedAt: time.Now().Unix(), + } + if err := s.saveKBCloneProgress(ctx, progress); err != nil { + logger.Errorf(ctx, "Failed to update KB clone progress: %v", err) + } + + // Get source and target knowledge bases + srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID) + if err != nil { + logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) + handleError(progress, err, "Failed to copy knowledge base configuration") + return err + } + + // Use different sync strategies based on knowledge base type + if srcKB.Type == types.KnowledgeBaseTypeFAQ { + return s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError) + } + + // Document type: use Knowledge-level diff based on file_hash + addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge to add: %v", err) + handleError(progress, err, "Failed to calculate knowledge difference") + return err + } + + delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err) + handleError(progress, err, "Failed to calculate knowledge difference") + return err + } + + totalOperations := len(addKnowledge) + len(delKnowledge) + progress.Total = totalOperations + progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + + logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) + + processedCount := 0 + batch := 10 + + // Delete knowledge in target that doesn't exist in source + g, gctx := errgroup.WithContext(ctx) + for ids := range slices.Chunk(delKnowledge, batch) { + g.Go(func() error { + err := s.DeleteKnowledgeList(gctx, ids) + if err != nil { + logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err) + return err + } + return nil + }) + } + if err := g.Wait(); err != nil { + logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) + handleError(progress, err, "Failed to delete knowledge") + return err + } + + processedCount += len(delKnowledge) + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + + // Clone knowledge from source to target + g, gctx = errgroup.WithContext(ctx) + g.SetLimit(batch) + for _, knowledge := range addKnowledge { + g.Go(func() error { + srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) + if err != nil { + logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err) + return err + } + err = s.cloneKnowledge(gctx, srcKn, dstKB) + if err != nil { + logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err) + return err + } + + // Update progress + processedCount++ + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + + return nil + }) + } + if err := g.Wait(); err != nil { + logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) + handleError(progress, err, "Failed to clone knowledge") + return err + } + + // Mark as completed + progress.Status = types.KBCloneStatusCompleted + progress.Progress = 100 + progress.Processed = totalOperations + progress.Message = "Knowledge base clone completed successfully" + progress.UpdatedAt = time.Now().Unix() + if err := s.saveKBCloneProgress(ctx, progress); err != nil { + logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err) + } + + logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID) + return nil +} + +// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync +func (s *knowledgeService) cloneFAQKnowledgeBase( + ctx context.Context, + srcKB, dstKB *types.KnowledgeBase, + progress *types.KBCloneProgress, + handleError func(*types.KBCloneProgress, error, string), +) error { + // Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry) + srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err) + handleError(progress, err, "Failed to get source FAQ knowledge") + return err + } + if len(srcKnowledgeList) == 0 { + // Source has no FAQ knowledge, nothing to clone + progress.Status = types.KBCloneStatusCompleted + progress.Progress = 100 + progress.Message = "Source FAQ knowledge base is empty" + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + return nil + } + srcKnowledge := srcKnowledgeList[0] + + // Get chunk-level differences based on content_hash + chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err) + handleError(progress, err, "Failed to calculate FAQ chunk difference") + return err + } + + totalOperations := len(chunksToAdd) + len(chunksToDelete) + progress.Total = totalOperations + progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + + logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete)) + + // If nothing to do, mark as completed + if totalOperations == 0 { + progress.Status = types.KBCloneStatusCompleted + progress.Progress = 100 + progress.Message = "FAQ knowledge base is already in sync" + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + return nil + } + + // Get tenant info and initialize retrieve engine + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) + if err != nil { + logger.Errorf(ctx, "Failed to init retrieve engine: %v", err) + handleError(progress, err, "Failed to initialize retrieve engine") + return err + } + + // Get embedding model + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get embedding model: %v", err) + handleError(progress, err, "Failed to get embedding model") + return err + } + + processedCount := 0 + + // Delete FAQ chunks that don't exist in source + if len(chunksToDelete) > 0 { + // Delete from vector store + if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil { + logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err) + handleError(progress, err, "Failed to delete FAQ entries from vector store") + return err + } + // Delete from database + if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil { + logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err) + handleError(progress, err, "Failed to delete FAQ entries from database") + return err + } + processedCount += len(chunksToDelete) + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + } + + // Get or create the FAQ knowledge entry in destination + dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge) + if err != nil { + logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err) + handleError(progress, err, "Failed to prepare FAQ knowledge entry") + return err + } + + // Clone FAQ chunks from source to destination + batch := 50 + for i := 0; i < len(chunksToAdd); i += batch { + end := i + batch + if end > len(chunksToAdd) { + end = len(chunksToAdd) + } + batchIDs := chunksToAdd[i:end] + + // Get source chunks + srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs) + if err != nil { + logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err) + handleError(progress, err, "Failed to get source FAQ entries") + return err + } + + // Create new chunks for destination + newChunks := make([]*types.Chunk, 0, len(srcChunks)) + for _, srcChunk := range srcChunks { + newChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: dstKB.TenantID, + KnowledgeID: dstKnowledge.ID, + KnowledgeBaseID: dstKB.ID, + TagID: srcChunk.TagID, + Content: srcChunk.Content, + ChunkIndex: srcChunk.ChunkIndex, + IsEnabled: srcChunk.IsEnabled, + Flags: srcChunk.Flags, + ChunkType: types.ChunkTypeFAQ, + Metadata: srcChunk.Metadata, + ContentHash: srcChunk.ContentHash, + Status: int(types.ChunkStatusStored), // Initially stored, will be indexed + } + newChunks = append(newChunks, newChunk) + } + + // Save to database + if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil { + logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err) + handleError(progress, err, "Failed to create FAQ entries") + return err + } + + // Index in vector store using existing method + // This will index standard question + similar questions based on FAQConfig + if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil { + logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err) + handleError(progress, err, "Failed to index FAQ entries") + return err + } + + // Update chunk status to indexed + for _, chunk := range newChunks { + chunk.Status = int(types.ChunkStatusIndexed) + } + if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil { + logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err) + // Don't fail the whole operation for status update failure + } + + processedCount += len(batchIDs) + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + } + + // Mark as completed + progress.Status = types.KBCloneStatusCompleted + progress.Progress = 100 + progress.Processed = totalOperations + progress.Message = "FAQ knowledge base clone completed successfully" + progress.UpdatedAt = time.Now().Unix() + if err := s.saveKBCloneProgress(ctx, progress); err != nil { + logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err) + } + + return nil +} + +// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base +// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge +func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) { + // FAQ knowledge base should have exactly one Knowledge entry + knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID) + if err != nil { + return nil, err + } + + if len(knowledgeList) > 0 { + return knowledgeList[0], nil + } + + // Create a new FAQ knowledge entry, copying from source if available + knowledge := &types.Knowledge{ + ID: uuid.New().String(), + TenantID: kb.TenantID, + KnowledgeBaseID: kb.ID, + Type: types.KnowledgeTypeFAQ, + Title: "FAQ", + ParseStatus: "completed", + EnableStatus: "enabled", + EmbeddingModelID: kb.EmbeddingModelID, + } + + // Copy additional fields from source knowledge if available + if srcKnowledge != nil { + knowledge.Title = srcKnowledge.Title + knowledge.Description = srcKnowledge.Description + knowledge.Source = srcKnowledge.Source + knowledge.Metadata = srcKnowledge.Metadata + } + + if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { + return nil, err + } + return knowledge, nil +} + +// saveKBCloneProgress saves the KB clone progress to Redis +func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error { + key := getKBCloneProgressKey(progress.TaskID) + data, err := json.Marshal(progress) + if err != nil { + return fmt.Errorf("failed to marshal progress: %w", err) + } + return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err() +} + +// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use) +func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error { + return s.saveKBCloneProgress(ctx, progress) +} + +// GetKBCloneProgress retrieves the progress of a knowledge base clone task +func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) { + key := getKBCloneProgressKey(taskID) + data, err := s.redisClient.Get(ctx, key).Bytes() + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, werrors.NewNotFoundError("KB clone task not found") + } + return nil, fmt.Errorf("failed to get progress from Redis: %w", err) + } + + var progress types.KBCloneProgress + if err := json.Unmarshal(data, &progress); err != nil { + return nil, fmt.Errorf("failed to unmarshal progress: %w", err) + } + return &progress, nil +} diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.go index 5c4cc705..d1926540 100644 --- a/internal/handler/knowledgebase.go +++ b/internal/handler/knowledgebase.go @@ -2,8 +2,10 @@ package handler import ( "context" + "encoding/json" "net/http" "strconv" + "time" "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" @@ -11,20 +13,28 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/hibiken/asynq" ) // KnowledgeBaseHandler defines the HTTP handler for knowledge base operations type KnowledgeBaseHandler struct { service interfaces.KnowledgeBaseService knowledgeService interfaces.KnowledgeService + asynqClient *asynq.Client } // NewKnowledgeBaseHandler creates a new knowledge base handler instance func NewKnowledgeBaseHandler( service interfaces.KnowledgeBaseService, knowledgeService interfaces.KnowledgeService, + asynqClient *asynq.Client, ) *KnowledgeBaseHandler { - return &KnowledgeBaseHandler{service: service, knowledgeService: knowledgeService} + return &KnowledgeBaseHandler{ + service: service, + knowledgeService: knowledgeService, + asynqClient: asynqClient, + } } // HybridSearch godoc @@ -319,15 +329,23 @@ type CopyKnowledgeBaseRequest struct { TargetID string `json:"target_id"` } +// CopyKnowledgeBaseResponse defines the response for copy knowledge base +type CopyKnowledgeBaseResponse struct { + TaskID string `json:"task_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` + Message string `json:"message"` +} + // CopyKnowledgeBase godoc // @Summary 复制知识库 -// @Description 将一个知识库的内容复制到另一个知识库 +// @Description 将一个知识库的内容复制到另一个知识库(异步任务) // @Tags 知识库 // @Accept json // @Produce json -// @Param request body CopyKnowledgeBaseRequest true "复制请求" -// @Success 200 {object} map[string]interface{} "复制成功" -// @Failure 400 {object} errors.AppError "请求参数错误" +// @Param request body CopyKnowledgeBaseRequest true "复制请求" +// @Success 200 {object} map[string]interface{} "任务ID" +// @Failure 400 {object} errors.AppError "请求参数错误" // @Security Bearer // @Router /knowledge-bases/copy [post] func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { @@ -339,20 +357,102 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { return } - go func(ctx context.Context) { - err := h.knowledgeService.CloneKnowledgeBase(ctx, req.SourceID, req.TargetID) - if err != nil { - logger.Errorf(ctx, "Failed to copy knowledge base, ID: %s to ID: %s", - secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID)) - return - } - logger.Infof(ctx, "Knowledge base copy from ID: %s to ID: %s successfully", - secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID)) - }(logger.CloneContext(ctx)) + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + // Generate task ID + taskID := uuid.New().String() + + // Create KB clone payload + payload := types.KBClonePayload{ + TenantID: tenantID.(uint64), + TaskID: taskID, + SourceID: req.SourceID, + TargetID: req.TargetID, + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + logger.Errorf(ctx, "Failed to marshal KB clone payload: %v", err) + c.Error(errors.NewInternalServerError("Failed to create task")) + return + } + + // Enqueue KB clone task to Asynq + task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3)) + info, err := h.asynqClient.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err) + c.Error(errors.NewInternalServerError("Failed to enqueue task")) + return + } + + logger.Infof(ctx, "KB clone task enqueued: %s, asynq task ID: %s, source: %s, target: %s", + taskID, info.ID, secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID)) + + // Save initial progress to Redis so frontend can query immediately + initialProgress := &types.KBCloneProgress{ + TaskID: taskID, + SourceID: req.SourceID, + TargetID: req.TargetID, + Status: types.KBCloneStatusPending, + Progress: 0, + Message: "Task queued, waiting to start...", + CreatedAt: time.Now().Unix(), + UpdatedAt: time.Now().Unix(), + } + if err := h.knowledgeService.SaveKBCloneProgress(ctx, initialProgress); err != nil { + logger.Warnf(ctx, "Failed to save initial KB clone progress: %v", err) + // Don't fail the request, task is already enqueued + } c.JSON(http.StatusOK, gin.H{ "success": true, - "message": "Knowledge base copy successfully", + "data": CopyKnowledgeBaseResponse{ + TaskID: taskID, + SourceID: req.SourceID, + TargetID: req.TargetID, + Message: "Knowledge base copy task started", + }, + }) +} + +// GetKBCloneProgress godoc +// @Summary 获取知识库复制进度 +// @Description 获取知识库复制任务的进度 +// @Tags 知识库 +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} map[string]interface{} "进度信息" +// @Failure 404 {object} errors.AppError "任务不存在" +// @Security Bearer +// @Router /knowledge-bases/copy/progress/{task_id} [get] +func (h *KnowledgeBaseHandler) GetKBCloneProgress(c *gin.Context) { + ctx := c.Request.Context() + + taskID := c.Param("task_id") + if taskID == "" { + logger.Error(ctx, "Task ID is empty") + c.Error(errors.NewBadRequestError("Task ID cannot be empty")) + return + } + + progress, err := h.knowledgeService.GetKBCloneProgress(ctx, taskID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(err) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": progress, }) } diff --git a/internal/router/router.go b/internal/router/router.go index 9c2ad7d8..78502592 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -208,6 +208,8 @@ func RegisterKnowledgeBaseRoutes(r *gin.RouterGroup, handler *handler.KnowledgeB kb.GET("/:id/hybrid-search", handler.HybridSearch) // 拷贝知识库 kb.POST("/copy", handler.CopyKnowledgeBase) + // 获取知识库复制进度 + kb.GET("/copy/progress/:task_id", handler.GetKBCloneProgress) } } diff --git a/internal/router/task.go b/internal/router/task.go index e860c2fd..c04aa6de 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -73,6 +73,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register summary generation handler mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration) + // Register KB clone handler + mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone) + go func() { // Start the server if err := params.Server.Run(mux); err != nil { diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index aae79d9c..082967f2 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -6,6 +6,7 @@ const ( TypeFAQImport = "faq:import" // FAQ导入任务 TypeQuestionGeneration = "question:generation" // 问题生成任务 TypeSummaryGeneration = "summary:generation" // 摘要生成任务 + TypeKBClone = "kb:clone" // 知识库复制任务 ) // ExtractChunkPayload represents the extract chunk task payload @@ -56,6 +57,39 @@ type SummaryGenerationPayload struct { KnowledgeID string `json:"knowledge_id"` } +// KBClonePayload represents the knowledge base clone task payload +type KBClonePayload struct { + TenantID uint64 `json:"tenant_id"` + TaskID string `json:"task_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` +} + +// KBCloneTaskStatus represents the status of a knowledge base clone task +type KBCloneTaskStatus string + +const ( + KBCloneStatusPending KBCloneTaskStatus = "pending" + KBCloneStatusProcessing KBCloneTaskStatus = "processing" + KBCloneStatusCompleted KBCloneTaskStatus = "completed" + KBCloneStatusFailed KBCloneTaskStatus = "failed" +) + +// KBCloneProgress represents the progress of a knowledge base clone task +type KBCloneProgress struct { + TaskID string `json:"task_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` + Status KBCloneTaskStatus `json:"status"` + Progress int `json:"progress"` // 0-100 + Total int `json:"total"` // 总知识数 + Processed int `json:"processed"` // 已处理数 + Message string `json:"message"` // 状态消息 + Error string `json:"error"` // 错误信息 + CreatedAt int64 `json:"created_at"` // 任务创建时间 + UpdatedAt int64 `json:"updated_at"` // 最后更新时间 +} + // ChunkContext represents chunk content with surrounding context type ChunkContext struct { ChunkID string `json:"chunk_id"` diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index e6caaf8e..57c5a90a 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -61,6 +61,9 @@ type ChunkRepository interface { // UpdateChunkFieldsByTagID updates fields for all chunks with the specified tag ID. // Supports updating is_enabled and flags fields. UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags) ([]string, error) + // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. + // Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src) + FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error) } // ChunkService defines the interface for chunk service operations diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index cf01b47b..2b8aafed 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -111,6 +111,12 @@ type KnowledgeService interface { ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error // ProcessSummaryGeneration handles Asynq summary generation tasks ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error + // ProcessKBClone handles Asynq knowledge base clone tasks + ProcessKBClone(ctx context.Context, t *asynq.Task) error + // GetKBCloneProgress retrieves the progress of a knowledge base clone task + GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) + // SaveKBCloneProgress saves the progress of a knowledge base clone task + SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error } // KnowledgeRepository defines the interface for knowledge repositories.