feat: 知识库异步复制功能支持进度追踪和增量同步

This commit is contained in:
wizardchen
2025-12-16 17:38:28 +08:00
committed by lyingbug
parent a7df6900eb
commit 509582a4d5
9 changed files with 689 additions and 21 deletions
+54 -5
View File
@@ -140,6 +140,29 @@ type CopyKnowledgeBaseRequest struct {
TargetID string `json:"target_id"`
}
// CopyKnowledgeBaseResponse represents the response from copy knowledge base API
type CopyKnowledgeBaseResponse struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Message string `json:"message"`
}
// KBCloneProgress represents the progress of a knowledge base clone task
type KBCloneProgress struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Status string `json:"status"` // pending, processing, completed, failed
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // Total operations count
Processed int `json:"processed"` // Processed operations count
Message string `json:"message"`
Error string `json:"error,omitempty"`
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// CreateKnowledgeBase creates a knowledge base
func (c *Client) CreateKnowledgeBase(ctx context.Context, knowledgeBase *KnowledgeBase) (*KnowledgeBase, error) {
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-bases", knowledgeBase, nil)
@@ -257,18 +280,44 @@ func (c *Client) HybridSearch(ctx context.Context, knowledgeBaseID string, param
return response.Data, nil
}
func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) error {
// CopyKnowledgeBase copies a knowledge base asynchronously and returns task info
func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) (*CopyKnowledgeBaseResponse, error) {
path := "/api/v1/knowledge-bases/copy"
resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil)
if err != nil {
return err
return nil, err
}
var response struct {
Success bool `json:"success"`
Message string `json:"message,omitempty"`
Success bool `json:"success"`
Data CopyKnowledgeBaseResponse `json:"data"`
}
return parseResponse(resp, &response)
if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return &response.Data, nil
}
// GetKBCloneProgress gets the progress of a knowledge base clone task
func (c *Client) GetKBCloneProgress(ctx context.Context, taskID string) (*KBCloneProgress, error) {
path := fmt.Sprintf("/api/v1/knowledge-bases/copy/progress/%s", taskID)
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil)
if err != nil {
return nil, err
}
var response struct {
Success bool `json:"success"`
Data KBCloneProgress `json:"data"`
}
if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return &response.Data, nil
}
+40
View File
@@ -586,3 +586,43 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
return affectedIDs, nil
}
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
// Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst),
//
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src)
func (r *chunkRepository) FAQChunkDiff(
ctx context.Context,
srcTenantID uint64, srcKBID string,
dstTenantID uint64, dstKBID string,
) (chunksToAdd []string, chunksToDelete []string, err error) {
// Get content_hash set from destination KB
dstHashSubQuery := r.db.Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
Select("content_hash")
// Find chunks in source that don't exist in destination (by content_hash)
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
Where("content_hash NOT IN (?)", dstHashSubQuery).
Pluck("id", &chunksToAdd).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err)
}
// Get content_hash set from source KB
srcHashSubQuery := r.db.Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
Select("content_hash")
// Find chunks in destination that don't exist in source (by content_hash)
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
Where("content_hash NOT IN (?)", srcHashSubQuery).
Pluck("id", &chunksToDelete).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err)
}
return chunksToAdd, chunksToDelete, nil
}
+431
View File
@@ -30,6 +30,7 @@ import (
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel/attribute"
"golang.org/x/sync/errgroup"
)
@@ -67,6 +68,7 @@ type knowledgeService struct {
modelService interfaces.ModelService
task *asynq.Client
graphEngine interfaces.RetrieveGraphRepository
redisClient *redis.Client
}
const (
@@ -91,6 +93,7 @@ func NewKnowledgeService(
task *asynq.Client,
graphEngine interfaces.RetrieveGraphRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
redisClient *redis.Client,
) (interfaces.KnowledgeService, error) {
return &knowledgeService{
config: config,
@@ -107,6 +110,7 @@ func NewKnowledgeService(
task: task,
graphEngine: graphEngine,
retrieveEngine: retrieveEngine,
redisClient: redisClient,
}, nil
}
@@ -5174,3 +5178,430 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
return nil
}
const (
kbCloneProgressKeyPrefix = "kb_clone_progress:"
kbCloneProgressTTL = 24 * time.Hour
)
// getKBCloneProgressKey returns the Redis key for storing KB clone progress
func getKBCloneProgressKey(taskID string) string {
return kbCloneProgressKeyPrefix + taskID
}
// ProcessKBClone handles Asynq knowledge base clone tasks
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
var payload types.KBClonePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
return fmt.Errorf("failed to unmarshal KB clone payload: %w", err)
}
// Add tenant ID to context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
// Check if this is the last retry
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
isLastRetry := retryCount >= maxRetry
logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d",
payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry)
// Helper function to handle errors - only mark as failed on last retry
handleError := func(progress *types.KBCloneProgress, err error, message string) {
if isLastRetry {
progress.Status = types.KBCloneStatusFailed
progress.Error = err.Error()
progress.Message = message
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
}
// Update progress to processing
progress := &types.KBCloneProgress{
TaskID: payload.TaskID,
SourceID: payload.SourceID,
TargetID: payload.TargetID,
Status: types.KBCloneStatusProcessing,
Progress: 0,
Message: "Starting knowledge base clone...",
UpdatedAt: time.Now().Unix(),
}
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress: %v", err)
}
// Get source and target knowledge bases
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID)
if err != nil {
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
handleError(progress, err, "Failed to copy knowledge base configuration")
return err
}
// Use different sync strategies based on knowledge base type
if srcKB.Type == types.KnowledgeBaseTypeFAQ {
return s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError)
}
// Document type: use Knowledge-level diff based on file_hash
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge to add: %v", err)
handleError(progress, err, "Failed to calculate knowledge difference")
return err
}
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err)
handleError(progress, err, "Failed to calculate knowledge difference")
return err
}
totalOperations := len(addKnowledge) + len(delKnowledge)
progress.Total = totalOperations
progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
processedCount := 0
batch := 10
// Delete knowledge in target that doesn't exist in source
g, gctx := errgroup.WithContext(ctx)
for ids := range slices.Chunk(delKnowledge, batch) {
g.Go(func() error {
err := s.DeleteKnowledgeList(gctx, ids)
if err != nil {
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
return err
}
return nil
})
}
if err := g.Wait(); err != nil {
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
handleError(progress, err, "Failed to delete knowledge")
return err
}
processedCount += len(delKnowledge)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
// Clone knowledge from source to target
g, gctx = errgroup.WithContext(ctx)
g.SetLimit(batch)
for _, knowledge := range addKnowledge {
g.Go(func() error {
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
if err != nil {
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
return err
}
err = s.cloneKnowledge(gctx, srcKn, dstKB)
if err != nil {
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
return err
}
// Update progress
processedCount++
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
})
}
if err := g.Wait(); err != nil {
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
handleError(progress, err, "Failed to clone knowledge")
return err
}
// Mark as completed
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Processed = totalOperations
progress.Message = "Knowledge base clone completed successfully"
progress.UpdatedAt = time.Now().Unix()
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
}
logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID)
return nil
}
// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync
func (s *knowledgeService) cloneFAQKnowledgeBase(
ctx context.Context,
srcKB, dstKB *types.KnowledgeBase,
progress *types.KBCloneProgress,
handleError func(*types.KBCloneProgress, error, string),
) error {
// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err)
handleError(progress, err, "Failed to get source FAQ knowledge")
return err
}
if len(srcKnowledgeList) == 0 {
// Source has no FAQ knowledge, nothing to clone
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Message = "Source FAQ knowledge base is empty"
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
}
srcKnowledge := srcKnowledgeList[0]
// Get chunk-level differences based on content_hash
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
handleError(progress, err, "Failed to calculate FAQ chunk difference")
return err
}
totalOperations := len(chunksToAdd) + len(chunksToDelete)
progress.Total = totalOperations
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
// If nothing to do, mark as completed
if totalOperations == 0 {
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Message = "FAQ knowledge base is already in sync"
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
}
// Get tenant info and initialize retrieve engine
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
handleError(progress, err, "Failed to initialize retrieve engine")
return err
}
// Get embedding model
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
handleError(progress, err, "Failed to get embedding model")
return err
}
processedCount := 0
// Delete FAQ chunks that don't exist in source
if len(chunksToDelete) > 0 {
// Delete from vector store
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil {
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
handleError(progress, err, "Failed to delete FAQ entries from vector store")
return err
}
// Delete from database
if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil {
logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err)
handleError(progress, err, "Failed to delete FAQ entries from database")
return err
}
processedCount += len(chunksToDelete)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
// Get or create the FAQ knowledge entry in destination
dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge)
if err != nil {
logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err)
handleError(progress, err, "Failed to prepare FAQ knowledge entry")
return err
}
// Clone FAQ chunks from source to destination
batch := 50
for i := 0; i < len(chunksToAdd); i += batch {
end := i + batch
if end > len(chunksToAdd) {
end = len(chunksToAdd)
}
batchIDs := chunksToAdd[i:end]
// Get source chunks
srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs)
if err != nil {
logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err)
handleError(progress, err, "Failed to get source FAQ entries")
return err
}
// Create new chunks for destination
newChunks := make([]*types.Chunk, 0, len(srcChunks))
for _, srcChunk := range srcChunks {
newChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: dstKB.TenantID,
KnowledgeID: dstKnowledge.ID,
KnowledgeBaseID: dstKB.ID,
TagID: srcChunk.TagID,
Content: srcChunk.Content,
ChunkIndex: srcChunk.ChunkIndex,
IsEnabled: srcChunk.IsEnabled,
Flags: srcChunk.Flags,
ChunkType: types.ChunkTypeFAQ,
Metadata: srcChunk.Metadata,
ContentHash: srcChunk.ContentHash,
Status: int(types.ChunkStatusStored), // Initially stored, will be indexed
}
newChunks = append(newChunks, newChunk)
}
// Save to database
if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil {
logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err)
handleError(progress, err, "Failed to create FAQ entries")
return err
}
// Index in vector store using existing method
// This will index standard question + similar questions based on FAQConfig
if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil {
logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err)
handleError(progress, err, "Failed to index FAQ entries")
return err
}
// Update chunk status to indexed
for _, chunk := range newChunks {
chunk.Status = int(types.ChunkStatusIndexed)
}
if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil {
logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err)
// Don't fail the whole operation for status update failure
}
processedCount += len(batchIDs)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
// Mark as completed
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Processed = totalOperations
progress.Message = "FAQ knowledge base clone completed successfully"
progress.UpdatedAt = time.Now().Unix()
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
}
return nil
}
// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base
// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge
func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) {
// FAQ knowledge base should have exactly one Knowledge entry
knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
if err != nil {
return nil, err
}
if len(knowledgeList) > 0 {
return knowledgeList[0], nil
}
// Create a new FAQ knowledge entry, copying from source if available
knowledge := &types.Knowledge{
ID: uuid.New().String(),
TenantID: kb.TenantID,
KnowledgeBaseID: kb.ID,
Type: types.KnowledgeTypeFAQ,
Title: "FAQ",
ParseStatus: "completed",
EnableStatus: "enabled",
EmbeddingModelID: kb.EmbeddingModelID,
}
// Copy additional fields from source knowledge if available
if srcKnowledge != nil {
knowledge.Title = srcKnowledge.Title
knowledge.Description = srcKnowledge.Description
knowledge.Source = srcKnowledge.Source
knowledge.Metadata = srcKnowledge.Metadata
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
return nil, err
}
return knowledge, nil
}
// saveKBCloneProgress saves the KB clone progress to Redis
func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
key := getKBCloneProgressKey(progress.TaskID)
data, err := json.Marshal(progress)
if err != nil {
return fmt.Errorf("failed to marshal progress: %w", err)
}
return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err()
}
// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use)
func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
return s.saveKBCloneProgress(ctx, progress)
}
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) {
key := getKBCloneProgressKey(taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, werrors.NewNotFoundError("KB clone task not found")
}
return nil, fmt.Errorf("failed to get progress from Redis: %w", err)
}
var progress types.KBCloneProgress
if err := json.Unmarshal(data, &progress); err != nil {
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
}
return &progress, nil
}
+116 -16
View File
@@ -2,8 +2,10 @@ package handler
import (
"context"
"encoding/json"
"net/http"
"strconv"
"time"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
@@ -11,20 +13,28 @@ import (
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
// KnowledgeBaseHandler defines the HTTP handler for knowledge base operations
type KnowledgeBaseHandler struct {
service interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
asynqClient *asynq.Client
}
// NewKnowledgeBaseHandler creates a new knowledge base handler instance
func NewKnowledgeBaseHandler(
service interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
asynqClient *asynq.Client,
) *KnowledgeBaseHandler {
return &KnowledgeBaseHandler{service: service, knowledgeService: knowledgeService}
return &KnowledgeBaseHandler{
service: service,
knowledgeService: knowledgeService,
asynqClient: asynqClient,
}
}
// HybridSearch godoc
@@ -319,15 +329,23 @@ type CopyKnowledgeBaseRequest struct {
TargetID string `json:"target_id"`
}
// CopyKnowledgeBaseResponse defines the response for copy knowledge base
type CopyKnowledgeBaseResponse struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Message string `json:"message"`
}
// CopyKnowledgeBase godoc
// @Summary 复制知识库
// @Description 将一个知识库的内容复制到另一个知识库
// @Description 将一个知识库的内容复制到另一个知识库(异步任务)
// @Tags 知识库
// @Accept json
// @Produce json
// @Param request body CopyKnowledgeBaseRequest true "复制请求"
// @Success 200 {object} map[string]interface{} "复制成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Param request body CopyKnowledgeBaseRequest true "复制请求"
// @Success 200 {object} map[string]interface{} "任务ID"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Router /knowledge-bases/copy [post]
func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
@@ -339,20 +357,102 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
return
}
go func(ctx context.Context) {
err := h.knowledgeService.CloneKnowledgeBase(ctx, req.SourceID, req.TargetID)
if err != nil {
logger.Errorf(ctx, "Failed to copy knowledge base, ID: %s to ID: %s",
secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
return
}
logger.Infof(ctx, "Knowledge base copy from ID: %s to ID: %s successfully",
secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
}(logger.CloneContext(ctx))
// Get tenant ID from context
tenantID, exists := c.Get(types.TenantIDContextKey.String())
if !exists {
logger.Error(ctx, "Failed to get tenant ID")
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
// Generate task ID
taskID := uuid.New().String()
// Create KB clone payload
payload := types.KBClonePayload{
TenantID: tenantID.(uint64),
TaskID: taskID,
SourceID: req.SourceID,
TargetID: req.TargetID,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal KB clone payload: %v", err)
c.Error(errors.NewInternalServerError("Failed to create task"))
return
}
// Enqueue KB clone task to Asynq
task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue task"))
return
}
logger.Infof(ctx, "KB clone task enqueued: %s, asynq task ID: %s, source: %s, target: %s",
taskID, info.ID, secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
// Save initial progress to Redis so frontend can query immediately
initialProgress := &types.KBCloneProgress{
TaskID: taskID,
SourceID: req.SourceID,
TargetID: req.TargetID,
Status: types.KBCloneStatusPending,
Progress: 0,
Message: "Task queued, waiting to start...",
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}
if err := h.knowledgeService.SaveKBCloneProgress(ctx, initialProgress); err != nil {
logger.Warnf(ctx, "Failed to save initial KB clone progress: %v", err)
// Don't fail the request, task is already enqueued
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge base copy successfully",
"data": CopyKnowledgeBaseResponse{
TaskID: taskID,
SourceID: req.SourceID,
TargetID: req.TargetID,
Message: "Knowledge base copy task started",
},
})
}
// GetKBCloneProgress godoc
// @Summary 获取知识库复制进度
// @Description 获取知识库复制任务的进度
// @Tags 知识库
// @Accept json
// @Produce json
// @Param task_id path string true "任务ID"
// @Success 200 {object} map[string]interface{} "进度信息"
// @Failure 404 {object} errors.AppError "任务不存在"
// @Security Bearer
// @Router /knowledge-bases/copy/progress/{task_id} [get]
func (h *KnowledgeBaseHandler) GetKBCloneProgress(c *gin.Context) {
ctx := c.Request.Context()
taskID := c.Param("task_id")
if taskID == "" {
logger.Error(ctx, "Task ID is empty")
c.Error(errors.NewBadRequestError("Task ID cannot be empty"))
return
}
progress, err := h.knowledgeService.GetKBCloneProgress(ctx, taskID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": progress,
})
}
+2
View File
@@ -208,6 +208,8 @@ func RegisterKnowledgeBaseRoutes(r *gin.RouterGroup, handler *handler.KnowledgeB
kb.GET("/:id/hybrid-search", handler.HybridSearch)
// 拷贝知识库
kb.POST("/copy", handler.CopyKnowledgeBase)
// 获取知识库复制进度
kb.GET("/copy/progress/:task_id", handler.GetKBCloneProgress)
}
}
+3
View File
@@ -73,6 +73,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register summary generation handler
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
// Register KB clone handler
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
go func() {
// Start the server
if err := params.Server.Run(mux); err != nil {
+34
View File
@@ -6,6 +6,7 @@ const (
TypeFAQImport = "faq:import" // FAQ导入任务
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -56,6 +57,39 @@ type SummaryGenerationPayload struct {
KnowledgeID string `json:"knowledge_id"`
}
// KBClonePayload represents the knowledge base clone task payload
type KBClonePayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string
const (
KBCloneStatusPending KBCloneTaskStatus = "pending"
KBCloneStatusProcessing KBCloneTaskStatus = "processing"
KBCloneStatusCompleted KBCloneTaskStatus = "completed"
KBCloneStatusFailed KBCloneTaskStatus = "failed"
)
// KBCloneProgress represents the progress of a knowledge base clone task
type KBCloneProgress struct {
TaskID string `json:"task_id"`
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
Status KBCloneTaskStatus `json:"status"`
Progress int `json:"progress"` // 0-100
Total int `json:"total"` // 总知识数
Processed int `json:"processed"` // 已处理数
Message string `json:"message"` // 状态消息
Error string `json:"error"` // 错误信息
CreatedAt int64 `json:"created_at"` // 任务创建时间
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}
// ChunkContext represents chunk content with surrounding context
type ChunkContext struct {
ChunkID string `json:"chunk_id"`
+3
View File
@@ -61,6 +61,9 @@ type ChunkRepository interface {
// UpdateChunkFieldsByTagID updates fields for all chunks with the specified tag ID.
// Supports updating is_enabled and flags fields.
UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags) ([]string, error)
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
// Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src)
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error)
}
// ChunkService defines the interface for chunk service operations
+6
View File
@@ -111,6 +111,12 @@ type KnowledgeService interface {
ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error
// ProcessSummaryGeneration handles Asynq summary generation tasks
ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error
// ProcessKBClone handles Asynq knowledge base clone tasks
ProcessKBClone(ctx context.Context, t *asynq.Task) error
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error)
// SaveKBCloneProgress saves the progress of a knowledge base clone task
SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error
}
// KnowledgeRepository defines the interface for knowledge repositories.