feat: 知识库异步复制功能支持进度追踪和增量同步
This commit is contained in:
+54
-5
@@ -140,6 +140,29 @@ type CopyKnowledgeBaseRequest struct {
|
||||
TargetID string `json:"target_id"`
|
||||
}
|
||||
|
||||
// CopyKnowledgeBaseResponse represents the response from copy knowledge base API
|
||||
type CopyKnowledgeBaseResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
TargetID string `json:"target_id"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// KBCloneProgress represents the progress of a knowledge base clone task
|
||||
type KBCloneProgress struct {
|
||||
TaskID string `json:"task_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
TargetID string `json:"target_id"`
|
||||
Status string `json:"status"` // pending, processing, completed, failed
|
||||
Progress int `json:"progress"` // 0-100
|
||||
Total int `json:"total"` // Total operations count
|
||||
Processed int `json:"processed"` // Processed operations count
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// CreateKnowledgeBase creates a knowledge base
|
||||
func (c *Client) CreateKnowledgeBase(ctx context.Context, knowledgeBase *KnowledgeBase) (*KnowledgeBase, error) {
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-bases", knowledgeBase, nil)
|
||||
@@ -257,18 +280,44 @@ func (c *Client) HybridSearch(ctx context.Context, knowledgeBaseID string, param
|
||||
return response.Data, nil
|
||||
}
|
||||
|
||||
func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) error {
|
||||
// CopyKnowledgeBase copies a knowledge base asynchronously and returns task info
|
||||
func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) (*CopyKnowledgeBaseResponse, error) {
|
||||
path := "/api/v1/knowledge-bases/copy"
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Success bool `json:"success"`
|
||||
Data CopyKnowledgeBaseResponse `json:"data"`
|
||||
}
|
||||
|
||||
return parseResponse(resp, &response)
|
||||
if err := parseResponse(resp, &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &response.Data, nil
|
||||
}
|
||||
|
||||
// GetKBCloneProgress gets the progress of a knowledge base clone task
|
||||
func (c *Client) GetKBCloneProgress(ctx context.Context, taskID string) (*KBCloneProgress, error) {
|
||||
path := fmt.Sprintf("/api/v1/knowledge-bases/copy/progress/%s", taskID)
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Data KBCloneProgress `json:"data"`
|
||||
}
|
||||
|
||||
if err := parseResponse(resp, &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &response.Data, nil
|
||||
}
|
||||
|
||||
@@ -586,3 +586,43 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
|
||||
|
||||
return affectedIDs, nil
|
||||
}
|
||||
|
||||
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
|
||||
// Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst),
|
||||
//
|
||||
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src)
|
||||
func (r *chunkRepository) FAQChunkDiff(
|
||||
ctx context.Context,
|
||||
srcTenantID uint64, srcKBID string,
|
||||
dstTenantID uint64, dstKBID string,
|
||||
) (chunksToAdd []string, chunksToDelete []string, err error) {
|
||||
// Get content_hash set from destination KB
|
||||
dstHashSubQuery := r.db.Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
|
||||
Select("content_hash")
|
||||
|
||||
// Find chunks in source that don't exist in destination (by content_hash)
|
||||
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
|
||||
Where("content_hash NOT IN (?)", dstHashSubQuery).
|
||||
Pluck("id", &chunksToAdd).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err)
|
||||
}
|
||||
|
||||
// Get content_hash set from source KB
|
||||
srcHashSubQuery := r.db.Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
|
||||
Select("content_hash")
|
||||
|
||||
// Find chunks in destination that don't exist in source (by content_hash)
|
||||
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
|
||||
Where("content_hash NOT IN (?)", srcHashSubQuery).
|
||||
Pluck("id", &chunksToDelete).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err)
|
||||
}
|
||||
|
||||
return chunksToAdd, chunksToDelete, nil
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -67,6 +68,7 @@ type knowledgeService struct {
|
||||
modelService interfaces.ModelService
|
||||
task *asynq.Client
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
redisClient *redis.Client
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -91,6 +93,7 @@ func NewKnowledgeService(
|
||||
task *asynq.Client,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
redisClient *redis.Client,
|
||||
) (interfaces.KnowledgeService, error) {
|
||||
return &knowledgeService{
|
||||
config: config,
|
||||
@@ -107,6 +110,7 @@ func NewKnowledgeService(
|
||||
task: task,
|
||||
graphEngine: graphEngine,
|
||||
retrieveEngine: retrieveEngine,
|
||||
redisClient: redisClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -5174,3 +5178,430 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
kbCloneProgressKeyPrefix = "kb_clone_progress:"
|
||||
kbCloneProgressTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// getKBCloneProgressKey returns the Redis key for storing KB clone progress
|
||||
func getKBCloneProgressKey(taskID string) string {
|
||||
return kbCloneProgressKeyPrefix + taskID
|
||||
}
|
||||
|
||||
// ProcessKBClone handles Asynq knowledge base clone tasks
|
||||
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.KBClonePayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal KB clone payload: %w", err)
|
||||
}
|
||||
|
||||
// Add tenant ID to context
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
// Check if this is the last retry
|
||||
retryCount, _ := asynq.GetRetryCount(ctx)
|
||||
maxRetry, _ := asynq.GetMaxRetry(ctx)
|
||||
isLastRetry := retryCount >= maxRetry
|
||||
|
||||
logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d",
|
||||
payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry)
|
||||
|
||||
// Helper function to handle errors - only mark as failed on last retry
|
||||
handleError := func(progress *types.KBCloneProgress, err error, message string) {
|
||||
if isLastRetry {
|
||||
progress.Status = types.KBCloneStatusFailed
|
||||
progress.Error = err.Error()
|
||||
progress.Message = message
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
}
|
||||
|
||||
// Update progress to processing
|
||||
progress := &types.KBCloneProgress{
|
||||
TaskID: payload.TaskID,
|
||||
SourceID: payload.SourceID,
|
||||
TargetID: payload.TargetID,
|
||||
Status: types.KBCloneStatusProcessing,
|
||||
Progress: 0,
|
||||
Message: "Starting knowledge base clone...",
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update KB clone progress: %v", err)
|
||||
}
|
||||
|
||||
// Get source and target knowledge bases
|
||||
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
|
||||
handleError(progress, err, "Failed to copy knowledge base configuration")
|
||||
return err
|
||||
}
|
||||
|
||||
// Use different sync strategies based on knowledge base type
|
||||
if srcKB.Type == types.KnowledgeBaseTypeFAQ {
|
||||
return s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError)
|
||||
}
|
||||
|
||||
// Document type: use Knowledge-level diff based on file_hash
|
||||
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge to add: %v", err)
|
||||
handleError(progress, err, "Failed to calculate knowledge difference")
|
||||
return err
|
||||
}
|
||||
|
||||
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err)
|
||||
handleError(progress, err, "Failed to calculate knowledge difference")
|
||||
return err
|
||||
}
|
||||
|
||||
totalOperations := len(addKnowledge) + len(delKnowledge)
|
||||
progress.Total = totalOperations
|
||||
progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
|
||||
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
|
||||
|
||||
processedCount := 0
|
||||
batch := 10
|
||||
|
||||
// Delete knowledge in target that doesn't exist in source
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
for ids := range slices.Chunk(delKnowledge, batch) {
|
||||
g.Go(func() error {
|
||||
err := s.DeleteKnowledgeList(gctx, ids)
|
||||
if err != nil {
|
||||
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
|
||||
handleError(progress, err, "Failed to delete knowledge")
|
||||
return err
|
||||
}
|
||||
|
||||
processedCount += len(delKnowledge)
|
||||
if totalOperations > 0 {
|
||||
progress.Progress = processedCount * 100 / totalOperations
|
||||
}
|
||||
progress.Processed = processedCount
|
||||
progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
|
||||
// Clone knowledge from source to target
|
||||
g, gctx = errgroup.WithContext(ctx)
|
||||
g.SetLimit(batch)
|
||||
for _, knowledge := range addKnowledge {
|
||||
g.Go(func() error {
|
||||
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
|
||||
if err != nil {
|
||||
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
|
||||
return err
|
||||
}
|
||||
err = s.cloneKnowledge(gctx, srcKn, dstKB)
|
||||
if err != nil {
|
||||
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Update progress
|
||||
processedCount++
|
||||
if totalOperations > 0 {
|
||||
progress.Progress = processedCount * 100 / totalOperations
|
||||
}
|
||||
progress.Processed = processedCount
|
||||
progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
|
||||
handleError(progress, err, "Failed to clone knowledge")
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark as completed
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
progress.Processed = totalOperations
|
||||
progress.Message = "Knowledge base clone completed successfully"
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync
|
||||
func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
ctx context.Context,
|
||||
srcKB, dstKB *types.KnowledgeBase,
|
||||
progress *types.KBCloneProgress,
|
||||
handleError func(*types.KBCloneProgress, error, string),
|
||||
) error {
|
||||
// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
|
||||
srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err)
|
||||
handleError(progress, err, "Failed to get source FAQ knowledge")
|
||||
return err
|
||||
}
|
||||
if len(srcKnowledgeList) == 0 {
|
||||
// Source has no FAQ knowledge, nothing to clone
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
progress.Message = "Source FAQ knowledge base is empty"
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
return nil
|
||||
}
|
||||
srcKnowledge := srcKnowledgeList[0]
|
||||
|
||||
// Get chunk-level differences based on content_hash
|
||||
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
|
||||
handleError(progress, err, "Failed to calculate FAQ chunk difference")
|
||||
return err
|
||||
}
|
||||
|
||||
totalOperations := len(chunksToAdd) + len(chunksToDelete)
|
||||
progress.Total = totalOperations
|
||||
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
|
||||
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
|
||||
|
||||
// If nothing to do, mark as completed
|
||||
if totalOperations == 0 {
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
progress.Message = "FAQ knowledge base is already in sync"
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get tenant info and initialize retrieve engine
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
handleError(progress, err, "Failed to initialize retrieve engine")
|
||||
return err
|
||||
}
|
||||
|
||||
// Get embedding model
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
|
||||
handleError(progress, err, "Failed to get embedding model")
|
||||
return err
|
||||
}
|
||||
|
||||
processedCount := 0
|
||||
|
||||
// Delete FAQ chunks that don't exist in source
|
||||
if len(chunksToDelete) > 0 {
|
||||
// Delete from vector store
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
|
||||
handleError(progress, err, "Failed to delete FAQ entries from vector store")
|
||||
return err
|
||||
}
|
||||
// Delete from database
|
||||
if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err)
|
||||
handleError(progress, err, "Failed to delete FAQ entries from database")
|
||||
return err
|
||||
}
|
||||
processedCount += len(chunksToDelete)
|
||||
if totalOperations > 0 {
|
||||
progress.Progress = processedCount * 100 / totalOperations
|
||||
}
|
||||
progress.Processed = processedCount
|
||||
progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
|
||||
// Get or create the FAQ knowledge entry in destination
|
||||
dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err)
|
||||
handleError(progress, err, "Failed to prepare FAQ knowledge entry")
|
||||
return err
|
||||
}
|
||||
|
||||
// Clone FAQ chunks from source to destination
|
||||
batch := 50
|
||||
for i := 0; i < len(chunksToAdd); i += batch {
|
||||
end := i + batch
|
||||
if end > len(chunksToAdd) {
|
||||
end = len(chunksToAdd)
|
||||
}
|
||||
batchIDs := chunksToAdd[i:end]
|
||||
|
||||
// Get source chunks
|
||||
srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err)
|
||||
handleError(progress, err, "Failed to get source FAQ entries")
|
||||
return err
|
||||
}
|
||||
|
||||
// Create new chunks for destination
|
||||
newChunks := make([]*types.Chunk, 0, len(srcChunks))
|
||||
for _, srcChunk := range srcChunks {
|
||||
newChunk := &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: dstKB.TenantID,
|
||||
KnowledgeID: dstKnowledge.ID,
|
||||
KnowledgeBaseID: dstKB.ID,
|
||||
TagID: srcChunk.TagID,
|
||||
Content: srcChunk.Content,
|
||||
ChunkIndex: srcChunk.ChunkIndex,
|
||||
IsEnabled: srcChunk.IsEnabled,
|
||||
Flags: srcChunk.Flags,
|
||||
ChunkType: types.ChunkTypeFAQ,
|
||||
Metadata: srcChunk.Metadata,
|
||||
ContentHash: srcChunk.ContentHash,
|
||||
Status: int(types.ChunkStatusStored), // Initially stored, will be indexed
|
||||
}
|
||||
newChunks = append(newChunks, newChunk)
|
||||
}
|
||||
|
||||
// Save to database
|
||||
if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err)
|
||||
handleError(progress, err, "Failed to create FAQ entries")
|
||||
return err
|
||||
}
|
||||
|
||||
// Index in vector store using existing method
|
||||
// This will index standard question + similar questions based on FAQConfig
|
||||
if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil {
|
||||
logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err)
|
||||
handleError(progress, err, "Failed to index FAQ entries")
|
||||
return err
|
||||
}
|
||||
|
||||
// Update chunk status to indexed
|
||||
for _, chunk := range newChunks {
|
||||
chunk.Status = int(types.ChunkStatusIndexed)
|
||||
}
|
||||
if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err)
|
||||
// Don't fail the whole operation for status update failure
|
||||
}
|
||||
|
||||
processedCount += len(batchIDs)
|
||||
if totalOperations > 0 {
|
||||
progress.Progress = processedCount * 100 / totalOperations
|
||||
}
|
||||
progress.Processed = processedCount
|
||||
progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
|
||||
// Mark as completed
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
progress.Processed = totalOperations
|
||||
progress.Message = "FAQ knowledge base clone completed successfully"
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base
|
||||
// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge
|
||||
func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) {
|
||||
// FAQ knowledge base should have exactly one Knowledge entry
|
||||
knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(knowledgeList) > 0 {
|
||||
return knowledgeList[0], nil
|
||||
}
|
||||
|
||||
// Create a new FAQ knowledge entry, copying from source if available
|
||||
knowledge := &types.Knowledge{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: kb.TenantID,
|
||||
KnowledgeBaseID: kb.ID,
|
||||
Type: types.KnowledgeTypeFAQ,
|
||||
Title: "FAQ",
|
||||
ParseStatus: "completed",
|
||||
EnableStatus: "enabled",
|
||||
EmbeddingModelID: kb.EmbeddingModelID,
|
||||
}
|
||||
|
||||
// Copy additional fields from source knowledge if available
|
||||
if srcKnowledge != nil {
|
||||
knowledge.Title = srcKnowledge.Title
|
||||
knowledge.Description = srcKnowledge.Description
|
||||
knowledge.Source = srcKnowledge.Source
|
||||
knowledge.Metadata = srcKnowledge.Metadata
|
||||
}
|
||||
|
||||
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return knowledge, nil
|
||||
}
|
||||
|
||||
// saveKBCloneProgress saves the KB clone progress to Redis
|
||||
func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
|
||||
key := getKBCloneProgressKey(progress.TaskID)
|
||||
data, err := json.Marshal(progress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal progress: %w", err)
|
||||
}
|
||||
return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err()
|
||||
}
|
||||
|
||||
// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use)
|
||||
func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
|
||||
return s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
|
||||
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
|
||||
func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) {
|
||||
key := getKBCloneProgressKey(taskID)
|
||||
data, err := s.redisClient.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, werrors.NewNotFoundError("KB clone task not found")
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get progress from Redis: %w", err)
|
||||
}
|
||||
|
||||
var progress types.KBCloneProgress
|
||||
if err := json.Unmarshal(data, &progress); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
|
||||
}
|
||||
return &progress, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -11,20 +13,28 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// KnowledgeBaseHandler defines the HTTP handler for knowledge base operations
|
||||
type KnowledgeBaseHandler struct {
|
||||
service interfaces.KnowledgeBaseService
|
||||
knowledgeService interfaces.KnowledgeService
|
||||
asynqClient *asynq.Client
|
||||
}
|
||||
|
||||
// NewKnowledgeBaseHandler creates a new knowledge base handler instance
|
||||
func NewKnowledgeBaseHandler(
|
||||
service interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
asynqClient *asynq.Client,
|
||||
) *KnowledgeBaseHandler {
|
||||
return &KnowledgeBaseHandler{service: service, knowledgeService: knowledgeService}
|
||||
return &KnowledgeBaseHandler{
|
||||
service: service,
|
||||
knowledgeService: knowledgeService,
|
||||
asynqClient: asynqClient,
|
||||
}
|
||||
}
|
||||
|
||||
// HybridSearch godoc
|
||||
@@ -319,15 +329,23 @@ type CopyKnowledgeBaseRequest struct {
|
||||
TargetID string `json:"target_id"`
|
||||
}
|
||||
|
||||
// CopyKnowledgeBaseResponse defines the response for copy knowledge base
|
||||
type CopyKnowledgeBaseResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
TargetID string `json:"target_id"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// CopyKnowledgeBase godoc
|
||||
// @Summary 复制知识库
|
||||
// @Description 将一个知识库的内容复制到另一个知识库
|
||||
// @Description 将一个知识库的内容复制到另一个知识库(异步任务)
|
||||
// @Tags 知识库
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body CopyKnowledgeBaseRequest true "复制请求"
|
||||
// @Success 200 {object} map[string]interface{} "复制成功"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Param request body CopyKnowledgeBaseRequest true "复制请求"
|
||||
// @Success 200 {object} map[string]interface{} "任务ID"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Security Bearer
|
||||
// @Router /knowledge-bases/copy [post]
|
||||
func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
@@ -339,20 +357,102 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
go func(ctx context.Context) {
|
||||
err := h.knowledgeService.CloneKnowledgeBase(ctx, req.SourceID, req.TargetID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to copy knowledge base, ID: %s to ID: %s",
|
||||
secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Knowledge base copy from ID: %s to ID: %s successfully",
|
||||
secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
|
||||
}(logger.CloneContext(ctx))
|
||||
// Get tenant ID from context
|
||||
tenantID, exists := c.Get(types.TenantIDContextKey.String())
|
||||
if !exists {
|
||||
logger.Error(ctx, "Failed to get tenant ID")
|
||||
c.Error(errors.NewUnauthorizedError("Unauthorized"))
|
||||
return
|
||||
}
|
||||
|
||||
// Generate task ID
|
||||
taskID := uuid.New().String()
|
||||
|
||||
// Create KB clone payload
|
||||
payload := types.KBClonePayload{
|
||||
TenantID: tenantID.(uint64),
|
||||
TaskID: taskID,
|
||||
SourceID: req.SourceID,
|
||||
TargetID: req.TargetID,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal KB clone payload: %v", err)
|
||||
c.Error(errors.NewInternalServerError("Failed to create task"))
|
||||
return
|
||||
}
|
||||
|
||||
// Enqueue KB clone task to Asynq
|
||||
task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
|
||||
c.Error(errors.NewInternalServerError("Failed to enqueue task"))
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "KB clone task enqueued: %s, asynq task ID: %s, source: %s, target: %s",
|
||||
taskID, info.ID, secutils.SanitizeForLog(req.SourceID), secutils.SanitizeForLog(req.TargetID))
|
||||
|
||||
// Save initial progress to Redis so frontend can query immediately
|
||||
initialProgress := &types.KBCloneProgress{
|
||||
TaskID: taskID,
|
||||
SourceID: req.SourceID,
|
||||
TargetID: req.TargetID,
|
||||
Status: types.KBCloneStatusPending,
|
||||
Progress: 0,
|
||||
Message: "Task queued, waiting to start...",
|
||||
CreatedAt: time.Now().Unix(),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
if err := h.knowledgeService.SaveKBCloneProgress(ctx, initialProgress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to save initial KB clone progress: %v", err)
|
||||
// Don't fail the request, task is already enqueued
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"message": "Knowledge base copy successfully",
|
||||
"data": CopyKnowledgeBaseResponse{
|
||||
TaskID: taskID,
|
||||
SourceID: req.SourceID,
|
||||
TargetID: req.TargetID,
|
||||
Message: "Knowledge base copy task started",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// GetKBCloneProgress godoc
|
||||
// @Summary 获取知识库复制进度
|
||||
// @Description 获取知识库复制任务的进度
|
||||
// @Tags 知识库
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param task_id path string true "任务ID"
|
||||
// @Success 200 {object} map[string]interface{} "进度信息"
|
||||
// @Failure 404 {object} errors.AppError "任务不存在"
|
||||
// @Security Bearer
|
||||
// @Router /knowledge-bases/copy/progress/{task_id} [get]
|
||||
func (h *KnowledgeBaseHandler) GetKBCloneProgress(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
taskID := c.Param("task_id")
|
||||
if taskID == "" {
|
||||
logger.Error(ctx, "Task ID is empty")
|
||||
c.Error(errors.NewBadRequestError("Task ID cannot be empty"))
|
||||
return
|
||||
}
|
||||
|
||||
progress, err := h.knowledgeService.GetKBCloneProgress(ctx, taskID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": progress,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -208,6 +208,8 @@ func RegisterKnowledgeBaseRoutes(r *gin.RouterGroup, handler *handler.KnowledgeB
|
||||
kb.GET("/:id/hybrid-search", handler.HybridSearch)
|
||||
// 拷贝知识库
|
||||
kb.POST("/copy", handler.CopyKnowledgeBase)
|
||||
// 获取知识库复制进度
|
||||
kb.GET("/copy/progress/:task_id", handler.GetKBCloneProgress)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -73,6 +73,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register summary generation handler
|
||||
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
|
||||
|
||||
// Register KB clone handler
|
||||
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
|
||||
|
||||
go func() {
|
||||
// Start the server
|
||||
if err := params.Server.Run(mux); err != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ const (
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
TypeKBClone = "kb:clone" // 知识库复制任务
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -56,6 +57,39 @@ type SummaryGenerationPayload struct {
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
}
|
||||
|
||||
// KBClonePayload represents the knowledge base clone task payload
|
||||
type KBClonePayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
TargetID string `json:"target_id"`
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
type KBCloneTaskStatus string
|
||||
|
||||
const (
|
||||
KBCloneStatusPending KBCloneTaskStatus = "pending"
|
||||
KBCloneStatusProcessing KBCloneTaskStatus = "processing"
|
||||
KBCloneStatusCompleted KBCloneTaskStatus = "completed"
|
||||
KBCloneStatusFailed KBCloneTaskStatus = "failed"
|
||||
)
|
||||
|
||||
// KBCloneProgress represents the progress of a knowledge base clone task
|
||||
type KBCloneProgress struct {
|
||||
TaskID string `json:"task_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
TargetID string `json:"target_id"`
|
||||
Status KBCloneTaskStatus `json:"status"`
|
||||
Progress int `json:"progress"` // 0-100
|
||||
Total int `json:"total"` // 总知识数
|
||||
Processed int `json:"processed"` // 已处理数
|
||||
Message string `json:"message"` // 状态消息
|
||||
Error string `json:"error"` // 错误信息
|
||||
CreatedAt int64 `json:"created_at"` // 任务创建时间
|
||||
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
|
||||
}
|
||||
|
||||
// ChunkContext represents chunk content with surrounding context
|
||||
type ChunkContext struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
|
||||
@@ -61,6 +61,9 @@ type ChunkRepository interface {
|
||||
// UpdateChunkFieldsByTagID updates fields for all chunks with the specified tag ID.
|
||||
// Supports updating is_enabled and flags fields.
|
||||
UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags) ([]string, error)
|
||||
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
|
||||
// Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src)
|
||||
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error)
|
||||
}
|
||||
|
||||
// ChunkService defines the interface for chunk service operations
|
||||
|
||||
@@ -111,6 +111,12 @@ type KnowledgeService interface {
|
||||
ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessSummaryGeneration handles Asynq summary generation tasks
|
||||
ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessKBClone handles Asynq knowledge base clone tasks
|
||||
ProcessKBClone(ctx context.Context, t *asynq.Task) error
|
||||
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
|
||||
GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error)
|
||||
// SaveKBCloneProgress saves the progress of a knowledge base clone task
|
||||
SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error
|
||||
}
|
||||
|
||||
// KnowledgeRepository defines the interface for knowledge repositories.
|
||||
|
||||
Reference in New Issue
Block a user