Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_clone_move.go
T
ochan.kwon e9980c6011 fix: deep-copy stored files and images when cloning a knowledge base
Cloning a knowledge base previously copied only the storage path strings
(knowledge.FilePath and chunk.ImageInfo.URL), so the source and the clone
shared the same physical objects in the storage backend. Once the original
file and extracted images are deleted on source removal, the clone is left
with dangling references and its document and images become unreadable —
data loss that occurs even for same-store clones.

Add a CopyFile primitive to the FileService interface and implement it in
every backend: server-side CopyObject on the object stores
(s3/obs/cos/oss/tos/ks3/minio), io.Copy on local, and a no-op on dummy.
Destinations use the knowledge-owned layout and reuse the existing
path/object-key guards; a sentinel ErrCrossBackendCopy is returned when the
source scheme does not match the backend.

Use CopyFile to deep-copy the document file in cloneKnowledge and the
extracted images in CloneChunk and cloneFAQKnowledgeBase via a shared
cloneChunkImageInfo helper that deduplicates identical image URLs per clone
and rewrites them to the new objects. Copied objects are cleaned up
best-effort if a clone fails partway through. A clone-time preflight rejects
cloning into a target bound to a different storage backend when the tenant
pins providers via StorageEngineConfig.

Adds unit tests for local CopyFile (independent copy survives source
deletion, traversal rejection, cross-backend rejection), cloneChunkImageInfo
(empty/multi/dedup/parse-failure/OriginalURL handling), and the storage
provider preflight.
2026-06-03 14:45:59 +08:00

1221 lines
43 KiB
Go

package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
"golang.org/x/sync/errgroup"
)
// copyOwnedObject performs a real copy of srcPath into a NEW object owned by
// (tenantID, knowledgeID) using the destination FileService, returning the new
// provider:// path. The same-backend check lives inside dstSvc.CopyFile, which
// returns file.ErrCrossBackendCopy when srcPath belongs to a different provider;
// that error is propagated unchanged so callers can fail the clone explicitly.
// srcSvc is accepted for symmetry with the read side but is not used directly:
// server-side copies are issued by the destination service.
func copyOwnedObject(
ctx context.Context,
srcSvc, dstSvc interfaces.FileService,
srcPath string,
tenantID uint64,
knowledgeID string,
) (string, error) {
_ = srcSvc // reserved for future cross-backend streaming fallback
return dstSvc.CopyFile(ctx, srcPath, tenantID, knowledgeID)
}
// cloneChunkImageInfo parses a chunk's image_info JSON, copies every referenced
// object into a NEW object owned by (tenantID, knowledgeID), and returns the
// re-serialized image_info plus the list of newly-created object URLs (for
// rollback on failure). urlCache dedups identical source objects across chunks
// so the same source image is copied at most once per clone.
//
// An empty srcImageInfo yields ("", nil, nil). A JSON parse failure returns an
// error (the clone fails) rather than silently inheriting the shared-reference
// bug. When an image's OriginalURL points at the same object as its URL (the
// common case for extracted images), OriginalURL is rewritten to the new path
// too; an OriginalURL from a different/external source is preserved.
func cloneChunkImageInfo(
ctx context.Context,
dstSvc interfaces.FileService,
srcImageInfo string,
tenantID uint64,
knowledgeID string,
urlCache map[string]string,
) (newImageInfo string, copiedURLs []string, err error) {
if srcImageInfo == "" {
return "", nil, nil
}
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(srcImageInfo), &images); err != nil {
return "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err)
}
for _, img := range images {
if img == nil || img.URL == "" {
continue
}
originalMatchedURL := img.OriginalURL == img.URL
newURL, cached := urlCache[img.URL]
if !cached {
newURL, err = copyOwnedObject(ctx, dstSvc, dstSvc, img.URL, tenantID, knowledgeID)
if err != nil {
return "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err)
}
urlCache[img.URL] = newURL
copiedURLs = append(copiedURLs, newURL)
}
if originalMatchedURL {
img.OriginalURL = newURL
}
img.URL = newURL
}
out, err := json.Marshal(images)
if err != nil {
return "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err)
}
return string(out), copiedURLs, nil
}
// cleanupCopiedObjects deletes objects that were newly created during a clone
// that subsequently failed, to avoid orphaning storage. It is best-effort:
// delete errors are logged but never returned (the original clone error wins).
func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths []string) {
if len(paths) == 0 || svc == nil {
return
}
logger.Infof(ctx, "Cleaning up %d copied objects after clone failure", len(paths))
for _, p := range paths {
if p == "" {
continue
}
if err := svc.DeleteFile(ctx, p); err != nil {
logger.Errorf(ctx, "Failed to clean up copied object %s: %v", p, err)
}
}
}
func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error {
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID)
if err != nil {
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
return err
}
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return err
}
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return err
}
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
batch := 10
g, gctx := errgroup.WithContext(ctx)
for ids := range slices.Chunk(delKnowledge, batch) {
g.Go(func() error {
err := s.DeleteKnowledgeList(gctx, ids)
if err != nil {
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
return err
}
return nil
})
}
err = g.Wait()
if err != nil {
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
return err
}
// Copy context out of auto-stop task
g, gctx = errgroup.WithContext(ctx)
g.SetLimit(batch)
for _, knowledge := range addKnowledge {
g.Go(func() error {
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
if err != nil {
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
return err
}
err = s.cloneKnowledge(gctx, srcKn, dstKB)
if err != nil {
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
return err
}
return nil
})
}
err = g.Wait()
if err != nil {
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
return err
}
return nil
}
// CloneChunk clone chunks from one knowledge to another
// This method transfers a chunk from a source knowledge document to a target knowledge document
// It handles the creation of new chunks in the target knowledge and updates the vector database accordingly
// Parameters:
// - ctx: Context with authentication and request information
// - src: Source knowledge document containing the chunk to move
// - dst: Target knowledge document where the chunk will be moved
//
// Returns:
// - error: Any error encountered during the move operation
//
// This method handles the chunk transfer logic, including creating new chunks in the target knowledge
// and updating the vector database representation of the moved chunks.
// It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained
// by mapping the source chunk IDs to the new target chunk IDs.
func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) (err error) {
chunkPage := 1
chunkPageSize := 100
srcTodst := map[string]string{}
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
targetChunks := make([]*types.Chunk, 0, 10)
chunkType := []types.ChunkType{
types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary,
types.ChunkTypeImageCaption, types.ChunkTypeImageOCR,
}
// Resolve the destination FileService so extracted images can be copied
// into objects owned by the destination knowledge. urlCache dedups identical
// source images across chunks; copiedURLs accumulates new objects so they can
// be cleaned up if the clone fails partway through.
dstKB, dstKBErr := s.kbService.GetKnowledgeBaseByID(ctx, dst.KnowledgeBaseID)
if dstKBErr != nil {
return fmt.Errorf("failed to load destination knowledge base for image copy: %w", dstKBErr)
}
dstSvc := s.resolveFileService(ctx, dstKB)
urlCache := map[string]string{}
var copiedURLs []string
defer func() {
if err != nil {
cleanupCopiedObjects(ctx, dstSvc, copiedURLs)
}
}()
for {
sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx,
src.TenantID,
src.ID,
&types.Pagination{
Page: chunkPage,
PageSize: chunkPageSize,
},
chunkType,
"",
"",
"",
"",
"",
)
chunkPage++
if err != nil {
return err
}
if len(sourceChunks) == 0 {
break
}
now := time.Now()
for _, sourceChunk := range sourceChunks {
// Map TagID to target knowledge base
targetTagID := ""
if sourceChunk.TagID != "" {
if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok {
targetTagID = mappedTagID
} else {
// Try to find or create the tag in target knowledge base
targetTagID = s.getOrCreateTagInTarget(ctx, src.TenantID, dst.TenantID, dst.KnowledgeBaseID, sourceChunk.TagID, tagIDMapping)
}
}
// Deep-copy extracted images into objects owned by the destination
// knowledge so deleting the source never breaks this clone.
newImageInfo, copied, copyErr := cloneChunkImageInfo(
ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache)
if copyErr != nil {
err = fmt.Errorf("clone chunk image copy failed: %w", copyErr)
return err
}
copiedURLs = append(copiedURLs, copied...)
targetChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: dst.TenantID,
KnowledgeID: dst.ID,
KnowledgeBaseID: dst.KnowledgeBaseID,
TagID: targetTagID,
Content: sourceChunk.Content,
ChunkIndex: sourceChunk.ChunkIndex,
IsEnabled: sourceChunk.IsEnabled,
Flags: sourceChunk.Flags,
Status: sourceChunk.Status,
StartAt: sourceChunk.StartAt,
EndAt: sourceChunk.EndAt,
PreChunkID: sourceChunk.PreChunkID,
NextChunkID: sourceChunk.NextChunkID,
ChunkType: sourceChunk.ChunkType,
ParentChunkID: sourceChunk.ParentChunkID,
Metadata: sourceChunk.Metadata,
ContentHash: sourceChunk.ContentHash,
ImageInfo: newImageInfo,
CreatedAt: now,
UpdatedAt: now,
}
targetChunks = append(targetChunks, targetChunk)
srcTodst[sourceChunk.ID] = targetChunk.ID
}
}
for _, targetChunk := range targetChunks {
if val, ok := srcTodst[targetChunk.PreChunkID]; ok {
targetChunk.PreChunkID = val
} else {
targetChunk.PreChunkID = ""
}
if val, ok := srcTodst[targetChunk.NextChunkID]; ok {
targetChunk.NextChunkID = val
} else {
targetChunk.NextChunkID = ""
}
if val, ok := srcTodst[targetChunk.ParentChunkID]; ok {
targetChunk.ParentChunkID = val
} else {
targetChunk.ParentChunkID = ""
}
}
for chunks := range slices.Chunk(targetChunks, chunkPageSize) {
err := s.chunkRepo.CreateChunks(ctx, chunks)
if err != nil {
return err
}
}
tenantID := types.MustTenantIDFromContext(ctx)
// Route CopyIndices via the source KB's bound store. This function does
// not handle cross-store copies — embeddings written by different
// VectorStore backends are not bit-compatible, so callers that allow
// source/target KBs to bind to different stores must perform their own
// cross-store migration before invoking this.
var sourceStoreID *string
if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil {
sourceStoreID = srcKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
if err != nil {
return err
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID)
if err != nil {
return err
}
if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID,
map[string]string{src.ID: dst.ID},
srcTodst,
embeddingModel.GetDimensions(),
dst.Type,
); err != nil {
return err
}
return nil
}
const (
kbCloneProgressKeyPrefix = "kb_clone_progress:"
kbCloneProgressTTL = 24 * time.Hour
)
// getKBCloneProgressKey returns the Redis key for storing KB clone progress
func getKBCloneProgressKey(taskID string) string {
return kbCloneProgressKeyPrefix + taskID
}
// ProcessKBClone handles Asynq knowledge base clone tasks
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
var payload types.KBClonePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
return fmt.Errorf("failed to unmarshal KB clone payload: %w", err)
}
// Add tenant ID to context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
// Get tenant info and add to context
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
// Check if this is the last retry
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
isLastRetry := retryCount >= maxRetry
logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d",
payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry)
// Helper function to handle errors - only mark as failed on last retry
handleError := func(progress *types.KBCloneProgress, err error, message string) {
if isLastRetry {
progress.Status = types.KBCloneStatusFailed
progress.Error = err.Error()
progress.Message = message
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
}
// Update progress to processing
progress := &types.KBCloneProgress{
TaskID: payload.TaskID,
SourceID: payload.SourceID,
TargetID: payload.TargetID,
Status: types.KBCloneStatusProcessing,
Progress: 0,
Message: "Starting knowledge base clone...",
UpdatedAt: time.Now().Unix(),
}
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress: %v", err)
}
// Get source and target knowledge bases
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID)
if err != nil {
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
handleError(progress, err, "Failed to copy knowledge base configuration")
return err
}
// Use different sync strategies based on knowledge base type
if srcKB.Type == types.KnowledgeBaseTypeFAQ {
return s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError)
}
// Document type: use Knowledge-level diff based on file_hash
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge to add: %v", err)
handleError(progress, err, "Failed to calculate knowledge difference")
return err
}
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err)
handleError(progress, err, "Failed to calculate knowledge difference")
return err
}
totalOperations := len(addKnowledge) + len(delKnowledge)
progress.Total = totalOperations
progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
processedCount := 0
batch := 10
// Delete knowledge in target that doesn't exist in source
g, gctx := errgroup.WithContext(ctx)
for ids := range slices.Chunk(delKnowledge, batch) {
g.Go(func() error {
err := s.DeleteKnowledgeList(gctx, ids)
if err != nil {
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
return err
}
return nil
})
}
if err := g.Wait(); err != nil {
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
handleError(progress, err, "Failed to delete knowledge")
return err
}
processedCount += len(delKnowledge)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
// Clone knowledge from source to target
g, gctx = errgroup.WithContext(ctx)
g.SetLimit(batch)
for _, knowledge := range addKnowledge {
g.Go(func() error {
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
if err != nil {
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
return err
}
err = s.cloneKnowledge(gctx, srcKn, dstKB)
if err != nil {
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
return err
}
// Update progress
processedCount++
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
})
}
if err := g.Wait(); err != nil {
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
handleError(progress, err, "Failed to clone knowledge")
return err
}
// Mark as completed
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Processed = totalOperations
progress.Message = "Knowledge base clone completed successfully"
progress.UpdatedAt = time.Now().Unix()
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
}
logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID)
return nil
}
// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync
func (s *knowledgeService) cloneFAQKnowledgeBase(
ctx context.Context,
srcKB, dstKB *types.KnowledgeBase,
progress *types.KBCloneProgress,
handleError func(*types.KBCloneProgress, error, string),
) (retErr error) {
// Deep-copy extracted FAQ images into objects owned by the destination KB.
// urlCache dedups identical source images across chunks; copiedURLs tracks
// new objects for best-effort cleanup if the clone fails partway through.
dstSvc := s.resolveFileService(ctx, dstKB)
imageURLCache := map[string]string{}
var copiedImageURLs []string
defer func() {
if retErr != nil {
cleanupCopiedObjects(ctx, dstSvc, copiedImageURLs)
}
}()
// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err)
handleError(progress, err, "Failed to get source FAQ knowledge")
return err
}
if len(srcKnowledgeList) == 0 {
// Source has no FAQ knowledge, nothing to clone
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Message = "Source FAQ knowledge base is empty"
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
}
srcKnowledge := srcKnowledgeList[0]
// Get chunk-level differences based on content_hash
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
handleError(progress, err, "Failed to calculate FAQ chunk difference")
return err
}
totalOperations := len(chunksToAdd) + len(chunksToDelete)
progress.Total = totalOperations
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
// If nothing to do, mark as completed
if totalOperations == 0 {
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Message = "FAQ knowledge base is already in sync"
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
return nil
}
// Route the FAQ clone through the source KB's bound store. Same
// constraint as CloneChunk: callers must ensure source and target share
// the same VectorStore (cross-store FAQ clone is not handled here).
var sourceStoreID *string
if srcKB != nil {
sourceStoreID = srcKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceStoreID)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
handleError(progress, err, "Failed to initialize retrieve engine")
return err
}
// Get embedding model
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
handleError(progress, err, "Failed to get embedding model")
return err
}
processedCount := 0
// Delete FAQ chunks that don't exist in source
if len(chunksToDelete) > 0 {
// Delete from vector store
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
handleError(progress, err, "Failed to delete FAQ entries from vector store")
return err
}
// Delete from database
if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil {
logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err)
handleError(progress, err, "Failed to delete FAQ entries from database")
return err
}
processedCount += len(chunksToDelete)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
// Get or create the FAQ knowledge entry in destination
dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge)
if err != nil {
logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err)
handleError(progress, err, "Failed to prepare FAQ knowledge entry")
return err
}
// Clone FAQ chunks from source to destination
batch := 50
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
for i := 0; i < len(chunksToAdd); i += batch {
end := i + batch
if end > len(chunksToAdd) {
end = len(chunksToAdd)
}
batchIDs := chunksToAdd[i:end]
// Get source chunks
srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs)
if err != nil {
logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err)
handleError(progress, err, "Failed to get source FAQ entries")
return err
}
// Create new chunks for destination
newChunks := make([]*types.Chunk, 0, len(srcChunks))
for _, srcChunk := range srcChunks {
// Map TagID to target knowledge base
targetTagID := ""
if srcChunk.TagID != "" {
if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok {
targetTagID = mappedTagID
} else {
// Try to find or create the tag in target knowledge base
targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping)
}
}
// Deep-copy extracted images into objects owned by the destination
// FAQ knowledge so deleting the source never breaks this clone.
newImageInfo, copied, copyErr := cloneChunkImageInfo(
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
if copyErr != nil {
logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)
handleError(progress, copyErr, "Failed to copy FAQ entry images")
retErr = copyErr
return retErr
}
copiedImageURLs = append(copiedImageURLs, copied...)
newChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: dstKB.TenantID,
KnowledgeID: dstKnowledge.ID,
KnowledgeBaseID: dstKB.ID,
TagID: targetTagID,
Content: srcChunk.Content,
ChunkIndex: srcChunk.ChunkIndex,
IsEnabled: srcChunk.IsEnabled,
Flags: srcChunk.Flags,
ChunkType: types.ChunkTypeFAQ,
Metadata: srcChunk.Metadata,
ContentHash: srcChunk.ContentHash,
ImageInfo: newImageInfo,
Status: int(types.ChunkStatusStored), // Initially stored, will be indexed
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
newChunks = append(newChunks, newChunk)
}
// Save to database
if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil {
logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err)
handleError(progress, err, "Failed to create FAQ entries")
return err
}
// Index in vector store using existing method
// This will index standard question + similar questions based on FAQConfig
if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil {
logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err)
handleError(progress, err, "Failed to index FAQ entries")
return err
}
// Update chunk status to indexed
for _, chunk := range newChunks {
chunk.Status = int(types.ChunkStatusIndexed)
}
if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil {
logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err)
// Don't fail the whole operation for status update failure
}
processedCount += len(batchIDs)
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
// Mark as completed
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
progress.Processed = totalOperations
progress.Message = "FAQ knowledge base clone completed successfully"
progress.UpdatedAt = time.Now().Unix()
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
}
return nil
}
// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base
// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge
func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) {
// FAQ knowledge base should have exactly one Knowledge entry
knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
if err != nil {
return nil, err
}
if len(knowledgeList) > 0 {
return knowledgeList[0], nil
}
// Create a new FAQ knowledge entry, copying from source if available
knowledge := &types.Knowledge{
ID: uuid.New().String(),
TenantID: kb.TenantID,
KnowledgeBaseID: kb.ID,
Type: types.KnowledgeTypeFAQ,
Channel: types.ChannelWeb,
Title: "FAQ",
ParseStatus: "completed",
EnableStatus: "enabled",
EmbeddingModelID: kb.EmbeddingModelID,
}
// Copy additional fields from source knowledge if available
if srcKnowledge != nil {
knowledge.Title = srcKnowledge.Title
knowledge.Description = srcKnowledge.Description
knowledge.Source = srcKnowledge.Source
knowledge.Channel = srcKnowledge.Channel
knowledge.Metadata = srcKnowledge.Metadata
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
return nil, err
}
return knowledge, nil
}
// saveKBCloneProgress saves the KB clone progress to Redis
func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
key := getKBCloneProgressKey(progress.TaskID)
data, err := json.Marshal(progress)
if err != nil {
return fmt.Errorf("failed to marshal progress: %w", err)
}
return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err()
}
// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use)
func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
return s.saveKBCloneProgress(ctx, progress)
}
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) {
key := getKBCloneProgressKey(taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, werrors.NewNotFoundError("KB clone task not found")
}
return nil, fmt.Errorf("failed to get progress from Redis: %w", err)
}
var progress types.KBCloneProgress
if err := json.Unmarshal(data, &progress); err != nil {
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
}
return &progress, nil
}
const (
knowledgeMoveProgressKeyPrefix = "knowledge_move_progress:"
knowledgeMoveProgressTTL = 24 * time.Hour
)
func getKnowledgeMoveProgressKey(taskID string) string {
return knowledgeMoveProgressKeyPrefix + taskID
}
func (s *knowledgeService) saveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
key := getKnowledgeMoveProgressKey(progress.TaskID)
data, err := json.Marshal(progress)
if err != nil {
return fmt.Errorf("failed to marshal move progress: %w", err)
}
return s.redisClient.Set(ctx, key, data, knowledgeMoveProgressTTL).Err()
}
// SaveKnowledgeMoveProgress saves the knowledge move progress to Redis (public method for handler use)
func (s *knowledgeService) SaveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
return s.saveKnowledgeMoveProgress(ctx, progress)
}
// GetKnowledgeMoveProgress retrieves the progress of a knowledge move task
func (s *knowledgeService) GetKnowledgeMoveProgress(ctx context.Context, taskID string) (*types.KnowledgeMoveProgress, error) {
key := getKnowledgeMoveProgressKey(taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, werrors.NewNotFoundError("Knowledge move task not found")
}
return nil, fmt.Errorf("failed to get move progress from Redis: %w", err)
}
var progress types.KnowledgeMoveProgress
if err := json.Unmarshal(data, &progress); err != nil {
return nil, fmt.Errorf("failed to unmarshal move progress: %w", err)
}
return &progress, nil
}
// ProcessKnowledgeMove handles Asynq knowledge move tasks
func (s *knowledgeService) ProcessKnowledgeMove(ctx context.Context, t *asynq.Task) error {
var payload types.KnowledgeMovePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
return fmt.Errorf("failed to unmarshal knowledge move payload: %w", err)
}
// Add tenant ID to context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
// Get tenant info and add to context
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "ProcessKnowledgeMove: failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
// Check if this is the last retry
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
isLastRetry := retryCount >= maxRetry
logger.Infof(ctx, "ProcessKnowledgeMove: task=%s, source=%s, target=%s, mode=%s, count=%d, retry=%d/%d",
payload.TaskID, payload.SourceKBID, payload.TargetKBID, payload.Mode, len(payload.KnowledgeIDs), retryCount, maxRetry)
// Helper function to handle errors - only mark as failed on last retry
handleError := func(progress *types.KnowledgeMoveProgress, err error, message string) {
if isLastRetry {
progress.Status = types.KBCloneStatusFailed
progress.Error = err.Error()
progress.Message = message
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKnowledgeMoveProgress(ctx, progress)
}
}
// Update progress to processing
progress := &types.KnowledgeMoveProgress{
TaskID: payload.TaskID,
SourceKBID: payload.SourceKBID,
TargetKBID: payload.TargetKBID,
Status: types.KBCloneStatusProcessing,
Total: len(payload.KnowledgeIDs),
Progress: 0,
Message: "Starting knowledge move...",
UpdatedAt: time.Now().Unix(),
}
_ = s.saveKnowledgeMoveProgress(ctx, progress)
// Get source and target knowledge bases
sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.SourceKBID)
if err != nil {
handleError(progress, err, "Failed to get source knowledge base")
return err
}
targetKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.TargetKBID)
if err != nil {
handleError(progress, err, "Failed to get target knowledge base")
return err
}
// Validate compatibility
if sourceKB.Type != targetKB.Type {
err := fmt.Errorf("type mismatch: source=%s, target=%s", sourceKB.Type, targetKB.Type)
handleError(progress, err, "Source and target knowledge bases must be the same type")
return err
}
if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID {
err := fmt.Errorf("embedding model mismatch: source=%s, target=%s", sourceKB.EmbeddingModelID, targetKB.EmbeddingModelID)
handleError(progress, err, "Source and target must use the same embedding model")
return err
}
// Process each knowledge item
for i, knowledgeID := range payload.KnowledgeIDs {
err := s.moveOneKnowledge(ctx, knowledgeID, sourceKB, targetKB, payload.Mode)
if err != nil {
logger.Errorf(ctx, "ProcessKnowledgeMove: failed to move knowledge %s: %v", knowledgeID, err)
progress.Failed++
}
progress.Processed = i + 1
if progress.Total > 0 {
progress.Progress = progress.Processed * 100 / progress.Total
}
progress.Message = fmt.Sprintf("Moved %d/%d knowledge items", progress.Processed, progress.Total)
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKnowledgeMoveProgress(ctx, progress)
}
// Mark as completed
if progress.Failed > 0 && progress.Failed == progress.Total {
progress.Status = types.KBCloneStatusFailed
progress.Message = fmt.Sprintf("Knowledge move failed: all %d items failed", progress.Total)
} else {
progress.Status = types.KBCloneStatusCompleted
progress.Message = fmt.Sprintf("Knowledge move completed: %d/%d succeeded", progress.Processed-progress.Failed, progress.Total)
}
progress.Progress = 100
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKnowledgeMoveProgress(ctx, progress)
logger.Infof(ctx, "ProcessKnowledgeMove: task=%s completed, processed=%d, failed=%d", payload.TaskID, progress.Processed, progress.Failed)
return nil
}
// moveOneKnowledge moves a single knowledge item from source KB to target KB.
func (s *knowledgeService) moveOneKnowledge(
ctx context.Context,
knowledgeID string,
sourceKB, targetKB *types.KnowledgeBase,
mode string,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// Get the knowledge item
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
return fmt.Errorf("failed to get knowledge %s: %w", knowledgeID, err)
}
// Only move completed items
if knowledge.ParseStatus != types.ParseStatusCompleted {
return fmt.Errorf("knowledge %s is not in completed status (current: %s)", knowledgeID, knowledge.ParseStatus)
}
// Mark as processing during move
knowledge.ParseStatus = types.ParseStatusProcessing
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
return fmt.Errorf("failed to mark knowledge as processing: %w", err)
}
switch mode {
case "reuse_vectors":
return s.moveKnowledgeReuseVectors(ctx, knowledge, sourceKB, targetKB)
case "reparse":
return s.moveKnowledgeReparse(ctx, knowledge, sourceKB, targetKB)
default:
return fmt.Errorf("unknown move mode: %s", mode)
}
}
// moveKnowledgeReuseVectors moves knowledge by copying vector indices and updating DB references.
func (s *knowledgeService) moveKnowledgeReuseVectors(
ctx context.Context,
knowledge *types.Knowledge,
sourceKB, targetKB *types.KnowledgeBase,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 1. Get old chunk IDs for vector index copy mapping
oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID)
if err != nil {
return fmt.Errorf("failed to list chunks: %w", err)
}
// Build identity mapping (same chunk IDs, just moving between KBs)
chunkIDMapping := make(map[string]string, len(oldChunks))
for _, c := range oldChunks {
chunkIDMapping[c.ID] = c.ID
}
// 2. Copy vector indices from source KB to target KB
if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" {
// reuse_vectors mode copies index entries directly between KBs, which
// only works inside the same VectorStore backend. Route through the
// source KB's binding; callers must reject cross-store moves before
// landing here.
var sourceStoreID *string
if sourceKB != nil {
sourceStoreID = sourceKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
if err != nil {
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID)
if err != nil {
return fmt.Errorf("failed to get embedding model: %w", err)
}
// Copy indices from source KB to target KB
knowledgeIDMapping := map[string]string{knowledge.ID: knowledge.ID}
if err := retrieveEngine.CopyIndices(ctx, sourceKB.ID, targetKB.ID,
knowledgeIDMapping, chunkIDMapping,
embeddingModel.GetDimensions(), sourceKB.Type,
); err != nil {
return fmt.Errorf("failed to copy indices: %w", err)
}
// Delete indices from source KB
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID},
embeddingModel.GetDimensions(), sourceKB.Type,
); err != nil {
logger.Warnf(ctx, "moveKnowledgeReuseVectors: failed to delete old indices for knowledge %s: %v", knowledge.ID, err)
// Non-fatal: indices will be orphaned but won't affect correctness
}
}
// 3. Update chunks' knowledge_base_id in DB
if err := s.chunkRepo.MoveChunksByKnowledgeID(ctx, tenantID, knowledge.ID, targetKB.ID); err != nil {
return fmt.Errorf("failed to move chunks: %w", err)
}
// 4. Update knowledge record
knowledge.KnowledgeBaseID = targetKB.ID
knowledge.TagID = "" // Clear tag since tags are KB-scoped
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
return fmt.Errorf("failed to update knowledge: %w", err)
}
return nil
}
// moveKnowledgeReparse moves knowledge to target KB and re-parses it with target KB's configuration.
func (s *knowledgeService) moveKnowledgeReparse(
ctx context.Context,
knowledge *types.Knowledge,
_, targetKB *types.KnowledgeBase,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 1. Clean up existing chunks and vector indices
if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
logger.Warnf(ctx, "moveKnowledgeReparse: cleanup partial error for knowledge %s: %v", knowledge.ID, err)
// Continue - partial cleanup is acceptable
}
// 2. Update knowledge to belong to target KB
knowledge.KnowledgeBaseID = targetKB.ID
knowledge.EmbeddingModelID = targetKB.EmbeddingModelID
knowledge.TagID = "" // Clear tag since tags are KB-scoped
knowledge.ParseStatus = types.ParseStatusPending
knowledge.EnableStatus = "disabled"
knowledge.Description = ""
knowledge.ProcessedAt = nil
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
return fmt.Errorf("failed to update knowledge: %w", err)
}
// 3. Enqueue document processing task with target KB's configuration
if knowledge.IsManual() {
meta, err := knowledge.ManualMetadata()
if err != nil || meta == nil {
return fmt.Errorf("failed to get manual metadata for reparse: %w", err)
}
s.triggerManualProcessing(ctx, targetKB, knowledge, meta.Content, false)
return nil
}
if knowledge.FilePath != "" {
enableMultimodel := targetKB.IsMultimodalEnabled()
enableQuestionGeneration := false
questionCount := 3
if targetKB.QuestionGenerationConfig != nil && targetKB.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if targetKB.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = targetKB.QuestionGenerationConfig.QuestionCount
}
}
lang, _ := types.LanguageFromContext(ctx)
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: targetKB.ID,
FilePath: knowledge.FilePath,
FileName: knowledge.FileName,
FileType: getFileType(knowledge.FileName),
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
Language: lang,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
return fmt.Errorf("failed to marshal document process payload: %w", err)
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...)
info, err := s.task.Enqueue(task)
if err != nil {
return fmt.Errorf("failed to enqueue document process task: %w", err)
}
logger.Infof(ctx, "moveKnowledgeReparse: enqueued reparse task id=%s for knowledge=%s", info.ID, knowledge.ID)
}
return nil
}
// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag.
// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB.
// If not found, it creates a new tag with the same properties.
// The mapping is cached in tagIDMapping for subsequent lookups.
func (s *knowledgeService) getOrCreateTagInTarget(
ctx context.Context,
srcTenantID, dstTenantID uint64,
dstKnowledgeBaseID string,
srcTagID string,
tagIDMapping map[string]string,
) string {
// Get source tag
srcTag, err := s.tagRepo.GetByID(ctx, srcTenantID, srcTagID)
if err != nil || srcTag == nil {
logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err)
tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups
return ""
}
// Try to find existing tag with same name in target KB
dstTag, err := s.tagRepo.GetByName(ctx, dstTenantID, dstKnowledgeBaseID, srcTag.Name)
if err == nil && dstTag != nil {
tagIDMapping[srcTagID] = dstTag.ID
return dstTag.ID
}
// Create new tag in target KB
// "未分类" tag should have the lowest sort order to appear first
sortOrder := srcTag.SortOrder
if srcTag.Name == types.UntaggedTagName {
sortOrder = -1
}
newTag := &types.KnowledgeTag{
ID: uuid.New().String(),
TenantID: dstTenantID,
KnowledgeBaseID: dstKnowledgeBaseID,
Name: srcTag.Name,
Color: srcTag.Color,
SortOrder: sortOrder,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
if err := s.tagRepo.Create(ctx, newTag); err != nil {
logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err)
tagIDMapping[srcTagID] = "" // Cache empty result
return ""
}
tagIDMapping[srcTagID] = newTag.ID
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dstKnowledgeBaseID)
return newTag.ID
}