e9980c6011
Cloning a knowledge base previously copied only the storage path strings (knowledge.FilePath and chunk.ImageInfo.URL), so the source and the clone shared the same physical objects in the storage backend. Once the original file and extracted images are deleted on source removal, the clone is left with dangling references and its document and images become unreadable — data loss that occurs even for same-store clones. Add a CopyFile primitive to the FileService interface and implement it in every backend: server-side CopyObject on the object stores (s3/obs/cos/oss/tos/ks3/minio), io.Copy on local, and a no-op on dummy. Destinations use the knowledge-owned layout and reuse the existing path/object-key guards; a sentinel ErrCrossBackendCopy is returned when the source scheme does not match the backend. Use CopyFile to deep-copy the document file in cloneKnowledge and the extracted images in CloneChunk and cloneFAQKnowledgeBase via a shared cloneChunkImageInfo helper that deduplicates identical image URLs per clone and rewrites them to the new objects. Copied objects are cleaned up best-effort if a clone fails partway through. A clone-time preflight rejects cloning into a target bound to a different storage backend when the tenant pins providers via StorageEngineConfig. Adds unit tests for local CopyFile (independent copy survives source deletion, traversal rejection, cross-backend rejection), cloneChunkImageInfo (empty/multi/dedup/parse-failure/OriginalURL handling), and the storage provider preflight.
1221 lines
43 KiB
Go
1221 lines
43 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
|
werrors "github.com/Tencent/WeKnora/internal/errors"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"github.com/google/uuid"
|
|
"github.com/hibiken/asynq"
|
|
"github.com/redis/go-redis/v9"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// copyOwnedObject performs a real copy of srcPath into a NEW object owned by
|
|
// (tenantID, knowledgeID) using the destination FileService, returning the new
|
|
// provider:// path. The same-backend check lives inside dstSvc.CopyFile, which
|
|
// returns file.ErrCrossBackendCopy when srcPath belongs to a different provider;
|
|
// that error is propagated unchanged so callers can fail the clone explicitly.
|
|
// srcSvc is accepted for symmetry with the read side but is not used directly:
|
|
// server-side copies are issued by the destination service.
|
|
func copyOwnedObject(
|
|
ctx context.Context,
|
|
srcSvc, dstSvc interfaces.FileService,
|
|
srcPath string,
|
|
tenantID uint64,
|
|
knowledgeID string,
|
|
) (string, error) {
|
|
_ = srcSvc // reserved for future cross-backend streaming fallback
|
|
return dstSvc.CopyFile(ctx, srcPath, tenantID, knowledgeID)
|
|
}
|
|
|
|
// cloneChunkImageInfo parses a chunk's image_info JSON, copies every referenced
|
|
// object into a NEW object owned by (tenantID, knowledgeID), and returns the
|
|
// re-serialized image_info plus the list of newly-created object URLs (for
|
|
// rollback on failure). urlCache dedups identical source objects across chunks
|
|
// so the same source image is copied at most once per clone.
|
|
//
|
|
// An empty srcImageInfo yields ("", nil, nil). A JSON parse failure returns an
|
|
// error (the clone fails) rather than silently inheriting the shared-reference
|
|
// bug. When an image's OriginalURL points at the same object as its URL (the
|
|
// common case for extracted images), OriginalURL is rewritten to the new path
|
|
// too; an OriginalURL from a different/external source is preserved.
|
|
func cloneChunkImageInfo(
|
|
ctx context.Context,
|
|
dstSvc interfaces.FileService,
|
|
srcImageInfo string,
|
|
tenantID uint64,
|
|
knowledgeID string,
|
|
urlCache map[string]string,
|
|
) (newImageInfo string, copiedURLs []string, err error) {
|
|
if srcImageInfo == "" {
|
|
return "", nil, nil
|
|
}
|
|
|
|
var images []*types.ImageInfo
|
|
if err := json.Unmarshal([]byte(srcImageInfo), &images); err != nil {
|
|
return "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err)
|
|
}
|
|
|
|
for _, img := range images {
|
|
if img == nil || img.URL == "" {
|
|
continue
|
|
}
|
|
originalMatchedURL := img.OriginalURL == img.URL
|
|
|
|
newURL, cached := urlCache[img.URL]
|
|
if !cached {
|
|
newURL, err = copyOwnedObject(ctx, dstSvc, dstSvc, img.URL, tenantID, knowledgeID)
|
|
if err != nil {
|
|
return "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err)
|
|
}
|
|
urlCache[img.URL] = newURL
|
|
copiedURLs = append(copiedURLs, newURL)
|
|
}
|
|
|
|
if originalMatchedURL {
|
|
img.OriginalURL = newURL
|
|
}
|
|
img.URL = newURL
|
|
}
|
|
|
|
out, err := json.Marshal(images)
|
|
if err != nil {
|
|
return "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err)
|
|
}
|
|
return string(out), copiedURLs, nil
|
|
}
|
|
|
|
// cleanupCopiedObjects deletes objects that were newly created during a clone
|
|
// that subsequently failed, to avoid orphaning storage. It is best-effort:
|
|
// delete errors are logged but never returned (the original clone error wins).
|
|
func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths []string) {
|
|
if len(paths) == 0 || svc == nil {
|
|
return
|
|
}
|
|
logger.Infof(ctx, "Cleaning up %d copied objects after clone failure", len(paths))
|
|
for _, p := range paths {
|
|
if p == "" {
|
|
continue
|
|
}
|
|
if err := svc.DeleteFile(ctx, p); err != nil {
|
|
logger.Errorf(ctx, "Failed to clean up copied object %s: %v", p, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error {
|
|
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
|
|
return err
|
|
}
|
|
|
|
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
|
return err
|
|
}
|
|
|
|
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
|
return err
|
|
}
|
|
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
|
|
|
|
batch := 10
|
|
g, gctx := errgroup.WithContext(ctx)
|
|
for ids := range slices.Chunk(delKnowledge, batch) {
|
|
g.Go(func() error {
|
|
err := s.DeleteKnowledgeList(gctx, ids)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
err = g.Wait()
|
|
if err != nil {
|
|
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
|
|
return err
|
|
}
|
|
|
|
// Copy context out of auto-stop task
|
|
g, gctx = errgroup.WithContext(ctx)
|
|
g.SetLimit(batch)
|
|
for _, knowledge := range addKnowledge {
|
|
g.Go(func() error {
|
|
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
|
|
return err
|
|
}
|
|
err = s.cloneKnowledge(gctx, srcKn, dstKB)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
err = g.Wait()
|
|
if err != nil {
|
|
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CloneChunk clone chunks from one knowledge to another
|
|
// This method transfers a chunk from a source knowledge document to a target knowledge document
|
|
// It handles the creation of new chunks in the target knowledge and updates the vector database accordingly
|
|
// Parameters:
|
|
// - ctx: Context with authentication and request information
|
|
// - src: Source knowledge document containing the chunk to move
|
|
// - dst: Target knowledge document where the chunk will be moved
|
|
//
|
|
// Returns:
|
|
// - error: Any error encountered during the move operation
|
|
//
|
|
// This method handles the chunk transfer logic, including creating new chunks in the target knowledge
|
|
// and updating the vector database representation of the moved chunks.
|
|
// It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained
|
|
// by mapping the source chunk IDs to the new target chunk IDs.
|
|
func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) (err error) {
|
|
chunkPage := 1
|
|
chunkPageSize := 100
|
|
srcTodst := map[string]string{}
|
|
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
|
|
targetChunks := make([]*types.Chunk, 0, 10)
|
|
chunkType := []types.ChunkType{
|
|
types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary,
|
|
types.ChunkTypeImageCaption, types.ChunkTypeImageOCR,
|
|
}
|
|
|
|
// Resolve the destination FileService so extracted images can be copied
|
|
// into objects owned by the destination knowledge. urlCache dedups identical
|
|
// source images across chunks; copiedURLs accumulates new objects so they can
|
|
// be cleaned up if the clone fails partway through.
|
|
dstKB, dstKBErr := s.kbService.GetKnowledgeBaseByID(ctx, dst.KnowledgeBaseID)
|
|
if dstKBErr != nil {
|
|
return fmt.Errorf("failed to load destination knowledge base for image copy: %w", dstKBErr)
|
|
}
|
|
dstSvc := s.resolveFileService(ctx, dstKB)
|
|
urlCache := map[string]string{}
|
|
var copiedURLs []string
|
|
defer func() {
|
|
if err != nil {
|
|
cleanupCopiedObjects(ctx, dstSvc, copiedURLs)
|
|
}
|
|
}()
|
|
|
|
for {
|
|
sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx,
|
|
src.TenantID,
|
|
src.ID,
|
|
&types.Pagination{
|
|
Page: chunkPage,
|
|
PageSize: chunkPageSize,
|
|
},
|
|
chunkType,
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
)
|
|
chunkPage++
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(sourceChunks) == 0 {
|
|
break
|
|
}
|
|
now := time.Now()
|
|
for _, sourceChunk := range sourceChunks {
|
|
// Map TagID to target knowledge base
|
|
targetTagID := ""
|
|
if sourceChunk.TagID != "" {
|
|
if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok {
|
|
targetTagID = mappedTagID
|
|
} else {
|
|
// Try to find or create the tag in target knowledge base
|
|
targetTagID = s.getOrCreateTagInTarget(ctx, src.TenantID, dst.TenantID, dst.KnowledgeBaseID, sourceChunk.TagID, tagIDMapping)
|
|
}
|
|
}
|
|
|
|
// Deep-copy extracted images into objects owned by the destination
|
|
// knowledge so deleting the source never breaks this clone.
|
|
newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
|
ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache)
|
|
if copyErr != nil {
|
|
err = fmt.Errorf("clone chunk image copy failed: %w", copyErr)
|
|
return err
|
|
}
|
|
copiedURLs = append(copiedURLs, copied...)
|
|
|
|
targetChunk := &types.Chunk{
|
|
ID: uuid.New().String(),
|
|
TenantID: dst.TenantID,
|
|
KnowledgeID: dst.ID,
|
|
KnowledgeBaseID: dst.KnowledgeBaseID,
|
|
TagID: targetTagID,
|
|
Content: sourceChunk.Content,
|
|
ChunkIndex: sourceChunk.ChunkIndex,
|
|
IsEnabled: sourceChunk.IsEnabled,
|
|
Flags: sourceChunk.Flags,
|
|
Status: sourceChunk.Status,
|
|
StartAt: sourceChunk.StartAt,
|
|
EndAt: sourceChunk.EndAt,
|
|
PreChunkID: sourceChunk.PreChunkID,
|
|
NextChunkID: sourceChunk.NextChunkID,
|
|
ChunkType: sourceChunk.ChunkType,
|
|
ParentChunkID: sourceChunk.ParentChunkID,
|
|
Metadata: sourceChunk.Metadata,
|
|
ContentHash: sourceChunk.ContentHash,
|
|
ImageInfo: newImageInfo,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
targetChunks = append(targetChunks, targetChunk)
|
|
srcTodst[sourceChunk.ID] = targetChunk.ID
|
|
}
|
|
}
|
|
for _, targetChunk := range targetChunks {
|
|
if val, ok := srcTodst[targetChunk.PreChunkID]; ok {
|
|
targetChunk.PreChunkID = val
|
|
} else {
|
|
targetChunk.PreChunkID = ""
|
|
}
|
|
if val, ok := srcTodst[targetChunk.NextChunkID]; ok {
|
|
targetChunk.NextChunkID = val
|
|
} else {
|
|
targetChunk.NextChunkID = ""
|
|
}
|
|
if val, ok := srcTodst[targetChunk.ParentChunkID]; ok {
|
|
targetChunk.ParentChunkID = val
|
|
} else {
|
|
targetChunk.ParentChunkID = ""
|
|
}
|
|
}
|
|
for chunks := range slices.Chunk(targetChunks, chunkPageSize) {
|
|
err := s.chunkRepo.CreateChunks(ctx, chunks)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
tenantID := types.MustTenantIDFromContext(ctx)
|
|
// Route CopyIndices via the source KB's bound store. This function does
|
|
// not handle cross-store copies — embeddings written by different
|
|
// VectorStore backends are not bit-compatible, so callers that allow
|
|
// source/target KBs to bind to different stores must perform their own
|
|
// cross-store migration before invoking this.
|
|
var sourceStoreID *string
|
|
if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil {
|
|
sourceStoreID = srcKB.VectorStoreID
|
|
}
|
|
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
|
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID,
|
|
map[string]string{src.ID: dst.ID},
|
|
srcTodst,
|
|
embeddingModel.GetDimensions(),
|
|
dst.Type,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const (
|
|
kbCloneProgressKeyPrefix = "kb_clone_progress:"
|
|
kbCloneProgressTTL = 24 * time.Hour
|
|
)
|
|
|
|
// getKBCloneProgressKey returns the Redis key for storing KB clone progress
|
|
func getKBCloneProgressKey(taskID string) string {
|
|
return kbCloneProgressKeyPrefix + taskID
|
|
}
|
|
|
|
// ProcessKBClone handles Asynq knowledge base clone tasks
|
|
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
|
|
var payload types.KBClonePayload
|
|
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
|
return fmt.Errorf("failed to unmarshal KB clone payload: %w", err)
|
|
}
|
|
|
|
// Add tenant ID to context
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
|
|
|
// Get tenant info and add to context
|
|
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
|
|
return fmt.Errorf("failed to get tenant info: %w", err)
|
|
}
|
|
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
|
|
|
// Check if this is the last retry
|
|
retryCount, _ := asynq.GetRetryCount(ctx)
|
|
maxRetry, _ := asynq.GetMaxRetry(ctx)
|
|
isLastRetry := retryCount >= maxRetry
|
|
|
|
logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d",
|
|
payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry)
|
|
|
|
// Helper function to handle errors - only mark as failed on last retry
|
|
handleError := func(progress *types.KBCloneProgress, err error, message string) {
|
|
if isLastRetry {
|
|
progress.Status = types.KBCloneStatusFailed
|
|
progress.Error = err.Error()
|
|
progress.Message = message
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
}
|
|
}
|
|
|
|
// Update progress to processing
|
|
progress := &types.KBCloneProgress{
|
|
TaskID: payload.TaskID,
|
|
SourceID: payload.SourceID,
|
|
TargetID: payload.TargetID,
|
|
Status: types.KBCloneStatusProcessing,
|
|
Progress: 0,
|
|
Message: "Starting knowledge base clone...",
|
|
UpdatedAt: time.Now().Unix(),
|
|
}
|
|
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
|
logger.Errorf(ctx, "Failed to update KB clone progress: %v", err)
|
|
}
|
|
|
|
// Get source and target knowledge bases
|
|
srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
|
|
handleError(progress, err, "Failed to copy knowledge base configuration")
|
|
return err
|
|
}
|
|
|
|
// Use different sync strategies based on knowledge base type
|
|
if srcKB.Type == types.KnowledgeBaseTypeFAQ {
|
|
return s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError)
|
|
}
|
|
|
|
// Document type: use Knowledge-level diff based on file_hash
|
|
addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get knowledge to add: %v", err)
|
|
handleError(progress, err, "Failed to calculate knowledge difference")
|
|
return err
|
|
}
|
|
|
|
delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err)
|
|
handleError(progress, err, "Failed to calculate knowledge difference")
|
|
return err
|
|
}
|
|
|
|
totalOperations := len(addKnowledge) + len(delKnowledge)
|
|
progress.Total = totalOperations
|
|
progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
|
|
logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))
|
|
|
|
processedCount := 0
|
|
batch := 10
|
|
|
|
// Delete knowledge in target that doesn't exist in source
|
|
g, gctx := errgroup.WithContext(ctx)
|
|
for ids := range slices.Chunk(delKnowledge, batch) {
|
|
g.Go(func() error {
|
|
err := s.DeleteKnowledgeList(gctx, ids)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
|
|
handleError(progress, err, "Failed to delete knowledge")
|
|
return err
|
|
}
|
|
|
|
processedCount += len(delKnowledge)
|
|
if totalOperations > 0 {
|
|
progress.Progress = processedCount * 100 / totalOperations
|
|
}
|
|
progress.Processed = processedCount
|
|
progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
|
|
// Clone knowledge from source to target
|
|
g, gctx = errgroup.WithContext(ctx)
|
|
g.SetLimit(batch)
|
|
for _, knowledge := range addKnowledge {
|
|
g.Go(func() error {
|
|
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
|
|
return err
|
|
}
|
|
err = s.cloneKnowledge(gctx, srcKn, dstKB)
|
|
if err != nil {
|
|
logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
|
|
return err
|
|
}
|
|
|
|
// Update progress
|
|
processedCount++
|
|
if totalOperations > 0 {
|
|
progress.Progress = processedCount * 100 / totalOperations
|
|
}
|
|
progress.Processed = processedCount
|
|
progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
|
|
return nil
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
|
|
handleError(progress, err, "Failed to clone knowledge")
|
|
return err
|
|
}
|
|
|
|
// Mark as completed
|
|
progress.Status = types.KBCloneStatusCompleted
|
|
progress.Progress = 100
|
|
progress.Processed = totalOperations
|
|
progress.Message = "Knowledge base clone completed successfully"
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
|
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
|
|
}
|
|
|
|
logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID)
|
|
return nil
|
|
}
|
|
|
|
// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync
|
|
func (s *knowledgeService) cloneFAQKnowledgeBase(
|
|
ctx context.Context,
|
|
srcKB, dstKB *types.KnowledgeBase,
|
|
progress *types.KBCloneProgress,
|
|
handleError func(*types.KBCloneProgress, error, string),
|
|
) (retErr error) {
|
|
// Deep-copy extracted FAQ images into objects owned by the destination KB.
|
|
// urlCache dedups identical source images across chunks; copiedURLs tracks
|
|
// new objects for best-effort cleanup if the clone fails partway through.
|
|
dstSvc := s.resolveFileService(ctx, dstKB)
|
|
imageURLCache := map[string]string{}
|
|
var copiedImageURLs []string
|
|
defer func() {
|
|
if retErr != nil {
|
|
cleanupCopiedObjects(ctx, dstSvc, copiedImageURLs)
|
|
}
|
|
}()
|
|
|
|
// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
|
|
srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err)
|
|
handleError(progress, err, "Failed to get source FAQ knowledge")
|
|
return err
|
|
}
|
|
if len(srcKnowledgeList) == 0 {
|
|
// Source has no FAQ knowledge, nothing to clone
|
|
progress.Status = types.KBCloneStatusCompleted
|
|
progress.Progress = 100
|
|
progress.Message = "Source FAQ knowledge base is empty"
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
return nil
|
|
}
|
|
srcKnowledge := srcKnowledgeList[0]
|
|
|
|
// Get chunk-level differences based on content_hash
|
|
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
|
|
handleError(progress, err, "Failed to calculate FAQ chunk difference")
|
|
return err
|
|
}
|
|
|
|
totalOperations := len(chunksToAdd) + len(chunksToDelete)
|
|
progress.Total = totalOperations
|
|
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
|
|
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
|
|
|
|
// If nothing to do, mark as completed
|
|
if totalOperations == 0 {
|
|
progress.Status = types.KBCloneStatusCompleted
|
|
progress.Progress = 100
|
|
progress.Message = "FAQ knowledge base is already in sync"
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
return nil
|
|
}
|
|
|
|
// Route the FAQ clone through the source KB's bound store. Same
|
|
// constraint as CloneChunk: callers must ensure source and target share
|
|
// the same VectorStore (cross-store FAQ clone is not handled here).
|
|
var sourceStoreID *string
|
|
if srcKB != nil {
|
|
sourceStoreID = srcKB.VectorStoreID
|
|
}
|
|
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
|
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceStoreID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
|
handleError(progress, err, "Failed to initialize retrieve engine")
|
|
return err
|
|
}
|
|
|
|
// Get embedding model
|
|
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
|
|
handleError(progress, err, "Failed to get embedding model")
|
|
return err
|
|
}
|
|
|
|
processedCount := 0
|
|
|
|
// Delete FAQ chunks that don't exist in source
|
|
if len(chunksToDelete) > 0 {
|
|
// Delete from vector store
|
|
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
|
|
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
|
|
handleError(progress, err, "Failed to delete FAQ entries from vector store")
|
|
return err
|
|
}
|
|
// Delete from database
|
|
if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil {
|
|
logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err)
|
|
handleError(progress, err, "Failed to delete FAQ entries from database")
|
|
return err
|
|
}
|
|
processedCount += len(chunksToDelete)
|
|
if totalOperations > 0 {
|
|
progress.Progress = processedCount * 100 / totalOperations
|
|
}
|
|
progress.Processed = processedCount
|
|
progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
}
|
|
|
|
// Get or create the FAQ knowledge entry in destination
|
|
dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err)
|
|
handleError(progress, err, "Failed to prepare FAQ knowledge entry")
|
|
return err
|
|
}
|
|
|
|
// Clone FAQ chunks from source to destination
|
|
batch := 50
|
|
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
|
|
for i := 0; i < len(chunksToAdd); i += batch {
|
|
end := i + batch
|
|
if end > len(chunksToAdd) {
|
|
end = len(chunksToAdd)
|
|
}
|
|
batchIDs := chunksToAdd[i:end]
|
|
|
|
// Get source chunks
|
|
srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err)
|
|
handleError(progress, err, "Failed to get source FAQ entries")
|
|
return err
|
|
}
|
|
|
|
// Create new chunks for destination
|
|
newChunks := make([]*types.Chunk, 0, len(srcChunks))
|
|
for _, srcChunk := range srcChunks {
|
|
// Map TagID to target knowledge base
|
|
targetTagID := ""
|
|
if srcChunk.TagID != "" {
|
|
if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok {
|
|
targetTagID = mappedTagID
|
|
} else {
|
|
// Try to find or create the tag in target knowledge base
|
|
targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping)
|
|
}
|
|
}
|
|
|
|
// Deep-copy extracted images into objects owned by the destination
|
|
// FAQ knowledge so deleting the source never breaks this clone.
|
|
newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
|
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
|
|
if copyErr != nil {
|
|
logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)
|
|
handleError(progress, copyErr, "Failed to copy FAQ entry images")
|
|
retErr = copyErr
|
|
return retErr
|
|
}
|
|
copiedImageURLs = append(copiedImageURLs, copied...)
|
|
|
|
newChunk := &types.Chunk{
|
|
ID: uuid.New().String(),
|
|
TenantID: dstKB.TenantID,
|
|
KnowledgeID: dstKnowledge.ID,
|
|
KnowledgeBaseID: dstKB.ID,
|
|
TagID: targetTagID,
|
|
Content: srcChunk.Content,
|
|
ChunkIndex: srcChunk.ChunkIndex,
|
|
IsEnabled: srcChunk.IsEnabled,
|
|
Flags: srcChunk.Flags,
|
|
ChunkType: types.ChunkTypeFAQ,
|
|
Metadata: srcChunk.Metadata,
|
|
ContentHash: srcChunk.ContentHash,
|
|
ImageInfo: newImageInfo,
|
|
Status: int(types.ChunkStatusStored), // Initially stored, will be indexed
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
newChunks = append(newChunks, newChunk)
|
|
}
|
|
|
|
// Save to database
|
|
if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil {
|
|
logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err)
|
|
handleError(progress, err, "Failed to create FAQ entries")
|
|
return err
|
|
}
|
|
|
|
// Index in vector store using existing method
|
|
// This will index standard question + similar questions based on FAQConfig
|
|
if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil {
|
|
logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err)
|
|
handleError(progress, err, "Failed to index FAQ entries")
|
|
return err
|
|
}
|
|
|
|
// Update chunk status to indexed
|
|
for _, chunk := range newChunks {
|
|
chunk.Status = int(types.ChunkStatusIndexed)
|
|
}
|
|
if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil {
|
|
logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err)
|
|
// Don't fail the whole operation for status update failure
|
|
}
|
|
|
|
processedCount += len(batchIDs)
|
|
if totalOperations > 0 {
|
|
progress.Progress = processedCount * 100 / totalOperations
|
|
}
|
|
progress.Processed = processedCount
|
|
progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd))
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKBCloneProgress(ctx, progress)
|
|
}
|
|
|
|
// Mark as completed
|
|
progress.Status = types.KBCloneStatusCompleted
|
|
progress.Progress = 100
|
|
progress.Processed = totalOperations
|
|
progress.Message = "FAQ knowledge base clone completed successfully"
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
if err := s.saveKBCloneProgress(ctx, progress); err != nil {
|
|
logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base
|
|
// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge
|
|
func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) {
|
|
// FAQ knowledge base should have exactly one Knowledge entry
|
|
knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(knowledgeList) > 0 {
|
|
return knowledgeList[0], nil
|
|
}
|
|
|
|
// Create a new FAQ knowledge entry, copying from source if available
|
|
knowledge := &types.Knowledge{
|
|
ID: uuid.New().String(),
|
|
TenantID: kb.TenantID,
|
|
KnowledgeBaseID: kb.ID,
|
|
Type: types.KnowledgeTypeFAQ,
|
|
Channel: types.ChannelWeb,
|
|
Title: "FAQ",
|
|
ParseStatus: "completed",
|
|
EnableStatus: "enabled",
|
|
EmbeddingModelID: kb.EmbeddingModelID,
|
|
}
|
|
|
|
// Copy additional fields from source knowledge if available
|
|
if srcKnowledge != nil {
|
|
knowledge.Title = srcKnowledge.Title
|
|
knowledge.Description = srcKnowledge.Description
|
|
knowledge.Source = srcKnowledge.Source
|
|
knowledge.Channel = srcKnowledge.Channel
|
|
knowledge.Metadata = srcKnowledge.Metadata
|
|
}
|
|
|
|
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
|
|
return nil, err
|
|
}
|
|
return knowledge, nil
|
|
}
|
|
|
|
// saveKBCloneProgress saves the KB clone progress to Redis
|
|
func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
|
|
key := getKBCloneProgressKey(progress.TaskID)
|
|
data, err := json.Marshal(progress)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal progress: %w", err)
|
|
}
|
|
return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err()
|
|
}
|
|
|
|
// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use)
|
|
func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
|
|
return s.saveKBCloneProgress(ctx, progress)
|
|
}
|
|
|
|
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
|
|
func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) {
|
|
key := getKBCloneProgressKey(taskID)
|
|
data, err := s.redisClient.Get(ctx, key).Bytes()
|
|
if err != nil {
|
|
if errors.Is(err, redis.Nil) {
|
|
return nil, werrors.NewNotFoundError("KB clone task not found")
|
|
}
|
|
return nil, fmt.Errorf("failed to get progress from Redis: %w", err)
|
|
}
|
|
|
|
var progress types.KBCloneProgress
|
|
if err := json.Unmarshal(data, &progress); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
|
|
}
|
|
return &progress, nil
|
|
}
|
|
|
|
const (
|
|
knowledgeMoveProgressKeyPrefix = "knowledge_move_progress:"
|
|
knowledgeMoveProgressTTL = 24 * time.Hour
|
|
)
|
|
|
|
func getKnowledgeMoveProgressKey(taskID string) string {
|
|
return knowledgeMoveProgressKeyPrefix + taskID
|
|
}
|
|
|
|
func (s *knowledgeService) saveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
|
|
key := getKnowledgeMoveProgressKey(progress.TaskID)
|
|
data, err := json.Marshal(progress)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal move progress: %w", err)
|
|
}
|
|
return s.redisClient.Set(ctx, key, data, knowledgeMoveProgressTTL).Err()
|
|
}
|
|
|
|
// SaveKnowledgeMoveProgress saves the knowledge move progress to Redis (public method for handler use)
|
|
func (s *knowledgeService) SaveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
|
|
return s.saveKnowledgeMoveProgress(ctx, progress)
|
|
}
|
|
|
|
// GetKnowledgeMoveProgress retrieves the progress of a knowledge move task
|
|
func (s *knowledgeService) GetKnowledgeMoveProgress(ctx context.Context, taskID string) (*types.KnowledgeMoveProgress, error) {
|
|
key := getKnowledgeMoveProgressKey(taskID)
|
|
data, err := s.redisClient.Get(ctx, key).Bytes()
|
|
if err != nil {
|
|
if errors.Is(err, redis.Nil) {
|
|
return nil, werrors.NewNotFoundError("Knowledge move task not found")
|
|
}
|
|
return nil, fmt.Errorf("failed to get move progress from Redis: %w", err)
|
|
}
|
|
|
|
var progress types.KnowledgeMoveProgress
|
|
if err := json.Unmarshal(data, &progress); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal move progress: %w", err)
|
|
}
|
|
return &progress, nil
|
|
}
|
|
|
|
// ProcessKnowledgeMove handles Asynq knowledge move tasks
|
|
func (s *knowledgeService) ProcessKnowledgeMove(ctx context.Context, t *asynq.Task) error {
|
|
var payload types.KnowledgeMovePayload
|
|
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
|
return fmt.Errorf("failed to unmarshal knowledge move payload: %w", err)
|
|
}
|
|
|
|
// Add tenant ID to context
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
|
|
|
// Get tenant info and add to context
|
|
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "ProcessKnowledgeMove: failed to get tenant info: %v", err)
|
|
return fmt.Errorf("failed to get tenant info: %w", err)
|
|
}
|
|
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
|
|
|
// Check if this is the last retry
|
|
retryCount, _ := asynq.GetRetryCount(ctx)
|
|
maxRetry, _ := asynq.GetMaxRetry(ctx)
|
|
isLastRetry := retryCount >= maxRetry
|
|
|
|
logger.Infof(ctx, "ProcessKnowledgeMove: task=%s, source=%s, target=%s, mode=%s, count=%d, retry=%d/%d",
|
|
payload.TaskID, payload.SourceKBID, payload.TargetKBID, payload.Mode, len(payload.KnowledgeIDs), retryCount, maxRetry)
|
|
|
|
// Helper function to handle errors - only mark as failed on last retry
|
|
handleError := func(progress *types.KnowledgeMoveProgress, err error, message string) {
|
|
if isLastRetry {
|
|
progress.Status = types.KBCloneStatusFailed
|
|
progress.Error = err.Error()
|
|
progress.Message = message
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKnowledgeMoveProgress(ctx, progress)
|
|
}
|
|
}
|
|
|
|
// Update progress to processing
|
|
progress := &types.KnowledgeMoveProgress{
|
|
TaskID: payload.TaskID,
|
|
SourceKBID: payload.SourceKBID,
|
|
TargetKBID: payload.TargetKBID,
|
|
Status: types.KBCloneStatusProcessing,
|
|
Total: len(payload.KnowledgeIDs),
|
|
Progress: 0,
|
|
Message: "Starting knowledge move...",
|
|
UpdatedAt: time.Now().Unix(),
|
|
}
|
|
_ = s.saveKnowledgeMoveProgress(ctx, progress)
|
|
|
|
// Get source and target knowledge bases
|
|
sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.SourceKBID)
|
|
if err != nil {
|
|
handleError(progress, err, "Failed to get source knowledge base")
|
|
return err
|
|
}
|
|
targetKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.TargetKBID)
|
|
if err != nil {
|
|
handleError(progress, err, "Failed to get target knowledge base")
|
|
return err
|
|
}
|
|
|
|
// Validate compatibility
|
|
if sourceKB.Type != targetKB.Type {
|
|
err := fmt.Errorf("type mismatch: source=%s, target=%s", sourceKB.Type, targetKB.Type)
|
|
handleError(progress, err, "Source and target knowledge bases must be the same type")
|
|
return err
|
|
}
|
|
if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID {
|
|
err := fmt.Errorf("embedding model mismatch: source=%s, target=%s", sourceKB.EmbeddingModelID, targetKB.EmbeddingModelID)
|
|
handleError(progress, err, "Source and target must use the same embedding model")
|
|
return err
|
|
}
|
|
|
|
// Process each knowledge item
|
|
for i, knowledgeID := range payload.KnowledgeIDs {
|
|
err := s.moveOneKnowledge(ctx, knowledgeID, sourceKB, targetKB, payload.Mode)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "ProcessKnowledgeMove: failed to move knowledge %s: %v", knowledgeID, err)
|
|
progress.Failed++
|
|
}
|
|
progress.Processed = i + 1
|
|
if progress.Total > 0 {
|
|
progress.Progress = progress.Processed * 100 / progress.Total
|
|
}
|
|
progress.Message = fmt.Sprintf("Moved %d/%d knowledge items", progress.Processed, progress.Total)
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKnowledgeMoveProgress(ctx, progress)
|
|
}
|
|
|
|
// Mark as completed
|
|
if progress.Failed > 0 && progress.Failed == progress.Total {
|
|
progress.Status = types.KBCloneStatusFailed
|
|
progress.Message = fmt.Sprintf("Knowledge move failed: all %d items failed", progress.Total)
|
|
} else {
|
|
progress.Status = types.KBCloneStatusCompleted
|
|
progress.Message = fmt.Sprintf("Knowledge move completed: %d/%d succeeded", progress.Processed-progress.Failed, progress.Total)
|
|
}
|
|
progress.Progress = 100
|
|
progress.UpdatedAt = time.Now().Unix()
|
|
_ = s.saveKnowledgeMoveProgress(ctx, progress)
|
|
|
|
logger.Infof(ctx, "ProcessKnowledgeMove: task=%s completed, processed=%d, failed=%d", payload.TaskID, progress.Processed, progress.Failed)
|
|
return nil
|
|
}
|
|
|
|
// moveOneKnowledge moves a single knowledge item from source KB to target KB.
|
|
func (s *knowledgeService) moveOneKnowledge(
|
|
ctx context.Context,
|
|
knowledgeID string,
|
|
sourceKB, targetKB *types.KnowledgeBase,
|
|
mode string,
|
|
) error {
|
|
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
|
|
|
// Get the knowledge item
|
|
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get knowledge %s: %w", knowledgeID, err)
|
|
}
|
|
|
|
// Only move completed items
|
|
if knowledge.ParseStatus != types.ParseStatusCompleted {
|
|
return fmt.Errorf("knowledge %s is not in completed status (current: %s)", knowledgeID, knowledge.ParseStatus)
|
|
}
|
|
|
|
// Mark as processing during move
|
|
knowledge.ParseStatus = types.ParseStatusProcessing
|
|
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
|
return fmt.Errorf("failed to mark knowledge as processing: %w", err)
|
|
}
|
|
|
|
switch mode {
|
|
case "reuse_vectors":
|
|
return s.moveKnowledgeReuseVectors(ctx, knowledge, sourceKB, targetKB)
|
|
case "reparse":
|
|
return s.moveKnowledgeReparse(ctx, knowledge, sourceKB, targetKB)
|
|
default:
|
|
return fmt.Errorf("unknown move mode: %s", mode)
|
|
}
|
|
}
|
|
|
|
// moveKnowledgeReuseVectors moves knowledge by copying vector indices and updating DB references.
|
|
func (s *knowledgeService) moveKnowledgeReuseVectors(
|
|
ctx context.Context,
|
|
knowledge *types.Knowledge,
|
|
sourceKB, targetKB *types.KnowledgeBase,
|
|
) error {
|
|
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
|
|
|
// 1. Get old chunk IDs for vector index copy mapping
|
|
oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to list chunks: %w", err)
|
|
}
|
|
|
|
// Build identity mapping (same chunk IDs, just moving between KBs)
|
|
chunkIDMapping := make(map[string]string, len(oldChunks))
|
|
for _, c := range oldChunks {
|
|
chunkIDMapping[c.ID] = c.ID
|
|
}
|
|
|
|
// 2. Copy vector indices from source KB to target KB
|
|
if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" {
|
|
// reuse_vectors mode copies index entries directly between KBs, which
|
|
// only works inside the same VectorStore backend. Route through the
|
|
// source KB's binding; callers must reject cross-store moves before
|
|
// landing here.
|
|
var sourceStoreID *string
|
|
if sourceKB != nil {
|
|
sourceStoreID = sourceKB.VectorStoreID
|
|
}
|
|
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
|
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
|
}
|
|
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get embedding model: %w", err)
|
|
}
|
|
|
|
// Copy indices from source KB to target KB
|
|
knowledgeIDMapping := map[string]string{knowledge.ID: knowledge.ID}
|
|
if err := retrieveEngine.CopyIndices(ctx, sourceKB.ID, targetKB.ID,
|
|
knowledgeIDMapping, chunkIDMapping,
|
|
embeddingModel.GetDimensions(), sourceKB.Type,
|
|
); err != nil {
|
|
return fmt.Errorf("failed to copy indices: %w", err)
|
|
}
|
|
|
|
// Delete indices from source KB
|
|
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID},
|
|
embeddingModel.GetDimensions(), sourceKB.Type,
|
|
); err != nil {
|
|
logger.Warnf(ctx, "moveKnowledgeReuseVectors: failed to delete old indices for knowledge %s: %v", knowledge.ID, err)
|
|
// Non-fatal: indices will be orphaned but won't affect correctness
|
|
}
|
|
}
|
|
|
|
// 3. Update chunks' knowledge_base_id in DB
|
|
if err := s.chunkRepo.MoveChunksByKnowledgeID(ctx, tenantID, knowledge.ID, targetKB.ID); err != nil {
|
|
return fmt.Errorf("failed to move chunks: %w", err)
|
|
}
|
|
|
|
// 4. Update knowledge record
|
|
knowledge.KnowledgeBaseID = targetKB.ID
|
|
knowledge.TagID = "" // Clear tag since tags are KB-scoped
|
|
knowledge.ParseStatus = types.ParseStatusCompleted
|
|
knowledge.UpdatedAt = time.Now()
|
|
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
|
return fmt.Errorf("failed to update knowledge: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// moveKnowledgeReparse moves knowledge to target KB and re-parses it with target KB's configuration.
|
|
func (s *knowledgeService) moveKnowledgeReparse(
|
|
ctx context.Context,
|
|
knowledge *types.Knowledge,
|
|
_, targetKB *types.KnowledgeBase,
|
|
) error {
|
|
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
|
|
|
// 1. Clean up existing chunks and vector indices
|
|
if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
|
|
logger.Warnf(ctx, "moveKnowledgeReparse: cleanup partial error for knowledge %s: %v", knowledge.ID, err)
|
|
// Continue - partial cleanup is acceptable
|
|
}
|
|
|
|
// 2. Update knowledge to belong to target KB
|
|
knowledge.KnowledgeBaseID = targetKB.ID
|
|
knowledge.EmbeddingModelID = targetKB.EmbeddingModelID
|
|
knowledge.TagID = "" // Clear tag since tags are KB-scoped
|
|
knowledge.ParseStatus = types.ParseStatusPending
|
|
knowledge.EnableStatus = "disabled"
|
|
knowledge.Description = ""
|
|
knowledge.ProcessedAt = nil
|
|
knowledge.UpdatedAt = time.Now()
|
|
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
|
return fmt.Errorf("failed to update knowledge: %w", err)
|
|
}
|
|
|
|
// 3. Enqueue document processing task with target KB's configuration
|
|
if knowledge.IsManual() {
|
|
meta, err := knowledge.ManualMetadata()
|
|
if err != nil || meta == nil {
|
|
return fmt.Errorf("failed to get manual metadata for reparse: %w", err)
|
|
}
|
|
s.triggerManualProcessing(ctx, targetKB, knowledge, meta.Content, false)
|
|
return nil
|
|
}
|
|
|
|
if knowledge.FilePath != "" {
|
|
enableMultimodel := targetKB.IsMultimodalEnabled()
|
|
enableQuestionGeneration := false
|
|
questionCount := 3
|
|
if targetKB.QuestionGenerationConfig != nil && targetKB.QuestionGenerationConfig.Enabled {
|
|
enableQuestionGeneration = true
|
|
if targetKB.QuestionGenerationConfig.QuestionCount > 0 {
|
|
questionCount = targetKB.QuestionGenerationConfig.QuestionCount
|
|
}
|
|
}
|
|
|
|
lang, _ := types.LanguageFromContext(ctx)
|
|
taskPayload := types.DocumentProcessPayload{
|
|
TenantID: tenantID,
|
|
KnowledgeID: knowledge.ID,
|
|
KnowledgeBaseID: targetKB.ID,
|
|
FilePath: knowledge.FilePath,
|
|
FileName: knowledge.FileName,
|
|
FileType: getFileType(knowledge.FileName),
|
|
EnableMultimodel: enableMultimodel,
|
|
EnableQuestionGeneration: enableQuestionGeneration,
|
|
QuestionCount: questionCount,
|
|
Language: lang,
|
|
}
|
|
|
|
langfuse.InjectTracing(ctx, &taskPayload)
|
|
payloadBytes, err := json.Marshal(taskPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal document process payload: %w", err)
|
|
}
|
|
|
|
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes,
|
|
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...)
|
|
info, err := s.task.Enqueue(task)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to enqueue document process task: %w", err)
|
|
}
|
|
logger.Infof(ctx, "moveKnowledgeReparse: enqueued reparse task id=%s for knowledge=%s", info.ID, knowledge.ID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag.
|
|
// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB.
|
|
// If not found, it creates a new tag with the same properties.
|
|
// The mapping is cached in tagIDMapping for subsequent lookups.
|
|
func (s *knowledgeService) getOrCreateTagInTarget(
|
|
ctx context.Context,
|
|
srcTenantID, dstTenantID uint64,
|
|
dstKnowledgeBaseID string,
|
|
srcTagID string,
|
|
tagIDMapping map[string]string,
|
|
) string {
|
|
// Get source tag
|
|
srcTag, err := s.tagRepo.GetByID(ctx, srcTenantID, srcTagID)
|
|
if err != nil || srcTag == nil {
|
|
logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err)
|
|
tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups
|
|
return ""
|
|
}
|
|
|
|
// Try to find existing tag with same name in target KB
|
|
dstTag, err := s.tagRepo.GetByName(ctx, dstTenantID, dstKnowledgeBaseID, srcTag.Name)
|
|
if err == nil && dstTag != nil {
|
|
tagIDMapping[srcTagID] = dstTag.ID
|
|
return dstTag.ID
|
|
}
|
|
|
|
// Create new tag in target KB
|
|
// "未分类" tag should have the lowest sort order to appear first
|
|
sortOrder := srcTag.SortOrder
|
|
if srcTag.Name == types.UntaggedTagName {
|
|
sortOrder = -1
|
|
}
|
|
newTag := &types.KnowledgeTag{
|
|
ID: uuid.New().String(),
|
|
TenantID: dstTenantID,
|
|
KnowledgeBaseID: dstKnowledgeBaseID,
|
|
Name: srcTag.Name,
|
|
Color: srcTag.Color,
|
|
SortOrder: sortOrder,
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
if err := s.tagRepo.Create(ctx, newTag); err != nil {
|
|
logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err)
|
|
tagIDMapping[srcTagID] = "" // Cache empty result
|
|
return ""
|
|
}
|
|
|
|
tagIDMapping[srcTagID] = newTag.ID
|
|
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dstKnowledgeBaseID)
|
|
return newTag.ID
|
|
}
|