fix(knowledge): delete extracted images from storage when knowledge is removed
Previously, only the original uploaded file was deleted via knowledge.FilePath. Images extracted by the builtin parser (stored as individual S3/MinIO/local files via ImageResolver) were never cleaned up, causing orphan files to accumulate. - Add ListImageInfoByKnowledgeIDs repository method (no chunk_type filter) - Add collectImageURLs utility with URL deduplication and JSON parse logging - Add deleteExtractedImages helper (standalone, best-effort with error logging) - Fix DeleteKnowledge: collect image URLs before errgroup, delete in file goroutine - Fix DeleteKnowledgeList: bulk collect with KB-level fileSvc mapping - Fix cleanupKnowledgeResources: collect before chunk deletion, delete after - Fix ProcessKBDelete: collect before chunk deletion for KB-wide cleanup
This commit is contained in:
@@ -425,6 +425,20 @@ func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantI
|
||||
).Delete(&types.Chunk{}).Error
|
||||
}
|
||||
|
||||
// ListImageInfoByKnowledgeIDs returns non-empty image_info values for the given knowledge IDs.
|
||||
// No chunk_type filter — collects from text, image_ocr, and image_caption chunks.
|
||||
func (r *chunkRepository) ListImageInfoByKnowledgeIDs(
|
||||
ctx context.Context, tenantID uint64, knowledgeIDs []string,
|
||||
) ([]interfaces.ChunkImageInfo, error) {
|
||||
var results []interfaces.ChunkImageInfo
|
||||
err := r.db.WithContext(ctx).
|
||||
Model(&types.Chunk{}).
|
||||
Select("knowledge_id, image_info").
|
||||
Where("tenant_id = ? AND knowledge_id IN ? AND image_info != ''", tenantID, knowledgeIDs).
|
||||
Scan(&results).Error
|
||||
return results, err
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeList deletes all chunks for a knowledge list
|
||||
func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error {
|
||||
return r.db.WithContext(ctx).Where(
|
||||
|
||||
@@ -1067,6 +1067,46 @@ func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Conte
|
||||
return types.NewPageResult(total, page, knowledges), nil
|
||||
}
|
||||
|
||||
// collectImageURLs extracts unique provider:// image URLs from image_info JSON strings.
|
||||
func collectImageURLs(ctx context.Context, imageInfos []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var urls []string
|
||||
for _, info := range imageInfos {
|
||||
if info == "" {
|
||||
continue
|
||||
}
|
||||
var images []*types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(info), &images); err != nil {
|
||||
logger.Warnf(ctx, "Failed to parse image_info JSON: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, img := range images {
|
||||
if img.URL != "" {
|
||||
if _, exists := seen[img.URL]; !exists {
|
||||
seen[img.URL] = struct{}{}
|
||||
urls = append(urls, img.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// deleteExtractedImages deletes all extracted image files from storage.
|
||||
// Standalone function — callable from both knowledgeService and knowledgeBaseService.
|
||||
// Errors are logged but do not fail the overall deletion.
|
||||
func deleteExtractedImages(ctx context.Context, fileSvc interfaces.FileService, imageURLs []string) {
|
||||
if len(imageURLs) == 0 {
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Deleting %d extracted images", len(imageURLs))
|
||||
for _, url := range imageURLs {
|
||||
if err := fileSvc.DeleteFile(ctx, url); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete extracted image %s: %v", url, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteKnowledge deletes a knowledge entry and all related resources
|
||||
func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error {
|
||||
// Get the knowledge entry
|
||||
@@ -1091,6 +1131,18 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
|
||||
kbFileSvc := s.resolveFileService(ctx, kb)
|
||||
|
||||
// Collect image URLs before chunks are deleted (ImageInfo references are lost after deletion)
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantID, []string{id})
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to collect image URLs for cleanup: %v", err)
|
||||
}
|
||||
var imageInfoStrs []string
|
||||
for _, ci := range chunkImageInfos {
|
||||
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
|
||||
}
|
||||
imageURLs := collectImageURLs(ctx, imageInfoStrs)
|
||||
|
||||
wg := errgroup.Group{}
|
||||
// Delete knowledge embeddings from vector store
|
||||
wg.Go(func() error {
|
||||
@@ -1124,13 +1176,14 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
return nil
|
||||
})
|
||||
|
||||
// Delete the physical file if it exists
|
||||
// Delete the physical file and extracted images if they exist
|
||||
wg.Go(func() error {
|
||||
if knowledge.FilePath != "" {
|
||||
if err := kbFileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed")
|
||||
}
|
||||
}
|
||||
deleteExtractedImages(ctx, kbFileSvc, imageURLs)
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
tenantInfo.StorageUsed -= knowledge.StorageSize
|
||||
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil {
|
||||
@@ -1189,6 +1242,25 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
}
|
||||
}
|
||||
|
||||
// Collect image URLs before chunks are deleted
|
||||
chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, ids)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to collect image URLs for batch cleanup: %v", err)
|
||||
}
|
||||
knowledgeToKB := make(map[string]string)
|
||||
for _, k := range knowledgeList {
|
||||
knowledgeToKB[k.ID] = k.KnowledgeBaseID
|
||||
}
|
||||
kbImageInfos := make(map[string][]string) // kbID → []imageInfo JSON
|
||||
for _, ci := range chunkImageInfos {
|
||||
kbID := knowledgeToKB[ci.KnowledgeID]
|
||||
kbImageInfos[kbID] = append(kbImageInfos[kbID], ci.ImageInfo)
|
||||
}
|
||||
kbImageURLs := make(map[string][]string) // kbID → []imageURL (deduplicated)
|
||||
for kbID, infos := range kbImageInfos {
|
||||
kbImageURLs[kbID] = collectImageURLs(ctx, infos)
|
||||
}
|
||||
|
||||
wg := errgroup.Group{}
|
||||
// 2. Delete knowledge embeddings from vector store
|
||||
wg.Go(func() error {
|
||||
@@ -1236,7 +1308,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
return nil
|
||||
})
|
||||
|
||||
// 4. Delete the physical file if it exists
|
||||
// 4. Delete the physical file and extracted images if they exist
|
||||
wg.Go(func() error {
|
||||
storageAdjust := int64(0)
|
||||
for _, knowledge := range knowledgeList {
|
||||
@@ -1248,6 +1320,15 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
}
|
||||
storageAdjust -= knowledge.StorageSize
|
||||
}
|
||||
// Delete extracted images per KB
|
||||
for kbID, urls := range kbImageURLs {
|
||||
fSvc := kbFileServices[kbID]
|
||||
if fSvc == nil {
|
||||
logger.Warnf(ctx, "No file service for KB %s, skipping %d image deletions", kbID, len(urls))
|
||||
continue
|
||||
}
|
||||
deleteExtractedImages(ctx, fSvc, urls)
|
||||
}
|
||||
tenantInfo.StorageUsed += storageAdjust
|
||||
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed")
|
||||
@@ -7024,11 +7105,28 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
|
||||
}
|
||||
}
|
||||
|
||||
// Collect image URLs before chunks are deleted
|
||||
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
|
||||
fileSvc := s.resolveFileService(ctx, kb)
|
||||
chunkImageInfos, imgErr := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, []string{knowledge.ID})
|
||||
if imgErr != nil {
|
||||
logger.GetLogger(ctx).WithField("error", imgErr).Error("Failed to collect image URLs for cleanup")
|
||||
cleanupErr = errors.Join(cleanupErr, imgErr)
|
||||
}
|
||||
var imageInfoStrs []string
|
||||
for _, ci := range chunkImageInfos {
|
||||
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
|
||||
}
|
||||
imageURLs := collectImageURLs(ctx, imageInfoStrs)
|
||||
|
||||
if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge chunks")
|
||||
cleanupErr = errors.Join(cleanupErr, err)
|
||||
}
|
||||
|
||||
// Delete extracted images after chunks are deleted
|
||||
deleteExtractedImages(ctx, fileSvc, imageURLs)
|
||||
|
||||
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
|
||||
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge graph data")
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCollectImageURLs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
imageInfos []string
|
||||
wantURLs []string
|
||||
}{
|
||||
{
|
||||
name: "empty input",
|
||||
imageInfos: nil,
|
||||
wantURLs: nil,
|
||||
},
|
||||
{
|
||||
name: "empty strings",
|
||||
imageInfos: []string{"", ""},
|
||||
wantURLs: nil,
|
||||
},
|
||||
{
|
||||
name: "single image",
|
||||
imageInfos: []string{
|
||||
`[{"url":"s3://bucket/prefix/123/exports/abc.png","original_url":"https://example.com/img.png"}]`,
|
||||
},
|
||||
wantURLs: []string{"s3://bucket/prefix/123/exports/abc.png"},
|
||||
},
|
||||
{
|
||||
name: "multiple images in one chunk",
|
||||
imageInfos: []string{
|
||||
`[{"url":"s3://bucket/a.png"},{"url":"s3://bucket/b.png"}]`,
|
||||
},
|
||||
wantURLs: []string{"s3://bucket/a.png", "s3://bucket/b.png"},
|
||||
},
|
||||
{
|
||||
name: "dedup across chunks — parent and child share same URL",
|
||||
imageInfos: []string{
|
||||
`[{"url":"s3://bucket/same.png"}]`,
|
||||
`[{"url":"s3://bucket/same.png"}]`,
|
||||
`[{"url":"s3://bucket/other.png"}]`,
|
||||
},
|
||||
wantURLs: []string{"s3://bucket/same.png", "s3://bucket/other.png"},
|
||||
},
|
||||
{
|
||||
name: "skip empty URL field",
|
||||
imageInfos: []string{
|
||||
`[{"url":"","original_url":"https://example.com/img.png"}]`,
|
||||
},
|
||||
wantURLs: nil,
|
||||
},
|
||||
{
|
||||
name: "invalid JSON — skipped with no panic",
|
||||
imageInfos: []string{
|
||||
`not valid json`,
|
||||
`[{"url":"s3://bucket/valid.png"}]`,
|
||||
},
|
||||
wantURLs: []string{"s3://bucket/valid.png"},
|
||||
},
|
||||
{
|
||||
name: "mixed providers",
|
||||
imageInfos: []string{
|
||||
`[{"url":"s3://bucket/img1.png"}]`,
|
||||
`[{"url":"minio://bucket/img2.jpg"}]`,
|
||||
`[{"url":"local://data/img3.webp"}]`,
|
||||
},
|
||||
wantURLs: []string{
|
||||
"s3://bucket/img1.png",
|
||||
"minio://bucket/img2.jpg",
|
||||
"local://data/img3.webp",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := collectImageURLs(ctx, tt.imageInfos)
|
||||
|
||||
if len(got) != len(tt.wantURLs) {
|
||||
t.Fatalf("collectImageURLs() returned %d URLs, want %d\ngot: %v\nwant: %v",
|
||||
len(got), len(tt.wantURLs), got, tt.wantURLs)
|
||||
}
|
||||
for i, url := range got {
|
||||
if url != tt.wantURLs[i] {
|
||||
t.Errorf("collectImageURLs()[%d] = %q, want %q", i, url, tt.wantURLs[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -453,6 +453,17 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
}
|
||||
}
|
||||
|
||||
// Collect image URLs before chunks are deleted
|
||||
chunkImageInfos, imgErr := s.chunkRepo.ListImageInfoByKnowledgeIDs(ctx, tenantID, knowledgeIDs)
|
||||
if imgErr != nil {
|
||||
logger.Warnf(ctx, "Failed to collect image URLs for KB delete: %v", imgErr)
|
||||
}
|
||||
var imageInfoStrs []string
|
||||
for _, ci := range chunkImageInfos {
|
||||
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
|
||||
}
|
||||
imageURLs := collectImageURLs(ctx, imageInfoStrs)
|
||||
|
||||
// Delete all chunks
|
||||
logger.Infof(ctx, "Deleting all chunks in knowledge base")
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
@@ -461,8 +472,8 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
}
|
||||
}
|
||||
|
||||
// Delete physical files and adjust storage
|
||||
logger.Infof(ctx, "Deleting physical files")
|
||||
// Delete physical files, extracted images, and adjust storage
|
||||
logger.Infof(ctx, "Deleting physical files and extracted images")
|
||||
storageAdjust := int64(0)
|
||||
for _, knowledge := range knowledgeList {
|
||||
if knowledge.FilePath != "" {
|
||||
@@ -472,6 +483,7 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
}
|
||||
storageAdjust -= knowledge.StorageSize
|
||||
}
|
||||
deleteExtractedImages(ctx, s.fileSvc, imageURLs)
|
||||
if storageAdjust != 0 {
|
||||
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantID, storageAdjust); err != nil {
|
||||
logger.Warnf(ctx, "Failed to adjust tenant storage: %v", err)
|
||||
|
||||
@@ -6,6 +6,12 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ChunkImageInfo holds (knowledge_id, image_info) pairs for image cleanup before chunk deletion.
|
||||
type ChunkImageInfo struct {
|
||||
KnowledgeID string `gorm:"column:knowledge_id"`
|
||||
ImageInfo string `gorm:"column:image_info"`
|
||||
}
|
||||
|
||||
// ChunkRepository defines the interface for chunk repository operations
|
||||
type ChunkRepository interface {
|
||||
// CreateChunks creates chunks
|
||||
@@ -58,6 +64,8 @@ type ChunkRepository interface {
|
||||
DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) error
|
||||
// DeleteByKnowledgeList deletes all chunks for a knowledge list
|
||||
DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error
|
||||
// ListImageInfoByKnowledgeIDs returns non-empty (knowledge_id, image_info) pairs for image cleanup.
|
||||
ListImageInfoByKnowledgeIDs(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]ChunkImageInfo, error)
|
||||
// MoveChunksByKnowledgeID updates knowledge_base_id for all chunks of a knowledge item
|
||||
MoveChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string, targetKBID string) error
|
||||
// DeleteChunksByTagID deletes all chunks with the specified tag ID
|
||||
|
||||
Reference in New Issue
Block a user