fix(knowledge): delete extracted images from storage when knowledge is removed

Previously, only the original uploaded file was deleted via knowledge.FilePath.
Images extracted by the builtin parser (stored as individual S3/MinIO/local files
via ImageResolver) were never cleaned up, causing orphan files to accumulate.

- Add ListImageInfoByKnowledgeIDs repository method (no chunk_type filter)
- Add collectImageURLs utility with URL deduplication and JSON parse logging
- Add deleteExtractedImages helper (standalone, best-effort with error logging)
- Fix DeleteKnowledge: collect image URLs before errgroup, delete in file goroutine
- Fix DeleteKnowledgeList: bulk collect with KB-level fileSvc mapping
- Fix cleanupKnowledgeResources: collect before chunk deletion, delete after
- Fix ProcessKBDelete: collect before chunk deletion for KB-wide cleanup
This commit is contained in:
ochan.kwon
2026-04-01 22:45:23 +09:00
committed by lyingbug
parent 8a375749f1
commit 33e919cc80
5 changed files with 230 additions and 4 deletions
+14
View File
@@ -425,6 +425,20 @@ func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantI
).Delete(&types.Chunk{}).Error
}
// ListImageInfoByKnowledgeIDs returns non-empty image_info values for the given knowledge IDs.
// No chunk_type filter — collects from text, image_ocr, and image_caption chunks.
func (r *chunkRepository) ListImageInfoByKnowledgeIDs(
ctx context.Context, tenantID uint64, knowledgeIDs []string,
) ([]interfaces.ChunkImageInfo, error) {
var results []interfaces.ChunkImageInfo
err := r.db.WithContext(ctx).
Model(&types.Chunk{}).
Select("knowledge_id, image_info").
Where("tenant_id = ? AND knowledge_id IN ? AND image_info != ''", tenantID, knowledgeIDs).
Scan(&results).Error
return results, err
}
// DeleteByKnowledgeList deletes all chunks for a knowledge list
func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error {
return r.db.WithContext(ctx).Where(
+100 -2
View File
@@ -1067,6 +1067,46 @@ func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Conte
return types.NewPageResult(total, page, knowledges), nil
}
// collectImageURLs extracts unique provider:// image URLs from image_info JSON strings.
func collectImageURLs(ctx context.Context, imageInfos []string) []string {
seen := make(map[string]struct{})
var urls []string
for _, info := range imageInfos {
if info == "" {
continue
}
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(info), &images); err != nil {
logger.Warnf(ctx, "Failed to parse image_info JSON: %v", err)
continue
}
for _, img := range images {
if img.URL != "" {
if _, exists := seen[img.URL]; !exists {
seen[img.URL] = struct{}{}
urls = append(urls, img.URL)
}
}
}
}
return urls
}
// deleteExtractedImages deletes all extracted image files from storage.
// Standalone function — callable from both knowledgeService and knowledgeBaseService.
// Errors are logged but do not fail the overall deletion.
func deleteExtractedImages(ctx context.Context, fileSvc interfaces.FileService, imageURLs []string) {
if len(imageURLs) == 0 {
return
}
logger.Infof(ctx, "Deleting %d extracted images", len(imageURLs))
for _, url := range imageURLs {
if err := fileSvc.DeleteFile(ctx, url); err != nil {
logger.Errorf(ctx, "Failed to delete extracted image %s: %v", url, err)
}
}
}
// DeleteKnowledge deletes a knowledge entry and all related resources
func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error {
// Get the knowledge entry
@@ -1091,6 +1131,18 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
kbFileSvc := s.resolveFileService(ctx, kb)
// Collect image URLs before chunks are deleted (ImageInfo references are lost after deletion)
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantID, []string{id})
if err != nil {
logger.Errorf(ctx, "Failed to collect image URLs for cleanup: %v", err)
}
var imageInfoStrs []string
for _, ci := range chunkImageInfos {
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
}
imageURLs := collectImageURLs(ctx, imageInfoStrs)
wg := errgroup.Group{}
// Delete knowledge embeddings from vector store
wg.Go(func() error {
@@ -1124,13 +1176,14 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
return nil
})
// Delete the physical file if it exists
// Delete the physical file and extracted images if they exist
wg.Go(func() error {
if knowledge.FilePath != "" {
if err := kbFileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed")
}
}
deleteExtractedImages(ctx, kbFileSvc, imageURLs)
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
tenantInfo.StorageUsed -= knowledge.StorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil {
@@ -1189,6 +1242,25 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
}
}
// Collect image URLs before chunks are deleted
chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, ids)
if err != nil {
logger.Errorf(ctx, "Failed to collect image URLs for batch cleanup: %v", err)
}
knowledgeToKB := make(map[string]string)
for _, k := range knowledgeList {
knowledgeToKB[k.ID] = k.KnowledgeBaseID
}
kbImageInfos := make(map[string][]string) // kbID → []imageInfo JSON
for _, ci := range chunkImageInfos {
kbID := knowledgeToKB[ci.KnowledgeID]
kbImageInfos[kbID] = append(kbImageInfos[kbID], ci.ImageInfo)
}
kbImageURLs := make(map[string][]string) // kbID → []imageURL (deduplicated)
for kbID, infos := range kbImageInfos {
kbImageURLs[kbID] = collectImageURLs(ctx, infos)
}
wg := errgroup.Group{}
// 2. Delete knowledge embeddings from vector store
wg.Go(func() error {
@@ -1236,7 +1308,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
return nil
})
// 4. Delete the physical file if it exists
// 4. Delete the physical file and extracted images if they exist
wg.Go(func() error {
storageAdjust := int64(0)
for _, knowledge := range knowledgeList {
@@ -1248,6 +1320,15 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
}
storageAdjust -= knowledge.StorageSize
}
// Delete extracted images per KB
for kbID, urls := range kbImageURLs {
fSvc := kbFileServices[kbID]
if fSvc == nil {
logger.Warnf(ctx, "No file service for KB %s, skipping %d image deletions", kbID, len(urls))
continue
}
deleteExtractedImages(ctx, fSvc, urls)
}
tenantInfo.StorageUsed += storageAdjust
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed")
@@ -7024,11 +7105,28 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
}
}
// Collect image URLs before chunks are deleted
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
fileSvc := s.resolveFileService(ctx, kb)
chunkImageInfos, imgErr := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, []string{knowledge.ID})
if imgErr != nil {
logger.GetLogger(ctx).WithField("error", imgErr).Error("Failed to collect image URLs for cleanup")
cleanupErr = errors.Join(cleanupErr, imgErr)
}
var imageInfoStrs []string
for _, ci := range chunkImageInfos {
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
}
imageURLs := collectImageURLs(ctx, imageInfoStrs)
if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil {
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge chunks")
cleanupErr = errors.Join(cleanupErr, err)
}
// Delete extracted images after chunks are deleted
deleteExtractedImages(ctx, fileSvc, imageURLs)
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge graph data")
@@ -0,0 +1,94 @@
package service
import (
"context"
"testing"
)
func TestCollectImageURLs(t *testing.T) {
ctx := context.Background()
tests := []struct {
name string
imageInfos []string
wantURLs []string
}{
{
name: "empty input",
imageInfos: nil,
wantURLs: nil,
},
{
name: "empty strings",
imageInfos: []string{"", ""},
wantURLs: nil,
},
{
name: "single image",
imageInfos: []string{
`[{"url":"s3://bucket/prefix/123/exports/abc.png","original_url":"https://example.com/img.png"}]`,
},
wantURLs: []string{"s3://bucket/prefix/123/exports/abc.png"},
},
{
name: "multiple images in one chunk",
imageInfos: []string{
`[{"url":"s3://bucket/a.png"},{"url":"s3://bucket/b.png"}]`,
},
wantURLs: []string{"s3://bucket/a.png", "s3://bucket/b.png"},
},
{
name: "dedup across chunks — parent and child share same URL",
imageInfos: []string{
`[{"url":"s3://bucket/same.png"}]`,
`[{"url":"s3://bucket/same.png"}]`,
`[{"url":"s3://bucket/other.png"}]`,
},
wantURLs: []string{"s3://bucket/same.png", "s3://bucket/other.png"},
},
{
name: "skip empty URL field",
imageInfos: []string{
`[{"url":"","original_url":"https://example.com/img.png"}]`,
},
wantURLs: nil,
},
{
name: "invalid JSON — skipped with no panic",
imageInfos: []string{
`not valid json`,
`[{"url":"s3://bucket/valid.png"}]`,
},
wantURLs: []string{"s3://bucket/valid.png"},
},
{
name: "mixed providers",
imageInfos: []string{
`[{"url":"s3://bucket/img1.png"}]`,
`[{"url":"minio://bucket/img2.jpg"}]`,
`[{"url":"local://data/img3.webp"}]`,
},
wantURLs: []string{
"s3://bucket/img1.png",
"minio://bucket/img2.jpg",
"local://data/img3.webp",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := collectImageURLs(ctx, tt.imageInfos)
if len(got) != len(tt.wantURLs) {
t.Fatalf("collectImageURLs() returned %d URLs, want %d\ngot: %v\nwant: %v",
len(got), len(tt.wantURLs), got, tt.wantURLs)
}
for i, url := range got {
if url != tt.wantURLs[i] {
t.Errorf("collectImageURLs()[%d] = %q, want %q", i, url, tt.wantURLs[i])
}
}
})
}
}
+14 -2
View File
@@ -453,6 +453,17 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
}
}
// Collect image URLs before chunks are deleted
chunkImageInfos, imgErr := s.chunkRepo.ListImageInfoByKnowledgeIDs(ctx, tenantID, knowledgeIDs)
if imgErr != nil {
logger.Warnf(ctx, "Failed to collect image URLs for KB delete: %v", imgErr)
}
var imageInfoStrs []string
for _, ci := range chunkImageInfos {
imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
}
imageURLs := collectImageURLs(ctx, imageInfoStrs)
// Delete all chunks
logger.Infof(ctx, "Deleting all chunks in knowledge base")
for _, knowledgeID := range knowledgeIDs {
@@ -461,8 +472,8 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
}
}
// Delete physical files and adjust storage
logger.Infof(ctx, "Deleting physical files")
// Delete physical files, extracted images, and adjust storage
logger.Infof(ctx, "Deleting physical files and extracted images")
storageAdjust := int64(0)
for _, knowledge := range knowledgeList {
if knowledge.FilePath != "" {
@@ -472,6 +483,7 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
}
storageAdjust -= knowledge.StorageSize
}
deleteExtractedImages(ctx, s.fileSvc, imageURLs)
if storageAdjust != 0 {
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantID, storageAdjust); err != nil {
logger.Warnf(ctx, "Failed to adjust tenant storage: %v", err)
+8
View File
@@ -6,6 +6,12 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// ChunkImageInfo holds (knowledge_id, image_info) pairs for image cleanup before chunk deletion.
type ChunkImageInfo struct {
KnowledgeID string `gorm:"column:knowledge_id"`
ImageInfo string `gorm:"column:image_info"`
}
// ChunkRepository defines the interface for chunk repository operations
type ChunkRepository interface {
// CreateChunks creates chunks
@@ -58,6 +64,8 @@ type ChunkRepository interface {
DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) error
// DeleteByKnowledgeList deletes all chunks for a knowledge list
DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error
// ListImageInfoByKnowledgeIDs returns non-empty (knowledge_id, image_info) pairs for image cleanup.
ListImageInfoByKnowledgeIDs(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]ChunkImageInfo, error)
// MoveChunksByKnowledgeID updates knowledge_base_id for all chunks of a knowledge item
MoveChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string, targetKBID string) error
// DeleteChunksByTagID deletes all chunks with the specified tag ID