diff --git a/internal/application/service/knowledge_clone_image_test.go b/internal/application/service/knowledge_clone_image_test.go index a3c53c07..6c2f9ccc 100644 --- a/internal/application/service/knowledge_clone_image_test.go +++ b/internal/application/service/knowledge_clone_image_test.go @@ -65,16 +65,13 @@ func mustImageInfoJSON(t *testing.T, imgs []types.ImageInfo) string { func TestCloneChunkImageInfo_Empty(t *testing.T) { svc := &countingFileService{} - content, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "some text", "", 1, "kb-1", map[string]string{}) + out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", 1, "kb-1", map[string]string{}) if err != nil { t.Fatalf("unexpected error: %v", err) } if out != "" || copied != nil { t.Fatalf("expected empty result, got out=%q copied=%v", out, copied) } - if content != "some text" { - t.Fatalf("content must pass through unchanged, got %q", content) - } if svc.copyCalls != 0 { t.Fatalf("expected 0 copies, got %d", svc.copyCalls) } @@ -85,7 +82,7 @@ func TestCloneChunkImageInfo_RewritesURLAndMatchedOriginal(t *testing.T) { src := mustImageInfoJSON(t, []types.ImageInfo{ {URL: "local://1/k0/a.png", OriginalURL: "local://1/k0/a.png", Caption: "cap"}, }) - _, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 7, "k-dst", map[string]string{}) + out, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 7, "k-dst", map[string]string{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -109,19 +106,41 @@ func TestCloneChunkImageInfo_RewritesURLAndMatchedOriginal(t *testing.T) { } } -func TestCloneChunkImageInfo_RewritesInContentMarkdownURLs(t *testing.T) { +// TestRewriteContentImageURLs_ParentTextChunk covers the core scenario: an +// image lives in an independent child chunk (so its image_info yields the +// old->new URL mapping in urlCache), while the PARENT text chunk carries the +// ![](url) reference with an empty image_info. The parent's content must still +// be rewritten from the shared urlCache. +func TestRewriteContentImageURLs_ParentTextChunk(t *testing.T) { svc := &countingFileService{} - src := mustImageInfoJSON(t, []types.ImageInfo{ + // Child image chunk populates urlCache via its image_info. + childImageInfo := mustImageInfoJSON(t, []types.ImageInfo{ {URL: "local://1/k0/a.png", OriginalURL: "local://1/k0/a.png"}, }) - content := "See ![diagram](local://1/k0/a.png) here." - newContent, _, _, err := cloneChunkImageInfo(context.Background(), svc, content, src, 7, "k-dst", map[string]string{}) - if err != nil { + urlCache := map[string]string{} + if _, _, err := cloneChunkImageInfo(context.Background(), svc, childImageInfo, 7, "k-dst", urlCache); err != nil { t.Fatalf("unexpected error: %v", err) } + + // Parent text chunk has NO image_info but embeds the markdown reference. + parentContent := "See ![diagram](local://1/k0/a.png) here." + got := rewriteContentImageURLs(parentContent, urlCache) want := "See ![diagram](local://7/k-dst/copy-of-local://1/k0/a.png) here." - if newContent != want { - t.Errorf("content image URL not rewritten:\n got %q\nwant %q", newContent, want) + if got != want { + t.Errorf("parent content image URL not rewritten:\n got %q\nwant %q", got, want) + } +} + +// TestRewriteContentImageURLs_NoMappingIsNoop ensures content without any known +// old URL is returned unchanged, and an empty cache is a no-op. +func TestRewriteContentImageURLs_NoMappingIsNoop(t *testing.T) { + content := "See ![diagram](local://1/k0/a.png) here." + if got := rewriteContentImageURLs(content, map[string]string{}); got != content { + t.Errorf("empty cache must be no-op, got %q", got) + } + cache := map[string]string{"local://1/k0/other.png": "local://7/k-dst/copy.png"} + if got := rewriteContentImageURLs(content, cache); got != content { + t.Errorf("unrelated mapping must be no-op, got %q", got) } } @@ -130,7 +149,7 @@ func TestCloneChunkImageInfo_PreservesUnmatchedOriginalURL(t *testing.T) { src := mustImageInfoJSON(t, []types.ImageInfo{ {URL: "local://1/k0/a.png", OriginalURL: "https://external.example.com/a.png"}, }) - _, out, _, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{}) + out, _, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -150,7 +169,7 @@ func TestCloneChunkImageInfo_DedupsIdenticalURLs(t *testing.T) { {URL: "local://1/k0/same.png"}, {URL: "local://1/k0/other.png"}, }) - _, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{}) + _, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -166,10 +185,10 @@ func TestCloneChunkImageInfo_DedupsAcrossCallsViaSharedCache(t *testing.T) { svc := &countingFileService{} cache := map[string]string{} src := mustImageInfoJSON(t, []types.ImageInfo{{URL: "local://1/k0/shared.png"}}) - if _, _, _, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", cache); err != nil { + if _, _, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", cache); err != nil { t.Fatalf("first call error: %v", err) } - if _, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", cache); err != nil { + if _, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", cache); err != nil { t.Fatalf("second call error: %v", err) } else if len(copied) != 0 { t.Fatalf("second call should reuse cache (0 new copies), got %v", copied) @@ -181,7 +200,7 @@ func TestCloneChunkImageInfo_DedupsAcrossCallsViaSharedCache(t *testing.T) { func TestCloneChunkImageInfo_ParseFailureAbortsClone(t *testing.T) { svc := &countingFileService{} - _, _, _, err := cloneChunkImageInfo(context.Background(), svc, "", "{not valid json", 1, "k-dst", map[string]string{}) + _, _, err := cloneChunkImageInfo(context.Background(), svc, "{not valid json", 1, "k-dst", map[string]string{}) if err == nil { t.Fatal("expected error on invalid image_info JSON, got nil") } @@ -196,7 +215,7 @@ func TestCloneChunkImageInfo_CopyFailureReturnsPartialForCleanup(t *testing.T) { {URL: "local://1/k0/good.png"}, {URL: "local://1/k0/bad.png"}, }) - _, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{}) + _, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{}) if err == nil { t.Fatal("expected error when an image copy fails") } @@ -209,7 +228,7 @@ func TestCloneChunkImageInfo_CopyFailureReturnsPartialForCleanup(t *testing.T) { func TestCloneChunkImageInfo_SkipsEmptyURL(t *testing.T) { svc := &countingFileService{} src := mustImageInfoJSON(t, []types.ImageInfo{{URL: "", Caption: "no-image"}}) - _, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{}) + out, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go index 1d478a48..594dbe0c 100644 --- a/internal/application/service/knowledge_clone_move.go +++ b/internal/application/service/knowledge_clone_move.go @@ -43,55 +43,43 @@ func copyOwnedObject( // object into a NEW object owned by (tenantID, knowledgeID), and returns the // re-serialized image_info plus the list of newly-created object URLs (for // rollback on failure). urlCache dedups identical source objects across chunks -// so the same source image is copied at most once per clone. +// so the same source image is copied at most once per clone AND accumulates the +// full old->new URL mapping so callers can rewrite in-content Markdown image +// references (see rewriteContentImageURLs). // -// srcContent is the chunk's Markdown content, which embeds the same image URLs -// as ![](url) references. Because the source objects are deleted after some -// clone/move flows, those in-content references must also be rewritten to the -// newly-copied objects; otherwise the content would point at soon-to-be-deleted -// storage while image_info points at the new copies. The rewritten content is -// returned alongside the rewritten image_info so both stay consistent. -// -// An empty srcImageInfo yields (srcContent, "", nil, nil). A JSON parse failure -// returns an error (the clone fails) rather than silently inheriting the -// shared-reference bug. When an image's OriginalURL points at the same object as -// its URL (the common case for extracted images), OriginalURL is rewritten to -// the new path too; an OriginalURL from a different/external source is preserved. +// An empty srcImageInfo yields ("", nil, nil). A JSON parse failure returns an +// error (the clone fails) rather than silently inheriting the shared-reference +// bug. When an image's OriginalURL points at the same object as its URL (the +// common case for extracted images), OriginalURL is rewritten to the new path +// too; an OriginalURL from a different/external source is preserved. func cloneChunkImageInfo( ctx context.Context, dstSvc interfaces.FileService, - srcContent string, srcImageInfo string, tenantID uint64, knowledgeID string, urlCache map[string]string, -) (newContent string, newImageInfo string, copiedURLs []string, err error) { +) (newImageInfo string, copiedURLs []string, err error) { if srcImageInfo == "" { - return srcContent, "", nil, nil + return "", nil, nil } var images []*types.ImageInfo if err := json.Unmarshal([]byte(srcImageInfo), &images); err != nil { - return srcContent, "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err) + return "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err) } - // urlReplacements maps every old (source) URL to its new (copied) URL so - // the in-content Markdown image references can be rewritten to match. - urlReplacements := make(map[string]string) - for _, img := range images { if img == nil || img.URL == "" { continue } - oldURL := img.URL - oldOriginalURL := img.OriginalURL originalMatchedURL := img.OriginalURL == img.URL newURL, cached := urlCache[img.URL] if !cached { newURL, err = copyOwnedObject(ctx, dstSvc, dstSvc, img.URL, tenantID, knowledgeID) if err != nil { - return srcContent, "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err) + return "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err) } urlCache[img.URL] = newURL copiedURLs = append(copiedURLs, newURL) @@ -101,30 +89,33 @@ func cloneChunkImageInfo( img.OriginalURL = newURL } img.URL = newURL - - urlReplacements[oldURL] = newURL - if oldOriginalURL != "" { - urlReplacements[oldOriginalURL] = img.OriginalURL - } } out, err := json.Marshal(images) if err != nil { - return srcContent, "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err) + return "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err) } - return rewriteContentImageURLs(srcContent, urlReplacements), string(out), copiedURLs, nil + return string(out), copiedURLs, nil } // rewriteContentImageURLs replaces every occurrence of an old image URL with its -// new URL in content. Replacements whose old and new URL are identical (e.g. an -// external OriginalURL that was preserved) are skipped. Longer URLs are applied -// first so a URL that is a prefix of another does not partially rewrite it. -func rewriteContentImageURLs(content string, replacements map[string]string) string { - if content == "" || len(replacements) == 0 { +// new (copied) URL in content, using the old->new mapping accumulated in +// urlCache. It is the second half of the image deep-copy: chunk Content embeds +// image URLs as Markdown ![](url) references, but for document knowledge the +// image objects live in independent image_ocr/image_caption child chunks — the +// parent text chunk carries the ![](url) reference with an empty image_info. So +// the old->new mapping is only known after every chunk's image_info has been +// processed; this rewrite must therefore run as a final pass once urlCache is +// complete, over ALL cloned chunks, not per-chunk. +// +// Replacements are applied longest-old-URL first so a URL that is a prefix of +// another is not partially rewritten. Entries whose old==new are skipped. +func rewriteContentImageURLs(content string, urlCache map[string]string) string { + if content == "" || len(urlCache) == 0 { return content } - oldURLs := make([]string, 0, len(replacements)) - for oldURL, newURL := range replacements { + oldURLs := make([]string, 0, len(urlCache)) + for oldURL, newURL := range urlCache { if oldURL == "" || oldURL == newURL { continue } @@ -132,7 +123,7 @@ func rewriteContentImageURLs(content string, replacements map[string]string) str } slices.SortFunc(oldURLs, func(a, b string) int { return len(b) - len(a) }) for _, oldURL := range oldURLs { - content = strings.ReplaceAll(content, oldURL, replacements[oldURL]) + content = strings.ReplaceAll(content, oldURL, urlCache[oldURL]) } return content } @@ -298,9 +289,13 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl } // Deep-copy extracted images into objects owned by the destination - // knowledge so deleting the source never breaks this clone. - newContent, newImageInfo, copied, copyErr := cloneChunkImageInfo( - ctx, dstSvc, sourceChunk.Content, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache) + // knowledge so deleting the source never breaks this clone. Content + // URL rewriting happens in a final pass below, once urlCache holds + // the complete old->new mapping (image objects live in independent + // child chunks, so a parent text chunk's ![](url) reference cannot be + // rewritten until its child image chunk has been processed). + newImageInfo, copied, copyErr := cloneChunkImageInfo( + ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache) if copyErr != nil { err = fmt.Errorf("clone chunk image copy failed: %w", copyErr) return err @@ -313,7 +308,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl KnowledgeID: dst.ID, KnowledgeBaseID: dst.KnowledgeBaseID, TagID: targetTagID, - Content: newContent, + Content: sourceChunk.Content, ChunkIndex: sourceChunk.ChunkIndex, IsEnabled: sourceChunk.IsEnabled, Flags: sourceChunk.Flags, @@ -335,6 +330,11 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl } } for _, targetChunk := range targetChunks { + // Rewrite in-content Markdown image URLs now that urlCache holds the + // complete old->new mapping across all chunks. This fixes parent text + // chunks whose ![](url) reference points at a source object copied while + // processing an independent image_ocr/image_caption child chunk. + targetChunk.Content = rewriteContentImageURLs(targetChunk.Content, urlCache) if val, ok := srcTodst[targetChunk.PreChunkID]; ok { targetChunk.PreChunkID = val } else { @@ -717,9 +717,12 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( } // Deep-copy extracted images into objects owned by the destination - // FAQ knowledge so deleting the source never breaks this clone. - newContent, newImageInfo, copied, copyErr := cloneChunkImageInfo( - ctx, dstSvc, srcChunk.Content, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache) + // FAQ knowledge so deleting the source never breaks this clone. A FAQ + // chunk is self-contained (its own Content + image_info), so its + // ![](url) references can be rewritten immediately using the mapping + // just accumulated in imageURLCache. + newImageInfo, copied, copyErr := cloneChunkImageInfo( + ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache) if copyErr != nil { logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr) handleError(progress, copyErr, "Failed to copy FAQ entry images") @@ -734,7 +737,7 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( KnowledgeID: dstKnowledge.ID, KnowledgeBaseID: dstKB.ID, TagID: targetTagID, - Content: newContent, + Content: rewriteContentImageURLs(srcChunk.Content, imageURLCache), ChunkIndex: srcChunk.ChunkIndex, IsEnabled: srcChunk.IsEnabled, Flags: srcChunk.Flags,