fix(knowledge): rewrite chunk content image URLs on clone
When cloning knowledge chunks, image_info was deep-copied to new storage objects but Content still embedded the source markdown image URLs. After the source knowledge is deleted, those in-content references break while image_info points at the new copies. Rewrite Content alongside image_info so both stay consistent.
This commit is contained in:
@@ -65,13 +65,16 @@ func mustImageInfoJSON(t *testing.T, imgs []types.ImageInfo) string {
|
||||
|
||||
func TestCloneChunkImageInfo_Empty(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", 1, "kb-1", map[string]string{})
|
||||
content, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "some text", "", 1, "kb-1", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if out != "" || copied != nil {
|
||||
t.Fatalf("expected empty result, got out=%q copied=%v", out, copied)
|
||||
}
|
||||
if content != "some text" {
|
||||
t.Fatalf("content must pass through unchanged, got %q", content)
|
||||
}
|
||||
if svc.copyCalls != 0 {
|
||||
t.Fatalf("expected 0 copies, got %d", svc.copyCalls)
|
||||
}
|
||||
@@ -82,7 +85,7 @@ func TestCloneChunkImageInfo_RewritesURLAndMatchedOriginal(t *testing.T) {
|
||||
src := mustImageInfoJSON(t, []types.ImageInfo{
|
||||
{URL: "local://1/k0/a.png", OriginalURL: "local://1/k0/a.png", Caption: "cap"},
|
||||
})
|
||||
out, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 7, "k-dst", map[string]string{})
|
||||
_, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 7, "k-dst", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
@@ -106,12 +109,28 @@ func TestCloneChunkImageInfo_RewritesURLAndMatchedOriginal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneChunkImageInfo_RewritesInContentMarkdownURLs(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
src := mustImageInfoJSON(t, []types.ImageInfo{
|
||||
{URL: "local://1/k0/a.png", OriginalURL: "local://1/k0/a.png"},
|
||||
})
|
||||
content := "See  here."
|
||||
newContent, _, _, err := cloneChunkImageInfo(context.Background(), svc, content, src, 7, "k-dst", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
want := "See  here."
|
||||
if newContent != want {
|
||||
t.Errorf("content image URL not rewritten:\n got %q\nwant %q", newContent, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneChunkImageInfo_PreservesUnmatchedOriginalURL(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
src := mustImageInfoJSON(t, []types.ImageInfo{
|
||||
{URL: "local://1/k0/a.png", OriginalURL: "https://external.example.com/a.png"},
|
||||
})
|
||||
out, _, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{})
|
||||
_, out, _, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
@@ -131,7 +150,7 @@ func TestCloneChunkImageInfo_DedupsIdenticalURLs(t *testing.T) {
|
||||
{URL: "local://1/k0/same.png"},
|
||||
{URL: "local://1/k0/other.png"},
|
||||
})
|
||||
_, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{})
|
||||
_, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
@@ -147,10 +166,10 @@ func TestCloneChunkImageInfo_DedupsAcrossCallsViaSharedCache(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
cache := map[string]string{}
|
||||
src := mustImageInfoJSON(t, []types.ImageInfo{{URL: "local://1/k0/shared.png"}})
|
||||
if _, _, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", cache); err != nil {
|
||||
if _, _, _, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", cache); err != nil {
|
||||
t.Fatalf("first call error: %v", err)
|
||||
}
|
||||
if _, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", cache); err != nil {
|
||||
if _, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", cache); err != nil {
|
||||
t.Fatalf("second call error: %v", err)
|
||||
} else if len(copied) != 0 {
|
||||
t.Fatalf("second call should reuse cache (0 new copies), got %v", copied)
|
||||
@@ -162,7 +181,7 @@ func TestCloneChunkImageInfo_DedupsAcrossCallsViaSharedCache(t *testing.T) {
|
||||
|
||||
func TestCloneChunkImageInfo_ParseFailureAbortsClone(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
_, _, err := cloneChunkImageInfo(context.Background(), svc, "{not valid json", 1, "k-dst", map[string]string{})
|
||||
_, _, _, err := cloneChunkImageInfo(context.Background(), svc, "", "{not valid json", 1, "k-dst", map[string]string{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error on invalid image_info JSON, got nil")
|
||||
}
|
||||
@@ -177,7 +196,7 @@ func TestCloneChunkImageInfo_CopyFailureReturnsPartialForCleanup(t *testing.T) {
|
||||
{URL: "local://1/k0/good.png"},
|
||||
{URL: "local://1/k0/bad.png"},
|
||||
})
|
||||
_, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{})
|
||||
_, _, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when an image copy fails")
|
||||
}
|
||||
@@ -190,7 +209,7 @@ func TestCloneChunkImageInfo_CopyFailureReturnsPartialForCleanup(t *testing.T) {
|
||||
func TestCloneChunkImageInfo_SkipsEmptyURL(t *testing.T) {
|
||||
svc := &countingFileService{}
|
||||
src := mustImageInfoJSON(t, []types.ImageInfo{{URL: "", Caption: "no-image"}})
|
||||
out, copied, err := cloneChunkImageInfo(context.Background(), svc, src, 1, "k-dst", map[string]string{})
|
||||
_, out, copied, err := cloneChunkImageInfo(context.Background(), svc, "", src, 1, "k-dst", map[string]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
@@ -44,39 +45,53 @@ func copyOwnedObject(
|
||||
// rollback on failure). urlCache dedups identical source objects across chunks
|
||||
// so the same source image is copied at most once per clone.
|
||||
//
|
||||
// An empty srcImageInfo yields ("", nil, nil). A JSON parse failure returns an
|
||||
// error (the clone fails) rather than silently inheriting the shared-reference
|
||||
// bug. When an image's OriginalURL points at the same object as its URL (the
|
||||
// common case for extracted images), OriginalURL is rewritten to the new path
|
||||
// too; an OriginalURL from a different/external source is preserved.
|
||||
// srcContent is the chunk's Markdown content, which embeds the same image URLs
|
||||
// as  references. Because the source objects are deleted after some
|
||||
// clone/move flows, those in-content references must also be rewritten to the
|
||||
// newly-copied objects; otherwise the content would point at soon-to-be-deleted
|
||||
// storage while image_info points at the new copies. The rewritten content is
|
||||
// returned alongside the rewritten image_info so both stay consistent.
|
||||
//
|
||||
// An empty srcImageInfo yields (srcContent, "", nil, nil). A JSON parse failure
|
||||
// returns an error (the clone fails) rather than silently inheriting the
|
||||
// shared-reference bug. When an image's OriginalURL points at the same object as
|
||||
// its URL (the common case for extracted images), OriginalURL is rewritten to
|
||||
// the new path too; an OriginalURL from a different/external source is preserved.
|
||||
func cloneChunkImageInfo(
|
||||
ctx context.Context,
|
||||
dstSvc interfaces.FileService,
|
||||
srcContent string,
|
||||
srcImageInfo string,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
urlCache map[string]string,
|
||||
) (newImageInfo string, copiedURLs []string, err error) {
|
||||
) (newContent string, newImageInfo string, copiedURLs []string, err error) {
|
||||
if srcImageInfo == "" {
|
||||
return "", nil, nil
|
||||
return srcContent, "", nil, nil
|
||||
}
|
||||
|
||||
var images []*types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(srcImageInfo), &images); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err)
|
||||
return srcContent, "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err)
|
||||
}
|
||||
|
||||
// urlReplacements maps every old (source) URL to its new (copied) URL so
|
||||
// the in-content Markdown image references can be rewritten to match.
|
||||
urlReplacements := make(map[string]string)
|
||||
|
||||
for _, img := range images {
|
||||
if img == nil || img.URL == "" {
|
||||
continue
|
||||
}
|
||||
oldURL := img.URL
|
||||
oldOriginalURL := img.OriginalURL
|
||||
originalMatchedURL := img.OriginalURL == img.URL
|
||||
|
||||
newURL, cached := urlCache[img.URL]
|
||||
if !cached {
|
||||
newURL, err = copyOwnedObject(ctx, dstSvc, dstSvc, img.URL, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err)
|
||||
return srcContent, "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err)
|
||||
}
|
||||
urlCache[img.URL] = newURL
|
||||
copiedURLs = append(copiedURLs, newURL)
|
||||
@@ -86,13 +101,40 @@ func cloneChunkImageInfo(
|
||||
img.OriginalURL = newURL
|
||||
}
|
||||
img.URL = newURL
|
||||
|
||||
urlReplacements[oldURL] = newURL
|
||||
if oldOriginalURL != "" {
|
||||
urlReplacements[oldOriginalURL] = img.OriginalURL
|
||||
}
|
||||
}
|
||||
|
||||
out, err := json.Marshal(images)
|
||||
if err != nil {
|
||||
return "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err)
|
||||
return srcContent, "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err)
|
||||
}
|
||||
return string(out), copiedURLs, nil
|
||||
return rewriteContentImageURLs(srcContent, urlReplacements), string(out), copiedURLs, nil
|
||||
}
|
||||
|
||||
// rewriteContentImageURLs replaces every occurrence of an old image URL with its
|
||||
// new URL in content. Replacements whose old and new URL are identical (e.g. an
|
||||
// external OriginalURL that was preserved) are skipped. Longer URLs are applied
|
||||
// first so a URL that is a prefix of another does not partially rewrite it.
|
||||
func rewriteContentImageURLs(content string, replacements map[string]string) string {
|
||||
if content == "" || len(replacements) == 0 {
|
||||
return content
|
||||
}
|
||||
oldURLs := make([]string, 0, len(replacements))
|
||||
for oldURL, newURL := range replacements {
|
||||
if oldURL == "" || oldURL == newURL {
|
||||
continue
|
||||
}
|
||||
oldURLs = append(oldURLs, oldURL)
|
||||
}
|
||||
slices.SortFunc(oldURLs, func(a, b string) int { return len(b) - len(a) })
|
||||
for _, oldURL := range oldURLs {
|
||||
content = strings.ReplaceAll(content, oldURL, replacements[oldURL])
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// cleanupCopiedObjects deletes objects that were newly created during a clone
|
||||
@@ -257,8 +299,8 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
|
||||
// Deep-copy extracted images into objects owned by the destination
|
||||
// knowledge so deleting the source never breaks this clone.
|
||||
newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
||||
ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache)
|
||||
newContent, newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
||||
ctx, dstSvc, sourceChunk.Content, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache)
|
||||
if copyErr != nil {
|
||||
err = fmt.Errorf("clone chunk image copy failed: %w", copyErr)
|
||||
return err
|
||||
@@ -271,7 +313,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
KnowledgeID: dst.ID,
|
||||
KnowledgeBaseID: dst.KnowledgeBaseID,
|
||||
TagID: targetTagID,
|
||||
Content: sourceChunk.Content,
|
||||
Content: newContent,
|
||||
ChunkIndex: sourceChunk.ChunkIndex,
|
||||
IsEnabled: sourceChunk.IsEnabled,
|
||||
Flags: sourceChunk.Flags,
|
||||
@@ -676,8 +718,8 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
|
||||
// Deep-copy extracted images into objects owned by the destination
|
||||
// FAQ knowledge so deleting the source never breaks this clone.
|
||||
newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
||||
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
|
||||
newContent, newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
||||
ctx, dstSvc, srcChunk.Content, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
|
||||
if copyErr != nil {
|
||||
logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)
|
||||
handleError(progress, copyErr, "Failed to copy FAQ entry images")
|
||||
@@ -692,7 +734,7 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
KnowledgeID: dstKnowledge.ID,
|
||||
KnowledgeBaseID: dstKB.ID,
|
||||
TagID: targetTagID,
|
||||
Content: srcChunk.Content,
|
||||
Content: newContent,
|
||||
ChunkIndex: srcChunk.ChunkIndex,
|
||||
IsEnabled: srcChunk.IsEnabled,
|
||||
Flags: srcChunk.Flags,
|
||||
|
||||
Reference in New Issue
Block a user