From f92302cc647b37f59677cc1dcc3a6b00e83976d1 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Sat, 30 May 2026 17:47:54 +0800 Subject: [PATCH] fix(knowledge): stitch chunks by text overlap, not by position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconstructing full document content from chunks relied on position math (offset = content length - (EndAt - lastEndAt)) that assumes len([]rune(Content)) == EndAt - StartAt. Two upstream behaviors break that invariant and caused the merged "全文" view to drop or duplicate text: - The parent-child chunker prepends a synthetic table header to split tables; that header is zero-width in position space (start == end), so Content is longer than EndAt - StartAt. - Chunk Content can retain HTML entities (", > ...), inflating the rune count relative to the source span. Replace position-based trimming with text-overlap matching: find where the accumulated text's suffix reappears at the head of the next chunk and join there. Positions are only used to size the search window. This naturally skips the prepended header and is immune to entity length drift. Extract the logic into searchutil.AppendWithOverlap / MergeTextChunks and reuse it across the three backend stitching sites (wiki_ingest reconstructContent, graph mergeChunkContents, chat_pipeline mergeOverlappingChunks). Mirror the same algorithm in the frontend doc-content merge. Also fix the markdown hr/heading styles (thin solid divider, primary heading color). --- frontend/src/components/css/markdown.less | 8 +- frontend/src/components/doc-content.vue | 86 ++++----- .../service/chat_pipeline/merge_overlap.go | 36 +--- internal/application/service/graph.go | 27 +-- internal/application/service/wiki_ingest.go | 43 +---- internal/searchutil/chunkmerge.go | 163 ++++++++++++++++++ internal/searchutil/chunkmerge_test.go | 91 ++++++++++ 7 files changed, 320 insertions(+), 134 deletions(-) create mode 100644 internal/searchutil/chunkmerge.go create mode 100644 internal/searchutil/chunkmerge_test.go diff --git a/frontend/src/components/css/markdown.less b/frontend/src/components/css/markdown.less index 4c6d2f77..203ef056 100644 --- a/frontend/src/components/css/markdown.less +++ b/frontend/src/components/css/markdown.less @@ -29,7 +29,7 @@ h6 { margin-top: 5px; font-weight: bold; - color: var(--td-text-color-placeholder); + color: var(--td-text-color-primary); font-family: var(--app-font-family); transition: all 0.2s ease-out; font-size: 20px; @@ -115,8 +115,10 @@ hr { padding: 0; - margin: 32px 0; - border-top: 0.5rem dotted var(--td-brand-color-focus); + margin: 24px 0; + height: 1px; + border: none; + background: var(--td-component-stroke); overflow: hidden; box-sizing: content-box; } diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index 6136d63a..c91804f6 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -225,9 +225,40 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged'); // 合并后的文档内容(在下方通过 computed 定义) /** - * 根据 start_at 和 end_at 字段合并有 overlap 的 chunks - * 返回合并后的完整文档内容 - * 实现逻辑与后端 Go 代码保持一致 + * 把已合并文本 acc 和下一个 chunk 内容 next 拼接,并去除两者的重叠部分。 + * + * 不再依赖 start_at / end_at 做位置裁剪,而是用「文本重叠匹配」:在 next 的 + * 开头窗口里找 acc 后缀首次出现的位置,从该位置之后接上。这样能同时兼容: + * 1. chunker 给拆分表格补写的表头(零宽 start/end,位置上不可见)——表头出现 + * 在重叠行之前,会被自然跳过; + * 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是 + * 文本本身,不受长度偏差影响。 + * + * @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。 + */ +const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => { + if (!acc) return next; + if (!next) return acc; + + const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略 + const span = Math.max(positionOverlap, 0); + // 搜索的后缀最大长度;按位置重叠量放大几倍兜底,并设下限 + const maxK = Math.min(acc.length, next.length, Math.max(span * 3, 400)); + // 重叠行之前最多允许多少前缀(补写的表头)被跳过 + const headSlack = Math.max(span * 2, 320); + + for (let k = maxK; k >= MIN_OVERLAP; k--) { + const suffix = acc.slice(acc.length - k); + const pos = next.indexOf(suffix); + if (pos !== -1 && pos <= headSlack) { + return acc + next.slice(pos + k); + } + } + return acc + next; +}; + +/** + * 合并分块内容,还原完整文档。chunks 按 start_at 排序后逐段用文本重叠匹配拼接。 */ const mergeChunks = (chunks: any[]): string => { if (!chunks || chunks.length === 0) return ''; @@ -239,54 +270,31 @@ const mergeChunks = (chunks: any[]): string => { return startA - startB; }); - // 初始化合并结果,第一个 chunk 直接加入 - const mergedChunks: Array<{ - content: string; - start_at: number; - end_at: number; - }> = [{ - content: sortedChunks[0].content || '', - start_at: sortedChunks[0].start_at ?? 0, - end_at: sortedChunks[0].end_at ?? 0 - }]; + let merged = sortedChunks[0].content || ''; + let mergedEnd = sortedChunks[0].end_at ?? 0; - // 从第二个 chunk 开始遍历 for (let i = 1; i < sortedChunks.length; i++) { const currentChunk = sortedChunks[i]; - const lastChunk = mergedChunks[mergedChunks.length - 1]; - const currentStartAt = currentChunk.start_at ?? 0; const currentEndAt = currentChunk.end_at ?? 0; const currentContent = currentChunk.content || ''; - // 如果当前 chunk 的起始位置在最后一个 chunk 的结束位置之后,直接添加 - if (currentStartAt > lastChunk.end_at) { - mergedChunks.push({ - content: currentContent, - start_at: currentStartAt, - end_at: currentEndAt - }); - continue; + if (!currentContent) continue; + + // 与上一段有明显间隙(位置不相邻),用空行分隔后整段拼接 + if (currentStartAt > mergedEnd && mergedEnd > 0) { + merged = merged + '\n\n' + currentContent; + } else { + const positionOverlap = mergedEnd - currentStartAt; + merged = appendChunkContent(merged, currentContent, positionOverlap); } - // 合并重叠的 chunks - if (currentEndAt > lastChunk.end_at) { - // 将内容转换为字符数组以正确处理多字节字符 - const contentRunes = Array.from(currentContent); - const contentLength = contentRunes.length; - - // 计算偏移量:内容长度 - (当前结束位置 - 上一个结束位置) - const offset = contentLength - (currentEndAt - lastChunk.end_at); - - // 拼接非重叠部分 - const newContent = contentRunes.slice(offset).join(''); - lastChunk.content = lastChunk.content + newContent; - lastChunk.end_at = currentEndAt; + if (currentEndAt > mergedEnd) { + mergedEnd = currentEndAt; } } - // 合并所有段落,用双换行符连接 - return mergedChunks.map(chunk => chunk.content).join('\n\n'); + return merged; }; onMounted(() => { diff --git a/internal/application/service/chat_pipeline/merge_overlap.go b/internal/application/service/chat_pipeline/merge_overlap.go index 009d11d3..7b786206 100644 --- a/internal/application/service/chat_pipeline/merge_overlap.go +++ b/internal/application/service/chat_pipeline/merge_overlap.go @@ -5,6 +5,7 @@ import ( "encoding/json" "sort" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" ) @@ -32,35 +33,14 @@ func (p *PluginMerge) mergeOverlappingChunks( // Partial overlap: append the non-overlapping suffix. // - // Offset math assumes len([]rune(Content)) == EndAt-StartAt, but a - // few upstream paths break that invariant: - // - Parent-child chunker prepends table headers, so Content may be - // longer than EndAt-StartAt. - // - Legacy data / mixed chunk sources may carry EndAt-StartAt that - // exceed the actual rune length of Content. - // We clamp offset into [0, len(contentRunes)] so the merge degrades - // gracefully instead of panicking with a negative slice bound. + // 重叠去重统一交给 searchutil.AppendWithOverlap(按文本匹配),它能 + // 兼容父子分块器补写的零宽表头,以及 HTML 实体导致的 content 长度与 + // EndAt-StartAt 不一致——这两种情况都会让按位置裁剪错位、丢字或重复。 + // StartAt/EndAt 仅用于估算搜索窗口大小。 if chunks[i].EndAt > lastChunk.EndAt { - contentRunes := []rune(chunks[i].Content) - suffixLen := chunks[i].EndAt - lastChunk.EndAt - offset := len(contentRunes) - suffixLen - if offset < 0 || offset > len(contentRunes) { - pipelineWarn(ctx, "Merge", "overlap_offset_clamp", map[string]interface{}{ - "knowledge_id": knowledgeID, - "chunk_id": chunks[i].ID, - "content_runes": len(contentRunes), - "chunk_start": chunks[i].StartAt, - "chunk_end": chunks[i].EndAt, - "last_end": lastChunk.EndAt, - "computed_offset": offset, - }) - if offset < 0 { - offset = 0 - } else { - offset = len(contentRunes) - } - } - lastChunk.Content = lastChunk.Content + string(contentRunes[offset:]) + lastChunk.Content = searchutil.AppendWithOverlap( + lastChunk.Content, chunks[i].Content, lastChunk.EndAt-chunks[i].StartAt, + ) lastChunk.EndAt = chunks[i].EndAt lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) diff --git a/internal/application/service/graph.go b/internal/application/service/graph.go index 3a63831d..767d5935 100644 --- a/internal/application/service/graph.go +++ b/internal/application/service/graph.go @@ -15,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" "golang.org/x/sync/errgroup" @@ -345,29 +346,9 @@ func (b *graphBuilder) findRelationChunkIDs(source, target string, entities []*t // mergeChunkContents merges content from multiple document chunks // It accounts for overlapping portions between chunks to ensure coherent content func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string { - if len(chunks) == 0 { - return "" - } - - chunkContents := chunks[0].Content - preChunk := chunks[0] - - for i := 1; i < len(chunks); i++ { - // Only add non-overlapping content parts - if preChunk.EndAt > chunks[i].StartAt { - // Calculate overlap starting position - startPos := preChunk.EndAt - chunks[i].StartAt - if startPos >= 0 && startPos < len([]rune(chunks[i].Content)) { - chunkContents = chunkContents + string([]rune(chunks[i].Content)[startPos:]) - } - } else { - // If there's no overlap between chunks, add all content - chunkContents = chunkContents + chunks[i].Content - } - preChunk = chunks[i] - } - - return chunkContents + // 重叠去重统一交给公共逻辑(按文本匹配,兼容补写表头 / HTML 实体)。 + // 无间隙分隔符,保持与原实现一致的直接拼接行为。 + return searchutil.MergeTextChunks(chunks, "") } // BuildGraph constructs the knowledge graph diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index f165806b..2f3622da 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -1878,47 +1878,8 @@ func reconstructContent(chunks []*types.Chunk) string { } } - // Sort by StartAt, then ChunkIndex - sort.Slice(textChunks, func(i, j int) bool { - if textChunks[i].StartAt == textChunks[j].StartAt { - return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex - } - return textChunks[i].StartAt < textChunks[j].StartAt - }) - - var sb strings.Builder - lastEndAt := -1 - for _, c := range textChunks { - toAppend := c.Content - - if c.StartAt > lastEndAt || c.EndAt == 0 { - // Non-overlapping or missing position info - if sb.Len() > 0 { - sb.WriteString("\n") - } - sb.WriteString(toAppend) - if c.EndAt > 0 { - lastEndAt = c.EndAt - } - } else if c.EndAt > lastEndAt { - // Partial overlap - contentRunes := []rune(toAppend) - offset := len(contentRunes) - (c.EndAt - lastEndAt) - if offset >= 0 && offset < len(contentRunes) { - sb.WriteString(string(contentRunes[offset:])) - } else { - // Fallback if offset calculation is invalid - if sb.Len() > 0 { - sb.WriteString("\n") - } - sb.WriteString(toAppend) - } - lastEndAt = c.EndAt - } - // If c.EndAt <= lastEndAt, it's fully contained, so skip appending text - } - - return sb.String() + // 重叠去重与排序统一交给公共逻辑(按文本匹配,兼容补写表头 / HTML 实体)。 + return searchutil.MergeTextChunks(textChunks, "\n") } // reconstructEnrichedContent rebuilds document text and inlines image_info diff --git a/internal/searchutil/chunkmerge.go b/internal/searchutil/chunkmerge.go new file mode 100644 index 00000000..c8a4bc2d --- /dev/null +++ b/internal/searchutil/chunkmerge.go @@ -0,0 +1,163 @@ +package searchutil + +import ( + "sort" + + "github.com/Tencent/WeKnora/internal/types" +) + +// 这里实现 chunk 内容的「重叠拼接」公共逻辑,供文档重建(reconstructContent)、 +// 知识图谱内容合并(graph mergeChunkContents)、检索结果重叠合并 +// (chat_pipeline mergeOverlappingChunks)等多处复用。 +// +// 历史上各处都用「按位置」的公式裁剪重叠(offset = len(content) - (EndAt - +// lastEndAt) 之类),它默认 len([]rune(Content)) == EndAt-StartAt。但有两类 +// 数据会破坏这个不变式,导致拼接错位、丢字或重复: +// 1. 父子分块器会给被拆开的表格「补写表头」,补进去的表头是零宽度的 +// (start == end),位置坐标无法表达它,content 比 EndAt-StartAt 更长; +// 2. content 里可能保留 HTML 实体(如 " / >),其字符数比原文区间长。 +// +// 因此这里改为「按文本」匹配重叠:在下一段开头的窗口里查找已合并文本的后缀首次 +// 出现的位置,从该位置之后接上。位置信息(StartAt/EndAt)仅用于估算搜索窗口 +// 大小,不再用于裁剪。 + +const ( + // minOverlapRunes 是参与匹配的最短后缀长度。太短(如表格分隔行 |---|) + // 容易误匹配,因此忽略。 + minOverlapRunes = 12 + // defaultSearchSpan 是搜索窗口的下限,保证即使位置信息缺失/为 0 也能 + // 检测到一定范围内的真实重叠。 + defaultSearchSpan = 400 +) + +// AppendWithOverlap 把 next 追加到 acc 之后,并去除二者之间的重叠部分。 +// +// positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于 +// 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。 +// 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。 +func AppendWithOverlap(acc, next string, positionOverlap int) string { + if acc == "" { + return next + } + if next == "" { + return acc + } + + accRunes := []rune(acc) + nextRunes := []rune(next) + + span := positionOverlap + if span < 0 { + span = 0 + } + + maxK := minInt(len(accRunes), len(nextRunes)) + if cap := maxInt(span*3, defaultSearchSpan); maxK > cap { + maxK = cap + } + // 重叠内容之前最多允许跳过多少前缀(即补写的表头等合成文本)。 + headSlack := maxInt(span*2, 320) + + for k := maxK; k >= minOverlapRunes; k-- { + needle := accRunes[len(accRunes)-k:] + if pos := indexRunes(nextRunes, needle, headSlack); pos >= 0 { + return acc + string(nextRunes[pos+k:]) + } + } + return acc + next +} + +// MergeTextChunks 按 StartAt(并列时按 ChunkIndex)排序后,用 AppendWithOverlap +// 把多个 chunk 的内容重建为完整文本。gapSep 用于位置不相邻(有间隙)的两段之间 +// 的分隔符(如 "\n"),传空串则直接拼接。 +// +// 调用方负责先做类型过滤(例如只保留文本 chunk);本函数不感知 ChunkType。 +func MergeTextChunks(chunks []*types.Chunk, gapSep string) string { + if len(chunks) == 0 { + return "" + } + + sorted := make([]*types.Chunk, len(chunks)) + copy(sorted, chunks) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].StartAt == sorted[j].StartAt { + return sorted[i].ChunkIndex < sorted[j].ChunkIndex + } + return sorted[i].StartAt < sorted[j].StartAt + }) + + merged := "" + mergedEnd := -1 + for _, c := range sorted { + if c == nil || c.Content == "" { + continue + } + if merged == "" { + merged = c.Content + if c.EndAt > 0 { + mergedEnd = c.EndAt + } + continue + } + + // 间隙 / 位置信息缺失(EndAt==0):作为独立段落拼接,不做重叠裁剪。 + if c.StartAt > mergedEnd || c.EndAt == 0 { + if gapSep != "" { + merged += gapSep + } + merged += c.Content + if c.EndAt > 0 { + mergedEnd = c.EndAt + } + continue + } + + // 部分重叠或首尾相接:按文本匹配去重叠后拼接。 + if c.EndAt > mergedEnd { + merged = AppendWithOverlap(merged, c.Content, mergedEnd-c.StartAt) + mergedEnd = c.EndAt + } + // 否则被上一段完全覆盖,跳过。 + } + + return merged +} + +// indexRunes 在 haystack 中查找 needle 首次出现的 rune 下标,且起始位置不超过 +// maxStart。找不到返回 -1。 +func indexRunes(haystack, needle []rune, maxStart int) int { + if len(needle) == 0 || len(needle) > len(haystack) { + return -1 + } + limit := len(haystack) - len(needle) + if maxStart < limit { + limit = maxStart + } + for i := 0; i <= limit; i++ { + match := true + for j := 0; j < len(needle); j++ { + if haystack[i+j] != needle[j] { + match = false + break + } + } + if match { + return i + } + } + return -1 +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/searchutil/chunkmerge_test.go b/internal/searchutil/chunkmerge_test.go new file mode 100644 index 00000000..05fe9efa --- /dev/null +++ b/internal/searchutil/chunkmerge_test.go @@ -0,0 +1,91 @@ +package searchutil + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestAppendWithOverlap_ContiguousNoTrim(t *testing.T) { + // 首尾相接(无重叠),且 next 含 HTML 实体使其字符数 > EndAt-StartAt。 + // 旧的按位置公式会多切掉开头,这里应当整段保留。 + acc := "## 第二节\n\n" + next := "| 列A | 列B |\n| 值1 | 含实体"引号"的内容 |\n" + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("contiguous merge mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_PrependedTableHeaderSkipped(t *testing.T) { + // 模拟 chunker 给拆分表格补写表头:next 开头多了一份表头(零宽,位置不可见), + // 真正的重叠行在表头之后。按位置裁剪会错位,按文本匹配应正确去重。 + header := "| 列1 | 列2 | 列3 |\n|:---|:---|:---|\n" + overlapRows := "| 第5行 | 内容5A | 内容5B |\n| 第6行 | 内容6A | 内容6B |\n" + accTail := "| 第4行 | 内容4A | 内容4B |\n" + overlapRows + acc := header + "| 第1行 | x | — |\n" + accTail + + newRows := "| 第7行 | 内容7A | 内容7B |\n| 第8行 | 内容8A | 内容8B |\n" + next := header + overlapRows + newRows + + // 位置重叠量大约是两行的长度(这里给个近似值即可,仅用于窗口估算)。 + got := AppendWithOverlap(acc, next, len([]rune(overlapRows))) + want := acc + newRows + if got != want { + t.Fatalf("prepended-header merge mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_PlainOverlap(t *testing.T) { + acc := "abcdefghijklmnopqrstuvwxyz0123" + // next 与 acc 末尾 "klmnopqrstuvwxyz0123" 重叠,再接新内容 + next := "klmnopqrstuvwxyz0123ABCDEFG" + got := AppendWithOverlap(acc, next, 20) + want := "abcdefghijklmnopqrstuvwxyz0123ABCDEFG" + if got != want { + t.Fatalf("plain overlap mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_NoOverlap(t *testing.T) { + acc := "hello world" + next := "completely different" + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("no overlap mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestMergeTextChunks_OrdersFiltersAndStitches(t *testing.T) { + header := "| a | b |\n|:--|:--|\n" + chunks := []*types.Chunk{ + { + Content: header + "| r1 | x |\n| r2 | y |\n", ChunkType: types.ChunkTypeText, + StartAt: 0, EndAt: 20, ChunkIndex: 0, + }, + { + // 补写表头 + 与上一段 r2 重叠 + 新行 r3 + Content: header + "| r2 | y |\n| r3 | z |\n", ChunkType: types.ChunkTypeText, + StartAt: 10, EndAt: 40, ChunkIndex: 1, + }, + } + got := MergeTextChunks(chunks, "\n") + want := header + "| r1 | x |\n| r2 | y |\n" + "| r3 | z |\n" + if got != want { + t.Fatalf("MergeTextChunks mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestMergeTextChunks_GapSeparator(t *testing.T) { + chunks := []*types.Chunk{ + {Content: "first", ChunkType: types.ChunkTypeText, StartAt: 0, EndAt: 5, ChunkIndex: 0}, + {Content: "second", ChunkType: types.ChunkTypeText, StartAt: 100, EndAt: 106, ChunkIndex: 1}, + } + got := MergeTextChunks(chunks, "\n") + want := "first\nsecond" + if got != want { + t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want) + } +}