diff --git a/frontend/src/components/css/markdown.less b/frontend/src/components/css/markdown.less index 4c6d2f77..203ef056 100644 --- a/frontend/src/components/css/markdown.less +++ b/frontend/src/components/css/markdown.less @@ -29,7 +29,7 @@ h6 { margin-top: 5px; font-weight: bold; - color: var(--td-text-color-placeholder); + color: var(--td-text-color-primary); font-family: var(--app-font-family); transition: all 0.2s ease-out; font-size: 20px; @@ -115,8 +115,10 @@ hr { padding: 0; - margin: 32px 0; - border-top: 0.5rem dotted var(--td-brand-color-focus); + margin: 24px 0; + height: 1px; + border: none; + background: var(--td-component-stroke); overflow: hidden; box-sizing: content-box; } diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index 6136d63a..c91804f6 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -225,9 +225,40 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged'); // 合并后的文档内容(在下方通过 computed 定义) /** - * 根据 start_at 和 end_at 字段合并有 overlap 的 chunks - * 返回合并后的完整文档内容 - * 实现逻辑与后端 Go 代码保持一致 + * 把已合并文本 acc 和下一个 chunk 内容 next 拼接,并去除两者的重叠部分。 + * + * 不再依赖 start_at / end_at 做位置裁剪,而是用「文本重叠匹配」:在 next 的 + * 开头窗口里找 acc 后缀首次出现的位置,从该位置之后接上。这样能同时兼容: + * 1. chunker 给拆分表格补写的表头(零宽 start/end,位置上不可见)——表头出现 + * 在重叠行之前,会被自然跳过; + * 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是 + * 文本本身,不受长度偏差影响。 + * + * @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。 + */ +const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => { + if (!acc) return next; + if (!next) return acc; + + const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略 + const span = Math.max(positionOverlap, 0); + // 搜索的后缀最大长度;按位置重叠量放大几倍兜底,并设下限 + const maxK = Math.min(acc.length, next.length, Math.max(span * 3, 400)); + // 重叠行之前最多允许多少前缀(补写的表头)被跳过 + const headSlack = Math.max(span * 2, 320); + + for (let k = maxK; k >= MIN_OVERLAP; k--) { + const suffix = acc.slice(acc.length - k); + const pos = next.indexOf(suffix); + if (pos !== -1 && pos <= headSlack) { + return acc + next.slice(pos + k); + } + } + return acc + next; +}; + +/** + * 合并分块内容,还原完整文档。chunks 按 start_at 排序后逐段用文本重叠匹配拼接。 */ const mergeChunks = (chunks: any[]): string => { if (!chunks || chunks.length === 0) return ''; @@ -239,54 +270,31 @@ const mergeChunks = (chunks: any[]): string => { return startA - startB; }); - // 初始化合并结果,第一个 chunk 直接加入 - const mergedChunks: Array<{ - content: string; - start_at: number; - end_at: number; - }> = [{ - content: sortedChunks[0].content || '', - start_at: sortedChunks[0].start_at ?? 0, - end_at: sortedChunks[0].end_at ?? 0 - }]; + let merged = sortedChunks[0].content || ''; + let mergedEnd = sortedChunks[0].end_at ?? 0; - // 从第二个 chunk 开始遍历 for (let i = 1; i < sortedChunks.length; i++) { const currentChunk = sortedChunks[i]; - const lastChunk = mergedChunks[mergedChunks.length - 1]; - const currentStartAt = currentChunk.start_at ?? 0; const currentEndAt = currentChunk.end_at ?? 0; const currentContent = currentChunk.content || ''; - // 如果当前 chunk 的起始位置在最后一个 chunk 的结束位置之后,直接添加 - if (currentStartAt > lastChunk.end_at) { - mergedChunks.push({ - content: currentContent, - start_at: currentStartAt, - end_at: currentEndAt - }); - continue; + if (!currentContent) continue; + + // 与上一段有明显间隙(位置不相邻),用空行分隔后整段拼接 + if (currentStartAt > mergedEnd && mergedEnd > 0) { + merged = merged + '\n\n' + currentContent; + } else { + const positionOverlap = mergedEnd - currentStartAt; + merged = appendChunkContent(merged, currentContent, positionOverlap); } - // 合并重叠的 chunks - if (currentEndAt > lastChunk.end_at) { - // 将内容转换为字符数组以正确处理多字节字符 - const contentRunes = Array.from(currentContent); - const contentLength = contentRunes.length; - - // 计算偏移量:内容长度 - (当前结束位置 - 上一个结束位置) - const offset = contentLength - (currentEndAt - lastChunk.end_at); - - // 拼接非重叠部分 - const newContent = contentRunes.slice(offset).join(''); - lastChunk.content = lastChunk.content + newContent; - lastChunk.end_at = currentEndAt; + if (currentEndAt > mergedEnd) { + mergedEnd = currentEndAt; } } - // 合并所有段落,用双换行符连接 - return mergedChunks.map(chunk => chunk.content).join('\n\n'); + return merged; }; onMounted(() => { diff --git a/internal/application/service/chat_pipeline/merge_overlap.go b/internal/application/service/chat_pipeline/merge_overlap.go index 009d11d3..7b786206 100644 --- a/internal/application/service/chat_pipeline/merge_overlap.go +++ b/internal/application/service/chat_pipeline/merge_overlap.go @@ -5,6 +5,7 @@ import ( "encoding/json" "sort" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" ) @@ -32,35 +33,14 @@ func (p *PluginMerge) mergeOverlappingChunks( // Partial overlap: append the non-overlapping suffix. // - // Offset math assumes len([]rune(Content)) == EndAt-StartAt, but a - // few upstream paths break that invariant: - // - Parent-child chunker prepends table headers, so Content may be - // longer than EndAt-StartAt. - // - Legacy data / mixed chunk sources may carry EndAt-StartAt that - // exceed the actual rune length of Content. - // We clamp offset into [0, len(contentRunes)] so the merge degrades - // gracefully instead of panicking with a negative slice bound. + // 重叠去重统一交给 searchutil.AppendWithOverlap(按文本匹配),它能 + // 兼容父子分块器补写的零宽表头,以及 HTML 实体导致的 content 长度与 + // EndAt-StartAt 不一致——这两种情况都会让按位置裁剪错位、丢字或重复。 + // StartAt/EndAt 仅用于估算搜索窗口大小。 if chunks[i].EndAt > lastChunk.EndAt { - contentRunes := []rune(chunks[i].Content) - suffixLen := chunks[i].EndAt - lastChunk.EndAt - offset := len(contentRunes) - suffixLen - if offset < 0 || offset > len(contentRunes) { - pipelineWarn(ctx, "Merge", "overlap_offset_clamp", map[string]interface{}{ - "knowledge_id": knowledgeID, - "chunk_id": chunks[i].ID, - "content_runes": len(contentRunes), - "chunk_start": chunks[i].StartAt, - "chunk_end": chunks[i].EndAt, - "last_end": lastChunk.EndAt, - "computed_offset": offset, - }) - if offset < 0 { - offset = 0 - } else { - offset = len(contentRunes) - } - } - lastChunk.Content = lastChunk.Content + string(contentRunes[offset:]) + lastChunk.Content = searchutil.AppendWithOverlap( + lastChunk.Content, chunks[i].Content, lastChunk.EndAt-chunks[i].StartAt, + ) lastChunk.EndAt = chunks[i].EndAt lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) diff --git a/internal/application/service/graph.go b/internal/application/service/graph.go index 3a63831d..767d5935 100644 --- a/internal/application/service/graph.go +++ b/internal/application/service/graph.go @@ -15,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" "golang.org/x/sync/errgroup" @@ -345,29 +346,9 @@ func (b *graphBuilder) findRelationChunkIDs(source, target string, entities []*t // mergeChunkContents merges content from multiple document chunks // It accounts for overlapping portions between chunks to ensure coherent content func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string { - if len(chunks) == 0 { - return "" - } - - chunkContents := chunks[0].Content - preChunk := chunks[0] - - for i := 1; i < len(chunks); i++ { - // Only add non-overlapping content parts - if preChunk.EndAt > chunks[i].StartAt { - // Calculate overlap starting position - startPos := preChunk.EndAt - chunks[i].StartAt - if startPos >= 0 && startPos < len([]rune(chunks[i].Content)) { - chunkContents = chunkContents + string([]rune(chunks[i].Content)[startPos:]) - } - } else { - // If there's no overlap between chunks, add all content - chunkContents = chunkContents + chunks[i].Content - } - preChunk = chunks[i] - } - - return chunkContents + // 重叠去重统一交给公共逻辑(按文本匹配,兼容补写表头 / HTML 实体)。 + // 无间隙分隔符,保持与原实现一致的直接拼接行为。 + return searchutil.MergeTextChunks(chunks, "") } // BuildGraph constructs the knowledge graph diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index f165806b..2f3622da 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -1878,47 +1878,8 @@ func reconstructContent(chunks []*types.Chunk) string { } } - // Sort by StartAt, then ChunkIndex - sort.Slice(textChunks, func(i, j int) bool { - if textChunks[i].StartAt == textChunks[j].StartAt { - return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex - } - return textChunks[i].StartAt < textChunks[j].StartAt - }) - - var sb strings.Builder - lastEndAt := -1 - for _, c := range textChunks { - toAppend := c.Content - - if c.StartAt > lastEndAt || c.EndAt == 0 { - // Non-overlapping or missing position info - if sb.Len() > 0 { - sb.WriteString("\n") - } - sb.WriteString(toAppend) - if c.EndAt > 0 { - lastEndAt = c.EndAt - } - } else if c.EndAt > lastEndAt { - // Partial overlap - contentRunes := []rune(toAppend) - offset := len(contentRunes) - (c.EndAt - lastEndAt) - if offset >= 0 && offset < len(contentRunes) { - sb.WriteString(string(contentRunes[offset:])) - } else { - // Fallback if offset calculation is invalid - if sb.Len() > 0 { - sb.WriteString("\n") - } - sb.WriteString(toAppend) - } - lastEndAt = c.EndAt - } - // If c.EndAt <= lastEndAt, it's fully contained, so skip appending text - } - - return sb.String() + // 重叠去重与排序统一交给公共逻辑(按文本匹配,兼容补写表头 / HTML 实体)。 + return searchutil.MergeTextChunks(textChunks, "\n") } // reconstructEnrichedContent rebuilds document text and inlines image_info diff --git a/internal/searchutil/chunkmerge.go b/internal/searchutil/chunkmerge.go new file mode 100644 index 00000000..c8a4bc2d --- /dev/null +++ b/internal/searchutil/chunkmerge.go @@ -0,0 +1,163 @@ +package searchutil + +import ( + "sort" + + "github.com/Tencent/WeKnora/internal/types" +) + +// 这里实现 chunk 内容的「重叠拼接」公共逻辑,供文档重建(reconstructContent)、 +// 知识图谱内容合并(graph mergeChunkContents)、检索结果重叠合并 +// (chat_pipeline mergeOverlappingChunks)等多处复用。 +// +// 历史上各处都用「按位置」的公式裁剪重叠(offset = len(content) - (EndAt - +// lastEndAt) 之类),它默认 len([]rune(Content)) == EndAt-StartAt。但有两类 +// 数据会破坏这个不变式,导致拼接错位、丢字或重复: +// 1. 父子分块器会给被拆开的表格「补写表头」,补进去的表头是零宽度的 +// (start == end),位置坐标无法表达它,content 比 EndAt-StartAt 更长; +// 2. content 里可能保留 HTML 实体(如 " / >),其字符数比原文区间长。 +// +// 因此这里改为「按文本」匹配重叠:在下一段开头的窗口里查找已合并文本的后缀首次 +// 出现的位置,从该位置之后接上。位置信息(StartAt/EndAt)仅用于估算搜索窗口 +// 大小,不再用于裁剪。 + +const ( + // minOverlapRunes 是参与匹配的最短后缀长度。太短(如表格分隔行 |---|) + // 容易误匹配,因此忽略。 + minOverlapRunes = 12 + // defaultSearchSpan 是搜索窗口的下限,保证即使位置信息缺失/为 0 也能 + // 检测到一定范围内的真实重叠。 + defaultSearchSpan = 400 +) + +// AppendWithOverlap 把 next 追加到 acc 之后,并去除二者之间的重叠部分。 +// +// positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于 +// 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。 +// 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。 +func AppendWithOverlap(acc, next string, positionOverlap int) string { + if acc == "" { + return next + } + if next == "" { + return acc + } + + accRunes := []rune(acc) + nextRunes := []rune(next) + + span := positionOverlap + if span < 0 { + span = 0 + } + + maxK := minInt(len(accRunes), len(nextRunes)) + if cap := maxInt(span*3, defaultSearchSpan); maxK > cap { + maxK = cap + } + // 重叠内容之前最多允许跳过多少前缀(即补写的表头等合成文本)。 + headSlack := maxInt(span*2, 320) + + for k := maxK; k >= minOverlapRunes; k-- { + needle := accRunes[len(accRunes)-k:] + if pos := indexRunes(nextRunes, needle, headSlack); pos >= 0 { + return acc + string(nextRunes[pos+k:]) + } + } + return acc + next +} + +// MergeTextChunks 按 StartAt(并列时按 ChunkIndex)排序后,用 AppendWithOverlap +// 把多个 chunk 的内容重建为完整文本。gapSep 用于位置不相邻(有间隙)的两段之间 +// 的分隔符(如 "\n"),传空串则直接拼接。 +// +// 调用方负责先做类型过滤(例如只保留文本 chunk);本函数不感知 ChunkType。 +func MergeTextChunks(chunks []*types.Chunk, gapSep string) string { + if len(chunks) == 0 { + return "" + } + + sorted := make([]*types.Chunk, len(chunks)) + copy(sorted, chunks) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].StartAt == sorted[j].StartAt { + return sorted[i].ChunkIndex < sorted[j].ChunkIndex + } + return sorted[i].StartAt < sorted[j].StartAt + }) + + merged := "" + mergedEnd := -1 + for _, c := range sorted { + if c == nil || c.Content == "" { + continue + } + if merged == "" { + merged = c.Content + if c.EndAt > 0 { + mergedEnd = c.EndAt + } + continue + } + + // 间隙 / 位置信息缺失(EndAt==0):作为独立段落拼接,不做重叠裁剪。 + if c.StartAt > mergedEnd || c.EndAt == 0 { + if gapSep != "" { + merged += gapSep + } + merged += c.Content + if c.EndAt > 0 { + mergedEnd = c.EndAt + } + continue + } + + // 部分重叠或首尾相接:按文本匹配去重叠后拼接。 + if c.EndAt > mergedEnd { + merged = AppendWithOverlap(merged, c.Content, mergedEnd-c.StartAt) + mergedEnd = c.EndAt + } + // 否则被上一段完全覆盖,跳过。 + } + + return merged +} + +// indexRunes 在 haystack 中查找 needle 首次出现的 rune 下标,且起始位置不超过 +// maxStart。找不到返回 -1。 +func indexRunes(haystack, needle []rune, maxStart int) int { + if len(needle) == 0 || len(needle) > len(haystack) { + return -1 + } + limit := len(haystack) - len(needle) + if maxStart < limit { + limit = maxStart + } + for i := 0; i <= limit; i++ { + match := true + for j := 0; j < len(needle); j++ { + if haystack[i+j] != needle[j] { + match = false + break + } + } + if match { + return i + } + } + return -1 +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/searchutil/chunkmerge_test.go b/internal/searchutil/chunkmerge_test.go new file mode 100644 index 00000000..05fe9efa --- /dev/null +++ b/internal/searchutil/chunkmerge_test.go @@ -0,0 +1,91 @@ +package searchutil + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestAppendWithOverlap_ContiguousNoTrim(t *testing.T) { + // 首尾相接(无重叠),且 next 含 HTML 实体使其字符数 > EndAt-StartAt。 + // 旧的按位置公式会多切掉开头,这里应当整段保留。 + acc := "## 第二节\n\n" + next := "| 列A | 列B |\n| 值1 | 含实体"引号"的内容 |\n" + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("contiguous merge mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_PrependedTableHeaderSkipped(t *testing.T) { + // 模拟 chunker 给拆分表格补写表头:next 开头多了一份表头(零宽,位置不可见), + // 真正的重叠行在表头之后。按位置裁剪会错位,按文本匹配应正确去重。 + header := "| 列1 | 列2 | 列3 |\n|:---|:---|:---|\n" + overlapRows := "| 第5行 | 内容5A | 内容5B |\n| 第6行 | 内容6A | 内容6B |\n" + accTail := "| 第4行 | 内容4A | 内容4B |\n" + overlapRows + acc := header + "| 第1行 | x | — |\n" + accTail + + newRows := "| 第7行 | 内容7A | 内容7B |\n| 第8行 | 内容8A | 内容8B |\n" + next := header + overlapRows + newRows + + // 位置重叠量大约是两行的长度(这里给个近似值即可,仅用于窗口估算)。 + got := AppendWithOverlap(acc, next, len([]rune(overlapRows))) + want := acc + newRows + if got != want { + t.Fatalf("prepended-header merge mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_PlainOverlap(t *testing.T) { + acc := "abcdefghijklmnopqrstuvwxyz0123" + // next 与 acc 末尾 "klmnopqrstuvwxyz0123" 重叠,再接新内容 + next := "klmnopqrstuvwxyz0123ABCDEFG" + got := AppendWithOverlap(acc, next, 20) + want := "abcdefghijklmnopqrstuvwxyz0123ABCDEFG" + if got != want { + t.Fatalf("plain overlap mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithOverlap_NoOverlap(t *testing.T) { + acc := "hello world" + next := "completely different" + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("no overlap mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestMergeTextChunks_OrdersFiltersAndStitches(t *testing.T) { + header := "| a | b |\n|:--|:--|\n" + chunks := []*types.Chunk{ + { + Content: header + "| r1 | x |\n| r2 | y |\n", ChunkType: types.ChunkTypeText, + StartAt: 0, EndAt: 20, ChunkIndex: 0, + }, + { + // 补写表头 + 与上一段 r2 重叠 + 新行 r3 + Content: header + "| r2 | y |\n| r3 | z |\n", ChunkType: types.ChunkTypeText, + StartAt: 10, EndAt: 40, ChunkIndex: 1, + }, + } + got := MergeTextChunks(chunks, "\n") + want := header + "| r1 | x |\n| r2 | y |\n" + "| r3 | z |\n" + if got != want { + t.Fatalf("MergeTextChunks mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestMergeTextChunks_GapSeparator(t *testing.T) { + chunks := []*types.Chunk{ + {Content: "first", ChunkType: types.ChunkTypeText, StartAt: 0, EndAt: 5, ChunkIndex: 0}, + {Content: "second", ChunkType: types.ChunkTypeText, StartAt: 100, EndAt: 106, ChunkIndex: 1}, + } + got := MergeTextChunks(chunks, "\n") + want := "first\nsecond" + if got != want { + t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want) + } +}