fix(knowledge): stitch chunks by text overlap, not by position
Reconstructing full document content from chunks relied on position math (offset = content length - (EndAt - lastEndAt)) that assumes len([]rune(Content)) == EndAt - StartAt. Two upstream behaviors break that invariant and caused the merged "全文" view to drop or duplicate text: - The parent-child chunker prepends a synthetic table header to split tables; that header is zero-width in position space (start == end), so Content is longer than EndAt - StartAt. - Chunk Content can retain HTML entities (", > ...), inflating the rune count relative to the source span. Replace position-based trimming with text-overlap matching: find where the accumulated text's suffix reappears at the head of the next chunk and join there. Positions are only used to size the search window. This naturally skips the prepended header and is immune to entity length drift. Extract the logic into searchutil.AppendWithOverlap / MergeTextChunks and reuse it across the three backend stitching sites (wiki_ingest reconstructContent, graph mergeChunkContents, chat_pipeline mergeOverlappingChunks). Mirror the same algorithm in the frontend doc-content merge. Also fix the markdown hr/heading styles (thin solid divider, primary heading color).
This commit is contained in:
@@ -29,7 +29,7 @@
|
||||
h6 {
|
||||
margin-top: 5px;
|
||||
font-weight: bold;
|
||||
color: var(--td-text-color-placeholder);
|
||||
color: var(--td-text-color-primary);
|
||||
font-family: var(--app-font-family);
|
||||
transition: all 0.2s ease-out;
|
||||
font-size: 20px;
|
||||
@@ -115,8 +115,10 @@
|
||||
|
||||
hr {
|
||||
padding: 0;
|
||||
margin: 32px 0;
|
||||
border-top: 0.5rem dotted var(--td-brand-color-focus);
|
||||
margin: 24px 0;
|
||||
height: 1px;
|
||||
border: none;
|
||||
background: var(--td-component-stroke);
|
||||
overflow: hidden;
|
||||
box-sizing: content-box;
|
||||
}
|
||||
|
||||
@@ -225,9 +225,40 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged');
|
||||
// 合并后的文档内容(在下方通过 computed 定义)
|
||||
|
||||
/**
|
||||
* 根据 start_at 和 end_at 字段合并有 overlap 的 chunks
|
||||
* 返回合并后的完整文档内容
|
||||
* 实现逻辑与后端 Go 代码保持一致
|
||||
* 把已合并文本 acc 和下一个 chunk 内容 next 拼接,并去除两者的重叠部分。
|
||||
*
|
||||
* 不再依赖 start_at / end_at 做位置裁剪,而是用「文本重叠匹配」:在 next 的
|
||||
* 开头窗口里找 acc 后缀首次出现的位置,从该位置之后接上。这样能同时兼容:
|
||||
* 1. chunker 给拆分表格补写的表头(零宽 start/end,位置上不可见)——表头出现
|
||||
* 在重叠行之前,会被自然跳过;
|
||||
* 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是
|
||||
* 文本本身,不受长度偏差影响。
|
||||
*
|
||||
* @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。
|
||||
*/
|
||||
const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => {
|
||||
if (!acc) return next;
|
||||
if (!next) return acc;
|
||||
|
||||
const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略
|
||||
const span = Math.max(positionOverlap, 0);
|
||||
// 搜索的后缀最大长度;按位置重叠量放大几倍兜底,并设下限
|
||||
const maxK = Math.min(acc.length, next.length, Math.max(span * 3, 400));
|
||||
// 重叠行之前最多允许多少前缀(补写的表头)被跳过
|
||||
const headSlack = Math.max(span * 2, 320);
|
||||
|
||||
for (let k = maxK; k >= MIN_OVERLAP; k--) {
|
||||
const suffix = acc.slice(acc.length - k);
|
||||
const pos = next.indexOf(suffix);
|
||||
if (pos !== -1 && pos <= headSlack) {
|
||||
return acc + next.slice(pos + k);
|
||||
}
|
||||
}
|
||||
return acc + next;
|
||||
};
|
||||
|
||||
/**
|
||||
* 合并分块内容,还原完整文档。chunks 按 start_at 排序后逐段用文本重叠匹配拼接。
|
||||
*/
|
||||
const mergeChunks = (chunks: any[]): string => {
|
||||
if (!chunks || chunks.length === 0) return '';
|
||||
@@ -239,54 +270,31 @@ const mergeChunks = (chunks: any[]): string => {
|
||||
return startA - startB;
|
||||
});
|
||||
|
||||
// 初始化合并结果,第一个 chunk 直接加入
|
||||
const mergedChunks: Array<{
|
||||
content: string;
|
||||
start_at: number;
|
||||
end_at: number;
|
||||
}> = [{
|
||||
content: sortedChunks[0].content || '',
|
||||
start_at: sortedChunks[0].start_at ?? 0,
|
||||
end_at: sortedChunks[0].end_at ?? 0
|
||||
}];
|
||||
let merged = sortedChunks[0].content || '';
|
||||
let mergedEnd = sortedChunks[0].end_at ?? 0;
|
||||
|
||||
// 从第二个 chunk 开始遍历
|
||||
for (let i = 1; i < sortedChunks.length; i++) {
|
||||
const currentChunk = sortedChunks[i];
|
||||
const lastChunk = mergedChunks[mergedChunks.length - 1];
|
||||
|
||||
const currentStartAt = currentChunk.start_at ?? 0;
|
||||
const currentEndAt = currentChunk.end_at ?? 0;
|
||||
const currentContent = currentChunk.content || '';
|
||||
|
||||
// 如果当前 chunk 的起始位置在最后一个 chunk 的结束位置之后,直接添加
|
||||
if (currentStartAt > lastChunk.end_at) {
|
||||
mergedChunks.push({
|
||||
content: currentContent,
|
||||
start_at: currentStartAt,
|
||||
end_at: currentEndAt
|
||||
});
|
||||
continue;
|
||||
if (!currentContent) continue;
|
||||
|
||||
// 与上一段有明显间隙(位置不相邻),用空行分隔后整段拼接
|
||||
if (currentStartAt > mergedEnd && mergedEnd > 0) {
|
||||
merged = merged + '\n\n' + currentContent;
|
||||
} else {
|
||||
const positionOverlap = mergedEnd - currentStartAt;
|
||||
merged = appendChunkContent(merged, currentContent, positionOverlap);
|
||||
}
|
||||
|
||||
// 合并重叠的 chunks
|
||||
if (currentEndAt > lastChunk.end_at) {
|
||||
// 将内容转换为字符数组以正确处理多字节字符
|
||||
const contentRunes = Array.from(currentContent);
|
||||
const contentLength = contentRunes.length;
|
||||
|
||||
// 计算偏移量:内容长度 - (当前结束位置 - 上一个结束位置)
|
||||
const offset = contentLength - (currentEndAt - lastChunk.end_at);
|
||||
|
||||
// 拼接非重叠部分
|
||||
const newContent = contentRunes.slice(offset).join('');
|
||||
lastChunk.content = lastChunk.content + newContent;
|
||||
lastChunk.end_at = currentEndAt;
|
||||
if (currentEndAt > mergedEnd) {
|
||||
mergedEnd = currentEndAt;
|
||||
}
|
||||
}
|
||||
|
||||
// 合并所有段落,用双换行符连接
|
||||
return mergedChunks.map(chunk => chunk.content).join('\n\n');
|
||||
return merged;
|
||||
};
|
||||
|
||||
onMounted(() => {
|
||||
|
||||
Reference in New Issue
Block a user