28 lines
820 B
Go
28 lines
820 B
Go
package service
|
|
|
|
import "github.com/Tencent/WeKnora/internal/types"
|
|
|
|
// processTextChunks selects indexing inputs without linking parent-child context
|
|
// as if it were a flat document.
|
|
func processTextChunks(chunks []*types.Chunk, hasParentChild bool) []*types.Chunk {
|
|
textChunks := make([]*types.Chunk, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
// Parent-less text can coexist with parent-child output (for example
|
|
// table chunks). Persisting it without indexing silently loses recall.
|
|
if chunk.ChunkType == types.ChunkTypeText {
|
|
textChunks = append(textChunks, chunk)
|
|
}
|
|
}
|
|
if !hasParentChild {
|
|
for i, chunk := range textChunks {
|
|
if i > 0 {
|
|
textChunks[i-1].NextChunkID = chunk.ID
|
|
}
|
|
if i < len(textChunks)-1 {
|
|
textChunks[i+1].PreChunkID = chunk.ID
|
|
}
|
|
}
|
|
}
|
|
return textChunks
|
|
}
|