Files
Enterprise-Knowledge-Base/internal/application/service/knowledge_text_chunks.go

28 lines
820 B
Go

package service
import "github.com/Tencent/WeKnora/internal/types"
// processTextChunks selects indexing inputs without linking parent-child context
// as if it were a flat document.
func processTextChunks(chunks []*types.Chunk, hasParentChild bool) []*types.Chunk {
textChunks := make([]*types.Chunk, 0, len(chunks))
for _, chunk := range chunks {
// Parent-less text can coexist with parent-child output (for example
// table chunks). Persisting it without indexing silently loses recall.
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
if !hasParentChild {
for i, chunk := range textChunks {
if i > 0 {
textChunks[i-1].NextChunkID = chunk.ID
}
if i < len(textChunks)-1 {
textChunks[i+1].PreChunkID = chunk.ID
}
}
}
return textChunks
}