Files
Enterprise-Knowledge-Base/internal/application/repository/chunk.go
T
wizardchen 4a2be2b878 feat: Enhance file upload functionality with progress tracking
- Updated the `uploadKnowledgeFile` function to accept an optional progress callback, enabling real-time upload progress tracking.
- Refactored the document upload handling in the menu component to support multiple file uploads with individual progress events.
- Removed deprecated document processing state management from the KnowledgeBase view, streamlining the upload process.
- Improved user feedback during file uploads by dispatching events for upload start, progress, and completion.
2025-11-24 00:21:42 +08:00

193 lines
6.6 KiB
Go

package repository
import (
"context"
"errors"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
)
// chunkRepository implements the ChunkRepository interface
type chunkRepository struct {
db *gorm.DB
}
// NewChunkRepository creates a new chunk repository
func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
return &chunkRepository{db: db}
}
// CreateChunks creates multiple chunks in batches
func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
}
return r.db.WithContext(ctx).CreateInBatches(chunks, 100).Error
}
// GetChunkByID retrieves a chunk by its ID and tenant ID
func (r *chunkRepository) GetChunkByID(ctx context.Context, tenantID uint, id string) (*types.Chunk, error) {
var chunk types.Chunk
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&chunk).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New("chunk not found")
}
return nil, err
}
return &chunk, nil
}
// ListChunksByID retrieves multiple chunks by their IDs
func (r *chunkRepository) ListChunksByID(
ctx context.Context, tenantID uint, ids []string,
) ([]*types.Chunk, error) {
var chunks []*types.Chunk
if err := r.db.Debug().WithContext(ctx).
Where("tenant_id = ? AND id IN ?", tenantID, ids).
Find(&chunks).Error; err != nil {
return nil, err
}
return chunks, nil
}
// ListChunksByKnowledgeID lists all chunks for a knowledge ID
func (r *chunkRepository) ListChunksByKnowledgeID(
ctx context.Context, tenantID uint, knowledgeID string,
) ([]*types.Chunk, error) {
var chunks []*types.Chunk
if err := r.db.WithContext(ctx).
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info").
Where("tenant_id = ? AND knowledge_id = ? and chunk_type = ?", tenantID, knowledgeID, "text").
Order("chunk_index ASC").
Find(&chunks).Error; err != nil {
return nil, err
}
return chunks, nil
}
// ListPagedChunksByKnowledgeID lists chunks for a knowledge ID with pagination
func (r *chunkRepository) ListPagedChunksByKnowledgeID(
ctx context.Context,
tenantID uint,
knowledgeID string,
page *types.Pagination,
chunkType []types.ChunkType,
tagID string,
) ([]*types.Chunk, int64, error) {
var chunks []*types.Chunk
var total int64
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
if tagID != "" {
query = query.Where("tag_id = ?", tagID)
}
// First query the total count
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
// Then query the paginated data
dataQuery := r.db.WithContext(ctx).
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id")
dataQuery = dataQuery.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
if tagID != "" {
dataQuery = dataQuery.Where("tag_id = ?", tagID)
}
if err := dataQuery.
Order("chunk_index ASC").
Offset(page.Offset()).
Limit(page.Limit()).
Find(&chunks).Error; err != nil {
return nil, 0, err
}
return chunks, total, nil
}
func (r *chunkRepository) ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) {
var chunks []*types.Chunk
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND parent_chunk_id = ?", tenantID, parentID).
Find(&chunks).Error; err != nil {
return nil, err
}
return chunks, nil
}
// UpdateChunk updates a chunk
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
return r.db.WithContext(ctx).Save(chunk).Error
}
// UpdateChunks updates chunks in batch
func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error {
if len(chunks) == 0 {
return nil
}
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
}
return r.db.WithContext(ctx).Save(chunks).Error
}
// DeleteChunk deletes a chunk by its ID
func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint, id string) error {
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error
}
// DeleteChunks deletes chunks by IDs in batch
func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint, ids []string) error {
if len(ids) == 0 {
return nil
}
return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error
}
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error {
return r.db.WithContext(ctx).Where(
"tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID,
).Delete(&types.Chunk{}).Error
}
// DeleteByKnowledgeList deletes all chunks for a knowledge list
func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID uint, knowledgeIDs []string) error {
return r.db.WithContext(ctx).Where(
"tenant_id = ? AND knowledge_id in ?", tenantID, knowledgeIDs,
).Delete(&types.Chunk{}).Error
}
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
func (r *chunkRepository) CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint, kbID string) (int64, error) {
var count int64
err := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
Count(&count).Error
return count, err
}
func (r *chunkRepository) DeleteChunksByChunkIndexRange(ctx context.Context, tenantID uint, knowledgeID string, startChunkIndex int, endChunkIndex int) ([]*types.Chunk, error) {
var chunks []*types.Chunk
// 先查询要删除的chunks
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
Find(&chunks).Error; err != nil {
return nil, err
}
// 然后删除它们
if len(chunks) > 0 {
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
Delete(&types.Chunk{}).Error; err != nil {
return nil, err
}
}
return chunks, nil
}