4a2be2b878
- Updated the `uploadKnowledgeFile` function to accept an optional progress callback, enabling real-time upload progress tracking. - Refactored the document upload handling in the menu component to support multiple file uploads with individual progress events. - Removed deprecated document processing state management from the KnowledgeBase view, streamlining the upload process. - Improved user feedback during file uploads by dispatching events for upload start, progress, and completion.
193 lines
6.6 KiB
Go
193 lines
6.6 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
|
|
"github.com/Tencent/WeKnora/internal/common"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// chunkRepository implements the ChunkRepository interface
|
|
type chunkRepository struct {
|
|
db *gorm.DB
|
|
}
|
|
|
|
// NewChunkRepository creates a new chunk repository
|
|
func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
|
|
return &chunkRepository{db: db}
|
|
}
|
|
|
|
// CreateChunks creates multiple chunks in batches
|
|
func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
|
|
for _, chunk := range chunks {
|
|
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
|
|
}
|
|
return r.db.WithContext(ctx).CreateInBatches(chunks, 100).Error
|
|
}
|
|
|
|
// GetChunkByID retrieves a chunk by its ID and tenant ID
|
|
func (r *chunkRepository) GetChunkByID(ctx context.Context, tenantID uint, id string) (*types.Chunk, error) {
|
|
var chunk types.Chunk
|
|
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&chunk).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil, errors.New("chunk not found")
|
|
}
|
|
return nil, err
|
|
}
|
|
return &chunk, nil
|
|
}
|
|
|
|
// ListChunksByID retrieves multiple chunks by their IDs
|
|
func (r *chunkRepository) ListChunksByID(
|
|
ctx context.Context, tenantID uint, ids []string,
|
|
) ([]*types.Chunk, error) {
|
|
var chunks []*types.Chunk
|
|
if err := r.db.Debug().WithContext(ctx).
|
|
Where("tenant_id = ? AND id IN ?", tenantID, ids).
|
|
Find(&chunks).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return chunks, nil
|
|
}
|
|
|
|
// ListChunksByKnowledgeID lists all chunks for a knowledge ID
|
|
func (r *chunkRepository) ListChunksByKnowledgeID(
|
|
ctx context.Context, tenantID uint, knowledgeID string,
|
|
) ([]*types.Chunk, error) {
|
|
var chunks []*types.Chunk
|
|
if err := r.db.WithContext(ctx).
|
|
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info").
|
|
Where("tenant_id = ? AND knowledge_id = ? and chunk_type = ?", tenantID, knowledgeID, "text").
|
|
Order("chunk_index ASC").
|
|
Find(&chunks).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return chunks, nil
|
|
}
|
|
|
|
// ListPagedChunksByKnowledgeID lists chunks for a knowledge ID with pagination
|
|
func (r *chunkRepository) ListPagedChunksByKnowledgeID(
|
|
ctx context.Context,
|
|
tenantID uint,
|
|
knowledgeID string,
|
|
page *types.Pagination,
|
|
chunkType []types.ChunkType,
|
|
tagID string,
|
|
) ([]*types.Chunk, int64, error) {
|
|
var chunks []*types.Chunk
|
|
var total int64
|
|
|
|
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
|
Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
|
|
if tagID != "" {
|
|
query = query.Where("tag_id = ?", tagID)
|
|
}
|
|
|
|
// First query the total count
|
|
if err := query.Count(&total).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
// Then query the paginated data
|
|
dataQuery := r.db.WithContext(ctx).
|
|
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id")
|
|
dataQuery = dataQuery.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
|
|
if tagID != "" {
|
|
dataQuery = dataQuery.Where("tag_id = ?", tagID)
|
|
}
|
|
|
|
if err := dataQuery.
|
|
Order("chunk_index ASC").
|
|
Offset(page.Offset()).
|
|
Limit(page.Limit()).
|
|
Find(&chunks).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
return chunks, total, nil
|
|
}
|
|
|
|
func (r *chunkRepository) ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) {
|
|
var chunks []*types.Chunk
|
|
if err := r.db.WithContext(ctx).
|
|
Where("tenant_id = ? AND parent_chunk_id = ?", tenantID, parentID).
|
|
Find(&chunks).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return chunks, nil
|
|
}
|
|
|
|
// UpdateChunk updates a chunk
|
|
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
|
|
return r.db.WithContext(ctx).Save(chunk).Error
|
|
}
|
|
|
|
// UpdateChunks updates chunks in batch
|
|
func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error {
|
|
if len(chunks) == 0 {
|
|
return nil
|
|
}
|
|
for _, chunk := range chunks {
|
|
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
|
|
}
|
|
return r.db.WithContext(ctx).Save(chunks).Error
|
|
}
|
|
|
|
// DeleteChunk deletes a chunk by its ID
|
|
func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint, id string) error {
|
|
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error
|
|
}
|
|
|
|
// DeleteChunks deletes chunks by IDs in batch
|
|
func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint, ids []string) error {
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error
|
|
}
|
|
|
|
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
|
|
func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error {
|
|
return r.db.WithContext(ctx).Where(
|
|
"tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID,
|
|
).Delete(&types.Chunk{}).Error
|
|
}
|
|
|
|
// DeleteByKnowledgeList deletes all chunks for a knowledge list
|
|
func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID uint, knowledgeIDs []string) error {
|
|
return r.db.WithContext(ctx).Where(
|
|
"tenant_id = ? AND knowledge_id in ?", tenantID, knowledgeIDs,
|
|
).Delete(&types.Chunk{}).Error
|
|
}
|
|
|
|
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
|
|
func (r *chunkRepository) CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint, kbID string) (int64, error) {
|
|
var count int64
|
|
err := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
|
Count(&count).Error
|
|
return count, err
|
|
}
|
|
|
|
func (r *chunkRepository) DeleteChunksByChunkIndexRange(ctx context.Context, tenantID uint, knowledgeID string, startChunkIndex int, endChunkIndex int) ([]*types.Chunk, error) {
|
|
var chunks []*types.Chunk
|
|
// 先查询要删除的chunks
|
|
if err := r.db.WithContext(ctx).
|
|
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
|
|
Find(&chunks).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
// 然后删除它们
|
|
if len(chunks) > 0 {
|
|
if err := r.db.WithContext(ctx).
|
|
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
|
|
Delete(&types.Chunk{}).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return chunks, nil
|
|
}
|