feat: 批量更新tag时更新索引字段
This commit is contained in:
@@ -1235,3 +1235,64 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
log.Infof("[ElasticsearchV7] Successfully batch updated chunk enabled status")
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkTagID(
|
||||
ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkTagMap) == 0 {
|
||||
log.Warnf("[ElasticsearchV7] Chunk tag map is empty, skipping update")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[ElasticsearchV7] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
|
||||
|
||||
// Group chunks by tag ID for batch updates
|
||||
tagGroups := make(map[string][]string)
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
|
||||
}
|
||||
|
||||
// Batch update chunks for each tag ID using update_by_query
|
||||
for tagID, chunkIDs := range tagGroups {
|
||||
query := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"chunk_id.keyword": chunkIDs,
|
||||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
"source": "ctx._source.tag_id = params.tag_id",
|
||||
"lang": "painless",
|
||||
"params": map[string]interface{}{
|
||||
"tag_id": tagID,
|
||||
},
|
||||
},
|
||||
}
|
||||
queryJSON, _ := json.Marshal(query)
|
||||
res, err := esapi.UpdateByQueryRequest{
|
||||
Index: []string{e.index},
|
||||
Body: strings.NewReader(string(queryJSON)),
|
||||
}.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
log.Errorf("[ElasticsearchV7] Failed to update chunks with tag_id %s: %v", tagID, err)
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.IsError() {
|
||||
var e map[string]interface{}
|
||||
if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
|
||||
log.Errorf("[ElasticsearchV7] Error parsing the response body: %v", err)
|
||||
} else {
|
||||
log.Errorf("[ElasticsearchV7] Error updating chunks with tag_id: %v", e["error"])
|
||||
}
|
||||
return fmt.Errorf("elasticsearch update_by_query failed with status: %d", res.StatusCode)
|
||||
}
|
||||
log.Infof("[ElasticsearchV7] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID)
|
||||
}
|
||||
|
||||
log.Infof("[ElasticsearchV7] Successfully batch updated chunk tag ID")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -690,3 +690,55 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
log.Infof("[Elasticsearch] Successfully batch updated chunk enabled status")
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkTagID(
|
||||
ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkTagMap) == 0 {
|
||||
log.Warnf("[Elasticsearch] Chunk tag map is empty, skipping update")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Elasticsearch] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
|
||||
|
||||
// Group chunks by tag ID for batch updates
|
||||
tagGroups := make(map[string][]string)
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
|
||||
}
|
||||
|
||||
// Batch update chunks for each tag ID using update_by_query
|
||||
for tagID, chunkIDs := range tagGroups {
|
||||
query := types.NewQuery()
|
||||
query.Bool = &types.BoolQuery{
|
||||
Must: []types.Query{
|
||||
{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"chunk_id.keyword": chunkIDs,
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
source := "ctx._source.tag_id = params.tag_id"
|
||||
lang := scriptlanguage.Painless
|
||||
script := types.Script{
|
||||
Source: &source,
|
||||
Lang: &lang,
|
||||
Params: map[string]json.RawMessage{
|
||||
"tag_id": json.RawMessage(`"` + tagID + `"`),
|
||||
},
|
||||
}
|
||||
_, err := e.client.UpdateByQuery(e.index).Query(query).Script(&script).Do(ctx)
|
||||
if err != nil {
|
||||
log.Errorf("[Elasticsearch] Failed to update chunks with tag_id %s: %v", tagID, err)
|
||||
return err
|
||||
}
|
||||
log.Infof("[Elasticsearch] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID)
|
||||
}
|
||||
|
||||
log.Infof("[Elasticsearch] Successfully batch updated chunk tag ID")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -585,3 +585,35 @@ func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkS
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk enabled status")
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
|
||||
if len(chunkTagMap) == 0 {
|
||||
logger.GetLogger(ctx).Warnf("[Postgres] Chunk tag map is empty, skipping update")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
|
||||
|
||||
// Group chunks by tag ID for batch updates
|
||||
tagGroups := make(map[string][]string)
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
|
||||
}
|
||||
|
||||
// Batch update chunks for each tag ID
|
||||
for tagID, chunkIDs := range tagGroups {
|
||||
result := g.db.WithContext(ctx).Model(&pgVector{}).
|
||||
Where("chunk_id IN ?", chunkIDs).
|
||||
Update("tag_id", tagID)
|
||||
if result.Error != nil {
|
||||
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update chunks with tag_id %s: %v", tagID, result.Error)
|
||||
return result.Error
|
||||
}
|
||||
logger.GetLogger(ctx).
|
||||
Infof("[Postgres] Updated %d chunks to tag_id=%s, rows affected: %d", len(chunkIDs), tagID, result.RowsAffected)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -427,6 +427,58 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (q *qdrantRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkTagMap) == 0 {
|
||||
log.Warn("[Qdrant] Empty chunk tag map provided, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
|
||||
|
||||
// Get all collections that match our base name pattern
|
||||
collections, err := q.client.ListCollections(ctx)
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to list collections: %v", err)
|
||||
return fmt.Errorf("failed to list collections: %w", err)
|
||||
}
|
||||
|
||||
// Group chunks by tag ID for batch updates
|
||||
tagGroups := make(map[string][]string)
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
|
||||
}
|
||||
|
||||
// Update in all matching collections
|
||||
for _, collectionName := range collections {
|
||||
// Only process collections that start with our base name
|
||||
if len(collectionName) <= len(q.collectionBaseName) ||
|
||||
collectionName[:len(q.collectionBaseName)] != q.collectionBaseName {
|
||||
continue
|
||||
}
|
||||
|
||||
// Update chunks for each tag ID
|
||||
for tagID, chunkIDs := range tagGroups {
|
||||
_, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{
|
||||
CollectionName: collectionName,
|
||||
Payload: qdrant.NewValueMap(map[string]any{fieldTagID: tagID}),
|
||||
PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatchKeywords(fieldChunkID, chunkIDs...),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
log.Warnf("[Qdrant] Failed to update chunks with tag_id %s in %s: %v", tagID, collectionName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Batch update chunk tag ID completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Filter {
|
||||
must := make([]*qdrant.Condition, 0)
|
||||
mustNot := make([]*qdrant.Condition, 0)
|
||||
|
||||
@@ -3558,6 +3558,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
enabledUpdates := make(map[string]bool)
|
||||
tagUpdates := make(map[string]string)
|
||||
|
||||
// Handle ByTag updates first
|
||||
if len(req.ByTag) > 0 {
|
||||
@@ -3583,9 +3584,16 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
}
|
||||
|
||||
// Collect affected IDs for retriever sync
|
||||
if update.IsEnabled != nil && len(affectedIDs) > 0 {
|
||||
for _, id := range affectedIDs {
|
||||
enabledUpdates[id] = *update.IsEnabled
|
||||
if len(affectedIDs) > 0 {
|
||||
if update.IsEnabled != nil {
|
||||
for _, id := range affectedIDs {
|
||||
enabledUpdates[id] = *update.IsEnabled
|
||||
}
|
||||
}
|
||||
if update.TagID != nil {
|
||||
for _, id := range affectedIDs {
|
||||
tagUpdates[id] = *update.TagID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3644,6 +3652,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
}
|
||||
if chunk.TagID != newTagID {
|
||||
chunk.TagID = newTagID
|
||||
tagUpdates[chunk.ID] = newTagID
|
||||
needUpdate = true
|
||||
}
|
||||
}
|
||||
@@ -3669,8 +3678,8 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
// Sync enabled status to retriever engines
|
||||
if len(enabledUpdates) > 0 {
|
||||
// Sync to retriever engines
|
||||
if len(enabledUpdates) > 0 || len(tagUpdates) > 0 {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
@@ -3679,8 +3688,15 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
|
||||
return err
|
||||
if len(enabledUpdates) > 0 {
|
||||
if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(tagUpdates) > 0 {
|
||||
if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3810,9 +3826,27 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e
|
||||
resolvedTagID = tag.ID
|
||||
}
|
||||
|
||||
// Check if tag actually changed
|
||||
if chunk.TagID == resolvedTagID {
|
||||
return nil
|
||||
}
|
||||
|
||||
chunk.TagID = resolvedTagID
|
||||
chunk.UpdatedAt = time.Now()
|
||||
return s.chunkRepo.UpdateChunk(ctx, chunk)
|
||||
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync tag update to retriever engines
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return retrieveEngine.BatchUpdateChunkTagID(ctx, map[string]string{chunk.ID: resolvedTagID})
|
||||
}
|
||||
|
||||
// UpdateFAQEntryTagBatch updates tags for FAQ entries in batch.
|
||||
@@ -3889,7 +3923,26 @@ func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID stri
|
||||
}
|
||||
|
||||
if len(chunksToUpdate) > 0 {
|
||||
return s.chunkRepo.UpdateChunks(ctx, chunksToUpdate)
|
||||
if err := s.chunkRepo.UpdateChunks(ctx, chunksToUpdate); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync tag updates to retriever engines
|
||||
tagUpdates := make(map[string]string)
|
||||
for _, chunk := range chunksToUpdate {
|
||||
tagUpdates[chunk.ID] = chunk.TagID
|
||||
}
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -115,6 +115,19 @@ func (c *CompositeRetrieveEngine) BatchUpdateChunkEnabledStatus(
|
||||
})
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (c *CompositeRetrieveEngine) BatchUpdateChunkTagID(
|
||||
ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.BatchUpdateChunkTagID(ctx, chunkTagMap); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// concurrentRetrieve is a helper function for concurrent processing of retrieval parameters
|
||||
// and collecting results
|
||||
func concurrentRetrieve(
|
||||
|
||||
@@ -181,3 +181,11 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkEnabledStatu
|
||||
) error {
|
||||
return v.indexRepository.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap)
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkTagID(
|
||||
ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
return v.indexRepository.BatchUpdateChunkTagID(ctx, chunkTagMap)
|
||||
}
|
||||
|
||||
@@ -56,6 +56,10 @@ type RetrieveEngineRepository interface {
|
||||
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
|
||||
BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
// chunkTagMap: map of chunk ID to tag ID (empty string means no tag)
|
||||
BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error
|
||||
|
||||
// RetrieveEngine retrieves the engine
|
||||
RetrieveEngine
|
||||
}
|
||||
@@ -119,6 +123,10 @@ type RetrieveEngineService interface {
|
||||
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
|
||||
BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error
|
||||
|
||||
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
|
||||
// chunkTagMap: map of chunk ID to tag ID (empty string means no tag)
|
||||
BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error
|
||||
|
||||
// RetrieveEngine retrieves the engine
|
||||
RetrieveEngine
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user