feat: 批量更新tag时更新索引字段

This commit is contained in:
wizardchen
2025-12-30 17:43:29 +08:00
committed by lyingbug
parent 8832d70e19
commit da1dcb0cda
8 changed files with 288 additions and 9 deletions
@@ -1235,3 +1235,64 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
log.Infof("[ElasticsearchV7] Successfully batch updated chunk enabled status")
return nil
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (e *elasticsearchRepository) BatchUpdateChunkTagID(
ctx context.Context,
chunkTagMap map[string]string,
) error {
log := logger.GetLogger(ctx)
if len(chunkTagMap) == 0 {
log.Warnf("[ElasticsearchV7] Chunk tag map is empty, skipping update")
return nil
}
log.Infof("[ElasticsearchV7] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
// Group chunks by tag ID for batch updates
tagGroups := make(map[string][]string)
for chunkID, tagID := range chunkTagMap {
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
}
// Batch update chunks for each tag ID using update_by_query
for tagID, chunkIDs := range tagGroups {
query := map[string]interface{}{
"query": map[string]interface{}{
"terms": map[string]interface{}{
"chunk_id.keyword": chunkIDs,
},
},
"script": map[string]interface{}{
"source": "ctx._source.tag_id = params.tag_id",
"lang": "painless",
"params": map[string]interface{}{
"tag_id": tagID,
},
},
}
queryJSON, _ := json.Marshal(query)
res, err := esapi.UpdateByQueryRequest{
Index: []string{e.index},
Body: strings.NewReader(string(queryJSON)),
}.Do(ctx, e.client)
if err != nil {
log.Errorf("[ElasticsearchV7] Failed to update chunks with tag_id %s: %v", tagID, err)
return err
}
defer res.Body.Close()
if res.IsError() {
var e map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
log.Errorf("[ElasticsearchV7] Error parsing the response body: %v", err)
} else {
log.Errorf("[ElasticsearchV7] Error updating chunks with tag_id: %v", e["error"])
}
return fmt.Errorf("elasticsearch update_by_query failed with status: %d", res.StatusCode)
}
log.Infof("[ElasticsearchV7] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID)
}
log.Infof("[ElasticsearchV7] Successfully batch updated chunk tag ID")
return nil
}
@@ -690,3 +690,55 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
log.Infof("[Elasticsearch] Successfully batch updated chunk enabled status")
return nil
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (e *elasticsearchRepository) BatchUpdateChunkTagID(
ctx context.Context,
chunkTagMap map[string]string,
) error {
log := logger.GetLogger(ctx)
if len(chunkTagMap) == 0 {
log.Warnf("[Elasticsearch] Chunk tag map is empty, skipping update")
return nil
}
log.Infof("[Elasticsearch] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
// Group chunks by tag ID for batch updates
tagGroups := make(map[string][]string)
for chunkID, tagID := range chunkTagMap {
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
}
// Batch update chunks for each tag ID using update_by_query
for tagID, chunkIDs := range tagGroups {
query := types.NewQuery()
query.Bool = &types.BoolQuery{
Must: []types.Query{
{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"chunk_id.keyword": chunkIDs,
},
}},
},
}
source := "ctx._source.tag_id = params.tag_id"
lang := scriptlanguage.Painless
script := types.Script{
Source: &source,
Lang: &lang,
Params: map[string]json.RawMessage{
"tag_id": json.RawMessage(`"` + tagID + `"`),
},
}
_, err := e.client.UpdateByQuery(e.index).Query(query).Script(&script).Do(ctx)
if err != nil {
log.Errorf("[Elasticsearch] Failed to update chunks with tag_id %s: %v", tagID, err)
return err
}
log.Infof("[Elasticsearch] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID)
}
log.Infof("[Elasticsearch] Successfully batch updated chunk tag ID")
return nil
}
@@ -585,3 +585,35 @@ func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkS
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk enabled status")
return nil
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
if len(chunkTagMap) == 0 {
logger.GetLogger(ctx).Warnf("[Postgres] Chunk tag map is empty, skipping update")
return nil
}
logger.GetLogger(ctx).Infof("[Postgres] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
// Group chunks by tag ID for batch updates
tagGroups := make(map[string][]string)
for chunkID, tagID := range chunkTagMap {
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
}
// Batch update chunks for each tag ID
for tagID, chunkIDs := range tagGroups {
result := g.db.WithContext(ctx).Model(&pgVector{}).
Where("chunk_id IN ?", chunkIDs).
Update("tag_id", tagID)
if result.Error != nil {
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update chunks with tag_id %s: %v", tagID, result.Error)
return result.Error
}
logger.GetLogger(ctx).
Infof("[Postgres] Updated %d chunks to tag_id=%s, rows affected: %d", len(chunkIDs), tagID, result.RowsAffected)
}
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID")
return nil
}
@@ -427,6 +427,58 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch
return nil
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (q *qdrantRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
log := logger.GetLogger(ctx)
if len(chunkTagMap) == 0 {
log.Warn("[Qdrant] Empty chunk tag map provided, skipping")
return nil
}
log.Infof("[Qdrant] Batch updating chunk tag ID, count: %d", len(chunkTagMap))
// Get all collections that match our base name pattern
collections, err := q.client.ListCollections(ctx)
if err != nil {
log.Errorf("[Qdrant] Failed to list collections: %v", err)
return fmt.Errorf("failed to list collections: %w", err)
}
// Group chunks by tag ID for batch updates
tagGroups := make(map[string][]string)
for chunkID, tagID := range chunkTagMap {
tagGroups[tagID] = append(tagGroups[tagID], chunkID)
}
// Update in all matching collections
for _, collectionName := range collections {
// Only process collections that start with our base name
if len(collectionName) <= len(q.collectionBaseName) ||
collectionName[:len(q.collectionBaseName)] != q.collectionBaseName {
continue
}
// Update chunks for each tag ID
for tagID, chunkIDs := range tagGroups {
_, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{
CollectionName: collectionName,
Payload: qdrant.NewValueMap(map[string]any{fieldTagID: tagID}),
PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldChunkID, chunkIDs...),
},
}),
})
if err != nil {
log.Warnf("[Qdrant] Failed to update chunks with tag_id %s in %s: %v", tagID, collectionName, err)
}
}
}
log.Infof("[Qdrant] Batch update chunk tag ID completed")
return nil
}
func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Filter {
must := make([]*qdrant.Condition, 0)
mustNot := make([]*qdrant.Condition, 0)
+62 -9
View File
@@ -3558,6 +3558,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enabledUpdates := make(map[string]bool)
tagUpdates := make(map[string]string)
// Handle ByTag updates first
if len(req.ByTag) > 0 {
@@ -3583,9 +3584,16 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
}
// Collect affected IDs for retriever sync
if update.IsEnabled != nil && len(affectedIDs) > 0 {
for _, id := range affectedIDs {
enabledUpdates[id] = *update.IsEnabled
if len(affectedIDs) > 0 {
if update.IsEnabled != nil {
for _, id := range affectedIDs {
enabledUpdates[id] = *update.IsEnabled
}
}
if update.TagID != nil {
for _, id := range affectedIDs {
tagUpdates[id] = *update.TagID
}
}
}
}
@@ -3644,6 +3652,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
}
if chunk.TagID != newTagID {
chunk.TagID = newTagID
tagUpdates[chunk.ID] = newTagID
needUpdate = true
}
}
@@ -3669,8 +3678,8 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
}
}
// Sync enabled status to retriever engines
if len(enabledUpdates) > 0 {
// Sync to retriever engines
if len(enabledUpdates) > 0 || len(tagUpdates) > 0 {
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
@@ -3679,8 +3688,15 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
if err != nil {
return err
}
if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
return err
if len(enabledUpdates) > 0 {
if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
return err
}
}
if len(tagUpdates) > 0 {
if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
return err
}
}
}
@@ -3810,9 +3826,27 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e
resolvedTagID = tag.ID
}
// Check if tag actually changed
if chunk.TagID == resolvedTagID {
return nil
}
chunk.TagID = resolvedTagID
chunk.UpdatedAt = time.Now()
return s.chunkRepo.UpdateChunk(ctx, chunk)
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
return err
}
// Sync tag update to retriever engines
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
if err != nil {
return err
}
return retrieveEngine.BatchUpdateChunkTagID(ctx, map[string]string{chunk.ID: resolvedTagID})
}
// UpdateFAQEntryTagBatch updates tags for FAQ entries in batch.
@@ -3889,7 +3923,26 @@ func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID stri
}
if len(chunksToUpdate) > 0 {
return s.chunkRepo.UpdateChunks(ctx, chunksToUpdate)
if err := s.chunkRepo.UpdateChunks(ctx, chunksToUpdate); err != nil {
return err
}
// Sync tag updates to retriever engines
tagUpdates := make(map[string]string)
for _, chunk := range chunksToUpdate {
tagUpdates[chunk.ID] = chunk.TagID
}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
if err != nil {
return err
}
if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
return err
}
}
return nil
@@ -115,6 +115,19 @@ func (c *CompositeRetrieveEngine) BatchUpdateChunkEnabledStatus(
})
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (c *CompositeRetrieveEngine) BatchUpdateChunkTagID(
ctx context.Context,
chunkTagMap map[string]string,
) error {
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
if err := engineInfo.retrieveEngine.BatchUpdateChunkTagID(ctx, chunkTagMap); err != nil {
return err
}
return nil
})
}
// concurrentRetrieve is a helper function for concurrent processing of retrieval parameters
// and collecting results
func concurrentRetrieve(
@@ -181,3 +181,11 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkEnabledStatu
) error {
return v.indexRepository.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap)
}
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkTagID(
ctx context.Context,
chunkTagMap map[string]string,
) error {
return v.indexRepository.BatchUpdateChunkTagID(ctx, chunkTagMap)
}
+8
View File
@@ -56,6 +56,10 @@ type RetrieveEngineRepository interface {
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
// chunkTagMap: map of chunk ID to tag ID (empty string means no tag)
BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error
// RetrieveEngine retrieves the engine
RetrieveEngine
}
@@ -119,6 +123,10 @@ type RetrieveEngineService interface {
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error
// BatchUpdateChunkTagID updates the tag ID of chunks in batch
// chunkTagMap: map of chunk ID to tag ID (empty string means no tag)
BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error
// RetrieveEngine retrieves the engine
RetrieveEngine
}