From da1dcb0cdaacb7052f2cd719ff101b65a774b069 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 30 Dec 2025 17:43:29 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=89=B9=E9=87=8F=E6=9B=B4=E6=96=B0tag?= =?UTF-8?q?=E6=97=B6=E6=9B=B4=E6=96=B0=E7=B4=A2=E5=BC=95=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../retriever/elasticsearch/v7/repository.go | 61 ++++++++++++++++ .../retriever/elasticsearch/v8/repository.go | 52 ++++++++++++++ .../retriever/postgres/repository.go | 32 +++++++++ .../repository/retriever/qdrant/repository.go | 52 ++++++++++++++ internal/application/service/knowledge.go | 71 ++++++++++++++++--- .../service/retriever/composite.go | 13 ++++ .../keywords_vector_hybrid_indexer.go | 8 +++ internal/types/interfaces/retriever.go | 8 +++ 8 files changed, 288 insertions(+), 9 deletions(-) diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 779ceff5..ac6e7c4c 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -1235,3 +1235,64 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( log.Infof("[ElasticsearchV7] Successfully batch updated chunk enabled status") return nil } + +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (e *elasticsearchRepository) BatchUpdateChunkTagID( + ctx context.Context, + chunkTagMap map[string]string, +) error { + log := logger.GetLogger(ctx) + if len(chunkTagMap) == 0 { + log.Warnf("[ElasticsearchV7] Chunk tag map is empty, skipping update") + return nil + } + + log.Infof("[ElasticsearchV7] Batch updating chunk tag ID, count: %d", len(chunkTagMap)) + + // Group chunks by tag ID for batch updates + tagGroups := make(map[string][]string) + for chunkID, tagID := range chunkTagMap { + tagGroups[tagID] = append(tagGroups[tagID], chunkID) + } + + // Batch update chunks for each tag ID using update_by_query + for tagID, chunkIDs := range tagGroups { + query := map[string]interface{}{ + "query": map[string]interface{}{ + "terms": map[string]interface{}{ + "chunk_id.keyword": chunkIDs, + }, + }, + "script": map[string]interface{}{ + "source": "ctx._source.tag_id = params.tag_id", + "lang": "painless", + "params": map[string]interface{}{ + "tag_id": tagID, + }, + }, + } + queryJSON, _ := json.Marshal(query) + res, err := esapi.UpdateByQueryRequest{ + Index: []string{e.index}, + Body: strings.NewReader(string(queryJSON)), + }.Do(ctx, e.client) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to update chunks with tag_id %s: %v", tagID, err) + return err + } + defer res.Body.Close() + if res.IsError() { + var e map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&e); err != nil { + log.Errorf("[ElasticsearchV7] Error parsing the response body: %v", err) + } else { + log.Errorf("[ElasticsearchV7] Error updating chunks with tag_id: %v", e["error"]) + } + return fmt.Errorf("elasticsearch update_by_query failed with status: %d", res.StatusCode) + } + log.Infof("[ElasticsearchV7] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID) + } + + log.Infof("[ElasticsearchV7] Successfully batch updated chunk tag ID") + return nil +} diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index f3f89fd1..364ab983 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -690,3 +690,55 @@ func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus( log.Infof("[Elasticsearch] Successfully batch updated chunk enabled status") return nil } + +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (e *elasticsearchRepository) BatchUpdateChunkTagID( + ctx context.Context, + chunkTagMap map[string]string, +) error { + log := logger.GetLogger(ctx) + if len(chunkTagMap) == 0 { + log.Warnf("[Elasticsearch] Chunk tag map is empty, skipping update") + return nil + } + + log.Infof("[Elasticsearch] Batch updating chunk tag ID, count: %d", len(chunkTagMap)) + + // Group chunks by tag ID for batch updates + tagGroups := make(map[string][]string) + for chunkID, tagID := range chunkTagMap { + tagGroups[tagID] = append(tagGroups[tagID], chunkID) + } + + // Batch update chunks for each tag ID using update_by_query + for tagID, chunkIDs := range tagGroups { + query := types.NewQuery() + query.Bool = &types.BoolQuery{ + Must: []types.Query{ + {Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "chunk_id.keyword": chunkIDs, + }, + }}, + }, + } + source := "ctx._source.tag_id = params.tag_id" + lang := scriptlanguage.Painless + script := types.Script{ + Source: &source, + Lang: &lang, + Params: map[string]json.RawMessage{ + "tag_id": json.RawMessage(`"` + tagID + `"`), + }, + } + _, err := e.client.UpdateByQuery(e.index).Query(query).Script(&script).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to update chunks with tag_id %s: %v", tagID, err) + return err + } + log.Infof("[Elasticsearch] Updated %d chunks to tag_id=%s", len(chunkIDs), tagID) + } + + log.Infof("[Elasticsearch] Successfully batch updated chunk tag ID") + return nil +} diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index fb55de8e..0f8cda4a 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -585,3 +585,35 @@ func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkS logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk enabled status") return nil } + +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error { + if len(chunkTagMap) == 0 { + logger.GetLogger(ctx).Warnf("[Postgres] Chunk tag map is empty, skipping update") + return nil + } + + logger.GetLogger(ctx).Infof("[Postgres] Batch updating chunk tag ID, count: %d", len(chunkTagMap)) + + // Group chunks by tag ID for batch updates + tagGroups := make(map[string][]string) + for chunkID, tagID := range chunkTagMap { + tagGroups[tagID] = append(tagGroups[tagID], chunkID) + } + + // Batch update chunks for each tag ID + for tagID, chunkIDs := range tagGroups { + result := g.db.WithContext(ctx).Model(&pgVector{}). + Where("chunk_id IN ?", chunkIDs). + Update("tag_id", tagID) + if result.Error != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to update chunks with tag_id %s: %v", tagID, result.Error) + return result.Error + } + logger.GetLogger(ctx). + Infof("[Postgres] Updated %d chunks to tag_id=%s, rows affected: %d", len(chunkIDs), tagID, result.RowsAffected) + } + + logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID") + return nil +} diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index d5fee479..49e4e38a 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -427,6 +427,58 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch return nil } +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (q *qdrantRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error { + log := logger.GetLogger(ctx) + if len(chunkTagMap) == 0 { + log.Warn("[Qdrant] Empty chunk tag map provided, skipping") + return nil + } + + log.Infof("[Qdrant] Batch updating chunk tag ID, count: %d", len(chunkTagMap)) + + // Get all collections that match our base name pattern + collections, err := q.client.ListCollections(ctx) + if err != nil { + log.Errorf("[Qdrant] Failed to list collections: %v", err) + return fmt.Errorf("failed to list collections: %w", err) + } + + // Group chunks by tag ID for batch updates + tagGroups := make(map[string][]string) + for chunkID, tagID := range chunkTagMap { + tagGroups[tagID] = append(tagGroups[tagID], chunkID) + } + + // Update in all matching collections + for _, collectionName := range collections { + // Only process collections that start with our base name + if len(collectionName) <= len(q.collectionBaseName) || + collectionName[:len(q.collectionBaseName)] != q.collectionBaseName { + continue + } + + // Update chunks for each tag ID + for tagID, chunkIDs := range tagGroups { + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: collectionName, + Payload: qdrant.NewValueMap(map[string]any{fieldTagID: tagID}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, chunkIDs...), + }, + }), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to update chunks with tag_id %s in %s: %v", tagID, collectionName, err) + } + } + } + + log.Infof("[Qdrant] Batch update chunk tag ID completed") + return nil +} + func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Filter { must := make([]*qdrant.Condition, 0) mustNot := make([]*qdrant.Condition, 0) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 694b76a6..3ab77270 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -3558,6 +3558,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, tenantID := ctx.Value(types.TenantIDContextKey).(uint64) enabledUpdates := make(map[string]bool) + tagUpdates := make(map[string]string) // Handle ByTag updates first if len(req.ByTag) > 0 { @@ -3583,9 +3584,16 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, } // Collect affected IDs for retriever sync - if update.IsEnabled != nil && len(affectedIDs) > 0 { - for _, id := range affectedIDs { - enabledUpdates[id] = *update.IsEnabled + if len(affectedIDs) > 0 { + if update.IsEnabled != nil { + for _, id := range affectedIDs { + enabledUpdates[id] = *update.IsEnabled + } + } + if update.TagID != nil { + for _, id := range affectedIDs { + tagUpdates[id] = *update.TagID + } } } } @@ -3644,6 +3652,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, } if chunk.TagID != newTagID { chunk.TagID = newTagID + tagUpdates[chunk.ID] = newTagID needUpdate = true } } @@ -3669,8 +3678,8 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, } } - // Sync enabled status to retriever engines - if len(enabledUpdates) > 0 { + // Sync to retriever engines + if len(enabledUpdates) > 0 || len(tagUpdates) > 0 { tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, @@ -3679,8 +3688,15 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, if err != nil { return err } - if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil { - return err + if len(enabledUpdates) > 0 { + if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil { + return err + } + } + if len(tagUpdates) > 0 { + if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil { + return err + } } } @@ -3810,9 +3826,27 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e resolvedTagID = tag.ID } + // Check if tag actually changed + if chunk.TagID == resolvedTagID { + return nil + } + chunk.TagID = resolvedTagID chunk.UpdatedAt = time.Now() - return s.chunkRepo.UpdateChunk(ctx, chunk) + if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil { + return err + } + + // Sync tag update to retriever engines + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine( + s.retrieveEngine, + tenantInfo.GetEffectiveEngines(), + ) + if err != nil { + return err + } + return retrieveEngine.BatchUpdateChunkTagID(ctx, map[string]string{chunk.ID: resolvedTagID}) } // UpdateFAQEntryTagBatch updates tags for FAQ entries in batch. @@ -3889,7 +3923,26 @@ func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID stri } if len(chunksToUpdate) > 0 { - return s.chunkRepo.UpdateChunks(ctx, chunksToUpdate) + if err := s.chunkRepo.UpdateChunks(ctx, chunksToUpdate); err != nil { + return err + } + + // Sync tag updates to retriever engines + tagUpdates := make(map[string]string) + for _, chunk := range chunksToUpdate { + tagUpdates[chunk.ID] = chunk.TagID + } + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine( + s.retrieveEngine, + tenantInfo.GetEffectiveEngines(), + ) + if err != nil { + return err + } + if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil { + return err + } } return nil diff --git a/internal/application/service/retriever/composite.go b/internal/application/service/retriever/composite.go index 959a622c..6173d216 100644 --- a/internal/application/service/retriever/composite.go +++ b/internal/application/service/retriever/composite.go @@ -115,6 +115,19 @@ func (c *CompositeRetrieveEngine) BatchUpdateChunkEnabledStatus( }) } +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (c *CompositeRetrieveEngine) BatchUpdateChunkTagID( + ctx context.Context, + chunkTagMap map[string]string, +) error { + return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.BatchUpdateChunkTagID(ctx, chunkTagMap); err != nil { + return err + } + return nil + }) +} + // concurrentRetrieve is a helper function for concurrent processing of retrieval parameters // and collecting results func concurrentRetrieve( diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index 226c3555..e803558f 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -181,3 +181,11 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkEnabledStatu ) error { return v.indexRepository.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap) } + +// BatchUpdateChunkTagID updates the tag ID of chunks in batch +func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkTagID( + ctx context.Context, + chunkTagMap map[string]string, +) error { + return v.indexRepository.BatchUpdateChunkTagID(ctx, chunkTagMap) +} diff --git a/internal/types/interfaces/retriever.go b/internal/types/interfaces/retriever.go index 549dd180..15bc3dfa 100644 --- a/internal/types/interfaces/retriever.go +++ b/internal/types/interfaces/retriever.go @@ -56,6 +56,10 @@ type RetrieveEngineRepository interface { // chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error + // BatchUpdateChunkTagID updates the tag ID of chunks in batch + // chunkTagMap: map of chunk ID to tag ID (empty string means no tag) + BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error + // RetrieveEngine retrieves the engine RetrieveEngine } @@ -119,6 +123,10 @@ type RetrieveEngineService interface { // chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error + // BatchUpdateChunkTagID updates the tag ID of chunks in batch + // chunkTagMap: map of chunk ID to tag ID (empty string means no tag) + BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error + // RetrieveEngine retrieves the engine RetrieveEngine }