diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0a7410f6..61e9b3c3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -54,6 +54,24 @@ services: retries: 3 networks: - WeKnora-network-dev + profiles: + - minio + - full + + qdrant: + image: qdrant/qdrant:v1.16.2 + container_name: WeKnora-qdrant-dev + ports: + - "${QDRANT_REST_PORT:-6333}:6333" + - "${QDRANT_PORT:-6334}:6334" + volumes: + - qdrant_data_dev:/qdrant/storage + networks: + - WeKnora-network-dev + restart: unless-stopped + profiles: + - qdrant + - full neo4j: image: neo4j:latest @@ -72,6 +90,9 @@ services: restart: always networks: - WeKnora-network-dev + profiles: + - neo4j + - full docreader: image: wechatopenai/weknora-docreader:main @@ -130,6 +151,9 @@ services: networks: - WeKnora-network-dev restart: unless-stopped + profiles: + - jaeger + - full networks: WeKnora-network-dev: @@ -141,4 +165,5 @@ volumes: minio_data_dev: neo4j-data-dev: jaeger_data_dev: + qdrant_data_dev: diff --git a/docker-compose.yml b/docker-compose.yml index 930a4240..0d95dee4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,11 @@ services: - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-} - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-} - ELASTICSEARCH_INDEX=${ELASTICSEARCH_INDEX:-} + - QDRANT_HOST=qdrant + - QDRANT_PORT=${QDRANT_PORT:-6334} + - QDRANT_COLLECTION=${QDRANT_COLLECTION:-weknora_embeddings} + - QDRANT_API_KEY=${QDRANT_API_KEY:-} + - QDRANT_USE_TLS=${QDRANT_USE_TLS:-false} - DOCREADER_ADDR=docreader:50051 - STORAGE_TYPE=${STORAGE_TYPE:-} - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-} @@ -240,6 +245,21 @@ services: - neo4j - full + qdrant: + image: qdrant/qdrant:v1.16.2 + container_name: WeKnora-qdrant + ports: + - "${QDRANT_REST_PORT:-6333}:6333" + - "${QDRANT_PORT:-6334}:6334" + volumes: + - qdrant_data:/qdrant/storage + networks: + - WeKnora-network + restart: unless-stopped + profiles: + - qdrant + - full + networks: WeKnora-network: driver: bridge @@ -250,3 +270,4 @@ volumes: jaeger_data: minio_data: neo4j-data: + qdrant_data: diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index 9095e061..e615f7fa 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -6,10 +6,11 @@ import ( "maps" "os" "slices" + "strings" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" - typesLocal "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" "github.com/qdrant/go-client/qdrant" @@ -33,32 +34,121 @@ func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.Retriev log := logger.GetLogger(context.Background()) log.Info("[Qdrant] Initializing Qdrant retriever engine repository") - collectionName := os.Getenv(envQdrantCollection) - if collectionName == "" { + collectionBaseName := os.Getenv(envQdrantCollection) + if collectionBaseName == "" { log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name") - collectionName = defaultCollectionName + collectionBaseName = defaultCollectionName } res := &qdrantRepository{ - client: client, - collectionName: collectionName, + client: client, + collectionBaseName: collectionBaseName, } log.Info("[Qdrant] Successfully initialized repository") return res } -func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType { - return typesLocal.QdrantRetrieverEngineType +// getCollectionName returns the collection name for a specific dimension +func (q *qdrantRepository) getCollectionName(dimension int) string { + return fmt.Sprintf("%s_%d", q.collectionBaseName, dimension) } -func (q *qdrantRepository) Support() []typesLocal.RetrieverType { - return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType} +// ensureCollection ensures the collection exists for the given dimension +func (q *qdrantRepository) ensureCollection(ctx context.Context, dimension int) error { + collectionName := q.getCollectionName(dimension) + + // Check cache first + if _, ok := q.initializedCollections.Load(dimension); ok { + return nil + } + + log := logger.GetLogger(ctx) + + // Check if collection exists + exists, err := q.client.CollectionExists(ctx, collectionName) + if err != nil { + log.Errorf("[Qdrant] Failed to check collection existence: %v", err) + return fmt.Errorf("failed to check collection existence: %w", err) + } + + if !exists { + log.Infof("[Qdrant] Creating collection %s with dimension %d", collectionName, dimension) + + err = q.client.CreateCollection(ctx, &qdrant.CreateCollection{ + CollectionName: collectionName, + VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{ + Size: uint64(dimension), + Distance: qdrant.Distance_Cosine, + }), + }) + if err != nil { + log.Errorf("[Qdrant] Failed to create collection: %v", err) + return fmt.Errorf("failed to create collection: %w", err) + } + + // Create payload indexes for filtering + indexFields := []string{fieldChunkID, fieldKnowledgeID, fieldKnowledgeBaseID, fieldSourceID} + for _, field := range indexFields { + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: field, + FieldType: qdrant.FieldType_FieldTypeKeyword.Enum(), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create index for field %s: %v", field, err) + } + } + + // Create bool index for is_enabled + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: fieldIsEnabled, + FieldType: qdrant.FieldType_FieldTypeBool.Enum(), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create index for field %s: %v", fieldIsEnabled, err) + } + + // Create text index for content (for keyword search) with multilingual tokenizer + // This supports Chinese, Japanese, Korean and other languages + lowercase := true + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: fieldContent, + FieldType: qdrant.FieldType_FieldTypeText.Enum(), + FieldIndexParams: &qdrant.PayloadIndexParams{ + IndexParams: &qdrant.PayloadIndexParams_TextIndexParams{ + TextIndexParams: &qdrant.TextIndexParams{ + Tokenizer: qdrant.TokenizerType_Multilingual, + Lowercase: &lowercase, + }, + }, + }, + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create text index for content: %v", err) + } + + log.Infof("[Qdrant] Successfully created collection %s", collectionName) + } + + // Mark as initialized + q.initializedCollections.Store(dimension, true) + return nil +} + +func (q *qdrantRepository) EngineType() types.RetrieverEngineType { + return types.QdrantRetrieverEngineType +} + +func (q *qdrantRepository) Support() []types.RetrieverType { + return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType} } // EstimateStorageSize calculates the estimated storage size for a list of indices func (q *qdrantRepository) EstimateStorageSize(ctx context.Context, - indexInfoList []*typesLocal.IndexInfo, params map[string]any, + indexInfoList []*types.IndexInfo, params map[string]any, ) int64 { var totalStorageSize int64 for _, embedding := range indexInfoList { @@ -73,7 +163,7 @@ func (q *qdrantRepository) EstimateStorageSize(ctx context.Context, // Save stores a single point in Qdrant func (q *qdrantRepository) Save(ctx context.Context, - embedding *typesLocal.IndexInfo, + embedding *types.IndexInfo, additionalParams map[string]any, ) error { log := logger.GetLogger(ctx) @@ -86,6 +176,12 @@ func (q *qdrantRepository) Save(ctx context.Context, return err } + dimension := len(embeddingDB.Embedding) + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + + collectionName := q.getCollectionName(dimension) pointID := uuid.New().String() point := &qdrant.PointStruct{ Id: qdrant.NewID(pointID), @@ -94,7 +190,7 @@ func (q *qdrantRepository) Save(ctx context.Context, } _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: []*qdrant.PointStruct{point}, }) if err != nil { @@ -108,7 +204,7 @@ func (q *qdrantRepository) Save(ctx context.Context, // BatchSave stores multiple points in Qdrant using batch upsert func (q *qdrantRepository) BatchSave(ctx context.Context, - embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, + embeddingList []*types.IndexInfo, additionalParams map[string]any, ) error { log := logger.GetLogger(ctx) if len(embeddingList) == 0 { @@ -118,7 +214,9 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList)) - points := make([]*qdrant.PointStruct, 0, len(embeddingList)) + // Group points by dimension + pointsByDimension := make(map[int][]*qdrant.PointStruct) + for _, embedding := range embeddingList { embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams) if len(embeddingDB.Embedding) == 0 { @@ -126,30 +224,42 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, continue } + dimension := len(embeddingDB.Embedding) point := &qdrant.PointStruct{ Id: qdrant.NewID(uuid.New().String()), Vectors: qdrant.NewVectors(embeddingDB.Embedding...), Payload: createPayload(embeddingDB), } - points = append(points, point) - log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID) + pointsByDimension[dimension] = append(pointsByDimension[dimension], point) + log.Debugf("[Qdrant] Added chunk ID %s to batch request (dimension: %d)", embedding.ChunkID, dimension) } - if len(points) == 0 { + if len(pointsByDimension) == 0 { log.Warn("[Qdrant] No valid points to save after filtering") return nil } - _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, - Points: points, - }) - if err != nil { - log.Errorf("[Qdrant] Failed to execute batch operation: %v", err) - return fmt.Errorf("failed to batch save: %w", err) + // Save points to each dimension-specific collection + totalSaved := 0 + for dimension, points := range pointsByDimension { + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + + collectionName := q.getCollectionName(dimension) + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: collectionName, + Points: points, + }) + if err != nil { + log.Errorf("[Qdrant] Failed to execute batch operation for dimension %d: %v", dimension, err) + return fmt.Errorf("failed to batch save (dimension %d): %w", dimension, err) + } + totalSaved += len(points) + log.Infof("[Qdrant] Saved %d points to collection %s", len(points), collectionName) } - log.Infof("[Qdrant] Successfully batch saved %d indices", len(points)) + log.Infof("[Qdrant] Successfully batch saved %d indices", totalSaved) return nil } @@ -161,10 +271,11 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList return nil } - log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by chunk IDs from %s, count: %d", collectionName, len(chunkIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...), @@ -190,10 +301,11 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context, return nil } - log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by knowledge IDs from %s, count: %d", collectionName, len(knowledgeIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...), @@ -219,10 +331,11 @@ func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context, return nil } - log.Infof("[Qdrant] Deleting indices by source IDs, count: %d", len(sourceIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by source IDs from %s, count: %d", collectionName, len(sourceIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldSourceID, sourceIDList...), @@ -239,6 +352,7 @@ func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context, } // BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch +// This method operates on all collections since dimension is not provided func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error { log := logger.GetLogger(ctx) if len(chunkStatusMap) == 0 { @@ -248,6 +362,13 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch log.Infof("[Qdrant] Batch updating chunk enabled status, count: %d", len(chunkStatusMap)) + // Get all collections that match our base name pattern + collections, err := q.client.ListCollections(ctx) + if err != nil { + log.Errorf("[Qdrant] Failed to list collections: %v", err) + return fmt.Errorf("failed to list collections: %w", err) + } + // Group chunks by enabled status for batch updates enabledChunkIDs := make([]string, 0) disabledChunkIDs := make([]string, 0) @@ -260,47 +381,52 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch } } - // Update enabled chunks - if len(enabledChunkIDs) > 0 { - _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ - CollectionName: q.collectionName, - Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: true}), - PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ - Must: []*qdrant.Condition{ - qdrant.NewMatchKeywords(fieldChunkID, enabledChunkIDs...), - }, - }), - }) - if err != nil { - log.Errorf("[Qdrant] Failed to update enabled chunks: %v", err) - return fmt.Errorf("failed to update enabled chunks: %w", err) + // Update in all matching collections + for _, collectionName := range collections { + // Only process collections that start with our base name + if len(collectionName) <= len(q.collectionBaseName) || + collectionName[:len(q.collectionBaseName)] != q.collectionBaseName { + continue } - log.Infof("[Qdrant] Successfully enabled %d chunks", len(enabledChunkIDs)) - } - // Update disabled chunks - if len(disabledChunkIDs) > 0 { - _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ - CollectionName: q.collectionName, - Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: false}), - PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ - Must: []*qdrant.Condition{ - qdrant.NewMatchKeywords(fieldChunkID, disabledChunkIDs...), - }, - }), - }) - if err != nil { - log.Errorf("[Qdrant] Failed to update disabled chunks: %v", err) - return fmt.Errorf("failed to update disabled chunks: %w", err) + // Update enabled chunks + if len(enabledChunkIDs) > 0 { + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: collectionName, + Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: true}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, enabledChunkIDs...), + }, + }), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to update enabled chunks in %s: %v", collectionName, err) + } + } + + // Update disabled chunks + if len(disabledChunkIDs) > 0 { + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: collectionName, + Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: false}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, disabledChunkIDs...), + }, + }), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to update disabled chunks in %s: %v", collectionName, err) + } } - log.Infof("[Qdrant] Successfully disabled %d chunks", len(disabledChunkIDs)) } log.Infof("[Qdrant] Batch update chunk enabled status completed") return nil } -func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter { +func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Filter { must := make([]*qdrant.Condition, 0) mustNot := make([]*qdrant.Condition, 0) @@ -327,15 +453,15 @@ func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdra // Retrieve dispatches the retrieval operation to the appropriate method based on retriever type func (q *qdrantRepository) Retrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType) switch params.RetrieverType { - case typesLocal.VectorRetrieverType: + case types.VectorRetrieverType: return q.VectorRetrieve(ctx, params) - case typesLocal.KeywordsRetrieverType: + case types.KeywordsRetrieverType: return q.KeywordsRetrieve(ctx, params) } @@ -346,11 +472,26 @@ func (q *qdrantRepository) Retrieve(ctx context.Context, // VectorRetrieve performs vector similarity search func (q *qdrantRepository) VectorRetrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) + dimension := len(params.Embedding) log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", - len(params.Embedding), params.TopK, params.Threshold) + dimension, params.TopK, params.Threshold) + + // Get collection name based on embedding dimension + collectionName := q.getCollectionName(dimension) + + // Check if collection exists + exists, err := q.client.CollectionExists(ctx, collectionName) + if err != nil { + log.Errorf("[Qdrant] Failed to check collection existence: %v", err) + return nil, fmt.Errorf("failed to check collection: %w", err) + } + if !exists { + log.Warnf("[Qdrant] Collection %s does not exist, returning empty results", collectionName) + return buildRetrieveResult(nil, types.VectorRetrieverType), nil + } filter := q.getBaseFilter(params) @@ -358,7 +499,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, scoreThreshold := float32(params.Threshold) searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Query: qdrant.NewQuery(params.Embedding...), Filter: filter, Limit: &limit, @@ -367,10 +508,10 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, }) if err != nil { log.Errorf("[Qdrant] Vector search failed: %v", err) - return nil, err + return nil, fmt.Errorf("%s: %w", collectionName, err) } - var results []*typesLocal.IndexWithScore + var results []*types.IndexWithScore for _, point := range searchResult { payload := point.Payload embedding := &QdrantVectorEmbeddingWithScore{ @@ -386,7 +527,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, } pointID := point.Id.GetUuid() - results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding)) + results = append(results, fromQdrantVectorEmbedding(pointID, embedding, types.MatchTypeEmbedding)) } if len(results) == 0 { @@ -396,58 +537,104 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score) } - return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil + return buildRetrieveResult(results, types.VectorRetrieverType), nil } // KeywordsRetrieve performs keyword-based search in document content +// This searches across all collections since keyword search doesn't depend on dimension func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK) - filter := q.getBaseFilter(params) - - filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query)) - - limit := uint32(params.TopK) - scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ - CollectionName: q.collectionName, - Filter: filter, - Limit: &limit, - WithPayload: qdrant.NewWithPayload(true), - }) + // Get all collections that match our base name pattern + collections, err := q.client.ListCollections(ctx) if err != nil { - log.Errorf("[Qdrant] Keywords search failed: %v", err) - return nil, err + log.Errorf("[Qdrant] Failed to list collections: %v", err) + return nil, fmt.Errorf("failed to list collections: %w", err) } - var results []*typesLocal.IndexWithScore - for _, point := range scrollResult { - payload := point.Payload - embedding := &QdrantVectorEmbeddingWithScore{ - QdrantVectorEmbedding: QdrantVectorEmbedding{ - Content: payload[fieldContent].GetStringValue(), - SourceID: payload[fieldSourceID].GetStringValue(), - SourceType: int(payload[fieldSourceType].GetIntegerValue()), - ChunkID: payload[fieldChunkID].GetStringValue(), - KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), - KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), - }, - Score: 1.0, + var allResults []*types.IndexWithScore + limit := uint32(params.TopK) + + log.Debugf("[Qdrant] Found %d collections, base name: %s", len(collections), q.collectionBaseName) + + // Tokenize query for OR-based search (better for Chinese and multi-word queries) + queryTokens := tokenizeQuery(params.Query) + log.Debugf("[Qdrant] Tokenized query into %d tokens: %v", len(queryTokens), queryTokens) + + // Search in all matching collections + for _, collectionName := range collections { + log.Debugf("[Qdrant] Checking collection: %s", collectionName) + // Only process collections that start with our base name + if len(collectionName) <= len(q.collectionBaseName) || + collectionName[:len(q.collectionBaseName)] != q.collectionBaseName { + log.Debugf("[Qdrant] Skipping collection %s (doesn't match base name %s)", collectionName, q.collectionBaseName) + continue } - pointID := point.Id.GetUuid() - results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords)) + filter := q.getBaseFilter(params) + + // Build should conditions for each token (OR logic) + // This allows matching documents that contain any of the query tokens + if len(queryTokens) > 0 { + shouldConditions := make([]*qdrant.Condition, 0, len(queryTokens)) + for _, token := range queryTokens { + shouldConditions = append(shouldConditions, qdrant.NewMatchText(fieldContent, token)) + } + filter.Should = shouldConditions + } else { + // Fallback to original query if tokenization fails + filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query)) + } + + log.Debugf("[Qdrant] Searching in collection %s with %d should conditions", collectionName, len(filter.Should)) + + scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ + CollectionName: collectionName, + Filter: filter, + Limit: &limit, + WithPayload: qdrant.NewWithPayload(true), + }) + if err != nil { + log.Warnf("[Qdrant] Keywords search failed in %s: %v", collectionName, err) + continue + } + + log.Debugf("[Qdrant] Found %d results in collection %s", len(scrollResult), collectionName) + + for _, point := range scrollResult { + payload := point.Payload + embedding := &QdrantVectorEmbeddingWithScore{ + QdrantVectorEmbedding: QdrantVectorEmbedding{ + Content: payload[fieldContent].GetStringValue(), + SourceID: payload[fieldSourceID].GetStringValue(), + SourceType: int(payload[fieldSourceType].GetIntegerValue()), + ChunkID: payload[fieldChunkID].GetStringValue(), + KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), + KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + }, + Score: 1.0, + } + + pointID := point.Id.GetUuid() + allResults = append(allResults, fromQdrantVectorEmbedding(pointID, embedding, types.MatchTypeKeywords)) + } } - if len(results) == 0 { + // Limit results to topK + if len(allResults) > params.TopK { + allResults = allResults[:params.TopK] + } + + if len(allResults) == 0 { log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query) } else { - log.Infof("[Qdrant] Keywords retrieval found %d results", len(results)) + log.Infof("[Qdrant] Keywords retrieval found %d results", len(allResults)) } - return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil + return buildRetrieveResult(allResults, types.KeywordsRetrieverType), nil } // CopyIndices copies index data from source knowledge base to target knowledge base @@ -460,8 +647,8 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, ) error { log := logger.GetLogger(ctx) log.Infof( - "[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d", - sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + "[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d, dimension: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), dimension, ) if len(sourceToTargetChunkIDMap) == 0 { @@ -469,13 +656,20 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, return nil } + collectionName := q.getCollectionName(dimension) + + // Ensure target collection exists + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + batchSize := uint32(64) var offset *qdrant.PointId = nil totalCopied := 0 for { scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Filter: &qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID), @@ -524,6 +718,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, fieldChunkID: targetChunkID, fieldKnowledgeID: targetKnowledgeID, fieldKnowledgeBaseID: targetKnowledgeBaseID, + fieldIsEnabled: true, }) var vectors *qdrant.Vectors @@ -549,7 +744,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, if len(targetPoints) > 0 { _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: targetPoints, }) if err != nil { @@ -588,11 +783,11 @@ func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value { return qdrant.NewValueMap(payload) } -func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult { - return []*typesLocal.RetrieveResult{ +func buildRetrieveResult(results []*types.IndexWithScore, retrieverType types.RetrieverType) []*types.RetrieveResult { + return []*types.RetrieveResult{ { Results: results, - RetrieverEngineType: typesLocal.QdrantRetrieverEngineType, + RetrieverEngineType: types.QdrantRetrieverEngineType, RetrieverType: retrieverType, Error: nil, }, @@ -667,3 +862,30 @@ func fromQdrantVectorEmbedding(id string, MatchType: matchType, } } + +// tokenizeQuery splits a query string into tokens for OR-based full-text search. +// It uses jieba for professional Chinese word segmentation. +func tokenizeQuery(query string) []string { + query = strings.TrimSpace(query) + if query == "" { + return nil + } + + // Use jieba for segmentation (search mode for better recall) + words := types.Jieba.CutForSearch(query, true) + + // Filter and deduplicate + seen := make(map[string]bool) + result := make([]string, 0, len(words)) + for _, word := range words { + word = strings.TrimSpace(strings.ToLower(word)) + // Skip empty, single-char, and already seen words + if utf8.RuneCountInString(word) < 2 || seen[word] { + continue + } + seen[word] = true + result = append(result, word) + } + + return result +} diff --git a/internal/application/repository/retriever/qdrant/structs.go b/internal/application/repository/retriever/qdrant/structs.go index 38ece9a2..9bae0d57 100644 --- a/internal/application/repository/retriever/qdrant/structs.go +++ b/internal/application/repository/retriever/qdrant/structs.go @@ -1,10 +1,16 @@ package qdrant -import "github.com/qdrant/go-client/qdrant" +import ( + "sync" + + "github.com/qdrant/go-client/qdrant" +) type qdrantRepository struct { - client *qdrant.Client - collectionName string + client *qdrant.Client + collectionBaseName string + // Cache for initialized collections (dimension -> true) + initializedCollections sync.Map } type QdrantVectorEmbedding struct { diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index 1720d3f4..dd0a9751 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -389,7 +389,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri sourceID := fmt.Sprintf("%s-%s", chunkID, questionID) tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": chunkID, diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 2e49b1ec..f8349054 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -737,7 +737,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") @@ -826,7 +826,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") @@ -1047,7 +1047,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, // 删除旧的索引数据 tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err == nil { if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err) @@ -1663,7 +1663,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to init retrieve engine: %v", err) return fmt.Errorf("failed to init retrieve engine: %w", err) @@ -1772,7 +1772,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to init retrieve engine: %v", err) return fmt.Errorf("failed to init retrieve engine: %w", err) @@ -2268,7 +2268,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -2539,7 +2539,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -3257,7 +3257,7 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { return err @@ -3306,7 +3306,7 @@ func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context, // Sync update to retriever engines chunkStatusMap := map[string]bool{chunk.ID: isEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -3366,7 +3366,7 @@ func (s *knowledgeService) UpdateFAQEntryStatusBatch(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { return err @@ -4105,7 +4105,7 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context, logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks)) tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -4212,7 +4212,7 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context, return err } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -4358,7 +4358,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup") @@ -4749,7 +4749,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) if err == nil { retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err == nil { chunkIDs := make([]string, 0, len(chunksDeleted)) diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index e8897354..1ea1d11d 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -249,7 +249,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.Warnf(ctx, "Failed to create retrieve engine: %v", err) @@ -438,7 +438,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) // Create a composite retrieval engine with tenant's configured retrievers - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to create retrieval engine: %v", err) return nil, err diff --git a/internal/application/service/user.go b/internal/application/service/user.go index c33c215d..d5786b7e 100644 --- a/internal/application/service/user.go +++ b/internal/application/service/user.go @@ -64,35 +64,6 @@ func NewUserService( } } -var engine = map[string][]types.RetrieverEngineParams{ - "postgres": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.PostgresRetrieverEngineType, - }, - { - RetrieverType: types.VectorRetrieverType, - RetrieverEngineType: types.PostgresRetrieverEngineType, - }, - }, - "elasticsearch_v7": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - }, - "elasticsearch_v8": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - { - RetrieverType: types.VectorRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - }, -} - // Register creates a new user account func (s *userService) Register(ctx context.Context, req *types.RegisterRequest) (*types.User, error) { logger.Info(ctx, "Start user registration") @@ -120,21 +91,12 @@ func (s *userService) Register(ctx context.Context, req *types.RegisterRequest) return nil, errors.New("failed to process password") } - egs := []types.RetrieverEngineParams{} - for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { - if val, ok := engine[driver]; ok { - egs = append(egs, val...) - } - } - egs = uniqueRetrieverEngine(egs) - logger.Debug(ctx, "user register retriever engines") - // Create default tenant for the user + // Note: RetrieverEngines is left empty - system will use defaults from RETRIEVE_DRIVER env tenant := &types.Tenant{ - Name: fmt.Sprintf("%s's Workspace", secutils.SanitizeForLog(req.Username)), - Description: "Default workspace", - Status: "active", - RetrieverEngines: types.RetrieverEngines{Engines: egs}, + Name: fmt.Sprintf("%s's Workspace", secutils.SanitizeForLog(req.Username)), + Description: "Default workspace", + Status: "active", } createdTenant, err := s.tenantService.CreateTenant(ctx, tenant) @@ -464,15 +426,3 @@ func (s *userService) GetCurrentUser(ctx context.Context) (*types.User, error) { return user, nil } - -func uniqueRetrieverEngine(engine []types.RetrieverEngineParams) []types.RetrieverEngineParams { - seen := make(map[types.RetrieverEngineParams]bool) - var result []types.RetrieverEngineParams - for _, v := range engine { - if !seen[v] { - seen[v] = true - result = append(result, v) - } - } - return result -} diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 0a597dc7..3430294e 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -3,11 +3,52 @@ package types import ( "database/sql/driver" "encoding/json" + "os" + "strings" "time" "gorm.io/gorm" ) +// retrieverEngineMapping maps RETRIEVE_DRIVER values to retriever engine configurations +var retrieverEngineMapping = map[string][]RetrieverEngineParams{ + "postgres": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: PostgresRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: PostgresRetrieverEngineType}, + }, + "elasticsearch_v7": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + }, + "elasticsearch_v8": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + }, + "qdrant": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType}, + }, +} + +// GetDefaultRetrieverEngines returns the default retriever engines based on RETRIEVE_DRIVER env +func GetDefaultRetrieverEngines() []RetrieverEngineParams { + result := []RetrieverEngineParams{} + seen := make(map[string]bool) + + for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { + driver = strings.TrimSpace(driver) + if params, ok := retrieverEngineMapping[driver]; ok { + for _, p := range params { + key := string(p.RetrieverType) + ":" + string(p.RetrieverEngineType) + if !seen[key] { + seen[key] = true + result = append(result, p) + } + } + } + } + return result +} + // Tenant represents the tenant type Tenant struct { // ID @@ -49,6 +90,14 @@ type RetrieverEngines struct { Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"` } +// GetEffectiveEngines returns the tenant's engines if configured, otherwise returns system defaults +func (t *Tenant) GetEffectiveEngines() []RetrieverEngineParams { + if len(t.RetrieverEngines.Engines) > 0 { + return t.RetrieverEngines.Engines + } + return GetDefaultRetrieverEngines() +} + // BeforeCreate is a hook function that is called before creating a tenant func (t *Tenant) BeforeCreate(tx *gorm.DB) error { if t.RetrieverEngines.Engines == nil { diff --git a/scripts/dev.sh b/scripts/dev.sh index 169e9c0f..13f5e7fe 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -52,10 +52,10 @@ detect_compose_cmd() { # 显示帮助信息 show_help() { printf "%b\n" "${GREEN}WeKnora 开发环境脚本${NC}" - echo "用法: $0 [命令]" + echo "用法: $0 [命令] [选项]" echo "" echo "命令:" - echo " start 启动基础设施服务(postgres, redis, minio, neo4j, docreader, jaeger)" + echo " start 启动基础设施服务(postgres, redis, docreader)" echo " stop 停止所有服务" echo " restart 重启所有服务" echo " logs 查看服务日志" @@ -64,10 +64,20 @@ show_help() { echo " frontend 启动前端开发服务器(本地运行)" echo " help 显示此帮助信息" echo "" + echo "可选 Profile(用于 start 命令):" + echo " --minio 启动 MinIO 对象存储" + echo " --qdrant 启动 Qdrant 向量数据库" + echo " --neo4j 启动 Neo4j 图数据库" + echo " --jaeger 启动 Jaeger 链路追踪" + echo " --full 启动所有可选服务" + echo "" echo "示例:" - echo " $0 start # 启动所有基础设施" - echo " $0 app # 在另一个终端启动后端" - echo " $0 frontend # 在另一个终端启动前端" + echo " $0 start # 启动基础服务" + echo " $0 start --qdrant # 启动基础服务 + Qdrant" + echo " $0 start --qdrant --jaeger # 启动基础服务 + Qdrant + Jaeger" + echo " $0 start --full # 启动所有服务" + echo " $0 app # 在另一个终端启动后端" + echo " $0 frontend # 在另一个终端启动前端" } # 检查 Docker @@ -107,8 +117,43 @@ start_services() { return 1 fi + # 解析 profile 参数 + shift # 移除 "start" 命令本身 + PROFILES="--profile full" + ENABLED_SERVICES="" + + while [ $# -gt 0 ]; do + case "$1" in + --minio) + PROFILES="$PROFILES --profile minio" + ENABLED_SERVICES="$ENABLED_SERVICES minio" + ;; + --qdrant) + PROFILES="$PROFILES --profile qdrant" + ENABLED_SERVICES="$ENABLED_SERVICES qdrant" + ;; + --neo4j) + PROFILES="$PROFILES --profile neo4j" + ENABLED_SERVICES="$ENABLED_SERVICES neo4j" + ;; + --jaeger) + PROFILES="$PROFILES --profile jaeger" + ENABLED_SERVICES="$ENABLED_SERVICES jaeger" + ;; + --full) + PROFILES="--profile full" + ENABLED_SERVICES="minio qdrant neo4j jaeger" + break + ;; + *) + log_warning "未知参数: $1" + ;; + esac + shift + done + # 启动服务 - "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml up -d + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d if [ $? -eq 0 ]; then log_success "基础设施服务已启动" @@ -116,10 +161,22 @@ start_services() { log_info "服务访问地址:" echo " - PostgreSQL: localhost:5432" echo " - Redis: localhost:6379" - echo " - MinIO: localhost:9000 (Console: localhost:9001)" - echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)" echo " - DocReader: localhost:50051" - echo " - Jaeger: localhost:16686" + + # 根据启用的 profile 显示额外服务 + if [[ "$ENABLED_SERVICES" == *"minio"* ]]; then + echo " - MinIO: localhost:9000 (Console: localhost:9001)" + fi + if [[ "$ENABLED_SERVICES" == *"qdrant"* ]]; then + echo " - Qdrant: localhost:6333 (gRPC: localhost:6334)" + fi + if [[ "$ENABLED_SERVICES" == *"neo4j"* ]]; then + echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)" + fi + if [[ "$ENABLED_SERVICES" == *"jaeger"* ]]; then + echo " - Jaeger: localhost:16686" + fi + echo "" log_info "接下来的步骤:" printf "%b\n" "${YELLOW}1. 在新终端运行后端:${NC} make dev-app" @@ -201,6 +258,7 @@ start_app() { export REDIS_ADDR=localhost:6379 export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317 export NEO4J_URI=bolt://localhost:7687 + export QDRANT_HOST=localhost # 确保必要的环境变量已设置 if [ -z "$DB_DRIVER" ]; then @@ -251,9 +309,10 @@ start_frontend() { } # 解析命令 -case "${1:-help}" in +CMD="${1:-help}" +case "$CMD" in start) - start_services + start_services "$@" ;; stop) stop_services @@ -277,7 +336,7 @@ case "${1:-help}" in show_help ;; *) - log_error "未知命令: $1" + log_error "未知命令: $CMD" show_help exit 1 ;;