From 135e33eca427a52446de735c0e7db44a54d0cff6 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 28 Nov 2025 16:10:37 +0800 Subject: [PATCH] refactor: Clean up code by removing unnecessary blank lines and improving comments for better readability and maintainability --- .golangci.yml | 14 + client/cmd/agent_test/main.go | 1 - internal/agent/engine.go | 3 - internal/agent/tools/database_query.go | 35 -- internal/agent/tools/knowledge_search.go | 11 +- internal/agent/tools/list_knowledge_chunks.go | 16 +- internal/agent/tools/web_fetch.go | 2 +- internal/application/repository/knowledge.go | 5 +- .../application/repository/mcp_service.go | 4 - .../repository/retriever/neo4j/repository.go | 8 - .../service/chat_pipline/extract_entity.go | 18 +- .../application/service/chat_pipline/merge.go | 1 - .../service/chat_pipline/rerank.go | 340 +++++++++--------- .../service/chat_pipline/search.go | 2 +- .../service/chat_pipline/tracing.go | 44 +-- internal/application/service/chunk.go | 1 - internal/application/service/file/minio.go | 3 +- internal/application/service/graph.go | 10 +- internal/application/service/knowledge.go | 226 +----------- internal/application/service/knowledgebase.go | 5 +- .../llmcontext/compression_strategies.go | 1 - .../application/service/metric/rouge_score.go | 5 - internal/application/service/metric_hook.go | 1 - .../application/service/retriever/registry.go | 4 +- internal/application/service/session.go | 1 - internal/container/container.go | 2 +- internal/handler/initialization.go | 46 ++- internal/mcp/errors.go | 1 - internal/models/chat/remote_api_test.go | 1 - internal/models/rerank/reranker.go | 1 - internal/stream/memory_manager.go | 4 +- internal/types/extract_graph.go | 9 +- internal/types/faq.go | 10 +- internal/types/knowledge.go | 5 +- internal/types/knowledgebase.go | 17 +- internal/types/model.go | 2 + internal/types/session.go | 2 + internal/types/tenant.go | 2 + 38 files changed, 318 insertions(+), 545 deletions(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..cf49aee7 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,14 @@ +version: 2 +linters-settings: + lll: + line-length: 120 + tab-width: 4 +linters: + enable: + - lll # 控制行宽 + - govet + - revive +formatters: + enable: + - gofmt + - gofumpt diff --git a/client/cmd/agent_test/main.go b/client/cmd/agent_test/main.go index c3877527..6e89457f 100644 --- a/client/cmd/agent_test/main.go +++ b/client/cmd/agent_test/main.go @@ -277,7 +277,6 @@ func (cli *CLI) askQuestion(query string) { return nil }) - if err != nil { fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset) return diff --git a/internal/agent/engine.go b/internal/agent/engine.go index 39cd27e9..023d4d4e 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -638,7 +638,6 @@ func (e *AgentEngine) streamReflectionToEventBus( } }, ) - if err != nil { logger.Warnf(ctx, "Reflection failed: %v", err) return "", err @@ -710,7 +709,6 @@ func (e *AgentEngine) streamThinkingToEventBus( } }, ) - if err != nil { logger.Errorf(ctx, "[Agent][Thinking][Iteration-%d] Thinking stream failed: %v", iteration+1, err) return nil, err @@ -811,7 +809,6 @@ func (e *AgentEngine) streamFinalAnswerToEventBus( } }, ) - if err != nil { logger.Errorf(ctx, "[Agent][FinalAnswer] Final answer generation failed: %v", err) common.PipelineError(ctx, "Agent", "final_answer_stream_failed", map[string]interface{}{ diff --git a/internal/agent/tools/database_query.go b/internal/agent/tools/database_query.go index 8bc3e708..464d8c16 100644 --- a/internal/agent/tools/database_query.go +++ b/internal/agent/tools/database_query.go @@ -2,7 +2,6 @@ package tools import ( "context" - "database/sql" "encoding/json" "fmt" "regexp" @@ -421,37 +420,3 @@ func (t *DatabaseQueryTool) formatQueryResults(columns []string, results []map[s return output } - -// Helper function to convert database value to proper type -func convertDatabaseValue(val interface{}) interface{} { - if val == nil { - return nil - } - - switch v := val.(type) { - case []byte: - return string(v) - case sql.NullString: - if v.Valid { - return v.String - } - return nil - case sql.NullInt64: - if v.Valid { - return v.Int64 - } - return nil - case sql.NullFloat64: - if v.Valid { - return v.Float64 - } - return nil - case sql.NullBool: - if v.Valid { - return v.Bool - } - return nil - default: - return v - } -} diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index 0605cce6..fd9ff38b 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -459,18 +459,19 @@ func (t *KnowledgeSearchTool) filterByThreshold( var threshold float64 // Special handling for history matches: use lower threshold - if r.MatchType == types.MatchTypeHistory { + switch r.MatchType { + case types.MatchTypeHistory: // Use the lower of the two thresholds, then reduce by 0.1 (minimum 0.5) th := vectorThreshold if keywordThreshold < th { th = keywordThreshold } threshold = math.Max(th-0.1, 0.5) - } else if r.MatchType == types.MatchTypeEmbedding { + case types.MatchTypeEmbedding: threshold = vectorThreshold - } else if r.MatchType == types.MatchTypeKeywords { + case types.MatchTypeKeywords: threshold = keywordThreshold - } else { + default: // For other match types (graph, nearby chunk, etc.), use the lower threshold threshold = vectorThreshold if keywordThreshold < threshold { @@ -1208,7 +1209,7 @@ func (t *KnowledgeSearchTool) findMissingChunkRanges(retrievedChunks map[int]boo } var ranges []chunkRange - var currentStart int = -1 + currentStart := -1 // Iterate through all possible chunk indices (0 to totalChunks-1) for i := 0; i < totalChunks; i++ { diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index 4bcfd921..c40c00a5 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -190,11 +190,11 @@ func (t *ListKnowledgeChunksTool) buildOutput( builder.WriteString("=== 知识文档分块 ===\n\n") if knowledgeTitle != "" { - builder.WriteString(fmt.Sprintf("文档: %s (%s)\n", knowledgeTitle, knowledgeID)) + fmt.Fprintf(builder, "文档: %s (%s)\n", knowledgeTitle, knowledgeID) } else { - builder.WriteString(fmt.Sprintf("文档 ID: %s\n", knowledgeID)) + fmt.Fprintf(builder, "文档 ID: %s\n", knowledgeID) } - builder.WriteString(fmt.Sprintf("总分块数: %d\n", total)) + fmt.Fprintf(builder, "总分块数: %d\n", total) if fetched == 0 { builder.WriteString("未找到任何分块，请确认文档是否已完成解析。\n") @@ -203,14 +203,14 @@ func (t *ListKnowledgeChunksTool) buildOutput( } return builder.String() } - builder.WriteString(fmt.Sprintf("本次拉取: %d 条， 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex)) + fmt.Fprintf(builder, "本次拉取: %d 条， 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex) builder.WriteString("=== 分块内容预览 ===\n\n") for idx, c := range chunks { - builder.WriteString(fmt.Sprintf("Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1)) - builder.WriteString(fmt.Sprintf(" chunk_id: %s\n", c.ID)) - builder.WriteString(fmt.Sprintf(" 类型: %s\n", c.ChunkType)) - builder.WriteString(fmt.Sprintf(" 内容: %s\n\n", summarizeContent(c.Content))) + fmt.Fprintf(builder, "Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1) + fmt.Fprintf(builder, " chunk_id: %s\n", c.ID) + fmt.Fprintf(builder, " 类型: %s\n", c.ChunkType) + fmt.Fprintf(builder, " 内容: %s\n\n", summarizeContent(c.Content)) } if int64(fetched) < total { diff --git a/internal/agent/tools/web_fetch.go b/internal/agent/tools/web_fetch.go index f1bde86a..17018563 100644 --- a/internal/agent/tools/web_fetch.go +++ b/internal/agent/tools/web_fetch.go @@ -502,7 +502,7 @@ func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Build isOrdered := nodeName == "ol" node.Find("li").Each(func(idx int, li *goquery.Selection) { if isOrdered { - markdown.WriteString(fmt.Sprintf("%d. ", idx+1)) + fmt.Fprintf(markdown, "%d. ", idx+1) } else { markdown.WriteString("- ") } diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 8a6b0bb5..3750f4e5 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -138,7 +138,8 @@ func (r *knowledgeRepository) CheckKnowledgeExists( query := r.db.WithContext(ctx).Model(&types.Knowledge{}). Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed") - if params.Type == "file" { + switch params.Type { + case "file": // If file hash exists, prioritize exact match using hash if params.FileHash != "" { var knowledge types.Knowledge @@ -167,7 +168,7 @@ func (r *knowledgeRepository) CheckKnowledgeExists( } return true, &knowledge, nil } - } else if params.Type == "url" { + case "url": // If file hash exists, prioritize exact match using hash if params.FileHash != "" { var knowledge types.Knowledge diff --git a/internal/application/repository/mcp_service.go b/internal/application/repository/mcp_service.go index 6da3803e..5b95f939 100644 --- a/internal/application/repository/mcp_service.go +++ b/internal/application/repository/mcp_service.go @@ -30,7 +30,6 @@ func (r *mcpServiceRepository) GetByID(ctx context.Context, tenantID uint64, id err := r.db.WithContext(ctx). Where("id = ? AND tenant_id = ?", id, tenantID). First(&service).Error - if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, nil @@ -48,7 +47,6 @@ func (r *mcpServiceRepository) List(ctx context.Context, tenantID uint64) ([]*ty Where("tenant_id = ?", tenantID). Order("created_at DESC"). Find(&services).Error - if err != nil { return nil, err } @@ -63,7 +61,6 @@ func (r *mcpServiceRepository) ListEnabled(ctx context.Context, tenantID uint64) Where("tenant_id = ? AND enabled = ?", tenantID, true). Order("created_at DESC"). Find(&services).Error - if err != nil { return nil, err } @@ -81,7 +78,6 @@ func (r *mcpServiceRepository) ListByIDs(ctx context.Context, tenantID uint64, i err := r.db.WithContext(ctx). Where("tenant_id = ? AND id IN ?", tenantID, ids). Find(&services).Error - if err != nil { return nil, err } diff --git a/internal/application/repository/retriever/neo4j/repository.go b/internal/application/repository/retriever/neo4j/repository.go index 028086e8..63288ff3 100644 --- a/internal/application/repository/retriever/neo4j/repository.go +++ b/internal/application/repository/retriever/neo4j/repository.go @@ -221,11 +221,3 @@ func listI2listS(list []any) []string { } return result } - -func mapI2mapS(prop map[string]any) map[string]string { - attributes := make(map[string]string) - for k, v := range prop { - attributes[k] = fmt.Sprintf("%v", v) - } - return attributes -} diff --git a/internal/application/service/chat_pipline/extract_entity.go b/internal/application/service/chat_pipline/extract_entity.go index 170a9652..677720f1 100644 --- a/internal/application/service/chat_pipline/extract_entity.go +++ b/internal/application/service/chat_pipline/extract_entity.go @@ -304,12 +304,12 @@ func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) { if text == "" { - return nil, errors.New("Empty or invalid input string.") + return nil, errors.New("empty or invalid input string") } content := f.extractContent(ctx, text) // logger.Debugf(ctx, "Extracted content: %s", content) if content == "" { - return nil, errors.New("Empty or invalid input string.") + return nil, errors.New("empty or invalid input string") } var parsed interface{} @@ -318,10 +318,10 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i err = json.Unmarshal([]byte(content), &parsed) } if err != nil { - return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error()) + return nil, fmt.Errorf("failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error()) } if parsed == nil { - return nil, fmt.Errorf("Content must be a list of extractions or a dict.") + return nil, fmt.Errorf("content must be a list of extractions or a dict") } var items []interface{} @@ -330,7 +330,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i } else if parsedList, ok := parsed.([]interface{}); ok { items = parsedList } else { - return nil, fmt.Errorf("Expected list or dict, got %T", parsed) + return nil, fmt.Errorf("expected list or dict, got %T", parsed) } itemsList := make([]map[string]interface{}, 0) @@ -338,7 +338,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i if itemMap, ok := item.(map[string]interface{}); ok { itemsList = append(itemsList, itemMap) } else { - return nil, fmt.Errorf("Each item in the sequence must be a mapping.") + return nil, fmt.Errorf("each item in the sequence must be a mapping.") } } return itemsList, nil @@ -350,7 +350,7 @@ func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphDat return nil, err } if len(matchData) == 0 { - logger.Debugf(ctx, "Received empty extraction data.") + logger.Debugf(ctx, "received empty extraction data.") return &types.GraphData{}, nil } // mm, _ := json.Marshal(matchData) @@ -403,9 +403,7 @@ func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) { node.Attributes = make([]string, 0) } if prenode.Attributes != nil { - for _, attr := range prenode.Attributes { - node.Attributes = append(node.Attributes, attr) - } + node.Attributes = append(node.Attributes, prenode.Attributes...) } continue } diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index f7ed48f9..cc29236e 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -438,7 +438,6 @@ func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatM "neighbor_len": runeLen(chunk.Content), }) } - } } diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 97d2155e..48bd7d4e 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -1,15 +1,15 @@ package chatpipline import ( - "context" - "encoding/json" - "fmt" - "math" - "strings" + "context" + "encoding/json" + "fmt" + "math" + "strings" - "github.com/Tencent/WeKnora/internal/models/rerank" - "github.com/Tencent/WeKnora/internal/types" - "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" ) // PluginRerank implements reranking functionality for chat pipeline @@ -33,7 +33,7 @@ func (p *PluginRerank) ActivationEvents() []types.EventType { // OnEvent handles reranking events in the chat pipeline func (p *PluginRerank) OnEvent(ctx context.Context, - eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{ "session_id": chatManage.SessionID, @@ -86,52 +86,52 @@ func (p *PluginRerank) OnEvent(ctx context.Context, } } - pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{ - "result_cnt": len(rerankResp), - }) - for i := range chatManage.SearchResult { - chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata) - } - reranked := make([]*types.SearchResult, 0, len(rerankResp)) - for _, rr := range rerankResp { - sr := chatManage.SearchResult[rr.Index] - base := sr.Score - sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base) - sr.Score = rr.RelevanceScore - sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage) - reranked = append(reranked, sr) - } - final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7) - chatManage.RerankResult = final + pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{ + "result_cnt": len(rerankResp), + }) + for i := range chatManage.SearchResult { + chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata) + } + reranked := make([]*types.SearchResult, 0, len(rerankResp)) + for _, rr := range rerankResp { + sr := chatManage.SearchResult[rr.Index] + base := sr.Score + sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base) + sr.Score = rr.RelevanceScore + sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage) + reranked = append(reranked, sr) + } + final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7) + chatManage.RerankResult = final - // Log composite top scores and MMR selection summary - topN := min(3, len(reranked)) - for i := 0; i < topN; i++ { - pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{ - "rank": i + 1, - "chunk_id": reranked[i].ID, - "base_score": reranked[i].Metadata["base_score"], - "final_score": fmt.Sprintf("%.4f", reranked[i].Score), - "intent": chatManage.QueryIntent, - }) - } + // Log composite top scores and MMR selection summary + topN := min(3, len(reranked)) + for i := 0; i < topN; i++ { + pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{ + "rank": i + 1, + "chunk_id": reranked[i].ID, + "base_score": reranked[i].Metadata["base_score"], + "final_score": fmt.Sprintf("%.4f", reranked[i].Score), + "intent": chatManage.QueryIntent, + }) + } - if len(chatManage.RerankResult) == 0 { - pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{ - "filtered_cnt": 0, - }) - return ErrSearchNothing - } + if len(chatManage.RerankResult) == 0 { + pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{ + "filtered_cnt": 0, + }) + return ErrSearchNothing + } pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{ "filtered_cnt": len(chatManage.RerankResult), }) - return next() + return next() } // rerank performs the actual reranking operation with given query and passages func (p *PluginRerank) rerank(ctx context.Context, - chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string, + chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string, ) []rerank.RankResult { pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{ "query_variant": query, @@ -173,149 +173,149 @@ func (p *PluginRerank) rerank(ctx context.Context, rankFilter = append(rankFilter, result) } } - return rankFilter + return rankFilter } func ensureMetadata(m map[string]string) map[string]string { - if m == nil { - return make(map[string]string) - } - return m + if m == nil { + return make(map[string]string) + } + return m } func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 { - sourceWeight := 1.0 - switch strings.ToLower(sr.KnowledgeSource) { - case "web_search": - sourceWeight = 0.95 - default: - sourceWeight = 1.0 - } - intentBoost := 1.0 - if chatManage.QueryIntent != "" { - switch chatManage.QueryIntent { - case "definition": - if sr.ChunkType == string(types.ChunkTypeSummary) { - intentBoost = 1.05 - } - case "howto": - if sr.EndAt-sr.StartAt > 300 { - intentBoost = 1.03 - } - case "compare": - intentBoost = 1.0 - } - } - positionPrior := 1.0 - if sr.StartAt >= 0 { - positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05) - } - composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight - composite *= intentBoost - composite *= positionPrior - if composite < 0 { - composite = 0 - } - if composite > 1 { - composite = 1 - } - return composite + sourceWeight := 1.0 + switch strings.ToLower(sr.KnowledgeSource) { + case "web_search": + sourceWeight = 0.95 + default: + sourceWeight = 1.0 + } + intentBoost := 1.0 + if chatManage.QueryIntent != "" { + switch chatManage.QueryIntent { + case "definition": + if sr.ChunkType == string(types.ChunkTypeSummary) { + intentBoost = 1.05 + } + case "howto": + if sr.EndAt-sr.StartAt > 300 { + intentBoost = 1.03 + } + case "compare": + intentBoost = 1.0 + } + } + positionPrior := 1.0 + if sr.StartAt >= 0 { + positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05) + } + composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight + composite *= intentBoost + composite *= positionPrior + if composite < 0 { + composite = 0 + } + if composite > 1 { + composite = 1 + } + return composite } func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult { - if k <= 0 || len(results) == 0 { - return nil - } - pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{ - "lambda": lambda, - "k": k, - "candidates": len(results), - }) - selected := make([]*types.SearchResult, 0, k) - candidates := make([]*types.SearchResult, len(results)) - copy(candidates, results) - tokenSets := make([]map[string]struct{}, len(candidates)) - for i, r := range candidates { - tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r)) - } - for len(selected) < k && len(candidates) > 0 { - bestIdx := 0 - bestScore := -1.0 - for i, r := range candidates { - relevance := r.Score - redundancy := 0.0 - for _, s := range selected { - redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s)))) - } - mmr := lambda*relevance - (1.0-lambda)*redundancy - if mmr > bestScore { - bestScore = mmr - bestIdx = i - } - } - selected = append(selected, candidates[bestIdx]) - candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...) - } - // Compute average redundancy among selected - avgRed := 0.0 - if len(selected) > 1 { - pairs := 0 - for i := 0; i < len(selected); i++ { - for j := i + 1; j < len(selected); j++ { - si := tokenizeSimple(getEnrichedPassage(ctx, selected[i])) - sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j])) - avgRed += jaccard(si, sj) - pairs++ - } - } - if pairs > 0 { - avgRed /= float64(pairs) - } - } - pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{ - "selected": len(selected), - "avg_redundancy": fmt.Sprintf("%.4f", avgRed), - }) - return selected + if k <= 0 || len(results) == 0 { + return nil + } + pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{ + "lambda": lambda, + "k": k, + "candidates": len(results), + }) + selected := make([]*types.SearchResult, 0, k) + candidates := make([]*types.SearchResult, len(results)) + copy(candidates, results) + tokenSets := make([]map[string]struct{}, len(candidates)) + for i, r := range candidates { + tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r)) + } + for len(selected) < k && len(candidates) > 0 { + bestIdx := 0 + bestScore := -1.0 + for i, r := range candidates { + relevance := r.Score + redundancy := 0.0 + for _, s := range selected { + redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s)))) + } + mmr := lambda*relevance - (1.0-lambda)*redundancy + if mmr > bestScore { + bestScore = mmr + bestIdx = i + } + } + selected = append(selected, candidates[bestIdx]) + candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...) + } + // Compute average redundancy among selected + avgRed := 0.0 + if len(selected) > 1 { + pairs := 0 + for i := 0; i < len(selected); i++ { + for j := i + 1; j < len(selected); j++ { + si := tokenizeSimple(getEnrichedPassage(ctx, selected[i])) + sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j])) + avgRed += jaccard(si, sj) + pairs++ + } + } + if pairs > 0 { + avgRed /= float64(pairs) + } + } + pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{ + "selected": len(selected), + "avg_redundancy": fmt.Sprintf("%.4f", avgRed), + }) + return selected } func tokenizeSimple(text string) map[string]struct{} { - text = strings.ToLower(text) - fields := strings.Fields(text) - set := make(map[string]struct{}, len(fields)) - for _, f := range fields { - if len(f) > 1 { - set[f] = struct{}{} - } - } - return set + text = strings.ToLower(text) + fields := strings.Fields(text) + set := make(map[string]struct{}, len(fields)) + for _, f := range fields { + if len(f) > 1 { + set[f] = struct{}{} + } + } + return set } func jaccard(a, b map[string]struct{}) float64 { - if len(a) == 0 && len(b) == 0 { - return 0 - } - inter := 0 - for k := range a { - if _, ok := b[k]; ok { - inter++ - } - } - union := len(a) + len(b) - inter - if union == 0 { - return 0 - } - return float64(inter) / float64(union) + if len(a) == 0 && len(b) == 0 { + return 0 + } + inter := 0 + for k := range a { + if _, ok := b[k]; ok { + inter++ + } + } + union := len(a) + len(b) - inter + if union == 0 { + return 0 + } + return float64(inter) / float64(union) } func clampFloat(v, minV, maxV float64) float64 { - if v < minV { - return minV - } - if v > maxV { - return maxV - } - return v + if v < minV { + return minV + } + if v > maxV { + return maxV + } + return v } // getEnrichedPassage 合并Content和ImageInfo的文本内容 diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 2026c7d3..d7ce4213 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -387,7 +387,7 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID // searchWebIfEnabled executes web search when enabled and returns converted results func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult { - if !(chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0) { + if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil || chatManage.TenantID <= 0 { return nil } tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) diff --git a/internal/application/service/chat_pipline/tracing.go b/internal/application/service/chat_pipline/tracing.go index fe3bde6a..33155cf6 100644 --- a/internal/application/service/chat_pipline/tracing.go +++ b/internal/application/service/chat_pipline/tracing.go @@ -77,7 +77,7 @@ func (p *PluginTracing) OnEvent(ctx context.Context, // Search traces search operations in the chat pipeline func (p *PluginTracing) Search(ctx context.Context, - eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search") defer span.End() @@ -87,23 +87,23 @@ func (p *PluginTracing) Search(ctx context.Context, attribute.Float64("keyword_threshold", chatManage.KeywordThreshold), attribute.Int("match_count", chatManage.EmbeddingTopK), ) - err := next() - searchResultJson, _ := json.Marshal(chatManage.SearchResult) - unique := make(map[string]struct{}) - for _, r := range chatManage.SearchResult { - unique[r.ID] = struct{}{} - } - span.SetAttributes( - attribute.String("hybrid_search", string(searchResultJson)), - attribute.String("processed_query", chatManage.ProcessedQuery), - attribute.Int("search_unique_count", len(unique)), - ) - return err + err := next() + searchResultJson, _ := json.Marshal(chatManage.SearchResult) + unique := make(map[string]struct{}) + for _, r := range chatManage.SearchResult { + unique[r.ID] = struct{}{} + } + span.SetAttributes( + attribute.String("hybrid_search", string(searchResultJson)), + attribute.String("processed_query", chatManage.ProcessedQuery), + attribute.Int("search_unique_count", len(unique)), + ) + return err } // Rerank traces rerank operations in the chat pipeline func (p *PluginTracing) Rerank(ctx context.Context, - eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank") defer span.End() @@ -114,14 +114,14 @@ func (p *PluginTracing) Rerank(ctx context.Context, attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold), attribute.Int("rerank_filter_topk", chatManage.RerankTopK), ) - err := next() - resultJson, _ := json.Marshal(chatManage.RerankResult) - span.SetAttributes( - attribute.Int("rerank_resp_count", len(chatManage.RerankResult)), - attribute.String("rerank_resp_results", string(resultJson)), - attribute.String("query_intent", chatManage.QueryIntent), - ) - return err + err := next() + resultJson, _ := json.Marshal(chatManage.RerankResult) + span.SetAttributes( + attribute.Int("rerank_resp_count", len(chatManage.RerankResult)), + attribute.String("rerank_resp_results", string(resultJson)), + attribute.String("query_intent", chatManage.QueryIntent), + ) + return err } // Merge traces merge operations in the chat pipeline diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index 384582a2..91a0cff8 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -58,7 +58,6 @@ func (s *chunkService) GetRepository() interfaces.ChunkRepository { // Returns: // - error: Any error encountered during chunk creation func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { - err := s.chunkRepository.CreateChunks(ctx, chunks) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ diff --git a/internal/application/service/file/minio.go b/internal/application/service/file/minio.go index a2841908..e1c05c53 100644 --- a/internal/application/service/file/minio.go +++ b/internal/application/service/file/minio.go @@ -21,7 +21,8 @@ type minioFileService struct { // NewMinioFileService creates a MinIO file service func NewMinioFileService(endpoint, - accessKeyID, secretAccessKey, bucketName string, useSSL bool) (interfaces.FileService, error) { + accessKeyID, secretAccessKey, bucketName string, useSSL bool, +) (interfaces.FileService, error) { // Initialize MinIO client client, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), diff --git a/internal/application/service/graph.go b/internal/application/service/graph.go index 644275d1..5ea12913 100644 --- a/internal/application/service/graph.go +++ b/internal/application/service/graph.go @@ -183,7 +183,8 @@ func (b *graphBuilder) extractEntities(ctx context.Context, chunk *types.Chunk) // extractRelationships extracts relationships between entities // It analyzes semantic connections between multiple entities and establishes relationships func (b *graphBuilder) extractRelationships(ctx context.Context, - chunks []*types.Chunk, entities []*types.Entity) error { + chunks []*types.Chunk, entities []*types.Entity, +) error { log := logger.GetLogger(ctx) log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks)) @@ -340,7 +341,7 @@ func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string { return "" } - var chunkContents = chunks[0].Content + chunkContents := chunks[0].Content preChunk := chunks[0] for i := 1; i < len(chunks); i++ { @@ -369,7 +370,7 @@ func (b *graphBuilder) BuildGraph(ctx context.Context, chunks []*types.Chunk) er startTime := time.Now() // Concurrently extract entities from each document chunk - var chunkEntities = make([][]*types.Entity, len(chunks)) + chunkEntities := make([][]*types.Entity, len(chunks)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency @@ -868,7 +869,8 @@ func (b *graphBuilder) getEntityByTitle(title string) *types.Entity { // dfs depth-first search to find connected components func dfs(entityTitle string, adjacencyList map[string]map[string]*types.Relationship, - visited map[string]bool, component *[]string) { + visited map[string]bool, component *[]string, +) { visited[entityTitle] = true *component = append(*component, entityTitle) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 72a02a84..aa4c9e7a 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -864,207 +864,6 @@ func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowle return } -// processDocument handles asynchronous processing of document files -func (s *knowledgeService) processDocument(ctx context.Context, - kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool, -) { - logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel) - - ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument") - defer span.End() - span.SetAttributes( - attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)), - attribute.String("knowledge_base_id", kb.ID), - attribute.Int("tenant_id", int(kb.TenantID)), - attribute.String("knowledge_id", knowledge.ID), - attribute.String("file_name", knowledge.FileName), - attribute.String("file_type", knowledge.FileType), - attribute.String("file_path", knowledge.FilePath), - attribute.Int64("file_size", knowledge.FileSize), - attribute.String("embedding_model", knowledge.EmbeddingModelID), - attribute.Bool("enable_multimodal", enableMultimodel), - ) - logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String()) - - if !enableMultimodel && IsImageType(knowledge.FileType) { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = ErrImageNotParse.Error() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(ErrImageNotParse) - return - } - - // Update status to processing - knowledge.ParseStatus = "processing" - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - span.RecordError(err) - return - } - - // Read and chunk the document - f, err := file.Open() - if err != nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - WithField("error", err).Errorf("processDocument open file failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(err) - return - } - defer f.Close() - - span.AddEvent("start read file") - contentBytes, err := io.ReadAll(f) - if err != nil { - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(err) - return - } - - // Split file into chunks using document reader service - logger.GetLogger(ctx).Infof("processDocument split file content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v", - len(contentBytes), knowledge.FileName, knowledge.FileType, kb.ChunkingConfig.Separators, enableMultimodel) - var vlmConfig *proto.VLMConfig - if enableMultimodel { - vlmConfig, err = s.getVLMProtoConfig(ctx, kb) - if err != nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - WithField("error", err).Errorf("processDocument build VLM config failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(err) - return - } - if vlmConfig == nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - Error("processDocument enable multimodal but VLM config missing") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = "VLM 配置缺失" - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(errors.New("vlm config missing")) - return - } - } - - resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ - FileContent: contentBytes, - FileName: knowledge.FileName, - FileType: knowledge.FileType, - ReadConfig: &proto.ReadConfig{ - ChunkSize: int32(kb.ChunkingConfig.ChunkSize), - ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), - Separators: kb.ChunkingConfig.Separators, - EnableMultimodal: enableMultimodel, - StorageConfig: &proto.StorageConfig{ - Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), - Region: kb.StorageConfig.Region, - BucketName: kb.StorageConfig.BucketName, - AccessKeyId: kb.StorageConfig.SecretID, - SecretAccessKey: kb.StorageConfig.SecretKey, - AppId: kb.StorageConfig.AppID, - PathPrefix: kb.StorageConfig.PathPrefix, - }, - VlmConfig: vlmConfig, - }, - RequestId: ctx.Value(types.RequestIDContextKey).(string), - }) - if err != nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - WithField("error", err).Errorf("processDocument read file failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - span.RecordError(err) - return - } - - // Process and store chunks - span.AddEvent("start process chunks") - s.processChunks(ctx, kb, knowledge, resp.Chunks) -} - -// processDocumentFromURL handles asynchronous processing of URL content -func (s *knowledgeService) processDocumentFromURL(ctx context.Context, - kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool, -) { - // Update status to processing - knowledge.ParseStatus = "processing" - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - return - } - logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel) - - // Fetch and chunk content from URL - var vlmConfig *proto.VLMConfig - if enableMultimodel { - cfg, cfgErr := s.getVLMProtoConfig(ctx, kb) - if cfgErr != nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - WithField("error", cfgErr).Errorf("processDocumentFromURL build VLM config failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = cfgErr.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - return - } - if cfg == nil { - logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). - Error("processDocumentFromURL enable multimodal but VLM config missing") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = "VLM 配置缺失" - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - return - } - vlmConfig = cfg - } - - resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{ - Url: url, - Title: knowledge.Title, - ReadConfig: &proto.ReadConfig{ - ChunkSize: int32(kb.ChunkingConfig.ChunkSize), - ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), - Separators: kb.ChunkingConfig.Separators, - EnableMultimodal: enableMultimodel, - StorageConfig: &proto.StorageConfig{ - Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), - Region: kb.StorageConfig.Region, - BucketName: kb.StorageConfig.BucketName, - AccessKeyId: kb.StorageConfig.SecretID, - SecretAccessKey: kb.StorageConfig.SecretKey, - AppId: kb.StorageConfig.AppID, - PathPrefix: kb.StorageConfig.PathPrefix, - }, - VlmConfig: vlmConfig, - }, - RequestId: ctx.Value(types.RequestIDContextKey).(string), - }) - if err != nil { - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) - return - } - - // Process and store chunks - s.processChunks(ctx, kb, knowledge, resp.Chunks) -} - // processDocumentFromPassage handles asynchronous processing of text passages func (s *knowledgeService) processDocumentFromPassage(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string, @@ -1767,7 +1566,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID batch := 10 g, gctx := errgroup.WithContext(ctx) for ids := range slices.Chunk(delKnowledge, batch) { - ids = ids g.Go(func() error { err := s.DeleteKnowledgeList(gctx, ids) if err != nil { @@ -1787,7 +1585,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID g, gctx = errgroup.WithContext(ctx) g.SetLimit(batch) for _, knowledge := range addKnowledge { - knowledge = knowledge g.Go(func() error { srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) if err != nil { @@ -1916,7 +1713,8 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri } // Mark that we've found chunks for this image - if child.ChunkType == types.ChunkTypeImageCaption { + switch child.ChunkType { + case types.ChunkTypeImageCaption: hasCaptionChunk = true // Update caption if it has changed if image.Caption != cImageInfo[0].Caption { @@ -1924,7 +1722,7 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri child.ImageInfo = imageInfo updateChunk = append(updateChunk, chunkChildren[i]) } - } else if child.ChunkType == types.ChunkTypeImageOCR { + case types.ChunkTypeImageOCR: hasOCRChunk = true // Update OCR if it has changed if image.OCRText != cImageInfo[0].OCRText { @@ -2240,7 +2038,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, // calculateAppendOperations 计算Append模式下需要处理的条目，跳过已存在且内容相同的条目 func (s *knowledgeService) calculateAppendOperations(ctx context.Context, - tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload) ([]types.FAQEntryPayload, int, error) { + tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload, +) ([]types.FAQEntryPayload, int, error) { if len(entries) == 0 { return []types.FAQEntryPayload{}, 0, nil } @@ -2249,7 +2048,8 @@ func (s *knowledgeService) calculateAppendOperations(ctx context.Context, // calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目 func (s *knowledgeService) calculateReplaceOperations(ctx context.Context, - tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload) ([]types.FAQEntryPayload, []*types.Chunk, int, error) { + tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload, +) ([]types.FAQEntryPayload, []*types.Chunk, int, error) { // 计算所有新条目的 content hash，并同时构建 hash 到 entry 的映射 type entryWithHash struct { entry types.FAQEntryPayload @@ -2316,7 +2116,8 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context, // executeFAQImport 执行实际的FAQ导入逻辑 func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string, - payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int) (err error) { + payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int, +) (err error) { // 保存知识库和embedding模型信息，用于清理索引 var kb *types.KnowledgeBase var embeddingModel embedding.Embedder @@ -2364,8 +2165,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, if payload.Mode == types.FAQBatchModeReplace { // Replace模式：计算需要删除、创建、更新的条目 - entriesToProcess, chunksToDelete, skippedCount, err = - s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries) + entriesToProcess, chunksToDelete, skippedCount, err = s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries) if err != nil { return fmt.Errorf("failed to calculate replace operations: %w", err) } @@ -3080,13 +2880,15 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint } func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, - progress, total, processed int, errorMsg string) error { + progress, total, processed int, errorMsg string, +) error { return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg) } // updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态，包含NextChunkIndex func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, - progress, total, processed int, errorMsg string) error { + progress, total, processed int, errorMsg string, +) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 77df990f..ba590510 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -131,14 +131,15 @@ func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types kb.EnsureDefaults() // Get knowledge count - if kb.Type == types.KnowledgeBaseTypeDocument { + switch kb.Type { + case types.KnowledgeBaseTypeDocument: knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID) if err != nil { logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err) } else { kb.KnowledgeCount = knowledgeCount } - } else if kb.Type == types.KnowledgeBaseTypeFAQ { + case types.KnowledgeBaseTypeFAQ: // Get chunk count chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID) if err != nil { diff --git a/internal/application/service/llmcontext/compression_strategies.go b/internal/application/service/llmcontext/compression_strategies.go index fd74d0c4..249b90f7 100644 --- a/internal/application/service/llmcontext/compression_strategies.go +++ b/internal/application/service/llmcontext/compression_strategies.go @@ -192,7 +192,6 @@ func (s *smartCompressionStrategy) summarizeMessages(ctx context.Context, messag Temperature: 0.3, // Lower temperature for more consistent summaries MaxTokens: 500, // Limit summary length }) - if err != nil { return "", fmt.Errorf("failed to generate summary: %w", err) } diff --git a/internal/application/service/metric/rouge_score.go b/internal/application/service/metric/rouge_score.go index dd51e14b..542ccfab 100644 --- a/internal/application/service/metric/rouge_score.go +++ b/internal/application/service/metric/rouge_score.go @@ -96,11 +96,6 @@ func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams { return getNgrams(n, words, exclusive) } -func lenLcs(x, y []string) int { - table := lcs(x, y) - return table[len(x)][len(y)] -} - func lcs(x, y []string) [][]int { n, m := len(x), len(y) table := make([][]int, n+1) diff --git a/internal/application/service/metric_hook.go b/internal/application/service/metric_hook.go index ec3e80b3..88b1f32f 100644 --- a/internal/application/service/metric_hook.go +++ b/internal/application/service/metric_hook.go @@ -83,7 +83,6 @@ func (m *MetricList) Avg() *types.MetricResult { // HookMetric tracks evaluation metrics for QA pairs type HookMetric struct { - task *types.EvaluationTask qaPairMetricList []*qaPairMetric // Per-QA pair metrics metricResults *MetricList // Aggregated results mu *sync.RWMutex // Thread safety diff --git a/internal/application/service/retriever/registry.go b/internal/application/service/retriever/registry.go index f489c585..3790ce49 100644 --- a/internal/application/service/retriever/registry.go +++ b/internal/application/service/retriever/registry.go @@ -27,7 +27,7 @@ func (r *RetrieveEngineRegistry) Register(repo interfaces.RetrieveEngineService) defer r.mu.Unlock() if _, exists := r.repositories[repo.EngineType()]; exists { - return fmt.Errorf("Repository type %s already registered", repo.EngineType()) + return fmt.Errorf("repository type %s already registered", repo.EngineType()) } r.repositories[repo.EngineType()] = repo @@ -43,7 +43,7 @@ func (r *RetrieveEngineRegistry) GetRetrieveEngineService(repoType types.Retriev repo, exists := r.repositories[repoType] if !exists { - return nil, fmt.Errorf("Repository of type %s not found", repoType) + return nil, fmt.Errorf("repository of type %s not found", repoType) } return repo, nil diff --git a/internal/application/service/session.go b/internal/application/service/session.go index 18d50057..aebbfb3b 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -587,7 +587,6 @@ func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, sess // 3. Session's SummaryModelID (if not Remote) // 4. First knowledge base's SummaryModelID func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) { - // First, check if session has a SummaryModelID and if it's a Remote model if session.SummaryModelID != "" { model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID) diff --git a/internal/container/container.go b/internal/container/container.go index 38962be7..d4e67eb5 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -124,7 +124,7 @@ func BuildContainer(container *dig.Container) *dig.Container { // Session service (depends on agent service) // SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed - must(container.Provide(service.NewSessionService)) + must(container.Provide(service.NewSessionService)) must(container.Provide(router.NewAsyncqClient)) must(container.Provide(router.NewAsynqServer)) diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 95ebfd3b..df19cf6c 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -26,7 +26,6 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/utils" - secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/ollama/ollama/api" @@ -198,7 +197,7 @@ type InitializationRequest struct { // UpdateKBConfig 根据知识库ID和模型ID更新配置（简化版） func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { ctx := c.Request.Context() - kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) + kbIdStr := utils.SanitizeForLog(c.Param("kbId")) var req KBModelConfigRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -210,7 +209,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { // 获取知识库信息 kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) if err != nil || kb == nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) c.Error(errors.NewNotFoundError("知识库不存在")) return } @@ -350,7 +349,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { // InitializeByKB 根据知识库ID执行配置更新 func (h *InitializationHandler) InitializeByKB(c *gin.Context) { ctx := c.Request.Context() - kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) + kbIdStr := utils.SanitizeForLog(c.Param("kbId")) var req InitializationRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -358,12 +357,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { c.Error(errors.NewBadRequestError(err.Error())) return } - logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", secutils.SanitizeForLog(kbIdStr), secutils.SanitizeForLog(utils.ToJSON(req))) + logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", utils.SanitizeForLog(kbIdStr), utils.SanitizeForLog(utils.ToJSON(req))) // 获取指定知识库信息 kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) if err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error())) return } @@ -450,18 +449,18 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { }{ { modelType: types.ModelTypeKnowledgeQA, - name: secutils.SanitizeForLog(req.LLM.ModelName), + name: utils.SanitizeForLog(req.LLM.ModelName), source: types.ModelSource(req.LLM.Source), description: "LLM Model for Knowledge QA", - baseURL: secutils.SanitizeForLog(req.LLM.BaseURL), + baseURL: utils.SanitizeForLog(req.LLM.BaseURL), apiKey: req.LLM.APIKey, }, { modelType: types.ModelTypeEmbedding, - name: secutils.SanitizeForLog(req.Embedding.ModelName), + name: utils.SanitizeForLog(req.Embedding.ModelName), source: types.ModelSource(req.Embedding.Source), description: "Embedding Model", - baseURL: secutils.SanitizeForLog(req.Embedding.BaseURL), + baseURL: utils.SanitizeForLog(req.Embedding.BaseURL), apiKey: req.Embedding.APIKey, dimension: req.Embedding.Dimension, }, @@ -480,10 +479,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { interfaceType string }{ modelType: types.ModelTypeRerank, - name: secutils.SanitizeForLog(req.Rerank.ModelName), + name: utils.SanitizeForLog(req.Rerank.ModelName), source: types.ModelSourceRemote, description: "Rerank Model", - baseURL: secutils.SanitizeForLog(req.Rerank.BaseURL), + baseURL: utils.SanitizeForLog(req.Rerank.BaseURL), apiKey: req.Rerank.APIKey, }) } @@ -501,10 +500,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { interfaceType string }{ modelType: types.ModelTypeVLLM, - name: secutils.SanitizeForLog(req.Multimodal.VLM.ModelName), + name: utils.SanitizeForLog(req.Multimodal.VLM.ModelName), source: types.ModelSourceRemote, description: "VLM Model", - baseURL: secutils.SanitizeForLog(req.Multimodal.VLM.BaseURL), + baseURL: utils.SanitizeForLog(req.Multimodal.VLM.BaseURL), apiKey: req.Multimodal.VLM.APIKey, interfaceType: req.Multimodal.VLM.InterfaceType, }) @@ -676,7 +675,7 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { err = h.kbRepository.UpdateKnowledgeBase(ctx, kb) if err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error())) return } @@ -775,7 +774,7 @@ func (h *InitializationHandler) CheckOllamaModels(c *gin.Context) { modelStatus[modelName] = available } - logger.Infof(ctx, "Model %s availability: %v", secutils.SanitizeForLog(modelName), modelStatus[modelName]) + logger.Infof(ctx, "Model %s availability: %v", utils.SanitizeForLog(modelName), modelStatus[modelName]) } c.JSON(http.StatusOK, gin.H{ @@ -1010,8 +1009,8 @@ func (h *InitializationHandler) pullModelWithProgress(ctx context.Context, // 使用Ollama客户端的Pull方法，带进度回调 err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error { - var progressPercent float64 = 0.0 - var message string = "下载中" + progressPercent := 0.0 + message := "下载中" if progress.Total > 0 && progress.Completed > 0 { progressPercent = float64(progress.Completed) / float64(progress.Total) * 100 @@ -1057,7 +1056,7 @@ func (h *InitializationHandler) updateTaskStatus( // GetCurrentConfigByKB 根据知识库ID获取配置信息 func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) { ctx := c.Request.Context() - kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) + kbIdStr := utils.SanitizeForLog(c.Param("kbId")) logger.Info(ctx, "Getting configuration for knowledge base") @@ -1103,10 +1102,7 @@ func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) { Page: 1, PageSize: 1, }, "") - hasFiles := false - if err == nil && knowledgeList != nil && knowledgeList.Total > 0 { - hasFiles = true - } + hasFiles := err == nil && knowledgeList != nil && knowledgeList.Total > 0 // 构建配置响应 config := h.buildConfigResponse(ctx, models, kb, hasFiles) @@ -1339,7 +1335,7 @@ func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) { emb, err := embedding.NewEmbedder(cfg) if err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": secutils.SanitizeForLog(req.ModelName)}) + logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": utils.SanitizeForLog(req.ModelName)}) c.JSON(http.StatusOK, gin.H{ "success": true, "data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0}, @@ -1595,7 +1591,7 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) { c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB")) return } - logger.Infof(ctx, "Processing image: %s", secutils.SanitizeForLog(header.Filename)) + logger.Infof(ctx, "Processing image: %s", utils.SanitizeForLog(header.Filename)) // 解析文档分割配置 chunkSizeInt32, err := strconv.ParseInt(req.ChunkSize, 10, 32) diff --git a/internal/mcp/errors.go b/internal/mcp/errors.go index 31985a59..5ff32944 100644 --- a/internal/mcp/errors.go +++ b/internal/mcp/errors.go @@ -30,4 +30,3 @@ var ( // ErrConnectionClosed is returned when connection is closed unexpectedly ErrConnectionClosed = errors.New("connection closed") ) - diff --git a/internal/models/chat/remote_api_test.go b/internal/models/chat/remote_api_test.go index 782d41f8..a6650311 100644 --- a/internal/models/chat/remote_api_test.go +++ b/internal/models/chat/remote_api_test.go @@ -122,7 +122,6 @@ func TestRemoteAPIChat(t *testing.T) { response.Usage.CompletionTokens, response.Usage.TotalTokens) }) - }) } } diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go index 20a61d4c..a236d199 100644 --- a/internal/models/rerank/reranker.go +++ b/internal/models/rerank/reranker.go @@ -29,7 +29,6 @@ type RankResult struct { // Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field func (r *RankResult) UnmarshalJSON(data []byte) error { - var temp struct { Index int `json:"index"` Document DocumentInfo `json:"document"` diff --git a/internal/stream/memory_manager.go b/internal/stream/memory_manager.go index 22f9d5b6..c4485696 100644 --- a/internal/stream/memory_manager.go +++ b/internal/stream/memory_manager.go @@ -73,7 +73,7 @@ func (m *MemoryStreamManager) AppendEvent(ctx context.Context, sessionID, messag // Append event stream.events = append(stream.events, event) - stream.lastUpdated = time.Now() + stream.lastUpdated = time.Now() return nil } @@ -104,7 +104,7 @@ func (m *MemoryStreamManager) GetEvents(ctx context.Context, sessionID, messageI copy(eventsCopy, events) return eventsCopy, nextOffset, nil - } +} // Ensure MemoryStreamManager implements StreamManager interface var _ interfaces.StreamManager = (*MemoryStreamManager)(nil) diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index cdd0b488..8d1c0c78 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -6,13 +6,14 @@ const ( TypeFAQImport = "faq:import" // FAQ导入任务 ) +// ExtractChunkPayload represents the extract chunk task payload type ExtractChunkPayload struct { TenantID uint64 `json:"tenant_id"` ChunkID string `json:"chunk_id"` ModelID string `json:"model_id"` } -// DocumentProcessPayload 文档处理任务payload +// DocumentProcessPayload represents the document process task payload type DocumentProcessPayload struct { RequestId string `json:"request_id"` TenantID uint64 `json:"tenant_id"` @@ -26,7 +27,7 @@ type DocumentProcessPayload struct { EnableMultimodel bool `json:"enable_multimodel"` } -// FAQImportPayload FAQ导入任务payload +// FAQImportPayload represents the FAQ import task payload type FAQImportPayload struct { TenantID uint64 `json:"tenant_id"` TaskID string `json:"task_id"` @@ -36,6 +37,7 @@ type FAQImportPayload struct { Mode string `json:"mode"` } +// PromptTemplateStructured represents the prompt template structured type PromptTemplateStructured struct { Description string `json:"description"` Tags []string `json:"tags"` @@ -48,6 +50,7 @@ type GraphNode struct { Attributes []string `json:"attributes,omitempty"` } +// GraphRelation represents the relation of the graph type GraphRelation struct { Node1 string `json:"node1,omitempty"` Node2 string `json:"node2,omitempty"` @@ -60,11 +63,13 @@ type GraphData struct { Relation []*GraphRelation `json:"relation,omitempty"` } +// NameSpace represents the name space of the knowledge base and knowledge type NameSpace struct { KnowledgeBase string `json:"knowledge_base"` Knowledge string `json:"knowledge"` } +// Labels returns the labels of the name space func (n NameSpace) Labels() []string { res := make([]string, 0) if n.KnowledgeBase != "" { diff --git a/internal/types/faq.go b/internal/types/faq.go index ce2f01e9..c7da4ca6 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -160,10 +160,14 @@ type FAQSearchRequest struct { type FAQImportTaskStatus string const ( - FAQImportStatusPending FAQImportTaskStatus = "pending" + // FAQImportStatusPending represents the pending status of the FAQ import task + FAQImportStatusPending FAQImportTaskStatus = "pending" + // FAQImportStatusProcessing represents the processing status of the FAQ import task FAQImportStatusProcessing FAQImportTaskStatus = "processing" - FAQImportStatusCompleted FAQImportTaskStatus = "completed" - FAQImportStatusFailed FAQImportTaskStatus = "failed" + // FAQImportStatusCompleted represents the completed status of the FAQ import task + FAQImportStatusCompleted FAQImportTaskStatus = "completed" + // FAQImportStatusFailed represents the failed status of the FAQ import task + FAQImportStatusFailed FAQImportTaskStatus = "failed" ) // FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息 diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 11096a7b..185a1ed1 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -10,10 +10,13 @@ import ( ) const ( + // KnowledgeTypeManual represents the manual knowledge type KnowledgeTypeManual = "manual" - KnowledgeTypeFAQ = "faq" + // KnowledgeTypeFAQ represents the FAQ knowledge type + KnowledgeTypeFAQ = "faq" ) +// ManualKnowledgeFormat represents the format of the manual knowledge const ( ManualKnowledgeFormatMarkdown = "markdown" ManualKnowledgeStatusDraft = "draft" diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 525efa80..a7f3263f 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -8,32 +8,34 @@ import ( "gorm.io/gorm" ) +// KnowledgeBaseType represents the type of the knowledge base const ( + // KnowledgeBaseTypeDocument represents the document knowledge base type KnowledgeBaseTypeDocument = "document" KnowledgeBaseTypeFAQ = "faq" ) -// FAQIndexMode 表示 FAQ 索引策略：仅索引标准问还是同时索引问答内容 +// FAQIndexMode represents the FAQ index mode: only index questions or index questions and answers type FAQIndexMode string const ( - // FAQIndexModeQuestionOnly 仅使用标准问及相似问向量化 + // FAQIndexModeQuestionOnly only index questions and similar questions FAQIndexModeQuestionOnly FAQIndexMode = "question_only" - // FAQIndexModeQuestionAnswer 将标准问与答案拼接后向量化 + // FAQIndexModeQuestionAnswer index questions and answers together FAQIndexModeQuestionAnswer FAQIndexMode = "question_answer" ) -// FAQQuestionIndexMode 表示 FAQ 问题索引方式：一起索引还是分别索引 +// FAQQuestionIndexMode represents the FAQ question index mode: index together or index separately type FAQQuestionIndexMode string const ( - // FAQQuestionIndexModeCombined 将标准问和相似问一起索引（当前方式） + // FAQQuestionIndexModeCombined index questions and similar questions together FAQQuestionIndexModeCombined FAQQuestionIndexMode = "combined" - // FAQQuestionIndexModeSeparate 将标准问和相似问分别索引 + // FAQQuestionIndexModeSeparate index questions and similar questions separately FAQQuestionIndexModeSeparate FAQQuestionIndexMode = "separate" ) -// KnowledgeBase represents a knowledge base +// KnowledgeBase represents a knowledge base entity type KnowledgeBase struct { // Unique identifier of the knowledge base ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"` @@ -195,6 +197,7 @@ func (c *VLMConfig) Scan(value interface{}) error { return json.Unmarshal(b, c) } +// ExtractConfig represents the extract configuration for a knowledge base type ExtractConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` Text string `yaml:"text" json:"text,omitempty"` diff --git a/internal/types/model.go b/internal/types/model.go index 1d86cbde..2f3724cd 100644 --- a/internal/types/model.go +++ b/internal/types/model.go @@ -19,6 +19,7 @@ const ( ModelTypeVLLM ModelType = "VLLM" // VLLM model ) +// ModelStatus represents the status of the model type ModelStatus string const ( @@ -36,6 +37,7 @@ const ( ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model ) +// EmbeddingParameters represents the embedding parameters for a model type EmbeddingParameters struct { Dimension int `yaml:"dimension" json:"dimension"` TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"` diff --git a/internal/types/session.go b/internal/types/session.go index 65865120..f26b6b6c 100644 --- a/internal/types/session.go +++ b/internal/types/session.go @@ -17,6 +17,7 @@ const ( FallbackStrategyModel FallbackStrategy = "model" // Model fallback response ) +// SummaryConfig represents the summary configuration for a session type SummaryConfig struct { // Max tokens MaxTokens int `json:"max_tokens"` @@ -109,6 +110,7 @@ func (s *Session) BeforeCreate(tx *gorm.DB) (err error) { return nil } +// StringArray represents a list of strings type StringArray []string // Value implements the driver.Valuer interface, used to convert StringArray to database value diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 904ab0ad..67e85287 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -44,10 +44,12 @@ type Tenant struct { DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"` } +// RetrieverEngines represents the retriever engines for a tenant type RetrieverEngines struct { Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"` } +// BeforeCreate is a hook function that is called before creating a tenant func (t *Tenant) BeforeCreate(tx *gorm.DB) error { if t.RetrieverEngines.Engines == nil { t.RetrieverEngines.Engines = []RetrieverEngineParams{}