refactor: Clean up code by removing unnecessary blank lines and improving comments for better readability and maintainability

This commit is contained in:
wizardchen
2025-11-28 16:10:37 +08:00
parent a318a08ab7
commit 135e33eca4
38 changed files with 318 additions and 545 deletions
+14
View File
@@ -0,0 +1,14 @@
version: 2
linters-settings:
lll:
line-length: 120
tab-width: 4
linters:
enable:
- lll # 控制行宽
- govet
- revive
formatters:
enable:
- gofmt
- gofumpt
-1
View File
@@ -277,7 +277,6 @@ func (cli *CLI) askQuestion(query string) {
return nil return nil
}) })
if err != nil { if err != nil {
fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset) fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset)
return return
-3
View File
@@ -638,7 +638,6 @@ func (e *AgentEngine) streamReflectionToEventBus(
} }
}, },
) )
if err != nil { if err != nil {
logger.Warnf(ctx, "Reflection failed: %v", err) logger.Warnf(ctx, "Reflection failed: %v", err)
return "", err return "", err
@@ -710,7 +709,6 @@ func (e *AgentEngine) streamThinkingToEventBus(
} }
}, },
) )
if err != nil { if err != nil {
logger.Errorf(ctx, "[Agent][Thinking][Iteration-%d] Thinking stream failed: %v", iteration+1, err) logger.Errorf(ctx, "[Agent][Thinking][Iteration-%d] Thinking stream failed: %v", iteration+1, err)
return nil, err return nil, err
@@ -811,7 +809,6 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
} }
}, },
) )
if err != nil { if err != nil {
logger.Errorf(ctx, "[Agent][FinalAnswer] Final answer generation failed: %v", err) logger.Errorf(ctx, "[Agent][FinalAnswer] Final answer generation failed: %v", err)
common.PipelineError(ctx, "Agent", "final_answer_stream_failed", map[string]interface{}{ common.PipelineError(ctx, "Agent", "final_answer_stream_failed", map[string]interface{}{
-35
View File
@@ -2,7 +2,6 @@ package tools
import ( import (
"context" "context"
"database/sql"
"encoding/json" "encoding/json"
"fmt" "fmt"
"regexp" "regexp"
@@ -421,37 +420,3 @@ func (t *DatabaseQueryTool) formatQueryResults(columns []string, results []map[s
return output return output
} }
// Helper function to convert database value to proper type
func convertDatabaseValue(val interface{}) interface{} {
if val == nil {
return nil
}
switch v := val.(type) {
case []byte:
return string(v)
case sql.NullString:
if v.Valid {
return v.String
}
return nil
case sql.NullInt64:
if v.Valid {
return v.Int64
}
return nil
case sql.NullFloat64:
if v.Valid {
return v.Float64
}
return nil
case sql.NullBool:
if v.Valid {
return v.Bool
}
return nil
default:
return v
}
}
+6 -5
View File
@@ -459,18 +459,19 @@ func (t *KnowledgeSearchTool) filterByThreshold(
var threshold float64 var threshold float64
// Special handling for history matches: use lower threshold // Special handling for history matches: use lower threshold
if r.MatchType == types.MatchTypeHistory { switch r.MatchType {
case types.MatchTypeHistory:
// Use the lower of the two thresholds, then reduce by 0.1 (minimum 0.5) // Use the lower of the two thresholds, then reduce by 0.1 (minimum 0.5)
th := vectorThreshold th := vectorThreshold
if keywordThreshold < th { if keywordThreshold < th {
th = keywordThreshold th = keywordThreshold
} }
threshold = math.Max(th-0.1, 0.5) threshold = math.Max(th-0.1, 0.5)
} else if r.MatchType == types.MatchTypeEmbedding { case types.MatchTypeEmbedding:
threshold = vectorThreshold threshold = vectorThreshold
} else if r.MatchType == types.MatchTypeKeywords { case types.MatchTypeKeywords:
threshold = keywordThreshold threshold = keywordThreshold
} else { default:
// For other match types (graph, nearby chunk, etc.), use the lower threshold // For other match types (graph, nearby chunk, etc.), use the lower threshold
threshold = vectorThreshold threshold = vectorThreshold
if keywordThreshold < threshold { if keywordThreshold < threshold {
@@ -1208,7 +1209,7 @@ func (t *KnowledgeSearchTool) findMissingChunkRanges(retrievedChunks map[int]boo
} }
var ranges []chunkRange var ranges []chunkRange
var currentStart int = -1 currentStart := -1
// Iterate through all possible chunk indices (0 to totalChunks-1) // Iterate through all possible chunk indices (0 to totalChunks-1)
for i := 0; i < totalChunks; i++ { for i := 0; i < totalChunks; i++ {
@@ -190,11 +190,11 @@ func (t *ListKnowledgeChunksTool) buildOutput(
builder.WriteString("=== 知识文档分块 ===\n\n") builder.WriteString("=== 知识文档分块 ===\n\n")
if knowledgeTitle != "" { if knowledgeTitle != "" {
builder.WriteString(fmt.Sprintf("文档: %s (%s)\n", knowledgeTitle, knowledgeID)) fmt.Fprintf(builder, "文档: %s (%s)\n", knowledgeTitle, knowledgeID)
} else { } else {
builder.WriteString(fmt.Sprintf("文档 ID: %s\n", knowledgeID)) fmt.Fprintf(builder, "文档 ID: %s\n", knowledgeID)
} }
builder.WriteString(fmt.Sprintf("总分块数: %d\n", total)) fmt.Fprintf(builder, "总分块数: %d\n", total)
if fetched == 0 { if fetched == 0 {
builder.WriteString("未找到任何分块,请确认文档是否已完成解析。\n") builder.WriteString("未找到任何分块,请确认文档是否已完成解析。\n")
@@ -203,14 +203,14 @@ func (t *ListKnowledgeChunksTool) buildOutput(
} }
return builder.String() return builder.String()
} }
builder.WriteString(fmt.Sprintf("本次拉取: %d 条, 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex)) fmt.Fprintf(builder, "本次拉取: %d 条, 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex)
builder.WriteString("=== 分块内容预览 ===\n\n") builder.WriteString("=== 分块内容预览 ===\n\n")
for idx, c := range chunks { for idx, c := range chunks {
builder.WriteString(fmt.Sprintf("Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1)) fmt.Fprintf(builder, "Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1)
builder.WriteString(fmt.Sprintf(" chunk_id: %s\n", c.ID)) fmt.Fprintf(builder, " chunk_id: %s\n", c.ID)
builder.WriteString(fmt.Sprintf(" 类型: %s\n", c.ChunkType)) fmt.Fprintf(builder, " 类型: %s\n", c.ChunkType)
builder.WriteString(fmt.Sprintf(" 内容: %s\n\n", summarizeContent(c.Content))) fmt.Fprintf(builder, " 内容: %s\n\n", summarizeContent(c.Content))
} }
if int64(fetched) < total { if int64(fetched) < total {
+1 -1
View File
@@ -502,7 +502,7 @@ func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Build
isOrdered := nodeName == "ol" isOrdered := nodeName == "ol"
node.Find("li").Each(func(idx int, li *goquery.Selection) { node.Find("li").Each(func(idx int, li *goquery.Selection) {
if isOrdered { if isOrdered {
markdown.WriteString(fmt.Sprintf("%d. ", idx+1)) fmt.Fprintf(markdown, "%d. ", idx+1)
} else { } else {
markdown.WriteString("- ") markdown.WriteString("- ")
} }
+3 -2
View File
@@ -138,7 +138,8 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
query := r.db.WithContext(ctx).Model(&types.Knowledge{}). query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed") Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed")
if params.Type == "file" { switch params.Type {
case "file":
// If file hash exists, prioritize exact match using hash // If file hash exists, prioritize exact match using hash
if params.FileHash != "" { if params.FileHash != "" {
var knowledge types.Knowledge var knowledge types.Knowledge
@@ -167,7 +168,7 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
} }
return true, &knowledge, nil return true, &knowledge, nil
} }
} else if params.Type == "url" { case "url":
// If file hash exists, prioritize exact match using hash // If file hash exists, prioritize exact match using hash
if params.FileHash != "" { if params.FileHash != "" {
var knowledge types.Knowledge var knowledge types.Knowledge
@@ -30,7 +30,6 @@ func (r *mcpServiceRepository) GetByID(ctx context.Context, tenantID uint64, id
err := r.db.WithContext(ctx). err := r.db.WithContext(ctx).
Where("id = ? AND tenant_id = ?", id, tenantID). Where("id = ? AND tenant_id = ?", id, tenantID).
First(&service).Error First(&service).Error
if err != nil { if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) { if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil return nil, nil
@@ -48,7 +47,6 @@ func (r *mcpServiceRepository) List(ctx context.Context, tenantID uint64) ([]*ty
Where("tenant_id = ?", tenantID). Where("tenant_id = ?", tenantID).
Order("created_at DESC"). Order("created_at DESC").
Find(&services).Error Find(&services).Error
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -63,7 +61,6 @@ func (r *mcpServiceRepository) ListEnabled(ctx context.Context, tenantID uint64)
Where("tenant_id = ? AND enabled = ?", tenantID, true). Where("tenant_id = ? AND enabled = ?", tenantID, true).
Order("created_at DESC"). Order("created_at DESC").
Find(&services).Error Find(&services).Error
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -81,7 +78,6 @@ func (r *mcpServiceRepository) ListByIDs(ctx context.Context, tenantID uint64, i
err := r.db.WithContext(ctx). err := r.db.WithContext(ctx).
Where("tenant_id = ? AND id IN ?", tenantID, ids). Where("tenant_id = ? AND id IN ?", tenantID, ids).
Find(&services).Error Find(&services).Error
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -221,11 +221,3 @@ func listI2listS(list []any) []string {
} }
return result return result
} }
func mapI2mapS(prop map[string]any) map[string]string {
attributes := make(map[string]string)
for k, v := range prop {
attributes[k] = fmt.Sprintf("%v", v)
}
return attributes
}
@@ -304,12 +304,12 @@ func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types
func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) { func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) {
if text == "" { if text == "" {
return nil, errors.New("Empty or invalid input string.") return nil, errors.New("empty or invalid input string")
} }
content := f.extractContent(ctx, text) content := f.extractContent(ctx, text)
// logger.Debugf(ctx, "Extracted content: %s", content) // logger.Debugf(ctx, "Extracted content: %s", content)
if content == "" { if content == "" {
return nil, errors.New("Empty or invalid input string.") return nil, errors.New("empty or invalid input string")
} }
var parsed interface{} var parsed interface{}
@@ -318,10 +318,10 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
err = json.Unmarshal([]byte(content), &parsed) err = json.Unmarshal([]byte(content), &parsed)
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error()) return nil, fmt.Errorf("failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
} }
if parsed == nil { if parsed == nil {
return nil, fmt.Errorf("Content must be a list of extractions or a dict.") return nil, fmt.Errorf("content must be a list of extractions or a dict")
} }
var items []interface{} var items []interface{}
@@ -330,7 +330,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
} else if parsedList, ok := parsed.([]interface{}); ok { } else if parsedList, ok := parsed.([]interface{}); ok {
items = parsedList items = parsedList
} else { } else {
return nil, fmt.Errorf("Expected list or dict, got %T", parsed) return nil, fmt.Errorf("expected list or dict, got %T", parsed)
} }
itemsList := make([]map[string]interface{}, 0) itemsList := make([]map[string]interface{}, 0)
@@ -338,7 +338,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
if itemMap, ok := item.(map[string]interface{}); ok { if itemMap, ok := item.(map[string]interface{}); ok {
itemsList = append(itemsList, itemMap) itemsList = append(itemsList, itemMap)
} else { } else {
return nil, fmt.Errorf("Each item in the sequence must be a mapping.") return nil, fmt.Errorf("each item in the sequence must be a mapping.")
} }
} }
return itemsList, nil return itemsList, nil
@@ -350,7 +350,7 @@ func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphDat
return nil, err return nil, err
} }
if len(matchData) == 0 { if len(matchData) == 0 {
logger.Debugf(ctx, "Received empty extraction data.") logger.Debugf(ctx, "received empty extraction data.")
return &types.GraphData{}, nil return &types.GraphData{}, nil
} }
// mm, _ := json.Marshal(matchData) // mm, _ := json.Marshal(matchData)
@@ -403,9 +403,7 @@ func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) {
node.Attributes = make([]string, 0) node.Attributes = make([]string, 0)
} }
if prenode.Attributes != nil { if prenode.Attributes != nil {
for _, attr := range prenode.Attributes { node.Attributes = append(node.Attributes, prenode.Attributes...)
node.Attributes = append(node.Attributes, attr)
}
} }
continue continue
} }
@@ -438,7 +438,6 @@ func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatM
"neighbor_len": runeLen(chunk.Content), "neighbor_len": runeLen(chunk.Content),
}) })
} }
} }
} }
@@ -1,15 +1,15 @@
package chatpipline package chatpipline
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"math" "math"
"strings" "strings"
"github.com/Tencent/WeKnora/internal/models/rerank" "github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/types/interfaces"
) )
// PluginRerank implements reranking functionality for chat pipeline // PluginRerank implements reranking functionality for chat pipeline
@@ -33,7 +33,7 @@ func (p *PluginRerank) ActivationEvents() []types.EventType {
// OnEvent handles reranking events in the chat pipeline // OnEvent handles reranking events in the chat pipeline
func (p *PluginRerank) OnEvent(ctx context.Context, func (p *PluginRerank) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError { ) *PluginError {
pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{
"session_id": chatManage.SessionID, "session_id": chatManage.SessionID,
@@ -86,52 +86,52 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
} }
} }
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
"result_cnt": len(rerankResp), "result_cnt": len(rerankResp),
}) })
for i := range chatManage.SearchResult { for i := range chatManage.SearchResult {
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata) chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
} }
reranked := make([]*types.SearchResult, 0, len(rerankResp)) reranked := make([]*types.SearchResult, 0, len(rerankResp))
for _, rr := range rerankResp { for _, rr := range rerankResp {
sr := chatManage.SearchResult[rr.Index] sr := chatManage.SearchResult[rr.Index]
base := sr.Score base := sr.Score
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base) sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
sr.Score = rr.RelevanceScore sr.Score = rr.RelevanceScore
sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage) sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage)
reranked = append(reranked, sr) reranked = append(reranked, sr)
} }
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7) final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
chatManage.RerankResult = final chatManage.RerankResult = final
// Log composite top scores and MMR selection summary // Log composite top scores and MMR selection summary
topN := min(3, len(reranked)) topN := min(3, len(reranked))
for i := 0; i < topN; i++ { for i := 0; i < topN; i++ {
pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{
"rank": i + 1, "rank": i + 1,
"chunk_id": reranked[i].ID, "chunk_id": reranked[i].ID,
"base_score": reranked[i].Metadata["base_score"], "base_score": reranked[i].Metadata["base_score"],
"final_score": fmt.Sprintf("%.4f", reranked[i].Score), "final_score": fmt.Sprintf("%.4f", reranked[i].Score),
"intent": chatManage.QueryIntent, "intent": chatManage.QueryIntent,
}) })
} }
if len(chatManage.RerankResult) == 0 { if len(chatManage.RerankResult) == 0 {
pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{ pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{
"filtered_cnt": 0, "filtered_cnt": 0,
}) })
return ErrSearchNothing return ErrSearchNothing
} }
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
"filtered_cnt": len(chatManage.RerankResult), "filtered_cnt": len(chatManage.RerankResult),
}) })
return next() return next()
} }
// rerank performs the actual reranking operation with given query and passages // rerank performs the actual reranking operation with given query and passages
func (p *PluginRerank) rerank(ctx context.Context, func (p *PluginRerank) rerank(ctx context.Context,
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string, chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
) []rerank.RankResult { ) []rerank.RankResult {
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
"query_variant": query, "query_variant": query,
@@ -173,149 +173,149 @@ func (p *PluginRerank) rerank(ctx context.Context,
rankFilter = append(rankFilter, result) rankFilter = append(rankFilter, result)
} }
} }
return rankFilter return rankFilter
} }
func ensureMetadata(m map[string]string) map[string]string { func ensureMetadata(m map[string]string) map[string]string {
if m == nil { if m == nil {
return make(map[string]string) return make(map[string]string)
} }
return m return m
} }
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 { func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
sourceWeight := 1.0 sourceWeight := 1.0
switch strings.ToLower(sr.KnowledgeSource) { switch strings.ToLower(sr.KnowledgeSource) {
case "web_search": case "web_search":
sourceWeight = 0.95 sourceWeight = 0.95
default: default:
sourceWeight = 1.0 sourceWeight = 1.0
} }
intentBoost := 1.0 intentBoost := 1.0
if chatManage.QueryIntent != "" { if chatManage.QueryIntent != "" {
switch chatManage.QueryIntent { switch chatManage.QueryIntent {
case "definition": case "definition":
if sr.ChunkType == string(types.ChunkTypeSummary) { if sr.ChunkType == string(types.ChunkTypeSummary) {
intentBoost = 1.05 intentBoost = 1.05
} }
case "howto": case "howto":
if sr.EndAt-sr.StartAt > 300 { if sr.EndAt-sr.StartAt > 300 {
intentBoost = 1.03 intentBoost = 1.03
} }
case "compare": case "compare":
intentBoost = 1.0 intentBoost = 1.0
} }
} }
positionPrior := 1.0 positionPrior := 1.0
if sr.StartAt >= 0 { if sr.StartAt >= 0 {
positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05) positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05)
} }
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
composite *= intentBoost composite *= intentBoost
composite *= positionPrior composite *= positionPrior
if composite < 0 { if composite < 0 {
composite = 0 composite = 0
} }
if composite > 1 { if composite > 1 {
composite = 1 composite = 1
} }
return composite return composite
} }
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult { func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
if k <= 0 || len(results) == 0 { if k <= 0 || len(results) == 0 {
return nil return nil
} }
pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{
"lambda": lambda, "lambda": lambda,
"k": k, "k": k,
"candidates": len(results), "candidates": len(results),
}) })
selected := make([]*types.SearchResult, 0, k) selected := make([]*types.SearchResult, 0, k)
candidates := make([]*types.SearchResult, len(results)) candidates := make([]*types.SearchResult, len(results))
copy(candidates, results) copy(candidates, results)
tokenSets := make([]map[string]struct{}, len(candidates)) tokenSets := make([]map[string]struct{}, len(candidates))
for i, r := range candidates { for i, r := range candidates {
tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r)) tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r))
} }
for len(selected) < k && len(candidates) > 0 { for len(selected) < k && len(candidates) > 0 {
bestIdx := 0 bestIdx := 0
bestScore := -1.0 bestScore := -1.0
for i, r := range candidates { for i, r := range candidates {
relevance := r.Score relevance := r.Score
redundancy := 0.0 redundancy := 0.0
for _, s := range selected { for _, s := range selected {
redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s)))) redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s))))
} }
mmr := lambda*relevance - (1.0-lambda)*redundancy mmr := lambda*relevance - (1.0-lambda)*redundancy
if mmr > bestScore { if mmr > bestScore {
bestScore = mmr bestScore = mmr
bestIdx = i bestIdx = i
} }
} }
selected = append(selected, candidates[bestIdx]) selected = append(selected, candidates[bestIdx])
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...) candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
} }
// Compute average redundancy among selected // Compute average redundancy among selected
avgRed := 0.0 avgRed := 0.0
if len(selected) > 1 { if len(selected) > 1 {
pairs := 0 pairs := 0
for i := 0; i < len(selected); i++ { for i := 0; i < len(selected); i++ {
for j := i + 1; j < len(selected); j++ { for j := i + 1; j < len(selected); j++ {
si := tokenizeSimple(getEnrichedPassage(ctx, selected[i])) si := tokenizeSimple(getEnrichedPassage(ctx, selected[i]))
sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j])) sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j]))
avgRed += jaccard(si, sj) avgRed += jaccard(si, sj)
pairs++ pairs++
} }
} }
if pairs > 0 { if pairs > 0 {
avgRed /= float64(pairs) avgRed /= float64(pairs)
} }
} }
pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{ pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{
"selected": len(selected), "selected": len(selected),
"avg_redundancy": fmt.Sprintf("%.4f", avgRed), "avg_redundancy": fmt.Sprintf("%.4f", avgRed),
}) })
return selected return selected
} }
func tokenizeSimple(text string) map[string]struct{} { func tokenizeSimple(text string) map[string]struct{} {
text = strings.ToLower(text) text = strings.ToLower(text)
fields := strings.Fields(text) fields := strings.Fields(text)
set := make(map[string]struct{}, len(fields)) set := make(map[string]struct{}, len(fields))
for _, f := range fields { for _, f := range fields {
if len(f) > 1 { if len(f) > 1 {
set[f] = struct{}{} set[f] = struct{}{}
} }
} }
return set return set
} }
func jaccard(a, b map[string]struct{}) float64 { func jaccard(a, b map[string]struct{}) float64 {
if len(a) == 0 && len(b) == 0 { if len(a) == 0 && len(b) == 0 {
return 0 return 0
} }
inter := 0 inter := 0
for k := range a { for k := range a {
if _, ok := b[k]; ok { if _, ok := b[k]; ok {
inter++ inter++
} }
} }
union := len(a) + len(b) - inter union := len(a) + len(b) - inter
if union == 0 { if union == 0 {
return 0 return 0
} }
return float64(inter) / float64(union) return float64(inter) / float64(union)
} }
func clampFloat(v, minV, maxV float64) float64 { func clampFloat(v, minV, maxV float64) float64 {
if v < minV { if v < minV {
return minV return minV
} }
if v > maxV { if v > maxV {
return maxV return maxV
} }
return v return v
} }
// getEnrichedPassage 合并Content和ImageInfo的文本内容 // getEnrichedPassage 合并Content和ImageInfo的文本内容
@@ -387,7 +387,7 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
// searchWebIfEnabled executes web search when enabled and returns converted results // searchWebIfEnabled executes web search when enabled and returns converted results
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult { func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
if !(chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0) { if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil || chatManage.TenantID <= 0 {
return nil return nil
} }
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
@@ -77,7 +77,7 @@ func (p *PluginTracing) OnEvent(ctx context.Context,
// Search traces search operations in the chat pipeline // Search traces search operations in the chat pipeline
func (p *PluginTracing) Search(ctx context.Context, func (p *PluginTracing) Search(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError { ) *PluginError {
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search") _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search")
defer span.End() defer span.End()
@@ -87,23 +87,23 @@ func (p *PluginTracing) Search(ctx context.Context,
attribute.Float64("keyword_threshold", chatManage.KeywordThreshold), attribute.Float64("keyword_threshold", chatManage.KeywordThreshold),
attribute.Int("match_count", chatManage.EmbeddingTopK), attribute.Int("match_count", chatManage.EmbeddingTopK),
) )
err := next() err := next()
searchResultJson, _ := json.Marshal(chatManage.SearchResult) searchResultJson, _ := json.Marshal(chatManage.SearchResult)
unique := make(map[string]struct{}) unique := make(map[string]struct{})
for _, r := range chatManage.SearchResult { for _, r := range chatManage.SearchResult {
unique[r.ID] = struct{}{} unique[r.ID] = struct{}{}
} }
span.SetAttributes( span.SetAttributes(
attribute.String("hybrid_search", string(searchResultJson)), attribute.String("hybrid_search", string(searchResultJson)),
attribute.String("processed_query", chatManage.ProcessedQuery), attribute.String("processed_query", chatManage.ProcessedQuery),
attribute.Int("search_unique_count", len(unique)), attribute.Int("search_unique_count", len(unique)),
) )
return err return err
} }
// Rerank traces rerank operations in the chat pipeline // Rerank traces rerank operations in the chat pipeline
func (p *PluginTracing) Rerank(ctx context.Context, func (p *PluginTracing) Rerank(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError { ) *PluginError {
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank") _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank")
defer span.End() defer span.End()
@@ -114,14 +114,14 @@ func (p *PluginTracing) Rerank(ctx context.Context,
attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold), attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold),
attribute.Int("rerank_filter_topk", chatManage.RerankTopK), attribute.Int("rerank_filter_topk", chatManage.RerankTopK),
) )
err := next() err := next()
resultJson, _ := json.Marshal(chatManage.RerankResult) resultJson, _ := json.Marshal(chatManage.RerankResult)
span.SetAttributes( span.SetAttributes(
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)), attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
attribute.String("rerank_resp_results", string(resultJson)), attribute.String("rerank_resp_results", string(resultJson)),
attribute.String("query_intent", chatManage.QueryIntent), attribute.String("query_intent", chatManage.QueryIntent),
) )
return err return err
} }
// Merge traces merge operations in the chat pipeline // Merge traces merge operations in the chat pipeline
-1
View File
@@ -58,7 +58,6 @@ func (s *chunkService) GetRepository() interfaces.ChunkRepository {
// Returns: // Returns:
// - error: Any error encountered during chunk creation // - error: Any error encountered during chunk creation
func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
err := s.chunkRepository.CreateChunks(ctx, chunks) err := s.chunkRepository.CreateChunks(ctx, chunks)
if err != nil { if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{ logger.ErrorWithFields(ctx, err, map[string]interface{}{
+2 -1
View File
@@ -21,7 +21,8 @@ type minioFileService struct {
// NewMinioFileService creates a MinIO file service // NewMinioFileService creates a MinIO file service
func NewMinioFileService(endpoint, func NewMinioFileService(endpoint,
accessKeyID, secretAccessKey, bucketName string, useSSL bool) (interfaces.FileService, error) { accessKeyID, secretAccessKey, bucketName string, useSSL bool,
) (interfaces.FileService, error) {
// Initialize MinIO client // Initialize MinIO client
client, err := minio.New(endpoint, &minio.Options{ client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+6 -4
View File
@@ -183,7 +183,8 @@ func (b *graphBuilder) extractEntities(ctx context.Context, chunk *types.Chunk)
// extractRelationships extracts relationships between entities // extractRelationships extracts relationships between entities
// It analyzes semantic connections between multiple entities and establishes relationships // It analyzes semantic connections between multiple entities and establishes relationships
func (b *graphBuilder) extractRelationships(ctx context.Context, func (b *graphBuilder) extractRelationships(ctx context.Context,
chunks []*types.Chunk, entities []*types.Entity) error { chunks []*types.Chunk, entities []*types.Entity,
) error {
log := logger.GetLogger(ctx) log := logger.GetLogger(ctx)
log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks)) log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks))
@@ -340,7 +341,7 @@ func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string {
return "" return ""
} }
var chunkContents = chunks[0].Content chunkContents := chunks[0].Content
preChunk := chunks[0] preChunk := chunks[0]
for i := 1; i < len(chunks); i++ { for i := 1; i < len(chunks); i++ {
@@ -369,7 +370,7 @@ func (b *graphBuilder) BuildGraph(ctx context.Context, chunks []*types.Chunk) er
startTime := time.Now() startTime := time.Now()
// Concurrently extract entities from each document chunk // Concurrently extract entities from each document chunk
var chunkEntities = make([][]*types.Entity, len(chunks)) chunkEntities := make([][]*types.Entity, len(chunks))
g, gctx := errgroup.WithContext(ctx) g, gctx := errgroup.WithContext(ctx)
g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency
@@ -868,7 +869,8 @@ func (b *graphBuilder) getEntityByTitle(title string) *types.Entity {
// dfs depth-first search to find connected components // dfs depth-first search to find connected components
func dfs(entityTitle string, func dfs(entityTitle string,
adjacencyList map[string]map[string]*types.Relationship, adjacencyList map[string]map[string]*types.Relationship,
visited map[string]bool, component *[]string) { visited map[string]bool, component *[]string,
) {
visited[entityTitle] = true visited[entityTitle] = true
*component = append(*component, entityTitle) *component = append(*component, entityTitle)
+14 -212
View File
@@ -864,207 +864,6 @@ func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowle
return return
} }
// processDocument handles asynchronous processing of document files
func (s *knowledgeService) processDocument(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool,
) {
logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel)
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument")
defer span.End()
span.SetAttributes(
attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)),
attribute.String("knowledge_base_id", kb.ID),
attribute.Int("tenant_id", int(kb.TenantID)),
attribute.String("knowledge_id", knowledge.ID),
attribute.String("file_name", knowledge.FileName),
attribute.String("file_type", knowledge.FileType),
attribute.String("file_path", knowledge.FilePath),
attribute.Int64("file_size", knowledge.FileSize),
attribute.String("embedding_model", knowledge.EmbeddingModelID),
attribute.Bool("enable_multimodal", enableMultimodel),
)
logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String())
if !enableMultimodel && IsImageType(knowledge.FileType) {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = ErrImageNotParse.Error()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(ErrImageNotParse)
return
}
// Update status to processing
knowledge.ParseStatus = "processing"
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
span.RecordError(err)
return
}
// Read and chunk the document
f, err := file.Open()
if err != nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", err).Errorf("processDocument open file failed")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(err)
return
}
defer f.Close()
span.AddEvent("start read file")
contentBytes, err := io.ReadAll(f)
if err != nil {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(err)
return
}
// Split file into chunks using document reader service
logger.GetLogger(ctx).Infof("processDocument split file content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v",
len(contentBytes), knowledge.FileName, knowledge.FileType, kb.ChunkingConfig.Separators, enableMultimodel)
var vlmConfig *proto.VLMConfig
if enableMultimodel {
vlmConfig, err = s.getVLMProtoConfig(ctx, kb)
if err != nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", err).Errorf("processDocument build VLM config failed")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(err)
return
}
if vlmConfig == nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Error("processDocument enable multimodal but VLM config missing")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "VLM 配置缺失"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(errors.New("vlm config missing"))
return
}
}
resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{
FileContent: contentBytes,
FileName: knowledge.FileName,
FileType: knowledge.FileType,
ReadConfig: &proto.ReadConfig{
ChunkSize: int32(kb.ChunkingConfig.ChunkSize),
ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap),
Separators: kb.ChunkingConfig.Separators,
EnableMultimodal: enableMultimodel,
StorageConfig: &proto.StorageConfig{
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
Region: kb.StorageConfig.Region,
BucketName: kb.StorageConfig.BucketName,
AccessKeyId: kb.StorageConfig.SecretID,
SecretAccessKey: kb.StorageConfig.SecretKey,
AppId: kb.StorageConfig.AppID,
PathPrefix: kb.StorageConfig.PathPrefix,
},
VlmConfig: vlmConfig,
},
RequestId: ctx.Value(types.RequestIDContextKey).(string),
})
if err != nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", err).Errorf("processDocument read file failed")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
span.RecordError(err)
return
}
// Process and store chunks
span.AddEvent("start process chunks")
s.processChunks(ctx, kb, knowledge, resp.Chunks)
}
// processDocumentFromURL handles asynchronous processing of URL content
func (s *knowledgeService) processDocumentFromURL(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool,
) {
// Update status to processing
knowledge.ParseStatus = "processing"
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
return
}
logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel)
// Fetch and chunk content from URL
var vlmConfig *proto.VLMConfig
if enableMultimodel {
cfg, cfgErr := s.getVLMProtoConfig(ctx, kb)
if cfgErr != nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", cfgErr).Errorf("processDocumentFromURL build VLM config failed")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = cfgErr.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return
}
if cfg == nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Error("processDocumentFromURL enable multimodal but VLM config missing")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "VLM 配置缺失"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return
}
vlmConfig = cfg
}
resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{
Url: url,
Title: knowledge.Title,
ReadConfig: &proto.ReadConfig{
ChunkSize: int32(kb.ChunkingConfig.ChunkSize),
ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap),
Separators: kb.ChunkingConfig.Separators,
EnableMultimodal: enableMultimodel,
StorageConfig: &proto.StorageConfig{
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
Region: kb.StorageConfig.Region,
BucketName: kb.StorageConfig.BucketName,
AccessKeyId: kb.StorageConfig.SecretID,
SecretAccessKey: kb.StorageConfig.SecretKey,
AppId: kb.StorageConfig.AppID,
PathPrefix: kb.StorageConfig.PathPrefix,
},
VlmConfig: vlmConfig,
},
RequestId: ctx.Value(types.RequestIDContextKey).(string),
})
if err != nil {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return
}
// Process and store chunks
s.processChunks(ctx, kb, knowledge, resp.Chunks)
}
// processDocumentFromPassage handles asynchronous processing of text passages // processDocumentFromPassage handles asynchronous processing of text passages
func (s *knowledgeService) processDocumentFromPassage(ctx context.Context, func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string, kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string,
@@ -1767,7 +1566,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID
batch := 10 batch := 10
g, gctx := errgroup.WithContext(ctx) g, gctx := errgroup.WithContext(ctx)
for ids := range slices.Chunk(delKnowledge, batch) { for ids := range slices.Chunk(delKnowledge, batch) {
ids = ids
g.Go(func() error { g.Go(func() error {
err := s.DeleteKnowledgeList(gctx, ids) err := s.DeleteKnowledgeList(gctx, ids)
if err != nil { if err != nil {
@@ -1787,7 +1585,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID
g, gctx = errgroup.WithContext(ctx) g, gctx = errgroup.WithContext(ctx)
g.SetLimit(batch) g.SetLimit(batch)
for _, knowledge := range addKnowledge { for _, knowledge := range addKnowledge {
knowledge = knowledge
g.Go(func() error { g.Go(func() error {
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
if err != nil { if err != nil {
@@ -1916,7 +1713,8 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri
} }
// Mark that we've found chunks for this image // Mark that we've found chunks for this image
if child.ChunkType == types.ChunkTypeImageCaption { switch child.ChunkType {
case types.ChunkTypeImageCaption:
hasCaptionChunk = true hasCaptionChunk = true
// Update caption if it has changed // Update caption if it has changed
if image.Caption != cImageInfo[0].Caption { if image.Caption != cImageInfo[0].Caption {
@@ -1924,7 +1722,7 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri
child.ImageInfo = imageInfo child.ImageInfo = imageInfo
updateChunk = append(updateChunk, chunkChildren[i]) updateChunk = append(updateChunk, chunkChildren[i])
} }
} else if child.ChunkType == types.ChunkTypeImageOCR { case types.ChunkTypeImageOCR:
hasOCRChunk = true hasOCRChunk = true
// Update OCR if it has changed // Update OCR if it has changed
if image.OCRText != cImageInfo[0].OCRText { if image.OCRText != cImageInfo[0].OCRText {
@@ -2240,7 +2038,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目 // calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
func (s *knowledgeService) calculateAppendOperations(ctx context.Context, func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload) ([]types.FAQEntryPayload, int, error) { tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload,
) ([]types.FAQEntryPayload, int, error) {
if len(entries) == 0 { if len(entries) == 0 {
return []types.FAQEntryPayload{}, 0, nil return []types.FAQEntryPayload{}, 0, nil
} }
@@ -2249,7 +2048,8 @@ func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
// calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目 // calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目
func (s *knowledgeService) calculateReplaceOperations(ctx context.Context, func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload) ([]types.FAQEntryPayload, []*types.Chunk, int, error) { tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload,
) ([]types.FAQEntryPayload, []*types.Chunk, int, error) {
// 计算所有新条目的 content hash,并同时构建 hash 到 entry 的映射 // 计算所有新条目的 content hash,并同时构建 hash 到 entry 的映射
type entryWithHash struct { type entryWithHash struct {
entry types.FAQEntryPayload entry types.FAQEntryPayload
@@ -2316,7 +2116,8 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
// executeFAQImport 执行实际的FAQ导入逻辑 // executeFAQImport 执行实际的FAQ导入逻辑
func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string, func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string,
payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int) (err error) { payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int,
) (err error) {
// 保存知识库和embedding模型信息,用于清理索引 // 保存知识库和embedding模型信息,用于清理索引
var kb *types.KnowledgeBase var kb *types.KnowledgeBase
var embeddingModel embedding.Embedder var embeddingModel embedding.Embedder
@@ -2364,8 +2165,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
if payload.Mode == types.FAQBatchModeReplace { if payload.Mode == types.FAQBatchModeReplace {
// Replace模式:计算需要删除、创建、更新的条目 // Replace模式:计算需要删除、创建、更新的条目
entriesToProcess, chunksToDelete, skippedCount, err = entriesToProcess, chunksToDelete, skippedCount, err = s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
if err != nil { if err != nil {
return fmt.Errorf("failed to calculate replace operations: %w", err) return fmt.Errorf("failed to calculate replace operations: %w", err)
} }
@@ -3080,13 +2880,15 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint
} }
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
progress, total, processed int, errorMsg string) error { progress, total, processed int, errorMsg string,
) error {
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg) return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
} }
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex // updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
progress, total, processed int, errorMsg string) error { progress, total, processed int, errorMsg string,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64) tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil { if err != nil {
@@ -131,14 +131,15 @@ func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types
kb.EnsureDefaults() kb.EnsureDefaults()
// Get knowledge count // Get knowledge count
if kb.Type == types.KnowledgeBaseTypeDocument { switch kb.Type {
case types.KnowledgeBaseTypeDocument:
knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID) knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID)
if err != nil { if err != nil {
logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err) logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err)
} else { } else {
kb.KnowledgeCount = knowledgeCount kb.KnowledgeCount = knowledgeCount
} }
} else if kb.Type == types.KnowledgeBaseTypeFAQ { case types.KnowledgeBaseTypeFAQ:
// Get chunk count // Get chunk count
chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID) chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID)
if err != nil { if err != nil {
@@ -192,7 +192,6 @@ func (s *smartCompressionStrategy) summarizeMessages(ctx context.Context, messag
Temperature: 0.3, // Lower temperature for more consistent summaries Temperature: 0.3, // Lower temperature for more consistent summaries
MaxTokens: 500, // Limit summary length MaxTokens: 500, // Limit summary length
}) })
if err != nil { if err != nil {
return "", fmt.Errorf("failed to generate summary: %w", err) return "", fmt.Errorf("failed to generate summary: %w", err)
} }
@@ -96,11 +96,6 @@ func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams {
return getNgrams(n, words, exclusive) return getNgrams(n, words, exclusive)
} }
func lenLcs(x, y []string) int {
table := lcs(x, y)
return table[len(x)][len(y)]
}
func lcs(x, y []string) [][]int { func lcs(x, y []string) [][]int {
n, m := len(x), len(y) n, m := len(x), len(y)
table := make([][]int, n+1) table := make([][]int, n+1)
@@ -83,7 +83,6 @@ func (m *MetricList) Avg() *types.MetricResult {
// HookMetric tracks evaluation metrics for QA pairs // HookMetric tracks evaluation metrics for QA pairs
type HookMetric struct { type HookMetric struct {
task *types.EvaluationTask
qaPairMetricList []*qaPairMetric // Per-QA pair metrics qaPairMetricList []*qaPairMetric // Per-QA pair metrics
metricResults *MetricList // Aggregated results metricResults *MetricList // Aggregated results
mu *sync.RWMutex // Thread safety mu *sync.RWMutex // Thread safety
@@ -27,7 +27,7 @@ func (r *RetrieveEngineRegistry) Register(repo interfaces.RetrieveEngineService)
defer r.mu.Unlock() defer r.mu.Unlock()
if _, exists := r.repositories[repo.EngineType()]; exists { if _, exists := r.repositories[repo.EngineType()]; exists {
return fmt.Errorf("Repository type %s already registered", repo.EngineType()) return fmt.Errorf("repository type %s already registered", repo.EngineType())
} }
r.repositories[repo.EngineType()] = repo r.repositories[repo.EngineType()] = repo
@@ -43,7 +43,7 @@ func (r *RetrieveEngineRegistry) GetRetrieveEngineService(repoType types.Retriev
repo, exists := r.repositories[repoType] repo, exists := r.repositories[repoType]
if !exists { if !exists {
return nil, fmt.Errorf("Repository of type %s not found", repoType) return nil, fmt.Errorf("repository of type %s not found", repoType)
} }
return repo, nil return repo, nil
-1
View File
@@ -587,7 +587,6 @@ func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, sess
// 3. Session's SummaryModelID (if not Remote) // 3. Session's SummaryModelID (if not Remote)
// 4. First knowledge base's SummaryModelID // 4. First knowledge base's SummaryModelID
func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) { func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) {
// First, check if session has a SummaryModelID and if it's a Remote model // First, check if session has a SummaryModelID and if it's a Remote model
if session.SummaryModelID != "" { if session.SummaryModelID != "" {
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID) model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
+1 -1
View File
@@ -124,7 +124,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// Session service (depends on agent service) // Session service (depends on agent service)
// SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed // SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed
must(container.Provide(service.NewSessionService)) must(container.Provide(service.NewSessionService))
must(container.Provide(router.NewAsyncqClient)) must(container.Provide(router.NewAsyncqClient))
must(container.Provide(router.NewAsynqServer)) must(container.Provide(router.NewAsynqServer))
+21 -25
View File
@@ -26,7 +26,6 @@ import (
"github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils" "github.com/Tencent/WeKnora/internal/utils"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/ollama/ollama/api" "github.com/ollama/ollama/api"
@@ -198,7 +197,7 @@ type InitializationRequest struct {
// UpdateKBConfig 根据知识库ID和模型ID更新配置(简化版) // UpdateKBConfig 根据知识库ID和模型ID更新配置(简化版)
func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
var req KBModelConfigRequest var req KBModelConfigRequest
if err := c.ShouldBindJSON(&req); err != nil { if err := c.ShouldBindJSON(&req); err != nil {
@@ -210,7 +209,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
// 获取知识库信息 // 获取知识库信息
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
if err != nil || kb == nil { if err != nil || kb == nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewNotFoundError("知识库不存在")) c.Error(errors.NewNotFoundError("知识库不存在"))
return return
} }
@@ -350,7 +349,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
// InitializeByKB 根据知识库ID执行配置更新 // InitializeByKB 根据知识库ID执行配置更新
func (h *InitializationHandler) InitializeByKB(c *gin.Context) { func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
var req InitializationRequest var req InitializationRequest
if err := c.ShouldBindJSON(&req); err != nil { if err := c.ShouldBindJSON(&req); err != nil {
@@ -358,12 +357,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
c.Error(errors.NewBadRequestError(err.Error())) c.Error(errors.NewBadRequestError(err.Error()))
return return
} }
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", secutils.SanitizeForLog(kbIdStr), secutils.SanitizeForLog(utils.ToJSON(req))) logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", utils.SanitizeForLog(kbIdStr), utils.SanitizeForLog(utils.ToJSON(req)))
// 获取指定知识库信息 // 获取指定知识库信息
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
if err != nil { if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error())) c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error()))
return return
} }
@@ -450,18 +449,18 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
}{ }{
{ {
modelType: types.ModelTypeKnowledgeQA, modelType: types.ModelTypeKnowledgeQA,
name: secutils.SanitizeForLog(req.LLM.ModelName), name: utils.SanitizeForLog(req.LLM.ModelName),
source: types.ModelSource(req.LLM.Source), source: types.ModelSource(req.LLM.Source),
description: "LLM Model for Knowledge QA", description: "LLM Model for Knowledge QA",
baseURL: secutils.SanitizeForLog(req.LLM.BaseURL), baseURL: utils.SanitizeForLog(req.LLM.BaseURL),
apiKey: req.LLM.APIKey, apiKey: req.LLM.APIKey,
}, },
{ {
modelType: types.ModelTypeEmbedding, modelType: types.ModelTypeEmbedding,
name: secutils.SanitizeForLog(req.Embedding.ModelName), name: utils.SanitizeForLog(req.Embedding.ModelName),
source: types.ModelSource(req.Embedding.Source), source: types.ModelSource(req.Embedding.Source),
description: "Embedding Model", description: "Embedding Model",
baseURL: secutils.SanitizeForLog(req.Embedding.BaseURL), baseURL: utils.SanitizeForLog(req.Embedding.BaseURL),
apiKey: req.Embedding.APIKey, apiKey: req.Embedding.APIKey,
dimension: req.Embedding.Dimension, dimension: req.Embedding.Dimension,
}, },
@@ -480,10 +479,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
interfaceType string interfaceType string
}{ }{
modelType: types.ModelTypeRerank, modelType: types.ModelTypeRerank,
name: secutils.SanitizeForLog(req.Rerank.ModelName), name: utils.SanitizeForLog(req.Rerank.ModelName),
source: types.ModelSourceRemote, source: types.ModelSourceRemote,
description: "Rerank Model", description: "Rerank Model",
baseURL: secutils.SanitizeForLog(req.Rerank.BaseURL), baseURL: utils.SanitizeForLog(req.Rerank.BaseURL),
apiKey: req.Rerank.APIKey, apiKey: req.Rerank.APIKey,
}) })
} }
@@ -501,10 +500,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
interfaceType string interfaceType string
}{ }{
modelType: types.ModelTypeVLLM, modelType: types.ModelTypeVLLM,
name: secutils.SanitizeForLog(req.Multimodal.VLM.ModelName), name: utils.SanitizeForLog(req.Multimodal.VLM.ModelName),
source: types.ModelSourceRemote, source: types.ModelSourceRemote,
description: "VLM Model", description: "VLM Model",
baseURL: secutils.SanitizeForLog(req.Multimodal.VLM.BaseURL), baseURL: utils.SanitizeForLog(req.Multimodal.VLM.BaseURL),
apiKey: req.Multimodal.VLM.APIKey, apiKey: req.Multimodal.VLM.APIKey,
interfaceType: req.Multimodal.VLM.InterfaceType, interfaceType: req.Multimodal.VLM.InterfaceType,
}) })
@@ -676,7 +675,7 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb) err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
if err != nil { if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)}) logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error())) c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
return return
} }
@@ -775,7 +774,7 @@ func (h *InitializationHandler) CheckOllamaModels(c *gin.Context) {
modelStatus[modelName] = available modelStatus[modelName] = available
} }
logger.Infof(ctx, "Model %s availability: %v", secutils.SanitizeForLog(modelName), modelStatus[modelName]) logger.Infof(ctx, "Model %s availability: %v", utils.SanitizeForLog(modelName), modelStatus[modelName])
} }
c.JSON(http.StatusOK, gin.H{ c.JSON(http.StatusOK, gin.H{
@@ -1010,8 +1009,8 @@ func (h *InitializationHandler) pullModelWithProgress(ctx context.Context,
// 使用Ollama客户端的Pull方法,带进度回调 // 使用Ollama客户端的Pull方法,带进度回调
err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error { err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error {
var progressPercent float64 = 0.0 progressPercent := 0.0
var message string = "下载中" message := "下载中"
if progress.Total > 0 && progress.Completed > 0 { if progress.Total > 0 && progress.Completed > 0 {
progressPercent = float64(progress.Completed) / float64(progress.Total) * 100 progressPercent = float64(progress.Completed) / float64(progress.Total) * 100
@@ -1057,7 +1056,7 @@ func (h *InitializationHandler) updateTaskStatus(
// GetCurrentConfigByKB 根据知识库ID获取配置信息 // GetCurrentConfigByKB 根据知识库ID获取配置信息
func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) { func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
kbIdStr := secutils.SanitizeForLog(c.Param("kbId")) kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
logger.Info(ctx, "Getting configuration for knowledge base") logger.Info(ctx, "Getting configuration for knowledge base")
@@ -1103,10 +1102,7 @@ func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) {
Page: 1, Page: 1,
PageSize: 1, PageSize: 1,
}, "") }, "")
hasFiles := false hasFiles := err == nil && knowledgeList != nil && knowledgeList.Total > 0
if err == nil && knowledgeList != nil && knowledgeList.Total > 0 {
hasFiles = true
}
// 构建配置响应 // 构建配置响应
config := h.buildConfigResponse(ctx, models, kb, hasFiles) config := h.buildConfigResponse(ctx, models, kb, hasFiles)
@@ -1339,7 +1335,7 @@ func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) {
emb, err := embedding.NewEmbedder(cfg) emb, err := embedding.NewEmbedder(cfg)
if err != nil { if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": secutils.SanitizeForLog(req.ModelName)}) logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": utils.SanitizeForLog(req.ModelName)})
c.JSON(http.StatusOK, gin.H{ c.JSON(http.StatusOK, gin.H{
"success": true, "success": true,
"data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0}, "data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0},
@@ -1595,7 +1591,7 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB")) c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB"))
return return
} }
logger.Infof(ctx, "Processing image: %s", secutils.SanitizeForLog(header.Filename)) logger.Infof(ctx, "Processing image: %s", utils.SanitizeForLog(header.Filename))
// 解析文档分割配置 // 解析文档分割配置
chunkSizeInt32, err := strconv.ParseInt(req.ChunkSize, 10, 32) chunkSizeInt32, err := strconv.ParseInt(req.ChunkSize, 10, 32)
-1
View File
@@ -30,4 +30,3 @@ var (
// ErrConnectionClosed is returned when connection is closed unexpectedly // ErrConnectionClosed is returned when connection is closed unexpectedly
ErrConnectionClosed = errors.New("connection closed") ErrConnectionClosed = errors.New("connection closed")
) )
-1
View File
@@ -122,7 +122,6 @@ func TestRemoteAPIChat(t *testing.T) {
response.Usage.CompletionTokens, response.Usage.CompletionTokens,
response.Usage.TotalTokens) response.Usage.TotalTokens)
}) })
}) })
} }
} }
-1
View File
@@ -29,7 +29,6 @@ type RankResult struct {
// Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field // Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field
func (r *RankResult) UnmarshalJSON(data []byte) error { func (r *RankResult) UnmarshalJSON(data []byte) error {
var temp struct { var temp struct {
Index int `json:"index"` Index int `json:"index"`
Document DocumentInfo `json:"document"` Document DocumentInfo `json:"document"`
+2 -2
View File
@@ -73,7 +73,7 @@ func (m *MemoryStreamManager) AppendEvent(ctx context.Context, sessionID, messag
// Append event // Append event
stream.events = append(stream.events, event) stream.events = append(stream.events, event)
stream.lastUpdated = time.Now() stream.lastUpdated = time.Now()
return nil return nil
} }
@@ -104,7 +104,7 @@ func (m *MemoryStreamManager) GetEvents(ctx context.Context, sessionID, messageI
copy(eventsCopy, events) copy(eventsCopy, events)
return eventsCopy, nextOffset, nil return eventsCopy, nextOffset, nil
} }
// Ensure MemoryStreamManager implements StreamManager interface // Ensure MemoryStreamManager implements StreamManager interface
var _ interfaces.StreamManager = (*MemoryStreamManager)(nil) var _ interfaces.StreamManager = (*MemoryStreamManager)(nil)
+7 -2
View File
@@ -6,13 +6,14 @@ const (
TypeFAQImport = "faq:import" // FAQ导入任务 TypeFAQImport = "faq:import" // FAQ导入任务
) )
// ExtractChunkPayload represents the extract chunk task payload
type ExtractChunkPayload struct { type ExtractChunkPayload struct {
TenantID uint64 `json:"tenant_id"` TenantID uint64 `json:"tenant_id"`
ChunkID string `json:"chunk_id"` ChunkID string `json:"chunk_id"`
ModelID string `json:"model_id"` ModelID string `json:"model_id"`
} }
// DocumentProcessPayload 文档处理任务payload // DocumentProcessPayload represents the document process task payload
type DocumentProcessPayload struct { type DocumentProcessPayload struct {
RequestId string `json:"request_id"` RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"` TenantID uint64 `json:"tenant_id"`
@@ -26,7 +27,7 @@ type DocumentProcessPayload struct {
EnableMultimodel bool `json:"enable_multimodel"` EnableMultimodel bool `json:"enable_multimodel"`
} }
// FAQImportPayload FAQ导入任务payload // FAQImportPayload represents the FAQ import task payload
type FAQImportPayload struct { type FAQImportPayload struct {
TenantID uint64 `json:"tenant_id"` TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"` TaskID string `json:"task_id"`
@@ -36,6 +37,7 @@ type FAQImportPayload struct {
Mode string `json:"mode"` Mode string `json:"mode"`
} }
// PromptTemplateStructured represents the prompt template structured
type PromptTemplateStructured struct { type PromptTemplateStructured struct {
Description string `json:"description"` Description string `json:"description"`
Tags []string `json:"tags"` Tags []string `json:"tags"`
@@ -48,6 +50,7 @@ type GraphNode struct {
Attributes []string `json:"attributes,omitempty"` Attributes []string `json:"attributes,omitempty"`
} }
// GraphRelation represents the relation of the graph
type GraphRelation struct { type GraphRelation struct {
Node1 string `json:"node1,omitempty"` Node1 string `json:"node1,omitempty"`
Node2 string `json:"node2,omitempty"` Node2 string `json:"node2,omitempty"`
@@ -60,11 +63,13 @@ type GraphData struct {
Relation []*GraphRelation `json:"relation,omitempty"` Relation []*GraphRelation `json:"relation,omitempty"`
} }
// NameSpace represents the name space of the knowledge base and knowledge
type NameSpace struct { type NameSpace struct {
KnowledgeBase string `json:"knowledge_base"` KnowledgeBase string `json:"knowledge_base"`
Knowledge string `json:"knowledge"` Knowledge string `json:"knowledge"`
} }
// Labels returns the labels of the name space
func (n NameSpace) Labels() []string { func (n NameSpace) Labels() []string {
res := make([]string, 0) res := make([]string, 0)
if n.KnowledgeBase != "" { if n.KnowledgeBase != "" {
+7 -3
View File
@@ -160,10 +160,14 @@ type FAQSearchRequest struct {
type FAQImportTaskStatus string type FAQImportTaskStatus string
const ( const (
FAQImportStatusPending FAQImportTaskStatus = "pending" // FAQImportStatusPending represents the pending status of the FAQ import task
FAQImportStatusPending FAQImportTaskStatus = "pending"
// FAQImportStatusProcessing represents the processing status of the FAQ import task
FAQImportStatusProcessing FAQImportTaskStatus = "processing" FAQImportStatusProcessing FAQImportTaskStatus = "processing"
FAQImportStatusCompleted FAQImportTaskStatus = "completed" // FAQImportStatusCompleted represents the completed status of the FAQ import task
FAQImportStatusFailed FAQImportTaskStatus = "failed" FAQImportStatusCompleted FAQImportTaskStatus = "completed"
// FAQImportStatusFailed represents the failed status of the FAQ import task
FAQImportStatusFailed FAQImportTaskStatus = "failed"
) )
// FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息 // FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息
+4 -1
View File
@@ -10,10 +10,13 @@ import (
) )
const ( const (
// KnowledgeTypeManual represents the manual knowledge type
KnowledgeTypeManual = "manual" KnowledgeTypeManual = "manual"
KnowledgeTypeFAQ = "faq" // KnowledgeTypeFAQ represents the FAQ knowledge type
KnowledgeTypeFAQ = "faq"
) )
// ManualKnowledgeFormat represents the format of the manual knowledge
const ( const (
ManualKnowledgeFormatMarkdown = "markdown" ManualKnowledgeFormatMarkdown = "markdown"
ManualKnowledgeStatusDraft = "draft" ManualKnowledgeStatusDraft = "draft"
+10 -7
View File
@@ -8,32 +8,34 @@ import (
"gorm.io/gorm" "gorm.io/gorm"
) )
// KnowledgeBaseType represents the type of the knowledge base
const ( const (
// KnowledgeBaseTypeDocument represents the document knowledge base type
KnowledgeBaseTypeDocument = "document" KnowledgeBaseTypeDocument = "document"
KnowledgeBaseTypeFAQ = "faq" KnowledgeBaseTypeFAQ = "faq"
) )
// FAQIndexMode 表示 FAQ 索引策略:仅索引标准问还是同时索引问答内容 // FAQIndexMode represents the FAQ index mode: only index questions or index questions and answers
type FAQIndexMode string type FAQIndexMode string
const ( const (
// FAQIndexModeQuestionOnly 仅使用标准问及相似问向量化 // FAQIndexModeQuestionOnly only index questions and similar questions
FAQIndexModeQuestionOnly FAQIndexMode = "question_only" FAQIndexModeQuestionOnly FAQIndexMode = "question_only"
// FAQIndexModeQuestionAnswer 将标准问与答案拼接后向量化 // FAQIndexModeQuestionAnswer index questions and answers together
FAQIndexModeQuestionAnswer FAQIndexMode = "question_answer" FAQIndexModeQuestionAnswer FAQIndexMode = "question_answer"
) )
// FAQQuestionIndexMode 表示 FAQ 问题索引方式:一起索引还是分别索引 // FAQQuestionIndexMode represents the FAQ question index mode: index together or index separately
type FAQQuestionIndexMode string type FAQQuestionIndexMode string
const ( const (
// FAQQuestionIndexModeCombined 将标准问和相似问一起索引(当前方式) // FAQQuestionIndexModeCombined index questions and similar questions together
FAQQuestionIndexModeCombined FAQQuestionIndexMode = "combined" FAQQuestionIndexModeCombined FAQQuestionIndexMode = "combined"
// FAQQuestionIndexModeSeparate 将标准问和相似问分别索引 // FAQQuestionIndexModeSeparate index questions and similar questions separately
FAQQuestionIndexModeSeparate FAQQuestionIndexMode = "separate" FAQQuestionIndexModeSeparate FAQQuestionIndexMode = "separate"
) )
// KnowledgeBase represents a knowledge base // KnowledgeBase represents a knowledge base entity
type KnowledgeBase struct { type KnowledgeBase struct {
// Unique identifier of the knowledge base // Unique identifier of the knowledge base
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"` ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
@@ -195,6 +197,7 @@ func (c *VLMConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c) return json.Unmarshal(b, c)
} }
// ExtractConfig represents the extract configuration for a knowledge base
type ExtractConfig struct { type ExtractConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"` Enabled bool `yaml:"enabled" json:"enabled"`
Text string `yaml:"text" json:"text,omitempty"` Text string `yaml:"text" json:"text,omitempty"`
+2
View File
@@ -19,6 +19,7 @@ const (
ModelTypeVLLM ModelType = "VLLM" // VLLM model ModelTypeVLLM ModelType = "VLLM" // VLLM model
) )
// ModelStatus represents the status of the model
type ModelStatus string type ModelStatus string
const ( const (
@@ -36,6 +37,7 @@ const (
ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model
) )
// EmbeddingParameters represents the embedding parameters for a model
type EmbeddingParameters struct { type EmbeddingParameters struct {
Dimension int `yaml:"dimension" json:"dimension"` Dimension int `yaml:"dimension" json:"dimension"`
TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"` TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"`
+2
View File
@@ -17,6 +17,7 @@ const (
FallbackStrategyModel FallbackStrategy = "model" // Model fallback response FallbackStrategyModel FallbackStrategy = "model" // Model fallback response
) )
// SummaryConfig represents the summary configuration for a session
type SummaryConfig struct { type SummaryConfig struct {
// Max tokens // Max tokens
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`
@@ -109,6 +110,7 @@ func (s *Session) BeforeCreate(tx *gorm.DB) (err error) {
return nil return nil
} }
// StringArray represents a list of strings
type StringArray []string type StringArray []string
// Value implements the driver.Valuer interface, used to convert StringArray to database value // Value implements the driver.Valuer interface, used to convert StringArray to database value
+2
View File
@@ -44,10 +44,12 @@ type Tenant struct {
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"` DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
} }
// RetrieverEngines represents the retriever engines for a tenant
type RetrieverEngines struct { type RetrieverEngines struct {
Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"` Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"`
} }
// BeforeCreate is a hook function that is called before creating a tenant
func (t *Tenant) BeforeCreate(tx *gorm.DB) error { func (t *Tenant) BeforeCreate(tx *gorm.DB) error {
if t.RetrieverEngines.Engines == nil { if t.RetrieverEngines.Engines == nil {
t.RetrieverEngines.Engines = []RetrieverEngineParams{} t.RetrieverEngines.Engines = []RetrieverEngineParams{}