refactor: Clean up code by removing unnecessary blank lines and improving comments for better readability and maintainability
This commit is contained in:
@@ -0,0 +1,14 @@
|
|||||||
|
version: 2
|
||||||
|
linters-settings:
|
||||||
|
lll:
|
||||||
|
line-length: 120
|
||||||
|
tab-width: 4
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
- lll # 控制行宽
|
||||||
|
- govet
|
||||||
|
- revive
|
||||||
|
formatters:
|
||||||
|
enable:
|
||||||
|
- gofmt
|
||||||
|
- gofumpt
|
||||||
@@ -277,7 +277,6 @@ func (cli *CLI) askQuestion(query string) {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset)
|
fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -638,7 +638,6 @@ func (e *AgentEngine) streamReflectionToEventBus(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf(ctx, "Reflection failed: %v", err)
|
logger.Warnf(ctx, "Reflection failed: %v", err)
|
||||||
return "", err
|
return "", err
|
||||||
@@ -710,7 +709,6 @@ func (e *AgentEngine) streamThinkingToEventBus(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Errorf(ctx, "[Agent][Thinking][Iteration-%d] Thinking stream failed: %v", iteration+1, err)
|
logger.Errorf(ctx, "[Agent][Thinking][Iteration-%d] Thinking stream failed: %v", iteration+1, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -811,7 +809,6 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Errorf(ctx, "[Agent][FinalAnswer] Final answer generation failed: %v", err)
|
logger.Errorf(ctx, "[Agent][FinalAnswer] Final answer generation failed: %v", err)
|
||||||
common.PipelineError(ctx, "Agent", "final_answer_stream_failed", map[string]interface{}{
|
common.PipelineError(ctx, "Agent", "final_answer_stream_failed", map[string]interface{}{
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package tools
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -421,37 +420,3 @@ func (t *DatabaseQueryTool) formatQueryResults(columns []string, results []map[s
|
|||||||
|
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper function to convert database value to proper type
|
|
||||||
func convertDatabaseValue(val interface{}) interface{} {
|
|
||||||
if val == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
switch v := val.(type) {
|
|
||||||
case []byte:
|
|
||||||
return string(v)
|
|
||||||
case sql.NullString:
|
|
||||||
if v.Valid {
|
|
||||||
return v.String
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case sql.NullInt64:
|
|
||||||
if v.Valid {
|
|
||||||
return v.Int64
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case sql.NullFloat64:
|
|
||||||
if v.Valid {
|
|
||||||
return v.Float64
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case sql.NullBool:
|
|
||||||
if v.Valid {
|
|
||||||
return v.Bool
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -459,18 +459,19 @@ func (t *KnowledgeSearchTool) filterByThreshold(
|
|||||||
var threshold float64
|
var threshold float64
|
||||||
|
|
||||||
// Special handling for history matches: use lower threshold
|
// Special handling for history matches: use lower threshold
|
||||||
if r.MatchType == types.MatchTypeHistory {
|
switch r.MatchType {
|
||||||
|
case types.MatchTypeHistory:
|
||||||
// Use the lower of the two thresholds, then reduce by 0.1 (minimum 0.5)
|
// Use the lower of the two thresholds, then reduce by 0.1 (minimum 0.5)
|
||||||
th := vectorThreshold
|
th := vectorThreshold
|
||||||
if keywordThreshold < th {
|
if keywordThreshold < th {
|
||||||
th = keywordThreshold
|
th = keywordThreshold
|
||||||
}
|
}
|
||||||
threshold = math.Max(th-0.1, 0.5)
|
threshold = math.Max(th-0.1, 0.5)
|
||||||
} else if r.MatchType == types.MatchTypeEmbedding {
|
case types.MatchTypeEmbedding:
|
||||||
threshold = vectorThreshold
|
threshold = vectorThreshold
|
||||||
} else if r.MatchType == types.MatchTypeKeywords {
|
case types.MatchTypeKeywords:
|
||||||
threshold = keywordThreshold
|
threshold = keywordThreshold
|
||||||
} else {
|
default:
|
||||||
// For other match types (graph, nearby chunk, etc.), use the lower threshold
|
// For other match types (graph, nearby chunk, etc.), use the lower threshold
|
||||||
threshold = vectorThreshold
|
threshold = vectorThreshold
|
||||||
if keywordThreshold < threshold {
|
if keywordThreshold < threshold {
|
||||||
@@ -1208,7 +1209,7 @@ func (t *KnowledgeSearchTool) findMissingChunkRanges(retrievedChunks map[int]boo
|
|||||||
}
|
}
|
||||||
|
|
||||||
var ranges []chunkRange
|
var ranges []chunkRange
|
||||||
var currentStart int = -1
|
currentStart := -1
|
||||||
|
|
||||||
// Iterate through all possible chunk indices (0 to totalChunks-1)
|
// Iterate through all possible chunk indices (0 to totalChunks-1)
|
||||||
for i := 0; i < totalChunks; i++ {
|
for i := 0; i < totalChunks; i++ {
|
||||||
|
|||||||
@@ -190,11 +190,11 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
|||||||
builder.WriteString("=== 知识文档分块 ===\n\n")
|
builder.WriteString("=== 知识文档分块 ===\n\n")
|
||||||
|
|
||||||
if knowledgeTitle != "" {
|
if knowledgeTitle != "" {
|
||||||
builder.WriteString(fmt.Sprintf("文档: %s (%s)\n", knowledgeTitle, knowledgeID))
|
fmt.Fprintf(builder, "文档: %s (%s)\n", knowledgeTitle, knowledgeID)
|
||||||
} else {
|
} else {
|
||||||
builder.WriteString(fmt.Sprintf("文档 ID: %s\n", knowledgeID))
|
fmt.Fprintf(builder, "文档 ID: %s\n", knowledgeID)
|
||||||
}
|
}
|
||||||
builder.WriteString(fmt.Sprintf("总分块数: %d\n", total))
|
fmt.Fprintf(builder, "总分块数: %d\n", total)
|
||||||
|
|
||||||
if fetched == 0 {
|
if fetched == 0 {
|
||||||
builder.WriteString("未找到任何分块,请确认文档是否已完成解析。\n")
|
builder.WriteString("未找到任何分块,请确认文档是否已完成解析。\n")
|
||||||
@@ -203,14 +203,14 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
|||||||
}
|
}
|
||||||
return builder.String()
|
return builder.String()
|
||||||
}
|
}
|
||||||
builder.WriteString(fmt.Sprintf("本次拉取: %d 条, 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex))
|
fmt.Fprintf(builder, "本次拉取: %d 条, 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex)
|
||||||
|
|
||||||
builder.WriteString("=== 分块内容预览 ===\n\n")
|
builder.WriteString("=== 分块内容预览 ===\n\n")
|
||||||
for idx, c := range chunks {
|
for idx, c := range chunks {
|
||||||
builder.WriteString(fmt.Sprintf("Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1))
|
fmt.Fprintf(builder, "Chunk #%d (Index %d)\n", idx+1, c.ChunkIndex+1)
|
||||||
builder.WriteString(fmt.Sprintf(" chunk_id: %s\n", c.ID))
|
fmt.Fprintf(builder, " chunk_id: %s\n", c.ID)
|
||||||
builder.WriteString(fmt.Sprintf(" 类型: %s\n", c.ChunkType))
|
fmt.Fprintf(builder, " 类型: %s\n", c.ChunkType)
|
||||||
builder.WriteString(fmt.Sprintf(" 内容: %s\n\n", summarizeContent(c.Content)))
|
fmt.Fprintf(builder, " 内容: %s\n\n", summarizeContent(c.Content))
|
||||||
}
|
}
|
||||||
|
|
||||||
if int64(fetched) < total {
|
if int64(fetched) < total {
|
||||||
|
|||||||
@@ -502,7 +502,7 @@ func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Build
|
|||||||
isOrdered := nodeName == "ol"
|
isOrdered := nodeName == "ol"
|
||||||
node.Find("li").Each(func(idx int, li *goquery.Selection) {
|
node.Find("li").Each(func(idx int, li *goquery.Selection) {
|
||||||
if isOrdered {
|
if isOrdered {
|
||||||
markdown.WriteString(fmt.Sprintf("%d. ", idx+1))
|
fmt.Fprintf(markdown, "%d. ", idx+1)
|
||||||
} else {
|
} else {
|
||||||
markdown.WriteString("- ")
|
markdown.WriteString("- ")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -138,7 +138,8 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
|
|||||||
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||||
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed")
|
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed")
|
||||||
|
|
||||||
if params.Type == "file" {
|
switch params.Type {
|
||||||
|
case "file":
|
||||||
// If file hash exists, prioritize exact match using hash
|
// If file hash exists, prioritize exact match using hash
|
||||||
if params.FileHash != "" {
|
if params.FileHash != "" {
|
||||||
var knowledge types.Knowledge
|
var knowledge types.Knowledge
|
||||||
@@ -167,7 +168,7 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
|
|||||||
}
|
}
|
||||||
return true, &knowledge, nil
|
return true, &knowledge, nil
|
||||||
}
|
}
|
||||||
} else if params.Type == "url" {
|
case "url":
|
||||||
// If file hash exists, prioritize exact match using hash
|
// If file hash exists, prioritize exact match using hash
|
||||||
if params.FileHash != "" {
|
if params.FileHash != "" {
|
||||||
var knowledge types.Knowledge
|
var knowledge types.Knowledge
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ func (r *mcpServiceRepository) GetByID(ctx context.Context, tenantID uint64, id
|
|||||||
err := r.db.WithContext(ctx).
|
err := r.db.WithContext(ctx).
|
||||||
Where("id = ? AND tenant_id = ?", id, tenantID).
|
Where("id = ? AND tenant_id = ?", id, tenantID).
|
||||||
First(&service).Error
|
First(&service).Error
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -48,7 +47,6 @@ func (r *mcpServiceRepository) List(ctx context.Context, tenantID uint64) ([]*ty
|
|||||||
Where("tenant_id = ?", tenantID).
|
Where("tenant_id = ?", tenantID).
|
||||||
Order("created_at DESC").
|
Order("created_at DESC").
|
||||||
Find(&services).Error
|
Find(&services).Error
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -63,7 +61,6 @@ func (r *mcpServiceRepository) ListEnabled(ctx context.Context, tenantID uint64)
|
|||||||
Where("tenant_id = ? AND enabled = ?", tenantID, true).
|
Where("tenant_id = ? AND enabled = ?", tenantID, true).
|
||||||
Order("created_at DESC").
|
Order("created_at DESC").
|
||||||
Find(&services).Error
|
Find(&services).Error
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -81,7 +78,6 @@ func (r *mcpServiceRepository) ListByIDs(ctx context.Context, tenantID uint64, i
|
|||||||
err := r.db.WithContext(ctx).
|
err := r.db.WithContext(ctx).
|
||||||
Where("tenant_id = ? AND id IN ?", tenantID, ids).
|
Where("tenant_id = ? AND id IN ?", tenantID, ids).
|
||||||
Find(&services).Error
|
Find(&services).Error
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -221,11 +221,3 @@ func listI2listS(list []any) []string {
|
|||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapI2mapS(prop map[string]any) map[string]string {
|
|
||||||
attributes := make(map[string]string)
|
|
||||||
for k, v := range prop {
|
|
||||||
attributes[k] = fmt.Sprintf("%v", v)
|
|
||||||
}
|
|
||||||
return attributes
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -304,12 +304,12 @@ func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types
|
|||||||
|
|
||||||
func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) {
|
func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) {
|
||||||
if text == "" {
|
if text == "" {
|
||||||
return nil, errors.New("Empty or invalid input string.")
|
return nil, errors.New("empty or invalid input string")
|
||||||
}
|
}
|
||||||
content := f.extractContent(ctx, text)
|
content := f.extractContent(ctx, text)
|
||||||
// logger.Debugf(ctx, "Extracted content: %s", content)
|
// logger.Debugf(ctx, "Extracted content: %s", content)
|
||||||
if content == "" {
|
if content == "" {
|
||||||
return nil, errors.New("Empty or invalid input string.")
|
return nil, errors.New("empty or invalid input string")
|
||||||
}
|
}
|
||||||
|
|
||||||
var parsed interface{}
|
var parsed interface{}
|
||||||
@@ -318,10 +318,10 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
|
|||||||
err = json.Unmarshal([]byte(content), &parsed)
|
err = json.Unmarshal([]byte(content), &parsed)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
|
return nil, fmt.Errorf("failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
|
||||||
}
|
}
|
||||||
if parsed == nil {
|
if parsed == nil {
|
||||||
return nil, fmt.Errorf("Content must be a list of extractions or a dict.")
|
return nil, fmt.Errorf("content must be a list of extractions or a dict")
|
||||||
}
|
}
|
||||||
|
|
||||||
var items []interface{}
|
var items []interface{}
|
||||||
@@ -330,7 +330,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
|
|||||||
} else if parsedList, ok := parsed.([]interface{}); ok {
|
} else if parsedList, ok := parsed.([]interface{}); ok {
|
||||||
items = parsedList
|
items = parsedList
|
||||||
} else {
|
} else {
|
||||||
return nil, fmt.Errorf("Expected list or dict, got %T", parsed)
|
return nil, fmt.Errorf("expected list or dict, got %T", parsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
itemsList := make([]map[string]interface{}, 0)
|
itemsList := make([]map[string]interface{}, 0)
|
||||||
@@ -338,7 +338,7 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
|
|||||||
if itemMap, ok := item.(map[string]interface{}); ok {
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
||||||
itemsList = append(itemsList, itemMap)
|
itemsList = append(itemsList, itemMap)
|
||||||
} else {
|
} else {
|
||||||
return nil, fmt.Errorf("Each item in the sequence must be a mapping.")
|
return nil, fmt.Errorf("each item in the sequence must be a mapping.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return itemsList, nil
|
return itemsList, nil
|
||||||
@@ -350,7 +350,7 @@ func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphDat
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if len(matchData) == 0 {
|
if len(matchData) == 0 {
|
||||||
logger.Debugf(ctx, "Received empty extraction data.")
|
logger.Debugf(ctx, "received empty extraction data.")
|
||||||
return &types.GraphData{}, nil
|
return &types.GraphData{}, nil
|
||||||
}
|
}
|
||||||
// mm, _ := json.Marshal(matchData)
|
// mm, _ := json.Marshal(matchData)
|
||||||
@@ -403,9 +403,7 @@ func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) {
|
|||||||
node.Attributes = make([]string, 0)
|
node.Attributes = make([]string, 0)
|
||||||
}
|
}
|
||||||
if prenode.Attributes != nil {
|
if prenode.Attributes != nil {
|
||||||
for _, attr := range prenode.Attributes {
|
node.Attributes = append(node.Attributes, prenode.Attributes...)
|
||||||
node.Attributes = append(node.Attributes, attr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -438,7 +438,6 @@ func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatM
|
|||||||
"neighbor_len": runeLen(chunk.Content),
|
"neighbor_len": runeLen(chunk.Content),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
package chatpipline
|
package chatpipline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PluginRerank implements reranking functionality for chat pipeline
|
// PluginRerank implements reranking functionality for chat pipeline
|
||||||
@@ -33,7 +33,7 @@ func (p *PluginRerank) ActivationEvents() []types.EventType {
|
|||||||
|
|
||||||
// OnEvent handles reranking events in the chat pipeline
|
// OnEvent handles reranking events in the chat pipeline
|
||||||
func (p *PluginRerank) OnEvent(ctx context.Context,
|
func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||||
) *PluginError {
|
) *PluginError {
|
||||||
pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{
|
||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
@@ -86,52 +86,52 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
|
||||||
"result_cnt": len(rerankResp),
|
"result_cnt": len(rerankResp),
|
||||||
})
|
})
|
||||||
for i := range chatManage.SearchResult {
|
for i := range chatManage.SearchResult {
|
||||||
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
|
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
|
||||||
}
|
}
|
||||||
reranked := make([]*types.SearchResult, 0, len(rerankResp))
|
reranked := make([]*types.SearchResult, 0, len(rerankResp))
|
||||||
for _, rr := range rerankResp {
|
for _, rr := range rerankResp {
|
||||||
sr := chatManage.SearchResult[rr.Index]
|
sr := chatManage.SearchResult[rr.Index]
|
||||||
base := sr.Score
|
base := sr.Score
|
||||||
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
|
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
|
||||||
sr.Score = rr.RelevanceScore
|
sr.Score = rr.RelevanceScore
|
||||||
sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage)
|
sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage)
|
||||||
reranked = append(reranked, sr)
|
reranked = append(reranked, sr)
|
||||||
}
|
}
|
||||||
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
|
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
|
||||||
chatManage.RerankResult = final
|
chatManage.RerankResult = final
|
||||||
|
|
||||||
// Log composite top scores and MMR selection summary
|
// Log composite top scores and MMR selection summary
|
||||||
topN := min(3, len(reranked))
|
topN := min(3, len(reranked))
|
||||||
for i := 0; i < topN; i++ {
|
for i := 0; i < topN; i++ {
|
||||||
pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{
|
||||||
"rank": i + 1,
|
"rank": i + 1,
|
||||||
"chunk_id": reranked[i].ID,
|
"chunk_id": reranked[i].ID,
|
||||||
"base_score": reranked[i].Metadata["base_score"],
|
"base_score": reranked[i].Metadata["base_score"],
|
||||||
"final_score": fmt.Sprintf("%.4f", reranked[i].Score),
|
"final_score": fmt.Sprintf("%.4f", reranked[i].Score),
|
||||||
"intent": chatManage.QueryIntent,
|
"intent": chatManage.QueryIntent,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chatManage.RerankResult) == 0 {
|
if len(chatManage.RerankResult) == 0 {
|
||||||
pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{
|
pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{
|
||||||
"filtered_cnt": 0,
|
"filtered_cnt": 0,
|
||||||
})
|
})
|
||||||
return ErrSearchNothing
|
return ErrSearchNothing
|
||||||
}
|
}
|
||||||
|
|
||||||
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
|
||||||
"filtered_cnt": len(chatManage.RerankResult),
|
"filtered_cnt": len(chatManage.RerankResult),
|
||||||
})
|
})
|
||||||
return next()
|
return next()
|
||||||
}
|
}
|
||||||
|
|
||||||
// rerank performs the actual reranking operation with given query and passages
|
// rerank performs the actual reranking operation with given query and passages
|
||||||
func (p *PluginRerank) rerank(ctx context.Context,
|
func (p *PluginRerank) rerank(ctx context.Context,
|
||||||
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
|
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
|
||||||
) []rerank.RankResult {
|
) []rerank.RankResult {
|
||||||
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
|
||||||
"query_variant": query,
|
"query_variant": query,
|
||||||
@@ -173,149 +173,149 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
|||||||
rankFilter = append(rankFilter, result)
|
rankFilter = append(rankFilter, result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return rankFilter
|
return rankFilter
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureMetadata(m map[string]string) map[string]string {
|
func ensureMetadata(m map[string]string) map[string]string {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return make(map[string]string)
|
return make(map[string]string)
|
||||||
}
|
}
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
|
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
|
||||||
sourceWeight := 1.0
|
sourceWeight := 1.0
|
||||||
switch strings.ToLower(sr.KnowledgeSource) {
|
switch strings.ToLower(sr.KnowledgeSource) {
|
||||||
case "web_search":
|
case "web_search":
|
||||||
sourceWeight = 0.95
|
sourceWeight = 0.95
|
||||||
default:
|
default:
|
||||||
sourceWeight = 1.0
|
sourceWeight = 1.0
|
||||||
}
|
}
|
||||||
intentBoost := 1.0
|
intentBoost := 1.0
|
||||||
if chatManage.QueryIntent != "" {
|
if chatManage.QueryIntent != "" {
|
||||||
switch chatManage.QueryIntent {
|
switch chatManage.QueryIntent {
|
||||||
case "definition":
|
case "definition":
|
||||||
if sr.ChunkType == string(types.ChunkTypeSummary) {
|
if sr.ChunkType == string(types.ChunkTypeSummary) {
|
||||||
intentBoost = 1.05
|
intentBoost = 1.05
|
||||||
}
|
}
|
||||||
case "howto":
|
case "howto":
|
||||||
if sr.EndAt-sr.StartAt > 300 {
|
if sr.EndAt-sr.StartAt > 300 {
|
||||||
intentBoost = 1.03
|
intentBoost = 1.03
|
||||||
}
|
}
|
||||||
case "compare":
|
case "compare":
|
||||||
intentBoost = 1.0
|
intentBoost = 1.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
positionPrior := 1.0
|
positionPrior := 1.0
|
||||||
if sr.StartAt >= 0 {
|
if sr.StartAt >= 0 {
|
||||||
positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05)
|
positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05)
|
||||||
}
|
}
|
||||||
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
|
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
|
||||||
composite *= intentBoost
|
composite *= intentBoost
|
||||||
composite *= positionPrior
|
composite *= positionPrior
|
||||||
if composite < 0 {
|
if composite < 0 {
|
||||||
composite = 0
|
composite = 0
|
||||||
}
|
}
|
||||||
if composite > 1 {
|
if composite > 1 {
|
||||||
composite = 1
|
composite = 1
|
||||||
}
|
}
|
||||||
return composite
|
return composite
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
|
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
|
||||||
if k <= 0 || len(results) == 0 {
|
if k <= 0 || len(results) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{
|
||||||
"lambda": lambda,
|
"lambda": lambda,
|
||||||
"k": k,
|
"k": k,
|
||||||
"candidates": len(results),
|
"candidates": len(results),
|
||||||
})
|
})
|
||||||
selected := make([]*types.SearchResult, 0, k)
|
selected := make([]*types.SearchResult, 0, k)
|
||||||
candidates := make([]*types.SearchResult, len(results))
|
candidates := make([]*types.SearchResult, len(results))
|
||||||
copy(candidates, results)
|
copy(candidates, results)
|
||||||
tokenSets := make([]map[string]struct{}, len(candidates))
|
tokenSets := make([]map[string]struct{}, len(candidates))
|
||||||
for i, r := range candidates {
|
for i, r := range candidates {
|
||||||
tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r))
|
tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r))
|
||||||
}
|
}
|
||||||
for len(selected) < k && len(candidates) > 0 {
|
for len(selected) < k && len(candidates) > 0 {
|
||||||
bestIdx := 0
|
bestIdx := 0
|
||||||
bestScore := -1.0
|
bestScore := -1.0
|
||||||
for i, r := range candidates {
|
for i, r := range candidates {
|
||||||
relevance := r.Score
|
relevance := r.Score
|
||||||
redundancy := 0.0
|
redundancy := 0.0
|
||||||
for _, s := range selected {
|
for _, s := range selected {
|
||||||
redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s))))
|
redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s))))
|
||||||
}
|
}
|
||||||
mmr := lambda*relevance - (1.0-lambda)*redundancy
|
mmr := lambda*relevance - (1.0-lambda)*redundancy
|
||||||
if mmr > bestScore {
|
if mmr > bestScore {
|
||||||
bestScore = mmr
|
bestScore = mmr
|
||||||
bestIdx = i
|
bestIdx = i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
selected = append(selected, candidates[bestIdx])
|
selected = append(selected, candidates[bestIdx])
|
||||||
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
|
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
|
||||||
}
|
}
|
||||||
// Compute average redundancy among selected
|
// Compute average redundancy among selected
|
||||||
avgRed := 0.0
|
avgRed := 0.0
|
||||||
if len(selected) > 1 {
|
if len(selected) > 1 {
|
||||||
pairs := 0
|
pairs := 0
|
||||||
for i := 0; i < len(selected); i++ {
|
for i := 0; i < len(selected); i++ {
|
||||||
for j := i + 1; j < len(selected); j++ {
|
for j := i + 1; j < len(selected); j++ {
|
||||||
si := tokenizeSimple(getEnrichedPassage(ctx, selected[i]))
|
si := tokenizeSimple(getEnrichedPassage(ctx, selected[i]))
|
||||||
sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j]))
|
sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j]))
|
||||||
avgRed += jaccard(si, sj)
|
avgRed += jaccard(si, sj)
|
||||||
pairs++
|
pairs++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if pairs > 0 {
|
if pairs > 0 {
|
||||||
avgRed /= float64(pairs)
|
avgRed /= float64(pairs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{
|
pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{
|
||||||
"selected": len(selected),
|
"selected": len(selected),
|
||||||
"avg_redundancy": fmt.Sprintf("%.4f", avgRed),
|
"avg_redundancy": fmt.Sprintf("%.4f", avgRed),
|
||||||
})
|
})
|
||||||
return selected
|
return selected
|
||||||
}
|
}
|
||||||
|
|
||||||
func tokenizeSimple(text string) map[string]struct{} {
|
func tokenizeSimple(text string) map[string]struct{} {
|
||||||
text = strings.ToLower(text)
|
text = strings.ToLower(text)
|
||||||
fields := strings.Fields(text)
|
fields := strings.Fields(text)
|
||||||
set := make(map[string]struct{}, len(fields))
|
set := make(map[string]struct{}, len(fields))
|
||||||
for _, f := range fields {
|
for _, f := range fields {
|
||||||
if len(f) > 1 {
|
if len(f) > 1 {
|
||||||
set[f] = struct{}{}
|
set[f] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return set
|
return set
|
||||||
}
|
}
|
||||||
|
|
||||||
func jaccard(a, b map[string]struct{}) float64 {
|
func jaccard(a, b map[string]struct{}) float64 {
|
||||||
if len(a) == 0 && len(b) == 0 {
|
if len(a) == 0 && len(b) == 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
inter := 0
|
inter := 0
|
||||||
for k := range a {
|
for k := range a {
|
||||||
if _, ok := b[k]; ok {
|
if _, ok := b[k]; ok {
|
||||||
inter++
|
inter++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
union := len(a) + len(b) - inter
|
union := len(a) + len(b) - inter
|
||||||
if union == 0 {
|
if union == 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
return float64(inter) / float64(union)
|
return float64(inter) / float64(union)
|
||||||
}
|
}
|
||||||
|
|
||||||
func clampFloat(v, minV, maxV float64) float64 {
|
func clampFloat(v, minV, maxV float64) float64 {
|
||||||
if v < minV {
|
if v < minV {
|
||||||
return minV
|
return minV
|
||||||
}
|
}
|
||||||
if v > maxV {
|
if v > maxV {
|
||||||
return maxV
|
return maxV
|
||||||
}
|
}
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
// getEnrichedPassage 合并Content和ImageInfo的文本内容
|
// getEnrichedPassage 合并Content和ImageInfo的文本内容
|
||||||
|
|||||||
@@ -387,7 +387,7 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
|
|||||||
|
|
||||||
// searchWebIfEnabled executes web search when enabled and returns converted results
|
// searchWebIfEnabled executes web search when enabled and returns converted results
|
||||||
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
||||||
if !(chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0) {
|
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil || chatManage.TenantID <= 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ func (p *PluginTracing) OnEvent(ctx context.Context,
|
|||||||
|
|
||||||
// Search traces search operations in the chat pipeline
|
// Search traces search operations in the chat pipeline
|
||||||
func (p *PluginTracing) Search(ctx context.Context,
|
func (p *PluginTracing) Search(ctx context.Context,
|
||||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||||
) *PluginError {
|
) *PluginError {
|
||||||
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search")
|
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search")
|
||||||
defer span.End()
|
defer span.End()
|
||||||
@@ -87,23 +87,23 @@ func (p *PluginTracing) Search(ctx context.Context,
|
|||||||
attribute.Float64("keyword_threshold", chatManage.KeywordThreshold),
|
attribute.Float64("keyword_threshold", chatManage.KeywordThreshold),
|
||||||
attribute.Int("match_count", chatManage.EmbeddingTopK),
|
attribute.Int("match_count", chatManage.EmbeddingTopK),
|
||||||
)
|
)
|
||||||
err := next()
|
err := next()
|
||||||
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
|
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
|
||||||
unique := make(map[string]struct{})
|
unique := make(map[string]struct{})
|
||||||
for _, r := range chatManage.SearchResult {
|
for _, r := range chatManage.SearchResult {
|
||||||
unique[r.ID] = struct{}{}
|
unique[r.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
span.SetAttributes(
|
span.SetAttributes(
|
||||||
attribute.String("hybrid_search", string(searchResultJson)),
|
attribute.String("hybrid_search", string(searchResultJson)),
|
||||||
attribute.String("processed_query", chatManage.ProcessedQuery),
|
attribute.String("processed_query", chatManage.ProcessedQuery),
|
||||||
attribute.Int("search_unique_count", len(unique)),
|
attribute.Int("search_unique_count", len(unique)),
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rerank traces rerank operations in the chat pipeline
|
// Rerank traces rerank operations in the chat pipeline
|
||||||
func (p *PluginTracing) Rerank(ctx context.Context,
|
func (p *PluginTracing) Rerank(ctx context.Context,
|
||||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||||
) *PluginError {
|
) *PluginError {
|
||||||
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank")
|
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank")
|
||||||
defer span.End()
|
defer span.End()
|
||||||
@@ -114,14 +114,14 @@ func (p *PluginTracing) Rerank(ctx context.Context,
|
|||||||
attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold),
|
attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold),
|
||||||
attribute.Int("rerank_filter_topk", chatManage.RerankTopK),
|
attribute.Int("rerank_filter_topk", chatManage.RerankTopK),
|
||||||
)
|
)
|
||||||
err := next()
|
err := next()
|
||||||
resultJson, _ := json.Marshal(chatManage.RerankResult)
|
resultJson, _ := json.Marshal(chatManage.RerankResult)
|
||||||
span.SetAttributes(
|
span.SetAttributes(
|
||||||
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
|
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
|
||||||
attribute.String("rerank_resp_results", string(resultJson)),
|
attribute.String("rerank_resp_results", string(resultJson)),
|
||||||
attribute.String("query_intent", chatManage.QueryIntent),
|
attribute.String("query_intent", chatManage.QueryIntent),
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Merge traces merge operations in the chat pipeline
|
// Merge traces merge operations in the chat pipeline
|
||||||
|
|||||||
@@ -58,7 +58,6 @@ func (s *chunkService) GetRepository() interfaces.ChunkRepository {
|
|||||||
// Returns:
|
// Returns:
|
||||||
// - error: Any error encountered during chunk creation
|
// - error: Any error encountered during chunk creation
|
||||||
func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
|
func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
|
||||||
|
|
||||||
err := s.chunkRepository.CreateChunks(ctx, chunks)
|
err := s.chunkRepository.CreateChunks(ctx, chunks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ type minioFileService struct {
|
|||||||
|
|
||||||
// NewMinioFileService creates a MinIO file service
|
// NewMinioFileService creates a MinIO file service
|
||||||
func NewMinioFileService(endpoint,
|
func NewMinioFileService(endpoint,
|
||||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool) (interfaces.FileService, error) {
|
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||||
|
) (interfaces.FileService, error) {
|
||||||
// Initialize MinIO client
|
// Initialize MinIO client
|
||||||
client, err := minio.New(endpoint, &minio.Options{
|
client, err := minio.New(endpoint, &minio.Options{
|
||||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||||
|
|||||||
@@ -183,7 +183,8 @@ func (b *graphBuilder) extractEntities(ctx context.Context, chunk *types.Chunk)
|
|||||||
// extractRelationships extracts relationships between entities
|
// extractRelationships extracts relationships between entities
|
||||||
// It analyzes semantic connections between multiple entities and establishes relationships
|
// It analyzes semantic connections between multiple entities and establishes relationships
|
||||||
func (b *graphBuilder) extractRelationships(ctx context.Context,
|
func (b *graphBuilder) extractRelationships(ctx context.Context,
|
||||||
chunks []*types.Chunk, entities []*types.Entity) error {
|
chunks []*types.Chunk, entities []*types.Entity,
|
||||||
|
) error {
|
||||||
log := logger.GetLogger(ctx)
|
log := logger.GetLogger(ctx)
|
||||||
log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks))
|
log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks))
|
||||||
|
|
||||||
@@ -340,7 +341,7 @@ func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
var chunkContents = chunks[0].Content
|
chunkContents := chunks[0].Content
|
||||||
preChunk := chunks[0]
|
preChunk := chunks[0]
|
||||||
|
|
||||||
for i := 1; i < len(chunks); i++ {
|
for i := 1; i < len(chunks); i++ {
|
||||||
@@ -369,7 +370,7 @@ func (b *graphBuilder) BuildGraph(ctx context.Context, chunks []*types.Chunk) er
|
|||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
// Concurrently extract entities from each document chunk
|
// Concurrently extract entities from each document chunk
|
||||||
var chunkEntities = make([][]*types.Entity, len(chunks))
|
chunkEntities := make([][]*types.Entity, len(chunks))
|
||||||
g, gctx := errgroup.WithContext(ctx)
|
g, gctx := errgroup.WithContext(ctx)
|
||||||
g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency
|
g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency
|
||||||
|
|
||||||
@@ -868,7 +869,8 @@ func (b *graphBuilder) getEntityByTitle(title string) *types.Entity {
|
|||||||
// dfs depth-first search to find connected components
|
// dfs depth-first search to find connected components
|
||||||
func dfs(entityTitle string,
|
func dfs(entityTitle string,
|
||||||
adjacencyList map[string]map[string]*types.Relationship,
|
adjacencyList map[string]map[string]*types.Relationship,
|
||||||
visited map[string]bool, component *[]string) {
|
visited map[string]bool, component *[]string,
|
||||||
|
) {
|
||||||
visited[entityTitle] = true
|
visited[entityTitle] = true
|
||||||
*component = append(*component, entityTitle)
|
*component = append(*component, entityTitle)
|
||||||
|
|
||||||
|
|||||||
@@ -864,207 +864,6 @@ func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowle
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// processDocument handles asynchronous processing of document files
|
|
||||||
func (s *knowledgeService) processDocument(ctx context.Context,
|
|
||||||
kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool,
|
|
||||||
) {
|
|
||||||
logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel)
|
|
||||||
|
|
||||||
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument")
|
|
||||||
defer span.End()
|
|
||||||
span.SetAttributes(
|
|
||||||
attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)),
|
|
||||||
attribute.String("knowledge_base_id", kb.ID),
|
|
||||||
attribute.Int("tenant_id", int(kb.TenantID)),
|
|
||||||
attribute.String("knowledge_id", knowledge.ID),
|
|
||||||
attribute.String("file_name", knowledge.FileName),
|
|
||||||
attribute.String("file_type", knowledge.FileType),
|
|
||||||
attribute.String("file_path", knowledge.FilePath),
|
|
||||||
attribute.Int64("file_size", knowledge.FileSize),
|
|
||||||
attribute.String("embedding_model", knowledge.EmbeddingModelID),
|
|
||||||
attribute.Bool("enable_multimodal", enableMultimodel),
|
|
||||||
)
|
|
||||||
logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String())
|
|
||||||
|
|
||||||
if !enableMultimodel && IsImageType(knowledge.FileType) {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = ErrImageNotParse.Error()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(ErrImageNotParse)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update status to processing
|
|
||||||
knowledge.ParseStatus = "processing"
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
|
||||||
span.RecordError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read and chunk the document
|
|
||||||
f, err := file.Open()
|
|
||||||
if err != nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
WithField("error", err).Errorf("processDocument open file failed")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = err.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
span.AddEvent("start read file")
|
|
||||||
contentBytes, err := io.ReadAll(f)
|
|
||||||
if err != nil {
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = err.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Split file into chunks using document reader service
|
|
||||||
logger.GetLogger(ctx).Infof("processDocument split file content size: %d, file name: %s, file type: %s, separators: %v, enable multimodal: %v",
|
|
||||||
len(contentBytes), knowledge.FileName, knowledge.FileType, kb.ChunkingConfig.Separators, enableMultimodel)
|
|
||||||
var vlmConfig *proto.VLMConfig
|
|
||||||
if enableMultimodel {
|
|
||||||
vlmConfig, err = s.getVLMProtoConfig(ctx, kb)
|
|
||||||
if err != nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
WithField("error", err).Errorf("processDocument build VLM config failed")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = err.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if vlmConfig == nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
Error("processDocument enable multimodal but VLM config missing")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = "VLM 配置缺失"
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(errors.New("vlm config missing"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{
|
|
||||||
FileContent: contentBytes,
|
|
||||||
FileName: knowledge.FileName,
|
|
||||||
FileType: knowledge.FileType,
|
|
||||||
ReadConfig: &proto.ReadConfig{
|
|
||||||
ChunkSize: int32(kb.ChunkingConfig.ChunkSize),
|
|
||||||
ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap),
|
|
||||||
Separators: kb.ChunkingConfig.Separators,
|
|
||||||
EnableMultimodal: enableMultimodel,
|
|
||||||
StorageConfig: &proto.StorageConfig{
|
|
||||||
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
|
|
||||||
Region: kb.StorageConfig.Region,
|
|
||||||
BucketName: kb.StorageConfig.BucketName,
|
|
||||||
AccessKeyId: kb.StorageConfig.SecretID,
|
|
||||||
SecretAccessKey: kb.StorageConfig.SecretKey,
|
|
||||||
AppId: kb.StorageConfig.AppID,
|
|
||||||
PathPrefix: kb.StorageConfig.PathPrefix,
|
|
||||||
},
|
|
||||||
VlmConfig: vlmConfig,
|
|
||||||
},
|
|
||||||
RequestId: ctx.Value(types.RequestIDContextKey).(string),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
WithField("error", err).Errorf("processDocument read file failed")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = err.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
span.RecordError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process and store chunks
|
|
||||||
span.AddEvent("start process chunks")
|
|
||||||
s.processChunks(ctx, kb, knowledge, resp.Chunks)
|
|
||||||
}
|
|
||||||
|
|
||||||
// processDocumentFromURL handles asynchronous processing of URL content
|
|
||||||
func (s *knowledgeService) processDocumentFromURL(ctx context.Context,
|
|
||||||
kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool,
|
|
||||||
) {
|
|
||||||
// Update status to processing
|
|
||||||
knowledge.ParseStatus = "processing"
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel)
|
|
||||||
|
|
||||||
// Fetch and chunk content from URL
|
|
||||||
var vlmConfig *proto.VLMConfig
|
|
||||||
if enableMultimodel {
|
|
||||||
cfg, cfgErr := s.getVLMProtoConfig(ctx, kb)
|
|
||||||
if cfgErr != nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
WithField("error", cfgErr).Errorf("processDocumentFromURL build VLM config failed")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = cfgErr.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if cfg == nil {
|
|
||||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
|
||||||
Error("processDocumentFromURL enable multimodal but VLM config missing")
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = "VLM 配置缺失"
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
vlmConfig = cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{
|
|
||||||
Url: url,
|
|
||||||
Title: knowledge.Title,
|
|
||||||
ReadConfig: &proto.ReadConfig{
|
|
||||||
ChunkSize: int32(kb.ChunkingConfig.ChunkSize),
|
|
||||||
ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap),
|
|
||||||
Separators: kb.ChunkingConfig.Separators,
|
|
||||||
EnableMultimodal: enableMultimodel,
|
|
||||||
StorageConfig: &proto.StorageConfig{
|
|
||||||
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
|
|
||||||
Region: kb.StorageConfig.Region,
|
|
||||||
BucketName: kb.StorageConfig.BucketName,
|
|
||||||
AccessKeyId: kb.StorageConfig.SecretID,
|
|
||||||
SecretAccessKey: kb.StorageConfig.SecretKey,
|
|
||||||
AppId: kb.StorageConfig.AppID,
|
|
||||||
PathPrefix: kb.StorageConfig.PathPrefix,
|
|
||||||
},
|
|
||||||
VlmConfig: vlmConfig,
|
|
||||||
},
|
|
||||||
RequestId: ctx.Value(types.RequestIDContextKey).(string),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
knowledge.ParseStatus = "failed"
|
|
||||||
knowledge.ErrorMessage = err.Error()
|
|
||||||
knowledge.UpdatedAt = time.Now()
|
|
||||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process and store chunks
|
|
||||||
s.processChunks(ctx, kb, knowledge, resp.Chunks)
|
|
||||||
}
|
|
||||||
|
|
||||||
// processDocumentFromPassage handles asynchronous processing of text passages
|
// processDocumentFromPassage handles asynchronous processing of text passages
|
||||||
func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
|
func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
|
||||||
kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string,
|
kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string,
|
||||||
@@ -1767,7 +1566,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID
|
|||||||
batch := 10
|
batch := 10
|
||||||
g, gctx := errgroup.WithContext(ctx)
|
g, gctx := errgroup.WithContext(ctx)
|
||||||
for ids := range slices.Chunk(delKnowledge, batch) {
|
for ids := range slices.Chunk(delKnowledge, batch) {
|
||||||
ids = ids
|
|
||||||
g.Go(func() error {
|
g.Go(func() error {
|
||||||
err := s.DeleteKnowledgeList(gctx, ids)
|
err := s.DeleteKnowledgeList(gctx, ids)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1787,7 +1585,6 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID
|
|||||||
g, gctx = errgroup.WithContext(ctx)
|
g, gctx = errgroup.WithContext(ctx)
|
||||||
g.SetLimit(batch)
|
g.SetLimit(batch)
|
||||||
for _, knowledge := range addKnowledge {
|
for _, knowledge := range addKnowledge {
|
||||||
knowledge = knowledge
|
|
||||||
g.Go(func() error {
|
g.Go(func() error {
|
||||||
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
|
srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1916,7 +1713,8 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Mark that we've found chunks for this image
|
// Mark that we've found chunks for this image
|
||||||
if child.ChunkType == types.ChunkTypeImageCaption {
|
switch child.ChunkType {
|
||||||
|
case types.ChunkTypeImageCaption:
|
||||||
hasCaptionChunk = true
|
hasCaptionChunk = true
|
||||||
// Update caption if it has changed
|
// Update caption if it has changed
|
||||||
if image.Caption != cImageInfo[0].Caption {
|
if image.Caption != cImageInfo[0].Caption {
|
||||||
@@ -1924,7 +1722,7 @@ func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID stri
|
|||||||
child.ImageInfo = imageInfo
|
child.ImageInfo = imageInfo
|
||||||
updateChunk = append(updateChunk, chunkChildren[i])
|
updateChunk = append(updateChunk, chunkChildren[i])
|
||||||
}
|
}
|
||||||
} else if child.ChunkType == types.ChunkTypeImageOCR {
|
case types.ChunkTypeImageOCR:
|
||||||
hasOCRChunk = true
|
hasOCRChunk = true
|
||||||
// Update OCR if it has changed
|
// Update OCR if it has changed
|
||||||
if image.OCRText != cImageInfo[0].OCRText {
|
if image.OCRText != cImageInfo[0].OCRText {
|
||||||
@@ -2240,7 +2038,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
|||||||
|
|
||||||
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
|
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
|
||||||
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
|
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
|
||||||
tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload) ([]types.FAQEntryPayload, int, error) {
|
tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload,
|
||||||
|
) ([]types.FAQEntryPayload, int, error) {
|
||||||
if len(entries) == 0 {
|
if len(entries) == 0 {
|
||||||
return []types.FAQEntryPayload{}, 0, nil
|
return []types.FAQEntryPayload{}, 0, nil
|
||||||
}
|
}
|
||||||
@@ -2249,7 +2048,8 @@ func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
|
|||||||
|
|
||||||
// calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目
|
// calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目
|
||||||
func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
|
func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
|
||||||
tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload) ([]types.FAQEntryPayload, []*types.Chunk, int, error) {
|
tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload,
|
||||||
|
) ([]types.FAQEntryPayload, []*types.Chunk, int, error) {
|
||||||
// 计算所有新条目的 content hash,并同时构建 hash 到 entry 的映射
|
// 计算所有新条目的 content hash,并同时构建 hash 到 entry 的映射
|
||||||
type entryWithHash struct {
|
type entryWithHash struct {
|
||||||
entry types.FAQEntryPayload
|
entry types.FAQEntryPayload
|
||||||
@@ -2316,7 +2116,8 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
|
|||||||
|
|
||||||
// executeFAQImport 执行实际的FAQ导入逻辑
|
// executeFAQImport 执行实际的FAQ导入逻辑
|
||||||
func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string,
|
func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string,
|
||||||
payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int) (err error) {
|
payload *types.FAQBatchUpsertPayload, tenantID uint64, processedCount int,
|
||||||
|
) (err error) {
|
||||||
// 保存知识库和embedding模型信息,用于清理索引
|
// 保存知识库和embedding模型信息,用于清理索引
|
||||||
var kb *types.KnowledgeBase
|
var kb *types.KnowledgeBase
|
||||||
var embeddingModel embedding.Embedder
|
var embeddingModel embedding.Embedder
|
||||||
@@ -2364,8 +2165,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
|||||||
|
|
||||||
if payload.Mode == types.FAQBatchModeReplace {
|
if payload.Mode == types.FAQBatchModeReplace {
|
||||||
// Replace模式:计算需要删除、创建、更新的条目
|
// Replace模式:计算需要删除、创建、更新的条目
|
||||||
entriesToProcess, chunksToDelete, skippedCount, err =
|
entriesToProcess, chunksToDelete, skippedCount, err = s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
|
||||||
s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to calculate replace operations: %w", err)
|
return fmt.Errorf("failed to calculate replace operations: %w", err)
|
||||||
}
|
}
|
||||||
@@ -3080,13 +2880,15 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
||||||
progress, total, processed int, errorMsg string) error {
|
progress, total, processed int, errorMsg string,
|
||||||
|
) error {
|
||||||
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
|
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
|
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
|
||||||
func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
||||||
progress, total, processed int, errorMsg string) error {
|
progress, total, processed int, errorMsg string,
|
||||||
|
) error {
|
||||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
|
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -131,14 +131,15 @@ func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types
|
|||||||
kb.EnsureDefaults()
|
kb.EnsureDefaults()
|
||||||
|
|
||||||
// Get knowledge count
|
// Get knowledge count
|
||||||
if kb.Type == types.KnowledgeBaseTypeDocument {
|
switch kb.Type {
|
||||||
|
case types.KnowledgeBaseTypeDocument:
|
||||||
knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID)
|
knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err)
|
logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err)
|
||||||
} else {
|
} else {
|
||||||
kb.KnowledgeCount = knowledgeCount
|
kb.KnowledgeCount = knowledgeCount
|
||||||
}
|
}
|
||||||
} else if kb.Type == types.KnowledgeBaseTypeFAQ {
|
case types.KnowledgeBaseTypeFAQ:
|
||||||
// Get chunk count
|
// Get chunk count
|
||||||
chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID)
|
chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -192,7 +192,6 @@ func (s *smartCompressionStrategy) summarizeMessages(ctx context.Context, messag
|
|||||||
Temperature: 0.3, // Lower temperature for more consistent summaries
|
Temperature: 0.3, // Lower temperature for more consistent summaries
|
||||||
MaxTokens: 500, // Limit summary length
|
MaxTokens: 500, // Limit summary length
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("failed to generate summary: %w", err)
|
return "", fmt.Errorf("failed to generate summary: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,11 +96,6 @@ func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams {
|
|||||||
return getNgrams(n, words, exclusive)
|
return getNgrams(n, words, exclusive)
|
||||||
}
|
}
|
||||||
|
|
||||||
func lenLcs(x, y []string) int {
|
|
||||||
table := lcs(x, y)
|
|
||||||
return table[len(x)][len(y)]
|
|
||||||
}
|
|
||||||
|
|
||||||
func lcs(x, y []string) [][]int {
|
func lcs(x, y []string) [][]int {
|
||||||
n, m := len(x), len(y)
|
n, m := len(x), len(y)
|
||||||
table := make([][]int, n+1)
|
table := make([][]int, n+1)
|
||||||
|
|||||||
@@ -83,7 +83,6 @@ func (m *MetricList) Avg() *types.MetricResult {
|
|||||||
|
|
||||||
// HookMetric tracks evaluation metrics for QA pairs
|
// HookMetric tracks evaluation metrics for QA pairs
|
||||||
type HookMetric struct {
|
type HookMetric struct {
|
||||||
task *types.EvaluationTask
|
|
||||||
qaPairMetricList []*qaPairMetric // Per-QA pair metrics
|
qaPairMetricList []*qaPairMetric // Per-QA pair metrics
|
||||||
metricResults *MetricList // Aggregated results
|
metricResults *MetricList // Aggregated results
|
||||||
mu *sync.RWMutex // Thread safety
|
mu *sync.RWMutex // Thread safety
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func (r *RetrieveEngineRegistry) Register(repo interfaces.RetrieveEngineService)
|
|||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
if _, exists := r.repositories[repo.EngineType()]; exists {
|
if _, exists := r.repositories[repo.EngineType()]; exists {
|
||||||
return fmt.Errorf("Repository type %s already registered", repo.EngineType())
|
return fmt.Errorf("repository type %s already registered", repo.EngineType())
|
||||||
}
|
}
|
||||||
|
|
||||||
r.repositories[repo.EngineType()] = repo
|
r.repositories[repo.EngineType()] = repo
|
||||||
@@ -43,7 +43,7 @@ func (r *RetrieveEngineRegistry) GetRetrieveEngineService(repoType types.Retriev
|
|||||||
|
|
||||||
repo, exists := r.repositories[repoType]
|
repo, exists := r.repositories[repoType]
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, fmt.Errorf("Repository of type %s not found", repoType)
|
return nil, fmt.Errorf("repository of type %s not found", repoType)
|
||||||
}
|
}
|
||||||
|
|
||||||
return repo, nil
|
return repo, nil
|
||||||
|
|||||||
@@ -587,7 +587,6 @@ func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, sess
|
|||||||
// 3. Session's SummaryModelID (if not Remote)
|
// 3. Session's SummaryModelID (if not Remote)
|
||||||
// 4. First knowledge base's SummaryModelID
|
// 4. First knowledge base's SummaryModelID
|
||||||
func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) {
|
func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) {
|
||||||
|
|
||||||
// First, check if session has a SummaryModelID and if it's a Remote model
|
// First, check if session has a SummaryModelID and if it's a Remote model
|
||||||
if session.SummaryModelID != "" {
|
if session.SummaryModelID != "" {
|
||||||
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
|
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||||||
|
|
||||||
// Session service (depends on agent service)
|
// Session service (depends on agent service)
|
||||||
// SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed
|
// SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed
|
||||||
must(container.Provide(service.NewSessionService))
|
must(container.Provide(service.NewSessionService))
|
||||||
|
|
||||||
must(container.Provide(router.NewAsyncqClient))
|
must(container.Provide(router.NewAsyncqClient))
|
||||||
must(container.Provide(router.NewAsynqServer))
|
must(container.Provide(router.NewAsynqServer))
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ import (
|
|||||||
"github.com/Tencent/WeKnora/internal/types"
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||||
"github.com/Tencent/WeKnora/internal/utils"
|
"github.com/Tencent/WeKnora/internal/utils"
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/ollama/ollama/api"
|
"github.com/ollama/ollama/api"
|
||||||
@@ -198,7 +197,7 @@ type InitializationRequest struct {
|
|||||||
// UpdateKBConfig 根据知识库ID和模型ID更新配置(简化版)
|
// UpdateKBConfig 根据知识库ID和模型ID更新配置(简化版)
|
||||||
func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
kbIdStr := secutils.SanitizeForLog(c.Param("kbId"))
|
kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
|
||||||
|
|
||||||
var req KBModelConfigRequest
|
var req KBModelConfigRequest
|
||||||
if err := c.ShouldBindJSON(&req); err != nil {
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
@@ -210,7 +209,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
|||||||
// 获取知识库信息
|
// 获取知识库信息
|
||||||
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
||||||
if err != nil || kb == nil {
|
if err != nil || kb == nil {
|
||||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
|
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||||
c.Error(errors.NewNotFoundError("知识库不存在"))
|
c.Error(errors.NewNotFoundError("知识库不存在"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -350,7 +349,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
|||||||
// InitializeByKB 根据知识库ID执行配置更新
|
// InitializeByKB 根据知识库ID执行配置更新
|
||||||
func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
kbIdStr := secutils.SanitizeForLog(c.Param("kbId"))
|
kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
|
||||||
|
|
||||||
var req InitializationRequest
|
var req InitializationRequest
|
||||||
if err := c.ShouldBindJSON(&req); err != nil {
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
@@ -358,12 +357,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
|||||||
c.Error(errors.NewBadRequestError(err.Error()))
|
c.Error(errors.NewBadRequestError(err.Error()))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", secutils.SanitizeForLog(kbIdStr), secutils.SanitizeForLog(utils.ToJSON(req)))
|
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", utils.SanitizeForLog(kbIdStr), utils.SanitizeForLog(utils.ToJSON(req)))
|
||||||
|
|
||||||
// 获取指定知识库信息
|
// 获取指定知识库信息
|
||||||
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
|
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||||
c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error()))
|
c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error()))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -450,18 +449,18 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
modelType: types.ModelTypeKnowledgeQA,
|
modelType: types.ModelTypeKnowledgeQA,
|
||||||
name: secutils.SanitizeForLog(req.LLM.ModelName),
|
name: utils.SanitizeForLog(req.LLM.ModelName),
|
||||||
source: types.ModelSource(req.LLM.Source),
|
source: types.ModelSource(req.LLM.Source),
|
||||||
description: "LLM Model for Knowledge QA",
|
description: "LLM Model for Knowledge QA",
|
||||||
baseURL: secutils.SanitizeForLog(req.LLM.BaseURL),
|
baseURL: utils.SanitizeForLog(req.LLM.BaseURL),
|
||||||
apiKey: req.LLM.APIKey,
|
apiKey: req.LLM.APIKey,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
modelType: types.ModelTypeEmbedding,
|
modelType: types.ModelTypeEmbedding,
|
||||||
name: secutils.SanitizeForLog(req.Embedding.ModelName),
|
name: utils.SanitizeForLog(req.Embedding.ModelName),
|
||||||
source: types.ModelSource(req.Embedding.Source),
|
source: types.ModelSource(req.Embedding.Source),
|
||||||
description: "Embedding Model",
|
description: "Embedding Model",
|
||||||
baseURL: secutils.SanitizeForLog(req.Embedding.BaseURL),
|
baseURL: utils.SanitizeForLog(req.Embedding.BaseURL),
|
||||||
apiKey: req.Embedding.APIKey,
|
apiKey: req.Embedding.APIKey,
|
||||||
dimension: req.Embedding.Dimension,
|
dimension: req.Embedding.Dimension,
|
||||||
},
|
},
|
||||||
@@ -480,10 +479,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
|||||||
interfaceType string
|
interfaceType string
|
||||||
}{
|
}{
|
||||||
modelType: types.ModelTypeRerank,
|
modelType: types.ModelTypeRerank,
|
||||||
name: secutils.SanitizeForLog(req.Rerank.ModelName),
|
name: utils.SanitizeForLog(req.Rerank.ModelName),
|
||||||
source: types.ModelSourceRemote,
|
source: types.ModelSourceRemote,
|
||||||
description: "Rerank Model",
|
description: "Rerank Model",
|
||||||
baseURL: secutils.SanitizeForLog(req.Rerank.BaseURL),
|
baseURL: utils.SanitizeForLog(req.Rerank.BaseURL),
|
||||||
apiKey: req.Rerank.APIKey,
|
apiKey: req.Rerank.APIKey,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -501,10 +500,10 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
|||||||
interfaceType string
|
interfaceType string
|
||||||
}{
|
}{
|
||||||
modelType: types.ModelTypeVLLM,
|
modelType: types.ModelTypeVLLM,
|
||||||
name: secutils.SanitizeForLog(req.Multimodal.VLM.ModelName),
|
name: utils.SanitizeForLog(req.Multimodal.VLM.ModelName),
|
||||||
source: types.ModelSourceRemote,
|
source: types.ModelSourceRemote,
|
||||||
description: "VLM Model",
|
description: "VLM Model",
|
||||||
baseURL: secutils.SanitizeForLog(req.Multimodal.VLM.BaseURL),
|
baseURL: utils.SanitizeForLog(req.Multimodal.VLM.BaseURL),
|
||||||
apiKey: req.Multimodal.VLM.APIKey,
|
apiKey: req.Multimodal.VLM.APIKey,
|
||||||
interfaceType: req.Multimodal.VLM.InterfaceType,
|
interfaceType: req.Multimodal.VLM.InterfaceType,
|
||||||
})
|
})
|
||||||
@@ -676,7 +675,7 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
|||||||
|
|
||||||
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
|
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
|
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||||
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
|
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -775,7 +774,7 @@ func (h *InitializationHandler) CheckOllamaModels(c *gin.Context) {
|
|||||||
modelStatus[modelName] = available
|
modelStatus[modelName] = available
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Infof(ctx, "Model %s availability: %v", secutils.SanitizeForLog(modelName), modelStatus[modelName])
|
logger.Infof(ctx, "Model %s availability: %v", utils.SanitizeForLog(modelName), modelStatus[modelName])
|
||||||
}
|
}
|
||||||
|
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
@@ -1010,8 +1009,8 @@ func (h *InitializationHandler) pullModelWithProgress(ctx context.Context,
|
|||||||
|
|
||||||
// 使用Ollama客户端的Pull方法,带进度回调
|
// 使用Ollama客户端的Pull方法,带进度回调
|
||||||
err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error {
|
err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error {
|
||||||
var progressPercent float64 = 0.0
|
progressPercent := 0.0
|
||||||
var message string = "下载中"
|
message := "下载中"
|
||||||
|
|
||||||
if progress.Total > 0 && progress.Completed > 0 {
|
if progress.Total > 0 && progress.Completed > 0 {
|
||||||
progressPercent = float64(progress.Completed) / float64(progress.Total) * 100
|
progressPercent = float64(progress.Completed) / float64(progress.Total) * 100
|
||||||
@@ -1057,7 +1056,7 @@ func (h *InitializationHandler) updateTaskStatus(
|
|||||||
// GetCurrentConfigByKB 根据知识库ID获取配置信息
|
// GetCurrentConfigByKB 根据知识库ID获取配置信息
|
||||||
func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) {
|
func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) {
|
||||||
ctx := c.Request.Context()
|
ctx := c.Request.Context()
|
||||||
kbIdStr := secutils.SanitizeForLog(c.Param("kbId"))
|
kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
|
||||||
|
|
||||||
logger.Info(ctx, "Getting configuration for knowledge base")
|
logger.Info(ctx, "Getting configuration for knowledge base")
|
||||||
|
|
||||||
@@ -1103,10 +1102,7 @@ func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) {
|
|||||||
Page: 1,
|
Page: 1,
|
||||||
PageSize: 1,
|
PageSize: 1,
|
||||||
}, "")
|
}, "")
|
||||||
hasFiles := false
|
hasFiles := err == nil && knowledgeList != nil && knowledgeList.Total > 0
|
||||||
if err == nil && knowledgeList != nil && knowledgeList.Total > 0 {
|
|
||||||
hasFiles = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构建配置响应
|
// 构建配置响应
|
||||||
config := h.buildConfigResponse(ctx, models, kb, hasFiles)
|
config := h.buildConfigResponse(ctx, models, kb, hasFiles)
|
||||||
@@ -1339,7 +1335,7 @@ func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) {
|
|||||||
|
|
||||||
emb, err := embedding.NewEmbedder(cfg)
|
emb, err := embedding.NewEmbedder(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": secutils.SanitizeForLog(req.ModelName)})
|
logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": utils.SanitizeForLog(req.ModelName)})
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"success": true,
|
"success": true,
|
||||||
"data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0},
|
"data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0},
|
||||||
@@ -1595,7 +1591,7 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
|
|||||||
c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB"))
|
c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
logger.Infof(ctx, "Processing image: %s", secutils.SanitizeForLog(header.Filename))
|
logger.Infof(ctx, "Processing image: %s", utils.SanitizeForLog(header.Filename))
|
||||||
|
|
||||||
// 解析文档分割配置
|
// 解析文档分割配置
|
||||||
chunkSizeInt32, err := strconv.ParseInt(req.ChunkSize, 10, 32)
|
chunkSizeInt32, err := strconv.ParseInt(req.ChunkSize, 10, 32)
|
||||||
|
|||||||
@@ -30,4 +30,3 @@ var (
|
|||||||
// ErrConnectionClosed is returned when connection is closed unexpectedly
|
// ErrConnectionClosed is returned when connection is closed unexpectedly
|
||||||
ErrConnectionClosed = errors.New("connection closed")
|
ErrConnectionClosed = errors.New("connection closed")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -122,7 +122,6 @@ func TestRemoteAPIChat(t *testing.T) {
|
|||||||
response.Usage.CompletionTokens,
|
response.Usage.CompletionTokens,
|
||||||
response.Usage.TotalTokens)
|
response.Usage.TotalTokens)
|
||||||
})
|
})
|
||||||
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ type RankResult struct {
|
|||||||
|
|
||||||
// Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field
|
// Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field
|
||||||
func (r *RankResult) UnmarshalJSON(data []byte) error {
|
func (r *RankResult) UnmarshalJSON(data []byte) error {
|
||||||
|
|
||||||
var temp struct {
|
var temp struct {
|
||||||
Index int `json:"index"`
|
Index int `json:"index"`
|
||||||
Document DocumentInfo `json:"document"`
|
Document DocumentInfo `json:"document"`
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ func (m *MemoryStreamManager) AppendEvent(ctx context.Context, sessionID, messag
|
|||||||
|
|
||||||
// Append event
|
// Append event
|
||||||
stream.events = append(stream.events, event)
|
stream.events = append(stream.events, event)
|
||||||
stream.lastUpdated = time.Now()
|
stream.lastUpdated = time.Now()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -104,7 +104,7 @@ func (m *MemoryStreamManager) GetEvents(ctx context.Context, sessionID, messageI
|
|||||||
copy(eventsCopy, events)
|
copy(eventsCopy, events)
|
||||||
|
|
||||||
return eventsCopy, nextOffset, nil
|
return eventsCopy, nextOffset, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure MemoryStreamManager implements StreamManager interface
|
// Ensure MemoryStreamManager implements StreamManager interface
|
||||||
var _ interfaces.StreamManager = (*MemoryStreamManager)(nil)
|
var _ interfaces.StreamManager = (*MemoryStreamManager)(nil)
|
||||||
|
|||||||
@@ -6,13 +6,14 @@ const (
|
|||||||
TypeFAQImport = "faq:import" // FAQ导入任务
|
TypeFAQImport = "faq:import" // FAQ导入任务
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ExtractChunkPayload represents the extract chunk task payload
|
||||||
type ExtractChunkPayload struct {
|
type ExtractChunkPayload struct {
|
||||||
TenantID uint64 `json:"tenant_id"`
|
TenantID uint64 `json:"tenant_id"`
|
||||||
ChunkID string `json:"chunk_id"`
|
ChunkID string `json:"chunk_id"`
|
||||||
ModelID string `json:"model_id"`
|
ModelID string `json:"model_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DocumentProcessPayload 文档处理任务payload
|
// DocumentProcessPayload represents the document process task payload
|
||||||
type DocumentProcessPayload struct {
|
type DocumentProcessPayload struct {
|
||||||
RequestId string `json:"request_id"`
|
RequestId string `json:"request_id"`
|
||||||
TenantID uint64 `json:"tenant_id"`
|
TenantID uint64 `json:"tenant_id"`
|
||||||
@@ -26,7 +27,7 @@ type DocumentProcessPayload struct {
|
|||||||
EnableMultimodel bool `json:"enable_multimodel"`
|
EnableMultimodel bool `json:"enable_multimodel"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// FAQImportPayload FAQ导入任务payload
|
// FAQImportPayload represents the FAQ import task payload
|
||||||
type FAQImportPayload struct {
|
type FAQImportPayload struct {
|
||||||
TenantID uint64 `json:"tenant_id"`
|
TenantID uint64 `json:"tenant_id"`
|
||||||
TaskID string `json:"task_id"`
|
TaskID string `json:"task_id"`
|
||||||
@@ -36,6 +37,7 @@ type FAQImportPayload struct {
|
|||||||
Mode string `json:"mode"`
|
Mode string `json:"mode"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PromptTemplateStructured represents the prompt template structured
|
||||||
type PromptTemplateStructured struct {
|
type PromptTemplateStructured struct {
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Tags []string `json:"tags"`
|
Tags []string `json:"tags"`
|
||||||
@@ -48,6 +50,7 @@ type GraphNode struct {
|
|||||||
Attributes []string `json:"attributes,omitempty"`
|
Attributes []string `json:"attributes,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GraphRelation represents the relation of the graph
|
||||||
type GraphRelation struct {
|
type GraphRelation struct {
|
||||||
Node1 string `json:"node1,omitempty"`
|
Node1 string `json:"node1,omitempty"`
|
||||||
Node2 string `json:"node2,omitempty"`
|
Node2 string `json:"node2,omitempty"`
|
||||||
@@ -60,11 +63,13 @@ type GraphData struct {
|
|||||||
Relation []*GraphRelation `json:"relation,omitempty"`
|
Relation []*GraphRelation `json:"relation,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NameSpace represents the name space of the knowledge base and knowledge
|
||||||
type NameSpace struct {
|
type NameSpace struct {
|
||||||
KnowledgeBase string `json:"knowledge_base"`
|
KnowledgeBase string `json:"knowledge_base"`
|
||||||
Knowledge string `json:"knowledge"`
|
Knowledge string `json:"knowledge"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Labels returns the labels of the name space
|
||||||
func (n NameSpace) Labels() []string {
|
func (n NameSpace) Labels() []string {
|
||||||
res := make([]string, 0)
|
res := make([]string, 0)
|
||||||
if n.KnowledgeBase != "" {
|
if n.KnowledgeBase != "" {
|
||||||
|
|||||||
@@ -160,10 +160,14 @@ type FAQSearchRequest struct {
|
|||||||
type FAQImportTaskStatus string
|
type FAQImportTaskStatus string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FAQImportStatusPending FAQImportTaskStatus = "pending"
|
// FAQImportStatusPending represents the pending status of the FAQ import task
|
||||||
|
FAQImportStatusPending FAQImportTaskStatus = "pending"
|
||||||
|
// FAQImportStatusProcessing represents the processing status of the FAQ import task
|
||||||
FAQImportStatusProcessing FAQImportTaskStatus = "processing"
|
FAQImportStatusProcessing FAQImportTaskStatus = "processing"
|
||||||
FAQImportStatusCompleted FAQImportTaskStatus = "completed"
|
// FAQImportStatusCompleted represents the completed status of the FAQ import task
|
||||||
FAQImportStatusFailed FAQImportTaskStatus = "failed"
|
FAQImportStatusCompleted FAQImportTaskStatus = "completed"
|
||||||
|
// FAQImportStatusFailed represents the failed status of the FAQ import task
|
||||||
|
FAQImportStatusFailed FAQImportTaskStatus = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息
|
// FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息
|
||||||
|
|||||||
@@ -10,10 +10,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
// KnowledgeTypeManual represents the manual knowledge type
|
||||||
KnowledgeTypeManual = "manual"
|
KnowledgeTypeManual = "manual"
|
||||||
KnowledgeTypeFAQ = "faq"
|
// KnowledgeTypeFAQ represents the FAQ knowledge type
|
||||||
|
KnowledgeTypeFAQ = "faq"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ManualKnowledgeFormat represents the format of the manual knowledge
|
||||||
const (
|
const (
|
||||||
ManualKnowledgeFormatMarkdown = "markdown"
|
ManualKnowledgeFormatMarkdown = "markdown"
|
||||||
ManualKnowledgeStatusDraft = "draft"
|
ManualKnowledgeStatusDraft = "draft"
|
||||||
|
|||||||
@@ -8,32 +8,34 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// KnowledgeBaseType represents the type of the knowledge base
|
||||||
const (
|
const (
|
||||||
|
// KnowledgeBaseTypeDocument represents the document knowledge base type
|
||||||
KnowledgeBaseTypeDocument = "document"
|
KnowledgeBaseTypeDocument = "document"
|
||||||
KnowledgeBaseTypeFAQ = "faq"
|
KnowledgeBaseTypeFAQ = "faq"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FAQIndexMode 表示 FAQ 索引策略:仅索引标准问还是同时索引问答内容
|
// FAQIndexMode represents the FAQ index mode: only index questions or index questions and answers
|
||||||
type FAQIndexMode string
|
type FAQIndexMode string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// FAQIndexModeQuestionOnly 仅使用标准问及相似问向量化
|
// FAQIndexModeQuestionOnly only index questions and similar questions
|
||||||
FAQIndexModeQuestionOnly FAQIndexMode = "question_only"
|
FAQIndexModeQuestionOnly FAQIndexMode = "question_only"
|
||||||
// FAQIndexModeQuestionAnswer 将标准问与答案拼接后向量化
|
// FAQIndexModeQuestionAnswer index questions and answers together
|
||||||
FAQIndexModeQuestionAnswer FAQIndexMode = "question_answer"
|
FAQIndexModeQuestionAnswer FAQIndexMode = "question_answer"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FAQQuestionIndexMode 表示 FAQ 问题索引方式:一起索引还是分别索引
|
// FAQQuestionIndexMode represents the FAQ question index mode: index together or index separately
|
||||||
type FAQQuestionIndexMode string
|
type FAQQuestionIndexMode string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// FAQQuestionIndexModeCombined 将标准问和相似问一起索引(当前方式)
|
// FAQQuestionIndexModeCombined index questions and similar questions together
|
||||||
FAQQuestionIndexModeCombined FAQQuestionIndexMode = "combined"
|
FAQQuestionIndexModeCombined FAQQuestionIndexMode = "combined"
|
||||||
// FAQQuestionIndexModeSeparate 将标准问和相似问分别索引
|
// FAQQuestionIndexModeSeparate index questions and similar questions separately
|
||||||
FAQQuestionIndexModeSeparate FAQQuestionIndexMode = "separate"
|
FAQQuestionIndexModeSeparate FAQQuestionIndexMode = "separate"
|
||||||
)
|
)
|
||||||
|
|
||||||
// KnowledgeBase represents a knowledge base
|
// KnowledgeBase represents a knowledge base entity
|
||||||
type KnowledgeBase struct {
|
type KnowledgeBase struct {
|
||||||
// Unique identifier of the knowledge base
|
// Unique identifier of the knowledge base
|
||||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||||
@@ -195,6 +197,7 @@ func (c *VLMConfig) Scan(value interface{}) error {
|
|||||||
return json.Unmarshal(b, c)
|
return json.Unmarshal(b, c)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExtractConfig represents the extract configuration for a knowledge base
|
||||||
type ExtractConfig struct {
|
type ExtractConfig struct {
|
||||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||||
Text string `yaml:"text" json:"text,omitempty"`
|
Text string `yaml:"text" json:"text,omitempty"`
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ const (
|
|||||||
ModelTypeVLLM ModelType = "VLLM" // VLLM model
|
ModelTypeVLLM ModelType = "VLLM" // VLLM model
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ModelStatus represents the status of the model
|
||||||
type ModelStatus string
|
type ModelStatus string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -36,6 +37,7 @@ const (
|
|||||||
ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model
|
ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// EmbeddingParameters represents the embedding parameters for a model
|
||||||
type EmbeddingParameters struct {
|
type EmbeddingParameters struct {
|
||||||
Dimension int `yaml:"dimension" json:"dimension"`
|
Dimension int `yaml:"dimension" json:"dimension"`
|
||||||
TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"`
|
TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"`
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const (
|
|||||||
FallbackStrategyModel FallbackStrategy = "model" // Model fallback response
|
FallbackStrategyModel FallbackStrategy = "model" // Model fallback response
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// SummaryConfig represents the summary configuration for a session
|
||||||
type SummaryConfig struct {
|
type SummaryConfig struct {
|
||||||
// Max tokens
|
// Max tokens
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
@@ -109,6 +110,7 @@ func (s *Session) BeforeCreate(tx *gorm.DB) (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StringArray represents a list of strings
|
||||||
type StringArray []string
|
type StringArray []string
|
||||||
|
|
||||||
// Value implements the driver.Valuer interface, used to convert StringArray to database value
|
// Value implements the driver.Valuer interface, used to convert StringArray to database value
|
||||||
|
|||||||
@@ -44,10 +44,12 @@ type Tenant struct {
|
|||||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetrieverEngines represents the retriever engines for a tenant
|
||||||
type RetrieverEngines struct {
|
type RetrieverEngines struct {
|
||||||
Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"`
|
Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BeforeCreate is a hook function that is called before creating a tenant
|
||||||
func (t *Tenant) BeforeCreate(tx *gorm.DB) error {
|
func (t *Tenant) BeforeCreate(tx *gorm.DB) error {
|
||||||
if t.RetrieverEngines.Engines == nil {
|
if t.RetrieverEngines.Engines == nil {
|
||||||
t.RetrieverEngines.Engines = []RetrieverEngineParams{}
|
t.RetrieverEngines.Engines = []RetrieverEngineParams{}
|
||||||
|
|||||||
Reference in New Issue
Block a user