3bf2a14499
- Updated agent configuration to support separate system prompts for web search enabled and disabled states. - Removed deprecated agent configuration parameters to streamline settings management. - Enhanced UI components in AgentSettings.vue to allow configuration of custom prompts based on web search status. - Improved localization in English, Russian, and Chinese for new prompt settings and UI elements. - Refactored related API and service logic to accommodate changes in agent configuration structure.
334 lines
9.0 KiB
Go
334 lines
9.0 KiB
Go
package chatpipline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"regexp"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/models/chat"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"github.com/yanyiwu/gojieba"
|
|
)
|
|
|
|
// PluginPreprocess Query preprocessing plugin
|
|
type PluginPreprocess struct {
|
|
config *config.Config
|
|
jieba *gojieba.Jieba
|
|
stopwords map[string]struct{}
|
|
modelService interfaces.ModelService
|
|
}
|
|
|
|
// Regular expressions for text cleaning
|
|
var (
|
|
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
|
|
urlRegex = regexp.MustCompile(`https?://\S+`) // URLs
|
|
emailRegex = regexp.MustCompile(`\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,}\b`) // Email addresses
|
|
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
|
|
)
|
|
|
|
const maxProcessedTokens = 12
|
|
|
|
// NewPluginPreprocess Creates a new query preprocessing plugin
|
|
func NewPluginPreprocess(
|
|
eventManager *EventManager,
|
|
config *config.Config,
|
|
cleaner interfaces.ResourceCleaner,
|
|
modelService interfaces.ModelService,
|
|
) *PluginPreprocess {
|
|
// Use default dictionary for Jieba tokenizer
|
|
jieba := gojieba.NewJieba()
|
|
|
|
// Load stopwords from built-in stopword library
|
|
stopwords := loadStopwords()
|
|
|
|
res := &PluginPreprocess{
|
|
config: config,
|
|
jieba: jieba,
|
|
stopwords: stopwords,
|
|
modelService: modelService,
|
|
}
|
|
|
|
// Register resource cleanup function
|
|
if cleaner != nil {
|
|
cleaner.RegisterWithName("JiebaPreprocessor", func() error {
|
|
res.Close()
|
|
return nil
|
|
})
|
|
}
|
|
|
|
eventManager.Register(res)
|
|
return res
|
|
}
|
|
|
|
// Load stopwords
|
|
func loadStopwords() map[string]struct{} {
|
|
// Directly use some common stopwords built into Jieba
|
|
commonStopwords := []string{
|
|
"的", "了", "和", "是", "在", "我", "你", "他", "她", "它",
|
|
"这", "那", "什么", "怎么", "如何", "为什么", "哪里", "什么时候",
|
|
"the", "is", "are", "am", "I", "you", "he", "she", "it", "this",
|
|
"that", "what", "how", "a", "an", "and", "or", "but", "if", "of",
|
|
"to", "in", "on", "at", "by", "for", "with", "about", "from",
|
|
"有", "无", "好", "来", "去", "说", "看", "想", "会", "可以",
|
|
"吗", "呢", "啊", "吧", "的话", "就是", "只是", "因为", "所以",
|
|
}
|
|
|
|
result := make(map[string]struct{}, len(commonStopwords))
|
|
for _, word := range commonStopwords {
|
|
result[word] = struct{}{}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// ActivationEvents Register activation events
|
|
func (p *PluginPreprocess) ActivationEvents() []types.EventType {
|
|
return []types.EventType{types.PREPROCESS_QUERY}
|
|
}
|
|
|
|
// OnEvent Process events
|
|
func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError {
|
|
rawQuery := strings.TrimSpace(chatManage.RewriteQuery)
|
|
if rawQuery == "" {
|
|
return next()
|
|
}
|
|
|
|
pipelineInfo(ctx, "Preprocess", "input", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"rewrite_query": rawQuery,
|
|
})
|
|
|
|
normalized := normalizeWhitespace(rawQuery)
|
|
sanitized := strings.TrimSpace(p.cleanText(normalized))
|
|
if sanitized == "" {
|
|
sanitized = normalized
|
|
}
|
|
|
|
var (
|
|
processed = sanitized
|
|
strategy = "original"
|
|
tokenPreview string
|
|
tokenCount int
|
|
)
|
|
|
|
switch {
|
|
case containsChineseCharacters(sanitized):
|
|
segments := p.segmentText(sanitized)
|
|
tokens := p.selectMeaningfulTokens(segments)
|
|
tokenCount = len(tokens)
|
|
if len(tokens) >= 2 {
|
|
processed = strings.Join(tokens, " ")
|
|
strategy = "zh_tokens"
|
|
tokenPreview = strings.Join(tokens, ",")
|
|
} else {
|
|
strategy = "fallback_original"
|
|
}
|
|
case containsLatinLetters(sanitized):
|
|
processed = normalizeLatinQuery(sanitized)
|
|
if processed != sanitized {
|
|
strategy = "latin_normalize"
|
|
}
|
|
default:
|
|
strategy = "original"
|
|
}
|
|
|
|
if strings.TrimSpace(processed) == "" {
|
|
processed = rawQuery
|
|
strategy = "fallback_original"
|
|
}
|
|
|
|
chatManage.ProcessedQuery = processed
|
|
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
|
|
|
|
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"processed_query": processed,
|
|
"strategy": strategy,
|
|
"token_count": tokenCount,
|
|
"token_preview": tokenPreview,
|
|
"query_intent": chatManage.QueryIntent,
|
|
})
|
|
|
|
return next()
|
|
}
|
|
|
|
// cleanText Basic text cleaning
|
|
func (p *PluginPreprocess) cleanText(text string) string {
|
|
// Remove URLs
|
|
text = urlRegex.ReplaceAllString(text, " ")
|
|
|
|
// Remove email addresses
|
|
text = emailRegex.ReplaceAllString(text, " ")
|
|
|
|
// Remove excessive spaces
|
|
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
|
|
|
// Remove punctuation marks
|
|
text = punctRegex.ReplaceAllString(text, " ")
|
|
|
|
// Trim leading and trailing spaces
|
|
text = strings.TrimSpace(text)
|
|
|
|
return text
|
|
}
|
|
|
|
// segmentText Text tokenization
|
|
func (p *PluginPreprocess) segmentText(text string) []string {
|
|
// Use Jieba tokenizer for tokenization, using search engine mode
|
|
segments := p.jieba.CutForSearch(text, true)
|
|
return segments
|
|
}
|
|
|
|
// filterStopwords Filter stopwords
|
|
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
|
|
var tokens []string
|
|
seen := make(map[string]struct{})
|
|
|
|
for _, word := range segments {
|
|
word = strings.TrimSpace(word)
|
|
if word == "" {
|
|
continue
|
|
}
|
|
if _, stop := p.stopwords[word]; stop {
|
|
continue
|
|
}
|
|
if _, exists := seen[word]; exists {
|
|
continue
|
|
}
|
|
if !isInformativeToken(word) {
|
|
continue
|
|
}
|
|
|
|
seen[word] = struct{}{}
|
|
tokens = append(tokens, word)
|
|
if len(tokens) >= maxProcessedTokens {
|
|
break
|
|
}
|
|
}
|
|
|
|
return tokens
|
|
}
|
|
|
|
// isBlank Check if a string is blank
|
|
func isInformativeToken(token string) bool {
|
|
if token == "" {
|
|
return false
|
|
}
|
|
|
|
runeCount := utf8.RuneCountInString(token)
|
|
if runeCount == 1 {
|
|
r, _ := utf8.DecodeRuneInString(token)
|
|
if unicode.IsDigit(r) {
|
|
return true
|
|
}
|
|
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func containsChineseCharacters(text string) bool {
|
|
for _, r := range text {
|
|
if unicode.Is(unicode.Han, r) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func containsLatinLetters(text string) bool {
|
|
for _, r := range text {
|
|
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func normalizeWhitespace(text string) string {
|
|
text = strings.TrimSpace(text)
|
|
return multiSpaceRegex.ReplaceAllString(text, " ")
|
|
}
|
|
|
|
func normalizeLatinQuery(text string) string {
|
|
text = strings.ToLower(text)
|
|
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
|
return strings.TrimSpace(text)
|
|
}
|
|
|
|
type intentResp struct {
|
|
Intent string `json:"intent"`
|
|
Confidence float64 `json:"confidence"`
|
|
}
|
|
|
|
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
|
|
if p.modelService == nil || chatManage.ChatModelID == "" {
|
|
pipelineWarn(ctx, "IntentDetect", "skip", map[string]interface{}{"reason": "no_model", "session_id": chatManage.SessionID})
|
|
return "general"
|
|
}
|
|
chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "IntentDetect", "get_model_failed", map[string]interface{}{"error": err.Error(), "model_id": chatManage.ChatModelID})
|
|
return "general"
|
|
}
|
|
pipelineInfo(ctx, "IntentDetect", "start", map[string]interface{}{"session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID})
|
|
sys := "You are a query intent classifier. Classify the user's query into one of: definition, howto, compare, qa, general. Respond ONLY with a JSON object {\"intent\": \"...\", \"confidence\": 0.0 } inside a markdown fenced block."
|
|
usr := text
|
|
think := false
|
|
resp, err := chatModel.Chat(ctx, []chat.Message{
|
|
{Role: "system", Content: sys},
|
|
{Role: "user", Content: usr},
|
|
}, &chat.ChatOptions{Temperature: 0.0, MaxCompletionTokens: 64, Thinking: &think})
|
|
if err != nil || resp.Content == "" {
|
|
pipelineWarn(ctx, "IntentDetect", "model_call_failed", map[string]interface{}{"error": err})
|
|
return "general"
|
|
}
|
|
body := extractJSONBody(resp.Content)
|
|
var ir intentResp
|
|
if err := json.Unmarshal([]byte(body), &ir); err != nil {
|
|
pipelineWarn(ctx, "IntentDetect", "parse_failed", map[string]interface{}{"body": body, "error": err.Error()})
|
|
return "general"
|
|
}
|
|
pipelineInfo(ctx, "IntentDetect", "result", map[string]interface{}{"intent": ir.Intent, "confidence": ir.Confidence})
|
|
switch strings.ToLower(strings.TrimSpace(ir.Intent)) {
|
|
case "definition", "howto", "compare", "qa", "general":
|
|
return strings.ToLower(ir.Intent)
|
|
default:
|
|
return "general"
|
|
}
|
|
}
|
|
|
|
func extractJSONBody(text string) string {
|
|
t := strings.TrimSpace(text)
|
|
// Try fenced block first
|
|
if i := strings.Index(t, "{"); i >= 0 {
|
|
j := strings.LastIndex(t, "}")
|
|
if j > i {
|
|
return t[i : j+1]
|
|
}
|
|
}
|
|
return "{}"
|
|
}
|
|
|
|
// Ensure resources are properly released
|
|
func (p *PluginPreprocess) Close() {
|
|
if p.jieba != nil {
|
|
p.jieba.Free()
|
|
p.jieba = nil
|
|
}
|
|
}
|
|
|
|
// ShutdownHandler Returns shutdown function
|
|
func (p *PluginPreprocess) ShutdownHandler() func() {
|
|
return func() {
|
|
p.Close()
|
|
}
|
|
}
|