Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipline/preprocess.go
T
wizardchen 3bf2a14499 feat: Enhance agent configuration and localization for web search prompts
- Updated agent configuration to support separate system prompts for web search enabled and disabled states.
- Removed deprecated agent configuration parameters to streamline settings management.
- Enhanced UI components in AgentSettings.vue to allow configuration of custom prompts based on web search status.
- Improved localization in English, Russian, and Chinese for new prompt settings and UI elements.
- Refactored related API and service logic to accommodate changes in agent configuration structure.
2025-11-19 19:27:25 +08:00

334 lines
9.0 KiB
Go

package chatpipline
import (
"context"
"encoding/json"
"regexp"
"strings"
"unicode"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/yanyiwu/gojieba"
)
// PluginPreprocess Query preprocessing plugin
type PluginPreprocess struct {
config *config.Config
jieba *gojieba.Jieba
stopwords map[string]struct{}
modelService interfaces.ModelService
}
// Regular expressions for text cleaning
var (
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
urlRegex = regexp.MustCompile(`https?://\S+`) // URLs
emailRegex = regexp.MustCompile(`\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,}\b`) // Email addresses
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
)
const maxProcessedTokens = 12
// NewPluginPreprocess Creates a new query preprocessing plugin
func NewPluginPreprocess(
eventManager *EventManager,
config *config.Config,
cleaner interfaces.ResourceCleaner,
modelService interfaces.ModelService,
) *PluginPreprocess {
// Use default dictionary for Jieba tokenizer
jieba := gojieba.NewJieba()
// Load stopwords from built-in stopword library
stopwords := loadStopwords()
res := &PluginPreprocess{
config: config,
jieba: jieba,
stopwords: stopwords,
modelService: modelService,
}
// Register resource cleanup function
if cleaner != nil {
cleaner.RegisterWithName("JiebaPreprocessor", func() error {
res.Close()
return nil
})
}
eventManager.Register(res)
return res
}
// Load stopwords
func loadStopwords() map[string]struct{} {
// Directly use some common stopwords built into Jieba
commonStopwords := []string{
"的", "了", "和", "是", "在", "我", "你", "他", "她", "它",
"这", "那", "什么", "怎么", "如何", "为什么", "哪里", "什么时候",
"the", "is", "are", "am", "I", "you", "he", "she", "it", "this",
"that", "what", "how", "a", "an", "and", "or", "but", "if", "of",
"to", "in", "on", "at", "by", "for", "with", "about", "from",
"有", "无", "好", "来", "去", "说", "看", "想", "会", "可以",
"吗", "呢", "啊", "吧", "的话", "就是", "只是", "因为", "所以",
}
result := make(map[string]struct{}, len(commonStopwords))
for _, word := range commonStopwords {
result[word] = struct{}{}
}
return result
}
// ActivationEvents Register activation events
func (p *PluginPreprocess) ActivationEvents() []types.EventType {
return []types.EventType{types.PREPROCESS_QUERY}
}
// OnEvent Process events
func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError {
rawQuery := strings.TrimSpace(chatManage.RewriteQuery)
if rawQuery == "" {
return next()
}
pipelineInfo(ctx, "Preprocess", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": rawQuery,
})
normalized := normalizeWhitespace(rawQuery)
sanitized := strings.TrimSpace(p.cleanText(normalized))
if sanitized == "" {
sanitized = normalized
}
var (
processed = sanitized
strategy = "original"
tokenPreview string
tokenCount int
)
switch {
case containsChineseCharacters(sanitized):
segments := p.segmentText(sanitized)
tokens := p.selectMeaningfulTokens(segments)
tokenCount = len(tokens)
if len(tokens) >= 2 {
processed = strings.Join(tokens, " ")
strategy = "zh_tokens"
tokenPreview = strings.Join(tokens, ",")
} else {
strategy = "fallback_original"
}
case containsLatinLetters(sanitized):
processed = normalizeLatinQuery(sanitized)
if processed != sanitized {
strategy = "latin_normalize"
}
default:
strategy = "original"
}
if strings.TrimSpace(processed) == "" {
processed = rawQuery
strategy = "fallback_original"
}
chatManage.ProcessedQuery = processed
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"processed_query": processed,
"strategy": strategy,
"token_count": tokenCount,
"token_preview": tokenPreview,
"query_intent": chatManage.QueryIntent,
})
return next()
}
// cleanText Basic text cleaning
func (p *PluginPreprocess) cleanText(text string) string {
// Remove URLs
text = urlRegex.ReplaceAllString(text, " ")
// Remove email addresses
text = emailRegex.ReplaceAllString(text, " ")
// Remove excessive spaces
text = multiSpaceRegex.ReplaceAllString(text, " ")
// Remove punctuation marks
text = punctRegex.ReplaceAllString(text, " ")
// Trim leading and trailing spaces
text = strings.TrimSpace(text)
return text
}
// segmentText Text tokenization
func (p *PluginPreprocess) segmentText(text string) []string {
// Use Jieba tokenizer for tokenization, using search engine mode
segments := p.jieba.CutForSearch(text, true)
return segments
}
// filterStopwords Filter stopwords
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
var tokens []string
seen := make(map[string]struct{})
for _, word := range segments {
word = strings.TrimSpace(word)
if word == "" {
continue
}
if _, stop := p.stopwords[word]; stop {
continue
}
if _, exists := seen[word]; exists {
continue
}
if !isInformativeToken(word) {
continue
}
seen[word] = struct{}{}
tokens = append(tokens, word)
if len(tokens) >= maxProcessedTokens {
break
}
}
return tokens
}
// isBlank Check if a string is blank
func isInformativeToken(token string) bool {
if token == "" {
return false
}
runeCount := utf8.RuneCountInString(token)
if runeCount == 1 {
r, _ := utf8.DecodeRuneInString(token)
if unicode.IsDigit(r) {
return true
}
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
return false
}
return true
}
func containsChineseCharacters(text string) bool {
for _, r := range text {
if unicode.Is(unicode.Han, r) {
return true
}
}
return false
}
func containsLatinLetters(text string) bool {
for _, r := range text {
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
}
return false
}
func normalizeWhitespace(text string) string {
text = strings.TrimSpace(text)
return multiSpaceRegex.ReplaceAllString(text, " ")
}
func normalizeLatinQuery(text string) string {
text = strings.ToLower(text)
text = multiSpaceRegex.ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}
type intentResp struct {
Intent string `json:"intent"`
Confidence float64 `json:"confidence"`
}
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
if p.modelService == nil || chatManage.ChatModelID == "" {
pipelineWarn(ctx, "IntentDetect", "skip", map[string]interface{}{"reason": "no_model", "session_id": chatManage.SessionID})
return "general"
}
chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
pipelineWarn(ctx, "IntentDetect", "get_model_failed", map[string]interface{}{"error": err.Error(), "model_id": chatManage.ChatModelID})
return "general"
}
pipelineInfo(ctx, "IntentDetect", "start", map[string]interface{}{"session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID})
sys := "You are a query intent classifier. Classify the user's query into one of: definition, howto, compare, qa, general. Respond ONLY with a JSON object {\"intent\": \"...\", \"confidence\": 0.0 } inside a markdown fenced block."
usr := text
think := false
resp, err := chatModel.Chat(ctx, []chat.Message{
{Role: "system", Content: sys},
{Role: "user", Content: usr},
}, &chat.ChatOptions{Temperature: 0.0, MaxCompletionTokens: 64, Thinking: &think})
if err != nil || resp.Content == "" {
pipelineWarn(ctx, "IntentDetect", "model_call_failed", map[string]interface{}{"error": err})
return "general"
}
body := extractJSONBody(resp.Content)
var ir intentResp
if err := json.Unmarshal([]byte(body), &ir); err != nil {
pipelineWarn(ctx, "IntentDetect", "parse_failed", map[string]interface{}{"body": body, "error": err.Error()})
return "general"
}
pipelineInfo(ctx, "IntentDetect", "result", map[string]interface{}{"intent": ir.Intent, "confidence": ir.Confidence})
switch strings.ToLower(strings.TrimSpace(ir.Intent)) {
case "definition", "howto", "compare", "qa", "general":
return strings.ToLower(ir.Intent)
default:
return "general"
}
}
func extractJSONBody(text string) string {
t := strings.TrimSpace(text)
// Try fenced block first
if i := strings.Index(t, "{"); i >= 0 {
j := strings.LastIndex(t, "}")
if j > i {
return t[i : j+1]
}
}
return "{}"
}
// Ensure resources are properly released
func (p *PluginPreprocess) Close() {
if p.jieba != nil {
p.jieba.Free()
p.jieba = nil
}
}
// ShutdownHandler Returns shutdown function
func (p *PluginPreprocess) ShutdownHandler() func() {
return func() {
p.Close()
}
}