knowledge-governance/internal/qaopsfields/language.go

package qaopsfields

import (
	"regexp"
	"strings"
	"unicode"
)

const LanguageRuleVersion = "original-question-local-v1"

var nonLanguage = []*regexp.Regexp{
	regexp.MustCompile("(?s)```.*?```|`[^`]*`"),
	regexp.MustCompile(`(?i)https?://[^\s]+|[\w.+-]+@[\w.-]+\.[a-z]{2,}`),
	regexp.MustCompile(`(?i)\b(?:DobotStudio(?:\s+Pro)?|DobotSCStudio|DobotAI|DOBOT|MG400|M1\s*Pro|VX500|X-Trainer|FlexPendant|Magician(?:\s+(?:Lite|E6))?|Nova\d*|ATOM(?:-W)?|CRA-IP68|CR\d+A-Ex|CR(?:AF|A|V2?|H)?\d*[AHV]?|Panel|SDK|API|PLC|TCP/IP|Modbus(?:\s+TCP)?|ROS|DH|STP|LUA|CE(?:-MD|-EMC)?|RoHS|RDS)\b`),
	regexp.MustCompile(`(?i)\b\d+(?:\.\d+)?\s*(?:kg|mm|cm|ms|hz|khz|mhz|gb|mb|rpm|v|a|w|kw)?\b`),
}
var words = map[string]string{
	"en": "how what why when where which can could does do is are the a to of for with without connect connection use using reset read write support supported install set configure current please",
	"fr": "comment pourquoi quel quelle quelles est sont avec pour les une connecter connexion faire puis peux",
	"de": "wie warum welche welcher kann ich mit verbinden verbindung funktioniert einstellen werden wird das die der",
	"es": "cómo cómo por qué cuáles cuál puedo conectar conexión para los las una hay tiene configurar",
	"pt": "você vocês qual quais ligar conexão não robô configuração preciso posso configurar",
	"it": "come perché quale quali posso collegare collegamento impostare robotica della degli",
	"da": "hvordan hvad hvorfor hvilken hvilke tilslutter forbinde indstille jeg",
	"pl": "jak dlaczego jaki jaka jakie połączyć połączenie ustawić można jest",
	"vi": "tiếng việt làm thế nào kết nối được không cách tôi với của",
	"ru": "как что почему какой какие можно подключить подключение настроить работает для",
}
var languageNames = map[string]string{"zh": "中文", "en": "英文", "ja": "日文", "ko": "韩文", "fr": "法文", "de": "德文", "es": "西班牙文", "pt": "葡萄牙文", "it": "意大利文", "da": "丹麦文", "pl": "波兰文", "vi": "越南文", "ru": "俄文", "th": "泰文", "ar": "阿拉伯文", "mul": "多语言", "und": "未识别"}

func LanguageName(code string) string {
	if name, ok := languageNames[code]; ok {
		return name
	}
	return "未识别"
}

// Product identifiers and technical tokens carry no reliable language signal.
// Keep uncertain short input und instead of silently defaulting to English.
func DetectLanguage(question string) string {
	text := NaturalText(question)
	han, kana, hangul, thai, arabic, latin := 0, 0, 0, 0, 0, 0
	for _, r := range text {
		switch {
		case unicode.In(r, unicode.Hiragana, unicode.Katakana):
			kana++
		case unicode.In(r, unicode.Hangul):
			hangul++
		case unicode.In(r, unicode.Han):
			han++
		case unicode.In(r, unicode.Thai):
			thai++
		case unicode.In(r, unicode.Arabic):
			arabic++
		case unicode.In(r, unicode.Latin):
			latin++
		}
	}
	scripts := 0
	for _, n := range []int{kana, hangul, thai, arabic} {
		if n > 0 {
			scripts++
		}
	}
	if scripts > 1 {
		return "mul"
	}
	if kana > 0 {
		if latin > 2*(kana+han)+12 {
			return "mul"
		}
		return "ja"
	}
	if hangul > 0 {
		if latin > 2*hangul+12 {
			return "mul"
		}
		return "ko"
	}
	if thai > 0 {
		return "th"
	}
	if arabic > 0 {
		return "ar"
	}
	tokens := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { return !unicode.IsLetter(r) })
	if han > 0 {
		if len(tokens) >= 5 && latin > 2*han+12 {
			return "mul"
		}
		return "zh"
	}
	scores := map[string]int{}
	for lang, vocabulary := range words {
		set := map[string]bool{}
		for _, w := range strings.Fields(vocabulary) {
			set[w] = true
		}
		for _, token := range tokens {
			if set[token] {
				scores[lang]++
			}
		}
	}
	best, maxScore, tied := "und", 0, false
	for lang, score := range scores {
		if score > maxScore {
			best, maxScore, tied = lang, score, false
		} else if score == maxScore && score > 0 {
			tied = true
		}
	}
	if tied || maxScore == 0 {
		return "und"
	}
	return best
}

// NaturalText is a detection-only view; callers retain the original question verbatim.
func NaturalText(question string) string {
	text := question
	for _, pattern := range nonLanguage {
		text = pattern.ReplaceAllString(text, " ")
	}
	return text
}