package coverage import ( "strings" "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" ) // noAnswerPhrases lists explicit "no answer" phrases that the knowledge base // returns when it cannot find relevant information. These are matched as // substrings after trimming whitespace. var noAnswerPhrases = []string{ "未找到", "没有找到", "无法确定", "暂无相关", "暂无已发布的权威知识", "知识库中没有", "产品不存在于资料中", } // retrievalGapPhrases are only considered a no-answer signal when the answer // also directs the user to a human channel. Requiring both signals avoids // misclassifying valid negative results such as "未查询到异常记录". var retrievalGapPhrases = []string{ "未查询到", "未检索到", "未能查询到", "未能检索到", } var humanEscalationPhrases = []string{ "建议您联系", "建议联系", "请联系", "联系技术支持", "联系官方销售", } // Evaluator classifies a successful turn into a coverage verdict. type Evaluator struct{} // NewEvaluator creates the current Evaluator. func NewEvaluator() *Evaluator { return &Evaluator{} } // Version returns the current evaluator version. func (e *Evaluator) Version() string { return EvaluatorVersion } // Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation // rules (in priority order): // 1. Failed/processing turns → ErrTurnNotSucceeded // 2. Answer contains an explicit "no answer" phrase → uncovered_explicit_no_answer // 3. Image turns (image media present) are classified by whether the question // actually triggered a knowledge-base search: // - has references → covered_has_references // - no references + knowledge searched → uncovered_no_references // - no references + no knowledge search → covered_visual_recognition // (pure visual recognition is not a coverage miss) // 4. Text turns with zero references are classified by whether the question // actually triggered a knowledge-base search: // - knowledge searched → uncovered_no_references // - no knowledge search → covered_non_knowledge_response // 5. Has references → covered_has_references // // v1 never uses a score threshold. TopScore captures the highest reference score // (0 if no references). func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) { if turn.Status != "succeeded" { return Decision{}, ErrTurnNotSucceeded } // Normalize answer for no-answer phrase detection. trimmed := strings.TrimSpace(turn.Answer) // Check for explicit no-answer phrases first — this takes priority // over reference presence. if isNoAnswer(trimmed) { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredExplicitNoAnswer, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } // Image turns are classified by whether knowledge search occurred. if hasImageMedia(turn.Media) { if len(turn.References) > 0 { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredHasReferences, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } if turn.KnowledgeSearched { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredNoReferences, ReferenceCount: 0, TopScore: 0, }, nil } return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredVisualRecognition, ReferenceCount: 0, TopScore: 0, }, nil } // Text turns without references are only uncovered if they actually // triggered knowledge search. This prevents greetings and other normal // non-knowledge responses from generating artificial support cases. if len(turn.References) == 0 { if !turn.KnowledgeSearched { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredNonKnowledgeResponse, ReferenceCount: 0, TopScore: 0, }, nil } return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredNoReferences, ReferenceCount: 0, TopScore: 0, }, nil } // Has references → covered (v1 never uses score threshold). return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredHasReferences, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } // hasImageMedia reports whether the turn carried at least one image input. func hasImageMedia(media []core.MediaAudit) bool { for _, m := range media { if strings.EqualFold(strings.TrimSpace(m.Type), "image") { return true } } return false } func isNoAnswer(answer string) bool { lower := strings.ToLower(answer) for _, phrase := range noAnswerPhrases { if strings.Contains(lower, strings.ToLower(phrase)) { return true } } return containsAny(lower, retrievalGapPhrases) && containsAny(lower, humanEscalationPhrases) } func containsAny(value string, phrases []string) bool { for _, phrase := range phrases { if strings.Contains(value, strings.ToLower(phrase)) { return true } } return false } func maxScore(refs []core.Reference) float64 { var max float64 for _, r := range refs { if r.Score > max { max = r.Score } } return max }