package coverage import ( "strings" "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" ) // canonicalNoAnswer is the only response that carries the explicit-no-answer // machine contract. It must be the complete answer, never a substring match: // an otherwise supported answer may legitimately say that one specific detail // was not found. const canonicalNoAnswer = "当前暂无已发布的权威知识,暂不建议操作。" // Evaluator classifies a successful turn into a coverage verdict. type Evaluator struct{} // NewEvaluator creates the current Evaluator. func NewEvaluator() *Evaluator { return &Evaluator{} } // Version returns the current evaluator version. func (e *Evaluator) Version() string { return EvaluatorVersion } // Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation // rules (in priority order): // 1. Failed/processing turns → ErrTurnNotSucceeded // 2. Has references → covered_has_references (references are the coverage // evidence; wording in a supported answer never overrides them) // 3. Complete canonical no-answer response with no references // → uncovered_explicit_no_answer // 4. Image turns (image media present) are classified by whether the question // actually triggered a knowledge-base search: // - no references + knowledge searched → uncovered_no_references // - no references + no knowledge search → covered_visual_recognition // 5. Text turns with zero references are classified by whether the question // actually triggered a knowledge-base search: // - knowledge searched → uncovered_no_references // - no knowledge search → covered_non_knowledge_response // // v5 never uses a score threshold. TopScore captures the highest reference score // (0 if no references). func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) { if turn.Status != "succeeded" { return Decision{}, ErrTurnNotSucceeded } // Normalize answer for no-answer phrase detection. trimmed := strings.TrimSpace(turn.Answer) // References are the durable coverage evidence. A supported answer may // correctly note that one sub-detail is absent, so its wording cannot turn // the entire turn into a knowledge gap. if len(turn.References) > 0 { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredHasReferences, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } // An explicit abstention is a stable machine contract only when it is the // complete answer. Do not use broad natural-language substring matching. if isCanonicalNoAnswer(trimmed) { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredExplicitNoAnswer, ReferenceCount: 0, TopScore: 0, }, nil } // Image turns are classified by whether knowledge search occurred. if hasImageMedia(turn.Media) { if len(turn.References) > 0 { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredHasReferences, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } if turn.KnowledgeSearched { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredNoReferences, ReferenceCount: 0, TopScore: 0, }, nil } return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredVisualRecognition, ReferenceCount: 0, TopScore: 0, }, nil } // Text turns without references are only uncovered if they actually // triggered knowledge search. This prevents greetings and other normal // non-knowledge responses from generating artificial support cases. if len(turn.References) == 0 { if !turn.KnowledgeSearched { return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredNonKnowledgeResponse, ReferenceCount: 0, TopScore: 0, }, nil } return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictUncovered, ReasonCode: ReasonUncoveredNoReferences, ReferenceCount: 0, TopScore: 0, }, nil } // Has references → covered (v1 never uses score threshold). return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, Channel: turn.Channel, EvaluatorVersion: EvaluatorVersion, Verdict: VerdictCovered, ReasonCode: ReasonCoveredHasReferences, ReferenceCount: len(turn.References), TopScore: maxScore(turn.References), }, nil } // hasImageMedia reports whether the turn carried at least one image input. func hasImageMedia(media []core.MediaAudit) bool { for _, m := range media { if strings.EqualFold(strings.TrimSpace(m.Type), "image") { return true } } return false } func isCanonicalNoAnswer(answer string) bool { return strings.TrimSpace(answer) == canonicalNoAnswer } func maxScore(refs []core.Reference) float64 { var max float64 for _, r := range refs { if r.Score > max { max = r.Score } } return max }