Files
dobotai-platform/internal/coverage/evaluator.go
T

187 lines
6.0 KiB
Go

package coverage
import (
"strings"
"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
)
// canonicalNoAnswer is the only response that carries the explicit-no-answer
// machine contract. It must be the complete answer, never a substring match:
// an otherwise supported answer may legitimately say that one specific detail
// was not found.
const canonicalNoAnswer = "当前暂无已发布的权威知识,暂不建议操作。"
// Evaluator classifies a successful turn into a coverage verdict.
type Evaluator struct{}
// NewEvaluator creates the current Evaluator.
func NewEvaluator() *Evaluator {
return &Evaluator{}
}
// Version returns the current evaluator version.
func (e *Evaluator) Version() string {
return EvaluatorVersion
}
// Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation
// rules (in priority order):
// 1. Failed/processing turns → ErrTurnNotSucceeded
// 2. Has references → covered_has_references (references are the coverage
// evidence; wording in a supported answer never overrides them)
// 3. Complete canonical no-answer response with no references
// → uncovered_explicit_no_answer
// 4. Image turns (image media present) are classified by whether the question
// actually triggered a knowledge-base search:
// - no references + knowledge searched → uncovered_no_references
// - no references + no knowledge search → covered_visual_recognition
// 5. Text turns with zero references are classified by whether the question
// actually triggered a knowledge-base search:
// - knowledge searched → uncovered_no_references
// - no knowledge search → covered_non_knowledge_response
//
// v5 never uses a score threshold. TopScore captures the highest reference score
// (0 if no references).
func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
if turn.Status != "succeeded" {
return Decision{}, ErrTurnNotSucceeded
}
// Normalize answer for no-answer phrase detection.
trimmed := strings.TrimSpace(turn.Answer)
// References are the durable coverage evidence. A supported answer may
// correctly note that one sub-detail is absent, so its wording cannot turn
// the entire turn into a knowledge gap.
if len(turn.References) > 0 {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredHasReferences,
ReferenceCount: len(turn.References),
TopScore: maxScore(turn.References),
}, nil
}
// An explicit abstention is a stable machine contract only when it is the
// complete answer. Do not use broad natural-language substring matching.
if isCanonicalNoAnswer(trimmed) {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictUncovered,
ReasonCode: ReasonUncoveredExplicitNoAnswer,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
// Image turns are classified by whether knowledge search occurred.
if hasImageMedia(turn.Media) {
if len(turn.References) > 0 {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredHasReferences,
ReferenceCount: len(turn.References),
TopScore: maxScore(turn.References),
}, nil
}
if turn.KnowledgeSearched {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictUncovered,
ReasonCode: ReasonUncoveredNoReferences,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredVisualRecognition,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
// Text turns without references are only uncovered if they actually
// triggered knowledge search. This prevents greetings and other normal
// non-knowledge responses from generating artificial support cases.
if len(turn.References) == 0 {
if !turn.KnowledgeSearched {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredNonKnowledgeResponse,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictUncovered,
ReasonCode: ReasonUncoveredNoReferences,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
// Has references → covered (v1 never uses score threshold).
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredHasReferences,
ReferenceCount: len(turn.References),
TopScore: maxScore(turn.References),
}, nil
}
// hasImageMedia reports whether the turn carried at least one image input.
func hasImageMedia(media []core.MediaAudit) bool {
for _, m := range media {
if strings.EqualFold(strings.TrimSpace(m.Type), "image") {
return true
}
}
return false
}
func isCanonicalNoAnswer(answer string) bool {
return strings.TrimSpace(answer) == canonicalNoAnswer
}
func maxScore(refs []core.Reference) float64 {
var max float64
for _, r := range refs {
if r.Score > max {
max = r.Score
}
}
return max
}