187 lines
6.0 KiB
Go
187 lines
6.0 KiB
Go
package coverage
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
|
|
)
|
|
|
|
// canonicalNoAnswer is the only response that carries the explicit-no-answer
|
|
// machine contract. It must be the complete answer, never a substring match:
|
|
// an otherwise supported answer may legitimately say that one specific detail
|
|
// was not found.
|
|
const canonicalNoAnswer = "当前暂无已发布的权威知识,暂不建议操作。"
|
|
|
|
// Evaluator classifies a successful turn into a coverage verdict.
|
|
type Evaluator struct{}
|
|
|
|
// NewEvaluator creates the current Evaluator.
|
|
func NewEvaluator() *Evaluator {
|
|
return &Evaluator{}
|
|
}
|
|
|
|
// Version returns the current evaluator version.
|
|
func (e *Evaluator) Version() string {
|
|
return EvaluatorVersion
|
|
}
|
|
|
|
// Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation
|
|
// rules (in priority order):
|
|
// 1. Failed/processing turns → ErrTurnNotSucceeded
|
|
// 2. Has references → covered_has_references (references are the coverage
|
|
// evidence; wording in a supported answer never overrides them)
|
|
// 3. Complete canonical no-answer response with no references
|
|
// → uncovered_explicit_no_answer
|
|
// 4. Image turns (image media present) are classified by whether the question
|
|
// actually triggered a knowledge-base search:
|
|
// - no references + knowledge searched → uncovered_no_references
|
|
// - no references + no knowledge search → covered_visual_recognition
|
|
// 5. Text turns with zero references are classified by whether the question
|
|
// actually triggered a knowledge-base search:
|
|
// - knowledge searched → uncovered_no_references
|
|
// - no knowledge search → covered_non_knowledge_response
|
|
//
|
|
// v5 never uses a score threshold. TopScore captures the highest reference score
|
|
// (0 if no references).
|
|
func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
|
|
if turn.Status != "succeeded" {
|
|
return Decision{}, ErrTurnNotSucceeded
|
|
}
|
|
|
|
// Normalize answer for no-answer phrase detection.
|
|
trimmed := strings.TrimSpace(turn.Answer)
|
|
|
|
// References are the durable coverage evidence. A supported answer may
|
|
// correctly note that one sub-detail is absent, so its wording cannot turn
|
|
// the entire turn into a knowledge gap.
|
|
if len(turn.References) > 0 {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredHasReferences,
|
|
ReferenceCount: len(turn.References),
|
|
TopScore: maxScore(turn.References),
|
|
}, nil
|
|
}
|
|
|
|
// An explicit abstention is a stable machine contract only when it is the
|
|
// complete answer. Do not use broad natural-language substring matching.
|
|
if isCanonicalNoAnswer(trimmed) {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictUncovered,
|
|
ReasonCode: ReasonUncoveredExplicitNoAnswer,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
|
|
// Image turns are classified by whether knowledge search occurred.
|
|
if hasImageMedia(turn.Media) {
|
|
if len(turn.References) > 0 {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredHasReferences,
|
|
ReferenceCount: len(turn.References),
|
|
TopScore: maxScore(turn.References),
|
|
}, nil
|
|
}
|
|
if turn.KnowledgeSearched {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictUncovered,
|
|
ReasonCode: ReasonUncoveredNoReferences,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredVisualRecognition,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
|
|
// Text turns without references are only uncovered if they actually
|
|
// triggered knowledge search. This prevents greetings and other normal
|
|
// non-knowledge responses from generating artificial support cases.
|
|
if len(turn.References) == 0 {
|
|
if !turn.KnowledgeSearched {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredNonKnowledgeResponse,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictUncovered,
|
|
ReasonCode: ReasonUncoveredNoReferences,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
|
|
// Has references → covered (v1 never uses score threshold).
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredHasReferences,
|
|
ReferenceCount: len(turn.References),
|
|
TopScore: maxScore(turn.References),
|
|
}, nil
|
|
}
|
|
|
|
// hasImageMedia reports whether the turn carried at least one image input.
|
|
func hasImageMedia(media []core.MediaAudit) bool {
|
|
for _, m := range media {
|
|
if strings.EqualFold(strings.TrimSpace(m.Type), "image") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isCanonicalNoAnswer(answer string) bool {
|
|
return strings.TrimSpace(answer) == canonicalNoAnswer
|
|
}
|
|
|
|
func maxScore(refs []core.Reference) float64 {
|
|
var max float64
|
|
for _, r := range refs {
|
|
if r.Score > max {
|
|
max = r.Score
|
|
}
|
|
}
|
|
return max
|
|
}
|