112 lines
3.0 KiB
Go
112 lines
3.0 KiB
Go
package coverage
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
|
|
)
|
|
|
|
// noAnswerPhrases lists explicit "no answer" phrases that the knowledge base
|
|
// returns when it cannot find relevant information. These are matched as
|
|
// substrings after trimming whitespace.
|
|
var noAnswerPhrases = []string{
|
|
"未找到",
|
|
"没有找到",
|
|
"无法确定",
|
|
"暂无相关",
|
|
"暂无已发布的权威知识",
|
|
"知识库中没有",
|
|
}
|
|
|
|
// Evaluator classifies a successful turn into a coverage verdict.
|
|
type Evaluator struct{}
|
|
|
|
// NewEvaluator creates the current Evaluator.
|
|
func NewEvaluator() *Evaluator {
|
|
return &Evaluator{}
|
|
}
|
|
|
|
// Version returns the current evaluator version.
|
|
func (e *Evaluator) Version() string {
|
|
return EvaluatorVersion
|
|
}
|
|
|
|
// Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation
|
|
// rules (in priority order):
|
|
// 1. Failed/processing turns → ErrTurnNotSucceeded
|
|
// 2. Answer contains an explicit "no answer" phrase → uncovered_explicit_no_answer
|
|
// 3. Zero references (nil or empty) → uncovered_no_references
|
|
// 4. Has references → covered_has_references
|
|
//
|
|
// v1 never uses a score threshold. TopScore captures the highest reference score
|
|
// (0 if no references).
|
|
func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
|
|
if turn.Status != "succeeded" {
|
|
return Decision{}, ErrTurnNotSucceeded
|
|
}
|
|
|
|
// Normalize answer for no-answer phrase detection.
|
|
trimmed := strings.TrimSpace(turn.Answer)
|
|
|
|
// Check for explicit no-answer phrases first — this takes priority
|
|
// over reference presence.
|
|
if isNoAnswer(trimmed) {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictUncovered,
|
|
ReasonCode: ReasonUncoveredExplicitNoAnswer,
|
|
ReferenceCount: len(turn.References),
|
|
TopScore: maxScore(turn.References),
|
|
}, nil
|
|
}
|
|
|
|
// No references → uncovered.
|
|
if len(turn.References) == 0 {
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictUncovered,
|
|
ReasonCode: ReasonUncoveredNoReferences,
|
|
ReferenceCount: 0,
|
|
TopScore: 0,
|
|
}, nil
|
|
}
|
|
|
|
// Has references → covered (v1 never uses score threshold).
|
|
return Decision{
|
|
TurnID: turn.ID,
|
|
TenantKey: turn.TenantKey,
|
|
Channel: turn.Channel,
|
|
EvaluatorVersion: EvaluatorVersion,
|
|
Verdict: VerdictCovered,
|
|
ReasonCode: ReasonCoveredHasReferences,
|
|
ReferenceCount: len(turn.References),
|
|
TopScore: maxScore(turn.References),
|
|
}, nil
|
|
}
|
|
|
|
func isNoAnswer(answer string) bool {
|
|
lower := strings.ToLower(answer)
|
|
for _, phrase := range noAnswerPhrases {
|
|
if strings.Contains(lower, strings.ToLower(phrase)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func maxScore(refs []core.Reference) float64 {
|
|
var max float64
|
|
for _, r := range refs {
|
|
if r.Score > max {
|
|
max = r.Score
|
|
}
|
|
}
|
|
return max
|
|
}
|