fix: detect retrieval gaps with human escalation
This commit is contained in:
@@ -18,6 +18,24 @@ var noAnswerPhrases = []string{
|
||||
"知识库中没有",
|
||||
}
|
||||
|
||||
// retrievalGapPhrases are only considered a no-answer signal when the answer
|
||||
// also directs the user to a human channel. Requiring both signals avoids
|
||||
// misclassifying valid negative results such as "未查询到异常记录".
|
||||
var retrievalGapPhrases = []string{
|
||||
"未查询到",
|
||||
"未检索到",
|
||||
"未能查询到",
|
||||
"未能检索到",
|
||||
}
|
||||
|
||||
var humanEscalationPhrases = []string{
|
||||
"建议您联系",
|
||||
"建议联系",
|
||||
"请联系",
|
||||
"联系技术支持",
|
||||
"联系官方销售",
|
||||
}
|
||||
|
||||
// Evaluator classifies a successful turn into a coverage verdict.
|
||||
type Evaluator struct{}
|
||||
|
||||
@@ -97,6 +115,15 @@ func isNoAnswer(answer string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return containsAny(lower, retrievalGapPhrases) && containsAny(lower, humanEscalationPhrases)
|
||||
}
|
||||
|
||||
func containsAny(value string, phrases []string) bool {
|
||||
for _, phrase := range phrases {
|
||||
if strings.Contains(value, strings.ToLower(phrase)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
func TestEvaluatorVersion(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
if e.Version() != "v2" {
|
||||
t.Errorf("Evaluator version = %q, want v2", e.Version())
|
||||
if e.Version() != "v3" {
|
||||
t.Errorf("Evaluator version = %q, want v3", e.Version())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,6 +251,54 @@ func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluator_RetrievalGapRequiresHumanEscalation(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
cases := []struct {
|
||||
name string
|
||||
answer string
|
||||
verdict string
|
||||
reason string
|
||||
}{
|
||||
{
|
||||
name: "production NOVA price answer",
|
||||
answer: "当前暂未查询到Nova系列机器人的公开销售价格信息。\n\n如需获取报价,建议您联系越疆科技官方销售渠道或对应区域的销售工程师咨询具体价格方案。",
|
||||
verdict: VerdictUncovered,
|
||||
reason: ReasonUncoveredExplicitNoAnswer,
|
||||
},
|
||||
{
|
||||
name: "retrieval gap without escalation is a valid negative result",
|
||||
answer: "系统暂未查询到异常记录,设备运行正常。",
|
||||
verdict: VerdictCovered,
|
||||
reason: ReasonCoveredHasReferences,
|
||||
},
|
||||
{
|
||||
name: "human escalation alone is not a knowledge gap",
|
||||
answer: "设备状态正常,如需维护服务请联系技术支持。",
|
||||
verdict: VerdictCovered,
|
||||
reason: ReasonCoveredHasReferences,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
turn := core.Turn{
|
||||
ID: "nova-price-turn",
|
||||
Status: "succeeded",
|
||||
Answer: tc.answer,
|
||||
References: []core.Reference{
|
||||
{ID: "ref-1", Score: 0.0164},
|
||||
},
|
||||
}
|
||||
d, err := e.Evaluate(turn)
|
||||
if err != nil {
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
if d.Verdict != tc.verdict || d.ReasonCode != tc.reason {
|
||||
t.Fatalf("verdict/reason = %s/%s, want %s/%s", d.Verdict, d.ReasonCode, tc.verdict, tc.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
answers := []string{
|
||||
|
||||
@@ -31,7 +31,7 @@ const (
|
||||
)
|
||||
|
||||
// EvaluatorVersion is the current version of the evaluator.
|
||||
const EvaluatorVersion = "v2"
|
||||
const EvaluatorVersion = "v3"
|
||||
|
||||
var (
|
||||
ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded")
|
||||
|
||||
Reference in New Issue
Block a user