fix: detect unavailable authoritative knowledge answers

This commit is contained in:
姚章浩
2026-07-22 13:21:59 +08:00
parent 774ccd9454
commit d29b28be13
3 changed files with 43 additions and 14 deletions
+3 -2
View File
@@ -14,18 +14,19 @@ var noAnswerPhrases = []string{
"没有找到",
"无法确定",
"暂无相关",
"暂无已发布的权威知识",
"知识库中没有",
}
// Evaluator classifies a successful turn into a coverage verdict.
type Evaluator struct{}
// NewEvaluator creates a v1 Evaluator.
// NewEvaluator creates the current Evaluator.
func NewEvaluator() *Evaluator {
return &Evaluator{}
}
// Version returns the evaluator version, always "v1".
// Version returns the current evaluator version.
func (e *Evaluator) Version() string {
return EvaluatorVersion
}
+39 -11
View File
@@ -8,8 +8,8 @@ import (
func TestEvaluatorVersion(t *testing.T) {
e := NewEvaluator()
if e.Version() != "v1" {
t.Errorf("Evaluator version = %q, want v1", e.Version())
if e.Version() != "v2" {
t.Errorf("Evaluator version = %q, want v2", e.Version())
}
}
@@ -228,28 +228,56 @@ func TestEvaluator_NoAnswerPhraseMatchesInsideAnswer(t *testing.T) {
}
}
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "test-turn",
ID: "cra-price-turn",
Status: "succeeded",
Answer: "没有相关限制,设备可以正常运行。",
Answer: "当前暂无已发布的权威知识明确说明CRA系列机器人的价格信息。若需要获取对应报价,建议联系越疆官方销售渠道或技术支持进行咨询。",
References: []core.Reference{
{ID: "r1", Score: 0.9},
{ID: "ref-1", Score: 0.0164},
},
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatalf("unexpected error: %v", err)
t.Fatalf("Evaluate: %v", err)
}
if d.Verdict != VerdictCovered {
t.Fatalf("verdict = %s, want covered", d.Verdict)
if d.Verdict != VerdictUncovered {
t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictUncovered)
}
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
}
}
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
e := NewEvaluator()
answers := []string{
"没有相关限制,设备可以正常运行。",
"当前暂无已发布价格调整计划,现有价格继续执行。",
}
for _, answer := range answers {
turn := core.Turn{
ID: "test-turn",
Status: "succeeded",
Answer: answer,
References: []core.Reference{
{ID: "r1", Score: 0.9},
},
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatalf("answer %q: unexpected error: %v", answer, err)
}
if d.Verdict != VerdictCovered {
t.Fatalf("answer %q: verdict = %s, want covered", answer, d.Verdict)
}
}
}
func TestEvaluator_NoScoreThreshold(t *testing.T) {
// v1 evaluator must never use score threshold to determine covered/uncovered.
// The evaluator must never use score threshold to determine covered/uncovered.
// Even with very low score references, if there are references and no explicit
// no-answer, it should be covered.
e := NewEvaluator()
@@ -266,7 +294,7 @@ func TestEvaluator_NoScoreThreshold(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
if d.Verdict != VerdictCovered {
t.Errorf("verdict = %s, want covered (v1 never uses score threshold)", d.Verdict)
t.Errorf("verdict = %s, want covered (score threshold is not used)", d.Verdict)
}
if d.ReasonCode != ReasonCoveredHasReferences {
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
+1 -1
View File
@@ -31,7 +31,7 @@ const (
)
// EvaluatorVersion is the current version of the evaluator.
const EvaluatorVersion = "v1"
const EvaluatorVersion = "v2"
var (
ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded")