fix: detect unavailable authoritative knowledge answers
This commit is contained in:
@@ -14,18 +14,19 @@ var noAnswerPhrases = []string{
|
||||
"没有找到",
|
||||
"无法确定",
|
||||
"暂无相关",
|
||||
"暂无已发布的权威知识",
|
||||
"知识库中没有",
|
||||
}
|
||||
|
||||
// Evaluator classifies a successful turn into a coverage verdict.
|
||||
type Evaluator struct{}
|
||||
|
||||
// NewEvaluator creates a v1 Evaluator.
|
||||
// NewEvaluator creates the current Evaluator.
|
||||
func NewEvaluator() *Evaluator {
|
||||
return &Evaluator{}
|
||||
}
|
||||
|
||||
// Version returns the evaluator version, always "v1".
|
||||
// Version returns the current evaluator version.
|
||||
func (e *Evaluator) Version() string {
|
||||
return EvaluatorVersion
|
||||
}
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
func TestEvaluatorVersion(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
if e.Version() != "v1" {
|
||||
t.Errorf("Evaluator version = %q, want v1", e.Version())
|
||||
if e.Version() != "v2" {
|
||||
t.Errorf("Evaluator version = %q, want v2", e.Version())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,28 +228,56 @@ func TestEvaluator_NoAnswerPhraseMatchesInsideAnswer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
|
||||
func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
turn := core.Turn{
|
||||
ID: "test-turn",
|
||||
ID: "cra-price-turn",
|
||||
Status: "succeeded",
|
||||
Answer: "没有相关限制,设备可以正常运行。",
|
||||
Answer: "当前暂无已发布的权威知识明确说明CRA系列机器人的价格信息。若需要获取对应报价,建议联系越疆官方销售渠道或技术支持进行咨询。",
|
||||
References: []core.Reference{
|
||||
{ID: "r1", Score: 0.9},
|
||||
{ID: "ref-1", Score: 0.0164},
|
||||
},
|
||||
}
|
||||
|
||||
d, err := e.Evaluate(turn)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
if d.Verdict != VerdictCovered {
|
||||
t.Fatalf("verdict = %s, want covered", d.Verdict)
|
||||
if d.Verdict != VerdictUncovered {
|
||||
t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictUncovered)
|
||||
}
|
||||
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
|
||||
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
answers := []string{
|
||||
"没有相关限制,设备可以正常运行。",
|
||||
"当前暂无已发布价格调整计划,现有价格继续执行。",
|
||||
}
|
||||
for _, answer := range answers {
|
||||
turn := core.Turn{
|
||||
ID: "test-turn",
|
||||
Status: "succeeded",
|
||||
Answer: answer,
|
||||
References: []core.Reference{
|
||||
{ID: "r1", Score: 0.9},
|
||||
},
|
||||
}
|
||||
d, err := e.Evaluate(turn)
|
||||
if err != nil {
|
||||
t.Fatalf("answer %q: unexpected error: %v", answer, err)
|
||||
}
|
||||
if d.Verdict != VerdictCovered {
|
||||
t.Fatalf("answer %q: verdict = %s, want covered", answer, d.Verdict)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluator_NoScoreThreshold(t *testing.T) {
|
||||
// v1 evaluator must never use score threshold to determine covered/uncovered.
|
||||
// The evaluator must never use score threshold to determine covered/uncovered.
|
||||
// Even with very low score references, if there are references and no explicit
|
||||
// no-answer, it should be covered.
|
||||
e := NewEvaluator()
|
||||
@@ -266,7 +294,7 @@ func TestEvaluator_NoScoreThreshold(t *testing.T) {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if d.Verdict != VerdictCovered {
|
||||
t.Errorf("verdict = %s, want covered (v1 never uses score threshold)", d.Verdict)
|
||||
t.Errorf("verdict = %s, want covered (score threshold is not used)", d.Verdict)
|
||||
}
|
||||
if d.ReasonCode != ReasonCoveredHasReferences {
|
||||
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
||||
|
||||
@@ -31,7 +31,7 @@ const (
|
||||
)
|
||||
|
||||
// EvaluatorVersion is the current version of the evaluator.
|
||||
const EvaluatorVersion = "v1"
|
||||
const EvaluatorVersion = "v2"
|
||||
|
||||
var (
|
||||
ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded")
|
||||
|
||||
Reference in New Issue
Block a user