403 lines
11 KiB
Go
403 lines
11 KiB
Go
package coverage
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
|
|
)
|
|
|
|
func TestEvaluatorVersion(t *testing.T) {
|
|
e := NewEvaluator()
|
|
if e.Version() != "v5" {
|
|
t.Errorf("Evaluator version = %q, want v5", e.Version())
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_RejectsFailedTurn(t *testing.T) {
|
|
e := NewEvaluator()
|
|
// Failed turn should be rejected/return error
|
|
_, err := e.Evaluate(core.Turn{Status: "failed"})
|
|
if err == nil {
|
|
t.Fatal("expected error for failed turn, got nil")
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_RejectsProcessingTurn(t *testing.T) {
|
|
e := NewEvaluator()
|
|
_, err := e.Evaluate(core.Turn{Status: "processing"})
|
|
if err == nil {
|
|
t.Fatal("expected error for processing turn, got nil")
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_UncoveredCanonicalNoAnswerWithoutReferences(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: canonicalNoAnswer,
|
|
KnowledgeSearched: true,
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
|
|
t.Fatalf("verdict/reason = %s/%s, want %s/%s", d.Verdict, d.ReasonCode, VerdictUncovered, ReasonUncoveredExplicitNoAnswer)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_NovaPartialNotFoundWithSevenReferencesIsCovered(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "nova-partial-turn",
|
|
Status: "succeeded",
|
|
Answer: "已找到Nova系列的相关资料;未找到VX500 2.5D视觉一体方案的明确说明。",
|
|
References: []core.Reference{
|
|
{ID: "r1"}, {ID: "r2"}, {ID: "r3"}, {ID: "r4"},
|
|
{ID: "r5"}, {ID: "r6"}, {ID: "r7"},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Verdict != VerdictCovered || d.ReasonCode != ReasonCoveredHasReferences || d.ReferenceCount != 7 {
|
|
t.Fatalf("verdict/reason/references = %s/%s/%d, want %s/%s/7", d.Verdict, d.ReasonCode, d.ReferenceCount, VerdictCovered, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_UncoveredNoReferences(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "这是一个正常的回答,但没有任何引用",
|
|
References: []core.Reference{},
|
|
KnowledgeSearched: true,
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictUncovered {
|
|
t.Errorf("verdict = %s, want uncovered", d.Verdict)
|
|
}
|
|
if d.ReasonCode != ReasonUncoveredNoReferences {
|
|
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredNoReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_UncoveredNilReferences(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "这是一个正常的回答,但引用为nil",
|
|
KnowledgeSearched: true,
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictUncovered {
|
|
t.Errorf("verdict = %s, want uncovered", d.Verdict)
|
|
}
|
|
if d.ReasonCode != ReasonUncoveredNoReferences {
|
|
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredNoReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_CoveredHasReferences(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "根据知识库中的信息,这个问题的答案是...",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.75},
|
|
{ID: "r2", Score: 0.92},
|
|
{ID: "r3", Score: 0.60},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Errorf("verdict = %s, want covered", d.Verdict)
|
|
}
|
|
if d.ReasonCode != ReasonCoveredHasReferences {
|
|
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_TopScore(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "根据知识库中的信息...",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.75},
|
|
{ID: "r2", Score: 0.92},
|
|
{ID: "r3", Score: 0.60},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.TopScore != 0.92 {
|
|
t.Errorf("top_score = %f, want 0.92", d.TopScore)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_ReferenceCount(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "根据知识库中的信息...",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.75},
|
|
{ID: "r2", Score: 0.92},
|
|
{ID: "r3", Score: 0.60},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.ReferenceCount != 3 {
|
|
t.Fatalf("reference_count = %d, want 3", d.ReferenceCount)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_TopScoreZeroWhenNoReferences(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "没有引用",
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.TopScore != 0 {
|
|
t.Errorf("top_score = %f, want 0", d.TopScore)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_ReferencesTrumpNaturalLanguageNoAnswer(t *testing.T) {
|
|
// References are durable coverage evidence; a natural-language phrase must
|
|
// not override them.
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "未找到相关结果,请换个方式提问",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.99},
|
|
{ID: "r2", Score: 0.88},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Errorf("answer with references: verdict = %s, want covered", d.Verdict)
|
|
}
|
|
if d.ReasonCode != ReasonCoveredHasReferences {
|
|
t.Errorf("answer with references: reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_NoAnswerPhraseMatchesInsideAnswer(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "根据现有资料,知识库中没有对应说明,请联系技术支持。",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.9},
|
|
},
|
|
}
|
|
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.ReasonCode != ReasonCoveredHasReferences {
|
|
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_ProductionUnavailableAuthorityPhraseWithReferencesIsCovered(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "cra-price-turn",
|
|
Status: "succeeded",
|
|
Answer: "当前暂无已发布的权威知识明确说明CRA系列机器人的价格信息。若需要获取对应报价,建议联系越疆官方销售渠道或技术支持进行咨询。",
|
|
References: []core.Reference{
|
|
{ID: "ref-1", Score: 0.0164},
|
|
},
|
|
}
|
|
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("Evaluate: %v", err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictCovered)
|
|
}
|
|
if d.ReasonCode != ReasonCoveredHasReferences {
|
|
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_RetrievalGapRequiresHumanEscalation(t *testing.T) {
|
|
e := NewEvaluator()
|
|
cases := []struct {
|
|
name string
|
|
answer string
|
|
verdict string
|
|
reason string
|
|
}{
|
|
{
|
|
name: "retrieval wording with evidence remains covered",
|
|
answer: "当前暂未查询到Nova系列机器人的公开销售价格信息。\n\n如需获取报价,建议您联系越疆科技官方销售渠道或对应区域的销售工程师咨询具体价格方案。",
|
|
verdict: VerdictCovered,
|
|
reason: ReasonCoveredHasReferences,
|
|
},
|
|
{
|
|
name: "retrieval gap without escalation is a valid negative result",
|
|
answer: "系统暂未查询到异常记录,设备运行正常。",
|
|
verdict: VerdictCovered,
|
|
reason: ReasonCoveredHasReferences,
|
|
},
|
|
{
|
|
name: "human escalation alone is not a knowledge gap",
|
|
answer: "设备状态正常,如需维护服务请联系技术支持。",
|
|
verdict: VerdictCovered,
|
|
reason: ReasonCoveredHasReferences,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
turn := core.Turn{
|
|
ID: "nova-price-turn",
|
|
Status: "succeeded",
|
|
Answer: tc.answer,
|
|
References: []core.Reference{
|
|
{ID: "ref-1", Score: 0.0164},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("Evaluate: %v", err)
|
|
}
|
|
if d.Verdict != tc.verdict || d.ReasonCode != tc.reason {
|
|
t.Fatalf("verdict/reason = %s/%s, want %s/%s", d.Verdict, d.ReasonCode, tc.verdict, tc.reason)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
|
|
e := NewEvaluator()
|
|
answers := []string{
|
|
"没有相关限制,设备可以正常运行。",
|
|
"当前暂无已发布价格调整计划,现有价格继续执行。",
|
|
}
|
|
for _, answer := range answers {
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: answer,
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.9},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("answer %q: unexpected error: %v", answer, err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Fatalf("answer %q: verdict = %s, want covered", answer, d.Verdict)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_NoScoreThreshold(t *testing.T) {
|
|
// The evaluator must never use score threshold to determine covered/uncovered.
|
|
// Even with very low score references, if there are references and no explicit
|
|
// no-answer, it should be covered.
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "这是一个回答,引用分数很低",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.01},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Errorf("verdict = %s, want covered (score threshold is not used)", d.Verdict)
|
|
}
|
|
if d.ReasonCode != ReasonCoveredHasReferences {
|
|
t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_EmptyAnswerCovered(t *testing.T) {
|
|
// Empty answer is not a no-answer phrase. With references, it's covered.
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "test-turn",
|
|
Status: "succeeded",
|
|
Answer: "",
|
|
References: []core.Reference{
|
|
{ID: "r1", Score: 0.5},
|
|
},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.Verdict != VerdictCovered {
|
|
t.Errorf("verdict = %s, want covered", d.Verdict)
|
|
}
|
|
}
|
|
|
|
func TestEvaluator_DecisionHasTurnIdentity(t *testing.T) {
|
|
e := NewEvaluator()
|
|
turn := core.Turn{
|
|
ID: "turn-abc",
|
|
TenantKey: "tenant-1",
|
|
Channel: "dingtalk",
|
|
Status: "succeeded",
|
|
Answer: "正常回答",
|
|
References: []core.Reference{{ID: "r1", Score: 0.5}},
|
|
}
|
|
d, err := e.Evaluate(turn)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if d.TurnID != "turn-abc" {
|
|
t.Errorf("turn_id = %q, want turn-abc", d.TurnID)
|
|
}
|
|
if d.TenantKey != "tenant-1" {
|
|
t.Errorf("tenant_key = %q, want tenant-1", d.TenantKey)
|
|
}
|
|
if d.Channel != "dingtalk" {
|
|
t.Errorf("channel = %q, want dingtalk", d.Channel)
|
|
}
|
|
}
|