dobotai-platform/internal/coverage/evaluator_test.go

package coverage

import (
	"testing"

	"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
)

func TestEvaluatorVersion(t *testing.T) {
	e := NewEvaluator()
	if e.Version() != "v4" {
		t.Errorf("Evaluator version = %q, want v4", e.Version())
	}
}

func TestEvaluator_RejectsFailedTurn(t *testing.T) {
	e := NewEvaluator()
	// Failed turn should be rejected/return error
	_, err := e.Evaluate(core.Turn{Status: "failed"})
	if err == nil {
		t.Fatal("expected error for failed turn, got nil")
	}
}

func TestEvaluator_RejectsProcessingTurn(t *testing.T) {
	e := NewEvaluator()
	_, err := e.Evaluate(core.Turn{Status: "processing"})
	if err == nil {
		t.Fatal("expected error for processing turn, got nil")
	}
}

func TestEvaluator_UncoveredExplicitNoAnswer(t *testing.T) {
	e := NewEvaluator()
	noAnswerPhrases := []string{
		"未找到相关结果",
		"没有找到你需要的信息",
		"无法确定该问题的答案",
		"暂无相关信息",
		"知识库中没有找到相关内容",
		"这个问题知识库中没有可用的信息",
	}
	for _, answer := range noAnswerPhrases {
		turn := core.Turn{
			ID:     "test-turn",
			Status: "succeeded",
			Answer: answer,
			References: []core.Reference{
				{ID: "r1", Score: 0.9},
			},
		}
		d, err := e.Evaluate(turn)
		if err != nil {
			t.Fatalf("unexpected error for answer %q: %v", answer, err)
		}
		if d.Verdict != VerdictUncovered {
			t.Errorf("answer %q: verdict = %s, want uncovered", answer, d.Verdict)
		}
		if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
			t.Errorf("answer %q: reason_code = %s, want %s", answer, d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
		}
	}
}

func TestEvaluator_ProductionMissingProductPhraseTrumpsReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "missing-product-turn",
		Status: "succeeded",
		Answer: "根据提供的检索信息，无法提供对应手册。原因：产品不存在于资料中；检索资料未提及该产品。建议联系技术支持部门。",
		References: []core.Reference{
			{ID: "unrelated-1", Score: 0.395940},
			{ID: "unrelated-2", Score: 0.311595},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatal(err)
	}
	if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
		t.Fatalf("verdict=%s reason=%s", d.Verdict, d.ReasonCode)
	}
}

func TestEvaluator_UncoveredNoReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:                "test-turn",
		Status:            "succeeded",
		Answer:            "这是一个正常的回答，但没有任何引用",
		References:        []core.Reference{},
		KnowledgeSearched: true,
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictUncovered {
		t.Errorf("verdict = %s, want uncovered", d.Verdict)
	}
	if d.ReasonCode != ReasonUncoveredNoReferences {
		t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredNoReferences)
	}
}

func TestEvaluator_UncoveredNilReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:                "test-turn",
		Status:            "succeeded",
		Answer:            "这是一个正常的回答，但引用为nil",
		KnowledgeSearched: true,
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictUncovered {
		t.Errorf("verdict = %s, want uncovered", d.Verdict)
	}
	if d.ReasonCode != ReasonUncoveredNoReferences {
		t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredNoReferences)
	}
}

func TestEvaluator_CoveredHasReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "根据知识库中的信息，这个问题的答案是...",
		References: []core.Reference{
			{ID: "r1", Score: 0.75},
			{ID: "r2", Score: 0.92},
			{ID: "r3", Score: 0.60},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictCovered {
		t.Errorf("verdict = %s, want covered", d.Verdict)
	}
	if d.ReasonCode != ReasonCoveredHasReferences {
		t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
	}
}

func TestEvaluator_TopScore(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "根据知识库中的信息...",
		References: []core.Reference{
			{ID: "r1", Score: 0.75},
			{ID: "r2", Score: 0.92},
			{ID: "r3", Score: 0.60},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.TopScore != 0.92 {
		t.Errorf("top_score = %f, want 0.92", d.TopScore)
	}
}

func TestEvaluator_ReferenceCount(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "根据知识库中的信息...",
		References: []core.Reference{
			{ID: "r1", Score: 0.75},
			{ID: "r2", Score: 0.92},
			{ID: "r3", Score: 0.60},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.ReferenceCount != 3 {
		t.Fatalf("reference_count = %d, want 3", d.ReferenceCount)
	}
}

func TestEvaluator_TopScoreZeroWhenNoReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "没有引用",
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.TopScore != 0 {
		t.Errorf("top_score = %f, want 0", d.TopScore)
	}
}

func TestEvaluator_NoAnswerTrumpsReferences(t *testing.T) {
	// When answer contains explicit "no answer" phrase, it should be uncovered
	// even if references exist. No-answer detection takes priority.
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "未找到相关结果，请换个方式提问",
		References: []core.Reference{
			{ID: "r1", Score: 0.99},
			{ID: "r2", Score: 0.88},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictUncovered {
		t.Errorf("no-answer with references: verdict = %s, want uncovered", d.Verdict)
	}
	if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
		t.Errorf("no-answer with references: reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
	}
}

func TestEvaluator_NoAnswerPhraseMatchesInsideAnswer(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "根据现有资料，知识库中没有对应说明，请联系技术支持。",
		References: []core.Reference{
			{ID: "r1", Score: 0.9},
		},
	}

	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
		t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
	}
}

func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "cra-price-turn",
		Status: "succeeded",
		Answer: "当前暂无已发布的权威知识明确说明CRA系列机器人的价格信息。若需要获取对应报价，建议联系越疆官方销售渠道或技术支持进行咨询。",
		References: []core.Reference{
			{ID: "ref-1", Score: 0.0164},
		},
	}

	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if d.Verdict != VerdictUncovered {
		t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictUncovered)
	}
	if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
		t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
	}
}

func TestEvaluator_RetrievalGapRequiresHumanEscalation(t *testing.T) {
	e := NewEvaluator()
	cases := []struct {
		name    string
		answer  string
		verdict string
		reason  string
	}{
		{
			name:    "production NOVA price answer",
			answer:  "当前暂未查询到Nova系列机器人的公开销售价格信息。\n\n如需获取报价，建议您联系越疆科技官方销售渠道或对应区域的销售工程师咨询具体价格方案。",
			verdict: VerdictUncovered,
			reason:  ReasonUncoveredExplicitNoAnswer,
		},
		{
			name:    "retrieval gap without escalation is a valid negative result",
			answer:  "系统暂未查询到异常记录，设备运行正常。",
			verdict: VerdictCovered,
			reason:  ReasonCoveredHasReferences,
		},
		{
			name:    "human escalation alone is not a knowledge gap",
			answer:  "设备状态正常，如需维护服务请联系技术支持。",
			verdict: VerdictCovered,
			reason:  ReasonCoveredHasReferences,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			turn := core.Turn{
				ID:     "nova-price-turn",
				Status: "succeeded",
				Answer: tc.answer,
				References: []core.Reference{
					{ID: "ref-1", Score: 0.0164},
				},
			}
			d, err := e.Evaluate(turn)
			if err != nil {
				t.Fatalf("Evaluate: %v", err)
			}
			if d.Verdict != tc.verdict || d.ReasonCode != tc.reason {
				t.Fatalf("verdict/reason = %s/%s, want %s/%s", d.Verdict, d.ReasonCode, tc.verdict, tc.reason)
			}
		})
	}
}

func TestEvaluator_DoesNotExpandBeyondApprovedNoAnswerPhrases(t *testing.T) {
	e := NewEvaluator()
	answers := []string{
		"没有相关限制，设备可以正常运行。",
		"当前暂无已发布价格调整计划，现有价格继续执行。",
	}
	for _, answer := range answers {
		turn := core.Turn{
			ID:     "test-turn",
			Status: "succeeded",
			Answer: answer,
			References: []core.Reference{
				{ID: "r1", Score: 0.9},
			},
		}
		d, err := e.Evaluate(turn)
		if err != nil {
			t.Fatalf("answer %q: unexpected error: %v", answer, err)
		}
		if d.Verdict != VerdictCovered {
			t.Fatalf("answer %q: verdict = %s, want covered", answer, d.Verdict)
		}
	}
}

func TestEvaluator_NoScoreThreshold(t *testing.T) {
	// The evaluator must never use score threshold to determine covered/uncovered.
	// Even with very low score references, if there are references and no explicit
	// no-answer, it should be covered.
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "这是一个回答，引用分数很低",
		References: []core.Reference{
			{ID: "r1", Score: 0.01},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictCovered {
		t.Errorf("verdict = %s, want covered (score threshold is not used)", d.Verdict)
	}
	if d.ReasonCode != ReasonCoveredHasReferences {
		t.Errorf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
	}
}

func TestEvaluator_EmptyAnswerCovered(t *testing.T) {
	// Empty answer is not a no-answer phrase. With references, it's covered.
	e := NewEvaluator()
	turn := core.Turn{
		ID:     "test-turn",
		Status: "succeeded",
		Answer: "",
		References: []core.Reference{
			{ID: "r1", Score: 0.5},
		},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.Verdict != VerdictCovered {
		t.Errorf("verdict = %s, want covered", d.Verdict)
	}
}

func TestEvaluator_DecisionHasTurnIdentity(t *testing.T) {
	e := NewEvaluator()
	turn := core.Turn{
		ID:         "turn-abc",
		TenantKey:  "tenant-1",
		Channel:    "dingtalk",
		Status:     "succeeded",
		Answer:     "正常回答",
		References: []core.Reference{{ID: "r1", Score: 0.5}},
	}
	d, err := e.Evaluate(turn)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if d.TurnID != "turn-abc" {
		t.Errorf("turn_id = %q, want turn-abc", d.TurnID)
	}
	if d.TenantKey != "tenant-1" {
		t.Errorf("tenant_key = %q, want tenant-1", d.TenantKey)
	}
	if d.Channel != "dingtalk" {
		t.Errorf("channel = %q, want dingtalk", d.Channel)
	}
}