dobotai-platform/internal/coverage/evaluator.go

package coverage

import (
	"strings"

	"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
)

// noAnswerPhrases lists explicit "no answer" phrases that the knowledge base
// returns when it cannot find relevant information. These are matched as
// substrings after trimming whitespace.
var noAnswerPhrases = []string{
	"未找到",
	"没有找到",
	"无法确定",
	"暂无相关",
	"暂无已发布的权威知识",
	"知识库中没有",
	"产品不存在于资料中",
}

// retrievalGapPhrases are only considered a no-answer signal when the answer
// also directs the user to a human channel. Requiring both signals avoids
// misclassifying valid negative results such as "未查询到异常记录".
var retrievalGapPhrases = []string{
	"未查询到",
	"未检索到",
	"未能查询到",
	"未能检索到",
}

var humanEscalationPhrases = []string{
	"建议您联系",
	"建议联系",
	"请联系",
	"联系技术支持",
	"联系官方销售",
}

// Evaluator classifies a successful turn into a coverage verdict.
type Evaluator struct{}

// NewEvaluator creates the current Evaluator.
func NewEvaluator() *Evaluator {
	return &Evaluator{}
}

// Version returns the current evaluator version.
func (e *Evaluator) Version() string {
	return EvaluatorVersion
}

// Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation
// rules (in priority order):
//  1. Failed/processing turns → ErrTurnNotSucceeded
//  2. Answer contains an explicit "no answer" phrase → uncovered_explicit_no_answer
//  3. Image turns (image media present) are classified by whether the question
//     actually triggered a knowledge-base search:
//     - has references → covered_has_references
//     - no references + knowledge searched → uncovered_no_references
//     - no references + no knowledge search → covered_visual_recognition
//     (pure visual recognition is not a coverage miss)
//  4. Text turns with zero references are classified by whether the question
//     actually triggered a knowledge-base search:
//     - knowledge searched → uncovered_no_references
//     - no knowledge search → covered_non_knowledge_response
//  5. Has references → covered_has_references
//
// v1 never uses a score threshold. TopScore captures the highest reference score
// (0 if no references).
func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
	if turn.Status != "succeeded" {
		return Decision{}, ErrTurnNotSucceeded
	}

	// Normalize answer for no-answer phrase detection.
	trimmed := strings.TrimSpace(turn.Answer)

	// Check for explicit no-answer phrases first — this takes priority
	// over reference presence.
	if isNoAnswer(trimmed) {
		return Decision{
			TurnID:           turn.ID,
			TenantKey:        turn.TenantKey,
			Channel:          turn.Channel,
			EvaluatorVersion: EvaluatorVersion,
			Verdict:          VerdictUncovered,
			ReasonCode:       ReasonUncoveredExplicitNoAnswer,
			ReferenceCount:   len(turn.References),
			TopScore:         maxScore(turn.References),
		}, nil
	}

	// Image turns are classified by whether knowledge search occurred.
	if hasImageMedia(turn.Media) {
		if len(turn.References) > 0 {
			return Decision{
				TurnID:           turn.ID,
				TenantKey:        turn.TenantKey,
				Channel:          turn.Channel,
				EvaluatorVersion: EvaluatorVersion,
				Verdict:          VerdictCovered,
				ReasonCode:       ReasonCoveredHasReferences,
				ReferenceCount:   len(turn.References),
				TopScore:         maxScore(turn.References),
			}, nil
		}
		if turn.KnowledgeSearched {
			return Decision{
				TurnID:           turn.ID,
				TenantKey:        turn.TenantKey,
				Channel:          turn.Channel,
				EvaluatorVersion: EvaluatorVersion,
				Verdict:          VerdictUncovered,
				ReasonCode:       ReasonUncoveredNoReferences,
				ReferenceCount:   0,
				TopScore:         0,
			}, nil
		}
		return Decision{
			TurnID:           turn.ID,
			TenantKey:        turn.TenantKey,
			Channel:          turn.Channel,
			EvaluatorVersion: EvaluatorVersion,
			Verdict:          VerdictCovered,
			ReasonCode:       ReasonCoveredVisualRecognition,
			ReferenceCount:   0,
			TopScore:         0,
		}, nil
	}

	// Text turns without references are only uncovered if they actually
	// triggered knowledge search. This prevents greetings and other normal
	// non-knowledge responses from generating artificial support cases.
	if len(turn.References) == 0 {
		if !turn.KnowledgeSearched {
			return Decision{
				TurnID:           turn.ID,
				TenantKey:        turn.TenantKey,
				Channel:          turn.Channel,
				EvaluatorVersion: EvaluatorVersion,
				Verdict:          VerdictCovered,
				ReasonCode:       ReasonCoveredNonKnowledgeResponse,
				ReferenceCount:   0,
				TopScore:         0,
			}, nil
		}
		return Decision{
			TurnID:           turn.ID,
			TenantKey:        turn.TenantKey,
			Channel:          turn.Channel,
			EvaluatorVersion: EvaluatorVersion,
			Verdict:          VerdictUncovered,
			ReasonCode:       ReasonUncoveredNoReferences,
			ReferenceCount:   0,
			TopScore:         0,
		}, nil
	}

	// Has references → covered (v1 never uses score threshold).
	return Decision{
		TurnID:           turn.ID,
		TenantKey:        turn.TenantKey,
		Channel:          turn.Channel,
		EvaluatorVersion: EvaluatorVersion,
		Verdict:          VerdictCovered,
		ReasonCode:       ReasonCoveredHasReferences,
		ReferenceCount:   len(turn.References),
		TopScore:         maxScore(turn.References),
	}, nil
}

// hasImageMedia reports whether the turn carried at least one image input.
func hasImageMedia(media []core.MediaAudit) bool {
	for _, m := range media {
		if strings.EqualFold(strings.TrimSpace(m.Type), "image") {
			return true
		}
	}
	return false
}

func isNoAnswer(answer string) bool {
	lower := strings.ToLower(answer)
	for _, phrase := range noAnswerPhrases {
		if strings.Contains(lower, strings.ToLower(phrase)) {
			return true
		}
	}
	return containsAny(lower, retrievalGapPhrases) && containsAny(lower, humanEscalationPhrases)
}

func containsAny(value string, phrases []string) bool {
	for _, phrase := range phrases {
		if strings.Contains(value, strings.ToLower(phrase)) {
			return true
		}
	}
	return false
}

func maxScore(refs []core.Reference) float64 {
	var max float64
	for _, r := range refs {
		if r.Score > max {
			max = r.Score
		}
	}
	return max
}