Weknora/dobotai-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(coverage): let references override partial no-answer wording

Browse Source
This commit is contained in:
姚章浩
2026-09-15 16:53:12 +08:00
parent b49cb5eaf9
commit c7d2629a77
4 changed files with 87 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/coverage/evaluator.go
+35 -59
View File
@@ -6,36 +6,11 @@ import (
"gitea.zheromiku.top/Weknora/dobotai-platform/internal/core"
)
// noAnswerPhrases lists explicit "no answer" phrases that the knowledge base
// returns when it cannot find relevant information. These are matched as
// substrings after trimming whitespace.
var noAnswerPhrases = []string{
"未找到",
"没有找到",
"无法确定",
"暂无相关",
"暂无已发布的权威知识",
"知识库中没有",
"产品不存在于资料中",
}
// retrievalGapPhrases are only considered a no-answer signal when the answer
// also directs the user to a human channel. Requiring both signals avoids
// misclassifying valid negative results such as "未查询到异常记录".
var retrievalGapPhrases = []string{
"未查询到",
"未检索到",
"未能查询到",
"未能检索到",
}
var humanEscalationPhrases = []string{
"建议您联系",
"建议联系",
"请联系",
"联系技术支持",
"联系官方销售",
}
// canonicalNoAnswer is the only response that carries the explicit-no-answer
// machine contract. It must be the complete answer, never a substring match:
// an otherwise supported answer may legitimately say that one specific detail
// was not found.
const canonicalNoAnswer = "当前暂无已发布的权威知识，暂不建议操作。"
// Evaluator classifies a successful turn into a coverage verdict.
type Evaluator struct{}
@@ -53,20 +28,20 @@ func (e *Evaluator) Version() string {
// Evaluate classifies a turn. It rejects non-succeeded turns. The evaluation
// rules (in priority order):
// 1. Failed/processing turns → ErrTurnNotSucceeded
// 2. Answer contains an explicit "no answer" phrase → uncovered_explicit_no_answer
// 3. Image turns (image media present) are classified by whether the question
// 2. Has references → covered_has_references (references are the coverage
// evidence; wording in a supported answer never overrides them)
// 3. Complete canonical no-answer response with no references
// → uncovered_explicit_no_answer
// 4. Image turns (image media present) are classified by whether the question
// actually triggered a knowledge-base search:
// - has references → covered_has_references
// - no references + knowledge searched → uncovered_no_references
// - no references + no knowledge search → covered_visual_recognition
// (pure visual recognition is not a coverage miss)
// 4. Text turns with zero references are classified by whether the question
// 5. Text turns with zero references are classified by whether the question
// actually triggered a knowledge-base search:
// - knowledge searched → uncovered_no_references
// - no knowledge search → covered_non_knowledge_response
// 5. Has references → covered_has_references
//
// v1 never uses a score threshold. TopScore captures the highest reference score
// v5 never uses a score threshold. TopScore captures the highest reference score
// (0 if no references).
func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
if turn.Status != "succeeded" {
@@ -76,9 +51,25 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
// Normalize answer for no-answer phrase detection.
trimmed := strings.TrimSpace(turn.Answer)
// Check for explicit no-answer phrases first — this takes priority
// over reference presence.
if isNoAnswer(trimmed) {
// References are the durable coverage evidence. A supported answer may
// correctly note that one sub-detail is absent, so its wording cannot turn
// the entire turn into a knowledge gap.
if len(turn.References) > 0 {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredHasReferences,
ReferenceCount: len(turn.References),
TopScore: maxScore(turn.References),
}, nil
}
// An explicit abstention is a stable machine contract only when it is the
// complete answer. Do not use broad natural-language substring matching.
if isCanonicalNoAnswer(trimmed) {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
@@ -86,8 +77,8 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictUncovered,
ReasonCode: ReasonUncoveredExplicitNoAnswer,
ReferenceCount: len(turn.References),
TopScore: maxScore(turn.References),
ReferenceCount: 0,
TopScore: 0,
}, nil
}
@@ -180,23 +171,8 @@ func hasImageMedia(media []core.MediaAudit) bool {
return false
}
func isNoAnswer(answer string) bool {
lower := strings.ToLower(answer)
for _, phrase := range noAnswerPhrases {
if strings.Contains(lower, strings.ToLower(phrase)) {
return true
}
}
return containsAny(lower, retrievalGapPhrases) && containsAny(lower, humanEscalationPhrases)
}
func containsAny(value string, phrases []string) bool {
for _, phrase := range phrases {
if strings.Contains(value, strings.ToLower(phrase)) {
return true
}
}
return false
func isCanonicalNoAnswer(answer string) bool {
return strings.TrimSpace(answer) == canonicalNoAnswer
}
func maxScore(refs []core.Reference) float64 {
internal/coverage/evaluator_test.go
+45 -60
View File
@@ -8,8 +8,8 @@ import (
func TestEvaluatorVersion(t *testing.T) {
e := NewEvaluator()
if e.Version() != "v4" {
t.Errorf("Evaluator version = %q, want v4", e.Version())
if e.Version() != "v5" {
t.Errorf("Evaluator version = %q, want v5", e.Version())
}
}
@@ -30,55 +30,40 @@ func TestEvaluator_RejectsProcessingTurn(t *testing.T) {
}
}
func TestEvaluator_UncoveredExplicitNoAnswer(t *testing.T) {
e := NewEvaluator()
noAnswerPhrases := []string{
"未找到相关结果",
"没有找到你需要的信息",
"无法确定该问题的答案",
"暂无相关信息",
"知识库中没有找到相关内容",
"这个问题知识库中没有可用的信息",
}
for _, answer := range noAnswerPhrases {
turn := core.Turn{
ID: "test-turn",
Status: "succeeded",
Answer: answer,
References: []core.Reference{
{ID: "r1", Score: 0.9},
},
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatalf("unexpected error for answer %q: %v", answer, err)
}
if d.Verdict != VerdictUncovered {
t.Errorf("answer %q: verdict = %s, want uncovered", answer, d.Verdict)
}
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Errorf("answer %q: reason_code = %s, want %s", answer, d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
}
}
}
func TestEvaluator_ProductionMissingProductPhraseTrumpsReferences(t *testing.T) {
func TestEvaluator_UncoveredCanonicalNoAnswerWithoutReferences(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "missing-product-turn",
Status: "succeeded",
Answer: "根据提供的检索信息，无法提供对应手册。原因：产品不存在于资料中；检索资料未提及该产品。建议联系技术支持部门。",
References: []core.Reference{
{ID: "unrelated-1", Score: 0.395940},
{ID: "unrelated-2", Score: 0.311595},
},
ID: "test-turn",
Status: "succeeded",
Answer: canonicalNoAnswer,
KnowledgeSearched: true,
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatal(err)
}
if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Fatalf("verdict=%s reason=%s", d.Verdict, d.ReasonCode)
t.Fatalf("verdict/reason = %s/%s, want %s/%s", d.Verdict, d.ReasonCode, VerdictUncovered, ReasonUncoveredExplicitNoAnswer)
}
}
func TestEvaluator_NovaPartialNotFoundWithSevenReferencesIsCovered(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "nova-partial-turn",
Status: "succeeded",
Answer: "已找到Nova系列的相关资料；未找到VX500 2.5D视觉一体方案的明确说明。",
References: []core.Reference{
{ID: "r1"}, {ID: "r2"}, {ID: "r3"}, {ID: "r4"},
{ID: "r5"}, {ID: "r6"}, {ID: "r7"},
},
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatal(err)
}
if d.Verdict != VerdictCovered || d.ReasonCode != ReasonCoveredHasReferences || d.ReferenceCount != 7 {
t.Fatalf("verdict/reason/references = %s/%s/%d, want %s/%s/7", d.Verdict, d.ReasonCode, d.ReferenceCount, VerdictCovered, ReasonCoveredHasReferences)
}
}
@@ -205,9 +190,9 @@ func TestEvaluator_TopScoreZeroWhenNoReferences(t *testing.T) {
}
}
func TestEvaluator_NoAnswerTrumpsReferences(t *testing.T) {
// When answer contains explicit "no answer" phrase, it should be uncovered
// even if references exist. No-answer detection takes priority.
func TestEvaluator_ReferencesTrumpNaturalLanguageNoAnswer(t *testing.T) {
// References are durable coverage evidence; a natural-language phrase must
// not override them.
e := NewEvaluator()
turn := core.Turn{
ID: "test-turn",
@@ -222,11 +207,11 @@ func TestEvaluator_NoAnswerTrumpsReferences(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if d.Verdict != VerdictUncovered {
t.Errorf("no-answer with references: verdict = %s, want uncovered", d.Verdict)
if d.Verdict != VerdictCovered {
t.Errorf("answer with references: verdict = %s, want covered", d.Verdict)
}
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Errorf("no-answer with references: reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
if d.ReasonCode != ReasonCoveredHasReferences {
t.Errorf("answer with references: reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
}
}
@@ -245,12 +230,12 @@ func TestEvaluator_NoAnswerPhraseMatchesInsideAnswer(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
if d.ReasonCode != ReasonCoveredHasReferences {
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
}
}
func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testing.T) {
func TestEvaluator_ProductionUnavailableAuthorityPhraseWithReferencesIsCovered(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "cra-price-turn",
@@ -265,11 +250,11 @@ func TestEvaluator_ProductionUnavailableAuthorityPhraseTrumpsReferences(t *testi
if err != nil {
t.Fatalf("Evaluate: %v", err)
}
if d.Verdict != VerdictUncovered {
t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictUncovered)
if d.Verdict != VerdictCovered {
t.Fatalf("verdict = %s, want %s", d.Verdict, VerdictCovered)
}
if d.ReasonCode != ReasonUncoveredExplicitNoAnswer {
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonUncoveredExplicitNoAnswer)
if d.ReasonCode != ReasonCoveredHasReferences {
t.Fatalf("reason_code = %s, want %s", d.ReasonCode, ReasonCoveredHasReferences)
}
}
@@ -282,10 +267,10 @@ func TestEvaluator_RetrievalGapRequiresHumanEscalation(t *testing.T) {
reason string
}{
{
name: "production NOVA price answer",
name: "retrieval wording with evidence remains covered",
answer: "当前暂未查询到Nova系列机器人的公开销售价格信息。\n\n如需获取报价，建议您联系越疆科技官方销售渠道或对应区域的销售工程师咨询具体价格方案。",
verdict: VerdictUncovered,
reason: ReasonUncoveredExplicitNoAnswer,
verdict: VerdictCovered,
reason: ReasonCoveredHasReferences,
},
{
name: "retrieval gap without escalation is a valid negative result",
internal/coverage/service_test.go
+6 -8
View File
@@ -104,14 +104,12 @@ func TestService_EvaluateAndPersist_UncoveredExplicitNoAnswer(t *testing.T) {
s, _ := NewService(repo)
turn := core.Turn{
ID: "turn-3",
TenantKey: "t1",
Channel: "dingtalk",
Status: "succeeded",
Answer: "未找到相关结果",
References: []core.Reference{
{ID: "r1", Score: 0.5},
},
ID: "turn-3",
TenantKey: "t1",
Channel: "dingtalk",
Status: "succeeded",
Answer: canonicalNoAnswer,
KnowledgeSearched: true,
}
d, err := s.EvaluateAndPersist(context.Background(), turn)
internal/coverage/types.go
+1 -1
View File
@@ -33,7 +33,7 @@ const (
)
// EvaluatorVersion is the current version of the evaluator.
const EvaluatorVersion = "v4"
const EvaluatorVersion = "v5"
var (
ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded")