Weknora/dobotai-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(coverage): skip alerts for non-knowledge text turns

Browse Source
This commit is contained in:
姚章浩
2026-09-10 22:02:59 +08:00
parent 563329320b
commit 8027635330
9 changed files with 158 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/migrations/000021_coverage_non_knowledge_response_reason.down.sql
+18
View File
@@ -0,0 +1,18 @@
-- 000021_coverage_non_knowledge_response_reason down migration.
-- Map the v4 covered reason to the pre-v4 covered code before restoring the
-- legacy allowlist, so rollback remains valid with persisted v4 decisions.
UPDATE coverage_decisions
SET reason_code = 'covered_has_references'
WHERE reason_code = 'covered_non_knowledge_response';
ALTER TABLE coverage_decisions
DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check;
ALTER TABLE coverage_decisions
ADD CONSTRAINT coverage_decisions_reason_code_check
CHECK (reason_code IN (
'covered_has_references',
'uncovered_no_references',
'uncovered_explicit_no_answer',
'covered_visual_recognition'
));
db/migrations/000021_coverage_non_knowledge_response_reason.up.sql
+14
View File
@@ -0,0 +1,14 @@
-- 000021_coverage_non_knowledge_response_reason: distinguish text turns that
-- did not invoke knowledge search from actual retrieval misses.
ALTER TABLE coverage_decisions
DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check;
ALTER TABLE coverage_decisions
ADD CONSTRAINT coverage_decisions_reason_code_check
CHECK (reason_code IN (
'covered_has_references',
'uncovered_no_references',
'uncovered_explicit_no_answer',
'covered_visual_recognition',
'covered_non_knowledge_response'
));
deploy/dobotai-dingtalk/dobotai-dingtalk-start
+38 -2
View File
@@ -18,9 +18,41 @@ if systemctl is-active --quiet "$legacy_service"; then
systemctl stop "$legacy_service"
fi
previous_image="$(docker inspect --format '{{.Config.Image}}' dobotai-dingtalk 2>/dev/null || true)"
set_runtime_image() {
local image="$1"
python3 - "$runtime_env" "$image" <<'PY'
from pathlib import Path
import os
import sys
path = Path(sys.argv[1])
image = sys.argv[2]
lines = path.read_text(encoding="utf-8").splitlines()
updated = False
rendered = []
for line in lines:
if line.startswith("DOBOTAI_IMAGE="):
rendered.append("DOBOTAI_IMAGE=" + image)
updated = True
else:
rendered.append(line)
if not updated:
rendered.append("DOBOTAI_IMAGE=" + image)
temporary = path.with_name(path.name + ".new")
temporary.write_text("\n".join(rendered) + "\n", encoding="utf-8")
os.chmod(temporary, 0o600)
os.replace(temporary, path)
PY
}
rollback() {
docker compose --env-file "$runtime_env" -f "$compose_file" down --remove-orphans >/dev/null 2>&1 || true
if [[ "$legacy_was_active" == "true" ]]; then
if [[ -n "$previous_image" ]]; then
set_runtime_image "$previous_image"
docker compose --env-file "$runtime_env" -f "$compose_file" up -d --no-build --force-recreate >/dev/null 2>&1 || true
elif [[ "$legacy_was_active" == "true" ]]; then
systemctl start "$legacy_service" || true
fi
}
@@ -37,5 +69,9 @@ for _ in $(seq 1 30); do
sleep 1
done
echo "DobotAI 容器健康检查失败，已回滚到原生服务。" >&2
if [[ -n "$previous_image" ]]; then
echo "DobotAI 容器健康检查失败，已回滚到上一容器镜像。" >&2
else
echo "DobotAI 容器健康检查失败，已回滚到原生服务。" >&2
fi
exit 1
internal/coverage/evaluator.go
+19 -2
View File
@@ -60,7 +60,10 @@ func (e *Evaluator) Version() string {
// - no references + knowledge searched → uncovered_no_references
// - no references + no knowledge search → covered_visual_recognition
// (pure visual recognition is not a coverage miss)
// 4. Text turns: zero references → uncovered_no_references
// 4. Text turns with zero references are classified by whether the question
// actually triggered a knowledge-base search:
// - knowledge searched → uncovered_no_references
// - no knowledge search → covered_non_knowledge_response
// 5. Has references → covered_has_references
//
// v1 never uses a score threshold. TopScore captures the highest reference score
@@ -126,8 +129,22 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) {
}, nil
}
// No references → uncovered.
// Text turns without references are only uncovered if they actually
// triggered knowledge search. This prevents greetings and other normal
// non-knowledge responses from generating artificial support cases.
if len(turn.References) == 0 {
if !turn.KnowledgeSearched {
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
Channel: turn.Channel,
EvaluatorVersion: EvaluatorVersion,
Verdict: VerdictCovered,
ReasonCode: ReasonCoveredNonKnowledgeResponse,
ReferenceCount: 0,
TopScore: 0,
}, nil
}
return Decision{
TurnID: turn.ID,
TenantKey: turn.TenantKey,
internal/coverage/evaluator_image_test.go
+27 -7
View File
@@ -66,20 +66,40 @@ func TestEvaluator_ImageWithReferencesIsCovered(t *testing.T) {
}
}
func TestEvaluator_TextTurnNoReferencesStillUncovered(t *testing.T) {
func TestEvaluator_TextTurnSearchedButNoReferencesIsUncovered(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "text-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但没有任何引用",
MessageType: "text",
References: []core.Reference{},
ID: "text-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但没有任何引用",
MessageType: "text",
References: []core.Reference{},
KnowledgeSearched: true,
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredNoReferences {
t.Fatalf("verdict=%s reason=%s, want uncovered_no_references (text turn unchanged)", d.Verdict, d.ReasonCode)
t.Fatalf("verdict=%s reason=%s, want uncovered_no_references after knowledge search", d.Verdict, d.ReasonCode)
}
}
func TestEvaluator_TextTurnWithoutKnowledgeSearchIsCovered(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "smalltalk-turn",
Status: "succeeded",
Answer: "你好，我在。",
MessageType: "text",
KnowledgeSearched: false,
}
d, err := e.Evaluate(turn)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if d.Verdict != VerdictCovered || d.ReasonCode != ReasonCoveredNonKnowledgeResponse {
t.Fatalf("verdict=%s reason=%s, want covered_non_knowledge_response", d.Verdict, d.ReasonCode)
}
}
internal/coverage/evaluator_test.go
+11 -9
View File
@@ -8,8 +8,8 @@ import (
func TestEvaluatorVersion(t *testing.T) {
e := NewEvaluator()
if e.Version() != "v3" {
t.Errorf("Evaluator version = %q, want v3", e.Version())
if e.Version() != "v4" {
t.Errorf("Evaluator version = %q, want v4", e.Version())
}
}
@@ -85,10 +85,11 @@ func TestEvaluator_ProductionMissingProductPhraseTrumpsReferences(t *testing.T)
func TestEvaluator_UncoveredNoReferences(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "test-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但没有任何引用",
References: []core.Reference{},
ID: "test-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但没有任何引用",
References: []core.Reference{},
KnowledgeSearched: true,
}
d, err := e.Evaluate(turn)
if err != nil {
@@ -105,9 +106,10 @@ func TestEvaluator_UncoveredNoReferences(t *testing.T) {
func TestEvaluator_UncoveredNilReferences(t *testing.T) {
e := NewEvaluator()
turn := core.Turn{
ID: "test-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但引用为nil",
ID: "test-turn",
Status: "succeeded",
Answer: "这是一个正常的回答，但引用为nil",
KnowledgeSearched: true,
}
d, err := e.Evaluate(turn)
if err != nil {
internal/coverage/service_test.go
+6 -5
View File
@@ -76,11 +76,12 @@ func TestService_EvaluateAndPersist_UncoveredNoReferences(t *testing.T) {
s, _ := NewService(repo)
turn := core.Turn{
ID: "turn-2",
TenantKey: "t1",
Channel: "dingtalk",
Status: "succeeded",
Answer: "正常回答但没有引用",
ID: "turn-2",
TenantKey: "t1",
Channel: "dingtalk",
Status: "succeeded",
Answer: "正常回答但没有引用",
KnowledgeSearched: true,
}
d, err := s.EvaluateAndPersist(context.Background(), turn)
internal/coverage/types.go
+6 -5
View File
@@ -25,14 +25,15 @@ const (
// ReasonCode values.
const (
ReasonCoveredHasReferences = "covered_has_references"
ReasonUncoveredNoReferences = "uncovered_no_references"
ReasonUncoveredExplicitNoAnswer = "uncovered_explicit_no_answer"
ReasonCoveredVisualRecognition = "covered_visual_recognition"
ReasonCoveredHasReferences = "covered_has_references"
ReasonUncoveredNoReferences = "uncovered_no_references"
ReasonUncoveredExplicitNoAnswer = "uncovered_explicit_no_answer"
ReasonCoveredVisualRecognition = "covered_visual_recognition"
ReasonCoveredNonKnowledgeResponse = "covered_non_knowledge_response"
)
// EvaluatorVersion is the current version of the evaluator.
const EvaluatorVersion = "v3"
const EvaluatorVersion = "v4"
var (
ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded")
scripts/containerized_deployment_test.go
+19
View File
@@ -96,3 +96,22 @@ func TestDobotAIDingTalkCredentialConfigurationExplainsEveryInputWithRedactedExa
}
}
}
func TestDobotAIDingTalkStartRollsBackToPreviousContainerImage(t *testing.T) {
data, err := os.ReadFile("../deploy/dobotai-dingtalk/dobotai-dingtalk-start")
if err != nil {
t.Fatal(err)
}
source := string(data)
for _, required := range []string{
"previous_image=",
"set_runtime_image",
"docker inspect --format",
"set_runtime_image \"$previous_image\"",
"docker compose --env-file \"$runtime_env\" -f \"$compose_file\" up -d --no-build --force-recreate",
} {
if !strings.Contains(source, required) {
t.Errorf("start script missing container upgrade rollback %q", required)
}
}
}