diff --git a/db/migrations/000021_coverage_non_knowledge_response_reason.down.sql b/db/migrations/000021_coverage_non_knowledge_response_reason.down.sql new file mode 100644 index 0000000..bb080ab --- /dev/null +++ b/db/migrations/000021_coverage_non_knowledge_response_reason.down.sql @@ -0,0 +1,18 @@ +-- 000021_coverage_non_knowledge_response_reason down migration. +-- Map the v4 covered reason to the pre-v4 covered code before restoring the +-- legacy allowlist, so rollback remains valid with persisted v4 decisions. +UPDATE coverage_decisions +SET reason_code = 'covered_has_references' +WHERE reason_code = 'covered_non_knowledge_response'; + +ALTER TABLE coverage_decisions + DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check; + +ALTER TABLE coverage_decisions + ADD CONSTRAINT coverage_decisions_reason_code_check + CHECK (reason_code IN ( + 'covered_has_references', + 'uncovered_no_references', + 'uncovered_explicit_no_answer', + 'covered_visual_recognition' + )); \ No newline at end of file diff --git a/db/migrations/000021_coverage_non_knowledge_response_reason.up.sql b/db/migrations/000021_coverage_non_knowledge_response_reason.up.sql new file mode 100644 index 0000000..353358d --- /dev/null +++ b/db/migrations/000021_coverage_non_knowledge_response_reason.up.sql @@ -0,0 +1,14 @@ +-- 000021_coverage_non_knowledge_response_reason: distinguish text turns that +-- did not invoke knowledge search from actual retrieval misses. +ALTER TABLE coverage_decisions + DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check; + +ALTER TABLE coverage_decisions + ADD CONSTRAINT coverage_decisions_reason_code_check + CHECK (reason_code IN ( + 'covered_has_references', + 'uncovered_no_references', + 'uncovered_explicit_no_answer', + 'covered_visual_recognition', + 'covered_non_knowledge_response' + )); \ No newline at end of file diff --git a/deploy/dobotai-dingtalk/dobotai-dingtalk-start b/deploy/dobotai-dingtalk/dobotai-dingtalk-start index ab7f146..42c3bdd 100644 --- a/deploy/dobotai-dingtalk/dobotai-dingtalk-start +++ b/deploy/dobotai-dingtalk/dobotai-dingtalk-start @@ -18,9 +18,41 @@ if systemctl is-active --quiet "$legacy_service"; then systemctl stop "$legacy_service" fi +previous_image="$(docker inspect --format '{{.Config.Image}}' dobotai-dingtalk 2>/dev/null || true)" + +set_runtime_image() { + local image="$1" + python3 - "$runtime_env" "$image" <<'PY' +from pathlib import Path +import os +import sys + +path = Path(sys.argv[1]) +image = sys.argv[2] +lines = path.read_text(encoding="utf-8").splitlines() +updated = False +rendered = [] +for line in lines: + if line.startswith("DOBOTAI_IMAGE="): + rendered.append("DOBOTAI_IMAGE=" + image) + updated = True + else: + rendered.append(line) +if not updated: + rendered.append("DOBOTAI_IMAGE=" + image) +temporary = path.with_name(path.name + ".new") +temporary.write_text("\n".join(rendered) + "\n", encoding="utf-8") +os.chmod(temporary, 0o600) +os.replace(temporary, path) +PY +} + rollback() { docker compose --env-file "$runtime_env" -f "$compose_file" down --remove-orphans >/dev/null 2>&1 || true - if [[ "$legacy_was_active" == "true" ]]; then + if [[ -n "$previous_image" ]]; then + set_runtime_image "$previous_image" + docker compose --env-file "$runtime_env" -f "$compose_file" up -d --no-build --force-recreate >/dev/null 2>&1 || true + elif [[ "$legacy_was_active" == "true" ]]; then systemctl start "$legacy_service" || true fi } @@ -37,5 +69,9 @@ for _ in $(seq 1 30); do sleep 1 done -echo "DobotAI 容器健康检查失败,已回滚到原生服务。" >&2 +if [[ -n "$previous_image" ]]; then + echo "DobotAI 容器健康检查失败,已回滚到上一容器镜像。" >&2 +else + echo "DobotAI 容器健康检查失败,已回滚到原生服务。" >&2 +fi exit 1 diff --git a/internal/coverage/evaluator.go b/internal/coverage/evaluator.go index 2aa98f6..1d7a173 100644 --- a/internal/coverage/evaluator.go +++ b/internal/coverage/evaluator.go @@ -60,7 +60,10 @@ func (e *Evaluator) Version() string { // - no references + knowledge searched → uncovered_no_references // - no references + no knowledge search → covered_visual_recognition // (pure visual recognition is not a coverage miss) -// 4. Text turns: zero references → uncovered_no_references +// 4. Text turns with zero references are classified by whether the question +// actually triggered a knowledge-base search: +// - knowledge searched → uncovered_no_references +// - no knowledge search → covered_non_knowledge_response // 5. Has references → covered_has_references // // v1 never uses a score threshold. TopScore captures the highest reference score @@ -126,8 +129,22 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) { }, nil } - // No references → uncovered. + // Text turns without references are only uncovered if they actually + // triggered knowledge search. This prevents greetings and other normal + // non-knowledge responses from generating artificial support cases. if len(turn.References) == 0 { + if !turn.KnowledgeSearched { + return Decision{ + TurnID: turn.ID, + TenantKey: turn.TenantKey, + Channel: turn.Channel, + EvaluatorVersion: EvaluatorVersion, + Verdict: VerdictCovered, + ReasonCode: ReasonCoveredNonKnowledgeResponse, + ReferenceCount: 0, + TopScore: 0, + }, nil + } return Decision{ TurnID: turn.ID, TenantKey: turn.TenantKey, diff --git a/internal/coverage/evaluator_image_test.go b/internal/coverage/evaluator_image_test.go index 47c866d..7cedef2 100644 --- a/internal/coverage/evaluator_image_test.go +++ b/internal/coverage/evaluator_image_test.go @@ -66,20 +66,40 @@ func TestEvaluator_ImageWithReferencesIsCovered(t *testing.T) { } } -func TestEvaluator_TextTurnNoReferencesStillUncovered(t *testing.T) { +func TestEvaluator_TextTurnSearchedButNoReferencesIsUncovered(t *testing.T) { e := NewEvaluator() turn := core.Turn{ - ID: "text-turn", - Status: "succeeded", - Answer: "这是一个正常的回答,但没有任何引用", - MessageType: "text", - References: []core.Reference{}, + ID: "text-turn", + Status: "succeeded", + Answer: "这是一个正常的回答,但没有任何引用", + MessageType: "text", + References: []core.Reference{}, + KnowledgeSearched: true, } d, err := e.Evaluate(turn) if err != nil { t.Fatalf("unexpected error: %v", err) } if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredNoReferences { - t.Fatalf("verdict=%s reason=%s, want uncovered_no_references (text turn unchanged)", d.Verdict, d.ReasonCode) + t.Fatalf("verdict=%s reason=%s, want uncovered_no_references after knowledge search", d.Verdict, d.ReasonCode) + } +} + +func TestEvaluator_TextTurnWithoutKnowledgeSearchIsCovered(t *testing.T) { + e := NewEvaluator() + turn := core.Turn{ + ID: "smalltalk-turn", + Status: "succeeded", + Answer: "你好,我在。", + MessageType: "text", + KnowledgeSearched: false, + } + + d, err := e.Evaluate(turn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Verdict != VerdictCovered || d.ReasonCode != ReasonCoveredNonKnowledgeResponse { + t.Fatalf("verdict=%s reason=%s, want covered_non_knowledge_response", d.Verdict, d.ReasonCode) } } diff --git a/internal/coverage/evaluator_test.go b/internal/coverage/evaluator_test.go index 251f4d4..41f7947 100644 --- a/internal/coverage/evaluator_test.go +++ b/internal/coverage/evaluator_test.go @@ -8,8 +8,8 @@ import ( func TestEvaluatorVersion(t *testing.T) { e := NewEvaluator() - if e.Version() != "v3" { - t.Errorf("Evaluator version = %q, want v3", e.Version()) + if e.Version() != "v4" { + t.Errorf("Evaluator version = %q, want v4", e.Version()) } } @@ -85,10 +85,11 @@ func TestEvaluator_ProductionMissingProductPhraseTrumpsReferences(t *testing.T) func TestEvaluator_UncoveredNoReferences(t *testing.T) { e := NewEvaluator() turn := core.Turn{ - ID: "test-turn", - Status: "succeeded", - Answer: "这是一个正常的回答,但没有任何引用", - References: []core.Reference{}, + ID: "test-turn", + Status: "succeeded", + Answer: "这是一个正常的回答,但没有任何引用", + References: []core.Reference{}, + KnowledgeSearched: true, } d, err := e.Evaluate(turn) if err != nil { @@ -105,9 +106,10 @@ func TestEvaluator_UncoveredNoReferences(t *testing.T) { func TestEvaluator_UncoveredNilReferences(t *testing.T) { e := NewEvaluator() turn := core.Turn{ - ID: "test-turn", - Status: "succeeded", - Answer: "这是一个正常的回答,但引用为nil", + ID: "test-turn", + Status: "succeeded", + Answer: "这是一个正常的回答,但引用为nil", + KnowledgeSearched: true, } d, err := e.Evaluate(turn) if err != nil { diff --git a/internal/coverage/service_test.go b/internal/coverage/service_test.go index 8265ef0..e8a5eee 100644 --- a/internal/coverage/service_test.go +++ b/internal/coverage/service_test.go @@ -76,11 +76,12 @@ func TestService_EvaluateAndPersist_UncoveredNoReferences(t *testing.T) { s, _ := NewService(repo) turn := core.Turn{ - ID: "turn-2", - TenantKey: "t1", - Channel: "dingtalk", - Status: "succeeded", - Answer: "正常回答但没有引用", + ID: "turn-2", + TenantKey: "t1", + Channel: "dingtalk", + Status: "succeeded", + Answer: "正常回答但没有引用", + KnowledgeSearched: true, } d, err := s.EvaluateAndPersist(context.Background(), turn) diff --git a/internal/coverage/types.go b/internal/coverage/types.go index 086815a..c781849 100644 --- a/internal/coverage/types.go +++ b/internal/coverage/types.go @@ -25,14 +25,15 @@ const ( // ReasonCode values. const ( - ReasonCoveredHasReferences = "covered_has_references" - ReasonUncoveredNoReferences = "uncovered_no_references" - ReasonUncoveredExplicitNoAnswer = "uncovered_explicit_no_answer" - ReasonCoveredVisualRecognition = "covered_visual_recognition" + ReasonCoveredHasReferences = "covered_has_references" + ReasonUncoveredNoReferences = "uncovered_no_references" + ReasonUncoveredExplicitNoAnswer = "uncovered_explicit_no_answer" + ReasonCoveredVisualRecognition = "covered_visual_recognition" + ReasonCoveredNonKnowledgeResponse = "covered_non_knowledge_response" ) // EvaluatorVersion is the current version of the evaluator. -const EvaluatorVersion = "v3" +const EvaluatorVersion = "v4" var ( ErrTurnNotSucceeded = errors.New("coverage evaluator: turn status must be succeeded") diff --git a/scripts/containerized_deployment_test.go b/scripts/containerized_deployment_test.go index 6a1ddea..7f9a5ac 100644 --- a/scripts/containerized_deployment_test.go +++ b/scripts/containerized_deployment_test.go @@ -96,3 +96,22 @@ func TestDobotAIDingTalkCredentialConfigurationExplainsEveryInputWithRedactedExa } } } + +func TestDobotAIDingTalkStartRollsBackToPreviousContainerImage(t *testing.T) { + data, err := os.ReadFile("../deploy/dobotai-dingtalk/dobotai-dingtalk-start") + if err != nil { + t.Fatal(err) + } + source := string(data) + for _, required := range []string{ + "previous_image=", + "set_runtime_image", + "docker inspect --format", + "set_runtime_image \"$previous_image\"", + "docker compose --env-file \"$runtime_env\" -f \"$compose_file\" up -d --no-build --force-recreate", + } { + if !strings.Contains(source, required) { + t.Errorf("start script missing container upgrade rollback %q", required) + } + } +}