From 0798a2dac7220834df3a7bfea280b2a864b0601a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Wed, 9 Sep 2026 13:57:43 +0800 Subject: [PATCH] feat(dingtalk): stabilize multimodal answer workflows --- cmd/dobotai/main.go | 40 ++- cmd/dobotai/main_test.go | 55 +++- cmd/dobotai/outbound_images.go | 41 ++- cmd/dobotai/question_table_sync.go | 28 +- cmd/dobotai/question_table_sync_test.go | 10 +- .../000016_question_image_urls.down.sql | 3 + .../000016_question_image_urls.up.sql | 6 + .../000017_knowledge_searched.down.sql | 3 + .../000017_knowledge_searched.up.sql | 6 + ...overage_visual_recognition_reason.down.sql | 11 + ..._coverage_visual_recognition_reason.up.sql | 14 + internal/channels/dingtalk/card.go | 2 + .../channels/dingtalk/card_diagnostics.go | 64 ++++ .../dingtalk/card_diagnostics_test.go | 93 ++++++ internal/channels/dingtalk/card_pump.go | 6 + .../channels/dingtalk/display_sanitizer.go | 31 ++ .../dingtalk/display_sanitizer_test.go | 66 ++++ .../channels/dingtalk/download_codes_test.go | 33 ++ internal/channels/dingtalk/handler.go | 116 ++++++- internal/channels/dingtalk/handler_test.go | 149 ++++++++- .../channels/dingtalk/image_publish_test.go | 80 +++++ internal/channels/dingtalk/markdown_stream.go | 114 +++++++ .../dingtalk/markdown_stream_handler_test.go | 65 ++++ .../channels/dingtalk/markdown_stream_test.go | 101 ++++++ internal/channels/dingtalk/stream.go | 33 +- internal/channels/dingtalk/stream_test.go | 8 +- internal/core/types.go | 16 +- internal/coverage/evaluator.go | 58 +++- internal/coverage/evaluator_image_test.go | 85 +++++ internal/coverage/types.go | 1 + internal/outboundimage/aliyun_oss.go | 31 +- internal/outboundimage/aliyun_oss_test.go | 58 ++++ internal/outboundimage/publisher.go | 61 ++-- internal/outboundimage/publisher_test.go | 65 +++- internal/outboundimage/rewriter.go | 181 ++++++++++- internal/outboundimage/rewriter_test.go | 299 ++++++++++++++++++ internal/outboundimage/stable_handler.go | 48 +++ internal/outboundimage/stable_handler_test.go | 91 ++++++ internal/qa/service.go | 24 +- internal/questiontable/dispatcher.go | 44 ++- internal/questiontable/export_rewrite_test.go | 137 ++++++++ internal/questiontable/row.go | 47 ++- internal/settings/controller.go | 13 +- internal/settings/controller_test.go | 21 +- .../store/postgres/question_table_exports.go | 28 +- internal/store/postgres/store.go | 18 +- 46 files changed, 2309 insertions(+), 195 deletions(-) create mode 100644 db/migrations/000016_question_image_urls.down.sql create mode 100644 db/migrations/000016_question_image_urls.up.sql create mode 100644 db/migrations/000017_knowledge_searched.down.sql create mode 100644 db/migrations/000017_knowledge_searched.up.sql create mode 100644 db/migrations/000018_coverage_visual_recognition_reason.down.sql create mode 100644 db/migrations/000018_coverage_visual_recognition_reason.up.sql create mode 100644 internal/channels/dingtalk/card_diagnostics.go create mode 100644 internal/channels/dingtalk/card_diagnostics_test.go create mode 100644 internal/channels/dingtalk/display_sanitizer.go create mode 100644 internal/channels/dingtalk/display_sanitizer_test.go create mode 100644 internal/channels/dingtalk/download_codes_test.go create mode 100644 internal/channels/dingtalk/image_publish_test.go create mode 100644 internal/channels/dingtalk/markdown_stream.go create mode 100644 internal/channels/dingtalk/markdown_stream_handler_test.go create mode 100644 internal/channels/dingtalk/markdown_stream_test.go create mode 100644 internal/coverage/evaluator_image_test.go create mode 100644 internal/outboundimage/aliyun_oss_test.go create mode 100644 internal/outboundimage/stable_handler.go create mode 100644 internal/outboundimage/stable_handler_test.go create mode 100644 internal/questiontable/export_rewrite_test.go diff --git a/cmd/dobotai/main.go b/cmd/dobotai/main.go index 20be0cb..0a7a749 100644 --- a/cmd/dobotai/main.go +++ b/cmd/dobotai/main.go @@ -185,6 +185,24 @@ func main() { }() log.Printf("DingTalk support notification dispatcher started") } + + // Outbound image runtime (stable-image publishing) is built from environment + // variables independently of the Stream ingress, so both the question-table + // answer rewriter and the inbound-image publisher can share it. + outboundImages, err := buildOutboundImageRuntime(outboundImageConfig{ + WeKnoraBaseURL: os.Getenv("WEKNORA_BASE_URL"), + WeKnoraAPIKey: os.Getenv("WEKNORA_API_KEY"), + Endpoint: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ENDPOINT"), + Region: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_REGION"), + AccessKeyID: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_ID"), + AccessKeySecret: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_SECRET"), + Bucket: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_BUCKET"), + PublicBaseURL: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_PUBLIC_BASE_URL"), + }) + if err != nil { + log.Fatalf("DingTalk outbound image init failed: %v", err) + } + // Question table dispatcher (optional, fail-closed). qtCfg, err := readQuestionTableConfig() if err != nil { @@ -200,7 +218,7 @@ func main() { if err != nil { log.Fatal(err) } - qtDispatcher, err := buildQuestionTableDispatcher(qtRepo, dingCfg.ClientID, dingCfg.ClientSecret, qtCfg) + qtDispatcher, err := buildQuestionTableDispatcher(qtRepo, dingCfg.ClientID, dingCfg.ClientSecret, qtCfg, answerRewriterAdapter{rewriter: outboundImages.rewriter}) if err != nil { log.Fatalf("DingTalk question table dispatcher init failed: %v", err) } @@ -232,18 +250,6 @@ func main() { if err != nil { log.Fatalf("DingTalk directory resolver init failed: %v", err) } - outboundRewriter, err := buildOutboundImageRewriter(outboundImageConfig{ - WeKnoraBaseURL: os.Getenv("WEKNORA_BASE_URL"), - WeKnoraAPIKey: os.Getenv("WEKNORA_API_KEY"), - Endpoint: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ENDPOINT"), - Region: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_REGION"), - AccessKeyID: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_ID"), - AccessKeySecret: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_SECRET"), - Bucket: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_OSS_BUCKET"), - }) - if err != nil { - log.Fatalf("DingTalk outbound image init failed: %v", err) - } feedbackService, err := feedback.NewService(dingCfg.TenantKey, "dingtalk", store) if err != nil { log.Fatal(err) @@ -284,9 +290,10 @@ func main() { sender, dingtalk.HandlerOptions{ ImageDownloader: downloader, + ImagePublisher: outboundImages.publisher, CardSender: cardSender, Directory: directoryResolver, - OutboundRewriter: outboundRewriter, + OutboundRewriter: outboundImages.rewriter, SettingsRouter: &dingTalkSettingsRouter{controller: settingsRuntime.controller}, LanguageResolver: settingsRuntime.repository, }, @@ -334,6 +341,9 @@ func main() { }) rootHandler := http.NewServeMux() rootHandler.Handle("/", api.Handler()) + if outboundImages.handler != nil { + rootHandler.Handle(outboundImageRoute, outboundImages.handler) + } if strings.TrimSpace(os.Getenv("DOBOTAI_ADMIN_TOKEN")) != "" { rootHandler.Handle("/admin/", adminweb.New(settingsRuntime.admin, os.Getenv("DOBOTAI_ADMIN_TOKEN"))) } @@ -377,7 +387,7 @@ func outboundOSSConfigured() bool { for _, key := range []string{ "DOBOTAI_OUTBOUND_IMAGE_OSS_ENDPOINT", "DOBOTAI_OUTBOUND_IMAGE_OSS_REGION", "DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_ID", "DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_SECRET", - "DOBOTAI_OUTBOUND_IMAGE_OSS_BUCKET", + "DOBOTAI_OUTBOUND_IMAGE_OSS_BUCKET", "DOBOTAI_OUTBOUND_IMAGE_PUBLIC_BASE_URL", } { if strings.TrimSpace(os.Getenv(key)) == "" { return false diff --git a/cmd/dobotai/main_test.go b/cmd/dobotai/main_test.go index afb37e8..0fe67e2 100644 --- a/cmd/dobotai/main_test.go +++ b/cmd/dobotai/main_test.go @@ -116,21 +116,21 @@ func TestMainWiresOptionalDingTalkCardDebugUser(t *testing.T) { } } -func TestBuildOutboundImageRewriterConfig(t *testing.T) { +func TestBuildOutboundImageRuntimeConfig(t *testing.T) { t.Run("disabled", func(t *testing.T) { - rewriter, err := buildOutboundImageRewriter(outboundImageConfig{}) - if err != nil || rewriter != nil { - t.Fatalf("rewriter=%v err=%v", rewriter, err) + runtime, err := buildOutboundImageRuntime(outboundImageConfig{}) + if err != nil || runtime.rewriter != nil || runtime.handler != nil { + t.Fatalf("runtime=%+v err=%v", runtime, err) } }) t.Run("partial configuration rejected", func(t *testing.T) { - _, err := buildOutboundImageRewriter(outboundImageConfig{Bucket: "private-bucket"}) + _, err := buildOutboundImageRuntime(outboundImageConfig{Bucket: "private-bucket"}) if err == nil { t.Fatal("expected configuration error") } }) t.Run("complete configuration", func(t *testing.T) { - rewriter, err := buildOutboundImageRewriter(outboundImageConfig{ + runtime, err := buildOutboundImageRuntime(outboundImageConfig{ WeKnoraBaseURL: "http://127.0.0.1:8080", WeKnoraAPIKey: "api-key", Endpoint: "https://oss-cn-shenzhen.aliyuncs.com", @@ -138,9 +138,48 @@ func TestBuildOutboundImageRewriterConfig(t *testing.T) { AccessKeyID: "access-key-id", AccessKeySecret: "access-key-secret", Bucket: "private-bucket", + PublicBaseURL: "https://weknora-api-test.dobot.cn/dingtalk/i", }) - if err != nil || rewriter == nil { - t.Fatalf("rewriter=%v err=%v", rewriter, err) + if err != nil || runtime.rewriter == nil || runtime.handler == nil { + t.Fatalf("runtime=%+v err=%v", runtime, err) } }) } + +func TestMainWiresStableOutboundImageRuntime(t *testing.T) { + data, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + source := string(data) + for _, fragment := range []string{ + `buildOutboundImageRuntime(outboundImageConfig{`, + `PublicBaseURL: os.Getenv("DOBOTAI_OUTBOUND_IMAGE_PUBLIC_BASE_URL")`, + `OutboundRewriter: outboundImages.rewriter`, + `rootHandler.Handle(outboundImageRoute, outboundImages.handler)`, + } { + if !strings.Contains(source, fragment) { + t.Errorf("main stable image wiring missing %q", fragment) + } + } +} + +func TestOutboundOSSConfiguredRequiresStablePublicBaseURL(t *testing.T) { + for _, key := range []string{ + "DOBOTAI_OUTBOUND_IMAGE_OSS_ENDPOINT", + "DOBOTAI_OUTBOUND_IMAGE_OSS_REGION", + "DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_ID", + "DOBOTAI_OUTBOUND_IMAGE_OSS_ACCESS_KEY_SECRET", + "DOBOTAI_OUTBOUND_IMAGE_OSS_BUCKET", + } { + t.Setenv(key, "configured") + } + t.Setenv("DOBOTAI_OUTBOUND_IMAGE_PUBLIC_BASE_URL", "") + if outboundOSSConfigured() { + t.Fatal("stable public base URL must be required") + } + t.Setenv("DOBOTAI_OUTBOUND_IMAGE_PUBLIC_BASE_URL", "https://img.example/dingtalk/i") + if !outboundOSSConfigured() { + t.Fatal("complete stable image configuration should be active") + } +} diff --git a/cmd/dobotai/outbound_images.go b/cmd/dobotai/outbound_images.go index 055a568..d7a636d 100644 --- a/cmd/dobotai/outbound_images.go +++ b/cmd/dobotai/outbound_images.go @@ -13,8 +13,8 @@ import ( const ( outboundImageMaxCount = 5 outboundImageMaxBytes = 10 << 20 - outboundImageURLTTL = 2 * time.Hour outboundImagePrefix = "dobotai/im-images" + outboundImageRoute = "/dingtalk/i/" ) type outboundImageConfig struct { @@ -25,10 +25,17 @@ type outboundImageConfig struct { AccessKeyID string AccessKeySecret string Bucket string + PublicBaseURL string } -func buildOutboundImageRewriter(cfg outboundImageConfig) (dingtalk.OutboundContentRewriter, error) { - ossValues := []string{cfg.Endpoint, cfg.Region, cfg.AccessKeyID, cfg.AccessKeySecret, cfg.Bucket} +type outboundImageRuntime struct { + rewriter dingtalk.OutboundContentRewriter + publisher *outboundimage.StableHashPublisher + handler http.Handler +} + +func buildOutboundImageRuntime(cfg outboundImageConfig) (outboundImageRuntime, error) { + ossValues := []string{cfg.Endpoint, cfg.Region, cfg.AccessKeyID, cfg.AccessKeySecret, cfg.Bucket, cfg.PublicBaseURL} enabled := false for _, value := range ossValues { if strings.TrimSpace(value) != "" { @@ -37,11 +44,11 @@ func buildOutboundImageRewriter(cfg outboundImageConfig) (dingtalk.OutboundConte } } if !enabled { - return nil, nil + return outboundImageRuntime{}, nil } for _, value := range append([]string{cfg.WeKnoraBaseURL, cfg.WeKnoraAPIKey}, ossValues...) { if strings.TrimSpace(value) == "" { - return nil, errors.New("outbound image: complete WeKnora and OSS configuration is required") + return outboundImageRuntime{}, errors.New("outbound image: complete WeKnora, OSS, and public URL configuration is required") } } transport := http.DefaultTransport.(*http.Transport).Clone() @@ -53,7 +60,7 @@ func buildOutboundImageRewriter(cfg outboundImageConfig) (dingtalk.OutboundConte outboundImageMaxBytes, ) if err != nil { - return nil, err + return outboundImageRuntime{}, err } store, err := outboundimage.NewAliyunOSSStore( cfg.Endpoint, @@ -63,8 +70,24 @@ func buildOutboundImageRewriter(cfg outboundImageConfig) (dingtalk.OutboundConte cfg.Bucket, ) if err != nil { - return nil, err + return outboundImageRuntime{}, err } - publisher := outboundimage.NewHashPublisher(store, outboundImagePrefix, outboundImageURLTTL) - return outboundimage.NewRewriter(fetcher, publisher, outboundImageMaxCount), nil + publisher, err := outboundimage.NewStableHashPublisher(store, outboundImagePrefix, cfg.PublicBaseURL) + if err != nil { + return outboundImageRuntime{}, err + } + handler, err := outboundimage.NewStableImageHandler(store, outboundImagePrefix, outboundImageRoute, outboundImageMaxBytes) + if err != nil { + return outboundImageRuntime{}, err + } + return outboundImageRuntime{ + rewriter: outboundimage.NewRewriter(fetcher, publisher, outboundImageMaxCount), + publisher: publisher, + handler: handler, + }, nil +} + +func buildOutboundImageRewriter(cfg outboundImageConfig) (dingtalk.OutboundContentRewriter, error) { + runtime, err := buildOutboundImageRuntime(cfg) + return runtime.rewriter, err } diff --git a/cmd/dobotai/question_table_sync.go b/cmd/dobotai/question_table_sync.go index fffa7fe..817723b 100644 --- a/cmd/dobotai/question_table_sync.go +++ b/cmd/dobotai/question_table_sync.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/channels/dingtalk" "gitea.zheromiku.top/Weknora/dobotai-platform/internal/questiontable" ) @@ -49,9 +50,11 @@ func readQuestionTableConfig() (questionTableConfig, error) { // buildQuestionTableDispatcher creates a validated question-table dispatcher // when the repo is non-nil and all required credentials are present. -// No HTTP request occurs at build time. It returns (nil, nil) when the -// question-table feature is disabled (empty config). -func buildQuestionTableDispatcher(repo questiontable.ExportRepository, clientID, clientSecret string, cfg questionTableConfig) (*questiontable.Dispatcher, error) { +// The optional answer rewriter converts the stored answer into the same +// outbound form used for DingTalk delivery (resource images to stable HTTPS, +// document links, anchor cleanup). No HTTP request occurs at build time. +// It returns (nil, nil) when the question-table feature is disabled. +func buildQuestionTableDispatcher(repo questiontable.ExportRepository, clientID, clientSecret string, cfg questionTableConfig, rewriter questiontable.AnswerRewriter) (*questiontable.Dispatcher, error) { // Disabled: all blank config. if cfg.BaseID == "" && cfg.SheetID == "" && cfg.OperatorID == "" { return nil, nil @@ -73,7 +76,24 @@ func buildQuestionTableDispatcher(repo questiontable.ExportRepository, clientID, return nil, err } - return questiontable.NewDispatcher(repo, client, nil) + return questiontable.NewDispatcher(repo, client, &questiontable.DispatcherOptions{ + Rewriter: rewriter, + SendTimeout: 60 * time.Second, + }) +} + +// answerRewriterAdapter adapts a blocking outbound content rewriter to the +// AnswerRewriter contract (caller never gets an error, so the original answer +// is always written when conversion cannot run). +type answerRewriterAdapter struct { + rewriter dingtalk.OutboundContentRewriter +} + +func (a answerRewriterAdapter) Rewrite(ctx context.Context, content string) (string, error) { + if a.rewriter == nil { + return content, nil + } + return a.rewriter.Rewrite(ctx, content), nil } // runQuestionTablePoller runs the dispatcher immediately, then every interval. diff --git a/cmd/dobotai/question_table_sync_test.go b/cmd/dobotai/question_table_sync_test.go index 775b3e3..0117343 100644 --- a/cmd/dobotai/question_table_sync_test.go +++ b/cmd/dobotai/question_table_sync_test.go @@ -32,7 +32,7 @@ func TestReadQuestionTableConfigDisabledAllBlank(t *testing.T) { } func TestBuildQuestionTableDispatcherDisabled(t *testing.T) { - d, err := buildQuestionTableDispatcher(nil, "", "", questionTableConfig{}) + d, err := buildQuestionTableDispatcher(nil, "", "", questionTableConfig{}, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -87,7 +87,7 @@ func TestReadQuestionTableConfigPartialErrors(t *testing.T) { func TestBuildQuestionTableDispatcherEnabledMissingRepo(t *testing.T) { cfg := questionTableConfig{BaseID: "b1", SheetID: "sh1", OperatorID: "op1"} - _, err := buildQuestionTableDispatcher(nil, "cid", "secret", cfg) + _, err := buildQuestionTableDispatcher(nil, "cid", "secret", cfg, nil) if err == nil { t.Fatal("expected error when repo is nil") } @@ -99,7 +99,7 @@ func TestBuildQuestionTableDispatcherEnabledMissingRepo(t *testing.T) { func TestBuildQuestionTableDispatcherEnabledMissingClientID(t *testing.T) { repo := &questionTableExportRepoStub{} cfg := questionTableConfig{BaseID: "b1", SheetID: "sh1", OperatorID: "op1"} - _, err := buildQuestionTableDispatcher(repo, "", "secret", cfg) + _, err := buildQuestionTableDispatcher(repo, "", "secret", cfg, nil) if err == nil { t.Fatal("expected error when client id is empty") } @@ -108,7 +108,7 @@ func TestBuildQuestionTableDispatcherEnabledMissingClientID(t *testing.T) { func TestBuildQuestionTableDispatcherEnabledMissingSecret(t *testing.T) { repo := &questionTableExportRepoStub{} cfg := questionTableConfig{BaseID: "b1", SheetID: "sh1", OperatorID: "op1"} - _, err := buildQuestionTableDispatcher(repo, "cid", "", cfg) + _, err := buildQuestionTableDispatcher(repo, "cid", "", cfg, nil) if err == nil { t.Fatal("expected error when secret is empty") } @@ -117,7 +117,7 @@ func TestBuildQuestionTableDispatcherEnabledMissingSecret(t *testing.T) { func TestBuildQuestionTableDispatcherEnabledCompleteConfig(t *testing.T) { repo := &questionTableExportRepoStub{} cfg := questionTableConfig{BaseID: "b1", SheetID: "sh1", OperatorID: "op1"} - d, err := buildQuestionTableDispatcher(repo, "cid", "secret", cfg) + d, err := buildQuestionTableDispatcher(repo, "cid", "secret", cfg, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/migrations/000016_question_image_urls.down.sql b/db/migrations/000016_question_image_urls.down.sql new file mode 100644 index 0000000..ed17fd4 --- /dev/null +++ b/db/migrations/000016_question_image_urls.down.sql @@ -0,0 +1,3 @@ +-- 000016_question_image_urls down migration. +ALTER TABLE turns + DROP COLUMN IF EXISTS question_image_urls_json; \ No newline at end of file diff --git a/db/migrations/000016_question_image_urls.up.sql b/db/migrations/000016_question_image_urls.up.sql new file mode 100644 index 0000000..e384225 --- /dev/null +++ b/db/migrations/000016_question_image_urls.up.sql @@ -0,0 +1,6 @@ +-- 000016_question_image_urls: stable published URLs for inbound question images. +-- Stores the object-storage HTTPS URLs of images a user sent in a question, +-- so the question-table export can render a readable, traceable record without +-- persisting download codes or data URIs. +ALTER TABLE turns + ADD COLUMN IF NOT EXISTS question_image_urls_json JSONB NOT NULL DEFAULT '[]'::JSONB; \ No newline at end of file diff --git a/db/migrations/000017_knowledge_searched.down.sql b/db/migrations/000017_knowledge_searched.down.sql new file mode 100644 index 0000000..bdfac51 --- /dev/null +++ b/db/migrations/000017_knowledge_searched.down.sql @@ -0,0 +1,3 @@ +-- 000017_knowledge_searched down migration. +ALTER TABLE turns + DROP COLUMN IF EXISTS knowledge_searched; \ No newline at end of file diff --git a/db/migrations/000017_knowledge_searched.up.sql b/db/migrations/000017_knowledge_searched.up.sql new file mode 100644 index 0000000..f2eec20 --- /dev/null +++ b/db/migrations/000017_knowledge_searched.up.sql @@ -0,0 +1,6 @@ +-- 000017_knowledge_searched: records whether a turn actually triggered a +-- knowledge-base search (knowledge_search tool call). Coverage classification +-- uses this to distinguish pure visual recognition from a searched-but-missed +-- knowledge question. +ALTER TABLE turns + ADD COLUMN IF NOT EXISTS knowledge_searched BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/000018_coverage_visual_recognition_reason.down.sql b/db/migrations/000018_coverage_visual_recognition_reason.down.sql new file mode 100644 index 0000000..aad52a3 --- /dev/null +++ b/db/migrations/000018_coverage_visual_recognition_reason.down.sql @@ -0,0 +1,11 @@ +-- 000018_coverage_visual_recognition_reason down migration. +ALTER TABLE coverage_decisions + DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check; + +ALTER TABLE coverage_decisions + ADD CONSTRAINT coverage_decisions_reason_code_check + CHECK (reason_code IN ( + 'covered_has_references', + 'uncovered_no_references', + 'uncovered_explicit_no_answer' + )); \ No newline at end of file diff --git a/db/migrations/000018_coverage_visual_recognition_reason.up.sql b/db/migrations/000018_coverage_visual_recognition_reason.up.sql new file mode 100644 index 0000000..98f17fc --- /dev/null +++ b/db/migrations/000018_coverage_visual_recognition_reason.up.sql @@ -0,0 +1,14 @@ +-- 000018_coverage_visual_recognition_reason: extend the coverage_decisions +-- reason_code allowlist with the new image-visual-recognition reason emitted by +-- coverage Evaluator v3 (visual recognition is not a knowledge-coverage miss). +ALTER TABLE coverage_decisions + DROP CONSTRAINT IF EXISTS coverage_decisions_reason_code_check; + +ALTER TABLE coverage_decisions + ADD CONSTRAINT coverage_decisions_reason_code_check + CHECK (reason_code IN ( + 'covered_has_references', + 'uncovered_no_references', + 'uncovered_explicit_no_answer', + 'covered_visual_recognition' + )); \ No newline at end of file diff --git a/internal/channels/dingtalk/card.go b/internal/channels/dingtalk/card.go index f55375c..305e8c2 100644 --- a/internal/channels/dingtalk/card.go +++ b/internal/channels/dingtalk/card.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "strings" "sync" @@ -206,6 +207,7 @@ func (s *CardSender) UpdateCard(ctx context.Context, outTrackID, content string, if len(content) > 64*1024 { return errors.New("dingtalk card: content exceeds 64 KiB") } + log.Print(formatCardUpdateDiagnostic(content, isFinalize, isError)) token, err := s.accessToken(ctx) if err != nil { diff --git a/internal/channels/dingtalk/card_diagnostics.go b/internal/channels/dingtalk/card_diagnostics.go new file mode 100644 index 0000000..717e50e --- /dev/null +++ b/internal/channels/dingtalk/card_diagnostics.go @@ -0,0 +1,64 @@ +package dingtalk + +import ( + "fmt" + "strings" +) + +type cardContentShape struct { + bytes int + lines int + pipeLines int + leadingPipeLines int + delimiterLines int + confirmedTable bool + confirmedTablePos int +} + +func inspectCardContent(content string) cardContentShape { + shape := cardContentShape{ + bytes: len(content), + confirmedTablePos: -1, + } + if content == "" { + return shape + } + shape.lines = strings.Count(content, "\n") + 1 + + lines := splitMarkdownStreamLines(content) + for i, line := range lines { + if strings.Contains(line.text, "|") { + shape.pipeLines++ + } + if isTableHeaderCandidate(line.text) { + shape.leadingPipeLines++ + } + if !line.inFence && isCompleteTableDelimiter(line.text) { + shape.delimiterLines++ + } + if shape.confirmedTable || i == 0 || line.inFence || lines[i-1].inFence { + continue + } + if isTableHeaderCandidate(lines[i-1].text) && isCompleteTableDelimiter(line.text) { + shape.confirmedTable = true + shape.confirmedTablePos = lines[i-1].start + } + } + return shape +} + +func formatCardUpdateDiagnostic(content string, isFinalize, isError bool) string { + shape := inspectCardContent(content) + return fmt.Sprintf( + "DingTalk card update shape: finalize=%t error=%t bytes=%d lines=%d pipe_lines=%d leading_pipe_lines=%d delimiter_lines=%d confirmed_table=%t table_start=%d", + isFinalize, + isError, + shape.bytes, + shape.lines, + shape.pipeLines, + shape.leadingPipeLines, + shape.delimiterLines, + shape.confirmedTable, + shape.confirmedTablePos, + ) +} diff --git a/internal/channels/dingtalk/card_diagnostics_test.go b/internal/channels/dingtalk/card_diagnostics_test.go new file mode 100644 index 0000000..3477cf7 --- /dev/null +++ b/internal/channels/dingtalk/card_diagnostics_test.go @@ -0,0 +1,93 @@ +package dingtalk + +import ( + "bytes" + "context" + "fmt" + "log" + "strings" + "testing" +) + +func TestCardSenderUpdateCardLogsOnlySafeContentShape(t *testing.T) { + ts := newCardTestServer() + defer ts.Close() + sender, err := newCardSenderForTest("client-id", "client-secret", "template-id", ts.Client(), ts.URL()+"/token", ts.URL()+"/create") + if err != nil { + t.Fatal(err) + } + + var output bytes.Buffer + previousWriter := log.Writer() + previousFlags := log.Flags() + previousPrefix := log.Prefix() + log.SetOutput(&output) + log.SetFlags(0) + log.SetPrefix("") + t.Cleanup(func() { + log.SetOutput(previousWriter) + log.SetFlags(previousFlags) + log.SetPrefix(previousPrefix) + }) + + content := "DO_NOT_LOG_BOUNDARY_TEXT\n\n| 型号 | 负载 |\n| --- | --- |\n| CR5 | 5kg |" + if err := sender.UpdateCard(context.Background(), "track-diagnostic", content, false, false); err != nil { + t.Fatal(err) + } + + got := output.String() + if !strings.Contains(got, "DingTalk card update shape:") || !strings.Contains(got, "confirmed_table=true") || !strings.Contains(got, "finalize=false") { + t.Fatalf("missing safe boundary diagnostic: %q", got) + } + if strings.Contains(got, "DO_NOT_LOG_BOUNDARY_TEXT") || strings.Contains(got, "CR5") || strings.Contains(got, "track-diagnostic") { + t.Fatalf("boundary diagnostic leaked content or card identifier: %q", got) + } +} + +func TestFormatCardUpdateDiagnosticReportsStructureWithoutContent(t *testing.T) { + content := "DO_NOT_LOG_ANSWER_TEXT\n\n| 型号 | 负载 |\n| --- | --- |\n| CR5 | 5kg |" + + got := formatCardUpdateDiagnostic(content, false, false) + wantFields := []string{ + "DingTalk card update shape:", + "finalize=false", + "error=false", + fmt.Sprintf("bytes=%d", len(content)), + "lines=5", + "pipe_lines=3", + "leading_pipe_lines=3", + "delimiter_lines=1", + "confirmed_table=true", + fmt.Sprintf("table_start=%d", strings.Index(content, "| 型号")), + } + for _, field := range wantFields { + if !strings.Contains(got, field) { + t.Fatalf("diagnostic %q missing field %q", got, field) + } + } + if strings.Contains(got, "DO_NOT_LOG_ANSWER_TEXT") || strings.Contains(got, "CR5") { + t.Fatalf("diagnostic leaked answer content: %q", got) + } +} + +func TestFormatCardUpdateDiagnosticDistinguishesFinalPlainText(t *testing.T) { + got := formatCardUpdateDiagnostic("普通回答", true, false) + wantFields := []string{ + "finalize=true", + "error=false", + "lines=1", + "pipe_lines=0", + "leading_pipe_lines=0", + "delimiter_lines=0", + "confirmed_table=false", + "table_start=-1", + } + for _, field := range wantFields { + if !strings.Contains(got, field) { + t.Fatalf("diagnostic %q missing field %q", got, field) + } + } + if strings.Contains(got, "普通回答") { + t.Fatalf("diagnostic leaked answer content: %q", got) + } +} diff --git a/internal/channels/dingtalk/card_pump.go b/internal/channels/dingtalk/card_pump.go index dcdd22a..6b86f7a 100644 --- a/internal/channels/dingtalk/card_pump.go +++ b/internal/channels/dingtalk/card_pump.go @@ -39,6 +39,7 @@ func (p *cardUpdatePump) run(ctx context.Context, ticks <-chan time.Time, stopTi if stopTicks != nil { defer stopTicks() } + lastSuccessful := "" for { select { case <-ctx.Done(): @@ -52,9 +53,14 @@ func (p *cardUpdatePump) run(ctx context.Context, ticks <-chan time.Time, stopTi default: continue } + if latest == lastSuccessful { + continue + } if err := update(ctx, latest); err != nil { log.Printf("DingTalk async card update failed, continuing") + continue } + lastSuccessful = latest } } } diff --git a/internal/channels/dingtalk/display_sanitizer.go b/internal/channels/dingtalk/display_sanitizer.go new file mode 100644 index 0000000..21161e1 --- /dev/null +++ b/internal/channels/dingtalk/display_sanitizer.go @@ -0,0 +1,31 @@ +package dingtalk + +import ( + "regexp" + "strings" +) + +const displayReferenceMarkerPattern = `\[\[ref[\t ]+id[\t ]*=[\t ]*["']c[0-9]+["'][\t ]*\]\]` + +var ( + displayReferenceMarkerRE = regexp.MustCompile(displayReferenceMarkerPattern) + leadingReferenceGroupColonRE = regexp.MustCompile(`(?m)^[ ]*(?:` + displayReferenceMarkerPattern + `[ ]*)+[::][ ]*`) + spacedReferenceGroupRE = regexp.MustCompile(`[ ]+(?:` + displayReferenceMarkerPattern + `[ ]*)+`) + spaceBeforeDisplayPunctuation = regexp.MustCompile(`[ ]+([,。;:!?、,.;:!?])`) + spaceAfterDisplayColon = regexp.MustCompile(`([::])[\t ]+`) +) + +func sanitizeDisplayAnswer(content string) string { + if index := strings.LastIndex(content, "[[ref"); index >= 0 && !strings.Contains(content[index:], "]]") { + content = content[:index] + } + if !displayReferenceMarkerRE.MatchString(content) { + return content + } + content = leadingReferenceGroupColonRE.ReplaceAllString(content, "") + content = spacedReferenceGroupRE.ReplaceAllString(content, " ") + content = displayReferenceMarkerRE.ReplaceAllString(content, "") + content = spaceBeforeDisplayPunctuation.ReplaceAllString(content, "$1") + content = spaceAfterDisplayColon.ReplaceAllString(content, "$1") + return content +} diff --git a/internal/channels/dingtalk/display_sanitizer_test.go b/internal/channels/dingtalk/display_sanitizer_test.go new file mode 100644 index 0000000..b8acd90 --- /dev/null +++ b/internal/channels/dingtalk/display_sanitizer_test.go @@ -0,0 +1,66 @@ +package dingtalk + +import "testing" + +func TestSanitizeDisplayAnswerRemovesInternalReferenceMarkers(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "adjacent inline markers", + input: "结论支持该功能[[ref id=\"c2\"]][[ref id=\"c4\"]]。", + want: "结论支持该功能。", + }, + { + name: "leading marker group and chinese colon", + input: "[[ref id=\"c2\"]][[ref id=\"c4\"]]:建议先检查配置。", + want: "建议先检查配置。", + }, + { + name: "leading marker group and ascii colon on second line", + input: "第一段。\n[[ref id='c12']] : Second paragraph.", + want: "第一段。\nSecond paragraph.", + }, + { + name: "normal punctuation remains natural", + input: "标题:[[ref id=\"c1\"]] 详细说明", + want: "标题:详细说明", + }, + { + name: "spaces around marker collapse to one", + input: "Feature A [[ref id=\"c3\"]] remains supported.", + want: "Feature A remains supported.", + }, + { + name: "unrelated double bracket syntax is preserved", + input: "请保留 [[wiki|页面名称]] 和 [普通链接](https://example.com)。", + want: "请保留 [[wiki|页面名称]] 和 [普通链接](https://example.com)。", + }, + { + name: "content without references remains byte for byte unchanged", + input: "Protocol: value ; keep spacing\n indented", + want: "Protocol: value ; keep spacing\n indented", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sanitizeDisplayAnswer(tt.input); got != tt.want { + t.Fatalf("sanitizeDisplayAnswer()=%q, want %q", got, tt.want) + } + }) + } +} + +func TestSanitizeDisplayAnswerHoldsBackDanglingStreamingReference(t *testing.T) { + for _, input := range []string{ + "答案正文[[ref", + "答案正文[[ref id=\"c", + "答案正文[[ref id=\"c2\"", + } { + if got := sanitizeDisplayAnswer(input); got != "答案正文" { + t.Fatalf("sanitizeDisplayAnswer(%q)=%q", input, got) + } + } +} diff --git a/internal/channels/dingtalk/download_codes_test.go b/internal/channels/dingtalk/download_codes_test.go new file mode 100644 index 0000000..fbe47ca --- /dev/null +++ b/internal/channels/dingtalk/download_codes_test.go @@ -0,0 +1,33 @@ +package dingtalk + +import ( + "testing" +) + +func TestEffectiveDownloadCodesDeduplicatesAndPreservesOrder(t *testing.T) { + cb := BotCallback{ + DownloadCode: "code-first", + DownloadCodes: []string{"code-first", "code-second", " code-third "}, + } + codes := effectiveDownloadCodes(cb) + if len(codes) != 3 { + t.Fatalf("codes=%v, want 3", codes) + } + if codes[0] != "code-first" || codes[1] != "code-second" || codes[2] != "code-third" { + t.Fatalf("codes=%v, want ordered de-dup", codes) + } +} + +func TestEffectiveDownloadCodesSingleOnly(t *testing.T) { + cb := BotCallback{DownloadCode: " solo "} + codes := effectiveDownloadCodes(cb) + if len(codes) != 1 || codes[0] != "solo" { + t.Fatalf("codes=%v, want [solo]", codes) + } +} + +func TestEffectiveDownloadCodesEmpty(t *testing.T) { + if codes := effectiveDownloadCodes(BotCallback{}); len(codes) != 0 { + t.Fatalf("codes=%v, want empty", codes) + } +} \ No newline at end of file diff --git a/internal/channels/dingtalk/handler.go b/internal/channels/dingtalk/handler.go index e3af29e..ad557a4 100644 --- a/internal/channels/dingtalk/handler.go +++ b/internal/channels/dingtalk/handler.go @@ -2,6 +2,7 @@ package dingtalk import ( "context" + "encoding/base64" "errors" "fmt" "log" @@ -45,6 +46,7 @@ type BotCallback struct { SessionWebhook string Text string DownloadCode string + DownloadCodes []string } type QAAsker interface { @@ -75,6 +77,12 @@ type ImageDownloader interface { Download(ctx context.Context, downloadCode string) (core.ImageInput, error) } +// ImagePublisher publishes a downloaded image to object storage and returns a +// stable HTTPS URL. Implementation satisfies outboundimage.Publisher. +type ImagePublisher interface { + Publish(ctx context.Context, data []byte, contentType string) (string, error) +} + type UserDirectory interface { ResolveUser(ctx context.Context, tenantKey, staffID, fallbackName string) (directory.Profile, error) } @@ -83,6 +91,10 @@ type OutboundContentRewriter interface { Rewrite(ctx context.Context, content string) string } +type OutboundPartialSanitizer interface { + SanitizePartial(content string) string +} + type SettingsMessage struct { TenantKey string Channel string @@ -119,6 +131,7 @@ type StreamingCardSender interface { type HandlerOptions struct { ImageDownloader ImageDownloader + ImagePublisher ImagePublisher CardSender FinalCardSender Directory UserDirectory OutboundRewriter OutboundContentRewriter @@ -131,6 +144,7 @@ type Handler struct { qa QAAsker reply MarkdownSender downloader ImageDownloader + imagePublisher ImagePublisher card FinalCardSender directory UserDirectory outboundRewriter OutboundContentRewriter @@ -175,6 +189,7 @@ func NewHandlerWithOptions(cfg TenantConfig, qa QAAsker, reply MarkdownSender, o qa: qa, reply: reply, downloader: opts.ImageDownloader, + imagePublisher: opts.ImagePublisher, card: opts.CardSender, directory: opts.Directory, outboundRewriter: opts.OutboundRewriter, @@ -287,7 +302,8 @@ func (h *Handler) Handle(ctx context.Context, cb BotCallback) error { return errors.New("dingtalk handler: text message content is empty") } case msgTypePicture: - if strings.TrimSpace(cb.DownloadCode) == "" { + codes := effectiveDownloadCodes(cb) + if len(codes) == 0 { log.Printf("DingTalk image preparation failed: stage=missing_download_code") return h.sendReply(ctx, webhookURL, imageErrorReply) } @@ -295,16 +311,25 @@ func (h *Handler) Handle(ctx context.Context, cb BotCallback) error { log.Printf("DingTalk image preparation failed: stage=downloader_unavailable") return h.sendReply(ctx, webhookURL, imageErrorReply) } - image, err := h.downloader.Download(ctx, strings.TrimSpace(cb.DownloadCode)) - if err != nil { - log.Printf("DingTalk image preparation failed: stage=%s", imageFailureStage(err)) - return h.sendReply(ctx, webhookURL, imageErrorReply) + images := make([]core.ImageInput, 0, len(codes)) + for _, code := range codes { + image, err := h.downloader.Download(ctx, code) + if err != nil { + log.Printf("DingTalk image preparation failed: stage=%s", imageFailureStage(err)) + return h.sendReply(ctx, webhookURL, imageErrorReply) + } + if h.imagePublisher != nil { + if url := h.publishQuestionImage(ctx, image); url != "" { + image.PublishedURL = url + } + } + images = append(images, image) } req.Query = strings.TrimSpace(cb.Text) if req.Query == "" { req.Query = imageQuery } - req.Images = []core.ImageInput{image} + req.Images = images default: return h.sendReply(ctx, webhookURL, unsupportedMessageReply) } @@ -419,19 +444,26 @@ func (h *Handler) handleStreaming(ctx context.Context, webhookURL string, req co // Streaming observer with the configured update throttle (500ms by default). var lastUpdate time.Time + var lastPartialContent string observer := func(fullContent string) { if strings.TrimSpace(fullContent) == "" { return } now := h.now() if lastUpdate.IsZero() || now.Sub(lastUpdate) >= h.cardUpdateInterval { + partialContent := h.sanitizeOutboundPartial(fullContent) + if strings.TrimSpace(partialContent) == "" || partialContent == lastPartialContent { + return + } // Advance the throttle window before I/O so a failing DingTalk API // cannot be hammered once per model chunk. The final update below is // always attempted regardless of this timestamp. lastUpdate = now - if updateErr := streamingCard.UpdateCard(ctx, outTrackID, fullContent, false, false); updateErr != nil { + if updateErr := streamingCard.UpdateCard(ctx, outTrackID, partialContent, false, false); updateErr != nil { log.Printf("DingTalk streaming card update failed, continuing") + return } + lastPartialContent = partialContent } } @@ -477,7 +509,9 @@ func (h *Handler) handleEventStreaming(ctx context.Context, webhookURL string, r result, err := eventQA.AskEvents(ctx, req, func(event weknora.StreamEvent) { if event.ResponseType == "answer" && strings.TrimSpace(event.FullAnswer) != "" { answerStarted = true - pump.Offer(event.FullAnswer) + if partialContent := h.sanitizeOutboundPartial(event.FullAnswer); strings.TrimSpace(partialContent) != "" { + pump.Offer(partialContent) + } return } if answerStarted { @@ -538,7 +572,7 @@ func (h *Handler) finalizeCardOrMarkdown(ctx context.Context, webhookURL string, func (h *Handler) rewriteOutboundContent(ctx context.Context, content string) string { if h.outboundRewriter == nil { - return content + return sanitizeDisplayAnswer(content) } timeout := h.finalDeliveryTimeout if timeout <= 0 { @@ -546,7 +580,15 @@ func (h *Handler) rewriteOutboundContent(ctx context.Context, content string) st } rewriteCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) defer cancel() - return h.outboundRewriter.Rewrite(rewriteCtx, content) + return sanitizeDisplayAnswer(h.outboundRewriter.Rewrite(rewriteCtx, content)) +} + +func (h *Handler) sanitizeOutboundPartial(content string) string { + sanitizer, ok := h.outboundRewriter.(OutboundPartialSanitizer) + if ok { + content = sanitizer.SanitizePartial(content) + } + return stableMarkdownPrefix(sanitizeDisplayAnswer(content)) } func (h *Handler) recordDelivery(ctx context.Context, result core.QAResult, status, method, providerMessageID, failureCode string) { @@ -671,3 +713,57 @@ func (h *Handler) sendReply(ctx context.Context, webhookURL, text string) error } return nil } + +// publishQuestionImage best-effort publishes a downloaded image's bytes to +// object storage for traceability. It returns the stable HTTPS URL, or an +// empty string when publishing is unavailable or fails, so that an image +// question is never blocked by storage. +func (h *Handler) publishQuestionImage(ctx context.Context, image core.ImageInput) string { + if h.imagePublisher == nil { + return "" + } + data, err := decodeImageDataURI(image.DataURI) + if err != nil || len(data) == 0 { + log.Printf("DingTalk question image publish failed: stage=decode") + return "" + } + url, err := h.imagePublisher.Publish(ctx, data, image.MIMEType) + if err != nil || strings.TrimSpace(url) == "" { + log.Printf("DingTalk question image publish failed: stage=publish") + return "" + } + return strings.TrimSpace(url) +} + +// decodeImageDataURI strips the "data:;base64," prefix and decodes the +// payload into raw bytes. It is used only to publish inbound images and never +// writes the data URI to storage. +func decodeImageDataURI(dataURI string) ([]byte, error) { + const prefix = "base64," + index := strings.Index(dataURI, prefix) + if index < 0 { + return nil, errors.New("dingtalk image: data URI missing base64 payload") + } + return base64.StdEncoding.DecodeString(dataURI[index+len(prefix):]) +} + +// effectiveDownloadCodes returns the ordered, de-duplicated list of image +// download codes for a picture callback. DownloadCode equals the first code +// and DownloadCodes holds every code (richText with several pictures), so the +// single value is used to canonicalize the order without duplication. +func effectiveDownloadCodes(cb BotCallback) []string { + var codes []string + seen := make(map[string]struct{}) + for _, code := range append([]string{cb.DownloadCode}, cb.DownloadCodes...) { + code = strings.TrimSpace(code) + if code == "" { + continue + } + if _, ok := seen[code]; ok { + continue + } + seen[code] = struct{}{} + codes = append(codes, code) + } + return codes +} diff --git a/internal/channels/dingtalk/handler_test.go b/internal/channels/dingtalk/handler_test.go index 1183a69..40ffff9 100644 --- a/internal/channels/dingtalk/handler_test.go +++ b/internal/channels/dingtalk/handler_test.go @@ -3,6 +3,7 @@ package dingtalk import ( "context" "errors" + "reflect" "strings" "testing" "time" @@ -981,9 +982,10 @@ type cardUpdate struct { } type fakeOutboundRewriter struct { - calls int - from string - to string + calls int + partialCalls int + from string + to string } func (f *fakeOutboundRewriter) Rewrite(_ context.Context, content string) string { @@ -991,6 +993,11 @@ func (f *fakeOutboundRewriter) Rewrite(_ context.Context, content string) string return strings.ReplaceAll(content, f.from, f.to) } +func (f *fakeOutboundRewriter) SanitizePartial(content string) string { + f.partialCalls++ + return strings.ReplaceAll(content, f.from, "") +} + func (f *fakeStreamingCard) SendFinalCard(_ context.Context, req CardRequest) (string, error) { f.finalCalled = true f.finalReq = req @@ -1668,15 +1675,16 @@ func TestRenderProgressEventUsesOnlySafeFixedText(t *testing.T) { } func TestHandleEventStreamingFinalizesLatestAnswerWithoutMarkdown(t *testing.T) { + rawAnswer := "完整答案[[ref id=\"c8\"]]" card := &fakeStreamingCard{startID: "track-events"} qa := &fakeEventQA{ - fakeStreamingQA: fakeStreamingQA{result: core.QAResult{Answer: "完整答案"}}, + fakeStreamingQA: fakeStreamingQA{result: core.QAResult{Answer: rawAnswer}}, events: []weknora.StreamEvent{ {ResponseType: "tool_call", ToolName: "query_understand"}, {ResponseType: "tool_call", ToolName: "knowledge_search"}, {ResponseType: "references", ReferenceCount: 3}, - {ResponseType: "answer", FullAnswer: "部分"}, - {ResponseType: "answer", FullAnswer: "完整答案", Done: true}, + {ResponseType: "answer", FullAnswer: "部分[[ref"}, + {ResponseType: "answer", FullAnswer: rawAnswer, Done: true}, }, } reply := &fakeReply{} @@ -1698,6 +1706,9 @@ func TestHandleEventStreamingFinalizesLatestAnswerWithoutMarkdown(t *testing.T) if last.content != "完整答案" || !last.isFinalize || last.isError { t.Fatalf("final update=%+v", last) } + if qa.result.Answer != rawAnswer { + t.Fatalf("raw QA answer mutated: %q", qa.result.Answer) + } if reply.called { t.Fatal("SendMarkdown must not be called") } @@ -2080,8 +2091,8 @@ func TestHandleStreamingRewritesOnlyFinalOutboundImageContent(t *testing.T) { if len(card.updates) < 2 { t.Fatalf("updates=%d", len(card.updates)) } - if !strings.Contains(card.updates[0].content, resource) { - t.Fatalf("streaming intermediate content should remain untouched: %q", card.updates[0].content) + if strings.Contains(card.updates[0].content, resource) { + t.Fatalf("streaming intermediate content leaked internal resource: %q", card.updates[0].content) } final := card.updates[len(card.updates)-1] if !final.isFinalize || strings.Contains(final.content, resource) || !strings.Contains(final.content, "https://img.example/signed.jpg") { @@ -2090,4 +2101,126 @@ func TestHandleStreamingRewritesOnlyFinalOutboundImageContent(t *testing.T) { if rewriter.calls != 1 { t.Fatalf("rewrite calls=%d, want 1", rewriter.calls) } + if rewriter.partialCalls == 0 { + t.Fatal("streaming intermediate content was not sanitized") + } +} + +func TestHandleFiltersReferenceMarkersAtDingTalkDisplayBoundary(t *testing.T) { + rawAnswer := "结论[[ref id=\"c2\"]][[ref id=\"c4\"]]。" + wantDisplay := "结论。" + + t.Run("final card", func(t *testing.T) { + qa := &fakeQA{result: core.QAResult{Answer: rawAnswer}} + reply := &fakeReply{} + card := &fakeCard{returnID: "track-reference-filter"} + h := newTestHandlerWithCard(t, qa, reply, card) + + if err := h.Handle(context.Background(), directTextCallback()); err != nil { + t.Fatal(err) + } + if card.lastReq.Content != wantDisplay { + t.Fatalf("card content=%q, want %q", card.lastReq.Content, wantDisplay) + } + if qa.result.Answer != rawAnswer { + t.Fatalf("raw QA answer mutated: %q", qa.result.Answer) + } + }) + + t.Run("markdown fallback", func(t *testing.T) { + qa := &fakeQA{result: core.QAResult{Answer: rawAnswer}} + reply := &fakeReply{} + card := &fakeCard{err: errors.New("card unavailable")} + h := newTestHandlerWithCard(t, qa, reply, card) + + if err := h.Handle(context.Background(), directTextCallback()); err != nil { + t.Fatal(err) + } + if reply.lastText != wantDisplay { + t.Fatalf("markdown content=%q, want %q", reply.lastText, wantDisplay) + } + if qa.result.Answer != rawAnswer { + t.Fatalf("raw QA answer mutated: %q", qa.result.Answer) + } + }) +} + +func TestHandleStreamingHoldsBackPartialReferenceMarkers(t *testing.T) { + rawAnswer := "答案正文[[ref id=\"c2\"]]继续。" + qa := &fakeStreamingQA{ + chunks: []string{"答案正文[[ref", " id=\"c2\"]]继续。"}, + result: core.QAResult{Answer: rawAnswer}, + } + reply := &fakeReply{} + card := &fakeStreamingCard{startID: "track-stream-reference-filter"} + h := newTestHandlerWithStreaming(t, qa, reply, card) + + if err := h.Handle(context.Background(), directTextCallback()); err != nil { + t.Fatal(err) + } + if len(card.updates) < 2 { + t.Fatalf("updates=%d, want at least one partial plus final", len(card.updates)) + } + for i, update := range card.updates { + if strings.Contains(update.content, "[[ref") { + t.Fatalf("update[%d] leaked reference marker: %q", i, update.content) + } + } + if card.updates[0].content != "答案正文" { + t.Fatalf("first partial=%q, want dangling marker held back", card.updates[0].content) + } + final := card.updates[len(card.updates)-1] + if !final.isFinalize || final.content != "答案正文继续。" { + t.Fatalf("final content=%q finalize=%t", final.content, final.isFinalize) + } + if qa.result.Answer != rawAnswer { + t.Fatalf("raw QA answer mutated: %q", qa.result.Answer) + } +} + +func TestHandleStreamingPublishesTableOnlyOnFinalUpdate(t *testing.T) { + answer := "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n\n结论" + qa := &fakeStreamingQA{ + chunks: []string{ + "说明\n\n", + "| 检查项", + " | 操作要求 |\n", + "| --- | --", + "- |\n", + "| 接地检查 | 上电前确", + "认 |\n\n结论", + }, + result: core.QAResult{Answer: answer}, + } + reply := &fakeReply{} + card := &fakeStreamingCard{startID: "track-stream-table"} + h := newTestHandlerWithStreaming(t, qa, reply, card) + callCount := int64(0) + h.now = func() time.Time { + callCount++ + return time.Unix(0, callCount*int64(time.Second)) + } + h.cardUpdateInterval = time.Millisecond + + if err := h.Handle(context.Background(), directTextCallback()); err != nil { + t.Fatal(err) + } + + var partials []string + for _, update := range card.updates { + if !update.isFinalize { + partials = append(partials, update.content) + } + } + wantPartials := []string{"说明\n\n"} + if !reflect.DeepEqual(partials, wantPartials) { + t.Fatalf("partial updates=%q, want only pre-table stream %q", partials, wantPartials) + } + final := card.updates[len(card.updates)-1] + if !final.isFinalize || final.content != answer { + t.Fatalf("final=%+v, want unchanged complete answer", final) + } + if qa.result.Answer != answer { + t.Fatalf("raw QA answer mutated: %q", qa.result.Answer) + } } diff --git a/internal/channels/dingtalk/image_publish_test.go b/internal/channels/dingtalk/image_publish_test.go new file mode 100644 index 0000000..7f158a3 --- /dev/null +++ b/internal/channels/dingtalk/image_publish_test.go @@ -0,0 +1,80 @@ +package dingtalk + +import ( + "context" + "encoding/base64" + "errors" + "testing" + + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" +) + +type fakeImagePublisher struct { + published []byte + mime string + url string + err error + calls int +} + +func (f *fakeImagePublisher) Publish(_ context.Context, data []byte, contentType string) (string, error) { + f.calls++ + f.published = data + f.mime = contentType + return f.url, f.err +} + +func TestDecodeImageDataURI(t *testing.T) { + raw := []byte("hello-image-bytes") + uri := "data:image/png;base64," + base64.StdEncoding.EncodeToString(raw) + + decoded, err := decodeImageDataURI(uri) + if err != nil { + t.Fatalf("decodeImageDataURI: %v", err) + } + if string(decoded) != string(raw) { + t.Fatalf("decoded=%q, want %q", decoded, raw) + } + + if _, err := decodeImageDataURI("data:image/png;base64"); err == nil { + t.Fatal("expected error for missing base64 payload") + } + if _, err := decodeImageDataURI("no comma at all"); err == nil { + t.Fatal("expected error for non-data URI") + } +} + +func TestPublishQuestionImagePublishesDecodedBytes(t *testing.T) { + pub := &fakeImagePublisher{url: "https://weknora-api.example/i/001"} + raw := []byte{0x89, 0x50, 0x4e, 0x47} + image := core.ImageInput{ + DataURI: "data:image/png;base64," + base64.StdEncoding.EncodeToString(raw), + MIMEType: "image/png", + } + h := &Handler{imagePublisher: pub} + + got := h.publishQuestionImage(context.Background(), image) + if got != "https://weknora-api.example/i/001" { + t.Fatalf("url=%q, want published", got) + } + if pub.calls != 1 || pub.mime != "image/png" || string(pub.published) != string(raw) { + t.Fatalf("publisher calls=%d mime=%q bytes=%q", pub.calls, pub.mime, pub.published) + } +} + +func TestPublishQuestionImageFailsClosedWithoutURL(t *testing.T) { + h := &Handler{imagePublisher: nil} + if got := h.publishQuestionImage(context.Background(), core.ImageInput{}); got != "" { + t.Fatalf("expected empty URL without publisher, got %q", got) + } + + pub := &fakeImagePublisher{err: errors.New("oss down")} + h2 := &Handler{imagePublisher: pub} + image := core.ImageInput{ + DataURI: "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("x")), + MIMEType: "image/png", + } + if got := h2.publishQuestionImage(context.Background(), image); got != "" { + t.Fatalf("expected empty URL on publish error, got %q", got) + } +} \ No newline at end of file diff --git a/internal/channels/dingtalk/markdown_stream.go b/internal/channels/dingtalk/markdown_stream.go new file mode 100644 index 0000000..1830f70 --- /dev/null +++ b/internal/channels/dingtalk/markdown_stream.go @@ -0,0 +1,114 @@ +package dingtalk + +import "strings" + +type markdownStreamLine struct { + start int + text string + inFence bool +} + +// stableMarkdownPrefix returns the cumulative content that is safe to render +// while Markdown is still streaming. Once a table is confirmed, the table and +// every following token stay hidden until the terminal card update. +func stableMarkdownPrefix(content string) string { + lines := splitMarkdownStreamLines(content) + for i := 0; i < len(lines); i++ { + if lines[i].inFence || !isTableHeaderCandidate(lines[i].text) { + continue + } + + if i == len(lines)-1 { + return content[:lines[i].start] + } + if lines[i+1].inFence || !isTableDelimiterCandidate(lines[i+1].text) { + continue + } + if !isCompleteTableDelimiter(lines[i+1].text) { + return content[:lines[i].start] + } + // DingTalk only lays out Markdown tables reliably on the terminal + // isFinalize=true update. Keep the table and every following partial + // token hidden; the authoritative final update bypasses this helper. + return content[:lines[i].start] + } + return content +} + +func splitMarkdownStreamLines(content string) []markdownStreamLine { + lines := make([]markdownStreamLine, 0, strings.Count(content, "\n")+1) + fence := "" + for start := 0; start < len(content); { + end := strings.IndexByte(content[start:], '\n') + if end < 0 { + end = len(content) + } else { + end += start + 1 + } + text := strings.TrimSuffix(content[start:end], "\n") + text = strings.TrimSuffix(text, "\r") + trimmed := strings.TrimSpace(text) + line := markdownStreamLine{start: start, text: text, inFence: fence != ""} + + if marker := markdownFenceMarker(trimmed); marker != "" { + line.inFence = true + if fence == "" { + fence = marker + } else if marker == fence { + fence = "" + } + } + lines = append(lines, line) + start = end + } + return lines +} + +func markdownFenceMarker(trimmed string) string { + switch { + case strings.HasPrefix(trimmed, "```"): + return "```" + case strings.HasPrefix(trimmed, "~~~"): + return "~~~" + default: + return "" + } +} + +func isTableHeaderCandidate(line string) bool { + return strings.HasPrefix(strings.TrimSpace(line), "|") +} + +func isTableDelimiterCandidate(line string) bool { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "|") { + return false + } + for _, r := range trimmed { + switch r { + case '|', ':', '-', ' ', '\t': + default: + return false + } + } + return true +} + +func isCompleteTableDelimiter(line string) bool { + trimmed := strings.TrimSpace(line) + trimmed = strings.TrimPrefix(trimmed, "|") + trimmed = strings.TrimSuffix(trimmed, "|") + cells := strings.Split(trimmed, "|") + if len(cells) < 2 { + return false + } + for _, cell := range cells { + cell = strings.TrimSpace(cell) + cell = strings.TrimPrefix(cell, ":") + cell = strings.TrimSuffix(cell, ":") + if len(cell) < 3 || strings.Trim(cell, "-") != "" { + return false + } + } + return true +} diff --git a/internal/channels/dingtalk/markdown_stream_handler_test.go b/internal/channels/dingtalk/markdown_stream_handler_test.go new file mode 100644 index 0000000..212f9f0 --- /dev/null +++ b/internal/channels/dingtalk/markdown_stream_handler_test.go @@ -0,0 +1,65 @@ +package dingtalk + +import ( + "context" + "reflect" + "testing" + "time" + + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/weknora" +) + +type pacedTableEventQA struct { + fakeStreamingQA + events []weknora.StreamEvent + pause time.Duration +} + +func (f *pacedTableEventQA) AskEvents(_ context.Context, req core.QARequest, observer weknora.StreamObserver) (core.QAResult, error) { + f.lastReq = req + for _, event := range f.events { + if observer != nil { + observer(event) + } + time.Sleep(f.pause) + } + return f.result, f.err +} + +func TestHandleEventStreamingPublishesTableOnlyOnFinalUpdate(t *testing.T) { + answer := "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n\n结论" + qa := &pacedTableEventQA{ + fakeStreamingQA: fakeStreamingQA{result: core.QAResult{Answer: answer}}, + events: []weknora.StreamEvent{ + {ResponseType: "answer", FullAnswer: "说明\n\n"}, + {ResponseType: "answer", FullAnswer: "说明\n\n| 检查项"}, + {ResponseType: "answer", FullAnswer: "说明\n\n| 检查项 | 操作要求 |\n| --- | --"}, + {ResponseType: "answer", FullAnswer: answer, Done: true}, + }, + pause: 20 * time.Millisecond, + } + reply := &fakeReply{} + card := &fakeStreamingCard{startID: "track-event-table"} + h := newTestHandlerWithStreaming(t, qa, reply, card) + h.cardUpdateInterval = time.Millisecond + + if err := h.Handle(context.Background(), directTextCallback()); err != nil { + t.Fatal(err) + } + + var partials []string + for _, update := range card.updates { + if !update.isFinalize { + partials = append(partials, update.content) + } + } + wantPartials := []string{"说明\n\n"} + if !reflect.DeepEqual(partials, wantPartials) { + t.Fatalf("event partial updates=%q, want only pre-table stream %q", partials, wantPartials) + } + final := card.updates[len(card.updates)-1] + if !final.isFinalize || final.content != answer { + t.Fatalf("final=%+v, want unchanged complete answer", final) + } +} diff --git a/internal/channels/dingtalk/markdown_stream_test.go b/internal/channels/dingtalk/markdown_stream_test.go new file mode 100644 index 0000000..7ff1330 --- /dev/null +++ b/internal/channels/dingtalk/markdown_stream_test.go @@ -0,0 +1,101 @@ +package dingtalk + +import "testing" + +func TestStableMarkdownPrefixHoldsCompletedTableUntilFinalization(t *testing.T) { + content := "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n\n表后正文仍在生成" + want := "说明\n\n" + + if got := stableMarkdownPrefix(content); got != want { + t.Fatalf("stableMarkdownPrefix()=%q, want table and trailing stream held as %q", got, want) + } +} + +func TestStableMarkdownPrefixHoldsBackTableUntilFinalization(t *testing.T) { + tests := []struct { + name string + content string + want string + }{ + { + name: "split header", + content: "说明\n\n| 检查项", + want: "说明\n\n", + }, + { + name: "header waiting for delimiter", + content: "说明\n\n| 检查项 | 操作要求 |\n", + want: "说明\n\n", + }, + { + name: "partial delimiter", + content: "说明\n\n| 检查项 | 操作要求 |\n| --- | --", + want: "说明\n\n", + }, + { + name: "partial data row", + content: "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确", + want: "说明\n\n", + }, + { + name: "complete table without terminator", + content: "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n", + want: "说明\n\n", + }, + { + name: "complete table followed by blank line", + content: "说明\n\n| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n\n后文", + want: "说明\n\n", + }, + { + name: "complete table followed by heading", + content: "| 检查项 | 操作要求 |\n| --- | --- |\n| 接地检查 | 上电前确认 |\n## 后续", + want: "", + }, + { + name: "second consecutive table remains hidden", + content: "| A | B |\n| --- | --- |\n| 1 | 2 |\n\n| C | D", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := stableMarkdownPrefix(tt.content); got != tt.want { + t.Fatalf("stableMarkdownPrefix()=%q, want %q", got, tt.want) + } + }) + } +} + +func TestStableMarkdownPrefixLeavesNonTablesUnchanged(t *testing.T) { + tests := []struct { + name string + content string + }{ + { + name: "ordinary text", + content: "普通段落继续流式显示。", + }, + { + name: "ordinary inline pipe", + content: "状态使用 ready | running | done 表示。", + }, + { + name: "pipe inside fenced code", + content: "```text\n| 这不是 | 表格 |\n| --- | --- |\n```", + }, + { + name: "completed prose line starting with pipe", + content: "| 这是引用样式文本\n下一段普通文字", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := stableMarkdownPrefix(tt.content); got != tt.content { + t.Fatalf("stableMarkdownPrefix()=%q, want unchanged %q", got, tt.content) + } + }) + } +} diff --git a/internal/channels/dingtalk/stream.go b/internal/channels/dingtalk/stream.go index 2195c9f..e6706ad 100644 --- a/internal/channels/dingtalk/stream.go +++ b/internal/channels/dingtalk/stream.go @@ -86,10 +86,13 @@ func BotCallbackFromSDK(data *chatbot.BotCallbackDataModel) (BotCallback, error) Text: data.Text.Content, } if msgtype == "richtext" { - rtMsgtype, rtText, rtCode := parseRichText(data.Content) + rtMsgtype, rtText, rtCodes := parseRichText(data.Content) cb.Msgtype = rtMsgtype cb.Text = rtText - cb.DownloadCode = rtCode + cb.DownloadCodes = rtCodes + if len(rtCodes) > 0 { + cb.DownloadCode = rtCodes[0] + } return cb, nil } if msgtype == "link" { @@ -178,19 +181,19 @@ func safeHTTPURL(raw string) string { // 3. Nested envelope: {"content":{"richText":[...]}} // // Picture items are identified by a "type" field (case-insensitive trimmed) -// equal to "picture". The first non-empty downloadCode is used; subsequent -// picture items are ignored. Text items are joined with newlines. +// equal to "picture". All non-empty download codes are collected in order; +// text items are joined with newlines. // -// Returns: (msgtype, text, downloadCode). Msgtype is "picture" if any picture +// Returns: (msgtype, text, downloadCodes). Msgtype is "picture" if any picture // item exists, "text" if only text items, "richtext" if malformed/empty. -func parseRichText(raw interface{}) (string, string, string) { +func parseRichText(raw interface{}) (string, string, []string) { items := extractRichTextItems(raw) if len(items) == 0 { - return "richtext", "", "" + return "richtext", "", nil } var textParts []string - var downloadCode string + var downloadCodes []string hasPicture := false for _, item := range items { @@ -198,11 +201,9 @@ func parseRichText(raw interface{}) (string, string, string) { if itemType == "picture" { hasPicture = true - if downloadCode == "" { - code := strings.TrimSpace(item.DownloadCode) - if code != "" { - downloadCode = code - } + code := strings.TrimSpace(item.DownloadCode) + if code != "" { + downloadCodes = append(downloadCodes, code) } continue } @@ -216,12 +217,12 @@ func parseRichText(raw interface{}) (string, string, string) { text := strings.Join(textParts, "\n") if hasPicture { - return "picture", text, downloadCode + return "picture", text, downloadCodes } if text != "" { - return "text", text, "" + return "text", text, nil } - return "richtext", "", "" + return "richtext", "", nil } // richTextItem is a single item in a richText array. diff --git a/internal/channels/dingtalk/stream_test.go b/internal/channels/dingtalk/stream_test.go index 52fabe2..733bc30 100644 --- a/internal/channels/dingtalk/stream_test.go +++ b/internal/channels/dingtalk/stream_test.go @@ -2,6 +2,7 @@ package dingtalk import ( "context" + "reflect" "strings" "sync" "testing" @@ -186,7 +187,7 @@ func TestBotCallbackFromSDK(t *testing.T) { SessionWebhook: "https://oapi.dingtalk.com/robot/sendBySession?session=abc", Text: " 你好 ", } - if cb != want { + if !reflect.DeepEqual(cb, want) { t.Fatalf("BotCallbackFromSDK() = %+v, want %+v", cb, want) } }) @@ -450,7 +451,7 @@ func TestBotCallbackFromSDKRichText(t *testing.T) { } }) - t.Run("first non-empty downloadCode used, subsequent ignored", func(t *testing.T) { + t.Run("all picture downloadCodes collected in order", func(t *testing.T) { t.Parallel() data := &chatbot.BotCallbackDataModel{ ConversationType: "1", @@ -468,6 +469,9 @@ func TestBotCallbackFromSDKRichText(t *testing.T) { if cb.DownloadCode != "code-first" { t.Fatalf("DownloadCode = %q, want first code-first", cb.DownloadCode) } + if len(cb.DownloadCodes) != 2 || cb.DownloadCodes[0] != "code-first" || cb.DownloadCodes[1] != "code-second" { + t.Fatalf("DownloadCodes = %+v, want [code-first code-second]", cb.DownloadCodes) + } }) t.Run("picture item missing downloadCode still picture", func(t *testing.T) { diff --git a/internal/core/types.go b/internal/core/types.go index aea6acb..760d143 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -17,13 +17,15 @@ var ErrTurnInProgress = errors.New("turn is already processing") // ImageInput contains the image data for a QA request. The DataURI is a // full data: URI (e.g. "data:image/png;base64,...") and must never be -// persisted to the audit log. +// persisted to the audit log. PublishedURL is an optional stable HTTPS URL +// produced after the image is published to object storage for traceability. type ImageInput struct { - DataURI string - MIMEType string - SizeBytes int64 - Width int - Height int + DataURI string + MIMEType string + SizeBytes int64 + Width int + Height int + PublishedURL string } // MediaAudit records metadata about media in a turn for audit purposes. @@ -117,6 +119,8 @@ type Turn struct { Answer string References []Reference Media []MediaAudit + QuestionImageURLs []string + KnowledgeSearched bool Status string ErrorMessage string } diff --git a/internal/coverage/evaluator.go b/internal/coverage/evaluator.go index c1c8be0..2aa98f6 100644 --- a/internal/coverage/evaluator.go +++ b/internal/coverage/evaluator.go @@ -54,8 +54,14 @@ func (e *Evaluator) Version() string { // rules (in priority order): // 1. Failed/processing turns → ErrTurnNotSucceeded // 2. Answer contains an explicit "no answer" phrase → uncovered_explicit_no_answer -// 3. Zero references (nil or empty) → uncovered_no_references -// 4. Has references → covered_has_references +// 3. Image turns (image media present) are classified by whether the question +// actually triggered a knowledge-base search: +// - has references → covered_has_references +// - no references + knowledge searched → uncovered_no_references +// - no references + no knowledge search → covered_visual_recognition +// (pure visual recognition is not a coverage miss) +// 4. Text turns: zero references → uncovered_no_references +// 5. Has references → covered_has_references // // v1 never uses a score threshold. TopScore captures the highest reference score // (0 if no references). @@ -82,6 +88,44 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) { }, nil } + // Image turns are classified by whether knowledge search occurred. + if hasImageMedia(turn.Media) { + if len(turn.References) > 0 { + return Decision{ + TurnID: turn.ID, + TenantKey: turn.TenantKey, + Channel: turn.Channel, + EvaluatorVersion: EvaluatorVersion, + Verdict: VerdictCovered, + ReasonCode: ReasonCoveredHasReferences, + ReferenceCount: len(turn.References), + TopScore: maxScore(turn.References), + }, nil + } + if turn.KnowledgeSearched { + return Decision{ + TurnID: turn.ID, + TenantKey: turn.TenantKey, + Channel: turn.Channel, + EvaluatorVersion: EvaluatorVersion, + Verdict: VerdictUncovered, + ReasonCode: ReasonUncoveredNoReferences, + ReferenceCount: 0, + TopScore: 0, + }, nil + } + return Decision{ + TurnID: turn.ID, + TenantKey: turn.TenantKey, + Channel: turn.Channel, + EvaluatorVersion: EvaluatorVersion, + Verdict: VerdictCovered, + ReasonCode: ReasonCoveredVisualRecognition, + ReferenceCount: 0, + TopScore: 0, + }, nil + } + // No references → uncovered. if len(turn.References) == 0 { return Decision{ @@ -109,6 +153,16 @@ func (e *Evaluator) Evaluate(turn core.Turn) (Decision, error) { }, nil } +// hasImageMedia reports whether the turn carried at least one image input. +func hasImageMedia(media []core.MediaAudit) bool { + for _, m := range media { + if strings.EqualFold(strings.TrimSpace(m.Type), "image") { + return true + } + } + return false +} + func isNoAnswer(answer string) bool { lower := strings.ToLower(answer) for _, phrase := range noAnswerPhrases { diff --git a/internal/coverage/evaluator_image_test.go b/internal/coverage/evaluator_image_test.go new file mode 100644 index 0000000..47c866d --- /dev/null +++ b/internal/coverage/evaluator_image_test.go @@ -0,0 +1,85 @@ +package coverage + +import ( + "testing" + + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" +) + +func imageTurn(answer string, refs []core.Reference, searched bool) core.Turn { + return core.Turn{ + ID: "img-turn", + Status: "succeeded", + MessageType: "image", + Answer: answer, + References: refs, + Media: []core.MediaAudit{{Type: "image", MIMEType: "image/jpeg"}}, + KnowledgeSearched: searched, + } +} + +func TestEvaluator_ImageVisualRecognitionNoSearch(t *testing.T) { + e := NewEvaluator() + turn := imageTurn("这张图片展示了一个手写的知识库数据处理与发布流程图。", nil, false) + + d, err := e.Evaluate(turn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Verdict != VerdictCovered { + t.Fatalf("verdict=%s, want covered (visual recognition is not a coverage miss)", d.Verdict) + } + if d.ReasonCode != ReasonCoveredVisualRecognition { + t.Fatalf("reason_code=%s, want %s", d.ReasonCode, ReasonCoveredVisualRecognition) + } +} + +func TestEvaluator_ImageSearchButNoReferencesIsUncovered(t *testing.T) { + e := NewEvaluator() + turn := imageTurn("根据图片中的错误信息,未能在知识库中找到对应的解决方案。", nil, true) + + d, err := e.Evaluate(turn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Verdict != VerdictUncovered { + t.Fatalf("verdict=%s, want uncovered (image turn searched KB but got no refs)", d.Verdict) + } + if d.ReasonCode != ReasonUncoveredNoReferences { + t.Fatalf("reason_code=%s, want %s", d.ReasonCode, ReasonUncoveredNoReferences) + } +} + +func TestEvaluator_ImageWithReferencesIsCovered(t *testing.T) { + e := NewEvaluator() + turn := imageTurn("根据知识库资料,这个是故障代码E5,请检查...", []core.Reference{{ID: "r1", Score: 0.9}}, true) + + d, err := e.Evaluate(turn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Verdict != VerdictCovered { + t.Fatalf("verdict=%s, want covered", d.Verdict) + } + if d.ReasonCode != ReasonCoveredHasReferences { + t.Fatalf("reason_code=%s, want %s", d.ReasonCode, ReasonCoveredHasReferences) + } +} + +func TestEvaluator_TextTurnNoReferencesStillUncovered(t *testing.T) { + e := NewEvaluator() + turn := core.Turn{ + ID: "text-turn", + Status: "succeeded", + Answer: "这是一个正常的回答,但没有任何引用", + MessageType: "text", + References: []core.Reference{}, + } + d, err := e.Evaluate(turn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Verdict != VerdictUncovered || d.ReasonCode != ReasonUncoveredNoReferences { + t.Fatalf("verdict=%s reason=%s, want uncovered_no_references (text turn unchanged)", d.Verdict, d.ReasonCode) + } +} diff --git a/internal/coverage/types.go b/internal/coverage/types.go index 553c9d4..086815a 100644 --- a/internal/coverage/types.go +++ b/internal/coverage/types.go @@ -28,6 +28,7 @@ const ( ReasonCoveredHasReferences = "covered_has_references" ReasonUncoveredNoReferences = "uncovered_no_references" ReasonUncoveredExplicitNoAnswer = "uncovered_explicit_no_answer" + ReasonCoveredVisualRecognition = "covered_visual_recognition" ) // EvaluatorVersion is the current version of the evaluator. diff --git a/internal/outboundimage/aliyun_oss.go b/internal/outboundimage/aliyun_oss.go index a629f6a..6572fb4 100644 --- a/internal/outboundimage/aliyun_oss.go +++ b/internal/outboundimage/aliyun_oss.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "errors" + "io" "strings" - "time" "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" @@ -49,16 +49,31 @@ func (s *AliyunOSSStore) Put(ctx context.Context, key string, data []byte, conte return nil } -func (s *AliyunOSSStore) PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) { - if s == nil || s.client == nil || strings.TrimSpace(key) == "" || ttl <= 0 { - return "", errors.New("outbound image: invalid OSS presign request") +func (s *AliyunOSSStore) Get(ctx context.Context, key string, maxBytes int64) ([]byte, string, error) { + if s == nil || s.client == nil || strings.TrimSpace(key) == "" || maxBytes <= 0 { + return nil, "", errors.New("outbound image: invalid OSS get request") } - result, err := s.client.Presign(ctx, &oss.GetObjectRequest{ + result, err := s.client.GetObject(ctx, &oss.GetObjectRequest{ Bucket: oss.Ptr(s.bucket), Key: oss.Ptr(key), - }, oss.PresignExpires(ttl)) + }) if err != nil { - return "", errors.New("outbound image: OSS presign failed") + return nil, "", errors.New("outbound image: OSS get failed") } - return result.URL, nil + defer result.Body.Close() + if result.ContentLength > maxBytes { + return nil, "", errors.New("outbound image: OSS object exceeds size limit") + } + data, err := io.ReadAll(io.LimitReader(result.Body, maxBytes+1)) + if err != nil { + return nil, "", errors.New("outbound image: OSS read failed") + } + if int64(len(data)) > maxBytes { + return nil, "", errors.New("outbound image: OSS object exceeds size limit") + } + contentType := "" + if result.ContentType != nil { + contentType = strings.TrimSpace(*result.ContentType) + } + return data, contentType, nil } diff --git a/internal/outboundimage/aliyun_oss_test.go b/internal/outboundimage/aliyun_oss_test.go new file mode 100644 index 0000000..f79b8a4 --- /dev/null +++ b/internal/outboundimage/aliyun_oss_test.go @@ -0,0 +1,58 @@ +package outboundimage + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" +) + +func TestAliyunOSSStoreGetReturnsBoundedImageBytesAndMIME(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/private-bucket/dobotai/im-images/id" { + t.Fatalf("method=%s path=%s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Content-Length", "4") + _, _ = w.Write([]byte("PNG!")) + })) + defer server.Close() + + config := oss.LoadDefaultConfig(). + WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test-id", "test-secret", "")). + WithRegion("cn-test"). + WithEndpoint(server.URL). + WithUsePathStyle(true) + store := &AliyunOSSStore{client: oss.NewClient(config), bucket: "private-bucket"} + + data, contentType, err := store.Get(context.Background(), "dobotai/im-images/id", 4) + if err != nil { + t.Fatal(err) + } + if string(data) != "PNG!" || contentType != "image/png" { + t.Fatalf("data=%q contentType=%q", data, contentType) + } +} + +func TestAliyunOSSStoreGetRejectsOversizedObjects(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Content-Length", "5") + _, _ = w.Write([]byte("PNG!!")) + })) + defer server.Close() + + config := oss.LoadDefaultConfig(). + WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test-id", "test-secret", "")). + WithRegion("cn-test"). + WithEndpoint(server.URL). + WithUsePathStyle(true) + store := &AliyunOSSStore{client: oss.NewClient(config), bucket: "private-bucket"} + + if _, _, err := store.Get(context.Background(), "dobotai/im-images/id", 4); err == nil { + t.Fatal("expected oversized object error") + } +} diff --git a/internal/outboundimage/publisher.go b/internal/outboundimage/publisher.go index 9bbac11..3ec7e25 100644 --- a/internal/outboundimage/publisher.go +++ b/internal/outboundimage/publisher.go @@ -5,48 +5,57 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" + "net/url" "path" + "regexp" "strings" - "time" ) -type ObjectStore interface { +var stableImageIDRE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + +type ObjectWriter interface { Put(ctx context.Context, key string, data []byte, contentType string) error - PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) } -type HashPublisher struct { - store ObjectStore - prefix string - ttl time.Duration +type StableHashPublisher struct { + store ObjectWriter + prefix string + publicBaseURL string } -func NewHashPublisher(store ObjectStore, prefix string, ttl time.Duration) *HashPublisher { - return &HashPublisher{store: store, prefix: strings.Trim(strings.TrimSpace(prefix), "/"), ttl: ttl} -} - -func (p *HashPublisher) Publish(ctx context.Context, data []byte, contentType string) (string, error) { - if p == nil || p.store == nil || len(data) == 0 || p.prefix == "" || p.ttl <= 0 { - return "", errors.New("outbound image: invalid publisher configuration") +func NewStableHashPublisher(store ObjectWriter, prefix, publicBaseURL string) (*StableHashPublisher, error) { + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + publicBaseURL = strings.TrimRight(strings.TrimSpace(publicBaseURL), "/") + parsed, err := url.Parse(publicBaseURL) + if store == nil || prefix == "" || err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return nil, errors.New("outbound image: invalid stable publisher configuration") } - extension := imageExtension(contentType) - if extension == "" { + return &StableHashPublisher{ + store: store, + prefix: prefix, + publicBaseURL: publicBaseURL, + }, nil +} + +func (p *StableHashPublisher) Publish(ctx context.Context, data []byte, contentType string) (string, error) { + if p == nil || p.store == nil || len(data) == 0 || p.prefix == "" || p.publicBaseURL == "" { + return "", errors.New("outbound image: invalid stable publisher configuration") + } + if imageExtension(contentType) == "" { return "", errors.New("outbound image: unsupported image content type") } sum := sha256.Sum256(data) - digest := hex.EncodeToString(sum[:]) - key := path.Join(p.prefix, digest[:2], digest+extension) + idBytes := sum[:16] + idBytes[6] = (idBytes[6] & 0x0f) | 0x50 + idBytes[8] = (idBytes[8] & 0x3f) | 0x80 + idHex := hex.EncodeToString(idBytes) + id := fmt.Sprintf("%s-%s-%s-%s-%s", idHex[:8], idHex[8:12], idHex[12:16], idHex[16:20], idHex[20:]) + key := path.Join(p.prefix, id) if err := p.store.Put(ctx, key, data, contentType); err != nil { return "", errors.New("outbound image: publish object failed") } - signedURL, err := p.store.PresignGet(ctx, key, p.ttl) - if err != nil { - return "", errors.New("outbound image: sign object URL failed") - } - if !strings.HasPrefix(strings.TrimSpace(signedURL), "https://") { - return "", errors.New("outbound image: signed object URL must use HTTPS") - } - return signedURL, nil + return p.publicBaseURL + "/" + id, nil } func imageExtension(contentType string) string { diff --git a/internal/outboundimage/publisher_test.go b/internal/outboundimage/publisher_test.go index 9c83284..4cd076a 100644 --- a/internal/outboundimage/publisher_test.go +++ b/internal/outboundimage/publisher_test.go @@ -4,14 +4,12 @@ import ( "context" "strings" "testing" - "time" ) type fakeObjectStore struct { key string data []byte contentType string - ttl time.Duration } func (s *fakeObjectStore) Put(_ context.Context, key string, data []byte, contentType string) error { @@ -21,14 +19,16 @@ func (s *fakeObjectStore) Put(_ context.Context, key string, data []byte, conten return nil } -func (s *fakeObjectStore) PresignGet(_ context.Context, key string, ttl time.Duration) (string, error) { - s.ttl = ttl - return "https://private-oss.example/" + key + "?signature=short", nil -} - -func TestHashPublisherUsesDeterministicTemporaryKeyAndTwoHourHTTPSURL(t *testing.T) { +func TestStableHashPublisherUsesDeterministicUUIDURLWithoutPresigning(t *testing.T) { store := &fakeObjectStore{} - publisher := NewHashPublisher(store, "dobotai/im-images", 2*time.Hour) + publisher, err := NewStableHashPublisher( + store, + "dobotai/im-images", + "https://weknora-api-test.dobot.cn/dingtalk/i", + ) + if err != nil { + t.Fatal(err) + } first, err := publisher.Publish(context.Background(), []byte("same-image"), "image/jpeg") if err != nil { @@ -43,13 +43,48 @@ func TestHashPublisherUsesDeterministicTemporaryKeyAndTwoHourHTTPSURL(t *testing if first != second || firstKey != store.key { t.Fatalf("publish is not deterministic: url1=%q url2=%q key1=%q key2=%q", first, second, firstKey, store.key) } - if !strings.HasPrefix(store.key, "dobotai/im-images/") || !strings.HasSuffix(store.key, ".jpg") { - t.Fatalf("key=%q", store.key) + + wantURLPrefix := "https://weknora-api-test.dobot.cn/dingtalk/i/" + if !strings.HasPrefix(first, wantURLPrefix) { + t.Fatalf("url=%q", first) } - if strings.Contains(store.key, "same-image") { - t.Fatalf("key leaks content: %q", store.key) + id := strings.TrimPrefix(first, wantURLPrefix) + if !stableImageIDRE.MatchString(id) { + t.Fatalf("stable image id=%q", id) } - if store.ttl != 2*time.Hour || !strings.HasPrefix(first, "https://") { - t.Fatalf("ttl=%s url=%q", store.ttl, first) + if store.key != "dobotai/im-images/"+id { + t.Fatalf("key=%q id=%q", store.key, id) + } +} + +func TestStableHashPublisherRejectsUnsafeConfigurationAndNonImages(t *testing.T) { + store := &fakeObjectStore{} + for _, test := range []struct { + name string + store ObjectWriter + prefix string + baseURL string + }{ + {name: "nil store", prefix: "dobotai/im-images", baseURL: "https://img.example/i"}, + {name: "empty prefix", store: store, baseURL: "https://img.example/i"}, + {name: "http base", store: store, prefix: "dobotai/im-images", baseURL: "http://img.example/i"}, + {name: "query base", store: store, prefix: "dobotai/im-images", baseURL: "https://img.example/i?token=secret"}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := NewStableHashPublisher(test.store, test.prefix, test.baseURL); err == nil { + t.Fatal("expected configuration error") + } + }) + } + + publisher, err := NewStableHashPublisher(store, "dobotai/im-images", "https://img.example/i") + if err != nil { + t.Fatal(err) + } + if _, err := publisher.Publish(context.Background(), nil, "image/png"); err == nil { + t.Fatal("expected empty image error") + } + if _, err := publisher.Publish(context.Background(), []byte("text"), "text/plain"); err == nil { + t.Fatal("expected non-image error") } } diff --git a/internal/outboundimage/rewriter.go b/internal/outboundimage/rewriter.go index 635905f..0942c95 100644 --- a/internal/outboundimage/rewriter.go +++ b/internal/outboundimage/rewriter.go @@ -2,11 +2,29 @@ package outboundimage import ( "context" + "net" + "net/url" + "path" "regexp" "strings" ) -var resourceMarkdownImageRE = regexp.MustCompile(`!\[[^\]]*\]\((resource://[0-9A-Za-z_-]+)\)`) +var ( + resourceMarkdownImageRE = regexp.MustCompile(`!\[[^\]]*\]\((resource://[0-9A-Za-z_-]+)(?:\s+(?:"[^"\r\n]*"|'[^'\r\n]*'|\([^\)\r\n]*\)))?\)`) + resourceHTMLDoubleImageRE = regexp.MustCompile(`(?i)]*\bsrc\s*=\s*"(resource://[0-9A-Za-z_-]+)"[^>]*>`) + resourceHTMLSingleImageRE = regexp.MustCompile(`(?i)]*\bsrc\s*=\s*'(resource://[0-9A-Za-z_-]+)'[^>]*>`) + resourceHTMLUnsafeImageRE = regexp.MustCompile(`(?i)]*resource://[0-9A-Za-z_-]+[^>]*>`) + markdownImageRE = regexp.MustCompile(`!\[[^\]]*\]\(([^\s)]+)(?:\s+(?:"[^"\r\n]*"|'[^'\r\n]*'|\([^\)\r\n]*\)))?\)`) + incompleteMarkdownImageRE = regexp.MustCompile(`!\[[^\]\r\n]*\]\([^\)\r\n]*$`) + htmlImageRE = regexp.MustCompile(`(?i)]*>`) + incompleteHTMLImageRE = regexp.MustCompile(`(?i)]*$`) + htmlDoubleSrcRE = regexp.MustCompile(`(?i)\bsrc\s*=\s*"([^"]+)"`) + htmlSingleSrcRE = regexp.MustCompile(`(?i)\bsrc\s*=\s*'([^']+)'`) + htmlDoubleAltRE = regexp.MustCompile(`(?i)\balt\s*=\s*"([^"]*)"`) + htmlSingleAltRE = regexp.MustCompile(`(?i)\balt\s*=\s*'([^']*)'`) + resourceReferenceRE = regexp.MustCompile(`resource://[0-9A-Za-z_-]+`) + dataImageReferenceRE = regexp.MustCompile(`(?i)data:image[^\s]*`) +) type Fetcher interface { Fetch(ctx context.Context, reference string) ([]byte, string, error) @@ -28,21 +46,15 @@ func NewRewriter(fetcher Fetcher, publisher Publisher, maxImages int) *Rewriter func (r *Rewriter) Rewrite(ctx context.Context, markdown string) string { if r == nil || r.fetcher == nil || r.publisher == nil || r.maxImages <= 0 { - return resourceMarkdownImageRE.ReplaceAllString(markdown, "") + markdown = normalizePublicHTMLImages(stripResourceImages(markdown)) + markdown = resourceReferenceRE.ReplaceAllString(resourceHTMLUnsafeImageRE.ReplaceAllString(markdown, ""), "") + return sanitizePublicMarkdownImages(markdown) } urls := map[string]string{} attempted := 0 - return resourceMarkdownImageRE.ReplaceAllStringFunc(markdown, func(image string) string { - match := resourceMarkdownImageRE.FindStringSubmatch(image) - if len(match) != 2 { - return "" - } - reference := match[1] - if signedURL, ok := urls[reference]; ok { - if signedURL == "" { - return "" - } - return strings.Replace(image, reference, signedURL, 1) + resolve := func(reference string) string { + if publishedURL, ok := urls[reference]; ok { + return publishedURL } if attempted >= r.maxImages { urls[reference] = "" @@ -54,12 +66,147 @@ func (r *Rewriter) Rewrite(ctx context.Context, markdown string) string { urls[reference] = "" return "" } - signedURL, err := r.publisher.Publish(ctx, data, contentType) - if err != nil || !strings.HasPrefix(signedURL, "https://") { + publishedURL, err := r.publisher.Publish(ctx, data, contentType) + if err != nil || !strings.HasPrefix(publishedURL, "https://") { urls[reference] = "" return "" } - urls[reference] = signedURL - return strings.Replace(image, reference, signedURL, 1) + urls[reference] = publishedURL + return publishedURL + } + + markdown = resourceMarkdownImageRE.ReplaceAllStringFunc(markdown, func(image string) string { + match := resourceMarkdownImageRE.FindStringSubmatch(image) + if len(match) != 2 { + return "" + } + reference := match[1] + publishedURL := resolve(reference) + if publishedURL == "" { + return "" + } + return strings.Replace(image, reference, publishedURL, 1) + }) + markdown = rewriteHTMLImages(markdown, resourceHTMLDoubleImageRE, resolve) + markdown = rewriteHTMLImages(markdown, resourceHTMLSingleImageRE, resolve) + markdown = normalizePublicHTMLImages(markdown) + markdown = resourceReferenceRE.ReplaceAllString(resourceHTMLUnsafeImageRE.ReplaceAllString(markdown, ""), "") + return sanitizePublicMarkdownImages(markdown) +} + +func sanitizePublicMarkdownImages(markdown string) string { + markdown = markdownImageRE.ReplaceAllStringFunc(markdown, func(image string) string { + match := markdownImageRE.FindStringSubmatch(image) + if len(match) != 2 || !isSafePublicImageURL(match[1]) { + return "" + } + if isKnownDocumentURL(match[1]) { + return strings.TrimPrefix(image, "!") + } + return image + }) + markdown = incompleteMarkdownImageRE.ReplaceAllString(markdown, "") + markdown = incompleteHTMLImageRE.ReplaceAllString(markdown, "") + return dataImageReferenceRE.ReplaceAllString(markdown, "") +} + +func isKnownDocumentURL(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return false + } + switch strings.ToLower(path.Ext(u.Path)) { + case ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", + ".csv", ".txt", ".zip", ".rar", ".7z": + return true + default: + return false + } +} + +func rewriteHTMLImages(content string, pattern *regexp.Regexp, resolve func(string) string) string { + return pattern.ReplaceAllStringFunc(content, func(image string) string { + match := pattern.FindStringSubmatch(image) + if len(match) != 2 { + return "" + } + publishedURL := resolve(match[1]) + if publishedURL == "" { + return "" + } + alt := "图片" + for _, pattern := range []*regexp.Regexp{htmlDoubleAltRE, htmlSingleAltRE} { + if altMatch := pattern.FindStringSubmatch(image); len(altMatch) == 2 && strings.TrimSpace(altMatch[1]) != "" { + alt = strings.ReplaceAll(strings.TrimSpace(altMatch[1]), "]", `\]`) + break + } + } + return "![" + alt + "](" + publishedURL + ")" }) } + +func normalizePublicHTMLImages(content string) string { + return htmlImageRE.ReplaceAllStringFunc(content, func(image string) string { + source := "" + for _, pattern := range []*regexp.Regexp{htmlDoubleSrcRE, htmlSingleSrcRE} { + if match := pattern.FindStringSubmatch(image); len(match) == 2 { + source = strings.TrimSpace(match[1]) + break + } + } + if !isSafePublicImageURL(source) { + return "" + } + return "![" + imageAlt(image) + "](" + source + ")" + }) +} + +func imageAlt(image string) string { + for _, pattern := range []*regexp.Regexp{htmlDoubleAltRE, htmlSingleAltRE} { + if match := pattern.FindStringSubmatch(image); len(match) == 2 && strings.TrimSpace(match[1]) != "" { + return strings.ReplaceAll(strings.TrimSpace(match[1]), "]", `\]`) + } + } + return "图片" +} + +func isSafePublicImageURL(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil { + return false + } + hostname := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + if hostname == "" || hostname == "localhost" || strings.HasSuffix(hostname, ".localhost") || strings.HasSuffix(hostname, ".local") || strings.HasSuffix(hostname, ".internal") { + return false + } + if port := u.Port(); port != "" && port != "443" { + return false + } + if ip := net.ParseIP(hostname); ip != nil { + if !ip.IsGlobalUnicast() || ip.IsPrivate() { + return false + } + } else if !strings.Contains(hostname, ".") { + return false + } + for key := range u.Query() { + normalized := strings.ToLower(key) + if strings.Contains(normalized, "signature") || normalized == "ossaccesskeyid" || strings.Contains(normalized, "credential") { + return false + } + } + return true +} + +func stripResourceImages(content string) string { + content = resourceMarkdownImageRE.ReplaceAllString(content, "") + content = resourceHTMLDoubleImageRE.ReplaceAllString(content, "") + content = resourceHTMLSingleImageRE.ReplaceAllString(content, "") + return resourceHTMLUnsafeImageRE.ReplaceAllString(content, "") +} + +func (r *Rewriter) SanitizePartial(content string) string { + content = resourceReferenceRE.ReplaceAllString(stripResourceImages(content), "") + content = normalizePublicHTMLImages(content) + return sanitizePublicMarkdownImages(content) +} diff --git a/internal/outboundimage/rewriter_test.go b/internal/outboundimage/rewriter_test.go index 1c23809..d33c632 100644 --- a/internal/outboundimage/rewriter_test.go +++ b/internal/outboundimage/rewriter_test.go @@ -76,3 +76,302 @@ func TestRewriteMarkdownResourceImagesDropsFailuresAndImagesBeyondLimit(t *testi t.Fatalf("publisher calls=%d, want 2", publisher.calls) } } + +func TestRewriteHTMLResourceImagesAsDingTalkMarkdown(t *testing.T) { + resource := "resource://AbCdEfGhIjKlMnOpQrStUv" + fetcher := &fakeFetcher{} + publisher := &fakePublisher{} + r := NewRewriter(fetcher, publisher, 5) + input := "前文\n\"步骤图\"\n" + + "重复\n\n后文" + + got := r.Rewrite(context.Background(), input) + + if strings.Contains(got, "resource://") || strings.Contains(strings.ToLower(got), "\n" + + "内部引用 resource://BareRefAbCdEfGhIjKlMn" + + got := r.SanitizePartial(input) + + if strings.Contains(got, "resource://") || strings.Contains(strings.ToLower(got), "` + + got := r.Rewrite(context.Background(), input) + + if got != "![公网图](https://example.com/a.png)" { + t.Fatalf("normalized public HTML image=%q", got) + } +} + +func TestRewriteRemovesDataImageWithOptionalTitle(t *testing.T) { + r := NewRewriter(&fakeFetcher{}, &fakePublisher{}, 5) + input := "正文\n![占位图](data:image/png;base64,iVBORw0KGgo= \"\")\n后文" + + got := r.Rewrite(context.Background(), input) + + if strings.Contains(got, "data:image") || strings.Contains(got, "占位图") { + t.Fatalf("final content retained a data image: %s", got) + } + if !strings.Contains(got, "正文") || !strings.Contains(got, "后文") { + t.Fatalf("surrounding text was lost: %s", got) + } +} + +func TestSanitizePartialRemovesDataImageWithOptionalTitle(t *testing.T) { + r := NewRewriter(&fakeFetcher{}, &fakePublisher{}, 5) + input := "正文\n![占位图](data:image/png;base64,iVBORw0KGgo= \"\")\n后文" + + got := r.SanitizePartial(input) + + if strings.Contains(got, "data:image") || strings.Contains(got, "占位图") { + t.Fatalf("partial content retained a data image: %s", got) + } + if !strings.Contains(got, "正文") || !strings.Contains(got, "后文") { + t.Fatalf("surrounding text was lost: %s", got) + } +} + +func TestRewritePublishesResourceImageWithOptionalTitle(t *testing.T) { + resource := "resource://AbCdEfGhIjKlMnOpQrStUv" + fetcher := &fakeFetcher{} + publisher := &fakePublisher{} + r := NewRewriter(fetcher, publisher, 5) + input := "![步骤图](" + resource + " \"安装说明\")" + + got := r.Rewrite(context.Background(), input) + + if got != "![步骤图](https://img.example/1.png?sig=short \"安装说明\")" { + t.Fatalf("resource image with title=%q", got) + } + if fetcher.calls[resource] != 1 || publisher.calls != 1 { + t.Fatalf("calls fetch=%v publish=%d", fetcher.calls, publisher.calls) + } +} + +func TestSanitizePartialRemovesDataHTMLImage(t *testing.T) { + r := NewRewriter(&fakeFetcher{}, &fakePublisher{}, 5) + input := `正文占位图后文` + + got := r.SanitizePartial(input) + + if strings.Contains(got, "data:image") || strings.Contains(strings.ToLower(got), "` + if got, want := r.Rewrite(context.Background(), html), `[参数表](https://docs.example/manual.xlsx)`; got != want { + t.Fatalf("HTML document image rewrite=%q, want %q", got, want) + } + if got, want := r.SanitizePartial(html), `[参数表](https://docs.example/manual.xlsx)`; got != want { + t.Fatalf("partial HTML document image rewrite=%q, want %q", got, want) + } +} + +func TestDocumentImageDowngradePreservesImageAndLinkContracts(t *testing.T) { + r := NewRewriter(&fakeFetcher{}, &fakePublisher{}, 5) + input := strings.Join([]string{ + `![稳定图片](https://weknora-api.example/i/006d001d-7d0b-542e-b5ab-6ee7d7cdce54)`, + `[已有PDF链接](https://docs.example/manual.pdf)`, + `![危险PDF](https://127.0.0.1/manual.pdf)`, + }, "\n") + + got := r.Rewrite(context.Background(), input) + + if !strings.Contains(got, `![稳定图片](https://weknora-api.example/i/006d001d-7d0b-542e-b5ab-6ee7d7cdce54)`) { + t.Fatalf("extensionless stable image changed: %s", got) + } + if !strings.Contains(got, `[已有PDF链接](https://docs.example/manual.pdf)`) { + t.Fatalf("existing document link changed: %s", got) + } + if strings.Contains(got, "127.0.0.1") || strings.Contains(got, "危险PDF") { + t.Fatalf("unsafe document image became a link: %s", got) + } +} diff --git a/internal/outboundimage/stable_handler.go b/internal/outboundimage/stable_handler.go new file mode 100644 index 0000000..ff9c4a1 --- /dev/null +++ b/internal/outboundimage/stable_handler.go @@ -0,0 +1,48 @@ +package outboundimage + +import ( + "context" + "errors" + "net/http" + "path" + "strconv" + "strings" +) + +type ObjectReader interface { + Get(ctx context.Context, key string, maxBytes int64) ([]byte, string, error) +} + +func NewStableImageHandler(reader ObjectReader, objectPrefix, routePrefix string, maxBytes int64) (http.Handler, error) { + objectPrefix = strings.Trim(strings.TrimSpace(objectPrefix), "/") + routePrefix = "/" + strings.Trim(strings.TrimSpace(routePrefix), "/") + "/" + if reader == nil || objectPrefix == "" || maxBytes <= 0 { + return nil, errors.New("outbound image: invalid stable image handler configuration") + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.Header().Set("Allow", "GET, HEAD") + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, routePrefix) + if r.URL.Path == id || strings.Contains(id, "/") || !stableImageIDRE.MatchString(id) { + http.NotFound(w, r) + return + } + data, contentType, err := reader.Get(r.Context(), path.Join(objectPrefix, id), maxBytes) + if err != nil || len(data) == 0 || imageExtension(contentType) == "" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", strings.TrimSpace(strings.Split(contentType, ";")[0])) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("Cache-Control", "public, max-age=86400, immutable") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + if r.Method == http.MethodHead { + return + } + _, _ = w.Write(data) + }), nil +} diff --git a/internal/outboundimage/stable_handler_test.go b/internal/outboundimage/stable_handler_test.go new file mode 100644 index 0000000..26201cb --- /dev/null +++ b/internal/outboundimage/stable_handler_test.go @@ -0,0 +1,91 @@ +package outboundimage + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +type fakeImageReader struct { + key string + maxBytes int64 + data []byte + contentType string + err error +} + +func (r *fakeImageReader) Get(_ context.Context, key string, maxBytes int64) ([]byte, string, error) { + r.key = key + r.maxBytes = maxBytes + return append([]byte(nil), r.data...), r.contentType, r.err +} + +func TestStableImageHandlerReturnsImageBytesDirectly(t *testing.T) { + const id = "ca13f02b-58b6-5f8f-a80d-b41873a97425" + reader := &fakeImageReader{data: []byte("JPEG"), contentType: "image/jpeg"} + handler, err := NewStableImageHandler(reader, "dobotai/im-images", "/dingtalk/i/", 10<<20) + if err != nil { + t.Fatal(err) + } + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/dingtalk/i/"+id, nil)) + + if recorder.Code != http.StatusOK { + t.Fatalf("status=%d", recorder.Code) + } + if recorder.Header().Get("Location") != "" { + t.Fatalf("stable image endpoint must not redirect: location=%q", recorder.Header().Get("Location")) + } + if recorder.Header().Get("Content-Type") != "image/jpeg" { + t.Fatalf("content-type=%q", recorder.Header().Get("Content-Type")) + } + if recorder.Body.String() != "JPEG" { + t.Fatalf("body=%q", recorder.Body.String()) + } + if reader.key != "dobotai/im-images/"+id || reader.maxBytes != 10<<20 { + t.Fatalf("key=%q maxBytes=%d", reader.key, reader.maxBytes) + } +} + +func TestStableImageHandlerAllowsHeadWithoutBody(t *testing.T) { + const id = "ca13f02b-58b6-5f8f-a80d-b41873a97425" + reader := &fakeImageReader{data: []byte("JPEG"), contentType: "image/jpeg"} + handler, err := NewStableImageHandler(reader, "dobotai/im-images", "/dingtalk/i/", 10<<20) + if err != nil { + t.Fatal(err) + } + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodHead, "/dingtalk/i/"+id, nil)) + + if recorder.Code != http.StatusOK || recorder.Body.Len() != 0 { + t.Fatalf("status=%d body=%q", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Content-Length") != "4" { + t.Fatalf("content-length=%q", recorder.Header().Get("Content-Length")) + } +} + +func TestStableImageHandlerRejectsPostAndInvalidIDsWithoutReadingOSS(t *testing.T) { + reader := &fakeImageReader{data: []byte("JPEG"), contentType: "image/jpeg"} + handler, err := NewStableImageHandler(reader, "dobotai/im-images", "/dingtalk/i/", 10<<20) + if err != nil { + t.Fatal(err) + } + + post := httptest.NewRecorder() + handler.ServeHTTP(post, httptest.NewRequest(http.MethodPost, "/dingtalk/i/ca13f02b-58b6-5f8f-a80d-b41873a97425", nil)) + if post.Code != http.StatusMethodNotAllowed { + t.Fatalf("post status=%d", post.Code) + } + invalid := httptest.NewRecorder() + handler.ServeHTTP(invalid, httptest.NewRequest(http.MethodGet, "/dingtalk/i/../../secret", nil)) + if invalid.Code != http.StatusNotFound { + t.Fatalf("invalid status=%d", invalid.Code) + } + if reader.key != "" { + t.Fatalf("invalid requests reached OSS key=%q", reader.key) + } +} diff --git a/internal/qa/service.go b/internal/qa/service.go index 6b1dd4c..c2bff20 100644 --- a/internal/qa/service.go +++ b/internal/qa/service.go @@ -129,11 +129,15 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve } media := make([]core.MediaAudit, 0, len(request.Images)) + questionImageURLs := make([]string, 0, len(request.Images)) for _, image := range request.Images { media = append(media, core.MediaAudit{ Type: "image", MIMEType: image.MIMEType, SizeBytes: image.SizeBytes, Width: image.Width, Height: image.Height, }) + if url := strings.TrimSpace(image.PublishedURL); url != "" { + questionImageURLs = append(questionImageURLs, url) + } } messageType := normalizeMessageType(request) if messageType == "" { @@ -161,6 +165,7 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve RequestMessageID: newID(), Question: request.Query, Media: media, + QuestionImageURLs: questionImageURLs, Status: "processing", } stored, created, err := s.repository.BeginTurn(ctx, turn) @@ -214,20 +219,32 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve Query: query, AgentID: agentID, AgentEnabled: true, Channel: "im", EnableMemory: true, Images: images, } + // Track whether this turn actually triggered a knowledge-base search by + // observing knowledge_search tool-call events. The flag feeds coverage + // classification (image turns) and is persisted with the turn. + var knowledgeSearched bool + effectiveEventObserver := weknora.StreamObserver(func(e weknora.StreamEvent) { + if strings.EqualFold(strings.TrimSpace(e.ToolName), "knowledge_search") { + knowledgeSearched = true + } + if eventObserver != nil { + eventObserver(e) + } + }) runChat := func(targetSessionID string) (weknora.ChatResult, error) { var result weknora.ChatResult var runErr error if eventObserver != nil { // Prefer AgentChatEvents; fallback to AgentChatStream adaptation; then AgentChat. if eventStream, ok := s.weknora.(eventStreamingWeKnora); ok { - result, runErr = eventStream.AgentChatEvents(ctx, targetSessionID, chatRequest, eventObserver) + result, runErr = eventStream.AgentChatEvents(ctx, targetSessionID, chatRequest, effectiveEventObserver) } else if streaming, ok := s.weknora.(streamingWeKnora); ok { // Adapt AgentChatStream answer chunks to StreamEvent result, runErr = streaming.AgentChatStream(ctx, targetSessionID, chatRequest, func(full string) { if answerObserver != nil { answerObserver(full) } - eventObserver(weknora.StreamEvent{ + effectiveEventObserver(weknora.StreamEvent{ ResponseType: "answer", FullAnswer: full, }) @@ -238,7 +255,7 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve if answerObserver != nil { answerObserver(result.Answer) } - eventObserver(weknora.StreamEvent{ + effectiveEventObserver(weknora.StreamEvent{ ResponseType: "answer", FullAnswer: result.Answer, Done: true, @@ -284,6 +301,7 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve } turn.Status, turn.Answer, turn.References = "succeeded", chat.Answer, chat.References turn.WeKnoraRequestID = chat.RequestID + turn.KnowledgeSearched = knowledgeSearched turn.AnswerMessageID = newID() if err := s.repository.FinishTurn(ctx, turn); err != nil { return core.QAResult{}, fmt.Errorf("finish turn: %w", err) diff --git a/internal/questiontable/dispatcher.go b/internal/questiontable/dispatcher.go index 78dcc89..06d7e34 100644 --- a/internal/questiontable/dispatcher.go +++ b/internal/questiontable/dispatcher.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "strings" "time" ) @@ -38,6 +39,15 @@ type RecordInserter interface { InsertRecord(ctx context.Context, row Row) (string, error) } +// AnswerRewriter optionally transforms the answer before it is written to the +// question table, applying the same outbound conversion used for DingTalk +// delivery (resource images to stable HTTPS, document links, anchor cleanup). +// When nil or when Rewrite returns an error, the original answer is written so +// the record is never lost. +type AnswerRewriter interface { + Rewrite(ctx context.Context, content string) (string, error) +} + // DispatcherStats holds the outcome counters for a single RunOnce invocation. type DispatcherStats struct { Claimed int @@ -51,6 +61,7 @@ type DispatcherOptions struct { MaxAttempts int ClaimLease time.Duration SendTimeout time.Duration + Rewriter AnswerRewriter } // Dispatcher runs the background dispatch loop that claims pending exports @@ -58,6 +69,7 @@ type DispatcherOptions struct { type Dispatcher struct { repo ExportRepository inserter RecordInserter + rewriter AnswerRewriter batch int maxAttempts int @@ -96,8 +108,10 @@ func NewDispatcher(repo ExportRepository, inserter RecordInserter, opts *Dispatc maxAttempts := 5 claimLease := 5 * time.Minute sendTimeout := 10 * time.Second + var rewriter AnswerRewriter if opts != nil { + rewriter = opts.Rewriter if opts.Batch < 0 { return nil, fmt.Errorf("questiontable dispatcher: batch must be >= 0, got %d", opts.Batch) } @@ -130,6 +144,7 @@ func NewDispatcher(repo ExportRepository, inserter RecordInserter, opts *Dispatc return &Dispatcher{ repo: repo, inserter: inserter, + rewriter: rewriter, batch: batch, maxAttempts: maxAttempts, claimLease: claimLease, @@ -165,8 +180,18 @@ func (d *Dispatcher) RunOnce(ctx context.Context) (DispatcherStats, error) { return stats, fmt.Errorf("questiontable dispatcher: context cancelled during dispatch: %w", err) } + row := item.Row + if err := d.rewriteAnswer(ctx, &row); err != nil { + // Rewriting is best-effort: the original answer is written so the + // question-table record is never lost. Only the row is skipped when + // the parent context is cancelled. + if ctx.Err() != nil { + return stats, fmt.Errorf("questiontable dispatcher: context cancelled during rewrite: %w", ctx.Err()) + } + } + sendCtx, cancel := context.WithTimeout(ctx, d.sendTimeout) - recordID, insertErr := d.inserter.InsertRecord(sendCtx, item.Row) + recordID, insertErr := d.inserter.InsertRecord(sendCtx, row) cancel() if insertErr != nil { @@ -189,3 +214,20 @@ func (d *Dispatcher) RunOnce(ctx context.Context) (DispatcherStats, error) { return stats, nil } + +// rewriteAnswer transforms the outbound answer when an AnswerRewriter is +// configured. A nil rewriter or a rewriter error leaves the original answer +// intact; the row is never skipped for a rewriter error. +func (d *Dispatcher) rewriteAnswer(ctx context.Context, row *Row) error { + if d.rewriter == nil { + return nil + } + rewritten, err := d.rewriter.Rewrite(ctx, row.Answer) + if err != nil { + return err + } + if strings.TrimSpace(rewritten) != "" { + row.Answer = rewritten + } + return nil +} diff --git a/internal/questiontable/export_rewrite_test.go b/internal/questiontable/export_rewrite_test.go new file mode 100644 index 0000000..16a25d0 --- /dev/null +++ b/internal/questiontable/export_rewrite_test.go @@ -0,0 +1,137 @@ +package questiontable + +import ( + "context" + "strings" + "testing" + "time" +) + +type answerRewriterStub struct { + rewritten string + err error +} + +func (s answerRewriterStub) Rewrite(ctx context.Context, content string) (string, error) { + return s.rewritten, s.err +} + +func TestRunOnceRewritesAnswerBeforeInsert(t *testing.T) { + repo := newStubRepo() + inserter := newStubInserter() + inserter.recordID = "rec-rewrite-1" + repo.claimFn = func(ctx context.Context, limit int, maxAttempts int, staleBefore time.Time) ([]PendingExport, error) { + return []PendingExport{ + { + ExportID: "exp-rewrite", + Row: Row{ + Question: "机械臂如何学习使用呢", + Answer: "原始答案含[[3](#_Toc1)]和![图](resource://abc)", + + AskedTime: time.Now(), + AskerName: "张三", + AskerUnionID: "union-z", + TriggerPerson: "张三", + Department: "研发部", + Region: "华东", + QuestionImageURLs: []string{"https://weknora-api.example/i/001"}, + }, + Attempts: 0, + }, + }, nil + } + rw := answerRewriterStub{rewritten: "转换后答案[3]", err: nil} + d, err := NewDispatcher(repo, inserter, &DispatcherOptions{Rewriter: rw}) + if err != nil { + t.Fatalf("NewDispatcher: %v", err) + } + + stats, err := d.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Claimed != 1 || stats.Exported != 1 || stats.Failed != 0 { + t.Fatalf("stats=%+v", stats) + } + if len(inserter.rows) != 1 { + t.Fatalf("rows=%d, want 1", len(inserter.rows)) + } + row := inserter.rows[0] + if row.Answer != "转换后答案[3]" { + t.Fatalf("Answer=%q, want rewritten", row.Answer) + } +} + +func TestRunOnceRewriterErrorFallsBackToOriginalAnswer(t *testing.T) { + repo := newStubRepo() + inserter := newStubInserter() + inserter.recordID = "rec-fallback-1" + repo.claimFn = func(ctx context.Context, limit int, maxAttempts int, staleBefore time.Time) ([]PendingExport, error) { + return []PendingExport{ + { + ExportID: "exp-fallback", + Row: Row{ + Question: "问题", + Answer: "原始答案", + AskedTime: time.Now(), + }, + Attempts: 0, + }, + }, nil + } + rw := answerRewriterStub{rewritten: "", err: context.DeadlineExceeded} + d, err := NewDispatcher(repo, inserter, &DispatcherOptions{Rewriter: rw}) + if err != nil { + t.Fatalf("NewDispatcher: %v", err) + } + + stats, err := d.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 0 || stats.Exported != 1 { + t.Fatalf("stats=%+v", stats) + } + if len(inserter.rows) != 1 || inserter.rows[0].Answer != "原始答案" { + t.Fatalf("rows=%+v, want fallback original answer", inserter.rows) + } +} + +func TestRowToFieldsIncludesQuestionImageURLsAndImageMarker(t *testing.T) { + r := Row{ + Question: "请描述并分析这张图片。", + Answer: "回答了。", + AskedTime: time.Now(), + AskerName: "张三", + AskerUnionID: "union-z", + TriggerPerson: "张三", + Department: "研发部", + Region: "华东", + QuestionImageURLs: []string{ + "https://weknora-api.example/i/001", + "https://weknora-api.example/i/002", + }, + } + + fields := r.ToFields() + + original, ok := fields[FieldOriginalQuestion].(string) + if !ok { + t.Fatalf("OriginalQuestion type=%T", fields[FieldOriginalQuestion]) + } + if !strings.HasPrefix(original, "[图片]") { + t.Fatalf("OriginalQuestion=%q, want [图片] prefix", original) + } + // Image URLs must be rendered as markdown image syntax so the Notable cell + // previews them (same as the answer column), not as a plain-text link. + if !strings.Contains(original, "![图片1](https://weknora-api.example/i/001)") || + !strings.Contains(original, "![图片2](https://weknora-api.example/i/002)") { + t.Fatalf("OriginalQuestion=%q, missing markdown image syntax", original) + } + + // Without image URLs, no [图片] prefix. + plain := Row{Question: "普通问题", AskedTime: time.Now()} + if got := plain.ToFields()[FieldOriginalQuestion]; got != "普通问题" { + t.Fatalf("plain OriginalQuestion=%q, want unchanged", got) + } +} diff --git a/internal/questiontable/row.go b/internal/questiontable/row.go index c35ad58..87fdddf 100644 --- a/internal/questiontable/row.go +++ b/internal/questiontable/row.go @@ -1,6 +1,7 @@ package questiontable import ( + "strconv" "strings" "time" "unicode" @@ -23,14 +24,15 @@ const ( // Row holds the data needed to produce a Notable v1 typed record. type Row struct { - Question string - Answer string - AskedTime time.Time - AskerName string - AskerUnionID string - TriggerPerson string - Department string - Region string + Question string + Answer string + AskedTime time.Time + AskerName string + AskerUnionID string + TriggerPerson string + Department string + Region string + QuestionImageURLs []string } // ToFields returns a typed field-ID → value map suitable for the Notable v1 @@ -67,8 +69,8 @@ func (r Row) ToFields() map[string]interface{} { } } - // OriginalQuestion. - fields[FieldOriginalQuestion] = sanitise(r.Question) + // OriginalQuestion: include image marker and stable URLs for image turns. + fields[FieldOriginalQuestion] = sanitise(questionWithImageURLs(r.Question, r.QuestionImageURLs)) // Language. fields[FieldLanguage] = sanitise(lang) @@ -134,3 +136,28 @@ func sanitise(s string) string { } return s } + +// questionWithImageURLs prefixes an image marker and renders the stable image +// URLs as markdown image syntax when the turn carried image input. Markdown +// image syntax lets the Notable cell preview the image (same as the answer +// column); a plain-text URL would render as an unpreviewable link. +func questionWithImageURLs(question string, imageURLs []string) string { + if len(imageURLs) == 0 { + return question + } + var builder strings.Builder + builder.WriteString("[图片] ") + builder.WriteString(strings.TrimSpace(question)) + for index, imageURL := range imageURLs { + imageURL = strings.TrimSpace(imageURL) + if imageURL == "" { + continue + } + builder.WriteString("\n![图片") + builder.WriteString(strconv.Itoa(index + 1)) + builder.WriteString("](") + builder.WriteString(imageURL) + builder.WriteString(")") + } + return builder.String() +} diff --git a/internal/settings/controller.go b/internal/settings/controller.go index 34b5dce..1a591d3 100644 --- a/internal/settings/controller.go +++ b/internal/settings/controller.go @@ -85,7 +85,7 @@ func (c *Controller) Handle(ctx context.Context, in SettingInput) (string, error if err != nil { return "", err } - return "身份已验证,请回到后台确认首位管理员。", nil + return "身份已验证,请回到服务器命令行等待确认。", nil } if explicit && len(args) == 0 { admin, err := c.repo.IsAdministrator(ctx, in.TenantKey, in.StaffID) @@ -158,7 +158,7 @@ func (c *Controller) Handle(ctx context.Context, in SettingInput) (string, error return "", err } -func settingMenu(admin, direct bool) string { +func settingMenu(admin, _ bool) string { menu := "⚙️ 个人设置\n入口:/setting 或 /设置\n" + "回答语言:/setting language \n" + "- auto:跟随当前提问语言(推荐)\n" + @@ -172,14 +172,7 @@ func settingMenu(admin, direct bool) string { "- support:在目标技术支持群@机器人发送 /setting support 保存;再发送 /setting support test,测试成功后激活。\n" + "- question:私聊发送 /setting question,再在10分钟内发送问题AI表链接;Schema验证通过后激活并短暂重启。\n" + "- faq:私聊发送 /setting faq,再在10分钟内发送FAQ表链接;Schema验证通过后激活并短暂重启。" - if !direct { - return menu + "\n\n🌐 管理后台与管理员更换:请私聊机器人发送 /setting 查看。" - } - return menu + "\n\n🌐 管理后台(仅PC)\n" + - "1. 通过SSH隧道将本地8090转发到服务器127.0.0.1:8090。\n" + - "2. 浏览器打开:http://127.0.0.1:8090/admin/settings\n" + - "3. Basic Auth用户名任意;密码为服务器DOBOTAI_ADMIN_TOKEN(仅从服务器安全获取,禁止发送到钉钉)。\n" + - "4. 更换管理员:后台生成6位码 → 新管理员私聊 /setting <6位码> → 回后台确认 → 新管理员生效后再停用旧管理员。" + return menu } func tableBindingFailureReply(action string, err error) string { diff --git a/internal/settings/controller_test.go b/internal/settings/controller_test.go index 0750766..5e2eaec 100644 --- a/internal/settings/controller_test.go +++ b/internal/settings/controller_test.go @@ -88,7 +88,7 @@ func TestControllerSettingMenuExplainsLanguageWithoutAdminActionsToOrdinaryUser( } } -func TestControllerSettingMenuExplainsAdminActionsAndBackendAccess(t *testing.T) { +func TestControllerSettingMenuExplainsAdminActionsWithoutBackendAccess(t *testing.T) { r := newMemoryRepo() r.admins["t/a"] = true c := NewController(r) @@ -103,15 +103,21 @@ func TestControllerSettingMenuExplainsAdminActionsAndBackendAccess(t *testing.T) "/setting question", "/setting faq", "Schema验证通过后激活并短暂重启", + } { + if !strings.Contains(out, want) { + t.Fatalf("reply=%q must contain %q", out, want) + } + } + for _, forbidden := range []string{ + "管理后台", "SSH隧道", "127.0.0.1:8090/admin/settings", "Basic Auth", "DOBOTAI_ADMIN_TOKEN", - "新管理员私聊 /setting <6位码>", - "新管理员生效后再停用旧管理员", + "回后台确认", } { - if !strings.Contains(out, want) { - t.Fatalf("reply=%q must contain %q", out, want) + if strings.Contains(out, forbidden) { + t.Fatalf("reply=%q must not contain %q", out, forbidden) } } } @@ -312,11 +318,14 @@ func TestControllerDetectsBootstrapCandidate(t *testing.T) { r := newMemoryRepo() c := NewController(r) c.now = func() time.Time { return time.Unix(100, 0) } - _, err := c.Handle(context.Background(), SettingInput{TenantKey: "t", StaffID: "a", UnionID: "union", DisplayName: "Alice", Text: "/setting 123456", ConversationType: "direct"}) + out, err := c.Handle(context.Background(), SettingInput{TenantKey: "t", StaffID: "a", UnionID: "union", DisplayName: "Alice", Text: "/setting 123456", ConversationType: "direct"}) if err != nil { t.Fatal(err) } if r.candidate.Code != "123456" || r.candidate.StaffID != "a" { t.Fatalf("candidate=%+v", r.candidate) } + if !strings.Contains(out, "服务器命令行") || strings.Contains(out, "后台") { + t.Fatalf("reply=%q must direct confirmation back to the server CLI only", out) + } } diff --git a/internal/store/postgres/question_table_exports.go b/internal/store/postgres/question_table_exports.go index 21f987b..ec40ce7 100644 --- a/internal/store/postgres/question_table_exports.go +++ b/internal/store/postgres/question_table_exports.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -82,6 +83,7 @@ func (r *QuestionTableExportRepository) Claim(ctx context.Context, limit int, ma cl.attempts, t.question, t.answer, + t.question_image_urls_json, m.created_at, COALESCE(m.actor_name_snapshot, '') AS actor_name_snapshot, COALESCE(m.actor_union_id_snapshot, '') AS actor_union_id_snapshot, @@ -109,6 +111,7 @@ func (r *QuestionTableExportRepository) Claim(ctx context.Context, limit int, ma attempts int question string answer string + imageURLsJSON []byte askedTime time.Time actorNameSnapshot string actorUnionIDSnapshot string @@ -117,25 +120,32 @@ func (r *QuestionTableExportRepository) Claim(ctx context.Context, limit int, ma regionSnap string ) if err := rows.Scan(&exportID, &attempts, - &question, &answer, + &question, &answer, &imageURLsJSON, &askedTime, &actorNameSnapshot, &actorUnionIDSnapshot, &triggerPersonSnap, &departmentSnap, ®ionSnap); err != nil { rows.Close() return nil, fmt.Errorf("questiontable export claim scan: %w", err) } + var imageURLs []string + if len(imageURLsJSON) > 0 { + if err := json.Unmarshal(imageURLsJSON, &imageURLs); err != nil { + imageURLs = nil + } + } exports = append(exports, questiontable.PendingExport{ ExportID: exportID, Attempts: attempts, Row: questiontable.Row{ - Question: question, - Answer: answer, - AskedTime: askedTime, - AskerName: actorNameSnapshot, - AskerUnionID: actorUnionIDSnapshot, - TriggerPerson: triggerPersonSnap, - Department: departmentSnap, - Region: regionSnap, + Question: question, + Answer: answer, + AskedTime: askedTime, + AskerName: actorNameSnapshot, + AskerUnionID: actorUnionIDSnapshot, + TriggerPerson: triggerPersonSnap, + Department: departmentSnap, + Region: regionSnap, + QuestionImageURLs: imageURLs, }, }) } diff --git a/internal/store/postgres/store.go b/internal/store/postgres/store.go index 06b3e32..f9750e5 100644 --- a/internal/store/postgres/store.go +++ b/internal/store/postgres/store.go @@ -56,6 +56,14 @@ func (s *Store) BeginTurn(ctx context.Context, turn core.Turn) (core.Turn, bool, if err != nil { return core.Turn{}, false, err } + imageURLs := turn.QuestionImageURLs + if imageURLs == nil { + imageURLs = []string{} + } + imageURLsJSON, err := json.Marshal(imageURLs) + if err != nil { + return core.Turn{}, false, err + } tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return core.Turn{}, false, err @@ -106,10 +114,10 @@ func (s *Store) BeginTurn(ctx context.Context, turn core.Turn) (core.Turn, bool, } command, err := tx.Exec(ctx, `INSERT INTO turns - (id,tenant_key,channel,external_message_id,session_key,question,status,media_json,request_message_id,agent_id) - VALUES($1,$2,$3,$4,$5,$6,'processing',$7,$8,$9) ON CONFLICT DO NOTHING`, + (id,tenant_key,channel,external_message_id,session_key,question,status,media_json,request_message_id,agent_id,question_image_urls_json) + VALUES($1,$2,$3,$4,$5,$6,'processing',$7,$8,$9,$10) ON CONFLICT DO NOTHING`, turn.ID, turn.TenantKey, turn.Channel, turn.ExternalMessageID, turn.SessionKey, turn.Question, - mediaJSON, turn.RequestMessageID, turn.AgentID) + mediaJSON, turn.RequestMessageID, turn.AgentID, imageURLsJSON) if err != nil { return core.Turn{}, false, err } @@ -240,9 +248,9 @@ func (s *Store) FinishTurn(ctx context.Context, turn core.Turn) error { } } } - _, err = tx.Exec(ctx, `UPDATE turns SET weknora_session_id=$2,weknora_request_id=$3,answer=$4,references_json=$5,media_json=$6,status=$7,error_message=$8,answer_message_id=COALESCE($9::TEXT,answer_message_id),updated_at=NOW() WHERE id=$1`, + _, err = tx.Exec(ctx, `UPDATE turns SET weknora_session_id=$2,weknora_request_id=$3,answer=$4,references_json=$5,media_json=$6,status=$7,error_message=$8,answer_message_id=COALESCE($9::TEXT,answer_message_id),knowledge_searched=$10,updated_at=NOW() WHERE id=$1`, turn.ID, turn.WeKnoraSessionID, turn.WeKnoraRequestID, turn.Answer, references, mediaJSON, - turn.Status, turn.ErrorMessage, answerMessageID) + turn.Status, turn.ErrorMessage, answerMessageID, turn.KnowledgeSearched) if err != nil { return err }