From d857853ea34e44edef8b04e5dd364778b486689f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 21 Jul 2026 10:42:28 +0800 Subject: [PATCH 1/2] feat: add message audit foundation --- db/migrations/000003_message_audit.down.sql | 15 + db/migrations/000003_message_audit.up.sql | 90 +++++ .../000003_message_audit_contract_test.go | 366 ++++++++++++++++++ internal/channels/dingtalk/handler.go | 1 + internal/channels/dingtalk/handler_test.go | 6 + internal/core/types.go | 11 + internal/core/types_test.go | 101 +++++ internal/httpapi/server.go | 3 +- internal/httpapi/server_test.go | 28 ++ internal/qa/service.go | 47 ++- internal/qa/service_test.go | 133 ++++++- .../postgres/message_audit_contract_test.go | 107 +++++ internal/store/postgres/store.go | 169 +++++++- .../store/postgres/store_contract_test.go | 3 +- 14 files changed, 1063 insertions(+), 17 deletions(-) create mode 100644 db/migrations/000003_message_audit.down.sql create mode 100644 db/migrations/000003_message_audit.up.sql create mode 100644 db/migrations/000003_message_audit_contract_test.go create mode 100644 internal/core/types_test.go create mode 100644 internal/store/postgres/message_audit_contract_test.go diff --git a/db/migrations/000003_message_audit.down.sql b/db/migrations/000003_message_audit.down.sql new file mode 100644 index 0000000..58bb5ab --- /dev/null +++ b/db/migrations/000003_message_audit.down.sql @@ -0,0 +1,15 @@ +-- 000003_message_audit down migration +-- Drop in reverse dependency order: first remove turn columns, then drop tables + +ALTER TABLE turns + DROP COLUMN IF EXISTS request_message_id; + +ALTER TABLE turns + DROP COLUMN IF EXISTS answer_message_id; + +ALTER TABLE turns + DROP COLUMN IF EXISTS agent_id; + +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS channel_conversations; +DROP TABLE IF EXISTS channel_actors; \ No newline at end of file diff --git a/db/migrations/000003_message_audit.up.sql b/db/migrations/000003_message_audit.up.sql new file mode 100644 index 0000000..a0d9d13 --- /dev/null +++ b/db/migrations/000003_message_audit.up.sql @@ -0,0 +1,90 @@ +-- 000003_message_audit: unified identity and message audit log +-- Adds channel_actors, channel_conversations, messages tables +-- and extends turns with agent_id, request_message_id, answer_message_id + +-- ============================================================ +-- 1. Channel Actors +-- ============================================================ +CREATE TABLE IF NOT EXISTS channel_actors ( + id TEXT PRIMARY KEY, + tenant_key TEXT NOT NULL, + channel TEXT NOT NULL, + external_actor_id TEXT NOT NULL, + actor_type TEXT NOT NULL CHECK (actor_type IN ('user','bot','support')), + display_name TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (tenant_key, channel, external_actor_id) +); + +CREATE INDEX IF NOT EXISTS idx_channel_actors_lookup + ON channel_actors (tenant_key, channel, external_actor_id); + +-- ============================================================ +-- 2. Channel Conversations +-- ============================================================ +CREATE TABLE IF NOT EXISTS channel_conversations ( + id TEXT PRIMARY KEY, + tenant_key TEXT NOT NULL, + channel TEXT NOT NULL, + external_conversation_id TEXT NOT NULL, + conversation_type TEXT NOT NULL CHECK (conversation_type IN ('direct','group','support','web')), + title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','archived')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (tenant_key, channel, external_conversation_id) +); + +CREATE INDEX IF NOT EXISTS idx_channel_conversations_lookup + ON channel_conversations (tenant_key, channel, external_conversation_id); + +-- ============================================================ +-- 3. Messages +-- ============================================================ +CREATE TABLE IF NOT EXISTS messages ( + id TEXT PRIMARY KEY, + tenant_key TEXT NOT NULL, + channel TEXT NOT NULL, + conversation_id TEXT NOT NULL REFERENCES channel_conversations(id), + actor_id TEXT REFERENCES channel_actors(id), + external_message_id TEXT NOT NULL DEFAULT '', + correlation_id TEXT NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ('inbound','outbound','internal')), + role TEXT NOT NULL CHECK (role IN ('user','assistant','support','system')), + message_type TEXT NOT NULL CHECK (message_type IN ('text','image','rich_text','card','feedback')), + text_content TEXT NOT NULL DEFAULT '', + media_json JSONB NOT NULL DEFAULT '[]'::JSONB, + reply_to_message_id TEXT REFERENCES messages(id), + status TEXT NOT NULL CHECK (status IN ('received','processing','generated','sent','failed')), + actor_name_snapshot TEXT NOT NULL DEFAULT '', + provider_created_at TIMESTAMPTZ, + safe_metadata_json JSONB NOT NULL DEFAULT '{}'::JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Partial unique index: only enforce uniqueness when external_message_id is non-empty +CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_external_unique + ON messages (tenant_key, channel, external_message_id) + WHERE external_message_id != ''; + +-- Index for tenant + conversation time-series +CREATE INDEX IF NOT EXISTS idx_messages_tenant_conv_created + ON messages (tenant_key, conversation_id, created_at DESC); + +-- Index for correlation lookups +CREATE INDEX IF NOT EXISTS idx_messages_correlation + ON messages (correlation_id); + +-- ============================================================ +-- 4. Extend turns table +-- ============================================================ +ALTER TABLE turns + ADD COLUMN IF NOT EXISTS request_message_id TEXT REFERENCES messages(id); + +ALTER TABLE turns + ADD COLUMN IF NOT EXISTS answer_message_id TEXT REFERENCES messages(id); + +ALTER TABLE turns + ADD COLUMN IF NOT EXISTS agent_id TEXT NOT NULL DEFAULT ''; \ No newline at end of file diff --git a/db/migrations/000003_message_audit_contract_test.go b/db/migrations/000003_message_audit_contract_test.go new file mode 100644 index 0000000..020aba4 --- /dev/null +++ b/db/migrations/000003_message_audit_contract_test.go @@ -0,0 +1,366 @@ +// Package migrations_test contains source-level contract tests for the +// 000003_message_audit migration. No real database connection is used. +package migrations_test + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func read003UpMigration(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(".", "000003_message_audit.up.sql")) + if err != nil { + t.Fatalf("read up migration: %v", err) + } + return string(data) +} + +func read003DownMigration(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(".", "000003_message_audit.down.sql")) + if err != nil { + t.Fatalf("read down migration: %v", err) + } + return string(data) +} + +// TestUp003UsesIfNotExists ensures every CREATE uses IF NOT EXISTS. +func TestUp003UsesIfNotExists(t *testing.T) { + up := read003UpMigration(t) + + for _, stmt := range splitStatements(up) { + upper := strings.ToUpper(stmt) + if strings.Contains(upper, "CREATE TABLE") && !strings.Contains(upper, "CREATE TABLE IF NOT EXISTS") { + t.Errorf("CREATE TABLE without IF NOT EXISTS:\n%s", strings.TrimSpace(stmt)) + } + if strings.Contains(upper, "CREATE INDEX") && !strings.Contains(upper, "CREATE INDEX IF NOT EXISTS") { + t.Errorf("CREATE INDEX without IF NOT EXISTS:\n%s", strings.TrimSpace(stmt)) + } + if strings.Contains(upper, "CREATE UNIQUE INDEX") && !strings.Contains(upper, "IF NOT EXISTS") { + t.Errorf("CREATE UNIQUE INDEX without IF NOT EXISTS:\n%s", strings.TrimSpace(stmt)) + } + } + + // We expect 3 CREATE TABLE IF NOT EXISTS (channel_actors, channel_conversations, messages) + if got := countMatches(up, `(?i)CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS`); got < 3 { + t.Errorf("expected at least 3 CREATE TABLE IF NOT EXISTS statements, got %d", got) + } + // We expect at least 3 CREATE INDEX IF NOT EXISTS + if got := countMatches(up, `(?i)CREATE\s+(UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS`); got < 3 { + t.Errorf("expected at least 3 CREATE INDEX IF NOT EXISTS statements, got %d", got) + } +} + +// TestUp003CreatesChannelActorsTable checks the channel_actors table structure. +func TestUp003CreatesChannelActorsTable(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "CREATE TABLE IF NOT EXISTS CHANNEL_ACTORS") { + t.Fatal("up migration must create channel_actors table") + } + if !strings.Contains(upper, "ID TEXT PRIMARY KEY") { + t.Error("channel_actors must have id TEXT PRIMARY KEY") + } + if !strings.Contains(upper, "TENANT_KEY") { + t.Error("channel_actors must have tenant_key column") + } + if !strings.Contains(upper, "CHANNEL") { + t.Error("channel_actors must have channel column") + } + if !strings.Contains(upper, "EXTERNAL_ACTOR_ID") { + t.Error("channel_actors must have external_actor_id column") + } + if !strings.Contains(upper, "ACTOR_TYPE") { + t.Error("channel_actors must have actor_type column") + } + if !strings.Contains(upper, "DISPLAY_NAME") { + t.Error("channel_actors must have display_name column") + } + if !strings.Contains(upper, "ACTOR_TYPE IN ('USER','BOT','SUPPORT')") { + t.Error("channel_actors actor_type must have CHECK (user, bot, support)") + } + if !strings.Contains(upper, "UNIQUE") && strings.Contains(upper, "TENANT_KEY") && strings.Contains(upper, "CHANNEL") && strings.Contains(upper, "EXTERNAL_ACTOR_ID") { + t.Error("channel_actors must have UNIQUE(tenant_key, channel, external_actor_id)") + } +} + +// TestUp003CreatesChannelConversationsTable checks the channel_conversations table structure. +func TestUp003CreatesChannelConversationsTable(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "CREATE TABLE IF NOT EXISTS CHANNEL_CONVERSATIONS") { + t.Fatal("up migration must create channel_conversations table") + } + if !strings.Contains(upper, "ID TEXT PRIMARY KEY") { + t.Error("channel_conversations must have id TEXT PRIMARY KEY") + } + if !strings.Contains(upper, "EXTERNAL_CONVERSATION_ID") { + t.Error("channel_conversations must have external_conversation_id column") + } + if !strings.Contains(upper, "CONVERSATION_TYPE") { + t.Error("channel_conversations must have conversation_type column") + } + if !strings.Contains(upper, "TITLE") { + t.Error("channel_conversations must have title column") + } + if !strings.Contains(upper, "STATUS") { + t.Error("channel_conversations must have status column") + } + if !strings.Contains(upper, "CONVERSATION_TYPE IN ('DIRECT','GROUP','SUPPORT','WEB')") { + t.Error("channel_conversations conversation_type must have CHECK (direct, group, support, web)") + } + if !strings.Contains(upper, "STATUS IN ('ACTIVE','ARCHIVED')") { + t.Error("channel_conversations status must have CHECK (active, archived)") + } +} + +// TestUp003CreatesMessagesTable checks the messages table structure. +func TestUp003CreatesMessagesTable(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "CREATE TABLE IF NOT EXISTS MESSAGES") { + t.Fatal("up migration must create messages table") + } + if !strings.Contains(upper, "ID TEXT PRIMARY KEY") { + t.Error("messages must have id TEXT PRIMARY KEY") + } + + required := []string{ + "TENANT_KEY", "CHANNEL", "CONVERSATION_ID", "ACTOR_ID", + "EXTERNAL_MESSAGE_ID", "CORRELATION_ID", "DIRECTION", + "ROLE", "MESSAGE_TYPE", "TEXT_CONTENT", "MEDIA_JSON", + "REPLY_TO_MESSAGE_ID", "STATUS", "ACTOR_NAME_SNAPSHOT", + "PROVIDER_CREATED_AT", "SAFE_METADATA_JSON", + } + for _, col := range required { + if !strings.Contains(upper, col) { + t.Errorf("messages must have %s column", col) + } + } + + // Check constraints + if !strings.Contains(upper, "DIRECTION IN ('INBOUND','OUTBOUND','INTERNAL')") { + t.Error("messages direction must have CHECK (inbound, outbound, internal)") + } + if !strings.Contains(upper, "ROLE IN ('USER','ASSISTANT','SUPPORT','SYSTEM')") { + t.Error("messages role must have CHECK (user, assistant, support, system)") + } + if !strings.Contains(upper, "MESSAGE_TYPE IN ('TEXT','IMAGE','RICH_TEXT','CARD','FEEDBACK')") { + t.Error("messages message_type must have CHECK (text, image, rich_text, card, feedback)") + } + if !strings.Contains(upper, "STATUS IN ('RECEIVED','PROCESSING','GENERATED','SENT','FAILED')") { + t.Error("messages status must have CHECK (received, processing, generated, sent, failed)") + } +} + +// TestUp003MessagesHasNoForbiddenColumns ensures secret/protocol columns are absent. +func TestUp003MessagesHasNoForbiddenColumns(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + forbidden := []string{ + "RAW_PAYLOAD", "SESSIONWEBHOOK", "DOWNLOAD_CODE", + "DATAURI", "DATA_URI", "TOKEN", "REASONING", + } + for _, col := range forbidden { + if strings.Contains(upper, col) { + t.Errorf("messages must not contain forbidden column %q", col) + } + } +} + +// TestUp003MessagesPartialUniqueIndex ensures external_message_id has a partial unique index. +func TestUp003MessagesPartialUniqueIndex(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "EXTERNAL_MESSAGE_ID") { + t.Fatal("messages must have external_message_id") + } + // Partial unique index: UNIQUE (tenant_key, channel, external_message_id) WHERE external_message_id != '' + // The pattern should appear somewhere in the migration + hasPartial := strings.Contains(upper, "WHERE") && + strings.Contains(upper, "EXTERNAL_MESSAGE_ID") && + (strings.Contains(upper, "!=''") || strings.Contains(upper, "!= ''") || strings.Contains(upper, "IS NOT NULL")) + if !hasPartial { + t.Error("messages must have partial unique index on external_message_id (non-empty only)") + } +} + +// TestUp003MessagesIndexes checks required indexes on messages. +func TestUp003MessagesIndexes(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + // Tenant + conversation + created_at + if !strings.Contains(upper, "TENANT_KEY") || !strings.Contains(upper, "CONVERSATION_ID") || !strings.Contains(upper, "CREATED_AT") { + t.Error("messages must have index on tenant_key, conversation_id, created_at") + } + // correlation_id index + if !strings.Contains(upper, "CORRELATION_ID") { + t.Error("messages must have index on correlation_id") + } +} + +// TestUp003AltersTurnsColumns ensures ALTER TABLE turns uses ADD COLUMN IF NOT EXISTS. +func TestUp003AltersTurnsColumns(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + // Must add request_message_id, answer_message_id, agent_id to turns + if !strings.Contains(upper, "ALTER TABLE TURNS") { + t.Fatal("up migration must ALTER TABLE turns") + } + if !strings.Contains(upper, "ADD COLUMN IF NOT EXISTS REQUEST_MESSAGE_ID") { + t.Error("turns must add request_message_id column with IF NOT EXISTS") + } + if !strings.Contains(upper, "ADD COLUMN IF NOT EXISTS ANSWER_MESSAGE_ID") { + t.Error("turns must add answer_message_id column with IF NOT EXISTS") + } + if !strings.Contains(upper, "ADD COLUMN IF NOT EXISTS AGENT_ID") { + t.Error("turns must add agent_id column with IF NOT EXISTS") + } + // agent_id must have NOT NULL DEFAULT '' + if !strings.Contains(upper, "AGENT_ID TEXT NOT NULL DEFAULT ''") { + t.Error("turns agent_id must be TEXT NOT NULL DEFAULT ''") + } +} + +// TestDown003DropsInReverseOrder ensures tables are dropped in reverse dependency order. +func TestDown003DropsInReverseOrder(t *testing.T) { + down := read003DownMigration(t) + upper := strings.ToUpper(down) + + if !strings.Contains(upper, "DROP TABLE IF EXISTS MESSAGES") { + t.Error("down migration must drop messages") + } + if !strings.Contains(upper, "DROP TABLE IF EXISTS CHANNEL_CONVERSATIONS") { + t.Error("down migration must drop channel_conversations") + } + if !strings.Contains(upper, "DROP TABLE IF EXISTS CHANNEL_ACTORS") { + t.Error("down migration must drop channel_actors") + } + + // Verify reverse order: messages before conversations before actors + msgPos := strings.Index(upper, "DROP TABLE IF EXISTS MESSAGES") + convPos := strings.Index(upper, "DROP TABLE IF EXISTS CHANNEL_CONVERSATIONS") + actorPos := strings.Index(upper, "DROP TABLE IF EXISTS CHANNEL_ACTORS") + + if msgPos < 0 || convPos < 0 || actorPos < 0 { + t.Fatal("down migration missing DROP statements") + } + if msgPos > convPos || convPos > actorPos { + t.Error("down migration must drop messages first, then channel_conversations, then channel_actors (reverse dependency order)") + } +} + +// TestDown003DropsTurnColumns ensures down removes the added turn columns. +func TestDown003DropsTurnColumns(t *testing.T) { + down := read003DownMigration(t) + upper := strings.ToUpper(down) + + if !strings.Contains(upper, "DROP COLUMN IF EXISTS REQUEST_MESSAGE_ID") { + t.Error("down migration must drop request_message_id from turns") + } + if !strings.Contains(upper, "DROP COLUMN IF EXISTS ANSWER_MESSAGE_ID") { + t.Error("down migration must drop answer_message_id from turns") + } + if !strings.Contains(upper, "DROP COLUMN IF EXISTS AGENT_ID") { + t.Error("down migration must drop agent_id from turns") + } +} + +// TestUp003DoesNotModify000001Or000002 ensures migration 003 doesn't recreate existing tables. +func TestUp003DoesNotModifySessionMappingsOrTurnsTables(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + // Must not re-create session_mappings or turns tables + if strings.Contains(upper, "CREATE TABLE IF NOT EXISTS SESSION_MAPPINGS") { + t.Error("migration 003 must not recreate session_mappings table") + } + if strings.Contains(upper, "CREATE TABLE IF NOT EXISTS TURNS") { + t.Error("migration 003 must not recreate turns table") + } + + // Must not create or modify 000002's media_json (already exists) + // It's fine if it doesn't mention media_json at all +} + +// TestUp003JSONBDefaults ensures JSONB columns use NOT NULL DEFAULT '[]' or '{}'. +func TestUp003JSONBDefaults(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "MEDIA_JSON JSONB NOT NULL DEFAULT '[]'::JSONB") { + t.Error("messages media_json must be JSONB NOT NULL DEFAULT '[]'::JSONB") + } + if !strings.Contains(upper, "SAFE_METADATA_JSON JSONB NOT NULL DEFAULT '{}'::JSONB") { + t.Error("messages safe_metadata_json must be JSONB NOT NULL DEFAULT '{}'::JSONB") + } +} + +// TestUp003MessagesSelfReference ensures reply_to_message_id is nullable and self-referencing. +func TestUp003MessagesSelfReference(t *testing.T) { + up := read003UpMigration(t) + upper := strings.ToUpper(up) + + if !strings.Contains(upper, "REPLY_TO_MESSAGE_ID") { + t.Error("messages must have reply_to_message_id column") + } +} + +func TestUp003TurnMessageForeignKeys(t *testing.T) { + up := read003UpMigration(t) + for _, field := range []string{"request_message_id", "answer_message_id"} { + pattern := `(?i)` + field + `\s+text\s+references\s+messages\s*\(\s*id\s*\)` + if !regexp.MustCompile(pattern).MatchString(up) { + t.Errorf("turns.%s must reference messages(id)", field) + } + } +} + +// TestUp003MessagesForeignKeys ensures core relations are enforced by PostgreSQL. +func TestUp003MessagesForeignKeys(t *testing.T) { + up := read003UpMigration(t) + checks := []struct { + name string + pattern string + }{ + {"conversation", `(?i)conversation_id\s+text\s+not\s+null\s+references\s+channel_conversations\s*\(\s*id\s*\)`}, + {"actor", `(?i)actor_id\s+text\s+references\s+channel_actors\s*\(\s*id\s*\)`}, + {"reply", `(?i)reply_to_message_id\s+text\s+references\s+messages\s*\(\s*id\s*\)`}, + } + for _, check := range checks { + if !regexp.MustCompile(check.pattern).MatchString(up) { + t.Errorf("messages missing %s foreign key", check.name) + } + } +} + +// TestUp003NoReservedWords ensures no bare reserved words are used as column names. +func TestUp003NoReservedWords(t *testing.T) { + up := read003UpMigration(t) + + // Check for common reserved words used as column identifiers + // "references" is the most dangerous in PostgreSQL + bareRefs := regexp.MustCompile(`(?im)^\s*REFERENCES\s+`) + if bareRefs.MatchString(up) { + // Exclude FOREIGN KEY ... REFERENCES pattern + lines := strings.Split(up, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + upperLine := strings.ToUpper(trimmed) + if strings.HasPrefix(upperLine, "REFERENCES ") && !strings.Contains(upperLine, "FOREIGN KEY") { + t.Errorf("reserved word 'references' used as column name: %s", trimmed) + } + } + } +} diff --git a/internal/channels/dingtalk/handler.go b/internal/channels/dingtalk/handler.go index 0e3cc1a..c0096d3 100644 --- a/internal/channels/dingtalk/handler.go +++ b/internal/channels/dingtalk/handler.go @@ -172,6 +172,7 @@ func (h *Handler) Handle(ctx context.Context, cb BotCallback) error { ExternalUserID: externalUserID, ExternalMessageID: strings.TrimSpace(cb.MsgID), SenderName: cb.SenderNick, + MessageType: strings.ToLower(strings.TrimSpace(cb.Msgtype)), } switch cb.ConversationType { case conversationTypeGroup: diff --git a/internal/channels/dingtalk/handler_test.go b/internal/channels/dingtalk/handler_test.go index f14f915..11f118f 100644 --- a/internal/channels/dingtalk/handler_test.go +++ b/internal/channels/dingtalk/handler_test.go @@ -167,6 +167,9 @@ func TestHandleDirectTextCallsQAAndRepliesAnswer(t *testing.T) { if req.Query != "hello world" { t.Fatalf("Query = %q, want trimmed 'hello world'", req.Query) } + if req.MessageType != "text" { + t.Fatalf("MessageType = %q, want text", req.MessageType) + } if req.ExternalChatID != "" { t.Fatalf("ExternalChatID should be empty for direct, got %q", req.ExternalChatID) } @@ -531,6 +534,9 @@ func TestHandleRichTextMixedDownloadsAndCallsQAWithRealQuery(t *testing.T) { if !qa.called || len(qa.lastReq.Images) != 1 || qa.lastReq.Images[0] != image { t.Fatalf("QA request did not receive image: %+v", qa.lastReq) } + if qa.lastReq.MessageType != "picture" { + t.Fatalf("MessageType=%q, want raw DingTalk type picture", qa.lastReq.MessageType) + } // Key assert: mixed uses real query, not default imageQuery if qa.lastReq.Query != "请分析这张截图中的错误" { t.Fatalf("richText mixed query=%q, want real query", qa.lastReq.Query) diff --git a/internal/core/types.go b/internal/core/types.go index 105f1a3..736f634 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -46,6 +46,7 @@ type QARequest struct { SenderName string Query string Images []ImageInput + MessageType string } type Reference struct { @@ -84,6 +85,16 @@ type Turn struct { SessionKey string WeKnoraSessionID string WeKnoraRequestID string + AgentID string + ConversationType ConversationType + ExternalChatID string + ExternalUserID string + SenderName string + MessageType string + ActorID string + ConversationID string + RequestMessageID string + AnswerMessageID string Question string Answer string References []Reference diff --git a/internal/core/types_test.go b/internal/core/types_test.go new file mode 100644 index 0000000..e20130c --- /dev/null +++ b/internal/core/types_test.go @@ -0,0 +1,101 @@ +package core + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestTurnHasAuditFields(t *testing.T) { + // Verify Turn struct has all the new audit fields from B1.1 + turn := Turn{ + AgentID: "agent-1", + ConversationType: ConversationDirect, + ExternalChatID: "chat-1", + ExternalUserID: "user-1", + SenderName: "Alice", + MessageType: "text", + ActorID: "actor-1", + ConversationID: "conv-1", + RequestMessageID: "req-msg-1", + AnswerMessageID: "ans-msg-1", + } + + if turn.AgentID != "agent-1" { + t.Error("Turn must have AgentID field") + } + if turn.ConversationType != ConversationDirect { + t.Error("Turn must have ConversationType field") + } + if turn.ExternalChatID != "chat-1" { + t.Error("Turn must have ExternalChatID field") + } + if turn.ExternalUserID != "user-1" { + t.Error("Turn must have ExternalUserID field") + } + if turn.SenderName != "Alice" { + t.Error("Turn must have SenderName field") + } + if turn.MessageType != "text" { + t.Error("Turn must have MessageType field") + } + if turn.ActorID != "actor-1" { + t.Error("Turn must have ActorID field") + } + if turn.ConversationID != "conv-1" { + t.Error("Turn must have ConversationID field") + } + if turn.RequestMessageID != "req-msg-1" { + t.Error("Turn must have RequestMessageID field") + } + if turn.AnswerMessageID != "ans-msg-1" { + t.Error("Turn must have AnswerMessageID field") + } +} + +func TestQARequestHasMessageType(t *testing.T) { + req := QARequest{ + TenantKey: "tenant", + Channel: "dingtalk", + ConversationType: ConversationDirect, + ExternalUserID: "user-1", + ExternalMessageID: "msg-1", + Query: "hello", + MessageType: "text", + } + + if req.MessageType != "text" { + t.Error("QARequest must have MessageType field") + } +} + +func TestMessageTypeDefaults(t *testing.T) { + // MessageType is optional; empty string should be valid + req := QARequest{ + TenantKey: "tenant", + Channel: "dingtalk", + ConversationType: ConversationDirect, + ExternalUserID: "user-1", + ExternalMessageID: "msg-1", + Query: "hello", + } + + if req.MessageType != "" { + t.Error("QARequest.MessageType should default to empty string") + } +} + +func TestTurnNoDataURITags(t *testing.T) { + // Verify Turn struct has no DataURI field or tag + turn := Turn{} + raw, err := json.Marshal(turn) + if err != nil { + t.Fatalf("marshal Turn: %v", err) + } + s := string(raw) + for _, forbidden := range []string{"data_uri", "DataURI", "dataURI"} { + if strings.Contains(s, forbidden) { + t.Errorf("Turn JSON must not contain %q: %s", forbidden, s) + } + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 69d03e5..b0f033d 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -60,6 +60,7 @@ func (s *Server) ask(w http.ResponseWriter, r *http.Request) { ExternalUserID string `json:"external_user_id"` ExternalMessageID string `json:"external_message_id"` SenderName string `json:"sender_name"` + MessageType string `json:"message_type"` Query string `json:"query"` } if err := json.NewDecoder(r.Body).Decode(&request); err != nil { @@ -70,7 +71,7 @@ func (s *Server) ask(w http.ResponseWriter, r *http.Request) { } return } - result, err := s.service.Ask(r.Context(), core.QARequest{TenantKey: request.TenantKey, Channel: request.Channel, ConversationType: request.ConversationType, ExternalChatID: request.ExternalChatID, ExternalUserID: request.ExternalUserID, ExternalMessageID: request.ExternalMessageID, SenderName: request.SenderName, Query: request.Query}) + result, err := s.service.Ask(r.Context(), core.QARequest{TenantKey: request.TenantKey, Channel: request.Channel, ConversationType: request.ConversationType, ExternalChatID: request.ExternalChatID, ExternalUserID: request.ExternalUserID, ExternalMessageID: request.ExternalMessageID, SenderName: request.SenderName, MessageType: request.MessageType, Query: request.Query}) if err != nil { if errors.Is(err, core.ErrTurnInProgress) { writeError(w, http.StatusConflict, "message is already processing") diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 7aeffcb..65b5a53 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -16,6 +16,15 @@ func (fakeQA) Ask(context.Context, core.QARequest) (core.QAResult, error) { return core.QAResult{TurnID: "turn-1", Answer: "answer"}, nil } +type captureQA struct { + request core.QARequest +} + +func (f *captureQA) Ask(_ context.Context, request core.QARequest) (core.QAResult, error) { + f.request = request + return core.QAResult{TurnID: "turn-capture", Answer: "answer"}, nil +} + func TestQAEndpointRequiresTokenAndReturnsResult(t *testing.T) { server, err := New(fakeQA{}, "ingress-secret") if err != nil { @@ -36,3 +45,22 @@ func TestQAEndpointRequiresTokenAndReturnsResult(t *testing.T) { t.Fatalf("status=%d body=%s", result.Code, result.Body.String()) } } + +func TestQAEndpointPassesOptionalMessageType(t *testing.T) { + qa := &captureQA{} + server, err := New(qa, "ingress-secret") + if err != nil { + t.Fatal(err) + } + body := `{"tenant_key":"tenant","channel":"web","conversation_type":"direct","external_user_id":"user","external_message_id":"message-rich","message_type":"rich_text","query":"hello"}` + request := httptest.NewRequest(http.MethodPost, "/v1/qa", strings.NewReader(body)) + request.Header.Set("Authorization", "Bearer ingress-secret") + result := httptest.NewRecorder() + server.Handler().ServeHTTP(result, request) + if result.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", result.Code, result.Body.String()) + } + if qa.request.MessageType != "rich_text" { + t.Fatalf("MessageType=%q, want rich_text", qa.request.MessageType) + } +} diff --git a/internal/qa/service.go b/internal/qa/service.go index 706b18f..e81e889 100644 --- a/internal/qa/service.go +++ b/internal/qa/service.go @@ -83,9 +83,29 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve Width: image.Width, Height: image.Height, }) } - turn := core.Turn{ID: newID(), TenantKey: request.TenantKey, Channel: request.Channel, - ExternalMessageID: request.ExternalMessageID, SessionKey: sessionKey, - Question: request.Query, Media: media, Status: "processing"} + messageType := normalizeMessageType(request) + if messageType == "" { + return core.QAResult{}, errors.New("unsupported message_type") + } + turn := core.Turn{ + ID: newID(), + TenantKey: request.TenantKey, + Channel: request.Channel, + ExternalMessageID: request.ExternalMessageID, + SessionKey: sessionKey, + AgentID: agentID, + ConversationType: request.ConversationType, + ExternalChatID: request.ExternalChatID, + ExternalUserID: request.ExternalUserID, + SenderName: request.SenderName, + MessageType: messageType, + ActorID: newID(), + ConversationID: newID(), + RequestMessageID: newID(), + Question: request.Query, + Media: media, + Status: "processing", + } stored, created, err := s.repository.BeginTurn(ctx, turn) if err != nil { return core.QAResult{}, fmt.Errorf("begin turn: %w", err) @@ -177,6 +197,7 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve } turn.Status, turn.Answer, turn.References = "succeeded", chat.Answer, chat.References turn.WeKnoraRequestID = chat.RequestID + turn.AnswerMessageID = newID() if err := s.repository.FinishTurn(ctx, turn); err != nil { return core.QAResult{}, fmt.Errorf("finish turn: %w", err) } @@ -184,6 +205,26 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve WeKnoraRequestID: chat.RequestID, Answer: chat.Answer, References: chat.References}, nil } +func normalizeMessageType(request core.QARequest) string { + messageType := strings.ToLower(strings.TrimSpace(request.MessageType)) + if messageType == "" { + if len(request.Images) > 0 { + return "image" + } + return "text" + } + switch messageType { + case "picture", "image": + return "image" + case "richtext", "rich_text": + return "rich_text" + case "text", "card", "feedback": + return messageType + default: + return "" + } +} + func validateRequest(request core.QARequest) error { if strings.TrimSpace(request.TenantKey) == "" || strings.TrimSpace(request.Channel) == "" || strings.TrimSpace(request.ExternalMessageID) == "" || strings.TrimSpace(request.Query) == "" { diff --git a/internal/qa/service_test.go b/internal/qa/service_test.go index e53ff79..0f6d9c2 100644 --- a/internal/qa/service_test.go +++ b/internal/qa/service_test.go @@ -12,9 +12,11 @@ import ( ) type memoryRepository struct { - mu sync.Mutex - turns map[string]core.Turn - sessions map[string]string + mu sync.Mutex + turns map[string]core.Turn + sessions map[string]string + beginTurns []core.Turn + finishTurns []core.Turn } func newMemoryRepository() *memoryRepository { @@ -23,6 +25,7 @@ func newMemoryRepository() *memoryRepository { func (r *memoryRepository) BeginTurn(_ context.Context, turn core.Turn) (core.Turn, bool, error) { r.mu.Lock() defer r.mu.Unlock() + r.beginTurns = append(r.beginTurns, turn) key := turn.TenantKey + ":" + turn.Channel + ":" + turn.ExternalMessageID if old, ok := r.turns[key]; ok { return old, false, nil @@ -52,6 +55,7 @@ func (r *memoryRepository) ResolveSession(ctx context.Context, m core.SessionMap func (r *memoryRepository) FinishTurn(_ context.Context, turn core.Turn) error { r.mu.Lock() defer r.mu.Unlock() + r.finishTurns = append(r.finishTurns, turn) r.turns[turn.TenantKey+":"+turn.Channel+":"+turn.ExternalMessageID] = turn return nil } @@ -361,3 +365,126 @@ func TestAskStreamDuplicateReplayDoesNotNotifyAgain(t *testing.T) { t.Fatalf("result=%+v secondCalls=%d streamCalls=%d", result, secondCalls, client.streamCalls) } } + +func TestAskBuildsDirectAuditIdentityAndMessageIDs(t *testing.T) { + repo := newMemoryRepository() + service, err := New(repo, fixedResolver{}, &fakeWeKnora{}, session.GroupShared) + if err != nil { + t.Fatal(err) + } + req := core.QARequest{ + TenantKey: "tenant", Channel: "dingtalk", ConversationType: core.ConversationDirect, + ExternalUserID: "user-1", ExternalMessageID: "message-audit-direct", + SenderName: "Alice", Query: "original question", MessageType: "rich_text", + } + if _, err = service.Ask(context.Background(), req); err != nil { + t.Fatalf("Ask: %v", err) + } + repo.mu.Lock() + begin := repo.beginTurns[0] + finished := repo.finishTurns[len(repo.finishTurns)-1] + repo.mu.Unlock() + if begin.AgentID != "agent-1" || begin.ConversationType != core.ConversationDirect || + begin.ExternalUserID != "user-1" || begin.ExternalChatID != "" || begin.SenderName != "Alice" { + t.Fatalf("direct audit identity mismatch: %+v", begin) + } + if begin.MessageType != "rich_text" || begin.Question != "original question" { + t.Fatalf("request message audit mismatch: %+v", begin) + } + ids := []string{begin.ID, begin.ActorID, begin.ConversationID, begin.RequestMessageID} + seen := map[string]bool{} + for _, id := range ids { + if id == "" || seen[id] { + t.Fatalf("audit IDs must be non-empty and distinct: %v", ids) + } + seen[id] = true + } + if finished.AnswerMessageID == "" || seen[finished.AnswerMessageID] { + t.Fatalf("answer message ID must be new and non-empty: begin=%v answer=%q", ids, finished.AnswerMessageID) + } +} + +func TestAskBuildsGroupAuditWithoutPersistingSenderPrefix(t *testing.T) { + repo := newMemoryRepository() + client := &fakeWeKnora{} + service, err := New(repo, fixedResolver{}, client, session.GroupShared) + if err != nil { + t.Fatal(err) + } + req := core.QARequest{ + TenantKey: "tenant", Channel: "dingtalk", ConversationType: core.ConversationGroup, + ExternalChatID: "group-1", ExternalUserID: "user-9", ExternalMessageID: "message-audit-group", + SenderName: "Bob", Query: "raw question", MessageType: "text", + } + if _, err = service.Ask(context.Background(), req); err != nil { + t.Fatalf("Ask: %v", err) + } + repo.mu.Lock() + begin := repo.beginTurns[0] + repo.mu.Unlock() + if begin.ConversationType != core.ConversationGroup || begin.ExternalChatID != "group-1" || begin.ExternalUserID != "user-9" { + t.Fatalf("group audit identity mismatch: %+v", begin) + } + if begin.Question != "raw question" { + t.Fatalf("audit question must remain raw, got %q", begin.Question) + } + client.mu.Lock() + defer client.mu.Unlock() + if len(client.queries) != 1 || client.queries[0] != "Bob: raw question" { + t.Fatalf("WeKnora query must keep sender prefix: %v", client.queries) + } +} + +func TestAskDefaultsAndNormalizesAuditMessageType(t *testing.T) { + tests := []struct { + name string + messageType string + images []core.ImageInput + want string + }{ + {name: "text-default", want: "text"}, + {name: "image-default", images: []core.ImageInput{{DataURI: "data:image/png;base64,AAAA", MIMEType: "image/png"}}, want: "image"}, + {name: "dingtalk-picture", messageType: "picture", images: []core.ImageInput{{DataURI: "data:image/png;base64,AAAA", MIMEType: "image/png"}}, want: "image"}, + {name: "richtext", messageType: "richText", want: "rich_text"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + repo := newMemoryRepository() + service, err := New(repo, fixedResolver{}, &fakeWeKnora{}, session.GroupShared) + if err != nil { + t.Fatal(err) + } + req := core.QARequest{TenantKey: "tenant", Channel: "dingtalk", ConversationType: core.ConversationDirect, + ExternalUserID: "user-1", ExternalMessageID: "message-type-" + tc.name, Query: "question", + MessageType: tc.messageType, Images: tc.images} + if _, err = service.Ask(context.Background(), req); err != nil { + t.Fatalf("Ask: %v", err) + } + repo.mu.Lock() + got := repo.beginTurns[0].MessageType + repo.mu.Unlock() + if got != tc.want { + t.Fatalf("MessageType=%q want %q", got, tc.want) + } + }) + } +} + +func TestAskFailureDoesNotGenerateAnswerMessageID(t *testing.T) { + repo := newMemoryRepository() + client := &fakeStreamingWeKnora{fakeWeKnora: &fakeWeKnora{}, streamErr: errors.New("stream failed")} + service, err := New(repo, fixedResolver{}, client, session.GroupShared) + if err != nil { + t.Fatal(err) + } + _, err = service.AskStream(context.Background(), streamQARequest("message-no-answer-audit"), func(string) {}) + if err == nil { + t.Fatal("expected error") + } + repo.mu.Lock() + finished := repo.finishTurns[len(repo.finishTurns)-1] + repo.mu.Unlock() + if finished.Status != "failed" || finished.AnswerMessageID != "" { + t.Fatalf("failed turn must not generate answer message: %+v", finished) + } +} diff --git a/internal/store/postgres/message_audit_contract_test.go b/internal/store/postgres/message_audit_contract_test.go new file mode 100644 index 0000000..37b31d4 --- /dev/null +++ b/internal/store/postgres/message_audit_contract_test.go @@ -0,0 +1,107 @@ +package postgres + +import ( + "strings" + "testing" + + "gitea.zheromiku.top/Weknora/dobotai-platform/internal/core" +) + +func TestExternalConversationID(t *testing.T) { + tests := []struct { + name string + turn core.Turn + want string + }{ + {name: "direct user fallback", turn: core.Turn{ConversationType: core.ConversationDirect, ExternalUserID: "user-1"}, want: "user-1"}, + {name: "direct explicit chat", turn: core.Turn{ConversationType: core.ConversationDirect, ExternalChatID: "dm-1", ExternalUserID: "user-1"}, want: "dm-1"}, + {name: "group chat", turn: core.Turn{ConversationType: core.ConversationGroup, ExternalChatID: "group-1", ExternalUserID: "user-1"}, want: "group-1"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := externalConversationID(tc.turn); got != tc.want { + t.Fatalf("externalConversationID=%q want %q", got, tc.want) + } + }) + } +} + +func TestShouldCreateAnswerMessage(t *testing.T) { + tests := []struct { + name string + turn core.Turn + want bool + }{ + {name: "success", turn: core.Turn{Status: "succeeded", Answer: "answer", AnswerMessageID: "answer-1"}, want: true}, + {name: "failed", turn: core.Turn{Status: "failed", Answer: "answer", AnswerMessageID: "answer-1"}}, + {name: "empty answer", turn: core.Turn{Status: "succeeded", Answer: " ", AnswerMessageID: "answer-1"}}, + {name: "missing id", turn: core.Turn{Status: "succeeded", Answer: "answer"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := shouldCreateAnswerMessage(tc.turn); got != tc.want { + t.Fatalf("shouldCreateAnswerMessage=%v want %v", got, tc.want) + } + }) + } +} + +func TestBeginTurnSQLPersistsAuditFoundation(t *testing.T) { + src := strings.ToUpper(readStoreSource(t)) + for _, want := range []string{ + "INSERT INTO CHANNEL_ACTORS", + "ON CONFLICT (TENANT_KEY,CHANNEL,EXTERNAL_ACTOR_ID)", + "INSERT INTO CHANNEL_CONVERSATIONS", + "ON CONFLICT (TENANT_KEY,CHANNEL,EXTERNAL_CONVERSATION_ID)", + "RETURNING ID", + "INSERT INTO MESSAGES", + "DIRECTION,ROLE,MESSAGE_TYPE", + "'INBOUND','USER'", + "ON CONFLICT (TENANT_KEY,CHANNEL,EXTERNAL_MESSAGE_ID)", + "REQUEST_MESSAGE_ID,AGENT_ID", + } { + if !strings.Contains(src, want) { + t.Errorf("store missing BeginTurn audit SQL fragment %q", want) + } + } +} + +func TestFinishTurnSQLPersistsGeneratedAssistantMessage(t *testing.T) { + src := strings.ToUpper(readStoreSource(t)) + for _, want := range []string{ + "'OUTBOUND','ASSISTANT','TEXT'", + "'GENERATED'", + "REPLY_TO_MESSAGE_ID", + "ANSWER_MESSAGE_ID", + } { + if !strings.Contains(src, want) { + t.Errorf("store missing FinishTurn audit SQL fragment %q", want) + } + } +} + +func TestLoadTurnSQLRestoresAuditLinks(t *testing.T) { + src := strings.ToUpper(readStoreSource(t)) + for _, want := range []string{ + "T.REQUEST_MESSAGE_ID", + "T.ANSWER_MESSAGE_ID", + "T.AGENT_ID", + "JOIN MESSAGES", + "JOIN CHANNEL_CONVERSATIONS", + "JOIN CHANNEL_ACTORS", + } { + if !strings.Contains(src, want) { + t.Errorf("loadTurn missing audit SQL fragment %q", want) + } + } +} + +func TestMessageAuditWritesUseTransactions(t *testing.T) { + src := readStoreSource(t) + if strings.Count(src, "s.pool.BeginTx(ctx") < 3 { + t.Fatal("BeginTurn, ResolveSession and FinishTurn must each use a transaction") + } + if strings.Count(src, "tx.Commit(ctx)") < 3 { + t.Fatal("message audit transactions must commit explicitly") + } +} diff --git a/internal/store/postgres/store.go b/internal/store/postgres/store.go index dbc13b5..614994f 100644 --- a/internal/store/postgres/store.go +++ b/internal/store/postgres/store.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -43,18 +44,75 @@ func (s *Store) BeginTurn(ctx context.Context, turn core.Turn) (core.Turn, bool, if err != nil { return core.Turn{}, false, err } - command, err := s.pool.Exec(ctx, `INSERT INTO turns - (id, tenant_key, channel, external_message_id, session_key, question, status, media_json) - VALUES ($1,$2,$3,$4,$5,$6,'processing',$7) ON CONFLICT DO NOTHING`, - turn.ID, turn.TenantKey, turn.Channel, turn.ExternalMessageID, turn.SessionKey, turn.Question, mediaJSON) + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return core.Turn{}, false, err + } + defer func() { _ = tx.Rollback(ctx) }() + + if err = tx.QueryRow(ctx, `INSERT INTO channel_actors + (id,tenant_key,channel,external_actor_id,actor_type,display_name) + VALUES($1,$2,$3,$4,'user',$5) + ON CONFLICT (tenant_key,channel,external_actor_id) DO UPDATE SET + display_name=CASE WHEN EXCLUDED.display_name<>'' THEN EXCLUDED.display_name ELSE channel_actors.display_name END, + updated_at=NOW() + RETURNING id`, turn.ActorID, turn.TenantKey, turn.Channel, turn.ExternalUserID, turn.SenderName).Scan(&turn.ActorID); err != nil { + return core.Turn{}, false, err + } + + externalConversation := externalConversationID(turn) + if externalConversation == "" { + return core.Turn{}, false, errors.New("external conversation identity is required") + } + conversationTitle := "" + if turn.ConversationType == core.ConversationDirect { + conversationTitle = strings.TrimSpace(turn.SenderName) + } + if err = tx.QueryRow(ctx, `INSERT INTO channel_conversations + (id,tenant_key,channel,external_conversation_id,conversation_type,title,status) + VALUES($1,$2,$3,$4,$5,$6,'active') + ON CONFLICT (tenant_key,channel,external_conversation_id) DO UPDATE SET + title=CASE WHEN EXCLUDED.title<>'' THEN EXCLUDED.title ELSE channel_conversations.title END, + updated_at=NOW() + RETURNING id`, turn.ConversationID, turn.TenantKey, turn.Channel, externalConversation, + string(turn.ConversationType), conversationTitle).Scan(&turn.ConversationID); err != nil { + return core.Turn{}, false, err + } + + if err = tx.QueryRow(ctx, `INSERT INTO messages + (id,tenant_key,channel,conversation_id,actor_id,external_message_id,correlation_id, + direction,role,message_type,text_content,media_json,status,actor_name_snapshot,safe_metadata_json) + VALUES($1,$2,$3,$4,$5,$6,$7,'inbound','user',$8,$9,$10,'received',$11,'{}'::JSONB) + ON CONFLICT (tenant_key,channel,external_message_id) WHERE external_message_id!='' + DO UPDATE SET updated_at=messages.updated_at + RETURNING id`, turn.RequestMessageID, turn.TenantKey, turn.Channel, turn.ConversationID, + turn.ActorID, turn.ExternalMessageID, turn.ID, turn.MessageType, turn.Question, mediaJSON, + turn.SenderName).Scan(&turn.RequestMessageID); err != nil { + return core.Turn{}, false, err + } + + command, err := tx.Exec(ctx, `INSERT INTO turns + (id,tenant_key,channel,external_message_id,session_key,question,status,media_json,request_message_id,agent_id) + VALUES($1,$2,$3,$4,$5,$6,'processing',$7,$8,$9) ON CONFLICT DO NOTHING`, + turn.ID, turn.TenantKey, turn.Channel, turn.ExternalMessageID, turn.SessionKey, turn.Question, + mediaJSON, turn.RequestMessageID, turn.AgentID) if err != nil { return core.Turn{}, false, err } if command.RowsAffected() == 1 { + if err = tx.Commit(ctx); err != nil { + return core.Turn{}, false, err + } return turn, true, nil } - stored, err := s.loadTurn(ctx, turn.TenantKey, turn.Channel, turn.ExternalMessageID) - return stored, false, err + stored, err := loadTurnWith(ctx, tx, turn.TenantKey, turn.Channel, turn.ExternalMessageID) + if err != nil { + return core.Turn{}, false, err + } + if err = tx.Commit(ctx); err != nil { + return core.Turn{}, false, err + } + return stored, false, nil } func (s *Store) ResolveSession(ctx context.Context, mapping core.SessionMapping, create func(context.Context) (string, error)) (string, error) { @@ -100,18 +158,99 @@ func (s *Store) FinishTurn(ctx context.Context, turn core.Turn) error { if err != nil { return err } - _, err = s.pool.Exec(ctx, `UPDATE turns SET weknora_session_id=$2,weknora_request_id=$3,answer=$4,references_json=$5,media_json=$6,status=$7,error_message=$8,updated_at=NOW() WHERE id=$1`, turn.ID, turn.WeKnoraSessionID, turn.WeKnoraRequestID, turn.Answer, references, mediaJSON, turn.Status, turn.ErrorMessage) - return err + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + + var answerMessageID any + if shouldCreateAnswerMessage(turn) { + _, err = tx.Exec(ctx, `INSERT INTO messages + (id,tenant_key,channel,conversation_id,external_message_id,correlation_id, + direction,role,message_type,text_content,media_json,reply_to_message_id,status,actor_name_snapshot,safe_metadata_json) + VALUES($1,$2,$3,$4,'',$5,'outbound','assistant','text',$6,'[]'::JSONB,$7,'generated','','{}'::JSONB) + ON CONFLICT (id) DO NOTHING`, turn.AnswerMessageID, turn.TenantKey, turn.Channel, + turn.ConversationID, turn.ID, turn.Answer, turn.RequestMessageID) + if err != nil { + return err + } + answerMessageID = turn.AnswerMessageID + } + _, err = tx.Exec(ctx, `UPDATE turns SET weknora_session_id=$2,weknora_request_id=$3,answer=$4,references_json=$5,media_json=$6,status=$7,error_message=$8,answer_message_id=COALESCE($9::TEXT,answer_message_id),updated_at=NOW() WHERE id=$1`, + turn.ID, turn.WeKnoraSessionID, turn.WeKnoraRequestID, turn.Answer, references, mediaJSON, + turn.Status, turn.ErrorMessage, answerMessageID) + if err != nil { + return err + } + return tx.Commit(ctx) +} + +type rowQuerier interface { + QueryRow(context.Context, string, ...any) pgx.Row } func (s *Store) loadTurn(ctx context.Context, tenantKey, channel, messageID string) (core.Turn, error) { + return loadTurnWith(ctx, s.pool, tenantKey, channel, messageID) +} + +func loadTurnWith(ctx context.Context, query rowQuerier, tenantKey, channel, messageID string) (core.Turn, error) { var turn core.Turn var references []byte var mediaJSON []byte - err := s.pool.QueryRow(ctx, `SELECT id,tenant_key,channel,external_message_id,session_key,weknora_session_id,weknora_request_id,question,answer,references_json,media_json,status,error_message FROM turns WHERE tenant_key=$1 AND channel=$2 AND external_message_id=$3`, tenantKey, channel, messageID).Scan(&turn.ID, &turn.TenantKey, &turn.Channel, &turn.ExternalMessageID, &turn.SessionKey, &turn.WeKnoraSessionID, &turn.WeKnoraRequestID, &turn.Question, &turn.Answer, &references, &mediaJSON, &turn.Status, &turn.ErrorMessage) + var agentID, requestMessageID, answerMessageID *string + var actorID, conversationID, messageType, actorName *string + var conversationType, externalConversationID, externalActorID *string + err := query.QueryRow(ctx, `SELECT + t.id,t.tenant_key,t.channel,t.external_message_id,t.session_key,t.weknora_session_id, + t.weknora_request_id,t.question,t.answer,t.references_json,t.media_json,t.status,t.error_message, + t.agent_id,t.request_message_id,t.answer_message_id, + m.actor_id,m.conversation_id,m.message_type,m.actor_name_snapshot, + c.conversation_type,c.external_conversation_id,a.external_actor_id + FROM turns t + LEFT JOIN messages m ON m.id=t.request_message_id + LEFT JOIN channel_conversations c ON c.id=m.conversation_id + LEFT JOIN channel_actors a ON a.id=m.actor_id + WHERE t.tenant_key=$1 AND t.channel=$2 AND t.external_message_id=$3`, tenantKey, channel, messageID).Scan( + &turn.ID, &turn.TenantKey, &turn.Channel, &turn.ExternalMessageID, &turn.SessionKey, + &turn.WeKnoraSessionID, &turn.WeKnoraRequestID, &turn.Question, &turn.Answer, &references, + &mediaJSON, &turn.Status, &turn.ErrorMessage, &agentID, &requestMessageID, &answerMessageID, + &actorID, &conversationID, &messageType, &actorName, &conversationType, + &externalConversationID, &externalActorID) if err != nil { return core.Turn{}, err } + if agentID != nil { + turn.AgentID = *agentID + } + if requestMessageID != nil { + turn.RequestMessageID = *requestMessageID + } + if answerMessageID != nil { + turn.AnswerMessageID = *answerMessageID + } + if actorID != nil { + turn.ActorID = *actorID + } + if conversationID != nil { + turn.ConversationID = *conversationID + } + if messageType != nil { + turn.MessageType = *messageType + } + if actorName != nil { + turn.SenderName = *actorName + } + if externalActorID != nil { + turn.ExternalUserID = *externalActorID + } + if conversationType != nil { + turn.ConversationType = core.ConversationType(*conversationType) + } + if externalConversationID != nil && (turn.ConversationType == core.ConversationGroup || + (turn.ExternalUserID != "" && *externalConversationID != turn.ExternalUserID)) { + turn.ExternalChatID = *externalConversationID + } if len(references) > 0 { if err := json.Unmarshal(references, &turn.References); err != nil { return core.Turn{}, err @@ -124,3 +263,15 @@ func (s *Store) loadTurn(ctx context.Context, tenantKey, channel, messageID stri } return turn, nil } + +func shouldCreateAnswerMessage(turn core.Turn) bool { + return turn.Status == "succeeded" && strings.TrimSpace(turn.Answer) != "" && + strings.TrimSpace(turn.AnswerMessageID) != "" +} + +func externalConversationID(turn core.Turn) string { + if externalChatID := strings.TrimSpace(turn.ExternalChatID); externalChatID != "" { + return externalChatID + } + return strings.TrimSpace(turn.ExternalUserID) +} diff --git a/internal/store/postgres/store_contract_test.go b/internal/store/postgres/store_contract_test.go index 8ca3244..a77dd17 100644 --- a/internal/store/postgres/store_contract_test.go +++ b/internal/store/postgres/store_contract_test.go @@ -60,7 +60,8 @@ func TestStoreSQLUsesReferencesJSONColumn(t *testing.T) { if !strings.Contains(update, "references_json=$5") { t.Error("FinishTurn UPDATE must write to references_json column") } - if !strings.Contains(selectTurns, "answer,references_json,media_json,status") && !strings.Contains(selectTurns, "answer,references_json,status") { + normalizedSelect := strings.ReplaceAll(selectTurns, "t.", "") + if !strings.Contains(normalizedSelect, "answer,references_json,media_json,status") && !strings.Contains(normalizedSelect, "answer,references_json,status") { t.Error("loadTurn SELECT must read references_json column") } } -- 2.52.0 From 32b76b84cbda6b8bed5d95d5d2077c951fc9fae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 21 Jul 2026 10:52:59 +0800 Subject: [PATCH 2/2] fix: address message audit review findings --- db/migrations/000003_message_audit.up.sql | 8 +- .../000003_message_audit_contract_test.go | 23 +++-- internal/qa/service.go | 14 ++- internal/qa/service_test.go | 95 +++++++++++++++++++ 4 files changed, 124 insertions(+), 16 deletions(-) diff --git a/db/migrations/000003_message_audit.up.sql b/db/migrations/000003_message_audit.up.sql index a0d9d13..c1b505f 100644 --- a/db/migrations/000003_message_audit.up.sql +++ b/db/migrations/000003_message_audit.up.sql @@ -17,9 +17,6 @@ CREATE TABLE IF NOT EXISTS channel_actors ( UNIQUE (tenant_key, channel, external_actor_id) ); -CREATE INDEX IF NOT EXISTS idx_channel_actors_lookup - ON channel_actors (tenant_key, channel, external_actor_id); - -- ============================================================ -- 2. Channel Conversations -- ============================================================ @@ -36,9 +33,6 @@ CREATE TABLE IF NOT EXISTS channel_conversations ( UNIQUE (tenant_key, channel, external_conversation_id) ); -CREATE INDEX IF NOT EXISTS idx_channel_conversations_lookup - ON channel_conversations (tenant_key, channel, external_conversation_id); - -- ============================================================ -- 3. Messages -- ============================================================ @@ -75,7 +69,7 @@ CREATE INDEX IF NOT EXISTS idx_messages_tenant_conv_created -- Index for correlation lookups CREATE INDEX IF NOT EXISTS idx_messages_correlation - ON messages (correlation_id); + ON messages (tenant_key, correlation_id); -- ============================================================ -- 4. Extend turns table diff --git a/db/migrations/000003_message_audit_contract_test.go b/db/migrations/000003_message_audit_contract_test.go index 020aba4..6abcd00 100644 --- a/db/migrations/000003_message_audit_contract_test.go +++ b/db/migrations/000003_message_audit_contract_test.go @@ -197,15 +197,22 @@ func TestUp003MessagesPartialUniqueIndex(t *testing.T) { // TestUp003MessagesIndexes checks required indexes on messages. func TestUp003MessagesIndexes(t *testing.T) { up := read003UpMigration(t) - upper := strings.ToUpper(up) - - // Tenant + conversation + created_at - if !strings.Contains(upper, "TENANT_KEY") || !strings.Contains(upper, "CONVERSATION_ID") || !strings.Contains(upper, "CREATED_AT") { - t.Error("messages must have index on tenant_key, conversation_id, created_at") + conversationIndex := regexp.MustCompile(`(?is)CREATE\s+INDEX\s+IF\s+NOT\s+EXISTS\s+idx_messages_tenant_conv_created\s+ON\s+messages\s*\(\s*tenant_key\s*,\s*conversation_id\s*,\s*created_at\s+DESC\s*\)`) + if !conversationIndex.MatchString(up) { + t.Error("messages must have index on tenant_key, conversation_id, created_at DESC") } - // correlation_id index - if !strings.Contains(upper, "CORRELATION_ID") { - t.Error("messages must have index on correlation_id") + correlationIndex := regexp.MustCompile(`(?is)CREATE\s+INDEX\s+IF\s+NOT\s+EXISTS\s+idx_messages_correlation\s+ON\s+messages\s*\(\s*tenant_key\s*,\s*correlation_id\s*\)`) + if !correlationIndex.MatchString(up) { + t.Error("messages correlation index must be tenant-scoped: (tenant_key, correlation_id)") + } +} + +func TestUp003AvoidsRedundantLookupIndexes(t *testing.T) { + up := strings.ToLower(read003UpMigration(t)) + for _, redundant := range []string{"idx_channel_actors_lookup", "idx_channel_conversations_lookup"} { + if strings.Contains(up, redundant) { + t.Errorf("migration must not create redundant index %s; UNIQUE already provides it", redundant) + } } } diff --git a/internal/qa/service.go b/internal/qa/service.go index e81e889..bcf0139 100644 --- a/internal/qa/service.go +++ b/internal/qa/service.go @@ -60,6 +60,7 @@ func (s *Service) AskEvents(ctx context.Context, request core.QARequest, observe } func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserver weknora.AnswerObserver, eventObserver weknora.StreamObserver) (core.QAResult, error) { + request = normalizeRequestIdentity(request) if err := validateRequest(request); err != nil { return core.QAResult{}, err } @@ -205,6 +206,17 @@ func (s *Service) ask(ctx context.Context, request core.QARequest, answerObserve WeKnoraRequestID: chat.RequestID, Answer: chat.Answer, References: chat.References}, nil } +func normalizeRequestIdentity(request core.QARequest) core.QARequest { + request.TenantKey = strings.TrimSpace(request.TenantKey) + request.Channel = strings.TrimSpace(request.Channel) + request.ConversationType = core.ConversationType(strings.TrimSpace(string(request.ConversationType))) + request.ExternalChatID = strings.TrimSpace(request.ExternalChatID) + request.ExternalUserID = strings.TrimSpace(request.ExternalUserID) + request.ExternalMessageID = strings.TrimSpace(request.ExternalMessageID) + request.SenderName = strings.TrimSpace(request.SenderName) + return request +} + func normalizeMessageType(request core.QARequest) string { messageType := strings.ToLower(strings.TrimSpace(request.MessageType)) if messageType == "" { @@ -218,7 +230,7 @@ func normalizeMessageType(request core.QARequest) string { return "image" case "richtext", "rich_text": return "rich_text" - case "text", "card", "feedback": + case "text": return messageType default: return "" diff --git a/internal/qa/service_test.go b/internal/qa/service_test.go index 0f6d9c2..038f992 100644 --- a/internal/qa/service_test.go +++ b/internal/qa/service_test.go @@ -3,6 +3,7 @@ package qa import ( "context" "errors" + "strings" "sync" "testing" @@ -64,6 +65,17 @@ type fixedResolver struct{} func (fixedResolver) Resolve(_, _ string) (string, error) { return "agent-1", nil } +type recordingResolver struct { + channel string + externalChatID string +} + +func (r *recordingResolver) Resolve(channel, externalChatID string) (string, error) { + r.channel = channel + r.externalChatID = externalChatID + return "agent-1", nil +} + type fakeWeKnora struct { mu sync.Mutex sessions int @@ -488,3 +500,86 @@ func TestAskFailureDoesNotGenerateAnswerMessageID(t *testing.T) { t.Fatalf("failed turn must not generate answer message: %+v", finished) } } + +func TestAskCanonicalizesStableIdentityBeforeIdempotency(t *testing.T) { + repo := newMemoryRepository() + client := &fakeWeKnora{} + resolver := &recordingResolver{} + service, err := New(repo, resolver, client, session.GroupShared) + if err != nil { + t.Fatal(err) + } + first := core.QARequest{ + TenantKey: " tenant ", Channel: " dingtalk ", ConversationType: core.ConversationType(" direct "), + ExternalChatID: " dm-1 ", ExternalUserID: " user-1 ", ExternalMessageID: " message-trim ", + SenderName: " Alice ", MessageType: " Text ", Query: " preserve question whitespace ", + } + result, err := service.Ask(context.Background(), first) + if err != nil { + t.Fatalf("first Ask: %v", err) + } + second := first + second.TenantKey = "tenant" + second.Channel = "dingtalk" + second.ConversationType = core.ConversationDirect + second.ExternalChatID = "dm-1" + second.ExternalUserID = "user-1" + second.ExternalMessageID = "message-trim" + second.SenderName = "Alice" + replay, err := service.Ask(context.Background(), second) + if err != nil { + t.Fatalf("replay Ask: %v", err) + } + if !replay.Replayed || replay.TurnID != result.TurnID { + t.Fatalf("identity variants must replay the same turn: first=%+v replay=%+v", result, replay) + } + repo.mu.Lock() + begin := repo.beginTurns[0] + repo.mu.Unlock() + if begin.TenantKey != "tenant" || begin.Channel != "dingtalk" || begin.ConversationType != core.ConversationDirect || + begin.ExternalChatID != "dm-1" || begin.ExternalUserID != "user-1" || begin.ExternalMessageID != "message-trim" || + begin.SenderName != "Alice" { + t.Fatalf("stable identity was not canonicalized: %+v", begin) + } + if begin.Question != " preserve question whitespace " { + t.Fatalf("question content must not be identity-normalized: %q", begin.Question) + } + if resolver.channel != "dingtalk" || resolver.externalChatID != "dm-1" { + t.Fatalf("resolver received non-canonical identity: channel=%q chat=%q", resolver.channel, resolver.externalChatID) + } + client.mu.Lock() + defer client.mu.Unlock() + if len(client.requests) != 1 { + t.Fatalf("canonical duplicate must call WeKnora once, got %d", len(client.requests)) + } +} + +func TestAskRejectsNonQAMessageTypes(t *testing.T) { + for _, messageType := range []string{"card", "feedback"} { + t.Run(messageType, func(t *testing.T) { + repo := newMemoryRepository() + client := &fakeWeKnora{} + service, err := New(repo, fixedResolver{}, client, session.GroupShared) + if err != nil { + t.Fatal(err) + } + req := core.QARequest{ + TenantKey: "tenant", Channel: "dingtalk", ConversationType: core.ConversationDirect, + ExternalUserID: "user-1", ExternalMessageID: "message-" + messageType, + MessageType: messageType, Query: "not a QA message", + } + if _, err = service.Ask(context.Background(), req); err == nil || !strings.Contains(err.Error(), "unsupported message_type") { + t.Fatalf("expected unsupported message_type error, got %v", err) + } + repo.mu.Lock() + beginCount := len(repo.beginTurns) + repo.mu.Unlock() + client.mu.Lock() + requestCount := len(client.requests) + client.mu.Unlock() + if beginCount != 0 || requestCount != 0 { + t.Fatalf("non-QA type must have zero DB/model side effects: begins=%d requests=%d", beginCount, requestCount) + } + }) + } +} -- 2.52.0