feat: attachment_support
This commit is contained in:
@@ -95,6 +95,10 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
if chatManage.QuotedContext != "" {
|
||||
userContent += "\n\n" + chatManage.QuotedContext
|
||||
}
|
||||
// Inject attachment content (documents, audio transcripts, etc.)
|
||||
if len(chatManage.Attachments) > 0 {
|
||||
userContent += chatManage.Attachments.BuildPrompt()
|
||||
}
|
||||
|
||||
if tpl := chatManage.SummaryConfig.ContextTemplate; tpl != "" {
|
||||
chatManage.UserContent = types.RenderPromptPlaceholders(tpl, types.PlaceholderValues{
|
||||
@@ -180,6 +184,10 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
if chatManage.QuotedContext != "" {
|
||||
userContent += "\n\n" + chatManage.QuotedContext
|
||||
}
|
||||
// Inject attachment content (documents, audio transcripts, etc.)
|
||||
if len(chatManage.Attachments) > 0 {
|
||||
userContent += chatManage.Attachments.BuildPrompt()
|
||||
}
|
||||
|
||||
// Set formatted content back to chat management
|
||||
chatManage.UserContent = userContent
|
||||
|
||||
@@ -121,6 +121,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
Images: req.ImageURLs,
|
||||
VLMModelID: vlmModelID,
|
||||
ChatModelSupportsVision: chatModelSupportsVision,
|
||||
Attachments: req.Attachments,
|
||||
Language: types.LanguageNameFromContext(ctx),
|
||||
},
|
||||
PipelineState: types.PipelineState{
|
||||
@@ -154,6 +155,10 @@ func (s *sessionService) KnowledgeQA(
|
||||
if req.QuotedContext != "" {
|
||||
userContent += "\n\n" + req.QuotedContext
|
||||
}
|
||||
// Inject attachment content for pure-chat path (RAG path handles this in INTO_CHAT_MESSAGE).
|
||||
if len(req.Attachments) > 0 {
|
||||
userContent += req.Attachments.BuildPrompt()
|
||||
}
|
||||
chatManage.UserContent = userContent
|
||||
|
||||
pipeline = types.NewPipelineBuilder().
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxTextFileLines is the line limit for inline text content; excess lines are truncated.
|
||||
maxTextFileLines = 500
|
||||
// textFileExtensions lists plain-text extensions handled by the line-based reader.
|
||||
textFileExtensions = ".txt,.md,.markdown,.json,.xml,.yaml,.yml,.csv,.log"
|
||||
)
|
||||
|
||||
// AttachmentProcessor saves uploaded file attachments and extracts their text content
|
||||
// for injection into the LLM prompt.
|
||||
type AttachmentProcessor struct {
|
||||
fileService interfaces.FileService
|
||||
documentReader interfaces.DocumentReader
|
||||
imageResolver *docparser.ImageResolver
|
||||
modelService interfaces.ModelService // used to obtain the ASR model
|
||||
}
|
||||
|
||||
// NewAttachmentProcessor creates an AttachmentProcessor with the given dependencies.
|
||||
func NewAttachmentProcessor(
|
||||
fileService interfaces.FileService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
imageResolver *docparser.ImageResolver,
|
||||
modelService interfaces.ModelService,
|
||||
) *AttachmentProcessor {
|
||||
return &AttachmentProcessor{
|
||||
fileService: fileService,
|
||||
documentReader: documentReader,
|
||||
imageResolver: imageResolver,
|
||||
modelService: modelService,
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessAttachment validates, saves, and extracts content from a single uploaded file.
|
||||
// Content extraction is attempted for all supported types; errors are non-fatal (logged as warnings).
|
||||
func (p *AttachmentProcessor) ProcessAttachment(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileSize int64,
|
||||
tenantID uint64,
|
||||
asrModelID string, // optional; enables audio transcription when set
|
||||
) (*types.MessageAttachment, error) {
|
||||
logger.Infof(ctx, "processing attachment: fileName=%s, fileSize=%d", secutils.SanitizeForLog(fileName), fileSize)
|
||||
|
||||
// Validate filename (injection / path-traversal checks)
|
||||
safeFileName, isValid := secutils.ValidateInput(fileName)
|
||||
if !isValid {
|
||||
return nil, fmt.Errorf("invalid characters in file name")
|
||||
}
|
||||
|
||||
baseName, err := secutils.SafeFileName(safeFileName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsafe file name: %w", err)
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(baseName))
|
||||
if ext == "" {
|
||||
ext = ".txt"
|
||||
}
|
||||
|
||||
if !isValidFileType(baseName) {
|
||||
return nil, fmt.Errorf("unsupported file type: %s", ext)
|
||||
}
|
||||
|
||||
uniqueFileName := fmt.Sprintf("attachment_%s%s", uuid.New().String()[:12], ext)
|
||||
|
||||
storageURL, err := p.fileService.SaveBytes(ctx, data, tenantID, uniqueFileName, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to save attachment: %w", err)
|
||||
}
|
||||
|
||||
attachment := &types.MessageAttachment{
|
||||
URL: storageURL,
|
||||
FileName: baseName,
|
||||
FileType: ext,
|
||||
FileSize: fileSize,
|
||||
}
|
||||
|
||||
// Extract text content based on file type; errors are non-fatal.
|
||||
if p.isTextFile(ext) {
|
||||
if err := p.processTextFile(ctx, data, attachment); err != nil {
|
||||
logger.Warnf(ctx, "text file processing failed: %v", err)
|
||||
}
|
||||
} else if docparser.IsAudioFormat(ext) {
|
||||
if err := p.processAudioFile(ctx, data, baseName, attachment, asrModelID); err != nil {
|
||||
logger.Warnf(ctx, "audio transcription failed: %v, keeping placeholder", err)
|
||||
}
|
||||
} else if docparser.IsSimpleFormat(ext) {
|
||||
if err := p.processWithDocParser(ctx, data, baseName, ext, attachment, tenantID); err != nil {
|
||||
logger.Warnf(ctx, "SimpleFormatReader failed: %v", err)
|
||||
}
|
||||
} else {
|
||||
if err := p.processWithDocumentReader(ctx, data, baseName, ext, attachment, tenantID); err != nil {
|
||||
logger.Warnf(ctx, "DocumentReader failed: %v, keeping metadata only", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "attachment processed: fileName=%s, truncated=%v, contentLen=%d",
|
||||
secutils.SanitizeForLog(baseName), attachment.IsTruncated, len(attachment.Content))
|
||||
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
// isTextFile reports whether ext is a plain-text extension handled line-by-line.
|
||||
func (p *AttachmentProcessor) isTextFile(ext string) bool {
|
||||
return strings.Contains(textFileExtensions, ext)
|
||||
}
|
||||
|
||||
// processTextFile reads plain-text content line by line, truncating at maxTextFileLines.
|
||||
func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) error {
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
var lines []string
|
||||
lineCount := 0
|
||||
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
if lineCount <= maxTextFileLines {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
attachment.LineCount = lineCount
|
||||
attachment.Content = strings.Join(lines, "\n")
|
||||
|
||||
if lineCount > maxTextFileLines {
|
||||
attachment.IsTruncated = true
|
||||
logger.Infof(ctx, "text file truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processWithDocParser extracts content via SimpleFormatReader (md, csv, json, images, etc.).
|
||||
func (p *AttachmentProcessor) processWithDocParser(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileType string,
|
||||
attachment *types.MessageAttachment,
|
||||
tenantID uint64,
|
||||
) error {
|
||||
reader := &docparser.SimpleFormatReader{}
|
||||
result, err := reader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: data,
|
||||
FileName: fileName,
|
||||
FileType: fileType,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("SimpleFormatReader failed: %w", err)
|
||||
}
|
||||
|
||||
// Resolve embedded image refs to storage URLs.
|
||||
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
|
||||
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "image resolution failed: %v", err)
|
||||
} else {
|
||||
result.MarkdownContent = updatedMarkdown
|
||||
}
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
// processAudioFile transcribes audio via ASR. Falls back to a placeholder when no ASR model is configured.
|
||||
func (p *AttachmentProcessor) processAudioFile(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
attachment *types.MessageAttachment,
|
||||
asrModelID string,
|
||||
) error {
|
||||
if asrModelID == "" || p.modelService == nil {
|
||||
attachment.Content = fmt.Sprintf("[audio file: %s]", fileName)
|
||||
logger.Infof(ctx, "no ASR model configured, keeping audio placeholder")
|
||||
return nil
|
||||
}
|
||||
|
||||
asrInstance, err := p.modelService.GetASRModel(ctx, asrModelID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get ASR model: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "starting audio transcription: fileName=%s, size=%d", fileName, len(data))
|
||||
transcript, err := asrInstance.Transcribe(ctx, data, fileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("audio transcription failed: %w", err)
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, transcript, attachment)
|
||||
logger.Infof(ctx, "audio transcription done: textLen=%d", len(transcript))
|
||||
return nil
|
||||
}
|
||||
|
||||
// processWithDocumentReader extracts content from complex formats (pdf, docx, xlsx, etc.).
|
||||
func (p *AttachmentProcessor) processWithDocumentReader(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileType string,
|
||||
attachment *types.MessageAttachment,
|
||||
tenantID uint64,
|
||||
) error {
|
||||
if p.documentReader == nil {
|
||||
return fmt.Errorf("DocumentReader not configured")
|
||||
}
|
||||
|
||||
result, err := p.documentReader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: data,
|
||||
FileName: fileName,
|
||||
FileType: fileType,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("DocumentReader failed: %w", err)
|
||||
}
|
||||
|
||||
// Resolve embedded image refs to storage URLs.
|
||||
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
|
||||
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "image resolution failed: %v", err)
|
||||
} else {
|
||||
result.MarkdownContent = updatedMarkdown
|
||||
}
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyLineTruncation stores content into attachment, truncating at maxTextFileLines if needed.
|
||||
func (p *AttachmentProcessor) applyLineTruncation(ctx context.Context, content string, attachment *types.MessageAttachment) {
|
||||
lines := strings.Split(content, "\n")
|
||||
lineCount := len(lines)
|
||||
attachment.LineCount = lineCount
|
||||
|
||||
if lineCount > maxTextFileLines {
|
||||
attachment.Content = strings.Join(lines[:maxTextFileLines], "\n")
|
||||
attachment.IsTruncated = true
|
||||
logger.Infof(ctx, "content truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
|
||||
} else {
|
||||
attachment.Content = content
|
||||
}
|
||||
}
|
||||
|
||||
// isValidFileType reports whether fileName has a supported extension.
|
||||
// Kept in sync with the frontend SUPPORTED_TYPES list in AttachmentUpload.vue.
|
||||
func isValidFileType(fileName string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext == "" {
|
||||
return false
|
||||
}
|
||||
ext = strings.TrimPrefix(ext, ".")
|
||||
|
||||
supportedTypes := []string{
|
||||
// documents
|
||||
"docx", "doc", "pdf", "ppt", "pptx",
|
||||
// spreadsheets
|
||||
"xlsx", "xls",
|
||||
// text / markup
|
||||
"md", "markdown", "txt", "csv", "json", "xml", "yaml", "yml", "log", "html",
|
||||
// images
|
||||
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp",
|
||||
// audio
|
||||
"mp3", "wav", "m4a", "flac", "ogg", "aac",
|
||||
}
|
||||
|
||||
for _, t := range supportedTypes {
|
||||
if ext == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DecodeBase64Attachment decodes a base64 attachment payload, stripping any data URI prefix.
|
||||
// Tries Std, URL, RawStd, and RawURL encodings in order.
|
||||
func DecodeBase64Attachment(data string) ([]byte, error) {
|
||||
// Strip data URI prefix (e.g. "data:application/pdf;base64,")
|
||||
if idx := strings.Index(data, ","); idx != -1 {
|
||||
data = data[idx+1:]
|
||||
}
|
||||
data = strings.TrimSpace(data)
|
||||
|
||||
for _, enc := range []struct{ e *base64.Encoding }{
|
||||
{base64.StdEncoding},
|
||||
{base64.URLEncoding},
|
||||
{base64.RawStdEncoding},
|
||||
{base64.RawURLEncoding},
|
||||
} {
|
||||
if decoded, err := enc.e.DecodeString(data); err == nil {
|
||||
return decoded, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("base64 decode failed: unrecognised encoding")
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -24,6 +25,7 @@ type Handler struct {
|
||||
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
|
||||
fileService interfaces.FileService // Service for file storage (image uploads)
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
}
|
||||
|
||||
// NewHandler creates a new instance of Handler with all necessary dependencies
|
||||
@@ -38,6 +40,8 @@ func NewHandler(
|
||||
agentShareService interfaces.AgentShareService,
|
||||
fileService interfaces.FileService,
|
||||
modelService interfaces.ModelService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
imageResolver *docparser.ImageResolver,
|
||||
) *Handler {
|
||||
return &Handler{
|
||||
sessionService: sessionService,
|
||||
@@ -50,6 +54,12 @@ func NewHandler(
|
||||
agentShareService: agentShareService,
|
||||
fileService: fileService,
|
||||
modelService: modelService,
|
||||
attachmentProcessor: NewAttachmentProcessor(
|
||||
fileService,
|
||||
documentReader,
|
||||
imageResolver,
|
||||
modelService,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -166,7 +166,7 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
|
||||
}
|
||||
|
||||
// createUserMessage creates a user message and returns the created message.
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, channel string) (*types.Message, error) {
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, attachments types.MessageAttachments, channel string) (*types.Message, error) {
|
||||
return h.messageService.CreateMessage(ctx, &types.Message{
|
||||
SessionID: sessionID,
|
||||
Role: "user",
|
||||
@@ -176,6 +176,7 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
|
||||
IsCompleted: true,
|
||||
MentionedItems: mentionedItems,
|
||||
Images: images,
|
||||
Attachments: attachments,
|
||||
Channel: channel,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
@@ -37,6 +38,7 @@ type qaRequestContext struct {
|
||||
images []ImageAttachment // Uploaded images with analysis text
|
||||
userMessageID string // Created user message ID (populated after createUserMessage)
|
||||
channel string // Source channel: "web", "api", "im", etc.
|
||||
attachments types.MessageAttachments // Processed file attachments
|
||||
}
|
||||
|
||||
// buildQARequest converts the qaRequestContext into a types.QARequest for service invocation.
|
||||
@@ -55,6 +57,7 @@ func (rc *qaRequestContext) buildQARequest() *types.QARequest {
|
||||
UserMessageID: rc.userMessageID,
|
||||
WebSearchEnabled: rc.webSearchEnabled,
|
||||
EnableMemory: rc.enableMemory,
|
||||
Attachments: rc.attachments,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +138,67 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
|
||||
// - Normal pure-chat mode: runs in the async goroutine with progress events
|
||||
}
|
||||
|
||||
// Process file attachments: decode and save to storage, extract content
|
||||
var processedAttachments types.MessageAttachments
|
||||
if len(request.AttachmentUploads) > 0 {
|
||||
logger.Infof(ctx, "[%s] processing %d attachment(s)", logPrefix, len(request.AttachmentUploads))
|
||||
|
||||
maxSize := secutils.GetMaxFileSize()
|
||||
for i, upload := range request.AttachmentUploads {
|
||||
if upload.FileSize > maxSize {
|
||||
return nil, nil, errors.NewBadRequestError(
|
||||
fmt.Sprintf("attachment %d exceeds size limit of %dMB", i+1, secutils.GetMaxFileSizeMB()))
|
||||
}
|
||||
}
|
||||
|
||||
tenantID := c.GetUint64(types.TenantIDContextKey.String())
|
||||
|
||||
// Use ASR only when the agent has audio upload enabled.
|
||||
asrModelID := ""
|
||||
if customAgent != nil && customAgent.Config.AudioUploadEnabled && customAgent.Config.ASRModelID != "" {
|
||||
asrModelID = customAgent.Config.ASRModelID
|
||||
}
|
||||
|
||||
// Process all attachments concurrently.
|
||||
processedAttachments = make(types.MessageAttachments, len(request.AttachmentUploads))
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(request.AttachmentUploads))
|
||||
|
||||
for i, upload := range request.AttachmentUploads {
|
||||
wg.Add(1)
|
||||
go func(idx int, att AttachmentUpload) {
|
||||
defer wg.Done()
|
||||
|
||||
data, err := DecodeBase64Attachment(att.Data)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("attachment %d decode failed: %w", idx+1, err)
|
||||
return
|
||||
}
|
||||
|
||||
processed, err := h.attachmentProcessor.ProcessAttachment(
|
||||
ctx, data, att.FileName, att.FileSize, tenantID, asrModelID,
|
||||
)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("attachment %d processing failed: %w", idx+1, err)
|
||||
return
|
||||
}
|
||||
|
||||
processedAttachments[idx] = *processed
|
||||
}(i, upload)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
if len(errChan) > 0 {
|
||||
err := <-errChan
|
||||
logger.Errorf(ctx, "[%s] attachment processing failed: %v", logPrefix, err)
|
||||
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("attachment processing failed: %v", err))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[%s] all attachments processed", logPrefix)
|
||||
}
|
||||
|
||||
// Build request context
|
||||
reqCtx := &qaRequestContext{
|
||||
ctx: ctx,
|
||||
@@ -160,6 +224,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
|
||||
effectiveTenantID: effectiveTenantID,
|
||||
images: request.Images,
|
||||
channel: request.Channel,
|
||||
attachments: processedAttachments,
|
||||
}
|
||||
|
||||
return reqCtx, &request, nil
|
||||
@@ -478,7 +543,7 @@ func (h *Handler) executeQA(reqCtx *qaRequestContext, mode qaMode, generateTitle
|
||||
}
|
||||
|
||||
// Create user message
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.channel)
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.attachments, reqCtx.channel)
|
||||
if err != nil {
|
||||
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
|
||||
@@ -49,9 +49,17 @@ type CreateKnowledgeQARequest struct {
|
||||
DisableTitle bool `json:"disable_title"` // Whether to disable auto title generation
|
||||
EnableMemory bool `json:"enable_memory"` // Whether memory feature is enabled for this request
|
||||
Images []ImageAttachment `json:"images"` // Attached images for multimodal chat
|
||||
AttachmentUploads []AttachmentUpload `json:"attachment_uploads,omitempty"` // Attached files (documents, audio, etc.)
|
||||
Channel string `json:"channel"` // Source channel: "web", "api", "im", etc.
|
||||
}
|
||||
|
||||
// AttachmentUpload represents a file attachment upload from the client
|
||||
type AttachmentUpload struct {
|
||||
Data string `json:"data"` // Base64-encoded file content
|
||||
FileName string `json:"file_name"` // Original filename
|
||||
FileSize int64 `json:"file_size"` // File size in bytes
|
||||
}
|
||||
|
||||
// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization
|
||||
type SearchKnowledgeRequest struct {
|
||||
Query string `json:"query" binding:"required"` // Query text to search for
|
||||
|
||||
@@ -45,6 +45,9 @@ type PipelineRequest struct {
|
||||
VLMModelID string `json:"-"`
|
||||
ChatModelSupportsVision bool `json:"-"`
|
||||
|
||||
// File attachments support
|
||||
Attachments MessageAttachments `json:"-"`
|
||||
|
||||
// Misc request-scoped config
|
||||
TenantID uint64 `json:"-"`
|
||||
WebSearchEnabled bool `json:"-"`
|
||||
@@ -185,6 +188,7 @@ func (c *ChatManage) Clone() *ChatManage {
|
||||
Images: append([]string(nil), c.Images...),
|
||||
VLMModelID: c.VLMModelID,
|
||||
ChatModelSupportsVision: c.ChatModelSupportsVision,
|
||||
Attachments: append(MessageAttachments(nil), c.Attachments...),
|
||||
TenantID: c.TenantID,
|
||||
WebSearchEnabled: c.WebSearchEnabled,
|
||||
WebSearchProviderID: c.WebSearchProviderID,
|
||||
|
||||
@@ -120,6 +120,10 @@ type CustomAgentConfig struct {
|
||||
ImageUploadEnabled bool `yaml:"image_upload_enabled" json:"image_upload_enabled"`
|
||||
// VLM model ID for image analysis (optional, falls back to tenant-level VLM)
|
||||
VLMModelID string `yaml:"vlm_model_id" json:"vlm_model_id"`
|
||||
// Whether audio upload (ASR transcription) is enabled for this agent (default: false)
|
||||
AudioUploadEnabled bool `yaml:"audio_upload_enabled" json:"audio_upload_enabled"`
|
||||
// ASR model ID for audio transcription (optional)
|
||||
ASRModelID string `yaml:"asr_model_id" json:"asr_model_id"`
|
||||
// Storage provider for image uploads: "local", "minio", "cos", "tos"
|
||||
// Empty means use the global/tenant default provider.
|
||||
ImageStorageProvider string `yaml:"image_storage_provider" json:"image_storage_provider"`
|
||||
|
||||
@@ -4,6 +4,8 @@ package types
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -64,6 +66,79 @@ func (m *MessageImages) Scan(value interface{}) error {
|
||||
return json.Unmarshal(b, m)
|
||||
}
|
||||
|
||||
// MessageAttachment represents a file attachment in a chat message
|
||||
type MessageAttachment struct {
|
||||
URL string `json:"url"` // Storage URL (provider://path)
|
||||
FileName string `json:"file_name"` // Original filename
|
||||
FileType string `json:"file_type"` // File extension (e.g., ".pdf", ".docx")
|
||||
FileSize int64 `json:"file_size"` // File size in bytes
|
||||
Content string `json:"content,omitempty"` // Extracted text content (for small text files)
|
||||
IsTruncated bool `json:"is_truncated,omitempty"` // Whether content was truncated
|
||||
LineCount int `json:"line_count,omitempty"` // Total line count (for text files)
|
||||
}
|
||||
|
||||
// MessageAttachments is a slice of MessageAttachment for database storage
|
||||
type MessageAttachments []MessageAttachment
|
||||
|
||||
// BuildPrompt returns a formatted prompt section for all attachments,
|
||||
// injecting file metadata and extracted content into the LLM context.
|
||||
func (attachments MessageAttachments) BuildPrompt() string {
|
||||
if len(attachments) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("\n\n## 用户上传的附件\n\n")
|
||||
|
||||
for i, att := range attachments {
|
||||
sb.WriteString(fmt.Sprintf("### 附件 %d: %s\n\n", i+1, att.FileName))
|
||||
sb.WriteString(fmt.Sprintf("- **文件类型**: %s\n", att.FileType))
|
||||
sb.WriteString(fmt.Sprintf("- **文件大小**: %.2f KB\n\n", float64(att.FileSize)/1024))
|
||||
|
||||
if att.Content != "" {
|
||||
sb.WriteString("**文件内容**:\n\n")
|
||||
sb.WriteString(att.Content)
|
||||
sb.WriteString("\n\n")
|
||||
|
||||
if att.IsTruncated {
|
||||
sb.WriteString(fmt.Sprintf("*注意: 此文件共有 %d 行,已截取前 500 行显示。*\n\n",
|
||||
att.LineCount))
|
||||
}
|
||||
} else {
|
||||
sb.WriteString("*此文件内容提取失败或不支持。*\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface for database serialization
|
||||
func (m MessageAttachments) Value() (driver.Value, error) {
|
||||
if m == nil {
|
||||
return json.Marshal([]MessageAttachment{})
|
||||
}
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface for database deserialization
|
||||
func (m *MessageAttachments) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
*m = make(MessageAttachments, 0)
|
||||
return nil
|
||||
}
|
||||
var b []byte
|
||||
switch v := value.(type) {
|
||||
case []byte:
|
||||
b = v
|
||||
case string:
|
||||
b = []byte(v)
|
||||
default:
|
||||
*m = make(MessageAttachments, 0)
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, m)
|
||||
}
|
||||
|
||||
// MentionedItems is a slice of MentionedItem for database storage
|
||||
type MentionedItems []MentionedItem
|
||||
|
||||
@@ -119,6 +194,8 @@ type Message struct {
|
||||
MentionedItems MentionedItems `json:"mentioned_items,omitempty" gorm:"type:jsonb,column:mentioned_items"`
|
||||
// Attached images with OCR/Caption text (for user messages)
|
||||
Images MessageImages `json:"images,omitempty" gorm:"type:jsonb;column:images"`
|
||||
// Attached files (documents, audio, etc., for user messages)
|
||||
Attachments MessageAttachments `json:"attachments,omitempty" gorm:"type:jsonb;column:attachments"`
|
||||
// Whether message generation is complete
|
||||
IsCompleted bool `json:"is_completed"`
|
||||
// Whether this response is a fallback (no knowledge base match found)
|
||||
@@ -194,6 +271,9 @@ func (m *Message) BeforeCreate(tx *gorm.DB) (err error) {
|
||||
if m.Images == nil {
|
||||
m.Images = make(MessageImages, 0)
|
||||
}
|
||||
if m.Attachments == nil {
|
||||
m.Attachments = make(MessageAttachments, 0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,4 +17,5 @@ type QARequest struct {
|
||||
WebSearchEnabled bool // Whether web search is enabled for this request
|
||||
EnableMemory bool // Whether memory feature is enabled
|
||||
QuotedContext string // Quoted message content from IM quote-reply (appended at LLM prompt stage, not used for retrieval)
|
||||
Attachments MessageAttachments // File attachments (processed and ready for prompt injection)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user