feat: attachment_support
This commit is contained in:
@@ -0,0 +1,320 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxTextFileLines is the line limit for inline text content; excess lines are truncated.
|
||||
maxTextFileLines = 500
|
||||
// textFileExtensions lists plain-text extensions handled by the line-based reader.
|
||||
textFileExtensions = ".txt,.md,.markdown,.json,.xml,.yaml,.yml,.csv,.log"
|
||||
)
|
||||
|
||||
// AttachmentProcessor saves uploaded file attachments and extracts their text content
|
||||
// for injection into the LLM prompt.
|
||||
type AttachmentProcessor struct {
|
||||
fileService interfaces.FileService
|
||||
documentReader interfaces.DocumentReader
|
||||
imageResolver *docparser.ImageResolver
|
||||
modelService interfaces.ModelService // used to obtain the ASR model
|
||||
}
|
||||
|
||||
// NewAttachmentProcessor creates an AttachmentProcessor with the given dependencies.
|
||||
func NewAttachmentProcessor(
|
||||
fileService interfaces.FileService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
imageResolver *docparser.ImageResolver,
|
||||
modelService interfaces.ModelService,
|
||||
) *AttachmentProcessor {
|
||||
return &AttachmentProcessor{
|
||||
fileService: fileService,
|
||||
documentReader: documentReader,
|
||||
imageResolver: imageResolver,
|
||||
modelService: modelService,
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessAttachment validates, saves, and extracts content from a single uploaded file.
|
||||
// Content extraction is attempted for all supported types; errors are non-fatal (logged as warnings).
|
||||
func (p *AttachmentProcessor) ProcessAttachment(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileSize int64,
|
||||
tenantID uint64,
|
||||
asrModelID string, // optional; enables audio transcription when set
|
||||
) (*types.MessageAttachment, error) {
|
||||
logger.Infof(ctx, "processing attachment: fileName=%s, fileSize=%d", secutils.SanitizeForLog(fileName), fileSize)
|
||||
|
||||
// Validate filename (injection / path-traversal checks)
|
||||
safeFileName, isValid := secutils.ValidateInput(fileName)
|
||||
if !isValid {
|
||||
return nil, fmt.Errorf("invalid characters in file name")
|
||||
}
|
||||
|
||||
baseName, err := secutils.SafeFileName(safeFileName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsafe file name: %w", err)
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(baseName))
|
||||
if ext == "" {
|
||||
ext = ".txt"
|
||||
}
|
||||
|
||||
if !isValidFileType(baseName) {
|
||||
return nil, fmt.Errorf("unsupported file type: %s", ext)
|
||||
}
|
||||
|
||||
uniqueFileName := fmt.Sprintf("attachment_%s%s", uuid.New().String()[:12], ext)
|
||||
|
||||
storageURL, err := p.fileService.SaveBytes(ctx, data, tenantID, uniqueFileName, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to save attachment: %w", err)
|
||||
}
|
||||
|
||||
attachment := &types.MessageAttachment{
|
||||
URL: storageURL,
|
||||
FileName: baseName,
|
||||
FileType: ext,
|
||||
FileSize: fileSize,
|
||||
}
|
||||
|
||||
// Extract text content based on file type; errors are non-fatal.
|
||||
if p.isTextFile(ext) {
|
||||
if err := p.processTextFile(ctx, data, attachment); err != nil {
|
||||
logger.Warnf(ctx, "text file processing failed: %v", err)
|
||||
}
|
||||
} else if docparser.IsAudioFormat(ext) {
|
||||
if err := p.processAudioFile(ctx, data, baseName, attachment, asrModelID); err != nil {
|
||||
logger.Warnf(ctx, "audio transcription failed: %v, keeping placeholder", err)
|
||||
}
|
||||
} else if docparser.IsSimpleFormat(ext) {
|
||||
if err := p.processWithDocParser(ctx, data, baseName, ext, attachment, tenantID); err != nil {
|
||||
logger.Warnf(ctx, "SimpleFormatReader failed: %v", err)
|
||||
}
|
||||
} else {
|
||||
if err := p.processWithDocumentReader(ctx, data, baseName, ext, attachment, tenantID); err != nil {
|
||||
logger.Warnf(ctx, "DocumentReader failed: %v, keeping metadata only", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "attachment processed: fileName=%s, truncated=%v, contentLen=%d",
|
||||
secutils.SanitizeForLog(baseName), attachment.IsTruncated, len(attachment.Content))
|
||||
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
// isTextFile reports whether ext is a plain-text extension handled line-by-line.
|
||||
func (p *AttachmentProcessor) isTextFile(ext string) bool {
|
||||
return strings.Contains(textFileExtensions, ext)
|
||||
}
|
||||
|
||||
// processTextFile reads plain-text content line by line, truncating at maxTextFileLines.
|
||||
func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) error {
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
var lines []string
|
||||
lineCount := 0
|
||||
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
if lineCount <= maxTextFileLines {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
attachment.LineCount = lineCount
|
||||
attachment.Content = strings.Join(lines, "\n")
|
||||
|
||||
if lineCount > maxTextFileLines {
|
||||
attachment.IsTruncated = true
|
||||
logger.Infof(ctx, "text file truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processWithDocParser extracts content via SimpleFormatReader (md, csv, json, images, etc.).
|
||||
func (p *AttachmentProcessor) processWithDocParser(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileType string,
|
||||
attachment *types.MessageAttachment,
|
||||
tenantID uint64,
|
||||
) error {
|
||||
reader := &docparser.SimpleFormatReader{}
|
||||
result, err := reader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: data,
|
||||
FileName: fileName,
|
||||
FileType: fileType,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("SimpleFormatReader failed: %w", err)
|
||||
}
|
||||
|
||||
// Resolve embedded image refs to storage URLs.
|
||||
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
|
||||
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "image resolution failed: %v", err)
|
||||
} else {
|
||||
result.MarkdownContent = updatedMarkdown
|
||||
}
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
// processAudioFile transcribes audio via ASR. Falls back to a placeholder when no ASR model is configured.
|
||||
func (p *AttachmentProcessor) processAudioFile(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
attachment *types.MessageAttachment,
|
||||
asrModelID string,
|
||||
) error {
|
||||
if asrModelID == "" || p.modelService == nil {
|
||||
attachment.Content = fmt.Sprintf("[audio file: %s]", fileName)
|
||||
logger.Infof(ctx, "no ASR model configured, keeping audio placeholder")
|
||||
return nil
|
||||
}
|
||||
|
||||
asrInstance, err := p.modelService.GetASRModel(ctx, asrModelID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get ASR model: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "starting audio transcription: fileName=%s, size=%d", fileName, len(data))
|
||||
transcript, err := asrInstance.Transcribe(ctx, data, fileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("audio transcription failed: %w", err)
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, transcript, attachment)
|
||||
logger.Infof(ctx, "audio transcription done: textLen=%d", len(transcript))
|
||||
return nil
|
||||
}
|
||||
|
||||
// processWithDocumentReader extracts content from complex formats (pdf, docx, xlsx, etc.).
|
||||
func (p *AttachmentProcessor) processWithDocumentReader(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
fileName string,
|
||||
fileType string,
|
||||
attachment *types.MessageAttachment,
|
||||
tenantID uint64,
|
||||
) error {
|
||||
if p.documentReader == nil {
|
||||
return fmt.Errorf("DocumentReader not configured")
|
||||
}
|
||||
|
||||
result, err := p.documentReader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: data,
|
||||
FileName: fileName,
|
||||
FileType: fileType,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("DocumentReader failed: %w", err)
|
||||
}
|
||||
|
||||
// Resolve embedded image refs to storage URLs.
|
||||
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
|
||||
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "image resolution failed: %v", err)
|
||||
} else {
|
||||
result.MarkdownContent = updatedMarkdown
|
||||
}
|
||||
}
|
||||
|
||||
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyLineTruncation stores content into attachment, truncating at maxTextFileLines if needed.
|
||||
func (p *AttachmentProcessor) applyLineTruncation(ctx context.Context, content string, attachment *types.MessageAttachment) {
|
||||
lines := strings.Split(content, "\n")
|
||||
lineCount := len(lines)
|
||||
attachment.LineCount = lineCount
|
||||
|
||||
if lineCount > maxTextFileLines {
|
||||
attachment.Content = strings.Join(lines[:maxTextFileLines], "\n")
|
||||
attachment.IsTruncated = true
|
||||
logger.Infof(ctx, "content truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
|
||||
} else {
|
||||
attachment.Content = content
|
||||
}
|
||||
}
|
||||
|
||||
// isValidFileType reports whether fileName has a supported extension.
|
||||
// Kept in sync with the frontend SUPPORTED_TYPES list in AttachmentUpload.vue.
|
||||
func isValidFileType(fileName string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext == "" {
|
||||
return false
|
||||
}
|
||||
ext = strings.TrimPrefix(ext, ".")
|
||||
|
||||
supportedTypes := []string{
|
||||
// documents
|
||||
"docx", "doc", "pdf", "ppt", "pptx",
|
||||
// spreadsheets
|
||||
"xlsx", "xls",
|
||||
// text / markup
|
||||
"md", "markdown", "txt", "csv", "json", "xml", "yaml", "yml", "log", "html",
|
||||
// images
|
||||
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp",
|
||||
// audio
|
||||
"mp3", "wav", "m4a", "flac", "ogg", "aac",
|
||||
}
|
||||
|
||||
for _, t := range supportedTypes {
|
||||
if ext == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DecodeBase64Attachment decodes a base64 attachment payload, stripping any data URI prefix.
|
||||
// Tries Std, URL, RawStd, and RawURL encodings in order.
|
||||
func DecodeBase64Attachment(data string) ([]byte, error) {
|
||||
// Strip data URI prefix (e.g. "data:application/pdf;base64,")
|
||||
if idx := strings.Index(data, ","); idx != -1 {
|
||||
data = data[idx+1:]
|
||||
}
|
||||
data = strings.TrimSpace(data)
|
||||
|
||||
for _, enc := range []struct{ e *base64.Encoding }{
|
||||
{base64.StdEncoding},
|
||||
{base64.URLEncoding},
|
||||
{base64.RawStdEncoding},
|
||||
{base64.RawURLEncoding},
|
||||
} {
|
||||
if decoded, err := enc.e.DecodeString(data); err == nil {
|
||||
return decoded, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("base64 decode failed: unrecognised encoding")
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -24,6 +25,7 @@ type Handler struct {
|
||||
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
|
||||
fileService interfaces.FileService // Service for file storage (image uploads)
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
}
|
||||
|
||||
// NewHandler creates a new instance of Handler with all necessary dependencies
|
||||
@@ -38,6 +40,8 @@ func NewHandler(
|
||||
agentShareService interfaces.AgentShareService,
|
||||
fileService interfaces.FileService,
|
||||
modelService interfaces.ModelService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
imageResolver *docparser.ImageResolver,
|
||||
) *Handler {
|
||||
return &Handler{
|
||||
sessionService: sessionService,
|
||||
@@ -50,6 +54,12 @@ func NewHandler(
|
||||
agentShareService: agentShareService,
|
||||
fileService: fileService,
|
||||
modelService: modelService,
|
||||
attachmentProcessor: NewAttachmentProcessor(
|
||||
fileService,
|
||||
documentReader,
|
||||
imageResolver,
|
||||
modelService,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -166,7 +166,7 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
|
||||
}
|
||||
|
||||
// createUserMessage creates a user message and returns the created message.
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, channel string) (*types.Message, error) {
|
||||
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, attachments types.MessageAttachments, channel string) (*types.Message, error) {
|
||||
return h.messageService.CreateMessage(ctx, &types.Message{
|
||||
SessionID: sessionID,
|
||||
Role: "user",
|
||||
@@ -176,6 +176,7 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
|
||||
IsCompleted: true,
|
||||
MentionedItems: mentionedItems,
|
||||
Images: images,
|
||||
Attachments: attachments,
|
||||
Channel: channel,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
@@ -37,6 +38,7 @@ type qaRequestContext struct {
|
||||
images []ImageAttachment // Uploaded images with analysis text
|
||||
userMessageID string // Created user message ID (populated after createUserMessage)
|
||||
channel string // Source channel: "web", "api", "im", etc.
|
||||
attachments types.MessageAttachments // Processed file attachments
|
||||
}
|
||||
|
||||
// buildQARequest converts the qaRequestContext into a types.QARequest for service invocation.
|
||||
@@ -55,6 +57,7 @@ func (rc *qaRequestContext) buildQARequest() *types.QARequest {
|
||||
UserMessageID: rc.userMessageID,
|
||||
WebSearchEnabled: rc.webSearchEnabled,
|
||||
EnableMemory: rc.enableMemory,
|
||||
Attachments: rc.attachments,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +138,67 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
|
||||
// - Normal pure-chat mode: runs in the async goroutine with progress events
|
||||
}
|
||||
|
||||
// Process file attachments: decode and save to storage, extract content
|
||||
var processedAttachments types.MessageAttachments
|
||||
if len(request.AttachmentUploads) > 0 {
|
||||
logger.Infof(ctx, "[%s] processing %d attachment(s)", logPrefix, len(request.AttachmentUploads))
|
||||
|
||||
maxSize := secutils.GetMaxFileSize()
|
||||
for i, upload := range request.AttachmentUploads {
|
||||
if upload.FileSize > maxSize {
|
||||
return nil, nil, errors.NewBadRequestError(
|
||||
fmt.Sprintf("attachment %d exceeds size limit of %dMB", i+1, secutils.GetMaxFileSizeMB()))
|
||||
}
|
||||
}
|
||||
|
||||
tenantID := c.GetUint64(types.TenantIDContextKey.String())
|
||||
|
||||
// Use ASR only when the agent has audio upload enabled.
|
||||
asrModelID := ""
|
||||
if customAgent != nil && customAgent.Config.AudioUploadEnabled && customAgent.Config.ASRModelID != "" {
|
||||
asrModelID = customAgent.Config.ASRModelID
|
||||
}
|
||||
|
||||
// Process all attachments concurrently.
|
||||
processedAttachments = make(types.MessageAttachments, len(request.AttachmentUploads))
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(request.AttachmentUploads))
|
||||
|
||||
for i, upload := range request.AttachmentUploads {
|
||||
wg.Add(1)
|
||||
go func(idx int, att AttachmentUpload) {
|
||||
defer wg.Done()
|
||||
|
||||
data, err := DecodeBase64Attachment(att.Data)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("attachment %d decode failed: %w", idx+1, err)
|
||||
return
|
||||
}
|
||||
|
||||
processed, err := h.attachmentProcessor.ProcessAttachment(
|
||||
ctx, data, att.FileName, att.FileSize, tenantID, asrModelID,
|
||||
)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("attachment %d processing failed: %w", idx+1, err)
|
||||
return
|
||||
}
|
||||
|
||||
processedAttachments[idx] = *processed
|
||||
}(i, upload)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
if len(errChan) > 0 {
|
||||
err := <-errChan
|
||||
logger.Errorf(ctx, "[%s] attachment processing failed: %v", logPrefix, err)
|
||||
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("attachment processing failed: %v", err))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[%s] all attachments processed", logPrefix)
|
||||
}
|
||||
|
||||
// Build request context
|
||||
reqCtx := &qaRequestContext{
|
||||
ctx: ctx,
|
||||
@@ -160,6 +224,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
|
||||
effectiveTenantID: effectiveTenantID,
|
||||
images: request.Images,
|
||||
channel: request.Channel,
|
||||
attachments: processedAttachments,
|
||||
}
|
||||
|
||||
return reqCtx, &request, nil
|
||||
@@ -478,7 +543,7 @@ func (h *Handler) executeQA(reqCtx *qaRequestContext, mode qaMode, generateTitle
|
||||
}
|
||||
|
||||
// Create user message
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.channel)
|
||||
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.attachments, reqCtx.channel)
|
||||
if err != nil {
|
||||
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
|
||||
@@ -49,9 +49,17 @@ type CreateKnowledgeQARequest struct {
|
||||
DisableTitle bool `json:"disable_title"` // Whether to disable auto title generation
|
||||
EnableMemory bool `json:"enable_memory"` // Whether memory feature is enabled for this request
|
||||
Images []ImageAttachment `json:"images"` // Attached images for multimodal chat
|
||||
AttachmentUploads []AttachmentUpload `json:"attachment_uploads,omitempty"` // Attached files (documents, audio, etc.)
|
||||
Channel string `json:"channel"` // Source channel: "web", "api", "im", etc.
|
||||
}
|
||||
|
||||
// AttachmentUpload represents a file attachment upload from the client
|
||||
type AttachmentUpload struct {
|
||||
Data string `json:"data"` // Base64-encoded file content
|
||||
FileName string `json:"file_name"` // Original filename
|
||||
FileSize int64 `json:"file_size"` // File size in bytes
|
||||
}
|
||||
|
||||
// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization
|
||||
type SearchKnowledgeRequest struct {
|
||||
Query string `json:"query" binding:"required"` // Query text to search for
|
||||
|
||||
Reference in New Issue
Block a user