feat: attachment_support

This commit is contained in:
MaoMengww
2026-04-10 12:49:20 +08:00
parent 69785ad0a3
commit aaee5f044a
26 changed files with 1029 additions and 19 deletions
@@ -0,0 +1,320 @@
package session
import (
"bufio"
"bytes"
"context"
"encoding/base64"
"fmt"
"path/filepath"
"strings"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
)
const (
// maxTextFileLines is the line limit for inline text content; excess lines are truncated.
maxTextFileLines = 500
// textFileExtensions lists plain-text extensions handled by the line-based reader.
textFileExtensions = ".txt,.md,.markdown,.json,.xml,.yaml,.yml,.csv,.log"
)
// AttachmentProcessor saves uploaded file attachments and extracts their text content
// for injection into the LLM prompt.
type AttachmentProcessor struct {
fileService interfaces.FileService
documentReader interfaces.DocumentReader
imageResolver *docparser.ImageResolver
modelService interfaces.ModelService // used to obtain the ASR model
}
// NewAttachmentProcessor creates an AttachmentProcessor with the given dependencies.
func NewAttachmentProcessor(
fileService interfaces.FileService,
documentReader interfaces.DocumentReader,
imageResolver *docparser.ImageResolver,
modelService interfaces.ModelService,
) *AttachmentProcessor {
return &AttachmentProcessor{
fileService: fileService,
documentReader: documentReader,
imageResolver: imageResolver,
modelService: modelService,
}
}
// ProcessAttachment validates, saves, and extracts content from a single uploaded file.
// Content extraction is attempted for all supported types; errors are non-fatal (logged as warnings).
func (p *AttachmentProcessor) ProcessAttachment(
ctx context.Context,
data []byte,
fileName string,
fileSize int64,
tenantID uint64,
asrModelID string, // optional; enables audio transcription when set
) (*types.MessageAttachment, error) {
logger.Infof(ctx, "processing attachment: fileName=%s, fileSize=%d", secutils.SanitizeForLog(fileName), fileSize)
// Validate filename (injection / path-traversal checks)
safeFileName, isValid := secutils.ValidateInput(fileName)
if !isValid {
return nil, fmt.Errorf("invalid characters in file name")
}
baseName, err := secutils.SafeFileName(safeFileName)
if err != nil {
return nil, fmt.Errorf("unsafe file name: %w", err)
}
ext := strings.ToLower(filepath.Ext(baseName))
if ext == "" {
ext = ".txt"
}
if !isValidFileType(baseName) {
return nil, fmt.Errorf("unsupported file type: %s", ext)
}
uniqueFileName := fmt.Sprintf("attachment_%s%s", uuid.New().String()[:12], ext)
storageURL, err := p.fileService.SaveBytes(ctx, data, tenantID, uniqueFileName, false)
if err != nil {
return nil, fmt.Errorf("failed to save attachment: %w", err)
}
attachment := &types.MessageAttachment{
URL: storageURL,
FileName: baseName,
FileType: ext,
FileSize: fileSize,
}
// Extract text content based on file type; errors are non-fatal.
if p.isTextFile(ext) {
if err := p.processTextFile(ctx, data, attachment); err != nil {
logger.Warnf(ctx, "text file processing failed: %v", err)
}
} else if docparser.IsAudioFormat(ext) {
if err := p.processAudioFile(ctx, data, baseName, attachment, asrModelID); err != nil {
logger.Warnf(ctx, "audio transcription failed: %v, keeping placeholder", err)
}
} else if docparser.IsSimpleFormat(ext) {
if err := p.processWithDocParser(ctx, data, baseName, ext, attachment, tenantID); err != nil {
logger.Warnf(ctx, "SimpleFormatReader failed: %v", err)
}
} else {
if err := p.processWithDocumentReader(ctx, data, baseName, ext, attachment, tenantID); err != nil {
logger.Warnf(ctx, "DocumentReader failed: %v, keeping metadata only", err)
}
}
logger.Infof(ctx, "attachment processed: fileName=%s, truncated=%v, contentLen=%d",
secutils.SanitizeForLog(baseName), attachment.IsTruncated, len(attachment.Content))
return attachment, nil
}
// isTextFile reports whether ext is a plain-text extension handled line-by-line.
func (p *AttachmentProcessor) isTextFile(ext string) bool {
return strings.Contains(textFileExtensions, ext)
}
// processTextFile reads plain-text content line by line, truncating at maxTextFileLines.
func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) error {
scanner := bufio.NewScanner(bytes.NewReader(data))
var lines []string
lineCount := 0
for scanner.Scan() {
lineCount++
if lineCount <= maxTextFileLines {
lines = append(lines, scanner.Text())
}
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("failed to read file content: %w", err)
}
attachment.LineCount = lineCount
attachment.Content = strings.Join(lines, "\n")
if lineCount > maxTextFileLines {
attachment.IsTruncated = true
logger.Infof(ctx, "text file truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
}
return nil
}
// processWithDocParser extracts content via SimpleFormatReader (md, csv, json, images, etc.).
func (p *AttachmentProcessor) processWithDocParser(
ctx context.Context,
data []byte,
fileName string,
fileType string,
attachment *types.MessageAttachment,
tenantID uint64,
) error {
reader := &docparser.SimpleFormatReader{}
result, err := reader.Read(ctx, &types.ReadRequest{
FileContent: data,
FileName: fileName,
FileType: fileType,
})
if err != nil {
return fmt.Errorf("SimpleFormatReader failed: %w", err)
}
// Resolve embedded image refs to storage URLs.
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
if err != nil {
logger.Warnf(ctx, "image resolution failed: %v", err)
} else {
result.MarkdownContent = updatedMarkdown
}
}
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
return nil
}
// processAudioFile transcribes audio via ASR. Falls back to a placeholder when no ASR model is configured.
func (p *AttachmentProcessor) processAudioFile(
ctx context.Context,
data []byte,
fileName string,
attachment *types.MessageAttachment,
asrModelID string,
) error {
if asrModelID == "" || p.modelService == nil {
attachment.Content = fmt.Sprintf("[audio file: %s]", fileName)
logger.Infof(ctx, "no ASR model configured, keeping audio placeholder")
return nil
}
asrInstance, err := p.modelService.GetASRModel(ctx, asrModelID)
if err != nil {
return fmt.Errorf("failed to get ASR model: %w", err)
}
logger.Infof(ctx, "starting audio transcription: fileName=%s, size=%d", fileName, len(data))
transcript, err := asrInstance.Transcribe(ctx, data, fileName)
if err != nil {
return fmt.Errorf("audio transcription failed: %w", err)
}
p.applyLineTruncation(ctx, transcript, attachment)
logger.Infof(ctx, "audio transcription done: textLen=%d", len(transcript))
return nil
}
// processWithDocumentReader extracts content from complex formats (pdf, docx, xlsx, etc.).
func (p *AttachmentProcessor) processWithDocumentReader(
ctx context.Context,
data []byte,
fileName string,
fileType string,
attachment *types.MessageAttachment,
tenantID uint64,
) error {
if p.documentReader == nil {
return fmt.Errorf("DocumentReader not configured")
}
result, err := p.documentReader.Read(ctx, &types.ReadRequest{
FileContent: data,
FileName: fileName,
FileType: fileType,
})
if err != nil {
return fmt.Errorf("DocumentReader failed: %w", err)
}
// Resolve embedded image refs to storage URLs.
if len(result.ImageRefs) > 0 && p.imageResolver != nil {
updatedMarkdown, _, err := p.imageResolver.ResolveAndStore(ctx, result, p.fileService, tenantID)
if err != nil {
logger.Warnf(ctx, "image resolution failed: %v", err)
} else {
result.MarkdownContent = updatedMarkdown
}
}
p.applyLineTruncation(ctx, result.MarkdownContent, attachment)
return nil
}
// applyLineTruncation stores content into attachment, truncating at maxTextFileLines if needed.
func (p *AttachmentProcessor) applyLineTruncation(ctx context.Context, content string, attachment *types.MessageAttachment) {
lines := strings.Split(content, "\n")
lineCount := len(lines)
attachment.LineCount = lineCount
if lineCount > maxTextFileLines {
attachment.Content = strings.Join(lines[:maxTextFileLines], "\n")
attachment.IsTruncated = true
logger.Infof(ctx, "content truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
} else {
attachment.Content = content
}
}
// isValidFileType reports whether fileName has a supported extension.
// Kept in sync with the frontend SUPPORTED_TYPES list in AttachmentUpload.vue.
func isValidFileType(fileName string) bool {
ext := strings.ToLower(filepath.Ext(fileName))
if ext == "" {
return false
}
ext = strings.TrimPrefix(ext, ".")
supportedTypes := []string{
// documents
"docx", "doc", "pdf", "ppt", "pptx",
// spreadsheets
"xlsx", "xls",
// text / markup
"md", "markdown", "txt", "csv", "json", "xml", "yaml", "yml", "log", "html",
// images
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp",
// audio
"mp3", "wav", "m4a", "flac", "ogg", "aac",
}
for _, t := range supportedTypes {
if ext == t {
return true
}
}
return false
}
// DecodeBase64Attachment decodes a base64 attachment payload, stripping any data URI prefix.
// Tries Std, URL, RawStd, and RawURL encodings in order.
func DecodeBase64Attachment(data string) ([]byte, error) {
// Strip data URI prefix (e.g. "data:application/pdf;base64,")
if idx := strings.Index(data, ","); idx != -1 {
data = data[idx+1:]
}
data = strings.TrimSpace(data)
for _, enc := range []struct{ e *base64.Encoding }{
{base64.StdEncoding},
{base64.URLEncoding},
{base64.RawStdEncoding},
{base64.RawURLEncoding},
} {
if decoded, err := enc.e.DecodeString(data); err == nil {
return decoded, nil
}
}
return nil, fmt.Errorf("base64 decode failed: unrecognised encoding")
}
+10
View File
@@ -5,6 +5,7 @@ import (
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -24,6 +25,7 @@ type Handler struct {
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
fileService interfaces.FileService // Service for file storage (image uploads)
modelService interfaces.ModelService // Service for model management (VLM access)
attachmentProcessor *AttachmentProcessor // Processor for file attachments
}
// NewHandler creates a new instance of Handler with all necessary dependencies
@@ -38,6 +40,8 @@ func NewHandler(
agentShareService interfaces.AgentShareService,
fileService interfaces.FileService,
modelService interfaces.ModelService,
documentReader interfaces.DocumentReader,
imageResolver *docparser.ImageResolver,
) *Handler {
return &Handler{
sessionService: sessionService,
@@ -50,6 +54,12 @@ func NewHandler(
agentShareService: agentShareService,
fileService: fileService,
modelService: modelService,
attachmentProcessor: NewAttachmentProcessor(
fileService,
documentReader,
imageResolver,
modelService,
),
}
}
+2 -1
View File
@@ -166,7 +166,7 @@ func createAgentQueryEvent(sessionID, assistantMessageID string) interfaces.Stre
}
// createUserMessage creates a user message and returns the created message.
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, channel string) (*types.Message, error) {
func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, requestID string, mentionedItems types.MentionedItems, images types.MessageImages, attachments types.MessageAttachments, channel string) (*types.Message, error) {
return h.messageService.CreateMessage(ctx, &types.Message{
SessionID: sessionID,
Role: "user",
@@ -176,6 +176,7 @@ func (h *Handler) createUserMessage(ctx context.Context, sessionID, query, reque
IsCompleted: true,
MentionedItems: mentionedItems,
Images: images,
Attachments: attachments,
Channel: channel,
})
}
+66 -1
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"runtime"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/errors"
@@ -37,6 +38,7 @@ type qaRequestContext struct {
images []ImageAttachment // Uploaded images with analysis text
userMessageID string // Created user message ID (populated after createUserMessage)
channel string // Source channel: "web", "api", "im", etc.
attachments types.MessageAttachments // Processed file attachments
}
// buildQARequest converts the qaRequestContext into a types.QARequest for service invocation.
@@ -55,6 +57,7 @@ func (rc *qaRequestContext) buildQARequest() *types.QARequest {
UserMessageID: rc.userMessageID,
WebSearchEnabled: rc.webSearchEnabled,
EnableMemory: rc.enableMemory,
Attachments: rc.attachments,
}
}
@@ -135,6 +138,67 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
// - Normal pure-chat mode: runs in the async goroutine with progress events
}
// Process file attachments: decode and save to storage, extract content
var processedAttachments types.MessageAttachments
if len(request.AttachmentUploads) > 0 {
logger.Infof(ctx, "[%s] processing %d attachment(s)", logPrefix, len(request.AttachmentUploads))
maxSize := secutils.GetMaxFileSize()
for i, upload := range request.AttachmentUploads {
if upload.FileSize > maxSize {
return nil, nil, errors.NewBadRequestError(
fmt.Sprintf("attachment %d exceeds size limit of %dMB", i+1, secutils.GetMaxFileSizeMB()))
}
}
tenantID := c.GetUint64(types.TenantIDContextKey.String())
// Use ASR only when the agent has audio upload enabled.
asrModelID := ""
if customAgent != nil && customAgent.Config.AudioUploadEnabled && customAgent.Config.ASRModelID != "" {
asrModelID = customAgent.Config.ASRModelID
}
// Process all attachments concurrently.
processedAttachments = make(types.MessageAttachments, len(request.AttachmentUploads))
var wg sync.WaitGroup
errChan := make(chan error, len(request.AttachmentUploads))
for i, upload := range request.AttachmentUploads {
wg.Add(1)
go func(idx int, att AttachmentUpload) {
defer wg.Done()
data, err := DecodeBase64Attachment(att.Data)
if err != nil {
errChan <- fmt.Errorf("attachment %d decode failed: %w", idx+1, err)
return
}
processed, err := h.attachmentProcessor.ProcessAttachment(
ctx, data, att.FileName, att.FileSize, tenantID, asrModelID,
)
if err != nil {
errChan <- fmt.Errorf("attachment %d processing failed: %w", idx+1, err)
return
}
processedAttachments[idx] = *processed
}(i, upload)
}
wg.Wait()
close(errChan)
if len(errChan) > 0 {
err := <-errChan
logger.Errorf(ctx, "[%s] attachment processing failed: %v", logPrefix, err)
return nil, nil, errors.NewBadRequestError(fmt.Sprintf("attachment processing failed: %v", err))
}
logger.Infof(ctx, "[%s] all attachments processed", logPrefix)
}
// Build request context
reqCtx := &qaRequestContext{
ctx: ctx,
@@ -160,6 +224,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo
effectiveTenantID: effectiveTenantID,
images: request.Images,
channel: request.Channel,
attachments: processedAttachments,
}
return reqCtx, &request, nil
@@ -478,7 +543,7 @@ func (h *Handler) executeQA(reqCtx *qaRequestContext, mode qaMode, generateTitle
}
// Create user message
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.channel)
userMsg, err := h.createUserMessage(ctx, sessionID, reqCtx.query, reqCtx.requestID, reqCtx.mentionedItems, convertImageAttachments(reqCtx.images), reqCtx.attachments, reqCtx.channel)
if err != nil {
reqCtx.c.Error(errors.NewInternalServerError(err.Error()))
return
+8
View File
@@ -49,9 +49,17 @@ type CreateKnowledgeQARequest struct {
DisableTitle bool `json:"disable_title"` // Whether to disable auto title generation
EnableMemory bool `json:"enable_memory"` // Whether memory feature is enabled for this request
Images []ImageAttachment `json:"images"` // Attached images for multimodal chat
AttachmentUploads []AttachmentUpload `json:"attachment_uploads,omitempty"` // Attached files (documents, audio, etc.)
Channel string `json:"channel"` // Source channel: "web", "api", "im", etc.
}
// AttachmentUpload represents a file attachment upload from the client
type AttachmentUpload struct {
Data string `json:"data"` // Base64-encoded file content
FileName string `json:"file_name"` // Original filename
FileSize int64 `json:"file_size"` // File size in bytes
}
// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization
type SearchKnowledgeRequest struct {
Query string `json:"query" binding:"required"` // Query text to search for