feat: implement image upload and multimodal support

- Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A.
- Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes.
- Updated the backend to process images alongside text queries, including support for image analysis.
- Introduced new UI components for image previews and management, improving user interaction with uploaded content.
- Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages.

These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses.
This commit is contained in:
wizardchen
2026-03-11 17:17:32 +08:00
parent 8de1cfb6b5
commit 991bd29a14
42 changed files with 1554 additions and 226 deletions
+214 -2
View File
@@ -29,6 +29,48 @@ const { t } = useI18n();
let query = ref("");
const showKbSelector = ref(false);
// Image upload state
const uploadedImages = ref<Array<{ file: File; preview: string }>>([]);
const imageInputRef = ref<HTMLInputElement>();
const imageUploading = ref(false);
const handleImageSelect = (event: Event) => {
const input = event.target as HTMLInputElement;
if (!input.files) return;
addImageFiles(Array.from(input.files));
input.value = '';
};
const addImageFiles = (files: File[]) => {
if (!isImageUploadEnabledByAgent.value) return;
const allowed = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
const maxSize = 10 * 1024 * 1024;
for (const file of files) {
if (uploadedImages.value.length >= 5) {
MessagePlugin.warning(t('chat.imageTooMany'));
break;
}
if (!allowed.includes(file.type)) {
MessagePlugin.warning(t('chat.imageTypeSizeError'));
continue;
}
if (file.size > maxSize) {
MessagePlugin.warning(t('chat.imageTypeSizeError'));
continue;
}
uploadedImages.value.push({ file, preview: URL.createObjectURL(file) });
}
};
const removeImage = (index: number) => {
const removed = uploadedImages.value.splice(index, 1);
if (removed.length > 0) URL.revokeObjectURL(removed[0].preview);
};
const triggerImageUpload = () => {
imageInputRef.value?.click();
};
const atButtonRef = ref<HTMLElement>();
const showAgentModeSelector = ref(false);
const agentModeButtonRef = ref<HTMLElement>();
@@ -115,6 +157,11 @@ watch([selectedAgentId, agentKnowledgeBases, agentKBSelectionMode], ([newAgentId
if (showMention.value) {
loadMentionItems(mentionQuery.value, true);
}
// Clear images when switching to an agent that doesn't support image upload
if (!isImageUploadEnabledByAgent.value && uploadedImages.value.length > 0) {
uploadedImages.value.forEach(img => URL.revokeObjectURL(img.preview));
uploadedImages.value = [];
}
}
}, { immediate: true });
@@ -177,6 +224,12 @@ const agentSupportedFileTypes = computed(() => {
return currentAgentConfig.value?.supported_file_types || [];
});
// 智能体是否启用了图片上传(多模态)
const isImageUploadEnabledByAgent = computed(() => {
if (!hasAgentConfig.value) return false;
return currentAgentConfig.value?.image_upload_enabled === true;
});
// 模型选择是否被智能体锁定 - 已移除锁定逻辑,允许用户自由切换模型
const isModelLockedByAgent = computed(() => {
return false;
@@ -1321,7 +1374,11 @@ const createSession = async (val: string) => {
type: item.type,
kb_type: item.type === 'kb' ? (item.kbType || 'document') : undefined
}));
emit('send-msg', val, selectedModelId.value, mentionedItems);
const imageFiles = uploadedImages.value.map(img => img.file);
emit('send-msg', val, selectedModelId.value, mentionedItems, imageFiles);
// Clean up image previews
uploadedImages.value.forEach(img => URL.revokeObjectURL(img.preview));
uploadedImages.value = [];
clearvalue();
}
@@ -1554,6 +1611,36 @@ const onKeydown = (val: string, event: { e: { preventDefault(): unknown; keyCode
}
}
const onPaste = (e: ClipboardEvent) => {
const items = e.clipboardData?.items;
if (!items) return;
const imageFiles: File[] = [];
for (const item of items) {
if (item.type.startsWith('image/')) {
const file = item.getAsFile();
if (file) imageFiles.push(file);
}
}
if (imageFiles.length > 0 && isImageUploadEnabledByAgent.value) {
e.preventDefault();
addImageFiles(imageFiles);
}
};
const onDrop = (e: DragEvent) => {
e.preventDefault();
const files = e.dataTransfer?.files;
if (!files) return;
const imageFiles = Array.from(files).filter(f => f.type.startsWith('image/'));
if (imageFiles.length > 0 && isImageUploadEnabledByAgent.value) {
addImageFiles(imageFiles);
}
};
const onDragOver = (e: DragEvent) => {
e.preventDefault();
};
const handleGoToWebSearchSettings = () => {
uiStore.openSettings('websearch');
if (route.path !== '/platform/settings') {
@@ -1732,9 +1819,25 @@ onBeforeRouteUpdate((to, from, next) => {
</script>
<template>
<div class="answers-input">
<div class="answers-input" @drop="onDrop" @dragover="onDragOver">
<!-- Hidden file input for image upload -->
<input
ref="imageInputRef"
type="file"
accept="image/jpeg,image/png,image/gif,image/webp"
multiple
style="display:none"
@change="handleImageSelect"
/>
<!-- 富文本输入框容器 -->
<div class="rich-input-container">
<!-- 图片预览区域 -->
<div v-if="uploadedImages.length > 0" class="image-preview-bar">
<div v-for="(img, idx) in uploadedImages" :key="idx" class="image-preview-item">
<img :src="img.preview" class="image-preview-thumb" />
<span class="image-preview-remove" @click="removeImage(idx)">×</span>
</div>
</div>
<!-- 选中的知识库和文件标签(显示在输入框内顶部) -->
<div v-if="allSelectedItems.length > 0" class="selected-tags-inline">
<span
@@ -1771,6 +1874,7 @@ onBeforeRouteUpdate((to, from, next) => {
@input="onInput"
@compositionstart="onCompositionStart"
@compositionend="onCompositionEnd"
@paste="onPaste"
/>
</div>
@@ -1867,6 +1971,32 @@ onBeforeRouteUpdate((to, from, next) => {
</div>
</t-tooltip>
<!-- 图片上传按钮 -->
<t-tooltip placement="top" theme="light" :popupProps="{ overlayClassName: 'input-field-tooltip' }">
<template #content>
<div v-if="!isImageUploadEnabledByAgent" class="tooltip-with-link">
<span>{{ $t('input.imageUploadDisabledByAgent') }}</span>
<a href="#" @click.prevent="handleGoToAgentSettings('model')">{{ $t('input.goToAgentSettings') }}</a>
</div>
<span v-else>{{ $t('chat.imageUploadTooltip') }}</span>
</template>
<div
class="control-btn image-upload-btn"
:class="{
'active': uploadedImages.length > 0,
'disabled': !isImageUploadEnabledByAgent
}"
@click.stop="isImageUploadEnabledByAgent && triggerImageUpload()"
>
<svg width="18" height="18" viewBox="0 0 1024 1024" fill="currentColor" class="control-icon">
<path d="M896 128H128c-35.3 0-64 28.7-64 64v640c0 35.3 28.7 64 64 64h768c35.3 0 64-28.7 64-64V192c0-35.3-28.7-64-64-64zM128 832V192h768l0.1 640H128z"/>
<path d="M352 448a96 96 0 1 0 0-192 96 96 0 0 0 0 192z"/>
<path d="M128 768l224-288 160 160 192-256L896 640v128H128z"/>
</svg>
<span v-if="uploadedImages.length > 0" class="image-count">{{ uploadedImages.length }}</span>
</div>
</t-tooltip>
<!-- @ 知识库/文件选择按钮 -->
<t-tooltip placement="top" theme="light" :popupProps="{ overlayClassName: 'input-field-tooltip' }">
<template #content>
@@ -2379,6 +2509,88 @@ const getImgSrc = (url: string) => {
color: var(--td-brand-color);
}
/* Image upload */
.image-upload-btn {
width: 28px;
height: 28px;
padding: 0;
min-width: auto;
display: flex;
align-items: center;
justify-content: center;
position: relative;
color: var(--td-text-color-secondary, #666);
&:hover {
background: var(--td-bg-color-secondarycontainer-hover, #f0f0f0);
color: var(--td-text-color-primary, #333);
}
&.active {
background: rgba(16, 185, 129, 0.1);
color: #07C05F;
}
.image-count {
position: absolute;
top: -2px;
right: -2px;
background: #07C05F;
color: #fff;
font-size: 10px;
width: 14px;
height: 14px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
line-height: 1;
}
}
.image-preview-bar {
display: flex;
gap: 8px;
padding: 8px 12px 4px;
flex-wrap: wrap;
}
.image-preview-item {
position: relative;
width: 60px;
height: 60px;
border-radius: 8px;
overflow: hidden;
border: 1px solid var(--td-border-level-1-color, #e7e7e7);
.image-preview-thumb {
width: 100%;
height: 100%;
object-fit: cover;
}
.image-preview-remove {
position: absolute;
top: 2px;
right: 2px;
width: 16px;
height: 16px;
background: rgba(0, 0, 0, 0.5);
color: #fff;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 12px;
cursor: pointer;
line-height: 1;
&:hover {
background: rgba(0, 0, 0, 0.7);
}
}
}
.websearch-btn {
width: 28px;
height: 28px;
+14 -2
View File
@@ -230,6 +230,15 @@
</p>
</div>
<!-- Chat/VLLM: supports vision toggle -->
<div v-if="modelType === 'chat' || modelType === 'vllm'" class="form-item">
<label class="form-label">{{ $t('model.editor.supportsVisionLabel') }}</label>
<div style="display: flex; align-items: center; gap: 8px;">
<t-switch v-model="formData.supportsVision" />
<span class="form-desc">{{ $t('model.editor.supportsVisionDesc') }}</span>
</div>
</div>
</t-form>
</div>
@@ -266,6 +275,7 @@ interface ModelFormData {
dimension?: number
interfaceType?: 'ollama' | 'openai'
isDefault: boolean
supportsVision?: boolean
}
interface Props {
@@ -455,7 +465,8 @@ const formData = ref<ModelFormData>({
apiKey: '',
dimension: undefined,
interfaceType: 'ollama',
isDefault: false
isDefault: false,
supportsVision: false
})
const rules = computed(() => ({
@@ -597,7 +608,8 @@ const resetForm = () => {
apiKey: '',
dimension: undefined, // 默认不填,让用户手动输入或通过检测按钮获取
interfaceType: undefined,
isDefault: false
isDefault: false,
supportsVision: false
}
modelChecked.value = false
modelAvailable.value = false