feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
This commit is contained in:
@@ -42,57 +42,12 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Multimodal feature:仅选择多模态模型,存储引擎在「存储引擎」页配置 -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.label') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.description') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-switch
|
||||
v-model="localMultimodal.enabled"
|
||||
@change="handleMultimodalToggle"
|
||||
size="medium"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 多模态开启时仅配置 VLLM 模型 -->
|
||||
<div v-if="localMultimodal.enabled" class="subsection">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.vllmLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.vllmDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<ModelSelector
|
||||
ref="vllmSelectorRef"
|
||||
model-type="VLLM"
|
||||
:selected-model-id="localMultimodal.vllmModelId"
|
||||
:all-models="allModels"
|
||||
@update:selected-model-id="handleVLLMChange"
|
||||
@add-model="handleAddModel('vllm')"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.vllmPlaceholder')"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch } from 'vue'
|
||||
import ModelSelector from '@/components/ModelSelector.vue'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
|
||||
const uiStore = useUIStore()
|
||||
|
||||
interface MultimodalConfig {
|
||||
enabled: boolean
|
||||
vllmModelId?: string
|
||||
}
|
||||
|
||||
interface QuestionGenerationConfig {
|
||||
enabled: boolean
|
||||
@@ -100,7 +55,6 @@ interface QuestionGenerationConfig {
|
||||
}
|
||||
|
||||
interface Props {
|
||||
multimodal: MultimodalConfig
|
||||
questionGeneration?: QuestionGenerationConfig
|
||||
allModels?: any[]
|
||||
}
|
||||
@@ -108,21 +62,13 @@ interface Props {
|
||||
const props = defineProps<Props>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:multimodal': [value: MultimodalConfig]
|
||||
'update:questionGeneration': [value: QuestionGenerationConfig]
|
||||
}>()
|
||||
|
||||
const localMultimodal = ref<MultimodalConfig>({ ...props.multimodal })
|
||||
const localQuestionGeneration = ref<QuestionGenerationConfig>(
|
||||
props.questionGeneration || { enabled: false, questionCount: 3 }
|
||||
)
|
||||
|
||||
const vllmSelectorRef = ref()
|
||||
|
||||
watch(() => props.multimodal, (newVal) => {
|
||||
localMultimodal.value = { ...newVal }
|
||||
}, { deep: true })
|
||||
|
||||
watch(() => props.questionGeneration, (newVal) => {
|
||||
if (newVal) {
|
||||
localQuestionGeneration.value = { ...newVal }
|
||||
@@ -139,22 +85,6 @@ const handleQuestionGenerationToggle = () => {
|
||||
const handleQuestionGenerationChange = () => {
|
||||
emit('update:questionGeneration', localQuestionGeneration.value)
|
||||
}
|
||||
|
||||
const handleMultimodalToggle = () => {
|
||||
if (!localMultimodal.value.enabled) {
|
||||
localMultimodal.value.vllmModelId = ''
|
||||
}
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
const handleVLLMChange = (modelId: string) => {
|
||||
localMultimodal.value.vllmModelId = modelId
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
const handleAddModel = (subSection: string) => {
|
||||
uiStore.openSettings('models', subSection)
|
||||
}
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
@@ -250,4 +180,3 @@ const handleAddModel = (subSection: string) => {
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user