feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
This commit is contained in:
@@ -3,6 +3,7 @@ package chat
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -55,6 +56,7 @@ func readLocalStorageBytes(storagePath string) []byte {
|
||||
localPath := filepath.Join(baseDir, filepath.FromSlash(relPath))
|
||||
data, err := os.ReadFile(localPath)
|
||||
if err != nil {
|
||||
log.Printf("[image-resolve] failed to read local file %s: %v", localPath, err)
|
||||
return nil
|
||||
}
|
||||
return data
|
||||
|
||||
@@ -112,10 +112,7 @@ func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []s
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
|
||||
// Log the curl equivalent for debugging
|
||||
logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'",
|
||||
r.baseURL, r.apiKey, string(jsonData),
|
||||
)
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
@@ -80,11 +80,7 @@ func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []str
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
|
||||
// Log the curl equivalent for debugging (API key masked for security)
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
|
||||
r.baseURL, string(jsonData),
|
||||
)
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package rerank
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
maxLogDocuments = 3
|
||||
maxLogTextRunes = 120
|
||||
)
|
||||
|
||||
func buildRerankRequestDebug(model, endpoint, query string, documents []string) string {
|
||||
previews := make([]string, 0, maxLogDocuments)
|
||||
for i, doc := range documents {
|
||||
if i >= maxLogDocuments {
|
||||
break
|
||||
}
|
||||
previews = append(previews, compactForLog(doc, maxLogTextRunes))
|
||||
}
|
||||
|
||||
previewJSON, _ := json.Marshal(previews)
|
||||
return fmt.Sprintf(
|
||||
"rerank request endpoint=%s model=%s query_preview=%q query_runes=%d documents=%d preview_docs=%s",
|
||||
endpoint,
|
||||
model,
|
||||
compactForLog(query, maxLogTextRunes),
|
||||
utf8.RuneCountInString(query),
|
||||
len(documents),
|
||||
string(previewJSON),
|
||||
)
|
||||
}
|
||||
|
||||
func compactForLog(text string, maxRunes int) string {
|
||||
normalized := strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
|
||||
if utf8.RuneCountInString(normalized) <= maxRunes {
|
||||
return normalized
|
||||
}
|
||||
return string([]rune(normalized)[:maxRunes]) + "...(truncated)"
|
||||
}
|
||||
@@ -82,11 +82,7 @@ func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []s
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
|
||||
// Log the curl equivalent for debugging (API key masked for security)
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
|
||||
r.baseURL, string(jsonData),
|
||||
)
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
@@ -93,10 +93,7 @@ func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []st
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
|
||||
// Log the curl equivalent for debugging
|
||||
logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'",
|
||||
r.baseURL, r.apiKey, string(jsonData),
|
||||
)
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user