beca2b89a3
Previously the Langfuse integration only traced in-process HTTP requests (chat / search / eval), so file uploads and every downstream asynq task (document parse, chunk embedding, OCR/VLM, summary / question gen, wiki ingest, datasource sync, etc.) produced either disconnected shallow traces or no observation at all. This change threads one trace end-to-end: - tracer: add SPAN observation type and StartSpan; add ResumeTrace so a worker can attach to an upstream trace without emitting a duplicate trace-create; StartGeneration now auto-picks parentObservationId from ctx so nested trace -> span -> generation trees render correctly. - types.TracingContext + LangfuseTracingCarrier: embed on all 17 asynq payloads so trace_id / parent_obs_id / user_id / session_id serialise into every job. - langfuse.InjectTracing: injected at 28 enqueue sites before json.Marshal so the HTTP-layer trace survives the Redis hop. - langfuse.AsynqMiddleware: mux.Use hook that peeks the payload, either resumes the upstream trace or opens a standalone asynq.<type> trace for scheduled jobs, and wraps the handler in a SPAN with task metadata (id / queue / retry / payload_bytes) plus ERROR level on failure. - GinMiddleware.shouldTrace: whitelist ingestion / knowledge-mutation / FAQ / wiki / datasource endpoints so the root trace actually starts. - Tests: tracer_test.go covers span nesting, error status, and ResumeTrace no-trace-create guarantee; asynq_test.go covers InjectTracing round-trip, middleware resume path, and standalone trace fallback. - Docs: docs/Langfuse\u96c6\u6210.md now lists the covered task types and documents the cross-process propagation model. No behavioural change when Langfuse is disabled (all new code paths are no-ops and carriers serialise to empty strings with omitempty).
437 lines
16 KiB
Go
437 lines
16 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
|
|
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
|
"github.com/Tencent/WeKnora/internal/models/vlm"
|
|
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
"github.com/google/uuid"
|
|
"github.com/hibiken/asynq"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
const (
|
|
vlmOCRPrompt = "<system_prompt>\n" +
|
|
"You are an OCR assistant. Your task is to extract all body text content from this document image and output in pure Markdown format.\n" +
|
|
"</system_prompt>\n\n" +
|
|
"<instructions>\n" +
|
|
"1. Ignore headers and footers.\n" +
|
|
"2. Use Markdown table syntax for tables.\n" +
|
|
"3. Use LaTeX format for formulas (wrapped with $ or $$).\n" +
|
|
"4. Organize content in the original reading order.\n" +
|
|
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
|
|
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
|
|
"</instructions>"
|
|
vlmOCRScannedPDFPrompt = "<system_prompt>\n" +
|
|
"You are an OCR and document layout extraction assistant. The input image is a page from a scanned PDF document.\n" +
|
|
"Your task is to carefully extract all text and layout structure from the image, and output the result in pure Markdown format.\n" +
|
|
"</system_prompt>\n\n" +
|
|
"<instructions>\n" +
|
|
"1. Ignore headers, footers, and page numbers.\n" +
|
|
"2. Preserve the original document's paragraph and hierarchical structure as much as possible.\n" +
|
|
"3. If there are tables, use Markdown table syntax to represent them.\n" +
|
|
"4. If there are mathematical formulas, use LaTeX format wrapped in $ or $$.\n" +
|
|
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
|
|
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
|
|
"</instructions>"
|
|
vlmCaptionPrompt = "Provide a brief and concise description of the main content of the image in Chinese"
|
|
)
|
|
|
|
// ImageMultimodalService handles image:multimodal asynq tasks.
|
|
// It reads images from storage (via FileService for provider:// URLs),
|
|
// performs OCR and VLM caption, and creates child chunks.
|
|
type ImageMultimodalService struct {
|
|
chunkService interfaces.ChunkService
|
|
modelService interfaces.ModelService
|
|
kbService interfaces.KnowledgeBaseService
|
|
knowledgeRepo interfaces.KnowledgeRepository
|
|
tenantRepo interfaces.TenantRepository
|
|
retrieveEngine interfaces.RetrieveEngineRegistry
|
|
ollamaService *ollama.OllamaService
|
|
taskEnqueuer interfaces.TaskEnqueuer
|
|
redisClient *redis.Client
|
|
}
|
|
|
|
func NewImageMultimodalService(
|
|
chunkService interfaces.ChunkService,
|
|
modelService interfaces.ModelService,
|
|
kbService interfaces.KnowledgeBaseService,
|
|
knowledgeRepo interfaces.KnowledgeRepository,
|
|
tenantRepo interfaces.TenantRepository,
|
|
retrieveEngine interfaces.RetrieveEngineRegistry,
|
|
ollamaService *ollama.OllamaService,
|
|
taskEnqueuer interfaces.TaskEnqueuer,
|
|
redisClient *redis.Client,
|
|
) interfaces.TaskHandler {
|
|
return &ImageMultimodalService{
|
|
chunkService: chunkService,
|
|
modelService: modelService,
|
|
kbService: kbService,
|
|
knowledgeRepo: knowledgeRepo,
|
|
tenantRepo: tenantRepo,
|
|
retrieveEngine: retrieveEngine,
|
|
ollamaService: ollamaService,
|
|
taskEnqueuer: taskEnqueuer,
|
|
redisClient: redisClient,
|
|
}
|
|
}
|
|
|
|
// Handle implements asynq handler for TypeImageMultimodal.
|
|
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
|
|
var payload types.ImageMultimodalPayload
|
|
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
|
|
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
|
|
}
|
|
|
|
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, ocr=%v, caption=%v",
|
|
payload.ChunkID, payload.ImageURL, payload.EnableOCR, payload.EnableCaption)
|
|
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
|
if payload.Language != "" {
|
|
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
|
}
|
|
|
|
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
|
if err != nil {
|
|
return fmt.Errorf("resolve VLM: %w", err)
|
|
}
|
|
|
|
// Read image bytes: try provider:// via tenant-resolved FileService,
|
|
// then legacy local path, then HTTP URL.
|
|
var imgBytes []byte
|
|
if types.ParseProviderScheme(payload.ImageURL) != "" {
|
|
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
|
|
if fileSvc == nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Resolve tenant file service failed, fallback to URL/local: tenant=%d kb=%s",
|
|
payload.TenantID, payload.KnowledgeBaseID)
|
|
} else {
|
|
// provider:// scheme — read via FileService
|
|
reader, getErr := fileSvc.GetFile(ctx, payload.ImageURL)
|
|
if getErr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] FileService.GetFile(%s) failed: %v", payload.ImageURL, getErr)
|
|
} else {
|
|
imgBytes, err = io.ReadAll(reader)
|
|
reader.Close()
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Read provider file %s failed: %v", payload.ImageURL, err)
|
|
imgBytes = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if imgBytes == nil && payload.ImageLocalPath != "" {
|
|
imgBytes, err = os.ReadFile(payload.ImageLocalPath)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying URL", payload.ImageLocalPath, err)
|
|
imgBytes = nil
|
|
}
|
|
}
|
|
if imgBytes == nil {
|
|
imgBytes, err = downloadImageFromURL(payload.ImageURL)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
|
|
return fmt.Errorf("read image from URL %s failed: %w", payload.ImageURL, err)
|
|
}
|
|
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
|
|
}
|
|
|
|
imageInfo := types.ImageInfo{
|
|
URL: payload.ImageURL,
|
|
OriginalURL: payload.ImageURL,
|
|
}
|
|
|
|
if payload.EnableOCR {
|
|
prompt := vlmOCRPrompt
|
|
if payload.ImageSourceType == "scanned_pdf" {
|
|
prompt = vlmOCRScannedPDFPrompt
|
|
logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL)
|
|
}
|
|
|
|
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
|
|
if ocrErr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
|
|
} else {
|
|
ocrText = sanitizeOCRText(ocrText)
|
|
if ocrText != "" {
|
|
imageInfo.OCRText = ocrText
|
|
} else {
|
|
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
|
|
}
|
|
}
|
|
}
|
|
|
|
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
|
|
if capErr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
|
} else if caption != "" {
|
|
imageInfo.Caption = caption
|
|
}
|
|
|
|
// Build child chunks for OCR and caption results
|
|
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
|
|
var newChunks []*types.Chunk
|
|
|
|
if imageInfo.OCRText != "" {
|
|
newChunks = append(newChunks, &types.Chunk{
|
|
ID: uuid.New().String(),
|
|
TenantID: payload.TenantID,
|
|
KnowledgeID: payload.KnowledgeID,
|
|
KnowledgeBaseID: payload.KnowledgeBaseID,
|
|
Content: imageInfo.OCRText,
|
|
ChunkType: types.ChunkTypeImageOCR,
|
|
ParentChunkID: payload.ChunkID,
|
|
IsEnabled: true,
|
|
Flags: types.ChunkFlagRecommended,
|
|
ImageInfo: string(imageInfoJSON),
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
})
|
|
}
|
|
|
|
if imageInfo.Caption != "" {
|
|
newChunks = append(newChunks, &types.Chunk{
|
|
ID: uuid.New().String(),
|
|
TenantID: payload.TenantID,
|
|
KnowledgeID: payload.KnowledgeID,
|
|
KnowledgeBaseID: payload.KnowledgeBaseID,
|
|
Content: imageInfo.Caption,
|
|
ChunkType: types.ChunkTypeImageCaption,
|
|
ParentChunkID: payload.ChunkID,
|
|
IsEnabled: true,
|
|
Flags: types.ChunkFlagRecommended,
|
|
ImageInfo: string(imageInfoJSON),
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
})
|
|
}
|
|
|
|
if len(newChunks) == 0 {
|
|
s.checkAndFinalizeAllImages(ctx, payload)
|
|
return nil
|
|
}
|
|
|
|
// Persist chunks
|
|
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
|
|
return fmt.Errorf("create multimodal chunks: %w", err)
|
|
}
|
|
for _, c := range newChunks {
|
|
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
|
|
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
|
|
}
|
|
|
|
// Index chunks so they can be retrieved
|
|
s.indexChunks(ctx, payload, newChunks)
|
|
|
|
// Enqueue question generation for the caption/OCR content if KB has it enabled.
|
|
// During initial processChunks, question generation is skipped for image-type
|
|
// knowledge because the text chunk is just a markdown reference. Now that we
|
|
// have real textual content (caption/OCR), we can generate questions.
|
|
// Note: for documents with multiple images (e.g. PDFs), we also wait until
|
|
// all images are processed before triggering summary/question generation.
|
|
s.checkAndFinalizeAllImages(ctx, payload)
|
|
|
|
return nil
|
|
}
|
|
|
|
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
|
|
// so they can participate in semantic search.
|
|
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
|
|
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
|
if err != nil || kb == nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
|
|
return
|
|
}
|
|
|
|
// Skip vector/keyword indexing when the KB has no embedding-based pipeline enabled
|
|
// (e.g. Wiki-only KBs). Without this check, GetEmbeddingModel would fail because
|
|
// EmbeddingModelID is intentionally empty for such KBs. The multimodal chunks
|
|
// themselves are already persisted in the DB above, so skipping index here is safe.
|
|
if !kb.NeedsEmbeddingModel() {
|
|
logger.Infof(ctx, "[ImageMultimodal] Vector/keyword indexing disabled for KB %s, skipping index for %d multimodal chunks",
|
|
kb.ID, len(chunks))
|
|
// Still mark chunks as indexed so downstream finalization sees a consistent state.
|
|
for _, chunk := range chunks {
|
|
dbChunk, gerr := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
|
|
if gerr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, gerr)
|
|
continue
|
|
}
|
|
dbChunk.Status = int(types.ChunkStatusIndexed)
|
|
if uerr := s.chunkService.UpdateChunk(ctx, dbChunk); uerr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, uerr)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
|
|
return
|
|
}
|
|
|
|
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
|
|
return
|
|
}
|
|
|
|
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
|
|
return
|
|
}
|
|
|
|
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
|
Content: chunk.Content,
|
|
SourceID: chunk.ID,
|
|
SourceType: types.ChunkSourceType,
|
|
ChunkID: chunk.ID,
|
|
KnowledgeID: chunk.KnowledgeID,
|
|
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
|
})
|
|
}
|
|
|
|
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
|
|
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
|
|
return
|
|
}
|
|
|
|
// Mark chunks as indexed.
|
|
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
|
|
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
|
|
for _, chunk := range chunks {
|
|
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
|
|
continue
|
|
}
|
|
dbChunk.Status = int(types.ChunkStatusIndexed)
|
|
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
|
|
}
|
|
}
|
|
|
|
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
|
|
}
|
|
|
|
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
|
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
|
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
|
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
|
}
|
|
if kb == nil {
|
|
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
|
}
|
|
|
|
vlmCfg := kb.VLMConfig
|
|
if !vlmCfg.IsEnabled() {
|
|
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
|
}
|
|
|
|
// New-style: resolve model through ModelService
|
|
if vlmCfg.ModelID != "" {
|
|
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
|
}
|
|
|
|
// Legacy: create VLM from inline config
|
|
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
|
}
|
|
|
|
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
|
|
func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Context, payload types.ImageMultimodalPayload) interfaces.FileService {
|
|
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
|
if err != nil || tenant == nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] GetTenantByID failed: tenant=%d err=%v", payload.TenantID, err)
|
|
return nil
|
|
}
|
|
|
|
provider := types.ParseProviderScheme(payload.ImageURL)
|
|
if provider == "" {
|
|
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
|
if kbErr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
|
|
} else if kb != nil {
|
|
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
|
|
}
|
|
}
|
|
|
|
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
|
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
|
|
if svcErr != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed: tenant=%d provider=%s err=%v", payload.TenantID, provider, svcErr)
|
|
return nil
|
|
}
|
|
return fileSvc
|
|
}
|
|
|
|
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
|
|
func downloadImageFromURL(imageURL string) ([]byte, error) {
|
|
return secutils.DownloadBytes(imageURL)
|
|
}
|
|
|
|
func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, payload types.ImageMultimodalPayload) {
|
|
if s.redisClient == nil {
|
|
s.enqueueKnowledgePostProcessTask(ctx, payload)
|
|
return
|
|
}
|
|
|
|
redisKey := fmt.Sprintf("multimodal:pending:%s", payload.KnowledgeID)
|
|
|
|
pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result()
|
|
if err != nil && err != redis.Nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to decrement pending count for %s: %v", payload.KnowledgeID, err)
|
|
return
|
|
}
|
|
|
|
if pendingCount <= 0 {
|
|
logger.Infof(ctx, "[ImageMultimodal] All images processed for knowledge %s. Finalizing...", payload.KnowledgeID)
|
|
s.redisClient.Del(ctx, redisKey)
|
|
|
|
s.enqueueKnowledgePostProcessTask(ctx, payload)
|
|
}
|
|
}
|
|
|
|
func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Context, payload types.ImageMultimodalPayload) {
|
|
if s.taskEnqueuer == nil {
|
|
return
|
|
}
|
|
|
|
taskPayload := types.KnowledgePostProcessPayload{
|
|
TenantID: payload.TenantID,
|
|
KnowledgeID: payload.KnowledgeID,
|
|
KnowledgeBaseID: payload.KnowledgeBaseID,
|
|
Language: payload.Language,
|
|
}
|
|
langfuse.InjectTracing(ctx, &taskPayload)
|
|
payloadBytes, err := json.Marshal(taskPayload)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to marshal post process payload: %v", err)
|
|
return
|
|
}
|
|
|
|
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
|
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
|
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
|
|
} else {
|
|
logger.Infof(ctx, "[ImageMultimodal] Enqueued post process task for %s", payload.KnowledgeID)
|
|
}
|
|
}
|