feat(knowledge): offload manual processing to async tasks
This commit is contained in:
@@ -885,8 +885,14 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context,
|
||||
}
|
||||
|
||||
if status == types.ManualKnowledgeStatusPublish {
|
||||
logger.Infof(ctx, "Manual knowledge created, scheduling indexing, ID: %s", knowledge.ID)
|
||||
s.triggerManualProcessing(ctx, kb, knowledge, cleanContent, false)
|
||||
logger.Infof(ctx, "Manual knowledge created, enqueuing async processing task, ID: %s", knowledge.ID)
|
||||
if err := s.enqueueManualProcessing(ctx, knowledge, cleanContent, false); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue manual processing task for new knowledge: %v", err)
|
||||
// Non-fatal: mark as failed so user can retry
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "Failed to enqueue processing task"
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
}
|
||||
}
|
||||
|
||||
return knowledge, nil
|
||||
@@ -2471,6 +2477,8 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types
|
||||
}
|
||||
|
||||
// UpdateManualKnowledge updates manual Markdown knowledge content.
|
||||
// For publish status, the heavy operations (cleanup old indexes, re-chunking,
|
||||
// re-embedding) are offloaded to an Asynq task so the HTTP response returns quickly.
|
||||
func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
knowledgeID string, payload *types.ManualKnowledgePayload,
|
||||
) (*types.Knowledge, error) {
|
||||
@@ -2540,14 +2548,6 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
existing.Source = types.KnowledgeTypeManual
|
||||
existing.EnableStatus = "disabled"
|
||||
existing.UpdatedAt = time.Now()
|
||||
|
||||
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
existing.EmbeddingModelID = kb.EmbeddingModelID
|
||||
|
||||
if status == types.ManualKnowledgeStatusDraft {
|
||||
@@ -2562,6 +2562,7 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// Publish: persist pending status and enqueue async task for cleanup + re-indexing
|
||||
existing.ParseStatus = "pending"
|
||||
existing.Description = ""
|
||||
existing.ProcessedAt = nil
|
||||
@@ -2571,11 +2572,45 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Manual knowledge updated, scheduling indexing, ID: %s", existing.ID)
|
||||
s.triggerManualProcessing(ctx, kb, existing, cleanContent, false)
|
||||
logger.Infof(ctx, "Manual knowledge updated, enqueuing async processing task, ID: %s", existing.ID)
|
||||
if err := s.enqueueManualProcessing(ctx, existing, cleanContent, true); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue manual processing task: %v", err)
|
||||
// Non-fatal: mark as failed so user can retry
|
||||
existing.ParseStatus = "failed"
|
||||
existing.ErrorMessage = "Failed to enqueue processing task"
|
||||
s.repo.UpdateKnowledge(ctx, existing)
|
||||
return nil, werrors.NewInternalServerError("Failed to submit processing task")
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// enqueueManualProcessing enqueues a manual:process Asynq task for async cleanup + re-indexing.
|
||||
func (s *knowledgeService) enqueueManualProcessing(ctx context.Context,
|
||||
knowledge *types.Knowledge, content string, needCleanup bool,
|
||||
) error {
|
||||
requestID, _ := types.RequestIDFromContext(ctx)
|
||||
payload := types.ManualProcessPayload{
|
||||
RequestId: requestID,
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
Content: content,
|
||||
NeedCleanup: needCleanup,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal manual process payload: %w", err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeManualProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to enqueue manual process task: %w", err)
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued manual process task: knowledge_id=%s, asynq_id=%s", knowledge.ID, info.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
|
||||
// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing.
|
||||
func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) {
|
||||
@@ -2595,7 +2630,35 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 1: Clean up existing resources (chunks, embeddings, graph data)
|
||||
// For manual knowledge, use async manual processing (cleanup + re-indexing in worker)
|
||||
if existing.IsManual() {
|
||||
meta, metaErr := existing.ManualMetadata()
|
||||
if metaErr != nil || meta == nil {
|
||||
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", metaErr)
|
||||
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
|
||||
}
|
||||
|
||||
existing.ParseStatus = "pending"
|
||||
existing.EnableStatus = "disabled"
|
||||
existing.Description = ""
|
||||
existing.ProcessedAt = nil
|
||||
existing.EmbeddingModelID = kb.EmbeddingModelID
|
||||
|
||||
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.enqueueManualProcessing(ctx, existing, meta.Content, true); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err)
|
||||
existing.ParseStatus = "failed"
|
||||
existing.ErrorMessage = "Failed to enqueue processing task"
|
||||
s.repo.UpdateKnowledge(ctx, existing)
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// For non-manual knowledge, cleanup synchronously then enqueue document processing
|
||||
logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID)
|
||||
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
@@ -2619,17 +2682,6 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
// Step 3: Trigger async re-parsing based on knowledge type
|
||||
logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type)
|
||||
|
||||
// For manual knowledge, extract content from metadata and trigger manual processing
|
||||
if existing.IsManual() {
|
||||
meta, err := existing.ManualMetadata()
|
||||
if err != nil || meta == nil {
|
||||
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", err)
|
||||
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
|
||||
}
|
||||
s.triggerManualProcessing(ctx, kb, existing, meta.Content, false)
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// For file-based knowledge, enqueue document processing task
|
||||
if existing.FilePath != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
@@ -6779,6 +6831,22 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve remote images: download http(s) images, upload to storage, replace URLs.
|
||||
// This runs before chunking so that chunks contain stable provider:// URLs.
|
||||
var resolvedImages []docparser.StoredImage
|
||||
if s.imageResolver != nil {
|
||||
fileSvc := s.resolveFileService(ctx, kb)
|
||||
updatedContent, storedImages, resolveErr := s.imageResolver.ResolveRemoteImages(ctx, clean, fileSvc, knowledge.TenantID)
|
||||
if resolveErr != nil {
|
||||
logger.Warnf(ctx, "Remote image resolution partially failed: %v", resolveErr)
|
||||
}
|
||||
if len(storedImages) > 0 {
|
||||
logger.Infof(ctx, "Resolved %d remote images for manual knowledge %s", len(storedImages), knowledge.ID)
|
||||
clean = updatedContent
|
||||
resolvedImages = storedImages
|
||||
}
|
||||
}
|
||||
|
||||
// Manual content is markdown - chunk directly with Go chunker
|
||||
chunkCfg := chunker.SplitterConfig{
|
||||
ChunkSize: kb.ChunkingConfig.ChunkSize,
|
||||
@@ -6796,7 +6864,12 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
}
|
||||
|
||||
var parsed []types.ParsedChunk
|
||||
var opts ProcessChunksOptions
|
||||
opts := ProcessChunksOptions{
|
||||
// When the KB has VLM enabled and we resolved remote images, pass them
|
||||
// through so processChunks will enqueue image:multimodal tasks (OCR + caption).
|
||||
EnableMultimodel: kb.IsMultimodalEnabled() && len(resolvedImages) > 0,
|
||||
StoredImages: resolvedImages,
|
||||
}
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
@@ -7153,6 +7226,83 @@ func downloadFileFromURL(ctx context.Context, fileURL string, payloadFileName, p
|
||||
return contentBytes, nil
|
||||
}
|
||||
|
||||
// ProcessManualUpdate handles Asynq manual knowledge update tasks.
|
||||
// It performs cleanup of old indexes/chunks (when NeedCleanup is true) and re-indexes the content.
|
||||
func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.ManualProcessPayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
logger.Errorf(ctx, "failed to unmarshal manual process task payload: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx = logger.WithRequestID(ctx, payload.RequestId)
|
||||
ctx = logger.WithField(ctx, "manual_process", payload.KnowledgeID)
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "ProcessManualUpdate: failed to get tenant: %v", err)
|
||||
return nil
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge: %v", err)
|
||||
return nil
|
||||
}
|
||||
if knowledge == nil {
|
||||
logger.Warnf(ctx, "ProcessManualUpdate: knowledge not found: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip if already completed or being deleted
|
||||
if knowledge.ParseStatus == types.ParseStatusCompleted {
|
||||
logger.Infof(ctx, "ProcessManualUpdate: already completed, skipping: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
if knowledge.ParseStatus == types.ParseStatusDeleting {
|
||||
logger.Infof(ctx, "ProcessManualUpdate: being deleted, skipping: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge base: %v", err)
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = fmt.Sprintf("failed to get knowledge base: %v", err)
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update status to processing
|
||||
knowledge.ParseStatus = "processing"
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "ProcessManualUpdate: failed to update status to processing: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup old resources (indexes, chunks, graph) for update operations
|
||||
if payload.NeedCleanup {
|
||||
if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_id": payload.KnowledgeID,
|
||||
})
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = fmt.Sprintf("failed to cleanup old resources: %v", err)
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Run manual processing (image resolution + chunking + embedding) synchronously within the worker
|
||||
s.triggerManualProcessing(ctx, kb, knowledge, payload.Content, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProcessDocument handles Asynq document processing tasks
|
||||
func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.DocumentProcessPayload
|
||||
|
||||
@@ -3,14 +3,22 @@ package docparser
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -156,11 +164,188 @@ func extFromMime(mime string) string {
|
||||
}
|
||||
|
||||
// isProviderScheme checks if the path uses a provider:// scheme (local://, minio://, cos://, tos://).
|
||||
func isProviderScheme(path string) bool {
|
||||
func isProviderScheme(p string) bool {
|
||||
for _, prefix := range []string{"local://", "minio://", "cos://", "tos://"} {
|
||||
if strings.HasPrefix(path, prefix) {
|
||||
if strings.HasPrefix(p, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Remote image resolution (for manual / web-clipped markdown content)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
// maxRemoteImageSize is the maximum allowed size for a single remote image download.
|
||||
maxRemoteImageSize = 10 * 1024 * 1024 // 10 MB
|
||||
// maxRemoteImages is the maximum number of remote images to process per document.
|
||||
maxRemoteImages = 30
|
||||
// remoteImageFetchTimeout is the per-image HTTP request timeout.
|
||||
remoteImageFetchTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// imgMarkdownPattern matches Markdown image syntax: 
|
||||
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)\s]+)\)`)
|
||||
|
||||
// ResolveRemoteImages scans a Markdown string for image references whose URL
|
||||
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
|
||||
// uploads the bytes via fileSvc, and replaces the original URL with the
|
||||
// provider:// serving URL.
|
||||
//
|
||||
// Images that fail SSRF validation, exceed size limits, or cannot be downloaded
|
||||
// are left unchanged (the original URL is preserved).
|
||||
//
|
||||
// Returns the updated Markdown and a list of successfully stored images.
|
||||
func (r *ImageResolver) ResolveRemoteImages(
|
||||
ctx context.Context,
|
||||
markdown string,
|
||||
fileSvc interfaces.FileService,
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
matches := imgMarkdownPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
if len(matches) == 0 {
|
||||
return markdown, nil, nil
|
||||
}
|
||||
|
||||
// Build a shared SSRF-safe HTTP client for all downloads.
|
||||
httpClient := secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: remoteImageFetchTimeout,
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
|
||||
processed := 0
|
||||
|
||||
// Process in reverse order so that earlier indices stay valid after replacements.
|
||||
for i := len(matches) - 1; i >= 0; i-- {
|
||||
if processed >= maxRemoteImages {
|
||||
break
|
||||
}
|
||||
m := matches[i]
|
||||
imgURL := markdown[m[4]:m[5]] // group 2: the URL
|
||||
|
||||
// Only process remote http(s) URLs.
|
||||
if !strings.HasPrefix(imgURL, "http://") && !strings.HasPrefix(imgURL, "https://") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Already a provider scheme — skip.
|
||||
if isProviderScheme(imgURL) {
|
||||
continue
|
||||
}
|
||||
|
||||
// --- SSRF check ---
|
||||
if safe, reason := secutils.IsSSRFSafeURL(imgURL); !safe {
|
||||
log.Printf("WARN: remote image blocked by SSRF check (%s): %s", reason, imgURL)
|
||||
continue
|
||||
}
|
||||
|
||||
// --- Download ---
|
||||
data, mimeType, dlErr := downloadImage(ctx, httpClient, imgURL)
|
||||
if dlErr != nil {
|
||||
log.Printf("WARN: failed to download remote image %s: %v", imgURL, dlErr)
|
||||
continue
|
||||
}
|
||||
|
||||
// Filter out icons / tiny decorative images.
|
||||
if isIconImage(data) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine file extension.
|
||||
ext := extFromMime(mimeType)
|
||||
if ext == "" {
|
||||
ext = extFromURLPath(imgURL)
|
||||
}
|
||||
if ext == "" {
|
||||
ext = ".png" // safe default
|
||||
}
|
||||
|
||||
// --- Upload to storage ---
|
||||
fileName := uuid.New().String() + ext
|
||||
servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
|
||||
if saveErr != nil {
|
||||
log.Printf("WARN: failed to save remote image %s: %v", imgURL, saveErr)
|
||||
continue
|
||||
}
|
||||
|
||||
images = append(images, StoredImage{
|
||||
OriginalRef: imgURL,
|
||||
ServingURL: servingURL,
|
||||
MimeType: mimeType,
|
||||
})
|
||||
|
||||
// Replace URL in markdown.
|
||||
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
|
||||
processed++
|
||||
}
|
||||
|
||||
return markdown, images, nil
|
||||
}
|
||||
|
||||
// downloadImage fetches an image from remoteURL using the provided SSRF-safe
|
||||
// client. It validates Content-Type and enforces maxRemoteImageSize.
|
||||
func downloadImage(ctx context.Context, client *http.Client, remoteURL string) (data []byte, mimeType string, err error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL, nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
// Some CDNs require a browser-like User-Agent.
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WeKnora/1.0)")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("HTTP GET: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Determine MIME type from Content-Type header.
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
mimeType, _, _ = mime.ParseMediaType(ct)
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Only allow image content types (or octet-stream which we sniff later).
|
||||
if !strings.HasPrefix(mimeType, "image/") && mimeType != "application/octet-stream" {
|
||||
return nil, "", fmt.Errorf("non-image content type: %s", mimeType)
|
||||
}
|
||||
|
||||
// Read body with size limit.
|
||||
limited := io.LimitReader(resp.Body, maxRemoteImageSize+1)
|
||||
body, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if len(body) > maxRemoteImageSize {
|
||||
return nil, "", fmt.Errorf("image exceeds %d bytes limit", maxRemoteImageSize)
|
||||
}
|
||||
|
||||
// If MIME was octet-stream, sniff the real type from body.
|
||||
if mimeType == "application/octet-stream" {
|
||||
detected := http.DetectContentType(body)
|
||||
if strings.HasPrefix(detected, "image/") {
|
||||
mimeType = detected
|
||||
} else {
|
||||
return nil, "", fmt.Errorf("downloaded data is not an image (sniffed: %s)", detected)
|
||||
}
|
||||
}
|
||||
|
||||
return body, mimeType, nil
|
||||
}
|
||||
|
||||
// extFromURLPath extracts the image file extension from the URL path segment.
|
||||
func extFromURLPath(rawURL string) string {
|
||||
p := path.Ext(path.Base(rawURL))
|
||||
switch strings.ToLower(p) {
|
||||
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
|
||||
return strings.ToLower(p)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockFileService is a minimal FileService implementation for testing.
|
||||
type mockFileService struct {
|
||||
saved []savedEntry
|
||||
}
|
||||
|
||||
type savedEntry struct {
|
||||
Data []byte
|
||||
TenantID uint64
|
||||
FileName string
|
||||
}
|
||||
|
||||
func (m *mockFileService) CheckConnectivity(ctx context.Context) error { return nil }
|
||||
func (m *mockFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (m *mockFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
m.saved = append(m.saved, savedEntry{Data: data, TenantID: tenantID, FileName: fileName})
|
||||
return fmt.Sprintf("local://images/%s", fileName), nil
|
||||
}
|
||||
func (m *mockFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
return filePath, nil
|
||||
}
|
||||
func (m *mockFileService) DeleteFile(ctx context.Context, filePath string) error { return nil }
|
||||
|
||||
func TestResolveRemoteImages_NormalDownload(t *testing.T) {
|
||||
// Create a test HTTP server that serves a real PNG image.
|
||||
pngData := createTestPNG(200, 200)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(pngData)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
markdown := fmt.Sprintf("# Hello\n\n\n\nSome text", ts.URL)
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 42)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 1 {
|
||||
t.Fatalf("expected 1 stored image, got %d", len(images))
|
||||
}
|
||||
|
||||
// URL should have been replaced.
|
||||
if strings.Contains(updated, ts.URL) {
|
||||
t.Errorf("original URL should have been replaced in markdown, got: %s", updated)
|
||||
}
|
||||
if !strings.Contains(updated, "local://images/") {
|
||||
t.Errorf("expected local:// URL in markdown, got: %s", updated)
|
||||
}
|
||||
|
||||
// Verify saved data.
|
||||
if len(fSvc.saved) != 1 {
|
||||
t.Fatalf("expected 1 saved entry, got %d", len(fSvc.saved))
|
||||
}
|
||||
if fSvc.saved[0].TenantID != 42 {
|
||||
t.Errorf("expected tenantID 42, got %d", fSvc.saved[0].TenantID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_SSRFBlocked(t *testing.T) {
|
||||
// URLs pointing to private IPs should be blocked by SSRF check.
|
||||
markdown := "\n\n"
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// Both images should be left unchanged (SSRF blocked).
|
||||
if len(images) != 0 {
|
||||
t.Errorf("expected 0 stored images (SSRF blocked), got %d", len(images))
|
||||
}
|
||||
if updated != markdown {
|
||||
t.Errorf("markdown should be unchanged when SSRF blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_NonImageContentType(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("<html>not an image</html>"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
markdown := fmt.Sprintf("", ts.URL)
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 0 {
|
||||
t.Errorf("expected 0 images for non-image content type, got %d", len(images))
|
||||
}
|
||||
// Original URL should be preserved.
|
||||
if !strings.Contains(updated, ts.URL) {
|
||||
t.Errorf("original URL should be preserved for non-image content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_ProviderSchemeSkipped(t *testing.T) {
|
||||
markdown := "\n"
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 0 {
|
||||
t.Errorf("expected 0 images for provider:// URLs, got %d", len(images))
|
||||
}
|
||||
if updated != markdown {
|
||||
t.Errorf("markdown should be unchanged for provider:// URLs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_MultipleImages(t *testing.T) {
|
||||
pngData := createTestPNG(256, 256)
|
||||
callCount := 0
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
callCount++
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(pngData)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
markdown := fmt.Sprintf("\n\ntext\n\n\n\n",
|
||||
ts.URL, ts.URL, ts.URL)
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 3 {
|
||||
t.Fatalf("expected 3 stored images, got %d", len(images))
|
||||
}
|
||||
if callCount != 3 {
|
||||
t.Errorf("expected 3 HTTP requests, got %d", callCount)
|
||||
}
|
||||
if strings.Contains(updated, ts.URL) {
|
||||
t.Errorf("all original URLs should have been replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_NoImages(t *testing.T) {
|
||||
markdown := "# Just text\n\nNo images here."
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 0 {
|
||||
t.Errorf("expected 0 images, got %d", len(images))
|
||||
}
|
||||
if updated != markdown {
|
||||
t.Errorf("markdown should be unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRemoteImages_Server404(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
markdown := fmt.Sprintf("", ts.URL)
|
||||
|
||||
resolver := NewImageResolver()
|
||||
fSvc := &mockFileService{}
|
||||
|
||||
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(images) != 0 {
|
||||
t.Errorf("expected 0 images for 404, got %d", len(images))
|
||||
}
|
||||
// Original URL preserved on failure.
|
||||
if !strings.Contains(updated, ts.URL) {
|
||||
t.Errorf("original URL should be preserved on download failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtFromURLPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
url string
|
||||
expect string
|
||||
}{
|
||||
{"https://example.com/photo.jpg", ".jpg"},
|
||||
{"https://example.com/photo.JPEG", ".jpeg"},
|
||||
{"https://example.com/photo.png?v=2", ""}, // query param — path.Ext won't catch it cleanly but that's ok
|
||||
{"https://example.com/photo.gif", ".gif"},
|
||||
{"https://example.com/photo.webp", ".webp"},
|
||||
{"https://example.com/photo.bmp", ".bmp"},
|
||||
{"https://example.com/photo.svg", ".svg"},
|
||||
{"https://example.com/photo.pdf", ""},
|
||||
{"https://example.com/noext", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.url, func(t *testing.T) {
|
||||
got := extFromURLPath(tt.url)
|
||||
if got != tt.expect {
|
||||
t.Errorf("extFromURLPath(%q) = %q, want %q", tt.url, got, tt.expect)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,7 @@ func RegisterSyncHandlers(params SyncTaskParams) {
|
||||
params.Executor.RegisterHandler(types.TypeChunkExtract, params.ChunkExtractor.Handle)
|
||||
params.Executor.RegisterHandler(types.TypeDataTableSummary, params.DataTableSummary.Handle)
|
||||
params.Executor.RegisterHandler(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
|
||||
params.Executor.RegisterHandler(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
|
||||
params.Executor.RegisterHandler(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
|
||||
params.Executor.RegisterHandler(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
|
||||
params.Executor.RegisterHandler(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
|
||||
|
||||
@@ -78,6 +78,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register document processing handler
|
||||
mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
|
||||
|
||||
// Register manual knowledge processing handler (cleanup + re-indexing)
|
||||
mux.HandleFunc(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
|
||||
|
||||
// Register FAQ import handler (includes dry run mode)
|
||||
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ const (
|
||||
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
|
||||
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -126,6 +127,17 @@ type KnowledgeMoveProgress struct {
|
||||
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
|
||||
}
|
||||
|
||||
// ManualProcessPayload represents the manual knowledge processing task payload.
|
||||
// Used for both create (publish) and update operations.
|
||||
type ManualProcessPayload struct {
|
||||
RequestId string `json:"request_id"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
Content string `json:"content"` // cleaned markdown content
|
||||
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
|
||||
}
|
||||
|
||||
// ImageMultimodalPayload represents the image multimodal processing task payload.
|
||||
type ImageMultimodalPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
|
||||
@@ -129,6 +129,8 @@ type KnowledgeService interface {
|
||||
UpdateFAQEntryTagBatch(ctx context.Context, kbID string, updates map[int64]*int64) error
|
||||
// GetRepository gets the knowledge repository
|
||||
GetRepository() KnowledgeRepository
|
||||
// ProcessManualUpdate handles Asynq manual knowledge update tasks (cleanup + re-indexing)
|
||||
ProcessManualUpdate(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessDocument handles Asynq document processing tasks
|
||||
ProcessDocument(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessFAQImport handles Asynq FAQ import tasks
|
||||
|
||||
Reference in New Issue
Block a user