feat(knowledge): offload manual processing to async tasks

This commit is contained in:
wizardchen
2026-03-17 19:26:29 +08:00
committed by lyingbug
parent 49b5f1a739
commit 3450ed5e41
7 changed files with 629 additions and 27 deletions
+175 -25
View File
@@ -885,8 +885,14 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context,
}
if status == types.ManualKnowledgeStatusPublish {
logger.Infof(ctx, "Manual knowledge created, scheduling indexing, ID: %s", knowledge.ID)
s.triggerManualProcessing(ctx, kb, knowledge, cleanContent, false)
logger.Infof(ctx, "Manual knowledge created, enqueuing async processing task, ID: %s", knowledge.ID)
if err := s.enqueueManualProcessing(ctx, knowledge, cleanContent, false); err != nil {
logger.Errorf(ctx, "Failed to enqueue manual processing task for new knowledge: %v", err)
// Non-fatal: mark as failed so user can retry
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "Failed to enqueue processing task"
s.repo.UpdateKnowledge(ctx, knowledge)
}
}
return knowledge, nil
@@ -2471,6 +2477,8 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types
}
// UpdateManualKnowledge updates manual Markdown knowledge content.
// For publish status, the heavy operations (cleanup old indexes, re-chunking,
// re-embedding) are offloaded to an Asynq task so the HTTP response returns quickly.
func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
knowledgeID string, payload *types.ManualKnowledgePayload,
) (*types.Knowledge, error) {
@@ -2540,14 +2548,6 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
existing.Source = types.KnowledgeTypeManual
existing.EnableStatus = "disabled"
existing.UpdatedAt = time.Now()
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": knowledgeID,
})
return nil, err
}
existing.EmbeddingModelID = kb.EmbeddingModelID
if status == types.ManualKnowledgeStatusDraft {
@@ -2562,6 +2562,7 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
return existing, nil
}
// Publish: persist pending status and enqueue async task for cleanup + re-indexing
existing.ParseStatus = "pending"
existing.Description = ""
existing.ProcessedAt = nil
@@ -2571,11 +2572,45 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
return nil, err
}
logger.Infof(ctx, "Manual knowledge updated, scheduling indexing, ID: %s", existing.ID)
s.triggerManualProcessing(ctx, kb, existing, cleanContent, false)
logger.Infof(ctx, "Manual knowledge updated, enqueuing async processing task, ID: %s", existing.ID)
if err := s.enqueueManualProcessing(ctx, existing, cleanContent, true); err != nil {
logger.Errorf(ctx, "Failed to enqueue manual processing task: %v", err)
// Non-fatal: mark as failed so user can retry
existing.ParseStatus = "failed"
existing.ErrorMessage = "Failed to enqueue processing task"
s.repo.UpdateKnowledge(ctx, existing)
return nil, werrors.NewInternalServerError("Failed to submit processing task")
}
return existing, nil
}
// enqueueManualProcessing enqueues a manual:process Asynq task for async cleanup + re-indexing.
func (s *knowledgeService) enqueueManualProcessing(ctx context.Context,
knowledge *types.Knowledge, content string, needCleanup bool,
) error {
requestID, _ := types.RequestIDFromContext(ctx)
payload := types.ManualProcessPayload{
RequestId: requestID,
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: content,
NeedCleanup: needCleanup,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal manual process payload: %w", err)
}
task := asynq.NewTask(types.TypeManualProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
return fmt.Errorf("failed to enqueue manual process task: %w", err)
}
logger.Infof(ctx, "Enqueued manual process task: knowledge_id=%s, asynq_id=%s", knowledge.ID, info.ID)
return nil
}
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing.
func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) {
@@ -2595,7 +2630,35 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return nil, err
}
// Step 1: Clean up existing resources (chunks, embeddings, graph data)
// For manual knowledge, use async manual processing (cleanup + re-indexing in worker)
if existing.IsManual() {
meta, metaErr := existing.ManualMetadata()
if metaErr != nil || meta == nil {
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", metaErr)
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
}
existing.ParseStatus = "pending"
existing.EnableStatus = "disabled"
existing.Description = ""
existing.ProcessedAt = nil
existing.EmbeddingModelID = kb.EmbeddingModelID
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
return nil, err
}
if err := s.enqueueManualProcessing(ctx, existing, meta.Content, true); err != nil {
logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err)
existing.ParseStatus = "failed"
existing.ErrorMessage = "Failed to enqueue processing task"
s.repo.UpdateKnowledge(ctx, existing)
}
return existing, nil
}
// For non-manual knowledge, cleanup synchronously then enqueue document processing
logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID)
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -2619,17 +2682,6 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
// Step 3: Trigger async re-parsing based on knowledge type
logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type)
// For manual knowledge, extract content from metadata and trigger manual processing
if existing.IsManual() {
meta, err := existing.ManualMetadata()
if err != nil || meta == nil {
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", err)
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
}
s.triggerManualProcessing(ctx, kb, existing, meta.Content, false)
return existing, nil
}
// For file-based knowledge, enqueue document processing task
if existing.FilePath != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
@@ -6779,6 +6831,22 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
return
}
// Resolve remote images: download http(s) images, upload to storage, replace URLs.
// This runs before chunking so that chunks contain stable provider:// URLs.
var resolvedImages []docparser.StoredImage
if s.imageResolver != nil {
fileSvc := s.resolveFileService(ctx, kb)
updatedContent, storedImages, resolveErr := s.imageResolver.ResolveRemoteImages(ctx, clean, fileSvc, knowledge.TenantID)
if resolveErr != nil {
logger.Warnf(ctx, "Remote image resolution partially failed: %v", resolveErr)
}
if len(storedImages) > 0 {
logger.Infof(ctx, "Resolved %d remote images for manual knowledge %s", len(storedImages), knowledge.ID)
clean = updatedContent
resolvedImages = storedImages
}
}
// Manual content is markdown - chunk directly with Go chunker
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
@@ -6796,7 +6864,12 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
}
var parsed []types.ParsedChunk
var opts ProcessChunksOptions
opts := ProcessChunksOptions{
// When the KB has VLM enabled and we resolved remote images, pass them
// through so processChunks will enqueue image:multimodal tasks (OCR + caption).
EnableMultimodel: kb.IsMultimodalEnabled() && len(resolvedImages) > 0,
StoredImages: resolvedImages,
}
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
@@ -7153,6 +7226,83 @@ func downloadFileFromURL(ctx context.Context, fileURL string, payloadFileName, p
return contentBytes, nil
}
// ProcessManualUpdate handles Asynq manual knowledge update tasks.
// It performs cleanup of old indexes/chunks (when NeedCleanup is true) and re-indexes the content.
func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Task) error {
var payload types.ManualProcessPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "failed to unmarshal manual process task payload: %v", err)
return nil
}
ctx = logger.WithRequestID(ctx, payload.RequestId)
ctx = logger.WithField(ctx, "manual_process", payload.KnowledgeID)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to get tenant: %v", err)
return nil
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge: %v", err)
return nil
}
if knowledge == nil {
logger.Warnf(ctx, "ProcessManualUpdate: knowledge not found: %s", payload.KnowledgeID)
return nil
}
// Skip if already completed or being deleted
if knowledge.ParseStatus == types.ParseStatusCompleted {
logger.Infof(ctx, "ProcessManualUpdate: already completed, skipping: %s", payload.KnowledgeID)
return nil
}
if knowledge.ParseStatus == types.ParseStatusDeleting {
logger.Infof(ctx, "ProcessManualUpdate: being deleted, skipping: %s", payload.KnowledgeID)
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge base: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to get knowledge base: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// Update status to processing
knowledge.ParseStatus = "processing"
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to update status to processing: %v", err)
return nil
}
// Cleanup old resources (indexes, chunks, graph) for update operations
if payload.NeedCleanup {
if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": payload.KnowledgeID,
})
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to cleanup old resources: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
}
// Run manual processing (image resolution + chunking + embedding) synchronously within the worker
s.triggerManualProcessing(ctx, kb, knowledge, payload.Content, true)
return nil
}
// ProcessDocument handles Asynq document processing tasks
func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) error {
var payload types.DocumentProcessPayload
@@ -3,14 +3,22 @@ package docparser
import (
"bytes"
"context"
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"io"
"log"
"mime"
"net/http"
"path"
"path/filepath"
"regexp"
"strings"
"time"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -156,11 +164,188 @@ func extFromMime(mime string) string {
}
// isProviderScheme checks if the path uses a provider:// scheme (local://, minio://, cos://, tos://).
func isProviderScheme(path string) bool {
func isProviderScheme(p string) bool {
for _, prefix := range []string{"local://", "minio://", "cos://", "tos://"} {
if strings.HasPrefix(path, prefix) {
if strings.HasPrefix(p, prefix) {
return true
}
}
return false
}
// ---------------------------------------------------------------------------
// Remote image resolution (for manual / web-clipped markdown content)
// ---------------------------------------------------------------------------
const (
// maxRemoteImageSize is the maximum allowed size for a single remote image download.
maxRemoteImageSize = 10 * 1024 * 1024 // 10 MB
// maxRemoteImages is the maximum number of remote images to process per document.
maxRemoteImages = 30
// remoteImageFetchTimeout is the per-image HTTP request timeout.
remoteImageFetchTimeout = 15 * time.Second
)
// imgMarkdownPattern matches Markdown image syntax: ![alt](url)
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)\s]+)\)`)
// ResolveRemoteImages scans a Markdown string for image references whose URL
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
// uploads the bytes via fileSvc, and replaces the original URL with the
// provider:// serving URL.
//
// Images that fail SSRF validation, exceed size limits, or cannot be downloaded
// are left unchanged (the original URL is preserved).
//
// Returns the updated Markdown and a list of successfully stored images.
func (r *ImageResolver) ResolveRemoteImages(
ctx context.Context,
markdown string,
fileSvc interfaces.FileService,
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
matches := imgMarkdownPattern.FindAllStringSubmatchIndex(markdown, -1)
if len(matches) == 0 {
return markdown, nil, nil
}
// Build a shared SSRF-safe HTTP client for all downloads.
httpClient := secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
Timeout: remoteImageFetchTimeout,
MaxRedirects: 5,
})
processed := 0
// Process in reverse order so that earlier indices stay valid after replacements.
for i := len(matches) - 1; i >= 0; i-- {
if processed >= maxRemoteImages {
break
}
m := matches[i]
imgURL := markdown[m[4]:m[5]] // group 2: the URL
// Only process remote http(s) URLs.
if !strings.HasPrefix(imgURL, "http://") && !strings.HasPrefix(imgURL, "https://") {
continue
}
// Already a provider scheme — skip.
if isProviderScheme(imgURL) {
continue
}
// --- SSRF check ---
if safe, reason := secutils.IsSSRFSafeURL(imgURL); !safe {
log.Printf("WARN: remote image blocked by SSRF check (%s): %s", reason, imgURL)
continue
}
// --- Download ---
data, mimeType, dlErr := downloadImage(ctx, httpClient, imgURL)
if dlErr != nil {
log.Printf("WARN: failed to download remote image %s: %v", imgURL, dlErr)
continue
}
// Filter out icons / tiny decorative images.
if isIconImage(data) {
continue
}
// Determine file extension.
ext := extFromMime(mimeType)
if ext == "" {
ext = extFromURLPath(imgURL)
}
if ext == "" {
ext = ".png" // safe default
}
// --- Upload to storage ---
fileName := uuid.New().String() + ext
servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
if saveErr != nil {
log.Printf("WARN: failed to save remote image %s: %v", imgURL, saveErr)
continue
}
images = append(images, StoredImage{
OriginalRef: imgURL,
ServingURL: servingURL,
MimeType: mimeType,
})
// Replace URL in markdown.
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
processed++
}
return markdown, images, nil
}
// downloadImage fetches an image from remoteURL using the provided SSRF-safe
// client. It validates Content-Type and enforces maxRemoteImageSize.
func downloadImage(ctx context.Context, client *http.Client, remoteURL string) (data []byte, mimeType string, err error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL, nil)
if err != nil {
return nil, "", fmt.Errorf("create request: %w", err)
}
// Some CDNs require a browser-like User-Agent.
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WeKnora/1.0)")
resp, err := client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("HTTP GET: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode)
}
// Determine MIME type from Content-Type header.
ct := resp.Header.Get("Content-Type")
mimeType, _, _ = mime.ParseMediaType(ct)
if mimeType == "" {
mimeType = "application/octet-stream"
}
// Only allow image content types (or octet-stream which we sniff later).
if !strings.HasPrefix(mimeType, "image/") && mimeType != "application/octet-stream" {
return nil, "", fmt.Errorf("non-image content type: %s", mimeType)
}
// Read body with size limit.
limited := io.LimitReader(resp.Body, maxRemoteImageSize+1)
body, err := io.ReadAll(limited)
if err != nil {
return nil, "", fmt.Errorf("read body: %w", err)
}
if len(body) > maxRemoteImageSize {
return nil, "", fmt.Errorf("image exceeds %d bytes limit", maxRemoteImageSize)
}
// If MIME was octet-stream, sniff the real type from body.
if mimeType == "application/octet-stream" {
detected := http.DetectContentType(body)
if strings.HasPrefix(detected, "image/") {
mimeType = detected
} else {
return nil, "", fmt.Errorf("downloaded data is not an image (sniffed: %s)", detected)
}
}
return body, mimeType, nil
}
// extFromURLPath extracts the image file extension from the URL path segment.
func extFromURLPath(rawURL string) string {
p := path.Ext(path.Base(rawURL))
switch strings.ToLower(p) {
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
return strings.ToLower(p)
default:
return ""
}
}
@@ -0,0 +1,249 @@
package docparser
import (
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// mockFileService is a minimal FileService implementation for testing.
type mockFileService struct {
saved []savedEntry
}
type savedEntry struct {
Data []byte
TenantID uint64
FileName string
}
func (m *mockFileService) CheckConnectivity(ctx context.Context) error { return nil }
func (m *mockFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
return "", nil
}
func (m *mockFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
m.saved = append(m.saved, savedEntry{Data: data, TenantID: tenantID, FileName: fileName})
return fmt.Sprintf("local://images/%s", fileName), nil
}
func (m *mockFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
return nil, nil
}
func (m *mockFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
return filePath, nil
}
func (m *mockFileService) DeleteFile(ctx context.Context, filePath string) error { return nil }
func TestResolveRemoteImages_NormalDownload(t *testing.T) {
// Create a test HTTP server that serves a real PNG image.
pngData := createTestPNG(200, 200)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.WriteHeader(http.StatusOK)
w.Write(pngData)
}))
defer ts.Close()
markdown := fmt.Sprintf("# Hello\n\n![photo](%s/image.png)\n\nSome text", ts.URL)
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 42)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 1 {
t.Fatalf("expected 1 stored image, got %d", len(images))
}
// URL should have been replaced.
if strings.Contains(updated, ts.URL) {
t.Errorf("original URL should have been replaced in markdown, got: %s", updated)
}
if !strings.Contains(updated, "local://images/") {
t.Errorf("expected local:// URL in markdown, got: %s", updated)
}
// Verify saved data.
if len(fSvc.saved) != 1 {
t.Fatalf("expected 1 saved entry, got %d", len(fSvc.saved))
}
if fSvc.saved[0].TenantID != 42 {
t.Errorf("expected tenantID 42, got %d", fSvc.saved[0].TenantID)
}
}
func TestResolveRemoteImages_SSRFBlocked(t *testing.T) {
// URLs pointing to private IPs should be blocked by SSRF check.
markdown := "![evil](http://127.0.0.1:8080/secret.png)\n\n![also-evil](http://169.254.169.254/metadata)"
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Both images should be left unchanged (SSRF blocked).
if len(images) != 0 {
t.Errorf("expected 0 stored images (SSRF blocked), got %d", len(images))
}
if updated != markdown {
t.Errorf("markdown should be unchanged when SSRF blocked")
}
}
func TestResolveRemoteImages_NonImageContentType(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
w.Write([]byte("<html>not an image</html>"))
}))
defer ts.Close()
markdown := fmt.Sprintf("![bad](%s/page.html)", ts.URL)
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 0 {
t.Errorf("expected 0 images for non-image content type, got %d", len(images))
}
// Original URL should be preserved.
if !strings.Contains(updated, ts.URL) {
t.Errorf("original URL should be preserved for non-image content")
}
}
func TestResolveRemoteImages_ProviderSchemeSkipped(t *testing.T) {
markdown := "![already](local://images/abc.png)\n![also](minio://bucket/key.jpg)"
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 0 {
t.Errorf("expected 0 images for provider:// URLs, got %d", len(images))
}
if updated != markdown {
t.Errorf("markdown should be unchanged for provider:// URLs")
}
}
func TestResolveRemoteImages_MultipleImages(t *testing.T) {
pngData := createTestPNG(256, 256)
callCount := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
w.Header().Set("Content-Type", "image/png")
w.WriteHeader(http.StatusOK)
w.Write(pngData)
}))
defer ts.Close()
markdown := fmt.Sprintf("![img1](%s/a.png)\n\ntext\n\n![img2](%s/b.png)\n\n![img3](%s/c.png)",
ts.URL, ts.URL, ts.URL)
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 10)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 3 {
t.Fatalf("expected 3 stored images, got %d", len(images))
}
if callCount != 3 {
t.Errorf("expected 3 HTTP requests, got %d", callCount)
}
if strings.Contains(updated, ts.URL) {
t.Errorf("all original URLs should have been replaced")
}
}
func TestResolveRemoteImages_NoImages(t *testing.T) {
markdown := "# Just text\n\nNo images here."
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 0 {
t.Errorf("expected 0 images, got %d", len(images))
}
if updated != markdown {
t.Errorf("markdown should be unchanged")
}
}
func TestResolveRemoteImages_Server404(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
markdown := fmt.Sprintf("![missing](%s/nope.png)", ts.URL)
resolver := NewImageResolver()
fSvc := &mockFileService{}
updated, images, err := resolver.ResolveRemoteImages(context.Background(), markdown, fSvc, 1)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(images) != 0 {
t.Errorf("expected 0 images for 404, got %d", len(images))
}
// Original URL preserved on failure.
if !strings.Contains(updated, ts.URL) {
t.Errorf("original URL should be preserved on download failure")
}
}
func TestExtFromURLPath(t *testing.T) {
tests := []struct {
url string
expect string
}{
{"https://example.com/photo.jpg", ".jpg"},
{"https://example.com/photo.JPEG", ".jpeg"},
{"https://example.com/photo.png?v=2", ""}, // query param — path.Ext won't catch it cleanly but that's ok
{"https://example.com/photo.gif", ".gif"},
{"https://example.com/photo.webp", ".webp"},
{"https://example.com/photo.bmp", ".bmp"},
{"https://example.com/photo.svg", ".svg"},
{"https://example.com/photo.pdf", ""},
{"https://example.com/noext", ""},
}
for _, tt := range tests {
t.Run(tt.url, func(t *testing.T) {
got := extFromURLPath(tt.url)
if got != tt.expect {
t.Errorf("extFromURLPath(%q) = %q, want %q", tt.url, got, tt.expect)
}
})
}
}
+1
View File
@@ -86,6 +86,7 @@ func RegisterSyncHandlers(params SyncTaskParams) {
params.Executor.RegisterHandler(types.TypeChunkExtract, params.ChunkExtractor.Handle)
params.Executor.RegisterHandler(types.TypeDataTableSummary, params.DataTableSummary.Handle)
params.Executor.RegisterHandler(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
params.Executor.RegisterHandler(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
params.Executor.RegisterHandler(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
params.Executor.RegisterHandler(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
params.Executor.RegisterHandler(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
+3
View File
@@ -78,6 +78,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register document processing handler
mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument)
// Register manual knowledge processing handler (cleanup + re-indexing)
mux.HandleFunc(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate)
// Register FAQ import handler (includes dry run mode)
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
+12
View File
@@ -13,6 +13,7 @@ const (
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -126,6 +127,17 @@ type KnowledgeMoveProgress struct {
UpdatedAt int64 `json:"updated_at"` // 最后更新时间
}
// ManualProcessPayload represents the manual knowledge processing task payload.
// Used for both create (publish) and update operations.
type ManualProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Content string `json:"content"` // cleaned markdown content
NeedCleanup bool `json:"need_cleanup"` // true for update, false for create
}
// ImageMultimodalPayload represents the image multimodal processing task payload.
type ImageMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
+2
View File
@@ -129,6 +129,8 @@ type KnowledgeService interface {
UpdateFAQEntryTagBatch(ctx context.Context, kbID string, updates map[int64]*int64) error
// GetRepository gets the knowledge repository
GetRepository() KnowledgeRepository
// ProcessManualUpdate handles Asynq manual knowledge update tasks (cleanup + re-indexing)
ProcessManualUpdate(ctx context.Context, t *asynq.Task) error
// ProcessDocument handles Asynq document processing tasks
ProcessDocument(ctx context.Context, t *asynq.Task) error
// ProcessFAQImport handles Asynq FAQ import tasks