This commit is contained in:
HelloWeit
2026-05-28 16:31:43 +08:00
committed by lyingbug
parent f1a27e0e18
commit edfabf3135
4 changed files with 61 additions and 8 deletions
@@ -1008,7 +1008,8 @@ func (s *knowledgeService) moveKnowledgeReparse(
return fmt.Errorf("failed to marshal document process payload: %w", err)
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...)
info, err := s.task.Enqueue(task)
if err != nil {
return fmt.Errorf("failed to enqueue document process task: %w", err)
@@ -262,7 +262,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return knowledge, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue document process task: %v", err)
@@ -436,7 +440,11 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
return knowledge, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue URL process task: %v", err)
@@ -659,7 +667,11 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
return knowledge, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config)...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL process task: %v", err)
@@ -879,7 +891,11 @@ func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Contex
return knowledge, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue passage process task: %v", err)
@@ -1724,7 +1724,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
@@ -1778,7 +1782,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config)...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err)
@@ -1825,7 +1833,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
@@ -0,0 +1,24 @@
package service
import (
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/hibiken/asynq"
)
func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asynq.Option {
timeout := 2 * time.Hour
if cfg != nil &&
cfg.KnowledgeBase != nil &&
cfg.KnowledgeBase.DocumentProcessTimeout > 0 {
timeout = cfg.KnowledgeBase.DocumentProcessTimeout
}
opts := []asynq.Option{
asynq.Queue("default"),
asynq.Timeout(timeout),
}
opts = append(opts, extra...)
return opts
}