feat: 新增异步任务提取服务

This commit is contained in:
begoniezhao
2025-09-24 12:15:17 +08:00
parent 31d19095f4
commit b6d8be634a
30 changed files with 2514 additions and 117 deletions
+39
View File
@@ -29,6 +29,7 @@ import (
"github.com/Tencent/WeKnora/services/docreader/src/client"
"github.com/Tencent/WeKnora/services/docreader/src/proto"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"go.opentelemetry.io/otel/attribute"
"golang.org/x/sync/errgroup"
)
@@ -61,6 +62,8 @@ type knowledgeService struct {
chunkRepo interfaces.ChunkRepository
fileSvc interfaces.FileService
modelService interfaces.ModelService
task *asynq.Client
graphEngine interfaces.RetrieveGraphRepository
}
// NewKnowledgeService creates a new knowledge service instance
@@ -74,6 +77,8 @@ func NewKnowledgeService(
chunkRepo interfaces.ChunkRepository,
fileSvc interfaces.FileService,
modelService interfaces.ModelService,
task *asynq.Client,
graphEngine interfaces.RetrieveGraphRepository,
) (interfaces.KnowledgeService, error) {
return &knowledgeService{
config: config,
@@ -85,6 +90,8 @@ func NewKnowledgeService(
chunkRepo: chunkRepo,
fileSvc: fileSvc,
modelService: modelService,
task: task,
graphEngine: graphEngine,
}, nil
}
@@ -488,6 +495,16 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
return nil
})
// Delete the knowledge graph
wg.Go(func() error {
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
return err
}
return nil
})
if err = wg.Wait(); err != nil {
return err
}
@@ -561,6 +578,19 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
return nil
})
// Delete the knowledge graph
wg.Go(func() error {
namespaces := []types.NameSpace{}
for _, knowledge := range knowledgeList {
namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID})
}
if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
return err
}
return nil
})
if err = wg.Wait(); err != nil {
return err
}
@@ -1158,6 +1188,15 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
logger.Infof(ctx, "processChunks create relationship rag task")
for _, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed")
span.RecordError(err)
}
}
// Update knowledge status to completed
knowledge.ParseStatus = "completed"
knowledge.EnableStatus = "enabled"