feat(metadata): enhance document processing to include metadata extraction and handling
- Updated DocReaderServicer to pass metadata in responses. - Modified PipelineParser to accumulate and merge metadata from all parsers. - Enhanced StdWebParser to extract and log the title from web page content. - Implemented logic in knowledge service to update knowledge title based on extracted metadata.
This commit is contained in:
@@ -7529,6 +7529,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
if convertResult == nil {
|
||||
return nil
|
||||
}
|
||||
// Update knowledge title from extracted page title if not already set
|
||||
if knowledge.Title == "" || knowledge.Title == payload.URL {
|
||||
if extractedTitle := convertResult.Metadata["title"]; extractedTitle != "" {
|
||||
knowledge.Title = extractedTitle
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update knowledge title from extracted page title: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Updated knowledge title to extracted page title: %s", extractedTitle)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if len(payload.Passages) > 0 {
|
||||
// Text passage import - direct chunking, no conversion needed
|
||||
passageChunks := make([]types.ParsedChunk, 0, len(payload.Passages))
|
||||
|
||||
Reference in New Issue
Block a user