feat(metadata): enhance document processing to include metadata extraction and handling

- Updated DocReaderServicer to pass metadata in responses.
- Modified PipelineParser to accumulate and merge metadata from all parsers.
- Enhanced StdWebParser to extract and log the title from web page content.
- Implemented logic in knowledge service to update knowledge title based on extracted metadata.
This commit is contained in:
wizardchen
2026-04-01 14:35:31 +08:00
committed by lyingbug
parent ce7df624fb
commit c4f5db7e88
4 changed files with 32 additions and 4 deletions
+12
View File
@@ -7529,6 +7529,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
if convertResult == nil {
return nil
}
// Update knowledge title from extracted page title if not already set
if knowledge.Title == "" || knowledge.Title == payload.URL {
if extractedTitle := convertResult.Metadata["title"]; extractedTitle != "" {
knowledge.Title = extractedTitle
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "Failed to update knowledge title from extracted page title: %v", err)
} else {
logger.Infof(ctx, "Updated knowledge title to extracted page title: %s", extractedTitle)
}
}
}
} else if len(payload.Passages) > 0 {
// Text passage import - direct chunking, no conversion needed
passageChunks := make([]types.ParsedChunk, 0, len(payload.Passages))