feat(metadata): enhance document processing to include metadata extraction and handling
- Updated DocReaderServicer to pass metadata in responses. - Modified PipelineParser to accumulate and merge metadata from all parsers. - Enhanced StdWebParser to extract and log the title from web page content. - Implemented logic in knowledge service to update knowledge title based on extracted metadata.
This commit is contained in:
@@ -151,6 +151,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
markdown_content=_c(result.content),
|
||||
image_refs=image_refs,
|
||||
image_dir_path=image_dir,
|
||||
metadata={k: _c(str(v)) for k, v in result.metadata.items()} if result.metadata else {},
|
||||
)
|
||||
logger.info(
|
||||
"Read response: content_len=%d, images=%d",
|
||||
|
||||
@@ -132,8 +132,9 @@ class PipelineParser(BaseParser):
|
||||
Document: Final document after processing through all parsers,
|
||||
with accumulated images from all stages
|
||||
"""
|
||||
# Accumulate images from all parsers
|
||||
# Accumulate images and metadata from all parsers
|
||||
images: Dict[str, str] = {}
|
||||
metadata: Dict = {}
|
||||
document = Document()
|
||||
for p in self._parsers:
|
||||
logger.info(f"PipelineParser: using parser {p.__class__.__name__}")
|
||||
@@ -141,10 +142,12 @@ class PipelineParser(BaseParser):
|
||||
document = p.parse_into_text(content)
|
||||
# Convert document content back to bytes for next parser
|
||||
content = endecode.encode_bytes(document.content)
|
||||
# Accumulate images from this parser
|
||||
# Accumulate images and metadata from this parser
|
||||
images.update(document.images)
|
||||
# Merge all accumulated images into final document
|
||||
metadata.update(document.metadata)
|
||||
# Merge all accumulated images and metadata into final document
|
||||
document.images.update(images)
|
||||
document.metadata.update(metadata)
|
||||
return document
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from trafilatura import extract
|
||||
@@ -110,7 +111,18 @@ class StdWebParser(BaseParser):
|
||||
if not md_text:
|
||||
logger.error("Failed to parse web page")
|
||||
return Document(content=f"Error parsing web page: {url}")
|
||||
return Document(content=md_text)
|
||||
|
||||
# Extract title from trafilatura metadata output (e.g. "title: xxx" line)
|
||||
metadata = {}
|
||||
title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
|
||||
if title_match:
|
||||
extracted_title = title_match.group(1).strip()
|
||||
if extracted_title:
|
||||
metadata["title"] = extracted_title
|
||||
logger.info(f"Extracted article title from trafilatura: {extracted_title}")
|
||||
else:
|
||||
logger.info(f"No title found in trafilatura output, first 200 chars: {md_text[:200]!r}")
|
||||
return Document(content=md_text, metadata=metadata)
|
||||
|
||||
|
||||
class WebParser(PipelineParser):
|
||||
|
||||
@@ -7529,6 +7529,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
if convertResult == nil {
|
||||
return nil
|
||||
}
|
||||
// Update knowledge title from extracted page title if not already set
|
||||
if knowledge.Title == "" || knowledge.Title == payload.URL {
|
||||
if extractedTitle := convertResult.Metadata["title"]; extractedTitle != "" {
|
||||
knowledge.Title = extractedTitle
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update knowledge title from extracted page title: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Updated knowledge title to extracted page title: %s", extractedTitle)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if len(payload.Passages) > 0 {
|
||||
// Text passage import - direct chunking, no conversion needed
|
||||
passageChunks := make([]types.ParsedChunk, 0, len(payload.Passages))
|
||||
|
||||
Reference in New Issue
Block a user