From c4f5db7e88b00dcc2b9eae3f4c3b8b4582b28d53 Mon Sep 17 00:00:00 2001
From: wizardchen <wizardchen@tencent.com>
Date: Wed, 1 Apr 2026 14:35:31 +0800
Subject: [PATCH] feat(metadata): enhance document processing to include
 metadata extraction and handling

- Updated DocReaderServicer to pass metadata in responses.
- Modified PipelineParser to accumulate and merge metadata from all parsers.
- Enhanced StdWebParser to extract and log the title from web page content.
- Implemented logic in knowledge service to update knowledge title based on extracted metadata.
---
 docreader/main.py                         |  1 +
 docreader/parser/chain_parser.py          |  9 ++++++---
 docreader/parser/web_parser.py            | 14 +++++++++++++-
 internal/application/service/knowledge.go | 12 ++++++++++++
 4 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/docreader/main.py b/docreader/main.py
index 6972953e..1ea26752 100644
--- a/docreader/main.py
+++ b/docreader/main.py
@@ -151,6 +151,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
                     markdown_content=_c(result.content),
                     image_refs=image_refs,
                     image_dir_path=image_dir,
+                    metadata={k: _c(str(v)) for k, v in result.metadata.items()} if result.metadata else {},
                 )
                 logger.info(
                     "Read response: content_len=%d, images=%d",
diff --git a/docreader/parser/chain_parser.py b/docreader/parser/chain_parser.py
index 6e5698d0..499c3627 100644
--- a/docreader/parser/chain_parser.py
+++ b/docreader/parser/chain_parser.py
@@ -132,8 +132,9 @@ class PipelineParser(BaseParser):
             Document: Final document after processing through all parsers,
                      with accumulated images from all stages
         """
-        # Accumulate images from all parsers
+        # Accumulate images and metadata from all parsers
         images: Dict[str, str] = {}
+        metadata: Dict = {}
         document = Document()
         for p in self._parsers:
             logger.info(f"PipelineParser: using parser {p.__class__.__name__}")
@@ -141,10 +142,12 @@ class PipelineParser(BaseParser):
             document = p.parse_into_text(content)
             # Convert document content back to bytes for next parser
             content = endecode.encode_bytes(document.content)
-            # Accumulate images from this parser
+            # Accumulate images and metadata from this parser
             images.update(document.images)
-        # Merge all accumulated images into final document
+            metadata.update(document.metadata)
+        # Merge all accumulated images and metadata into final document
         document.images.update(images)
+        document.metadata.update(metadata)
         return document
 
     @classmethod
diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py
index 100399d6..0618647c 100644
--- a/docreader/parser/web_parser.py
+++ b/docreader/parser/web_parser.py
@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import re
 
 from playwright.async_api import async_playwright
 from trafilatura import extract
@@ -110,7 +111,18 @@ class StdWebParser(BaseParser):
         if not md_text:
             logger.error("Failed to parse web page")
             return Document(content=f"Error parsing web page: {url}")
-        return Document(content=md_text)
+
+        # Extract title from trafilatura metadata output (e.g. "title: xxx" line)
+        metadata = {}
+        title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
+        if title_match:
+            extracted_title = title_match.group(1).strip()
+            if extracted_title:
+                metadata["title"] = extracted_title
+                logger.info(f"Extracted article title from trafilatura: {extracted_title}")
+        else:
+            logger.info(f"No title found in trafilatura output, first 200 chars: {md_text[:200]!r}")
+        return Document(content=md_text, metadata=metadata)
 
 
 class WebParser(PipelineParser):
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go
index 3b738f15..1976f3f8 100644
--- a/internal/application/service/knowledge.go
+++ b/internal/application/service/knowledge.go
@@ -7529,6 +7529,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
 		if convertResult == nil {
 			return nil
 		}
+		// Update knowledge title from extracted page title if not already set
+		if knowledge.Title == "" || knowledge.Title == payload.URL {
+			if extractedTitle := convertResult.Metadata["title"]; extractedTitle != "" {
+				knowledge.Title = extractedTitle
+				knowledge.UpdatedAt = time.Now()
+				if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
+					logger.Warnf(ctx, "Failed to update knowledge title from extracted page title: %v", err)
+				} else {
+					logger.Infof(ctx, "Updated knowledge title to extracted page title: %s", extractedTitle)
+				}
+			}
+		}
 	} else if len(payload.Passages) > 0 {
 		// Text passage import - direct chunking, no conversion needed
 		passageChunks := make([]types.ParsedChunk, 0, len(payload.Passages))