diff --git a/config/prompt_templates/generate_questions.yaml b/config/prompt_templates/generate_questions.yaml index f7a9f899..b20b8c99 100644 --- a/config/prompt_templates/generate_questions.yaml +++ b/config/prompt_templates/generate_questions.yaml @@ -6,32 +6,37 @@ templates: description: "Generate related questions from document chunks to improve retrieval recall" default: true content: | - You are a professional question generation assistant. Your task is to generate related questions that users might ask based on the given [Main Content]. - + You are a question generation assistant optimizing for search retrieval. Your goal is to generate the questions that can BEST answer — the questions a user would ask when they truly need this information. + Note: (if present) is only for helping you understand better. Generate questions ONLY about . + {{context}} - ## Main Content (generate questions based on this content) + Document name: {{doc_name}} - Document content: + {{content}} + - ## Core Requirements - - Generated questions must be directly related to the [Main Content] - - Questions must NOT use any pronouns or referential words (such as "it", "this", "that document", "this article", "the text", "its", etc.); use specific names instead - - Questions must be complete and self-contained, understandable without additional context - - Questions should be natural questions that users would likely ask in real scenarios - - Questions should be diverse, covering different aspects of the content + ## Think Before Generating + First, silently identify: + 1. What is the CORE TOPIC of this content? + 2. What problem or need does this content address? + 3. If a user needed this information, what would they search for? + + ## Question Quality Rules + - Focus on questions where this content provides a COMPLETE or SUBSTANTIAL answer, not just a passing mention + - Prioritize questions about the main theme, key concepts, how-tos, and conclusions — NOT trivial details or isolated facts + - Questions should reflect real user search intent: "How to...", "What is...", "Why does...", "What are the best practices for..." + - Each question must be self-contained: NO pronouns or references (e.g., "it", "this", "that document"); use specific names + - Questions should be at a level where someone would genuinely search for the answer, not quiz-style trivia - Each question should be concise and clear, within 30 words - - Generate {{question_count}} questions - ## Suggested Question Types - - Definition: What is...? What does... mean? - - Reason: Why...? What is the reason for...? - - Method: How to...? What is the way to...? - - Comparison: What is the difference between... and...? - - Application: What scenarios can... be used for? + ## What NOT to Generate + - Do NOT generate questions about minor details that are only briefly mentioned + - Do NOT generate questions that can be answered with a single word or number extracted from the text + - Do NOT generate questions that are too broad to be meaningfully answered by this specific content - ## Output Format - Output the question list directly, one question per line, without numbering or other prefixes. + ## Output + Generate {{question_count}} questions, one per line, no numbering or prefixes. ## CRITICAL: Language Rule - Generate questions in {{language}} diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py index 0618647c..c34bc722 100644 --- a/docreader/parser/web_parser.py +++ b/docreader/parser/web_parser.py @@ -138,16 +138,34 @@ class WebParser(PipelineParser): if __name__ == "__main__": - # Configure logging for debugging - logging.basicConfig(level=logging.DEBUG) - logger.setLevel(logging.DEBUG) + import sys - # Example URL to scrape - url = "https://cloud.tencent.com/document/product/457/6759" + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + + url = sys.argv[1] if len(sys.argv) > 1 else "https://cloud.tencent.com/document/product/457/6759" + print(f"\n{'='*60}") + print(f"URL: {url}") + print(f"{'='*60}\n") - # Create parser instance and parse the web page parser = WebParser(title="") - cc = parser.parse_into_text(url.encode()) - # Save the parsed markdown content to file - with open("./tencent.md", "w") as f: - f.write(cc.content) + doc = parser.parse_into_text(url.encode()) + + print(f"--- metadata ---") + for k, v in doc.metadata.items(): + print(f" {k}: {v}") + + print(f"\n--- images ({len(doc.images)}) ---") + for path in list(doc.images.keys())[:10]: + print(f" {path} ({len(doc.images[path])} chars base64)") + + print(f"\n--- content ({len(doc.content)} chars) ---") + print(doc.content[:300000]) + if len(doc.content) > 300000: + print(f"\n... (truncated, total {len(doc.content)} chars)") + + print(f"\n--- chunks ({len(doc.chunks)}) ---") + for i, chunk in enumerate(doc.chunks[:5]): + print(f" [{i}] seq={chunk.seq} range=[{chunk.start}:{chunk.end}] len={len(chunk.content)}") + print(f" {chunk.content[:120]}{'...' if len(chunk.content) > 120 else ''}") + if len(doc.chunks) > 5: + print(f" ... ({len(doc.chunks) - 5} more chunks)") diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 1976f3f8..8e39fe82 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2369,14 +2369,14 @@ func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context, // Build context section var contextSection string if prevContent != "" || nextContent != "" { - contextSection = "## Context Information (for reference only, to help understand the main content)\n" + contextSection = "\n" if prevContent != "" { - contextSection += fmt.Sprintf("[Preceding Context] %s\n", prevContent) + contextSection += fmt.Sprintf("[Preceding Content]\n%s\n\n", prevContent) } if nextContent != "" { - contextSection += fmt.Sprintf("[Following Context] %s\n", nextContent) + contextSection += fmt.Sprintf("[Following Content]\n%s\n\n", nextContent) } - contextSection += "\n" + contextSection += "\n\n" } langName := types.LanguageNameFromContext(ctx) @@ -7577,6 +7577,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e // Step 2: Store images and update markdown references var storedImages []docparser.StoredImage + + // For URL imports, resolve relative image paths to absolute URLs + // so that ResolveRemoteImages can download them. + if payload.URL != "" && convertResult != nil { + convertResult.MarkdownContent = docparser.ResolveRelativeImageURLs(convertResult.MarkdownContent, payload.URL) + } + if s.imageResolver != nil && convertResult != nil { fileSvc := s.resolveFileService(ctx, kb) tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint64)