refactor(prompt_templates): improve question generation guidelines and context handling

- Updated the question generation template to clarify the role of surrounding context and main content.
- Enhanced quality rules for generated questions to better align with user search intent.
- Revised output format and added explicit instructions on what not to generate.
- Improved logging and output in the web parser for better visibility of parsed content and metadata.
This commit is contained in:
wizardchen
2026-04-01 21:04:22 +08:00
committed by lyingbug
parent 1b7070c1a2
commit 8e1cfaccb7
3 changed files with 63 additions and 33 deletions
+28 -10
View File
@@ -138,16 +138,34 @@ class WebParser(PipelineParser):
if __name__ == "__main__":
# Configure logging for debugging
logging.basicConfig(level=logging.DEBUG)
logger.setLevel(logging.DEBUG)
import sys
# Example URL to scrape
url = "https://cloud.tencent.com/document/product/457/6759"
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
url = sys.argv[1] if len(sys.argv) > 1 else "https://cloud.tencent.com/document/product/457/6759"
print(f"\n{'='*60}")
print(f"URL: {url}")
print(f"{'='*60}\n")
# Create parser instance and parse the web page
parser = WebParser(title="")
cc = parser.parse_into_text(url.encode())
# Save the parsed markdown content to file
with open("./tencent.md", "w") as f:
f.write(cc.content)
doc = parser.parse_into_text(url.encode())
print(f"--- metadata ---")
for k, v in doc.metadata.items():
print(f" {k}: {v}")
print(f"\n--- images ({len(doc.images)}) ---")
for path in list(doc.images.keys())[:10]:
print(f" {path} ({len(doc.images[path])} chars base64)")
print(f"\n--- content ({len(doc.content)} chars) ---")
print(doc.content[:300000])
if len(doc.content) > 300000:
print(f"\n... (truncated, total {len(doc.content)} chars)")
print(f"\n--- chunks ({len(doc.chunks)}) ---")
for i, chunk in enumerate(doc.chunks[:5]):
print(f" [{i}] seq={chunk.seq} range=[{chunk.start}:{chunk.end}] len={len(chunk.content)}")
print(f" {chunk.content[:120]}{'...' if len(chunk.content) > 120 else ''}")
if len(doc.chunks) > 5:
print(f" ... ({len(doc.chunks) - 5} more chunks)")