Wikipedia text cleanup

tools/browse_wikipedia.py

@@ -12,6 +12,20 @@ def strip_html(text: str) -> str:
     """Removes HTML tags from a string using regex to provide clean text to the LLM."""
     return re.sub(r'<[^>]*>', '', text)
 
+def clean_wikitext(text: str) -> str:
+    """
+    Removes the most distracting elements of raw Wikitext:
+    1. HTML comments (<!-- ... -->)
+    2. Citations (<ref /> and <ref>...</ref>)
+    """
+    # Remove HTML comments
+    text = re.sub(r'<!--.*?-->', '', text, flags=re.DOTALL)
+    # Remove self-closing citations FIRST to prevent them being seen as opening tags
+    text = re.sub(r'<ref[^>]*/>', '', text)
+    # Remove paired citations
+    text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
+    return text
+
 async def _make_request(params: Dict[str, Any]) -> Dict[str, Any]:
     """
     Asynchronous helper to make the API request using httpx.
@@ -92,7 +106,8 @@ async def _fetch_content(title: str, section_index: int) -> Optional[str]:
     if not revisions:
         return None
         
-    return revisions[0].get("*")
+    content = revisions[0].get("*")
+    return clean_wikitext(content) if content else None
 
 async def _fetch_toc(title: str) -> Optional[str]:
     """