diff --git a/tools/browse_wikipedia.py b/tools/browse_wikipedia.py
index bd530c9..f10a328 100644
--- a/tools/browse_wikipedia.py
+++ b/tools/browse_wikipedia.py
@@ -12,6 +12,20 @@ def strip_html(text: str) -> str:
"""Removes HTML tags from a string using regex to provide clean text to the LLM."""
return re.sub(r'<[^>]*>', '', text)
+def clean_wikitext(text: str) -> str:
+ """
+ Removes the most distracting elements of raw Wikitext:
+ 1. HTML comments ()
+ 2. Citations ( and [...])
+ """
+ # Remove HTML comments
+ text = re.sub(r'', '', text, flags=re.DOTALL)
+ # Remove self-closing citations FIRST to prevent them being seen as opening tags
+ text = re.sub(r'[]*/>', '', text)
+ # Remove paired citations
+ text = re.sub(r'][]*>.*?]', '', text, flags=re.DOTALL)
+ return text
+
async def _make_request(params: Dict[str, Any]) -> Dict[str, Any]:
"""
Asynchronous helper to make the API request using httpx.
@@ -92,7 +106,8 @@ async def _fetch_content(title: str, section_index: int) -> Optional[str]:
if not revisions:
return None
- return revisions[0].get("*")
+ content = revisions[0].get("*")
+ return clean_wikitext(content) if content else None
async def _fetch_toc(title: str) -> Optional[str]:
"""