From 8c18d0d40e7ce0de851825838533033f92a2f49e Mon Sep 17 00:00:00 2001
From: moosecrap <admin@long-cat.net>
Date: Mon, 3 Aug 2026 16:27:59 -0700
Subject: [PATCH] Wikipedia text cleanup

---
 tools/browse_wikipedia.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tools/browse_wikipedia.py b/tools/browse_wikipedia.py
index bd530c9..f10a328 100644
--- a/tools/browse_wikipedia.py
+++ b/tools/browse_wikipedia.py
@@ -12,6 +12,20 @@ def strip_html(text: str) -> str:
     """Removes HTML tags from a string using regex to provide clean text to the LLM."""
     return re.sub(r'<[^>]*>', '', text)
 
+def clean_wikitext(text: str) -> str:
+    """
+    Removes the most distracting elements of raw Wikitext:
+    1. HTML comments (<!-- ... -->)
+    2. Citations (<ref /> and <ref>...</ref>)
+    """
+    # Remove HTML comments
+    text = re.sub(r'<!--.*?-->', '', text, flags=re.DOTALL)
+    # Remove self-closing citations FIRST to prevent them being seen as opening tags
+    text = re.sub(r'<ref[^>]*/>', '', text)
+    # Remove paired citations
+    text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
+    return text
+
 async def _make_request(params: Dict[str, Any]) -> Dict[str, Any]:
     """
     Asynchronous helper to make the API request using httpx.
@@ -92,7 +106,8 @@ async def _fetch_content(title: str, section_index: int) -> Optional[str]:
     if not revisions:
         return None
         
-    return revisions[0].get("*")
+    content = revisions[0].get("*")
+    return clean_wikitext(content) if content else None
 
 async def _fetch_toc(title: str) -> Optional[str]:
     """