User agent, readme

tools/browse_wikipedia.py

@@ -1,5 +1,4 @@
-import urllib.request
-import urllib.parse
+import httpx
 import json
 import re
 import asyncio
@@ -13,22 +12,19 @@ def strip_html(text: str) -> str:
     """Removes HTML tags from a string using regex to provide clean text to the LLM."""
     return re.sub(r'<[^>]*>', '', text)
 
-def _make_request(params: Dict[str, Any]) -> Dict[str, Any]:
+async def _make_request(params: Dict[str, Any]) -> Dict[str, Any]:
     """
-    Synchronous helper to make the API request using urllib.
-    urllib is used instead of httpx to avoid TLS/HTTP fingerprinting 
-    that triggers 403 Forbidden responses from Wikipedia.
+    Asynchronous helper to make the API request using httpx.
+    A custom User-Agent is used to avoid bot detection.
     """
-    query_string = urllib.parse.urlencode(params)
-    url = f"{API_URL}?{query_string}"
-    
     headers = {
         "User-Agent": config.USER_AGENT
     }
     
-    req = urllib.request.Request(url, headers=headers)
-    with urllib.request.urlopen(req) as response:
-        return json.loads(response.read().decode('utf-8'))
+    async with httpx.AsyncClient(headers=headers, timeout=15.0) as client:
+        response = await client.get(API_URL, params=params)
+        response.raise_for_status()
+        return response.json()
 
 async def _search(title: str, limit: int = 5, fallback: bool = False) -> str:
     """
@@ -42,8 +38,8 @@ async def _search(title: str, limit: int = 5, fallback: bool = False) -> str:
         "format": "json",
         "srlimit": limit
     }
-    # Run synchronous urllib call in a thread to avoid blocking the event loop
-    data = await asyncio.to_thread(_make_request, params)
+    # Directly await the async request
+    data = await _make_request(params)
     
     search_results = data.get("query", {}).get("search", [])
     if not search_results:
@@ -80,7 +76,7 @@ async def _fetch_content(title: str, section_index: int) -> Optional[str]:
         "redirects": 1,
         "format": "json"
     }
-    data = await asyncio.to_thread(_make_request, params)
+    data = await _make_request(params)
     
     pages = data.get("query", {}).get("pages", {})
     if not pages:
@@ -109,7 +105,7 @@ async def _fetch_toc(title: str) -> Optional[str]:
         "format": "json",
         "redirects": 1
     }
-    data = await asyncio.to_thread(_make_request, params)
+    data = await _make_request(params)
     
     parse_data = data.get("parse")
     if not parse_data: