feat: web.py endpoint config + retrying HTTP helper

web.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Mechanical web-fetch helper for the /research skill.
+
+Talks to the local SearXNG (search) and Firecrawl (scrape) services. Stdlib
+only — no third-party runtime dependencies. The /research skill (Claude) does
+all the thinking; this file only fetches and parses.
+
+Endpoints resolve from env (WEBTOOLS_HOST / SEARXNG_PORT / FIRECRAWL_PORT) with
+defaults for the `webtools` box. Any failure exits nonzero with a stderr message
+so the calling skill can react.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+
+class WebToolsError(Exception):
+    """Raised on any unrecoverable search/scrape failure."""
+
+
+def endpoints() -> tuple[str, str]:
+    """Return (searxng_base, firecrawl_base) from env, with webtools defaults."""
+    host = os.environ.get("WEBTOOLS_HOST", "10.10.20.37")
+    searx_port = os.environ.get("SEARXNG_PORT", "8080")
+    fire_port = os.environ.get("FIRECRAWL_PORT", "3002")
+    return (f"http://{host}:{searx_port}", f"http://{host}:{fire_port}")
+
+
+def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5):
+    """GET (data=None) or POST JSON; parse JSON; retry transient errors."""
+    headers = dict(headers or {})
+    body = None
+    if data is not None:
+        body = json.dumps(data).encode("utf-8")
+        headers.setdefault("Content-Type", "application/json")
+
+    last = None
+    for attempt in range(retries + 1):
+        try:
+            req = urllib.request.Request(
+                url, data=body, headers=headers,
+                method="POST" if body is not None else "GET",
+            )
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except (urllib.error.URLError, TimeoutError, ValueError) as exc:
+            last = exc
+            if attempt < retries:
+                time.sleep(backoff ** attempt)
+    raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")