feat: web.py endpoint config + retrying HTTP helper

requirements.txt

@@ -13,3 +13,6 @@ jsonschema>=4.0
 #   markitdown      # DOCX/PPTX/XLSX/HTML -> md
 #   marker-pdf      # heavier, ML/GPU, OCRs scanned PDFs
 #   docling         # heavier, ML/GPU, messy layouts/tables
+
+# Dev/test tooling (not needed at runtime; web.py is stdlib-only).
+pytest>=8.0

tests/test_web.py

@@ -0,0 +1,70 @@
+import json
+import urllib.error
+
+import pytest
+
+import web
+
+
+class FakeResp:
+    """Minimal stand-in for an http.client.HTTPResponse context manager."""
+
+    def __init__(self, payload):
+        self._b = json.dumps(payload).encode("utf-8")
+
+    def read(self):
+        return self._b
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+def test_endpoints_defaults(monkeypatch):
+    for var in ("WEBTOOLS_HOST", "SEARXNG_PORT", "FIRECRAWL_PORT"):
+        monkeypatch.delenv(var, raising=False)
+    searx, fire = web.endpoints()
+    assert searx == "http://10.10.20.37:8080"
+    assert fire == "http://10.10.20.37:3002"
+
+
+def test_endpoints_env_override(monkeypatch):
+    monkeypatch.setenv("WEBTOOLS_HOST", "192.168.1.5")
+    monkeypatch.setenv("SEARXNG_PORT", "9999")
+    monkeypatch.setenv("FIRECRAWL_PORT", "4000")
+    searx, fire = web.endpoints()
+    assert searx == "http://192.168.1.5:9999"
+    assert fire == "http://192.168.1.5:4000"
+
+
+def test_request_parses_json(monkeypatch):
+    monkeypatch.setattr(web.urllib.request, "urlopen",
+                        lambda req, timeout=None: FakeResp({"ok": True}))
+    assert web._request("http://x/y") == {"ok": True}
+
+
+def test_request_retries_then_succeeds(monkeypatch):
+    calls = {"n": 0}
+
+    def flaky(req, timeout=None):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise urllib.error.URLError("boom")
+        return FakeResp({"ok": True})
+
+    monkeypatch.setattr(web.urllib.request, "urlopen", flaky)
+    monkeypatch.setattr(web.time, "sleep", lambda *_: None)
+    assert web._request("http://x/y", retries=2) == {"ok": True}
+    assert calls["n"] == 3
+
+
+def test_request_raises_after_exhausting_retries(monkeypatch):
+    def always_fail(req, timeout=None):
+        raise urllib.error.URLError("down")
+
+    monkeypatch.setattr(web.urllib.request, "urlopen", always_fail)
+    monkeypatch.setattr(web.time, "sleep", lambda *_: None)
+    with pytest.raises(web.WebToolsError):
+        web._request("http://x/y", retries=2)

web.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Mechanical web-fetch helper for the /research skill.
+
+Talks to the local SearXNG (search) and Firecrawl (scrape) services. Stdlib
+only — no third-party runtime dependencies. The /research skill (Claude) does
+all the thinking; this file only fetches and parses.
+
+Endpoints resolve from env (WEBTOOLS_HOST / SEARXNG_PORT / FIRECRAWL_PORT) with
+defaults for the `webtools` box. Any failure exits nonzero with a stderr message
+so the calling skill can react.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+
+class WebToolsError(Exception):
+    """Raised on any unrecoverable search/scrape failure."""
+
+
+def endpoints() -> tuple[str, str]:
+    """Return (searxng_base, firecrawl_base) from env, with webtools defaults."""
+    host = os.environ.get("WEBTOOLS_HOST", "10.10.20.37")
+    searx_port = os.environ.get("SEARXNG_PORT", "8080")
+    fire_port = os.environ.get("FIRECRAWL_PORT", "3002")
+    return (f"http://{host}:{searx_port}", f"http://{host}:{fire_port}")
+
+
+def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5):
+    """GET (data=None) or POST JSON; parse JSON; retry transient errors."""
+    headers = dict(headers or {})
+    body = None
+    if data is not None:
+        body = json.dumps(data).encode("utf-8")
+        headers.setdefault("Content-Type", "application/json")
+
+    last = None
+    for attempt in range(retries + 1):
+        try:
+            req = urllib.request.Request(
+                url, data=body, headers=headers,
+                method="POST" if body is not None else "GET",
+            )
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except (urllib.error.URLError, TimeoutError, ValueError) as exc:
+            last = exc
+            if attempt < retries:
+                time.sleep(backoff ** attempt)
+    raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")