fix: fail-fast on non-JSON responses, harden scrape, cover CLI branches

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C7rEog2rS7uKNMb1CvpVjr

web.py

@@ -36,7 +36,13 @@ def endpoints() -> tuple[str, str]:
 
 
 def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5):
-    """GET (data=None) or POST JSON; parse JSON; retry transient errors."""
+    """GET (data=None) or POST JSON; parse JSON; retry transient network errors.
+
+    Network failures (URLError/timeout) are retried with backoff. A non-JSON
+    response is NOT retried — it is almost always a deterministic
+    misconfiguration (e.g. SearXNG json format disabled), so we fail fast with
+    a clear message.
+    """
     headers = dict(headers or {})
     body = None
     if data is not None:
@@ -51,12 +57,21 @@ def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5
                 method="POST" if body is not None else "GET",
             )
             with urllib.request.urlopen(req, timeout=timeout) as resp:
-                return json.loads(resp.read().decode("utf-8"))
-        except (urllib.error.URLError, TimeoutError, ValueError) as exc:
+                raw = resp.read()
+            break
+        except (urllib.error.URLError, TimeoutError) as exc:
             last = exc
             if attempt < retries:
                 time.sleep(backoff ** attempt)
-    raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")
+    else:
+        raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")
+
+    try:
+        return json.loads(raw.decode("utf-8"))
+    except ValueError as exc:
+        raise WebToolsError(
+            f"non-JSON response from {url} (is SearXNG json format enabled?): {exc}"
+        )
 
 
 def search(query, max_results=12, timeout=30):
@@ -88,7 +103,7 @@ def scrape(url, timeout=60):
     )
     if not payload.get("success", True):
         raise WebToolsError(f"firecrawl failed for {url}: {payload.get('error')}")
-    markdown = payload.get("data", {}).get("markdown", "")
+    markdown = (payload.get("data") or {}).get("markdown", "")
     if not markdown:
         raise WebToolsError(f"firecrawl returned no markdown for {url}")
     return markdown