#!/usr/bin/env python3
"""Mechanical web-fetch helper for the /research skill.

Talks to the local SearXNG (search) and Firecrawl (scrape) services. Stdlib
only — no third-party runtime dependencies. The /research skill (Claude) does
all the thinking; this file only fetches and parses.

Endpoints resolve from env (WEBTOOLS_HOST / SEARXNG_PORT / FIRECRAWL_PORT) with
defaults for the `webtools` box. Any failure exits nonzero with a stderr message
so the calling skill can react.
"""

from __future__ import annotations

import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path


class WebToolsError(Exception):
    """Raised on any unrecoverable search/scrape failure."""


def endpoints() -> tuple[str, str]:
    """Return (searxng_base, firecrawl_base) from env, with webtools defaults."""
    host = os.environ.get("WEBTOOLS_HOST", "10.10.20.37")
    searx_port = os.environ.get("SEARXNG_PORT", "8080")
    fire_port = os.environ.get("FIRECRAWL_PORT", "3002")
    return (f"http://{host}:{searx_port}", f"http://{host}:{fire_port}")


def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5):
    """GET (data=None) or POST JSON; parse JSON; retry transient errors."""
    headers = dict(headers or {})
    body = None
    if data is not None:
        body = json.dumps(data).encode("utf-8")
        headers.setdefault("Content-Type", "application/json")

    last = None
    for attempt in range(retries + 1):
        try:
            req = urllib.request.Request(
                url, data=body, headers=headers,
                method="POST" if body is not None else "GET",
            )
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except (urllib.error.URLError, TimeoutError, ValueError) as exc:
            last = exc
            if attempt < retries:
                time.sleep(backoff ** attempt)
    raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")