feat: web.py scrape() via Firecrawl

tests/test_web.py

@@ -99,3 +99,37 @@ def test_search_builds_json_query_url(monkeypatch):
     assert captured["url"].startswith("http://10.10.20.37:8080/search?")
     assert "q=demon+hierarchy" in captured["url"]
     assert "format=json" in captured["url"]
+
+
+def test_scrape_returns_markdown(monkeypatch):
+    monkeypatch.setattr(web, "_request",
+                        lambda url, **kw: {"success": True, "data": {"markdown": "# Hi"}})
+    assert web.scrape("http://a") == "# Hi"
+
+
+def test_scrape_posts_url_and_format(monkeypatch):
+    captured = {}
+
+    def fake_request(url, **kw):
+        captured["url"] = url
+        captured["data"] = kw.get("data")
+        return {"success": True, "data": {"markdown": "ok"}}
+
+    monkeypatch.setattr(web, "_request", fake_request)
+    web.scrape("http://example.com/page")
+    assert captured["url"].endswith(":3002/v1/scrape")
+    assert captured["data"] == {"url": "http://example.com/page", "formats": ["markdown"]}
+
+
+def test_scrape_raises_on_failure_flag(monkeypatch):
+    monkeypatch.setattr(web, "_request",
+                        lambda url, **kw: {"success": False, "error": "blocked"})
+    with pytest.raises(web.WebToolsError):
+        web.scrape("http://a")
+
+
+def test_scrape_raises_on_empty_markdown(monkeypatch):
+    monkeypatch.setattr(web, "_request",
+                        lambda url, **kw: {"success": True, "data": {"markdown": ""}})
+    with pytest.raises(web.WebToolsError):
+        web.scrape("http://a")

web.py

@@ -76,3 +76,19 @@ def search(query, max_results=12, timeout=30):
             "engine": r.get("engine", ""),
         })
     return hits[:max_results]
+
+
+def scrape(url, timeout=60):
+    """Scrape a URL to markdown via Firecrawl. Raises WebToolsError on failure."""
+    _, fire_base = endpoints()
+    payload = _request(
+        f"{fire_base}/v1/scrape",
+        data={"url": url, "formats": ["markdown"]},
+        timeout=timeout,
+    )
+    if not payload.get("success", True):
+        raise WebToolsError(f"firecrawl failed for {url}: {payload.get('error')}")
+    markdown = payload.get("data", {}).get("markdown", "")
+    if not markdown:
+        raise WebToolsError(f"firecrawl returned no markdown for {url}")
+    return markdown