diff --git a/tests/test_web.py b/tests/test_web.py index 8de2292..15295f4 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -99,3 +99,37 @@ def test_search_builds_json_query_url(monkeypatch): assert captured["url"].startswith("http://10.10.20.37:8080/search?") assert "q=demon+hierarchy" in captured["url"] assert "format=json" in captured["url"] + + +def test_scrape_returns_markdown(monkeypatch): + monkeypatch.setattr(web, "_request", + lambda url, **kw: {"success": True, "data": {"markdown": "# Hi"}}) + assert web.scrape("http://a") == "# Hi" + + +def test_scrape_posts_url_and_format(monkeypatch): + captured = {} + + def fake_request(url, **kw): + captured["url"] = url + captured["data"] = kw.get("data") + return {"success": True, "data": {"markdown": "ok"}} + + monkeypatch.setattr(web, "_request", fake_request) + web.scrape("http://example.com/page") + assert captured["url"].endswith(":3002/v1/scrape") + assert captured["data"] == {"url": "http://example.com/page", "formats": ["markdown"]} + + +def test_scrape_raises_on_failure_flag(monkeypatch): + monkeypatch.setattr(web, "_request", + lambda url, **kw: {"success": False, "error": "blocked"}) + with pytest.raises(web.WebToolsError): + web.scrape("http://a") + + +def test_scrape_raises_on_empty_markdown(monkeypatch): + monkeypatch.setattr(web, "_request", + lambda url, **kw: {"success": True, "data": {"markdown": ""}}) + with pytest.raises(web.WebToolsError): + web.scrape("http://a") diff --git a/web.py b/web.py index 17d7455..3b0c14d 100644 --- a/web.py +++ b/web.py @@ -76,3 +76,19 @@ def search(query, max_results=12, timeout=30): "engine": r.get("engine", ""), }) return hits[:max_results] + + +def scrape(url, timeout=60): + """Scrape a URL to markdown via Firecrawl. Raises WebToolsError on failure.""" + _, fire_base = endpoints() + payload = _request( + f"{fire_base}/v1/scrape", + data={"url": url, "formats": ["markdown"]}, + timeout=timeout, + ) + if not payload.get("success", True): + raise WebToolsError(f"firecrawl failed for {url}: {payload.get('error')}") + markdown = payload.get("data", {}).get("markdown", "") + if not markdown: + raise WebToolsError(f"firecrawl returned no markdown for {url}") + return markdown