feat: web.py scrape() via Firecrawl

This commit is contained in:
2026-06-28 20:54:57 -05:00
parent 52707d4703
commit fb68c34cb6
2 changed files with 50 additions and 0 deletions

View File

@@ -99,3 +99,37 @@ def test_search_builds_json_query_url(monkeypatch):
assert captured["url"].startswith("http://10.10.20.37:8080/search?")
assert "q=demon+hierarchy" in captured["url"]
assert "format=json" in captured["url"]
def test_scrape_returns_markdown(monkeypatch):
monkeypatch.setattr(web, "_request",
lambda url, **kw: {"success": True, "data": {"markdown": "# Hi"}})
assert web.scrape("http://a") == "# Hi"
def test_scrape_posts_url_and_format(monkeypatch):
captured = {}
def fake_request(url, **kw):
captured["url"] = url
captured["data"] = kw.get("data")
return {"success": True, "data": {"markdown": "ok"}}
monkeypatch.setattr(web, "_request", fake_request)
web.scrape("http://example.com/page")
assert captured["url"].endswith(":3002/v1/scrape")
assert captured["data"] == {"url": "http://example.com/page", "formats": ["markdown"]}
def test_scrape_raises_on_failure_flag(monkeypatch):
monkeypatch.setattr(web, "_request",
lambda url, **kw: {"success": False, "error": "blocked"})
with pytest.raises(web.WebToolsError):
web.scrape("http://a")
def test_scrape_raises_on_empty_markdown(monkeypatch):
monkeypatch.setattr(web, "_request",
lambda url, **kw: {"success": True, "data": {"markdown": ""}})
with pytest.raises(web.WebToolsError):
web.scrape("http://a")