feat: web.py scrape() via Firecrawl
This commit is contained in:
@@ -99,3 +99,37 @@ def test_search_builds_json_query_url(monkeypatch):
|
||||
assert captured["url"].startswith("http://10.10.20.37:8080/search?")
|
||||
assert "q=demon+hierarchy" in captured["url"]
|
||||
assert "format=json" in captured["url"]
|
||||
|
||||
|
||||
def test_scrape_returns_markdown(monkeypatch):
|
||||
monkeypatch.setattr(web, "_request",
|
||||
lambda url, **kw: {"success": True, "data": {"markdown": "# Hi"}})
|
||||
assert web.scrape("http://a") == "# Hi"
|
||||
|
||||
|
||||
def test_scrape_posts_url_and_format(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_request(url, **kw):
|
||||
captured["url"] = url
|
||||
captured["data"] = kw.get("data")
|
||||
return {"success": True, "data": {"markdown": "ok"}}
|
||||
|
||||
monkeypatch.setattr(web, "_request", fake_request)
|
||||
web.scrape("http://example.com/page")
|
||||
assert captured["url"].endswith(":3002/v1/scrape")
|
||||
assert captured["data"] == {"url": "http://example.com/page", "formats": ["markdown"]}
|
||||
|
||||
|
||||
def test_scrape_raises_on_failure_flag(monkeypatch):
|
||||
monkeypatch.setattr(web, "_request",
|
||||
lambda url, **kw: {"success": False, "error": "blocked"})
|
||||
with pytest.raises(web.WebToolsError):
|
||||
web.scrape("http://a")
|
||||
|
||||
|
||||
def test_scrape_raises_on_empty_markdown(monkeypatch):
|
||||
monkeypatch.setattr(web, "_request",
|
||||
lambda url, **kw: {"success": True, "data": {"markdown": ""}})
|
||||
with pytest.raises(web.WebToolsError):
|
||||
web.scrape("http://a")
|
||||
|
||||
Reference in New Issue
Block a user