feat: web.py scrape() via Firecrawl

This commit is contained in:
2026-06-28 20:54:57 -05:00
parent 52707d4703
commit fb68c34cb6
2 changed files with 50 additions and 0 deletions

16
web.py
View File

@@ -76,3 +76,19 @@ def search(query, max_results=12, timeout=30):
"engine": r.get("engine", ""),
})
return hits[:max_results]
def scrape(url, timeout=60):
"""Scrape a URL to markdown via Firecrawl. Raises WebToolsError on failure."""
_, fire_base = endpoints()
payload = _request(
f"{fire_base}/v1/scrape",
data={"url": url, "formats": ["markdown"]},
timeout=timeout,
)
if not payload.get("success", True):
raise WebToolsError(f"firecrawl failed for {url}: {payload.get('error')}")
markdown = payload.get("data", {}).get("markdown", "")
if not markdown:
raise WebToolsError(f"firecrawl returned no markdown for {url}")
return markdown