feat: web.py scrape() via Firecrawl
This commit is contained in:
16
web.py
16
web.py
@@ -76,3 +76,19 @@ def search(query, max_results=12, timeout=30):
|
||||
"engine": r.get("engine", ""),
|
||||
})
|
||||
return hits[:max_results]
|
||||
|
||||
|
||||
def scrape(url, timeout=60):
|
||||
"""Scrape a URL to markdown via Firecrawl. Raises WebToolsError on failure."""
|
||||
_, fire_base = endpoints()
|
||||
payload = _request(
|
||||
f"{fire_base}/v1/scrape",
|
||||
data={"url": url, "formats": ["markdown"]},
|
||||
timeout=timeout,
|
||||
)
|
||||
if not payload.get("success", True):
|
||||
raise WebToolsError(f"firecrawl failed for {url}: {payload.get('error')}")
|
||||
markdown = payload.get("data", {}).get("markdown", "")
|
||||
if not markdown:
|
||||
raise WebToolsError(f"firecrawl returned no markdown for {url}")
|
||||
return markdown
|
||||
|
||||
Reference in New Issue
Block a user