feat: web.py CLI (search/scrape) with nonzero exit on failure

This commit is contained in:
2026-06-28 20:57:44 -05:00
parent fb68c34cb6
commit 8ef9e17904
2 changed files with 79 additions and 0 deletions

43
web.py
View File

@@ -92,3 +92,46 @@ def scrape(url, timeout=60):
if not markdown:
raise WebToolsError(f"firecrawl returned no markdown for {url}")
return markdown
def main(argv=None):
parser = argparse.ArgumentParser(
prog="web.py",
description="Search (SearXNG) and scrape (Firecrawl) on the webtools box.",
)
sub = parser.add_subparsers(dest="cmd", required=True)
sp = sub.add_parser("search", help="search SearXNG")
sp.add_argument("query")
sp.add_argument("--max", type=int, default=12, help="max results (default 12)")
sp.add_argument("--json", action="store_true", help="emit JSON")
cp = sub.add_parser("scrape", help="scrape a URL to markdown")
cp.add_argument("url")
cp.add_argument("--out", help="write markdown to this path instead of stdout")
args = parser.parse_args(argv)
try:
if args.cmd == "search":
hits = search(args.query, max_results=args.max)
if args.json:
print(json.dumps(hits, indent=2))
else:
for h in hits:
print(f"{h['title']}\n {h['url']}\n {h['snippet'][:160]}\n")
elif args.cmd == "scrape":
markdown = scrape(args.url)
if args.out:
Path(args.out).write_text(markdown, encoding="utf-8")
print(f"wrote {args.out}")
else:
print(markdown)
except WebToolsError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())