feat: web.py CLI (search/scrape) with nonzero exit on failure
This commit is contained in:
43
web.py
43
web.py
@@ -92,3 +92,46 @@ def scrape(url, timeout=60):
|
||||
if not markdown:
|
||||
raise WebToolsError(f"firecrawl returned no markdown for {url}")
|
||||
return markdown
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="web.py",
|
||||
description="Search (SearXNG) and scrape (Firecrawl) on the webtools box.",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sp = sub.add_parser("search", help="search SearXNG")
|
||||
sp.add_argument("query")
|
||||
sp.add_argument("--max", type=int, default=12, help="max results (default 12)")
|
||||
sp.add_argument("--json", action="store_true", help="emit JSON")
|
||||
|
||||
cp = sub.add_parser("scrape", help="scrape a URL to markdown")
|
||||
cp.add_argument("url")
|
||||
cp.add_argument("--out", help="write markdown to this path instead of stdout")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.cmd == "search":
|
||||
hits = search(args.query, max_results=args.max)
|
||||
if args.json:
|
||||
print(json.dumps(hits, indent=2))
|
||||
else:
|
||||
for h in hits:
|
||||
print(f"{h['title']}\n {h['url']}\n {h['snippet'][:160]}\n")
|
||||
elif args.cmd == "scrape":
|
||||
markdown = scrape(args.url)
|
||||
if args.out:
|
||||
Path(args.out).write_text(markdown, encoding="utf-8")
|
||||
print(f"wrote {args.out}")
|
||||
else:
|
||||
print(markdown)
|
||||
except WebToolsError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
Reference in New Issue
Block a user