From 8ef9e17904d5bfd0e641a9b3ea00348cac9e370a Mon Sep 17 00:00:00 2001
From: ptarrant <ptarrant@gmail.com>
Date: Sun, 28 Jun 2026 20:57:44 -0500
Subject: [PATCH] feat: web.py CLI (search/scrape) with nonzero exit on failure

---
 tests/test_web.py | 36 ++++++++++++++++++++++++++++++++++++
 web.py            | 43 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/tests/test_web.py b/tests/test_web.py
index 15295f4..db17d91 100644
--- a/tests/test_web.py
+++ b/tests/test_web.py
@@ -133,3 +133,39 @@ def test_scrape_raises_on_empty_markdown(monkeypatch):
                         lambda url, **kw: {"success": True, "data": {"markdown": ""}})
     with pytest.raises(web.WebToolsError):
         web.scrape("http://a")
+
+
+def test_main_search_json(monkeypatch, capsys):
+    monkeypatch.setattr(web, "search",
+                        lambda q, max_results=12: [{"title": "T", "url": "http://u",
+                                                    "snippet": "s", "engine": "e"}])
+    rc = web.main(["search", "demons", "--max", "5", "--json"])
+    assert rc == 0
+    out = capsys.readouterr().out
+    assert json.loads(out) == [{"title": "T", "url": "http://u",
+                                "snippet": "s", "engine": "e"}]
+
+
+def test_main_scrape_to_stdout(monkeypatch, capsys):
+    monkeypatch.setattr(web, "scrape", lambda url: "# Page")
+    rc = web.main(["scrape", "http://a"])
+    assert rc == 0
+    assert "# Page" in capsys.readouterr().out
+
+
+def test_main_scrape_to_file(monkeypatch, tmp_path):
+    monkeypatch.setattr(web, "scrape", lambda url: "# Saved")
+    out = tmp_path / "p.md"
+    rc = web.main(["scrape", "http://a", "--out", str(out)])
+    assert rc == 0
+    assert out.read_text(encoding="utf-8") == "# Saved"
+
+
+def test_main_returns_1_on_error(monkeypatch, capsys):
+    def boom(*a, **k):
+        raise web.WebToolsError("nope")
+
+    monkeypatch.setattr(web, "search", boom)
+    rc = web.main(["search", "x"])
+    assert rc == 1
+    assert "error:" in capsys.readouterr().err
diff --git a/web.py b/web.py
index 3b0c14d..9f7c98e 100644
--- a/web.py
+++ b/web.py
@@ -92,3 +92,46 @@ def scrape(url, timeout=60):
     if not markdown:
         raise WebToolsError(f"firecrawl returned no markdown for {url}")
     return markdown
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="web.py",
+        description="Search (SearXNG) and scrape (Firecrawl) on the webtools box.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    sp = sub.add_parser("search", help="search SearXNG")
+    sp.add_argument("query")
+    sp.add_argument("--max", type=int, default=12, help="max results (default 12)")
+    sp.add_argument("--json", action="store_true", help="emit JSON")
+
+    cp = sub.add_parser("scrape", help="scrape a URL to markdown")
+    cp.add_argument("url")
+    cp.add_argument("--out", help="write markdown to this path instead of stdout")
+
+    args = parser.parse_args(argv)
+
+    try:
+        if args.cmd == "search":
+            hits = search(args.query, max_results=args.max)
+            if args.json:
+                print(json.dumps(hits, indent=2))
+            else:
+                for h in hits:
+                    print(f"{h['title']}\n  {h['url']}\n  {h['snippet'][:160]}\n")
+        elif args.cmd == "scrape":
+            markdown = scrape(args.url)
+            if args.out:
+                Path(args.out).write_text(markdown, encoding="utf-8")
+                print(f"wrote {args.out}")
+            else:
+                print(markdown)
+    except WebToolsError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())