feat: web.py search() over SearXNG json API
This commit is contained in:
@@ -68,3 +68,34 @@ def test_request_raises_after_exhausting_retries(monkeypatch):
|
||||
monkeypatch.setattr(web.time, "sleep", lambda *_: None)
|
||||
with pytest.raises(web.WebToolsError):
|
||||
web._request("http://x/y", retries=2)
|
||||
|
||||
|
||||
def test_search_parses_and_caps(monkeypatch):
|
||||
payload = {"results": [
|
||||
{"title": "A", "url": "http://a", "content": "snip a", "engine": "ddg"},
|
||||
{"title": "B", "url": "http://b", "content": "snip b", "engine": "bing"},
|
||||
{"title": "no-url", "content": "dropped"},
|
||||
{"title": "C", "url": "http://c", "content": "snip c", "engine": "ddg"},
|
||||
]}
|
||||
monkeypatch.setattr(web, "_request", lambda url, **kw: payload)
|
||||
hits = web.search("demons", max_results=2)
|
||||
assert hits == [
|
||||
{"title": "A", "url": "http://a", "snippet": "snip a", "engine": "ddg"},
|
||||
{"title": "B", "url": "http://b", "snippet": "snip b", "engine": "bing"},
|
||||
]
|
||||
|
||||
|
||||
def test_search_builds_json_query_url(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_request(url, **kw):
|
||||
captured["url"] = url
|
||||
return {"results": []}
|
||||
|
||||
monkeypatch.setattr(web, "_request", fake_request)
|
||||
monkeypatch.setenv("WEBTOOLS_HOST", "10.10.20.37")
|
||||
monkeypatch.delenv("SEARXNG_PORT", raising=False)
|
||||
web.search("demon hierarchy")
|
||||
assert captured["url"].startswith("http://10.10.20.37:8080/search?")
|
||||
assert "q=demon+hierarchy" in captured["url"]
|
||||
assert "format=json" in captured["url"]
|
||||
|
||||
19
web.py
19
web.py
@@ -57,3 +57,22 @@ def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5
|
||||
if attempt < retries:
|
||||
time.sleep(backoff ** attempt)
|
||||
raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")
|
||||
|
||||
|
||||
def search(query, max_results=12, timeout=30):
|
||||
"""Query SearXNG and return up to max_results normalized hits."""
|
||||
searx_base, _ = endpoints()
|
||||
qs = urllib.parse.urlencode({"q": query, "format": "json"})
|
||||
payload = _request(f"{searx_base}/search?{qs}", timeout=timeout)
|
||||
hits = []
|
||||
for r in payload.get("results", []):
|
||||
url = r.get("url")
|
||||
if not url:
|
||||
continue
|
||||
hits.append({
|
||||
"title": r.get("title", ""),
|
||||
"url": url,
|
||||
"snippet": r.get("content", ""),
|
||||
"engine": r.get("engine", ""),
|
||||
})
|
||||
return hits[:max_results]
|
||||
|
||||
Reference in New Issue
Block a user