feat: web.py search() over SearXNG json API
This commit is contained in:
@@ -68,3 +68,34 @@ def test_request_raises_after_exhausting_retries(monkeypatch):
|
|||||||
monkeypatch.setattr(web.time, "sleep", lambda *_: None)
|
monkeypatch.setattr(web.time, "sleep", lambda *_: None)
|
||||||
with pytest.raises(web.WebToolsError):
|
with pytest.raises(web.WebToolsError):
|
||||||
web._request("http://x/y", retries=2)
|
web._request("http://x/y", retries=2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_parses_and_caps(monkeypatch):
|
||||||
|
payload = {"results": [
|
||||||
|
{"title": "A", "url": "http://a", "content": "snip a", "engine": "ddg"},
|
||||||
|
{"title": "B", "url": "http://b", "content": "snip b", "engine": "bing"},
|
||||||
|
{"title": "no-url", "content": "dropped"},
|
||||||
|
{"title": "C", "url": "http://c", "content": "snip c", "engine": "ddg"},
|
||||||
|
]}
|
||||||
|
monkeypatch.setattr(web, "_request", lambda url, **kw: payload)
|
||||||
|
hits = web.search("demons", max_results=2)
|
||||||
|
assert hits == [
|
||||||
|
{"title": "A", "url": "http://a", "snippet": "snip a", "engine": "ddg"},
|
||||||
|
{"title": "B", "url": "http://b", "snippet": "snip b", "engine": "bing"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_builds_json_query_url(monkeypatch):
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def fake_request(url, **kw):
|
||||||
|
captured["url"] = url
|
||||||
|
return {"results": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(web, "_request", fake_request)
|
||||||
|
monkeypatch.setenv("WEBTOOLS_HOST", "10.10.20.37")
|
||||||
|
monkeypatch.delenv("SEARXNG_PORT", raising=False)
|
||||||
|
web.search("demon hierarchy")
|
||||||
|
assert captured["url"].startswith("http://10.10.20.37:8080/search?")
|
||||||
|
assert "q=demon+hierarchy" in captured["url"]
|
||||||
|
assert "format=json" in captured["url"]
|
||||||
|
|||||||
19
web.py
19
web.py
@@ -57,3 +57,22 @@ def _request(url, *, data=None, headers=None, timeout=30, retries=2, backoff=1.5
|
|||||||
if attempt < retries:
|
if attempt < retries:
|
||||||
time.sleep(backoff ** attempt)
|
time.sleep(backoff ** attempt)
|
||||||
raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")
|
raise WebToolsError(f"request failed after {retries + 1} attempt(s): {url}: {last}")
|
||||||
|
|
||||||
|
|
||||||
|
def search(query, max_results=12, timeout=30):
|
||||||
|
"""Query SearXNG and return up to max_results normalized hits."""
|
||||||
|
searx_base, _ = endpoints()
|
||||||
|
qs = urllib.parse.urlencode({"q": query, "format": "json"})
|
||||||
|
payload = _request(f"{searx_base}/search?{qs}", timeout=timeout)
|
||||||
|
hits = []
|
||||||
|
for r in payload.get("results", []):
|
||||||
|
url = r.get("url")
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
hits.append({
|
||||||
|
"title": r.get("title", ""),
|
||||||
|
"url": url,
|
||||||
|
"snippet": r.get("content", ""),
|
||||||
|
"engine": r.get("engine", ""),
|
||||||
|
})
|
||||||
|
return hits[:max_results]
|
||||||
|
|||||||
Reference in New Issue
Block a user