mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-19 21:27:40 +03:00
29b4af6b98
Knowledge searches wrote the names and sources of the files they found to the server log at info level, and the SerpApi, Serply, Firecrawl, Yandex, Kagi, Mojeek and external web search engines wrote their whole results there, snippets included. Anyone who can read the logs could see what users looked up and what the model was given. These lines now log at debug, the same as the search queries, so they only show with GLOBAL_LOG_LEVEL=DEBUG. Fixes #34129
67 lines
1.9 KiB
Python
67 lines
1.9 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from urllib.parse import urlencode
|
|
|
|
import requests
|
|
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def search_serply(
|
|
api_key: str,
|
|
query: str,
|
|
count: int,
|
|
hl: str = 'us',
|
|
limit: int = 10,
|
|
device_type: str = 'desktop',
|
|
proxy_location: str = 'US',
|
|
filter_list: list[str] | None = None,
|
|
) -> list[SearchResult]:
|
|
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
|
|
|
|
Args:
|
|
api_key (str): A serply.io API key
|
|
query (str): The query to search for
|
|
hl (str): Host Language code to display results in (reference https://developers.google.com/custom-search/docs/xml_results?hl=en#wsInterfaceLanguages)
|
|
limit (int): The maximum number of results to return [10-100, defaults to 10]
|
|
"""
|
|
log.info('Searching with Serply')
|
|
|
|
url = 'https://api.serply.io/v1/search/'
|
|
|
|
query_payload = {
|
|
'q': query,
|
|
'language': 'en',
|
|
'num': limit,
|
|
'gl': proxy_location.upper(),
|
|
'hl': hl.lower(),
|
|
}
|
|
|
|
url = f'{url}{urlencode(query_payload)}'
|
|
headers = {
|
|
'X-API-KEY': api_key,
|
|
'X-User-Agent': device_type,
|
|
'User-Agent': 'open-webui',
|
|
'X-Proxy-Location': proxy_location,
|
|
}
|
|
|
|
response = requests.request('GET', url, headers=headers)
|
|
response.raise_for_status()
|
|
|
|
json_response = response.json()
|
|
log.debug('results from serply search: %s', json_response)
|
|
|
|
results = sorted(json_response.get('results', []), key=lambda x: x.get('realPosition', 0))
|
|
if filter_list:
|
|
results = get_filtered_results(results, filter_list)
|
|
return [
|
|
SearchResult(
|
|
link=result['link'],
|
|
title=result.get('title'),
|
|
snippet=result.get('description'),
|
|
)
|
|
for result in results[:count]
|
|
]
|