first commit

searx/network/client.py

@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring, global-statement
+
+import asyncio
+import logging
+import random
+from ssl import SSLContext
+import threading
+from typing import Any, Dict
+
+import httpx
+from httpx_socks import AsyncProxyTransport
+from python_socks import parse_proxy_url, ProxyConnectionError, ProxyTimeoutError, ProxyError
+import uvloop
+
+from searx import logger
+
+
+uvloop.install()
+
+
+logger = logger.getChild('searx.network.client')
+LOOP = None
+SSLCONTEXTS: Dict[Any, SSLContext] = {}
+
+
+def shuffle_ciphers(ssl_context: SSLContext):
+    """Shuffle httpx's default ciphers of a SSL context randomly.
+
+    From `What Is TLS Fingerprint and How to Bypass It`_
+
+    > When implementing TLS fingerprinting, servers can't operate based on a
+    > locked-in whitelist database of fingerprints.  New fingerprints appear
+    > when web clients or TLS libraries release new versions. So, they have to
+    > live off a blocklist database instead.
+    > ...
+    > It's safe to leave the first three as is but shuffle the remaining ciphers
+    > and you can bypass the TLS fingerprint check.
+
+    .. _What Is TLS Fingerprint and How to Bypass It:
+       https://www.zenrows.com/blog/what-is-tls-fingerprint#how-to-bypass-tls-fingerprinting
+
+    """
+    c_list = [cipher["name"] for cipher in ssl_context.get_ciphers()]
+    sc_list, c_list = c_list[:3], c_list[3:]
+    random.shuffle(c_list)
+    ssl_context.set_ciphers(":".join(sc_list + c_list))
+
+
+def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True):
+    key = (proxy_url, cert, verify, trust_env)
+    if key not in SSLCONTEXTS:
+        SSLCONTEXTS[key] = httpx.create_ssl_context(verify, cert, trust_env)
+    shuffle_ciphers(SSLCONTEXTS[key])
+    return SSLCONTEXTS[key]
+
+
+class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
+    """Block HTTP request
+
+    The constructor is blank because httpx.AsyncHTTPTransport.__init__ creates an SSLContext unconditionally:
+    https://github.com/encode/httpx/blob/0f61aa58d66680c239ce43c8cdd453e7dc532bfc/httpx/_transports/default.py#L271
+
+    Each SSLContext consumes more than 500kb of memory, since there is about one network per engine.
+
+    In consequence, this class overrides all public methods
+
+    For reference: https://github.com/encode/httpx/issues/2298
+    """
+
+    def __init__(self, *args, **kwargs):
+        # pylint: disable=super-init-not-called
+        # this on purpose if the base class is not called
+        pass
+
+    async def handle_async_request(self, request):
+        raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
+
+    async def aclose(self) -> None:
+        pass
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type=None,
+        exc_value=None,
+        traceback=None,
+    ) -> None:
+        pass
+
+
+class AsyncProxyTransportFixed(AsyncProxyTransport):
+    """Fix httpx_socks.AsyncProxyTransport
+
+    Map python_socks exceptions to httpx.ProxyError exceptions
+    """
+
+    async def handle_async_request(self, request):
+        try:
+            return await super().handle_async_request(request)
+        except ProxyConnectionError as e:
+            raise httpx.ProxyError("ProxyConnectionError: " + e.strerror, request=request) from e
+        except ProxyTimeoutError as e:
+            raise httpx.ProxyError("ProxyTimeoutError: " + e.args[0], request=request) from e
+        except ProxyError as e:
+            raise httpx.ProxyError("ProxyError: " + e.args[0], request=request) from e
+
+
+def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries):
+    # support socks5h (requests compatibility):
+    # https://requests.readthedocs.io/en/master/user/advanced/#socks
+    # socks5://   hostname is resolved on client side
+    # socks5h://  hostname is resolved on proxy side
+    rdns = False
+    socks5h = 'socks5h://'
+    if proxy_url.startswith(socks5h):
+        proxy_url = 'socks5://' + proxy_url[len(socks5h) :]
+        rdns = True
+
+    proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
+    verify = get_sslcontexts(proxy_url, None, verify, True) if verify is True else verify
+    return AsyncProxyTransportFixed(
+        proxy_type=proxy_type,
+        proxy_host=proxy_host,
+        proxy_port=proxy_port,
+        username=proxy_username,
+        password=proxy_password,
+        rdns=rdns,
+        loop=get_loop(),
+        verify=verify,
+        http2=http2,
+        local_address=local_address,
+        limits=limit,
+        retries=retries,
+    )
+
+
+def get_transport(verify, http2, local_address, proxy_url, limit, retries):
+    verify = get_sslcontexts(None, None, verify, True) if verify is True else verify
+    return httpx.AsyncHTTPTransport(
+        # pylint: disable=protected-access
+        verify=verify,
+        http2=http2,
+        limits=limit,
+        proxy=httpx._config.Proxy(proxy_url) if proxy_url else None,
+        local_address=local_address,
+        retries=retries,
+    )
+
+
+def new_client(
+    # pylint: disable=too-many-arguments
+    enable_http,
+    verify,
+    enable_http2,
+    max_connections,
+    max_keepalive_connections,
+    keepalive_expiry,
+    proxies,
+    local_address,
+    retries,
+    max_redirects,
+    hook_log_response,
+):
+    limit = httpx.Limits(
+        max_connections=max_connections,
+        max_keepalive_connections=max_keepalive_connections,
+        keepalive_expiry=keepalive_expiry,
+    )
+    # See https://www.python-httpx.org/advanced/#routing
+    mounts = {}
+    for pattern, proxy_url in proxies.items():
+        if not enable_http and pattern.startswith('http://'):
+            continue
+        if proxy_url.startswith('socks4://') or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://'):
+            mounts[pattern] = get_transport_for_socks_proxy(
+                verify, enable_http2, local_address, proxy_url, limit, retries
+            )
+        else:
+            mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries)
+
+    if not enable_http:
+        mounts['http://'] = AsyncHTTPTransportNoHttp()
+
+    transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
+
+    event_hooks = None
+    if hook_log_response:
+        event_hooks = {'response': [hook_log_response]}
+
+    return httpx.AsyncClient(
+        transport=transport,
+        mounts=mounts,
+        max_redirects=max_redirects,
+        event_hooks=event_hooks,
+    )
+
+
+def get_loop():
+    return LOOP
+
+
+def init():
+    # log
+    for logger_name in (
+        'httpx',
+        'httpcore.proxy',
+        'httpcore.connection',
+        'httpcore.http11',
+        'httpcore.http2',
+        'hpack.hpack',
+        'hpack.table',
+    ):
+        logging.getLogger(logger_name).setLevel(logging.WARNING)
+
+    # loop
+    def loop_thread():
+        global LOOP
+        LOOP = asyncio.new_event_loop()
+        LOOP.run_forever()
+
+    thread = threading.Thread(
+        target=loop_thread,
+        name='asyncio_loop',
+        daemon=True,
+    )
+    thread.start()
+
+
+init()