first commit

searx/engines/yacy.py

@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""YaCy_ is a free distributed search engine, built on the principles of
+peer-to-peer (P2P) networks.
+
+API: Dev:APIyacysearch_
+
+Releases:
+
+- https://github.com/yacy/yacy_search_server/tags
+- https://download.yacy.net/
+
+.. _Yacy: https://yacy.net/
+.. _Dev:APIyacysearch: https://wiki.yacy.net/index.php/Dev:APIyacysearch
+
+Configuration
+=============
+
+The engine has the following (additional) settings:
+
+- :py:obj:`http_digest_auth_user`
+- :py:obj:`http_digest_auth_pass`
+- :py:obj:`search_mode`
+- :py:obj:`search_type`
+
+The :py:obj:`base_url` has to be set in the engine named `yacy` and is used by
+all yacy engines.
+
+.. code:: yaml
+
+  - name: yacy
+    engine: yacy
+    categories: general
+    search_type: text
+    shortcut: ya
+    base_url:
+      - https://yacy.searchlab.eu
+      - https://search.lomig.me
+      - https://yacy.ecosys.eu
+      - https://search.webproject.link
+
+  - name: yacy images
+    engine: yacy
+    categories: images
+    search_type: image
+    shortcut: yai
+    disabled: true
+
+
+Implementations
+===============
+"""
+# pylint: disable=fixme
+
+from __future__ import annotations
+
+import random
+from json import loads
+from urllib.parse import urlencode
+from dateutil import parser
+
+from httpx import DigestAuth
+
+from searx.utils import html_to_text
+
+# about
+about = {
+    "website": 'https://yacy.net/',
+    "wikidata_id": 'Q1759675',
+    "official_api_documentation": 'https://wiki.yacy.net/index.php/Dev:API',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+
+# engine dependent config
+categories = ['general']
+paging = True
+number_of_results = 10
+http_digest_auth_user = ""
+"""HTTP digest user for the local YACY instance"""
+http_digest_auth_pass = ""
+"""HTTP digest password for the local YACY instance"""
+
+search_mode = 'global'
+"""Yacy search mode ``global`` or ``local``.  By default, Yacy operates in ``global``
+mode.
+
+``global``
+  Peer-to-Peer search
+
+``local``
+  Privacy or Stealth mode, restricts the search to local yacy instance.
+"""
+search_type = 'text'
+"""One of ``text``, ``image`` / The search-types ``app``, ``audio`` and
+``video`` are not yet implemented (Pull-Requests are welcome).
+"""
+
+base_url: list | str = 'https://yacy.searchlab.eu'
+"""The value is an URL or a list of URLs.  In the latter case instance will be
+selected randomly.
+"""
+
+
+def init(_):
+    valid_types = [
+        'text',
+        'image',
+        # 'app', 'audio', 'video',
+    ]
+    if search_type not in valid_types:
+        raise ValueError('search_type "%s" is  not one of %s' % (search_type, valid_types))
+
+
+def _base_url() -> str:
+    from searx.engines import engines  # pylint: disable=import-outside-toplevel
+
+    url = engines['yacy'].base_url  # type: ignore
+    if isinstance(url, list):
+        url = random.choice(url)
+    if url.endswith("/"):
+        url = url[:-1]
+    return url
+
+
+def request(query, params):
+
+    offset = (params['pageno'] - 1) * number_of_results
+    args = {
+        'query': query,
+        'startRecord': offset,
+        'maximumRecords': number_of_results,
+        'contentdom': search_type,
+        'resource': search_mode,
+    }
+
+    # add language tag if specified
+    if params['language'] != 'all':
+        args['lr'] = 'lang_' + params['language'].split('-')[0]
+
+    params["url"] = f"{_base_url()}/yacysearch.json?{urlencode(args)}"
+
+    if http_digest_auth_user and http_digest_auth_pass:
+        params['auth'] = DigestAuth(http_digest_auth_user, http_digest_auth_pass)
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    raw_search_results = loads(resp.text)
+
+    # return empty array if there are no results
+    if not raw_search_results:
+        return []
+
+    search_results = raw_search_results.get('channels', [])
+
+    if len(search_results) == 0:
+        return []
+
+    for result in search_results[0].get('items', []):
+        # parse image results
+        if search_type == 'image':
+            result_url = ''
+            if 'url' in result:
+                result_url = result['url']
+            elif 'link' in result:
+                result_url = result['link']
+            else:
+                continue
+
+            # append result
+            results.append(
+                {
+                    'url': result_url,
+                    'title': result['title'],
+                    'content': '',
+                    'img_src': result['image'],
+                    'template': 'images.html',
+                }
+            )
+
+        # parse general results
+        else:
+            publishedDate = None
+            if 'pubDate' in result:
+                publishedDate = parser.parse(result['pubDate'])
+
+            # append result
+            results.append(
+                {
+                    'url': result['link'] or '',
+                    'title': result['title'],
+                    'content': html_to_text(result['description']),
+                    'publishedDate': publishedDate,
+                }
+            )
+
+        # TODO parse video, audio and file results
+
+    return results