Replace file URI creating by yarl's URL()

src/tribler-common/tribler_common/rest_utils.py

@@ -0,0 +1,33 @@
+import os
+from typing import Any, Union
+
+from yarl import URL
+
+MAGNET_SCHEME = 'magnet'
+HTTP_SCHEME = 'http'
+FILE_SCHEME = 'file'
+
+
+def path_to_uri(file_path: Union[str, Any]) -> str:
+    """Convert path to url
+
+    Example:
+        '/path/to/file' -> 'file:///path/to/file'
+    """
+    if not isinstance(file_path, str):
+        file_path = str(file_path)
+    return str(URL().build(scheme=FILE_SCHEME, path=file_path))
+
+
+def uri_to_path(file_uri: str) -> str:
+    """Convert uri to path
+
+    Example:
+        'file:///path/to/file' -> '/path/to/file'
+    """
+    path = URL(file_uri).path
+    if os.name == 'nt':
+        # Removes first slash for win OS
+        # see https://github.com/aio-libs/yarl/issues/674
+        return path.lstrip('/')
+    return path

src/tribler-common/tribler_common/tests/test_rest_utils.py

@@ -0,0 +1,43 @@
+from unittest.mock import patch
+
+import pytest
+
+from tribler_common.rest_utils import path_to_uri, uri_to_path
+
+NIX_PATHS = [
+    ('/path/to/file', 'file:///path/to/file'),
+    ('/path/to/file with space', 'file:///path/to/file%20with%20space'),
+    ('/path/to/%20%21file', 'file:///path/to/%2520%2521file'),  # See: https://github.com/Tribler/tribler/issues/6700
+]
+
+WIN_PATHS = [
+    ('C:\\path\\to\\file', 'file:///C:%5Cpath%5Cto%5Cfile'),
+    ('C:\\path\\to\\file with space', 'file:///C:%5Cpath%5Cto%5Cfile%20with%20space'),
+    ('C:\\path\\to\\%20%21file', 'file:///C:%5Cpath%5Cto%5C%2520%2521file'),
+]
+
+
+# posix
+@pytest.mark.parametrize('path,uri', NIX_PATHS)
+@patch('os.name', 'posix')
+def test_path_to_uri(path, uri):
+    assert path_to_uri(path) == uri
+
+
+@pytest.mark.parametrize('path,uri', NIX_PATHS)
+@patch('os.name', 'posix')
+def test_uri_to_path(path, uri):
+    assert uri_to_path(uri) == path
+
+
+# win
+@pytest.mark.parametrize('path,uri', WIN_PATHS)
+@patch('os.name', 'nt')
+def test_path_to_uri_win(path, uri):
+    assert path_to_uri(path) == uri
+
+
+@pytest.mark.parametrize('path,uri', WIN_PATHS)
+@patch('os.name', 'nt')
+def test_uri_to_path_win(path, uri):
+    assert uri_to_path(uri) == path

src/tribler-common/tribler_common/tests/test_utils.py

@@ -1,17 +1,11 @@
-from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 from tribler_common.patch_import import patch_import
-from tribler_common.utilities import Query, extract_tags, parse_query, show_system_popup, to_fts_query, uri_to_path
+from tribler_common.utilities import Query, extract_tags, parse_query, show_system_popup, to_fts_query
 
 # pylint: disable=import-outside-toplevel, import-error
 # fmt: off
 
-def test_uri_to_path():
-    path = Path(__file__).parent / "bla%20foo.bar"
-    uri = path.as_uri()
-    assert uri_to_path(uri) == path
-
 
 def test_to_fts_query():
     assert to_fts_query(None) is None

src/tribler-common/tribler_common/utilities.py

@@ -1,14 +1,9 @@
 import itertools
-import os
 import platform
 import re
 import sys
 from dataclasses import dataclass, field
 from typing import Set, Tuple
-from urllib.parse import urlparse
-from urllib.request import url2pathname
-
-from tribler_core.utilities.path_util import Path
 
 
 def is_frozen():
@@ -23,12 +18,6 @@ def is_frozen():
     return True
 
 
-def uri_to_path(uri):
-    parsed = urlparse(uri)
-    host = "{0}{0}{mnt}{0}".format(os.path.sep, mnt=parsed.netloc)
-    return Path(host) / url2pathname(parsed.path)
-
-
 fts_query_re = re.compile(r'\w+', re.UNICODE)
 tags_re = re.compile(r'#[^\s^#]{3,50}(?=[#\s]|$)')
 

src/tribler-core/tribler_core/components/libtorrent/download_manager/download_manager.py

@@ -16,7 +16,7 @@
 from ipv8.taskmanager import TaskManager, task
 
 from tribler_common.network_utils import default_network_utils
-from tribler_common.rest_constants import FILE_PREFIX, HTTP_PREFIX, MAGNET_PREFIX
+from tribler_common.rest_utils import FILE_SCHEME, HTTP_SCHEME, MAGNET_SCHEME, uri_to_path
 from tribler_common.simpledefs import DLSTATUS_SEEDING, MAX_LIBTORRENT_RATE_LIMIT, NTFY, STATEDIR_CHECKPOINT_DIR
 
 from tribler_core.components.libtorrent.download_manager.dht_health_manager import DHTHealthManager
@@ -505,10 +505,10 @@ def _map_call_on_ltsessions(self, hops, funcname, *args, **kwargs):
             getattr(self.get_session(hops), funcname)(*args, **kwargs)
 
     async def start_download_from_uri(self, uri, config=None):
-        if uri.startswith(HTTP_PREFIX):
+        if uri.startswith(HTTP_SCHEME):
             tdef = await TorrentDef.load_from_url(uri)
             return self.start_download(tdef=tdef, config=config)
-        if uri.startswith(MAGNET_PREFIX):
+        if uri.startswith(MAGNET_SCHEME):
             name, infohash, _ = parse_magnetlink(uri)
             if infohash is None:
                 raise RuntimeError("Missing infohash")
@@ -517,8 +517,8 @@ async def start_download_from_uri(self, uri, config=None):
             else:
                 tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri)
             return self.start_download(tdef=tdef, config=config)
-        if uri.startswith(FILE_PREFIX):
-            file = uri[len(FILE_PREFIX) + 1:]
+        if uri.startswith(FILE_SCHEME):
+            file = uri_to_path(uri)
             return self.start_download(torrent_file=file, config=config)
         raise Exception("invalid uri")
 

src/tribler-core/tribler_core/components/libtorrent/libtorrent_component.py

@@ -1,3 +1,5 @@
+from tribler_common.rest_utils import path_to_uri
+
 from tribler_core.components.base import Component
 from tribler_core.components.key.key_component import KeyComponent
 from tribler_core.components.libtorrent.download_manager.download_manager import DownloadManager
@@ -33,7 +35,7 @@ async def run(self):
 
         if config.gui_test_mode:
             from tribler_core.tests.tools.common import TORRENT_WITH_DIRS  # pylint: disable=import-outside-toplevel
-            uri = f"file:{TORRENT_WITH_DIRS}"
+            uri = path_to_uri(TORRENT_WITH_DIRS)
             await self.download_manager.start_download_from_uri(uri)
 
     async def shutdown(self):

src/tribler-core/tribler_core/components/libtorrent/restapi/tests/test_downloads_endpoint.py

@@ -8,6 +8,7 @@
 
 import pytest
 
+from tribler_common.rest_utils import HTTP_SCHEME, path_to_uri
 from tribler_common.simpledefs import DLSTATUS_CIRCUITS, DLSTATUS_DOWNLOADING, DLSTATUS_EXIT_NODES, DLSTATUS_STOPPED
 
 from tribler_core.components.libtorrent.download_manager.download_state import DownloadState
@@ -184,11 +185,10 @@ async def test_start_download_from_file(test_download, mock_dlmgr, rest_api):
     Testing whether we can start a download from a file
     """
     mock_dlmgr.start_download_from_uri = lambda *_, **__: succeed(test_download)
-
-    post_data = {'uri': f"file:{TESTS_DATA_DIR / 'video.avi.torrent'}"}
+    uri = path_to_uri(TESTS_DATA_DIR / 'video.avi.torrent')
     expected_json = {'started': True, 'infohash': 'c9a19e7fe5d9a6c106d6ea3c01746ac88ca3c7a5'}
     await do_request(rest_api, 'downloads', expected_code=200, request_type='PUT',
-                     post_data=post_data, expected_json=expected_json)
+                     post_data={'uri': uri}, expected_json=expected_json)
 
 
 async def test_start_download_with_selected_files(test_download, mock_dlmgr, rest_api):
@@ -200,8 +200,8 @@ def mocked_start_download(*_, config=None):
         return succeed(test_download)
 
     mock_dlmgr.start_download_from_uri = mocked_start_download
-
-    post_data = {'uri': f"file:{TESTS_DATA_DIR / 'video.avi.torrent'}", 'selected_files': [0]}
+    uri = path_to_uri(TESTS_DATA_DIR / 'video.avi.torrent')
+    post_data = {'uri': uri, 'selected_files': [0]}
     expected_json = {'started': True, 'infohash': 'c9a19e7fe5d9a6c106d6ea3c01746ac88ca3c7a5'}
     await do_request(rest_api, 'downloads', expected_code=200, request_type='PUT',
                      post_data=post_data, expected_json=expected_json)
@@ -249,7 +249,7 @@ def mocked_start_download(*_, **__):
 
     mock_dlmgr.start_download_from_uri = mocked_start_download
 
-    post_data = {'uri': 'http://localhost:1234/test.torrent'}
+    post_data = {'uri': f'{HTTP_SCHEME}://localhost:1234/test.torrent'}
     result = await do_request(rest_api, 'downloads', expected_code=500, request_type='PUT', post_data=post_data)
     assert result["error"] == "test"
 

src/tribler-core/tribler_core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py

@@ -10,7 +10,7 @@
 
 import pytest
 
-from tribler_common.rest_constants import FILE_PREFIX
+from tribler_common.rest_utils import path_to_uri
 from tribler_common.simpledefs import NTFY
 
 from tribler_core.components.libtorrent.restapi.torrentinfo_endpoint import TorrentInfoEndpoint
@@ -63,9 +63,8 @@ async def test_get_torrentinfo_escaped_characters(tmp_path, rest_api):
     source = TORRENT_UBUNTU_FILE
     destination = tmp_path / 'ubuntu%20%21 15.04.torrent'
     shutil.copyfile(source, destination)
-
-    response = await do_request(rest_api, url='torrentinfo', params={'uri': f'{FILE_PREFIX}:{destination}'},
-                                expected_code=200)
+    uri = path_to_uri(destination)
+    response = await do_request(rest_api, url='torrentinfo', params={'uri': uri}, expected_code=200)
 
     assert 'metainfo' in response
 
@@ -76,7 +75,7 @@ async def test_get_torrentinfo(tmp_path, rest_api, endpoint: TorrentInfoEndpoint
     """
 
     def _path(file):
-        return f'{FILE_PREFIX}:{TESTS_DATA_DIR / file}'
+        return path_to_uri(TESTS_DATA_DIR / file)
 
     shutil.copyfile(TORRENT_UBUNTU_FILE, tmp_path / 'ubuntu.torrent')
 

src/tribler-core/tribler_core/components/libtorrent/restapi/torrentinfo_endpoint.py

@@ -10,7 +10,7 @@
 
 from marshmallow.fields import String
 
-from tribler_common.rest_constants import FILE_PREFIX, HTTP_PREFIX, MAGNET_PREFIX
+from tribler_common.rest_utils import FILE_SCHEME, HTTP_SCHEME, MAGNET_SCHEME, uri_to_path
 from tribler_common.simpledefs import NTFY
 
 from tribler_core.components.libtorrent.download_manager.download_manager import DownloadManager
@@ -82,15 +82,15 @@ async def get_torrent_info(self, request):
             return RESTResponse({"error": "uri parameter missing"}, status=HTTP_BAD_REQUEST)
 
         metainfo = None
-        if uri.startswith(FILE_PREFIX):
-            file = uri[len(FILE_PREFIX) + 1:]
+        if uri.startswith(FILE_SCHEME):
+            file = uri_to_path(uri)
             try:
                 tdef = TorrentDef.load(file)
                 metainfo = tdef.metainfo
             except (TypeError, RuntimeError):
                 return RESTResponse({"error": f"error while decoding torrent file: {file}"},
                                     status=HTTP_INTERNAL_SERVER_ERROR)
-        elif uri.startswith(HTTP_PREFIX):
+        elif uri.startswith(HTTP_SCHEME):
             try:
                 response = await query_http_uri(uri)
             except (ServerConnectionError, ClientResponseError) as e:
@@ -102,7 +102,7 @@ async def get_torrent_info(self, request):
                     metainfo = await self.download_manager.get_metainfo(infohash, timeout=60, hops=hops, url=response)
             else:
                 metainfo = bdecode_compat(response)
-        elif uri.startswith(MAGNET_PREFIX):
+        elif uri.startswith(MAGNET_SCHEME):
             infohash = parse_magnetlink(uri)[1]
             if infohash is None:
                 return RESTResponse({"error": "missing infohash"}, status=HTTP_BAD_REQUEST)

src/tribler-core/tribler_core/components/libtorrent/tests/test_download_api.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from tribler_common.rest_constants import FILE_PREFIX
+from tribler_common.rest_utils import path_to_uri
 from tribler_common.simpledefs import DLSTATUS_DOWNLOADING
 
 from tribler_core.tests.tools.common import TORRENT_UBUNTU_FILE
@@ -20,7 +20,8 @@ async def test_download_torrent_from_url(tmp_path, file_server, download_manager
 @pytest.mark.asyncio
 @pytest.mark.timeout(10)
 async def test_download_torrent_from_file(download_manager):
-    d = await download_manager.start_download_from_uri(TORRENT_UBUNTU_FILE.as_uri())
+    uri = path_to_uri(TORRENT_UBUNTU_FILE)
+    d = await download_manager.start_download_from_uri(uri)
     await d.wait_for_status(DLSTATUS_DOWNLOADING)
 
 
@@ -29,5 +30,6 @@ async def test_download_torrent_from_file(download_manager):
 async def test_download_torrent_from_file_with_escaped_characters(download_manager, tmp_path):
     destination = tmp_path / 'ubuntu%20%21 15.04.torrent'
     shutil.copyfile(TORRENT_UBUNTU_FILE, destination)
-    d = await download_manager.start_download_from_uri(f'{FILE_PREFIX}:{destination}')
+    uri = path_to_uri(destination)
+    d = await download_manager.start_download_from_uri(uri)
     await d.wait_for_status(DLSTATUS_DOWNLOADING)

src/tribler-core/tribler_core/requirements.txt

@@ -20,4 +20,4 @@ pyyaml==6.0
 sentry-sdk==1.5.0
 service-identity==21.1.0
 yappi==1.3.3
-yarl==1.7.0
+yarl==1.7.2 # keep this dependency higher than 1.6.3. See: https://github.com/aio-libs/yarl/issues/517

src/tribler-gui/tribler_gui/dialogs/startdownloaddialog.py

@@ -8,7 +8,7 @@
 from PyQt5.QtCore import QTimer, pyqtSignal
 from PyQt5.QtWidgets import QFileDialog, QSizePolicy
 
-from tribler_common.rest_constants import FILE_PREFIX, MAGNET_PREFIX
+from tribler_common.rest_utils import FILE_SCHEME, MAGNET_SCHEME, uri_to_path
 
 from tribler_gui.defs import METAINFO_MAX_RETRIES, METAINFO_TIMEOUT
 from tribler_gui.dialogs.confirmationdialog import ConfirmationDialog
@@ -34,9 +34,9 @@ def __init__(self, parent, download_uri):
         DialogContainer.__init__(self, parent)
 
         torrent_name = download_uri
-        if torrent_name.startswith(FILE_PREFIX):
-            torrent_name = torrent_name[len(FILE_PREFIX) + 1 :]
-        elif torrent_name.startswith(MAGNET_PREFIX):
+        if torrent_name.startswith(FILE_SCHEME):
+            torrent_name = uri_to_path(torrent_name)
+        elif torrent_name.startswith(MAGNET_SCHEME):
             torrent_name = unquote_plus(torrent_name)
 
         self.download_uri = download_uri

src/tribler-gui/tribler_gui/start_gui.py

@@ -5,6 +5,7 @@
 from PyQt5.QtCore import QSettings
 
 from tribler_common.logger import load_logger_config
+from tribler_common.rest_utils import path_to_uri
 from tribler_common.sentry_reporter.sentry_reporter import SentryStrategy
 
 from tribler_core.check_os import (
@@ -58,7 +59,7 @@ def run_gui(api_port, api_key, root_state_dir, parsed_args):
             logger.info('GUI Application is already running. Passing a torrent file path to it.')
             for arg in sys.argv[1:]:
                 if os.path.exists(arg) and arg.endswith(".torrent"):
-                    app.send_message(f"file:{arg}")
+                    app.send_message(path_to_uri(arg))
                 elif arg.startswith('magnet'):
                     app.send_message(arg)
             logger.info('Close the current application.')

src/tribler-gui/tribler_gui/tests/test_gui.py

@@ -12,6 +12,7 @@
 
 import tribler_common
 from tribler_common.reported_error import ReportedError
+from tribler_common.rest_utils import path_to_uri
 from tribler_common.sentry_reporter.sentry_reporter import SentryReporter
 from tribler_common.tag_constants import MIN_TAG_LENGTH
 
@@ -393,8 +394,8 @@ def test_add_download_url(window):
     go_to_and_wait_for_downloads(window)
     window.on_add_torrent_from_url()
     screenshot(window, name="add_torrent_url_dialog")
-
-    window.dialog.dialog_widget.dialog_input.setText("file:" + str(TORRENT_WITH_DIRS))
+    uri = path_to_uri(TORRENT_WITH_DIRS)
+    window.dialog.dialog_widget.dialog_input.setText(uri)
     QTest.mouseClick(window.dialog.buttons[0], Qt.LeftButton)
     QTest.qWait(200)
     screenshot(window, name="add_torrent_url_startdownload_dialog")