From da5911d680fd538821cf01168f5125e10f8d8df3 Mon Sep 17 00:00:00 2001 From: Quinten Stokkink Date: Tue, 21 Nov 2023 09:33:08 +0100 Subject: [PATCH] Integrated torrent file tree into Download/TorrentDef (#7694) --- .../core/components/libtorrent/conftest.py | 11 +- .../libtorrent/download_manager/download.py | 90 ++++++++- .../libtorrent/tests/test_download.py | 184 +++++++++++++++++- .../libtorrent/tests/test_torrent_def.py | 66 ++++++- .../tests/test_torrent_file_tree.py | 99 ++++++++++ .../libtorrent/torrent_file_tree.py | 49 +++-- .../core/components/libtorrent/torrentdef.py | 51 ++++- 7 files changed, 515 insertions(+), 35 deletions(-) diff --git a/src/tribler/core/components/libtorrent/conftest.py b/src/tribler/core/components/libtorrent/conftest.py index 455d84d1f2e..86a7f5e6e8b 100644 --- a/src/tribler/core/components/libtorrent/conftest.py +++ b/src/tribler/core/components/libtorrent/conftest.py @@ -6,7 +6,7 @@ from tribler.core.components.libtorrent.download_manager.download import Download from tribler.core.components.libtorrent.download_manager.download_config import DownloadConfig from tribler.core.components.libtorrent.torrentdef import TorrentDef -from tribler.core.tests.tools.common import TESTS_DATA_DIR +from tribler.core.tests.tools.common import TESTS_DATA_DIR, TORRENT_UBUNTU_FILE, TORRENT_VIDEO_FILE from tribler.core.utilities.unicode import hexlify @@ -83,3 +83,12 @@ def mock_lt_status(): lt_status.pieces = [] lt_status.finished_time = 10 return lt_status + + +@pytest.fixture +def dual_movie_tdef() -> TorrentDef: + tdef = TorrentDef() + tdef.add_content(TORRENT_VIDEO_FILE) + tdef.add_content(TORRENT_UBUNTU_FILE) + tdef.save() + return tdef diff --git a/src/tribler/core/components/libtorrent/download_manager/download.py b/src/tribler/core/components/libtorrent/download_manager/download.py index 398ffc6375d..2ef28a4c74c 100644 --- a/src/tribler/core/components/libtorrent/download_manager/download.py +++ b/src/tribler/core/components/libtorrent/download_manager/download.py @@ -3,13 +3,16 @@ Author(s): Arno Bakker, Egbert Bouman """ +from __future__ import annotations + import asyncio import base64 import itertools import logging -from asyncio import CancelledError, Future, iscoroutine, sleep, wait_for +from asyncio import CancelledError, Future, iscoroutine, sleep, wait_for, get_running_loop from collections import defaultdict from contextlib import suppress +from enum import Enum from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple from bitarray import bitarray @@ -21,6 +24,7 @@ from tribler.core.components.libtorrent.download_manager.download_state import DownloadState from tribler.core.components.libtorrent.download_manager.stream import Stream from tribler.core.components.libtorrent.settings import DownloadDefaultsSettings +from tribler.core.components.libtorrent.torrent_file_tree import TorrentFileTree from tribler.core.components.libtorrent.torrentdef import TorrentDef, TorrentDefNoMetainfo from tribler.core.components.libtorrent.utils.libtorrent_helper import libtorrent as lt from tribler.core.components.libtorrent.utils.torrent_utils import check_handle, get_info_from_handle, require_handle @@ -37,6 +41,15 @@ Getter = Callable[[Any], Any] +class IllegalFileIndex(Enum): + """ + Error codes for Download.get_file_index(). These are used by the GUI to render directories. + """ + collapsed_dir = -1 + expanded_dir = -2 + unloaded = -3 + + class Download(TaskManager): """ Download subclass that represents a libtorrent download.""" @@ -379,6 +392,9 @@ def on_metadata_received_alert(self, alert: lt.metadata_received_alert): try: self.tdef = TorrentDef.load_from_dict(metadata) + with suppress(RuntimeError): + # Try to load the torrent info in the background if we have a loop. + get_running_loop().run_in_executor(None, self.tdef.load_torrent_info) except ValueError as ve: self._logger.exception(ve) return @@ -794,3 +810,75 @@ def set_piece_deadline(self, piece, deadline, flags=0): @check_handle([]) def get_file_priorities(self): return self.handle.file_priorities() + + def file_piece_range(self, file_path: Path) -> list[int]: + """ + Get the piece range of a given file, specified by the path. + + Calling this method with anything but a file path will return an empty list. + """ + file_index = self.get_file_index(file_path) + if file_index < 0: + return [] + + start_piece = self.tdef.torrent_info.map_file(file_index, 0, 1).piece + # Note: next_piece contains the next piece that is NOT part of this file. + if file_index < self.tdef.torrent_info.num_files() - 1: + next_piece = self.tdef.torrent_info.map_file(file_index + 1, 0, 1).piece + else: + # There is no next file so the nex piece is the last piece index + 1 (num_pieces()). + next_piece = self.tdef.torrent_info.num_pieces() + + return list(range(start_piece, next_piece)) + + @check_handle(0.0) + def get_file_completion(self, path: Path) -> float: + """ + Calculate the completion of a given file or directory. + """ + total = 0 + have = 0 + for piece_index in self.file_piece_range(path): + have += self.handle.have_piece(piece_index) + total += 1 + if total == 0: + return 1.0 + return have/total + + def get_file_length(self, path: Path) -> int: + """ + Get the length of a file or directory in bytes. Returns 0 if the given path does not point to an existing path. + """ + result = self.tdef.torrent_file_tree.find(path) + if result is not None: + return result.size + return 0 + + def get_file_index(self, path: Path) -> int: + """ + Get the index of a file or directory in a torrent. Note that directories do not have real file indices. + + Special cases ("error codes"): + + - ``-1`` (IllegalFileIndex.collapsed_dir): the given path is not a file but a collapsed directory. + - ``-2`` (IllegalFileIndex.expanded_dir): the given path is not a file but an expanded directory. + - ``-3`` (IllegalFileIndex.unloaded): the data structure is not loaded or the path is not found. + """ + result = self.tdef.torrent_file_tree.find(path) + if isinstance(result, TorrentFileTree.File): + return self.tdef.torrent_file_tree.find(path).index + if isinstance(result, TorrentFileTree.Directory): + return (IllegalFileIndex.collapsed_dir.value if result.collapsed + else IllegalFileIndex.expanded_dir.value) + return IllegalFileIndex.unloaded.value + + def is_file_selected(self, file_path: Path) -> bool: + """ + Check if the given file path is selected. + + Calling this method with anything but a file path will return False. + """ + result = self.tdef.torrent_file_tree.find(file_path) + if isinstance(result, TorrentFileTree.File): + return result.selected + return False diff --git a/src/tribler/core/components/libtorrent/tests/test_download.py b/src/tribler/core/components/libtorrent/tests/test_download.py index 919019b76eb..515203678e8 100644 --- a/src/tribler/core/components/libtorrent/tests/test_download.py +++ b/src/tribler/core/components/libtorrent/tests/test_download.py @@ -8,14 +8,14 @@ from ipv8.util import succeed from libtorrent import bencode -from tribler.core.components.libtorrent.download_manager.download import Download +from tribler.core.components.libtorrent.download_manager.download import Download, IllegalFileIndex from tribler.core.components.libtorrent.download_manager.download_config import DownloadConfig -from tribler.core.components.libtorrent.torrentdef import TorrentDefNoMetainfo +from tribler.core.components.libtorrent.torrentdef import TorrentDef, TorrentDefNoMetainfo from tribler.core.components.libtorrent.utils.torrent_utils import get_info_from_handle from tribler.core.components.reporter.exception_handler import NoCrashException from tribler.core.exceptions import SaveResumeDataError from tribler.core.tests.tools.base_test import MockObject -from tribler.core.tests.tools.common import TESTS_DATA_DIR +from tribler.core.tests.tools.common import TESTS_DATA_DIR, TORRENT_UBUNTU_FILE, TORRENT_VIDEO_FILE from tribler.core.utilities.unicode import hexlify from tribler.core.utilities.utilities import bdecode_compat @@ -308,15 +308,15 @@ def mocked_checkpoint(): test_download.on_metadata_received_alert(None) -@patch('tribler.core.components.libtorrent.download_manager.download.get_info_from_handle', Mock()) -@patch('tribler.core.components.libtorrent.download_manager.download.bdecode_compat', Mock()) -def test_on_metadata_received_alert_unicode_error(test_download): +def test_on_metadata_received_alert_unicode_error(test_download, dual_movie_tdef): """ Test the the case the field 'url' is not unicode compatible. In this case no exceptions should be raised. See: https://github.com/Tribler/tribler/issues/7223 """ + test_download.tdef = dual_movie_tdef tracker = {'url': Mock(encode=Mock(side_effect=UnicodeDecodeError('', b'', 0, 0, '')))} - test_download.handle = MagicMock(trackers=Mock(return_value=[tracker])) + test_download.handle = MagicMock(trackers=Mock(return_value=[tracker]), + torrent_file=lambda: dual_movie_tdef.torrent_info) test_download.on_metadata_received_alert(MagicMock()) @@ -506,3 +506,173 @@ async def test_shutdown(test_download: Download): assert not test_download.futures assert test_download.stream.close.called + + +def test_file_piece_range_flat(test_download: Download) -> None: + """ + Test if the piece range of a single-file torrent is correctly determined. + """ + total_pieces = test_download.tdef.torrent_info.num_pieces() + + piece_range = test_download.file_piece_range(Path("video.avi")) + + assert piece_range == list(range(total_pieces)) + + +def test_file_piece_range_wide(dual_movie_tdef: TorrentDef) -> None: + """ + Test if the piece range of a two-file torrent is correctly determined. + + The torrent is no longer flat after adding content! Data is now in the "data" directory. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + + piece_range_video = download.file_piece_range(Path("data") / TORRENT_VIDEO_FILE.name) + piece_range_ubuntu = download.file_piece_range(Path("data") / TORRENT_UBUNTU_FILE.name) + last_piece = piece_range_video[-1] + 1 + + assert 0 < last_piece < download.tdef.torrent_info.num_pieces() + assert piece_range_video == list(range(0, last_piece)) + assert piece_range_ubuntu == list(range(last_piece, download.tdef.torrent_info.num_pieces())) + + +def test_file_piece_range_nonexistent(test_download: Download) -> None: + """ + Test if the piece range of a single-file torrent is correctly determined. + """ + piece_range = test_download.file_piece_range(Path("I don't exist")) + + assert piece_range == [] + + +def test_file_completion_full(test_download: Download) -> None: + """ + Test if a complete file shows 1.0 completion. + """ + test_download.handle = MagicMock(have_piece=Mock(return_value=True)) + + assert 1.0 == test_download.get_file_completion(Path("video.avi")) + + +def test_file_completion_nonexistent(test_download: Download) -> None: + """ + Test if an unknown path (does not exist in a torrent) shows 1.0 completion. + """ + test_download.handle = MagicMock(have_piece=Mock(return_value=True)) + + assert 1.0 == test_download.get_file_completion(Path("I don't exist")) + + +def test_file_completion_directory(dual_movie_tdef: TorrentDef) -> None: + """ + Test if a directory (does not exist in a torrent) shows 1.0 completion. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + download.handle = MagicMock(have_piece=Mock(return_value=True)) + + assert 1.0 == download.get_file_completion(Path("data")) + + +def test_file_completion_nohandle(test_download: Download) -> None: + """ + Test if a file shows 0.0 completion if the torrent handle is not valid. + """ + test_download.handle = MagicMock(is_valid=Mock(return_value=False)) + + assert 0.0 == test_download.get_file_completion(Path("video.avi")) + + +def test_file_completion_partial(test_download: Download) -> None: + """ + Test if a file shows 0.0 completion if the torrent handle is not valid. + """ + total_pieces = test_download.tdef.torrent_info.num_pieces() + expected = (total_pieces // 2) / total_pieces + + def fake_has_piece(piece_index: int) -> bool: + return piece_index > total_pieces / 2 # total_pieces // 2 will return True + test_download.handle = MagicMock(have_piece=fake_has_piece) + + result = test_download.get_file_completion(Path("video.avi")) + + assert round(expected, 4) == round(result, 4) # Round to make sure we don't get float rounding errors + + +def test_file_length(test_download: Download) -> None: + """ + Test if we can get the length of a file. + """ + assert 1942100 == test_download.get_file_length(Path("video.avi")) + + +def test_file_length_two(dual_movie_tdef: TorrentDef) -> None: + """ + Test if we can get the length of a file in a two-file torrent. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + + assert 291888 == download.get_file_length(Path("data") / TORRENT_VIDEO_FILE.name) + assert 44258 == download.get_file_length(Path("data") / TORRENT_UBUNTU_FILE.name) + + +def test_file_length_nonexistent(test_download: Download) -> None: + """ + Test if the length of a non-existent file is 0. + """ + assert 0 == test_download.get_file_length(Path("I don't exist")) + + +def test_file_index_unloaded(test_download: Download) -> None: + """ + Test if a non-existent path leads to the special unloaded index. + """ + assert IllegalFileIndex.unloaded.value == test_download.get_file_index(Path("I don't exist")) + + +def test_file_index_directory_collapsed(dual_movie_tdef: TorrentDef) -> None: + """ + Test if a collapsed-dir path leads to the special collapsed dir index. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + + assert IllegalFileIndex.collapsed_dir.value == download.get_file_index(Path("data")) + + +def test_file_index_directory_expanded(dual_movie_tdef: TorrentDef) -> None: + """ + Test if a expanded-dir path leads to the special expanded dir index. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + download.tdef.torrent_file_tree.expand(Path("data")) + + assert IllegalFileIndex.expanded_dir.value == download.get_file_index(Path("data")) + + +def test_file_index_file(test_download: Download) -> None: + """ + Test if we can get the index of a file. + """ + assert 0 == test_download.get_file_index(Path("video.avi")) + + +def test_file_selected_nonexistent(test_download: Download) -> None: + """ + Test if a non-existent file does not register as selected. + """ + assert not test_download.is_file_selected(Path("I don't exist")) + + +def test_file_selected_realfile(test_download: Download) -> None: + """ + Test if a file starts off as selected. + """ + assert test_download.is_file_selected(Path("video.avi")) + + +def test_file_selected_directory(dual_movie_tdef: TorrentDef) -> None: + """ + Test if a directory does not register as selected. + """ + download = Download(dual_movie_tdef, checkpoint_disabled=True) + + assert not download.is_file_selected(Path("data")) diff --git a/src/tribler/core/components/libtorrent/tests/test_torrent_def.py b/src/tribler/core/components/libtorrent/tests/test_torrent_def.py index 9512d326fdc..7f917d4d8d8 100644 --- a/src/tribler/core/components/libtorrent/tests/test_torrent_def.py +++ b/src/tribler/core/components/libtorrent/tests/test_torrent_def.py @@ -3,7 +3,6 @@ import pytest from aiohttp import ClientResponseError -from libtorrent import bencode from tribler.core.components.libtorrent.torrentdef import TorrentDef, TorrentDefNoMetainfo from tribler.core.tests.tools.common import TESTS_DATA_DIR, TORRENT_UBUNTU_FILE @@ -28,11 +27,17 @@ def test_create_invalid_tdef(): """ invalid_metainfo = {} with pytest.raises(ValueError): - TorrentDef.load_from_memory(bencode(invalid_metainfo)) + TorrentDef(metainfo=invalid_metainfo) + + with pytest.raises(ValueError): + TorrentDef(metainfo=invalid_metainfo, ignore_validation=False) invalid_metainfo = {b'info': {}} with pytest.raises(ValueError): - TorrentDef.load_from_memory(bencode(invalid_metainfo)) + TorrentDef(metainfo=invalid_metainfo) + + with pytest.raises(ValueError): + TorrentDef(metainfo=invalid_metainfo, ignore_validation=False) def test_add_content_dir(tdef): @@ -293,3 +298,58 @@ def test_get_files_with_length(tdef): tdef.metainfo = {b'info': {b'files': [{b'path.utf-8': [b'test\xff' + name_bytes], b'length': 123}, {b'path': [b'file.txt'], b'length': 456}]}} assert tdef.get_files_with_length() == [(Path('file.txt'), 456)] + + +def test_load_torrent_info(tdef: TorrentDef) -> None: + """ + Test if load_torrent_info() loads the torrent info. + """ + tdef.metainfo = { + b'info': { + b'name': 'torrent name', + b'files': [{b'path': [b'a.txt'], b'length': 123}], + b'piece length': 128, + b'pieces': b'\x00' * 20 + } + } + + assert not tdef.torrent_info_loaded() + tdef.load_torrent_info() + assert tdef.torrent_info_loaded() + assert tdef.torrent_info is not None + + +def test_lazy_load_torrent_info(tdef: TorrentDef) -> None: + """ + Test if accessing torrent_info loads the torrent info. + """ + tdef.metainfo = { + b'info': { + b'name': 'torrent name', + b'files': [{b'path': [b'a.txt'], b'length': 123}], + b'piece length': 128, + b'pieces': b'\x00' * 20 + } + } + + assert not tdef.torrent_info_loaded() + assert tdef.torrent_info is not None + assert tdef.torrent_info_loaded() + + +def test_generate_tree(tdef: TorrentDef) -> None: + """ + Test if a torrent tree can be generated from a TorrentDef. + """ + tdef.metainfo = { + b'info': { + b'name': 'torrent name', + b'files': [{b'path': [b'a.txt'], b'length': 123}], + b'piece length': 128, + b'pieces': b'\x00' * 20 + } + } + + tree = tdef.torrent_file_tree + + assert tree.find(Path("torrent name") / "a.txt").size == 123 diff --git a/src/tribler/core/components/libtorrent/tests/test_torrent_file_tree.py b/src/tribler/core/components/libtorrent/tests/test_torrent_file_tree.py index 4ad0abc9996..f7b7a772373 100644 --- a/src/tribler/core/components/libtorrent/tests/test_torrent_file_tree.py +++ b/src/tribler/core/components/libtorrent/tests/test_torrent_file_tree.py @@ -448,3 +448,102 @@ def test_view_over_expanded(file_storage_with_dirs): Path("torrent_create") / "def" / "file6.avi", Path("torrent_create") / "file1.txt" ] + + +def test_view_full_collapsed(file_storage_with_dirs): + """ + Test if we can loop through a expanded directory with only collapsed dirs. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + tree.expand(Path("") / "torrent_create") + + result = tree.view(Path(""), 4) + + assert [Path(r) for r in result] == [ + Path("torrent_create"), + Path("torrent_create") / "abc", + Path("torrent_create") / "def", + Path("torrent_create") / "file1.txt" + ] + + +def test_select_start_selected(file_storage_with_dirs): + """ + Test if all files start selected. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + + assert tree.find(Path("torrent_create") / "abc" / "file2.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file3.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file4.txt").selected + assert tree.find(Path("torrent_create") / "def" / "file5.txt").selected + assert tree.find(Path("torrent_create") / "def" / "file6.avi").selected + assert tree.find(Path("torrent_create") / "file1.txt").selected + + +def test_select_nonexistent(file_storage_with_dirs): + """ + Test selecting a non-existent path. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + tree.set_selected(Path("."), False) + + tree.set_selected(Path("I don't exist"), True) + + assert not tree.find(Path("torrent_create") / "abc" / "file2.txt").selected + assert not tree.find(Path("torrent_create") / "abc" / "file3.txt").selected + assert not tree.find(Path("torrent_create") / "abc" / "file4.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file5.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file6.avi").selected + assert not tree.find(Path("torrent_create") / "file1.txt").selected + + +def test_select_file(file_storage_with_dirs): + """ + Test selecting a path pointing to a single file. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + tree.set_selected(Path("."), False) + + tree.set_selected(Path("torrent_create") / "abc" / "file2.txt", True) + + assert tree.find(Path("torrent_create") / "abc" / "file2.txt").selected + assert not tree.find(Path("torrent_create") / "abc" / "file3.txt").selected + assert not tree.find(Path("torrent_create") / "abc" / "file4.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file5.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file6.avi").selected + assert not tree.find(Path("torrent_create") / "file1.txt").selected + + +def test_select_flatdir(file_storage_with_dirs): + """ + Test selecting a path pointing to a directory with no subdirectories, only files. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + tree.set_selected(Path("."), False) + + tree.set_selected(Path("torrent_create") / "abc", True) + + assert tree.find(Path("torrent_create") / "abc" / "file2.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file3.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file4.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file5.txt").selected + assert not tree.find(Path("torrent_create") / "def" / "file6.avi").selected + assert not tree.find(Path("torrent_create") / "file1.txt").selected + + +def test_select_deepdir(file_storage_with_dirs): + """ + Test selecting a path pointing to a directory with no bdirectories and files. + """ + tree = TorrentFileTree.from_lt_file_storage(file_storage_with_dirs) + tree.set_selected(Path("."), False) + + tree.set_selected(Path("torrent_create"), True) + + assert tree.find(Path("torrent_create") / "abc" / "file2.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file3.txt").selected + assert tree.find(Path("torrent_create") / "abc" / "file4.txt").selected + assert tree.find(Path("torrent_create") / "def" / "file5.txt").selected + assert tree.find(Path("torrent_create") / "def" / "file6.avi").selected + assert tree.find(Path("torrent_create") / "file1.txt").selected diff --git a/src/tribler/core/components/libtorrent/torrent_file_tree.py b/src/tribler/core/components/libtorrent/torrent_file_tree.py index d2abf2086b4..efefa7fdee1 100644 --- a/src/tribler/core/components/libtorrent/torrent_file_tree.py +++ b/src/tribler/core/components/libtorrent/torrent_file_tree.py @@ -70,6 +70,7 @@ class File: name: str index: int size: int = 0 + selected: bool = True _sort_pattern = re.compile('([0-9]+)') # We use this for natural sorting (see sort_key()) @@ -192,6 +193,21 @@ def collapse(self, path: Path) -> None: if isinstance(element, TorrentFileTree.Directory) and element != self.root: element.collapsed = True + def set_selected(self, path: Path, selected: bool) -> None: + """ + Set the selected status for a File or entire Directory. + """ + item = self.find(path) + if item is None: + return + if isinstance(item, TorrentFileTree.File): + item.selected = selected # pylint: disable=W0201 + return + for key in item.directories: + self.set_selected(path / key, selected) + for file in item.files: + file.selected = selected + def find(self, path: Path) -> Directory | File | None: """ Get the Directory or File object at the given path, or None if it does not exist. @@ -295,24 +311,21 @@ def _view_process_directories(self, number: int, directory_items: ItemsView[str, Note that we only need to process the first directory and the remainder is visited through recursion. """ view = [] - try: - dirname, dirobj = next(iter(directory_items)) - except StopIteration: - return view, number - - full_path = fetch_path / dirname - - # The subdirectory is an item of the tree itself. - view.append(str(full_path)) - number -= 1 - if number == 0: # Exit early if we don't need anymore items - return view, number - - # If the elements of the subdirectory are not collapsed, recurse into a view of those elements. - if not dirobj.collapsed: - elems = self.view((dirobj, full_path), number) - view += elems - number -= len(elems) + for dirname, dirobj in directory_items: + full_path = fetch_path / dirname + + # The subdirectory is an item of the tree itself. + view.append(str(full_path)) + number -= 1 + if number == 0: # Exit early if we don't need anymore items + return view, number + + # If the elements of the subdirectory are not collapsed, recurse into a view of those elements. + if not dirobj.collapsed: + elems = self.view((dirobj, full_path), number) + view += elems + number -= len(elems) + break # We exhausted all subdirectories (note that the number may still be larger than 0) return view, number diff --git a/src/tribler/core/components/libtorrent/torrentdef.py b/src/tribler/core/components/libtorrent/torrentdef.py index 98c15a1e0d3..6f3bda68fb9 100644 --- a/src/tribler/core/components/libtorrent/torrentdef.py +++ b/src/tribler/core/components/libtorrent/torrentdef.py @@ -4,10 +4,13 @@ import itertools import logging from asyncio import get_running_loop +from contextlib import suppress +from functools import cached_property from hashlib import sha1 import aiohttp +from tribler.core.components.libtorrent.torrent_file_tree import TorrentFileTree from tribler.core.components.libtorrent.utils.libtorrent_helper import libtorrent as lt from tribler.core.components.libtorrent.utils import torrent_utils from tribler.core.utilities import maketorrent, path_util @@ -46,7 +49,7 @@ class TorrentDef: It can be used to create new torrents, or analyze existing ones. """ - def __init__(self, metainfo=None, torrent_parameters=None, ignore_validation=False): + def __init__(self, metainfo=None, torrent_parameters=None, ignore_validation=True): """ Create a new TorrentDef object, possibly based on existing data. :param metainfo: A dictionary with metainfo, i.e. from a .torrent file. @@ -55,19 +58,26 @@ def __init__(self, metainfo=None, torrent_parameters=None, ignore_validation=Fal """ self._logger = logging.getLogger(self.__class__.__name__) self.torrent_parameters = {} - self.metainfo = None + self.metainfo = metainfo self.files_list = [] self.infohash = None + self._torrent_info = None if metainfo is not None: # First, make sure the passed metainfo is valid if not ignore_validation: try: - lt.torrent_info(metainfo) + self._torrent_info = lt.torrent_info(metainfo) + self.infohash = self._torrent_info.info_hash() except RuntimeError as exc: raise ValueError from exc - self.metainfo = metainfo - self.infohash = sha1(lt.bencode(self.metainfo[b'info'])).digest() + else: + try: + if not self.metainfo[b'info']: + raise ValueError("Empty metainfo!") + self.infohash = sha1(lt.bencode(self.metainfo[b'info'])).digest() + except (KeyError, RuntimeError) as exc: + raise ValueError from exc self.copy_metainfo_to_torrent_parameters() elif torrent_parameters: @@ -95,6 +105,34 @@ def copy_metainfo_to_torrent_parameters(self): if self.metainfo and key in self.metainfo[b'info']: self.torrent_parameters[key] = self.metainfo[b'info'][key] + @property + def torrent_info(self) -> lt.torrent_info: + """ + Get the libtorrent torrent info instance or load it from our metainfo. + """ + self.load_torrent_info() + return self._torrent_info + + def load_torrent_info(self) -> None: + """ + Load the torrent info into memory from our metainfo if it does not exist. + """ + if self._torrent_info is None: + self._torrent_info = lt.torrent_info(self.metainfo) + + def torrent_info_loaded(self) -> bool: + """ + Check if the libtorrent torrent info is loaded. + """ + return self._torrent_info is not None + + @cached_property + def torrent_file_tree(self) -> TorrentFileTree: + """ + Construct a file tree from this torrent definition. + """ + return TorrentFileTree.from_lt_file_storage(self.torrent_info.files()) + @staticmethod def _threaded_load_job(filepath): """ @@ -339,6 +377,9 @@ def save(self, torrent_filepath=None): """ torrent_dict = torrent_utils.create_torrent_file(self.files_list, self.torrent_parameters, torrent_filepath=torrent_filepath) + self._torrent_info = None + with suppress(AttributeError): + del self.torrent_file_tree # Remove the cache without retrieving it or checking if it exists (Error) self.metainfo = bdecode_compat(torrent_dict['metainfo']) self.copy_metainfo_to_torrent_parameters() self.infohash = torrent_dict['infohash']