Merge pull request #7534 from drew2a/refactoring/requirements

Remove unused core dependencies

requirements-core.txt

@@ -4,20 +4,17 @@ anyio==3.7.1
 chardet==5.1.0
 configobj==5.0.8
 cryptography==41.0.1
-decorator==5.1.1
 Faker==18.11.2
 libnacl==1.8.0
 lz4==4.3.2
 marshmallow==3.19.0
 networkx==3.1
 pony==0.7.16
 psutil==5.9.5
-pyasn1==0.4.8
 pydantic==1.10.11
 PyOpenSSL==23.2.0
 pyyaml==6.0
 sentry-sdk==1.27.1
-service-identity==23.1.0
 yappi==1.4.0
 yarl==1.9.2 # keep this dependency higher than 1.6.3. See: https://github.com/aio-libs/yarl/issues/517
 bitarray==2.7.6

src/tribler/core/components/libtorrent/restapi/create_torrent_endpoint.py

@@ -11,7 +11,7 @@
 from tribler.core.components.libtorrent.torrentdef import TorrentDef
 from tribler.core.components.restapi.rest.rest_endpoint import HTTP_BAD_REQUEST, RESTEndpoint, RESTResponse
 from tribler.core.components.restapi.rest.schema import HandledErrorSchema
-from tribler.core.components.restapi.rest.util import return_handled_exception
+from tribler.core.components.restapi.rest.utils import return_handled_exception
 from tribler.core.utilities.path_util import Path
 from tribler.core.utilities.unicode import ensure_unicode, recursive_bytes
 from tribler.core.utilities.utilities import bdecode_compat, froze_it

src/tribler/core/components/libtorrent/restapi/downloads_endpoint.py

@@ -21,7 +21,7 @@
     RESTResponse,
     RESTStreamResponse,
 )
-from tribler.core.components.restapi.rest.util import return_handled_exception
+from tribler.core.components.restapi.rest.utils import return_handled_exception
 from tribler.core.utilities.path_util import Path
 from tribler.core.utilities.simpledefs import (
     DOWNLOAD,

src/tribler/core/components/restapi/rest/debug_endpoint.py

@@ -14,9 +14,9 @@
 from tribler.core.components.reporter.exception_handler import CoreExceptionHandler
 from tribler.core.components.resource_monitor.implementation.base import ResourceMonitor
 from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint, RESTResponse
+from tribler.core.components.restapi.rest.utils import get_threads_info
 from tribler.core.components.tunnel.community.tunnel_community import TriblerTunnelCommunity
 from tribler.core.exceptions import TriblerCoreTestException
-from tribler.core.utilities.instrumentation import WatchDog
 from tribler.core.utilities.osutils import get_root_state_directory
 from tribler.core.utilities.path_util import Path, tail
 
@@ -164,8 +164,7 @@ async def get_open_sockets(self, request):
         }
     )
     async def get_threads(self, request):
-        watchdog = WatchDog()
-        return RESTResponse({"threads": watchdog.get_threads_info()})
+        return RESTResponse({"threads": get_threads_info()})
 
     @docs(
         tags=['Debug'],

src/tribler/core/components/restapi/rest/events_endpoint.py

@@ -14,7 +14,7 @@
 from tribler.core import notifications
 from tribler.core.components.reporter.reported_error import ReportedError
 from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint, RESTStreamResponse
-from tribler.core.components.restapi.rest.util import fix_unicode_dict
+from tribler.core.components.restapi.rest.utils import fix_unicode_dict
 from tribler.core.utilities.notifier import Notifier
 from tribler.core.utilities.utilities import froze_it
 from tribler.core.version import version_id

src/tribler/core/components/restapi/rest/tests/test_utils.py

@@ -0,0 +1,95 @@
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+from tribler.core.components.restapi.rest.utils import _format_frames, fix_unicode_array, fix_unicode_dict, \
+    get_parameter, shorten
+
+
+def test_get_parameter():
+    """
+    Testing the get_parameters method in REST API util class
+    """
+    assert get_parameter({'test': [42]}, 'test') == 42
+    assert get_parameter({}, 'test') is None
+
+
+def test_fix_unicode_array():
+    """
+    Testing the fix of a unicode array
+    """
+    arr1 = ['a', 'b', 'c', 'd']
+    assert fix_unicode_array(arr1) == ['a', 'b', 'c', 'd']
+    arr2 = ['a', b'\xa1']
+    assert fix_unicode_array(arr2) == ['a', '']
+    arr3 = [1, 2, 3, '4']
+    assert fix_unicode_array(arr3) == [1, 2, 3, '4']
+    arr4 = [{'a': 'b'}]
+    assert fix_unicode_array(arr4) == [{'a': 'b'}]
+
+
+def test_fix_unicode_dict():
+    """
+    Testing the fix of a unicode dictionary
+    """
+    dict1 = {'a': 'b', 'c': 'd'}
+    assert fix_unicode_dict(dict1) == {'a': 'b', 'c': 'd'}
+    dict2 = {'a': b'\xa2'}
+    assert fix_unicode_dict(dict2) == {'a': ''}
+    dict3 = {'a': [1, 2], 'b': ['1', '2']}
+    assert fix_unicode_dict(dict3) == {'a': [1, 2], 'b': ['1', '2']}
+    dict4 = {'a': ['1', b'2\xa3']}
+    assert fix_unicode_dict(dict4) == {'a': ['1', '2']}
+    dict5 = {'a': ('1', b'2\xa3')}
+    assert fix_unicode_dict(dict5) == {'a': ['1', '2']}
+    dict6 = {'a': {'b': b'c\xa4'}}
+    assert fix_unicode_dict(dict6) == {'a': {'b': 'c'}}
+    dict7 = {'a': 'ѡ'}
+    assert fix_unicode_dict(dict7) == {'a': 'ѡ'}
+    obj = Mock
+    dict8 = {'a': {'b': obj}}
+    assert fix_unicode_dict(dict8) == {'a': {'b': obj}}
+
+
+def test_shorten():
+    """ Test that `shorten` returns correct string"""
+    assert not shorten(None)
+    assert shorten('long string', width=100) == 'long string'
+    assert shorten('long string', width=3, placeholder='...') == 'lon...'
+    assert shorten('long string', width=3, placeholder='...', cut_at_the_end=False) == '...ing'
+
+
+def test_format_frames():
+    """ Test that `format_frames` returns correct string"""
+    assert not list(_format_frames(None))
+
+    frames = SimpleNamespace(
+        f_code=SimpleNamespace(
+            co_filename='short_file',
+            co_name='function'
+        ),
+        f_lineno=1,
+        f_locals={
+            'key': 'value'
+        },
+        f_back=SimpleNamespace(
+            f_code=SimpleNamespace(
+                co_filename='long_file' * 100,
+                co_name='function'
+            ),
+            f_lineno=1,
+            f_locals={
+                'key': 'long_value' * 100
+            },
+            f_back=None
+        )
+    )
+    expected = [
+        "short_file:, line 1, in function\n"
+        "    <source is unknown>\n"
+        "\tkey = 'valu[...]\n",
+        '[...]elong_file:, line 1, in function\n'
+        '    <source is unknown>\n'
+        "\tkey = 'long[...]\n"
+    ]
+
+    assert list(_format_frames(frames, file_width=10, value_width=5)) == expected

src/tribler/core/components/restapi/rest/util.py → src/tribler/core/components/restapi/rest/utils.py

@@ -1,7 +1,16 @@
 """
 This file contains some utility methods that are used by the API.
 """
+import linecache
+import sys
+import threading
+from types import FrameType
+from typing import Dict, Iterator, List, Optional
+
 from tribler.core.components.restapi.rest.rest_endpoint import HTTP_INTERNAL_SERVER_ERROR, RESTResponse
+from tribler.core.utilities.utilities import switch_interval
+
+ONE_SECOND = 1
 
 
 def return_handled_exception(request, exception):
@@ -66,3 +75,50 @@ def fix_unicode_array(arr):
             new_arr.append(item)
 
     return new_arr
+
+
+def shorten(s: Optional[str], width: int = 50, placeholder='[...]', cut_at_the_end: bool = True):
+    """ Shorten a string to a given width.
+        Example: shorten('hello world', 5) -> 'hello[...]'
+    """
+    if s and len(s) > width:
+        return f'{s[:width]}{placeholder}' if cut_at_the_end else f'{placeholder}{s[len(s) - width:]}'
+    return s
+
+
+def _format_frames(frame: Optional[FrameType], file_width: int = 50, value_width: int = 100) -> Iterator[str]:
+    """ Format a stack trace."""
+    while frame:
+        filename = shorten(frame.f_code.co_filename, width=file_width, cut_at_the_end=False)
+        header = f"{filename}:, line {frame.f_lineno}, in {frame.f_code.co_name}"
+        source_line = linecache.getline(frame.f_code.co_filename, frame.f_lineno).strip() or "<source is unknown>"
+        local = ''
+        for key, value in list(frame.f_locals.items()):
+            try:
+                value = repr(value)
+            except Exception as e:  # pylint: disable=broad-except
+                value = f'<exception when taking repr: {e.__class__.__name__}: {e}>'
+            value = shorten(value, width=value_width)
+            local += f'\t{key} = {value}\n'
+        frame = frame.f_back
+        yield f'{header}\n' \
+              f'    {source_line}\n' \
+              f'{local}'
+
+
+def get_threads_info() -> List[Dict]:
+    """
+    Return information about available threads.
+    """
+
+    result = []
+    with switch_interval(ONE_SECOND):
+        for t in threading.enumerate():
+            frame = sys._current_frames().get(t.ident, None)  # pylint: disable=protected-access
+
+            result.append({
+                'thread_id': t.ident,
+                'thread_name': t.name,
+                'frames': list(_format_frames(frame)),
+            })
+    return result