Merge pull request #6168 from drew2a/refactoring/pydantic

Add pydantic

doc/extract_swagger.py

@@ -13,11 +13,10 @@
 
 async def extract_swagger(destination_fn):
     session = Mock()
-    session.config = (TriblerConfig()
-                      .put('api', 'key', 'apikey')
-                      .put('api', 'http_enabled', False)
-                      .put('api', 'https_enabled', False))
-
+    session.config = TriblerConfig()
+    session.config.api.key = 'apikey'
+    session.config.api.http_enabled = False
+    session.config.api.https_enabled = False
     api_manager = RESTManager(session)
     await api_manager.start()
 

src/requirements.txt

@@ -21,4 +21,4 @@ Pillow
 netifaces
 pyqtgraph
 yappi
-
+pydantic

src/run_tribler.py

@@ -72,25 +72,25 @@ async def start_tribler():
         version_history.fork_state_directory_if_necessary()
         version_history.save_if_necessary()
         state_dir = version_history.code_version.directory
-        config = TriblerConfig(state_dir=state_dir)\
-            .load(file=state_dir / CONFIG_FILE_NAME, reset_config_on_error=True)
+        config = TriblerConfig.load(file=state_dir / CONFIG_FILE_NAME, state_dir=state_dir, reset_config_on_error=True)
 
-        if not config.get('error_handling', 'core_error_reporting_requires_user_consent'):
+        if not config.error_handling.core_error_reporting_requires_user_consent:
             SentryReporter.global_strategy = SentryStrategy.SEND_ALLOWED
 
-        config.put('api', 'http_port', int(api_port))
+        config.api.http_port = int(api_port)
         # If the API key is set to an empty string, it will remain disabled
-        if config.get('api', 'key') not in ('', api_key):
-            config.put('api', 'key', api_key)
+        if config.api.key not in ('', api_key):
+            config.api.key = api_key
             config.write()  # Immediately write the API key so other applications can use it
-        config.put('api', 'http_enabled', True)
+        config.api.http_enabled = True
 
-        priority_order = config.get('resource_monitor', 'cpu_priority')
+        priority_order = config.resource_monitor.cpu_priority
         set_process_priority(pid=os.getpid(), priority_order=priority_order)
 
         global trace_logger
         # Enable tracer if --trace-debug or --trace-exceptions flag is present in sys.argv
-        trace_logger = check_and_enable_code_tracing('core', config.get_path('general', 'log_dir'))
+        log_dir = config.general.get_path_as_absolute('log_dir', config.state_dir)
+        trace_logger = check_and_enable_code_tracing('core', log_dir)
 
         session = Session(config, core_test_mode=core_test_mode)
 

src/tribler-core/run_bandwidth_crawler.py

@@ -55,15 +55,16 @@ async def start_crawler(tribler_config):
                         action=PortAction, metavar='{0..65535}')
     args = parser.parse_args(sys.argv[1:])
 
-    config = TriblerConfig(Path(args.statedir).absolute())\
-        .load(file=Path(args.statedir) / 'triblerd.conf')\
-        .put('tunnel_community', 'enabled', False)\
-        .put('libtorrent', 'enabled', False)\
-        .put('bootstrap', 'enabled', False)\
-        .put('chant', 'enabled', False)\
-        .put('torrent_checking', 'enabled', False)\
-        .put('api', 'http_enabled', True)\
-        .put('api', 'http_port', args.restapi)
+    state_dir = Path(args.statedir).absolute()
+    config = TriblerConfig.load(file=state_dir / 'triblerd.conf', state_dir=state_dir)
+
+    config.tunnel_community.enabled = False
+    config.libtorrent.enabled = False
+    config.bootstrap.enabled = False
+    config.chant.enabled = False
+    config.torrent_checking.enabled = False
+    config.api.http_enabled = True
+    config.api.http_port = args.restapi
 
     loop = get_event_loop()
     coro = start_crawler(config)

src/tribler-core/run_tribler_headless.py

@@ -69,8 +69,7 @@ async def signal_handler(sig):
         signal.signal(signal.SIGTERM, lambda sig, _: ensure_future(signal_handler(sig)))
 
         statedir = Path(options.statedir or Path(get_appstate_dir(), '.Tribler'))
-        config = TriblerConfig(statedir)\
-            .load(file=statedir / 'triblerd.conf')
+        config = TriblerConfig.load(file=statedir / 'triblerd.conf', state_dir=statedir)
 
         # Check if we are already running a Tribler instance
         self.process_checker = ProcessChecker()
@@ -82,26 +81,24 @@ async def signal_handler(sig):
         print("Starting Tribler")
 
         if options.restapi > 0:
-            config\
-                .put('api', 'http_enabled', True)\
-                .put('api', 'http_port', options.restapi)
+            config.api.http_enabled = True
+            config.api.http_port = options.restapi
 
         if options.ipv8 > 0:
-            config.put('ipv8', 'port', options.ipv8)
+            config.ipv8.port = options.ipv8
         elif options.ipv8 == 0:
-            config.put('ipv8', 'enabled', False)
+            config.ipv8.enabled = False
 
         if options.libtorrent != -1 and options.libtorrent > 0:
-            config.put('libtorrent', 'port', options.libtorrent)
+            config.libtorrent.port = options.libtorrent
 
         if options.ipv8_bootstrap_override is not None:
-            config.put('ipv8', 'bootstrap_override', options.ipv8_bootstrap_override)
+            config.ipv8.bootstrap_override = options.ipv8_bootstrap_override
 
         if options.testnet:
-            config\
-                .put('tunnel_community', 'testnet', True)\
-                .put('chant', 'testnet', True)\
-                .put('bandwidth_accounting', 'testnet', True)
+            config.tunnel_community.testnet = True
+            config.chant.testnet = True
+            config.bandwidth_accounting.testnet = True
 
         self.session = Session(config)
         try:

src/tribler-core/run_tunnel_helper.py

@@ -115,46 +115,42 @@ async def start(self, options):
             ipv8_port = base_port + int(os.environ["HELPER_INDEX"]) * 5
 
         statedir = Path(os.path.join(get_root_state_directory(), "tunnel-%d") % ipv8_port)
-        config = TriblerConfig(statedir)\
-            .load(file=statedir / 'triblerd.conf')\
-            .put('tunnel_community', 'socks5_listen_ports', [])\
-            .put('tunnel_community', 'random_slots', options.random_slots)\
-            .put('tunnel_community', 'competing_slots', options.competing_slots)\
-            .put('torrent_checking', 'enabled', False)\
-            .put('ipv8', 'enabled', True)\
-            .put('libtorrent', 'enabled', False)\
-            .put('ipv8', 'port', ipv8_port)\
-            .put('ipv8', 'address', options.ipv8_address)\
-            .put('dht', 'enabled', True)\
-            .put('tunnel_community', 'exitnode_enabled', bool(options.exit))\
-            .put('popularity_community', 'enabled', False)\
-            .put('tunnel_community', 'testnet', bool(options.testnet))\
-            .put('chant', 'enabled', False)\
-            .put('bootstrap', 'enabled', False)
+        config = TriblerConfig.load(file=statedir / 'triblerd.conf', state_dir=statedir)
+        config.tunnel_community.socks5_listen_ports = []
+        config.tunnel_community.random_slots = options.random_slots
+        config.tunnel_community.competing_slots = options.competing_slots
+        config.torrent_checking.enabled = False
+        config.ipv8.enabled = True
+        config.libtorrent.enabled = False
+        config.ipv8.port = ipv8_port
+        config.ipv8.address = options.ipv8_address
+        config.dht.enabled = True
+        config.tunnel_community.exitnode_enabled = bool(options.exit)
+        config.popularity_community.enabled = False
+        config.tunnel_community.testnet = bool(options.testnet)
+        config.chant.enabled = False
+        config.bootstrap.enabled = False
 
         if not options.no_rest_api:
             https = bool(options.cert_file)
-            config\
-                .put('api', 'https_enabled', https)\
-                .put('api', 'http_enabled', not https)\
-                .put('api', 'key', options.api_key)
+            config.api.https_enabled = https
+            config.api.http_enabled = not https
+            config.api.key = options.api_key
 
             api_port = options.restapi
             if "HELPER_INDEX" in os.environ and "HELPER_BASE" in os.environ:
                 api_port = int(os.environ["HELPER_BASE"]) + 10000 + int(os.environ["HELPER_INDEX"])
             if https:
-                config\
-                    .put('api', 'https_port', api_port)        \
-                    .put_path('api', 'https_certfile', options.cert_file)
+                config.api.https_port = api_port
+                config.api.put_path_as_relative('https_certfile', options.cert_file, config.state_dir)
             else:
-                config.put('api', 'http_port', api_port)
+                config.api.http_port = api_port
         else:
-            config\
-                .put('api', 'https_enabled', False)\
-                .put('api', 'http_enabled', False)
+            config.api.https_enabled = False
+            config.api.http_enabled = False
 
         if options.ipv8_bootstrap_override is not None:
-            config.put('ipv8', 'bootstrap_override', options.ipv8_bootstrap_override)
+            config.ipv8.bootstrap_override = options.ipv8_bootstrap_override
 
         self.session = Session(config)
 

src/tribler-core/tribler_core/config/tests/test_tribler_config.py

@@ -1,81 +1,88 @@
 import shutil
 
-from configobj import ParseError
-
 import pytest
+from configobj import ParseError
 
 from tribler_core.config.tribler_config import TriblerConfig
 from tribler_core.modules.libtorrent.download_manager import DownloadManager
 from tribler_core.tests.tools.common import TESTS_DATA_DIR
 from tribler_core.utilities.path_util import Path
 
-# fmt: off
 CONFIG_PATH = TESTS_DATA_DIR / "config_files"
 
 
-@pytest.mark.asyncio
-async def test_copy(tribler_config):
-    tribler_config.put('api', 'http_port', 42)
-
-    cloned = tribler_config.copy()
-    assert cloned.get('api', 'http_port') == 42
+# fmt: off
 
 
 @pytest.mark.asyncio
-async def test_put_path_relative(tmpdir):
+async def test_create(tmpdir):
     config = TriblerConfig(state_dir=tmpdir)
+    assert config
+    assert config.state_dir == Path(tmpdir)
+
 
-    # put correct path
-    config.put_path(section_name='general', property_name='log_dir', value=Path(tmpdir))
-    assert config.config['general']['log_dir'] == '.'
+@pytest.mark.asyncio
+async def test_base_getters_and_setters(tmpdir):
+    config = TriblerConfig(state_dir=tmpdir)
+    assert config.state_dir == Path(tmpdir)
 
-    config.put_path(section_name='general', property_name='log_dir', value=Path(tmpdir) / '1')
-    assert config.config['general']['log_dir'] == '1'
+    config.set_state_dir('.')
+    assert config.state_dir == Path('.')
 
 
 @pytest.mark.asyncio
-async def test_put_path_absolute(tmpdir):
+async def test_load_write(tmpdir):
     config = TriblerConfig(state_dir=tmpdir)
+    filename = 'test_read_write.ini'
 
-    config.put_path(section_name='general', property_name='log_dir', value=None)
-    assert not config.config['general']['log_dir']
+    config.general.log_dir = '1'
+    config.general.version_checker_enabled = False
+    config.libtorrent.port = None
+    config.libtorrent.proxy_type = 2
+    config.libtorrent.anon_proxy_server_ports = ['3'] * 5
 
-    config.put_path(section_name='general', property_name='log_dir', value=Path(tmpdir).parent)
-    assert config.config['general']['log_dir'] == str(Path(tmpdir).parent)
+    assert not config.file
+    config.write(tmpdir / filename)
+    assert config.file == tmpdir / filename
 
-    config.put_path(section_name='general', property_name='log_dir', value=Path('/Tribler'))
-    assert config.config['general']['log_dir'] == str(Path('/Tribler'))
+    config = TriblerConfig.load(file=tmpdir / filename, state_dir=tmpdir)
+    assert config.general.log_dir == '1'
+    assert config.general.version_checker_enabled is False
+    assert config.libtorrent.port is None
+    assert config.libtorrent.proxy_type == 2
+    assert config.libtorrent.anon_proxy_server_ports == ['3'] * 5
+    assert config.file == tmpdir / filename
 
 
 @pytest.mark.asyncio
-async def test_get_path_relative(tmpdir):
-    config = TriblerConfig(state_dir=tmpdir)
-
-    config.config['general']['log_dir'] = None
-    assert not config.get_path(section_name='general', property_name='log_dir')
+async def test_copy(tmpdir):
+    config = TriblerConfig(state_dir=tmpdir, file=tmpdir / '1.txt')
+    config.api.http_port = 42
 
-    config.config['general']['log_dir'] = '.'
-    assert config.get_path(section_name='general', property_name='log_dir') == Path(tmpdir)
-
-    config.config['general']['log_dir'] = '1'
-    assert config.get_path(section_name='general', property_name='log_dir') == Path(tmpdir) / '1'
+    cloned = config.copy()
+    assert cloned.api.http_port == 42
+    assert cloned.state_dir == tmpdir
+    assert cloned.file == tmpdir / '1.txt'
 
 
 @pytest.mark.asyncio
-async def test_get_path_absolute(tmpdir):
+async def test_get_path_relative(tmpdir):
     config = TriblerConfig(state_dir=tmpdir)
+    config.general.log_dir = None
+    assert not config.general.log_dir
+
+    config.general.log_dir = '.'
+    assert config.general.get_path_as_absolute('log_dir', tmpdir) == Path(tmpdir)
 
-    config.config['general']['log_dir'] = str(Path(tmpdir).parent)
-    assert config.get_path(section_name='general', property_name='log_dir') == Path(tmpdir).parent
+    config.general.log_dir = '1'
+    assert config.general.get_path_as_absolute('log_dir', tmpdir) == Path(tmpdir) / '1'
 
 
 @pytest.mark.asyncio
-async def test_init_without_config():
-    """
-    A newly created TriblerConfig is valid.
-    """
-    config = TriblerConfig()
-    assert config.config
+async def test_get_path_absolute(tmpdir):
+    config = TriblerConfig(state_dir=tmpdir)
+    config.general.log_dir = str(Path(tmpdir).parent)
+    assert config.general.get_path_as_absolute(property_name='log_dir', state_dir=tmpdir) == Path(tmpdir).parent
 
 
 @pytest.mark.asyncio
@@ -85,18 +92,17 @@ async def test_invalid_config_recovers(tmpdir):
 
     # By default, recover_error set to False when loading the config file so
     # if the config file is corrupted, it should raise a ParseError.
-    config = TriblerConfig(tmpdir)
     with pytest.raises(ParseError):
-        config.load(file=default_config_file)
+        TriblerConfig.load(file=default_config_file, state_dir=tmpdir)
 
     # If recover_error is set to True, the config should successfully load using
     # the default config in case of corrupted config file and the error is saved.
-    config.load(file=default_config_file, reset_config_on_error=True)
+    config = TriblerConfig.load(file=default_config_file, state_dir=tmpdir, reset_config_on_error=True)
     assert "configobj.ParseError: Invalid line" in config.error
 
     # Since the config should be saved on previous recovery, subsequent instantiation of TriblerConfig
     # should work without the reset flag.
-    config.load(file=default_config_file)
+    config = TriblerConfig.load(file=default_config_file, state_dir=tmpdir)
     assert not config.error