Merge branch 'move-pylin-plugin' of https://github.com/chrisjsewell/a…

…iida_core into move-pylin-plugin

aiida/backends/djsite/db/migrations/0047_migrate_repository.py

@@ -35,10 +35,12 @@ def migrate_repository(apps, schema_editor):
     from aiida.common import exceptions
     from aiida.common.progress_reporter import get_progress_reporter, set_progress_bar_tqdm, set_progress_reporter
     from aiida.manage.configuration import get_profile
+    from aiida.manage.manager import get_manager
 
     DbNode = apps.get_model('db', 'DbNode')
 
     profile = get_profile()
+    backend = get_manager().get_backend()
     node_count = DbNode.objects.count()
     missing_node_uuids = []
     missing_repo_folder = []
@@ -107,7 +109,7 @@ def migrate_repository(apps, schema_editor):
     # Store the UUID of the repository container in the `DbSetting` table. Note that for new databases, the profile
     # setup will already have stored the UUID and so it should be skipped, or an exception for a duplicate key will be
     # raised. This migration step is only necessary for existing databases that are migrated.
-    container_id = profile.get_repository().uuid
+    container_id = backend.get_repository().uuid
     with schema_editor.connection.cursor() as cursor:
         cursor.execute(
             f"""

aiida/backends/sqlalchemy/migrations/versions/1feaea71bd5a_migrate_repository.py

@@ -35,6 +35,7 @@ def upgrade():
     from aiida.common import exceptions
     from aiida.common.progress_reporter import get_progress_reporter, set_progress_bar_tqdm, set_progress_reporter
     from aiida.manage.configuration import get_profile
+    from aiida.manage.manager import get_manager
 
     connection = op.get_bind()
 
@@ -46,6 +47,7 @@ def upgrade():
     )
 
     profile = get_profile()
+    backend = get_manager().get_backend()
     node_count = connection.execute(select(func.count()).select_from(DbNode)).scalar()
     missing_repo_folder = []
     shard_count = 256
@@ -106,7 +108,7 @@ def upgrade():
     # Store the UUID of the repository container in the `DbSetting` table. Note that for new databases, the profile
     # setup will already have stored the UUID and so it should be skipped, or an exception for a duplicate key will be
     # raised. This migration step is only necessary for existing databases that are migrated.
-    container_id = profile.get_repository().uuid
+    container_id = backend.get_repository().uuid
     statement = text(
         f"""
         INSERT INTO db_dbsetting (key, val, description)

aiida/backends/testbase.py

@@ -142,7 +142,7 @@ def initialise_repository(cls):
         """Initialise the repository"""
         from aiida.manage.configuration import get_profile
         profile = get_profile()
-        repository = profile.get_repository()
+        repository = cls.backend.get_repository()
         repository.initialise(clear=True, **profile.defaults['repository'])
 
     @classmethod

aiida/cmdline/commands/cmd_setup.py

@@ -94,7 +94,7 @@ def setup(
     # with that UUID and we have to make sure that the provided repository corresponds to it.
     backend_manager = manager.get_backend_manager()
     repository_uuid_database = backend_manager.get_repository_uuid()
-    repository_uuid_profile = profile.get_repository().uuid
+    repository_uuid_profile = backend.get_repository().uuid
 
     if repository_uuid_database != repository_uuid_profile:
         echo.echo_critical(

aiida/cmdline/commands/cmd_status.py

@@ -85,7 +85,7 @@ def verdi_status(print_traceback, no_rmq):
 
     # Getting the repository
     try:
-        repository = profile.get_repository()
+        repository = manager.get_backend().get_repository()
     except Exception as exc:
         message = 'Error with repository folder'
         print_status(ServiceStatus.ERROR, 'repository', message, exception=exc, print_traceback=print_traceback)

aiida/cmdline/utils/common.py

@@ -11,11 +11,15 @@
 import logging
 import os
 import sys
+from typing import TYPE_CHECKING
 
 from tabulate import tabulate
 
 from . import echo
 
+if TYPE_CHECKING:
+    from aiida.orm import WorkChainNode
+
 __all__ = ('is_verbose',)
 
 
@@ -306,7 +310,7 @@ def get_process_function_report(node):
     return '\n'.join(report)
 
 
-def get_workchain_report(node, levelname, indent_size=4, max_depth=None):
+def get_workchain_report(node: 'WorkChainNode', levelname, indent_size=4, max_depth=None):
     """
     Return a multi line string representation of the log messages and output of a given workchain
 
@@ -333,7 +337,7 @@ def get_subtree(uuid, level=0):
         Get a nested tree of work calculation nodes and their nesting level starting from this uuid.
         The result is a list of uuid of these nodes.
         """
-        builder = orm.QueryBuilder()
+        builder = orm.QueryBuilder(backend=node.backend)
         builder.append(cls=orm.WorkChainNode, filters={'uuid': uuid}, tag='workcalculation')
         builder.append(
             cls=orm.WorkChainNode,

aiida/manage/configuration/config.py

@@ -361,8 +361,9 @@ def delete_profile(
         profile = self.get_profile(name)
 
         if include_repository:
-            repository = profile.get_repository()
-            repository.delete()
+            folder = profile.repository_path
+            if folder.exists():
+                shutil.rmtree(folder)
 
         if include_database:
             postgres = Postgres.from_profile(profile)

aiida/manage/configuration/profile.py

@@ -128,16 +128,6 @@ def __init__(self, name, attributes, from_config=False):
         # Currently, whether a profile is a test profile is solely determined by its name starting with 'test_'
         self._test_profile = bool(self.name.startswith('test_'))
 
-    def get_repository(self) -> 'Repository':
-        """Return the repository configured for this profile."""
-        from disk_objectstore import Container
-
-        from aiida.repository import Repository
-        from aiida.repository.backend import DiskObjectStoreRepositoryBackend
-        container = Container(self.repository_path / 'container')
-        backend = DiskObjectStoreRepositoryBackend(container=container)
-        return Repository(backend=backend)
-
     @property
     def uuid(self):
         """Return the profile uuid.

aiida/manage/manager.py

@@ -132,11 +132,23 @@ def _load_backend(self, schema_check: bool = True, repository_check: bool = True
             backend_manager.load_backend_environment(profile, validate_schema=schema_check)
             configuration.BACKEND_UUID = profile.uuid
 
+        backend_type = profile.database_backend
+
+        # Can only import the backend classes after the backend has been loaded
+        if backend_type == BACKEND_DJANGO:
+            from aiida.orm.implementation.django.backend import DjangoBackend
+            self._backend = DjangoBackend()
+        elif backend_type == BACKEND_SQLA:
+            from aiida.orm.implementation.sqlalchemy.backend import SqlaBackend
+            self._backend = SqlaBackend()
+        else:
+            raise ValueError(f'unknown database backend type: {backend_type}')
+
         # Perform the check on the repository compatibility. Since this is new functionality and the stability is not
         # yet known, we issue a warning in the case the repo and database are incompatible. In the future this might
         # then become an exception once we have verified that it is working reliably.
         if repository_check and not profile.is_test_profile:
-            repository_uuid_config = profile.get_repository().uuid
+            repository_uuid_config = self._backend.get_repository().uuid
             repository_uuid_database = backend_manager.get_repository_uuid()
 
             from aiida.cmdline.utils import echo
@@ -149,16 +161,6 @@ def _load_backend(self, schema_check: bool = True, repository_check: bool = True
                     'Please make sure that the configuration of your profile is correct.\n'
                 )
 
-        backend_type = profile.database_backend
-
-        # Can only import the backend classes after the backend has been loaded
-        if backend_type == BACKEND_DJANGO:
-            from aiida.orm.implementation.django.backend import DjangoBackend
-            self._backend = DjangoBackend()
-        elif backend_type == BACKEND_SQLA:
-            from aiida.orm.implementation.sqlalchemy.backend import SqlaBackend
-            self._backend = SqlaBackend()
-
         # Reconfigure the logging with `with_orm=True` to make sure that profile specific logging configuration options
         # are taken into account and the `DbLogHandler` is configured.
         configure_logging(with_orm=True)

aiida/orm/computers.py

@@ -84,7 +84,7 @@ def objects(cls) -> ComputerCollection:  # pylint: disable=no-self-argument
     def __init__(  # pylint: disable=too-many-arguments
         self,
         label: str = None,
-        hostname: str = None,
+        hostname: str = '',
         description: str = '',
         transport_type: str = '',
         scheduler_type: str = '',
@@ -137,7 +137,7 @@ def _hostname_validator(cls, hostname: str) -> None:
         """
         Validates the hostname.
         """
-        if not hostname.strip():
+        if not (hostname or hostname.strip()):
             raise exceptions.ValidationError('No hostname specified')
 
     @classmethod

aiida/orm/implementation/backends.py

@@ -25,14 +25,24 @@
         BackendQueryBuilder,
         BackendUserCollection,
     )
+    from aiida.repository.backend.abstract import AbstractRepositoryBackend
 
 __all__ = ('Backend',)
 
 TransactionType = TypeVar('TransactionType')
 
 
 class Backend(abc.ABC):
-    """The public interface that defines a backend factory that creates backend specific concrete objects."""
+    """Abstraction for a backend to read/write persistent data for a profile's provenance graph.
+
+    AiiDA splits data storage into two sources:
+
+    - Searchable data, which is stored in the database and can be queried using the QueryBuilder
+    - Non-searchable data, which is stored in the repository and can be loaded using the RepositoryBackend
+
+    The two sources are inter-linked by the ``Node.repository_metadata``.
+    Once stored, the leaf values of this dictionary must be valid pointers to object keys in the repository.
+    """
 
     @abc.abstractmethod
     def migrate(self) -> None:
@@ -135,3 +145,7 @@ def delete_nodes_and_connections(self, pks_to_delete: Sequence[int]):
 
         :raises: ``AssertionError`` if a transaction is not active
         """
+
+    @abc.abstractmethod
+    def get_repository(self) -> 'AbstractRepositoryBackend':
+        """Return the object repository configured for this backend."""

aiida/orm/implementation/django/comments.py

@@ -168,7 +168,7 @@ def delete_many(self, filters):
             raise exceptions.ValidationError('filters must not be empty')
 
         # Apply filter and delete found entities
-        builder = QueryBuilder().append(Comment, filters=filters, project='id').all()
+        builder = QueryBuilder(backend=self.backend).append(Comment, filters=filters, project='id').all()
         entities_to_delete = [_[0] for _ in builder]
         for entity in entities_to_delete:
             self.delete(entity)

aiida/orm/implementation/django/logs.py

@@ -144,7 +144,7 @@ def delete_many(self, filters):
             raise exceptions.ValidationError('filters must not be empty')
 
         # Apply filter and delete found entities
-        builder = QueryBuilder().append(Log, filters=filters, project='id')
+        builder = QueryBuilder(backend=self.backend).append(Log, filters=filters, project='id')
         entities_to_delete = builder.all(flat=True)
         for entity in entities_to_delete:
             self.delete(entity)

aiida/orm/implementation/querybuilder.py

@@ -41,7 +41,7 @@ class EntityTypes(Enum):
 
 
 EntityRelationships: Dict[str, Set[str]] = {
-    'authinfo': set(),
+    'authinfo': {'with_computer', 'with_user'},
     'comment': {'with_node', 'with_user'},
     'computer': {'with_node'},
     'group': {'with_node', 'with_user'},

aiida/orm/implementation/sql/backends.py

@@ -9,17 +9,20 @@
 ###########################################################################
 """Generic backend related objects"""
 import abc
-import typing
+from typing import TYPE_CHECKING, Generic, TypeVar
 
 from .. import backends, entities
 
+if TYPE_CHECKING:
+    from aiida.repository.backend import DiskObjectStoreRepositoryBackend
+
 __all__ = ('SqlBackend',)
 
 # The template type for the base sqlalchemy/django ORM model type
-ModelType = typing.TypeVar('ModelType')  # pylint: disable=invalid-name
+ModelType = TypeVar('ModelType')  # pylint: disable=invalid-name
 
 
-class SqlBackend(typing.Generic[ModelType], backends.Backend):
+class SqlBackend(Generic[ModelType], backends.Backend):
     """
     A class for SQL based backends.  Assumptions are that:
         * there is an ORM
@@ -29,6 +32,17 @@ class SqlBackend(typing.Generic[ModelType], backends.Backend):
     if any of these assumptions do not fit then just implement a backend from :class:`aiida.orm.implementation.Backend`
     """
 
+    def get_repository(self) -> 'DiskObjectStoreRepositoryBackend':
+        from disk_objectstore import Container
+
+        from aiida.manage.manager import get_manager
+        from aiida.repository.backend import DiskObjectStoreRepositoryBackend
+
+        profile = get_manager().get_profile()
+        assert profile is not None, 'profile not loaded'
+        container = Container(profile.repository_path / 'container')
+        return DiskObjectStoreRepositoryBackend(container=container)
+
     @abc.abstractmethod
     def get_backend_entity(self, model: ModelType) -> entities.BackendEntity:
         """

aiida/orm/implementation/sqlalchemy/authinfos.py

@@ -8,8 +8,6 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Module for the SqlAlchemy backend implementation of the `AuthInfo` ORM class."""
-
-from aiida.backends.sqlalchemy import get_scoped_session
 from aiida.backends.sqlalchemy.models.authinfo import DbAuthInfo
 from aiida.common import exceptions
 from aiida.common.lang import type_check
@@ -123,7 +121,7 @@ def delete(self, pk):
         # pylint: disable=import-error,no-name-in-module
         from sqlalchemy.orm.exc import NoResultFound
 
-        session = get_scoped_session()
+        session = self.backend.get_session()
 
         try:
             session.query(DbAuthInfo).filter_by(id=pk).one().delete()
@@ -143,7 +141,7 @@ def get(self, computer, user):
         # pylint: disable=import-error,no-name-in-module
         from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
 
-        session = get_scoped_session()
+        session = self.backend.get_session()
 
         try:
             authinfo = session.query(DbAuthInfo).filter_by(dbcomputer_id=computer.id, aiidauser_id=user.id).one()

aiida/orm/implementation/sqlalchemy/comments.py

@@ -14,7 +14,6 @@
 
 from sqlalchemy.orm.exc import NoResultFound
 
-from aiida.backends.sqlalchemy import get_scoped_session
 from aiida.backends.sqlalchemy.models import comment as models
 from aiida.common import exceptions, lang
 
@@ -125,7 +124,7 @@ def delete(self, comment_id):
         if not isinstance(comment_id, int):
             raise TypeError('comment_id must be an int')
 
-        session = get_scoped_session()
+        session = self.backend.get_session()
 
         try:
             session.query(models.DbComment).filter_by(id=comment_id).one().delete()
@@ -140,7 +139,7 @@ def delete_all(self):
 
         :raises `~aiida.common.exceptions.IntegrityError`: if all Comments could not be deleted
         """
-        session = get_scoped_session()
+        session = self.backend.get_session()
 
         try:
             session.query(models.DbComment).delete()
@@ -171,7 +170,7 @@ def delete_many(self, filters):
             raise exceptions.ValidationError('filters must not be empty')
 
         # Apply filter and delete found entities
-        builder = QueryBuilder().append(Comment, filters=filters, project='id')
+        builder = QueryBuilder(backend=self.backend).append(Comment, filters=filters, project='id')
         entities_to_delete = builder.all(flat=True)
         for entity in entities_to_delete:
             self.delete(entity)

aiida/orm/implementation/sqlalchemy/computers.py

@@ -15,7 +15,6 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm.session import make_transient
 
-from aiida.backends.sqlalchemy import get_scoped_session
 from aiida.backends.sqlalchemy.models.computer import DbComputer
 from aiida.common import exceptions
 from aiida.orm.implementation.computers import BackendComputer, BackendComputerCollection
@@ -52,7 +51,7 @@ def is_stored(self):
 
     def copy(self):
         """Create an unstored clone of an already stored `Computer`."""
-        session = get_scoped_session()
+        session = self.backend.get_session()
 
         if not self.is_stored:
             raise exceptions.InvalidOperation('You can copy a computer only after having stored it')
@@ -128,14 +127,13 @@ class SqlaComputerCollection(BackendComputerCollection):
 
     ENTITY_CLASS = SqlaComputer
 
-    @staticmethod
-    def list_names():
-        session = get_scoped_session()
+    def list_names(self):
+        session = self.backend.get_session()
         return session.query(DbComputer.label).all()
 
     def delete(self, pk):
         try:
-            session = get_scoped_session()
+            session = self.backend.get_session()
             session.get(DbComputer, pk).delete()
             session.commit()
         except SQLAlchemyError as exc: