diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index 256d2e1d2c..c86eca6cef 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -120,6 +120,7 @@ jobs: - name: Run test suite env: + AIIDA_WARN_v3: 1 SQLALCHEMY_WARN_20: 1 run: .github/workflows/tests.sh diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index cd557ef025..98de807201 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -238,6 +238,7 @@ jobs: - name: Run test suite env: + AIIDA_WARN_v3: 1 SQLALCHEMY_WARN_20: 1 run: .github/workflows/tests.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07b1a60afd..24d9688979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,6 +97,7 @@ repos: aiida/orm/nodes/data/jsonable.py| aiida/orm/nodes/node.py| aiida/orm/nodes/process/.*py| + aiida/orm/nodes/repository.py| aiida/orm/utils/links.py| aiida/plugins/entry_point.py| aiida/plugins/factories.py| diff --git a/aiida/cmdline/commands/cmd_calcjob.py b/aiida/cmdline/commands/cmd_calcjob.py index 2d8dcdc387..119f7f1eb1 100644 --- a/aiida/cmdline/commands/cmd_calcjob.py +++ b/aiida/cmdline/commands/cmd_calcjob.py @@ -111,7 +111,7 @@ def calcjob_inputcat(calcjob, path): try: # When we `cat`, it makes sense to directly send the output to stdout as it is - with calcjob.open(path, mode='rb') as fhandle: + with calcjob.base.repository.open(path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. @@ -163,7 +163,7 @@ def calcjob_outputcat(calcjob, path): try: # When we `cat`, it makes sense to directly send the output to stdout as it is - with retrieved.open(path, mode='rb') as fhandle: + with retrieved.base.repository.open(path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. diff --git a/aiida/cmdline/commands/cmd_node.py b/aiida/cmdline/commands/cmd_node.py index 1607f992d0..28a049a5ac 100644 --- a/aiida/cmdline/commands/cmd_node.py +++ b/aiida/cmdline/commands/cmd_node.py @@ -44,7 +44,7 @@ def repo_cat(node, relative_path): import sys try: - with node.open(relative_path, mode='rb') as fhandle: + with node.base.repository.open(relative_path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. @@ -96,7 +96,7 @@ def _copy_tree(key, output_dir): # pylint: disable=too-many-branches Recursively copy the content at the ``key`` path in the given node to the ``output_dir``. """ - for file in node.list_objects(key): + for file in node.base.repository.list_objects(key): # Not using os.path.join here, because this is the "path" # in the AiiDA node, not an actual OS - level path. file_key = file.name if not key else f'{key}/{file.name}' @@ -110,7 +110,7 @@ def _copy_tree(key, output_dir): # pylint: disable=too-many-branches assert file.file_type == FileType.FILE out_file_path = output_dir / file.name assert not out_file_path.exists() - with node.open(file_key, 'rb') as in_file: + with node.base.repository.open(file_key, 'rb') as in_file: with out_file_path.open('wb') as out_file: shutil.copyfileobj(in_file, out_file) diff --git a/aiida/cmdline/utils/repository.py b/aiida/cmdline/utils/repository.py index c507488fb8..01c18a096b 100644 --- a/aiida/cmdline/utils/repository.py +++ b/aiida/cmdline/utils/repository.py @@ -20,6 +20,6 @@ def list_repository_contents(node, path, color): """ from aiida.repository import FileType - for entry in node.list_objects(path): + for entry in node.base.repository.list_objects(path): bold = bool(entry.file_type == FileType.DIRECTORY) echo.echo(entry.name, bold=bold, fg='blue' if color and entry.file_type == FileType.DIRECTORY else None) diff --git a/aiida/common/json.py b/aiida/common/json.py index 7151ded476..ed8f9fd1aa 100644 --- a/aiida/common/json.py +++ b/aiida/common/json.py @@ -17,13 +17,11 @@ """ import codecs import json -import warnings -from aiida.common.warnings import AiidaDeprecationWarning +from aiida.common.warnings import warn_deprecation -warnings.warn( - 'This module has been deprecated and should no longer be used. Use the `json` standard library instead.', - AiidaDeprecationWarning +warn_deprecation( + 'This module has been deprecated and should no longer be used. Use the `json` standard library instead.', version=3 ) diff --git a/aiida/common/warnings.py b/aiida/common/warnings.py index 12baa1ff2b..32b0793b48 100644 --- a/aiida/common/warnings.py +++ b/aiida/common/warnings.py @@ -8,6 +8,8 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Define warnings that can be thrown by AiiDA.""" +import os +import warnings class AiidaDeprecationWarning(Warning): @@ -32,3 +34,17 @@ class AiidaTestWarning(Warning): """ Class for warnings concerning the AiiDA testing infrastructure. """ + + +def warn_deprecation(message: str, version: int, stacklevel=2) -> None: + """Warns about a deprecation for a future aiida-core version. + + Warnings are activated if the `AIIDA_WARN_v{major}` environment variable is set to `True`. + + :param message: the message to be printed + :param version: the major version number of the future version + :param stacklevel: the stack level at which the warning is issued + """ + if os.environ.get(f'AIIDA_WARN_v{version}'): + message = f'{message} (this will be removed in v{version})' + warnings.warn(message, AiidaDeprecationWarning, stacklevel=stacklevel) diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py index 31d0f49e0b..e50397b147 100644 --- a/aiida/engine/daemon/execmanager.py +++ b/aiida/engine/daemon/execmanager.py @@ -175,12 +175,12 @@ def upload_calculation( for code in input_codes: if code.is_local(): # Note: this will possibly overwrite files - for filename in code.list_object_names(): + for filename in code.base.repository.list_object_names(): # Note, once #2579 is implemented, use the `node.open` method instead of the named temporary file in # combination with the new `Transport.put_object_from_filelike` # Since the content of the node could potentially be binary, we read the raw bytes and pass them on with NamedTemporaryFile(mode='wb+') as handle: - handle.write(code.get_object_content(filename, mode='rb')) + handle.write(code.base.repository.get_object_content(filename, mode='rb')) handle.flush() transport.put(handle.name, filename) transport.chmod(code.get_local_executable(), 0o755) # rwxr-xr-x @@ -212,14 +212,14 @@ def upload_calculation( filepath_target = pathlib.Path(folder.abspath) / filename_target filepath_target.parent.mkdir(parents=True, exist_ok=True) - if data_node.get_object(filename_source).file_type == FileType.DIRECTORY: + if data_node.base.repository.get_object(filename_source).file_type == FileType.DIRECTORY: # If the source object is a directory, we copy its entire contents - data_node.copy_tree(filepath_target, filename_source) - provenance_exclude_list.extend(data_node.list_object_names(filename_source)) + data_node.base.repository.copy_tree(filepath_target, filename_source) + provenance_exclude_list.extend(data_node.base.repository.list_object_names(filename_source)) else: # Otherwise, simply copy the file with folder.open(target, 'wb') as handle: - with data_node.open(filename, 'rb') as source: + with data_node.base.repository.open(filename, 'rb') as source: shutil.copyfileobj(source, handle) provenance_exclude_list.append(target) @@ -320,12 +320,12 @@ def upload_calculation( dirname not in provenance_exclude_list for dirname in dirnames ): with open(filepath, 'rb') as handle: # type: ignore[assignment] - node._repository.put_object_from_filelike(handle, relpath) # pylint: disable=protected-access + node.base.repository._repository.put_object_from_filelike(handle, relpath) # pylint: disable=protected-access # Since the node is already stored, we cannot use the normal repository interface since it will raise a # `ModificationNotAllowed` error. To bypass it, we go straight to the underlying repository instance to store the # files, however, this means we have to manually update the node's repository metadata. - node._update_repository_metadata() # pylint: disable=protected-access + node.base.repository._update_repository_metadata() # pylint: disable=protected-access if not dry_run: # Make sure that attaching the `remote_folder` with a link is the last thing we do. This gives the biggest @@ -465,7 +465,7 @@ def retrieve_calculation(calculation: CalcJobNode, transport: Transport, retriev with SandboxFolder() as folder: retrieve_files_from_list(calculation, transport, folder.abspath, retrieve_list) # Here I retrieved everything; now I store them inside the calculation - retrieved_files.put_object_from_tree(folder.abspath) + retrieved_files.base.repository.put_object_from_tree(folder.abspath) # Retrieve the temporary files in the retrieved_temporary_folder if any files were # specified in the 'retrieve_temporary_list' key diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index d815389ed8..c6fa45cfb0 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -519,7 +519,7 @@ def parse_scheduler_output(self, retrieved: orm.Node) -> Optional[ExitCode]: self.logger.warning('could not determine `stderr` filename because `scheduler_stderr` option was not set.') else: try: - scheduler_stderr = retrieved.get_object_content(filename_stderr) + scheduler_stderr = retrieved.base.repository.get_object_content(filename_stderr) except FileNotFoundError: scheduler_stderr = None self.logger.warning(f'could not parse scheduler output: the `{filename_stderr}` file is missing') @@ -528,7 +528,7 @@ def parse_scheduler_output(self, retrieved: orm.Node) -> Optional[ExitCode]: self.logger.warning('could not determine `stdout` filename because `scheduler_stdout` option was not set.') else: try: - scheduler_stdout = retrieved.get_object_content(filename_stdout) + scheduler_stdout = retrieved.base.repository.get_object_content(filename_stdout) except FileNotFoundError: scheduler_stdout = None self.logger.warning(f'could not parse scheduler output: the `{filename_stdout}` file is missing') diff --git a/aiida/manage/configuration/__init__.py b/aiida/manage/configuration/__init__.py index 987f1dbd4e..ebe7a580ec 100644 --- a/aiida/manage/configuration/__init__.py +++ b/aiida/manage/configuration/__init__.py @@ -54,7 +54,7 @@ from typing import TYPE_CHECKING, Any, Optional import warnings -from aiida.common.warnings import AiidaDeprecationWarning +from aiida.common.warnings import AiidaDeprecationWarning, warn_deprecation if TYPE_CHECKING: from aiida.manage.configuration import Config, Profile # pylint: disable=import-self @@ -114,9 +114,10 @@ def _merge_deprecated_cache_yaml(config, filepath): while not cache_path_backup or os.path.isfile(cache_path_backup): cache_path_backup = f"{cache_path}.{timezone.now().strftime('%Y%m%d-%H%M%S.%f')}" - warnings.warn( + warn_deprecation( 'cache_config.yml use is deprecated and support will be removed in `v3.0`. Merging into config.json and ' - f'moving to: {cache_path_backup}', AiidaDeprecationWarning + f'moving to: {cache_path_backup}', + version=3 ) import yaml with open(cache_path, 'r', encoding='utf8') as handle: diff --git a/aiida/manage/manager.py b/aiida/manage/manager.py index 29518107c3..9ebe0c8c3d 100644 --- a/aiida/manage/manager.py +++ b/aiida/manage/manager.py @@ -12,7 +12,6 @@ import asyncio import functools from typing import TYPE_CHECKING, Any, Optional, Union -from warnings import warn if TYPE_CHECKING: from kiwipy.rmq import RmqThreadCommunicator @@ -206,8 +205,8 @@ def get_backend(self) -> 'StorageBackend': Deprecated: use `get_profile_storage` instead. """ - from aiida.common.warnings import AiidaDeprecationWarning - warn('get_backend() is deprecated, use get_profile_storage() instead', AiidaDeprecationWarning) + from aiida.common.warnings import warn_deprecation + warn_deprecation('get_backend() is deprecated, use get_profile_storage() instead', version=3) return self.get_profile_storage() def get_profile_storage(self) -> 'StorageBackend': diff --git a/aiida/manage/tests/main.py b/aiida/manage/tests/main.py index 1865c1bc6b..eaec110f2c 100644 --- a/aiida/manage/tests/main.py +++ b/aiida/manage/tests/main.py @@ -15,9 +15,8 @@ import os import shutil import tempfile -import warnings -from aiida.common.warnings import AiidaDeprecationWarning +from aiida.common.warnings import warn_deprecation from aiida.manage import configuration, get_manager from aiida.manage.configuration.settings import create_instance_directories from aiida.manage.external.postgres import Postgres @@ -123,7 +122,7 @@ def has_profile_open(self): return self._manager and self._manager.has_profile_open() def reset_db(self): - warnings.warn('reset_db() is deprecated, use clear_profile() instead', AiidaDeprecationWarning) + warn_deprecation('reset_db() is deprecated, use clear_profile() instead', version=3) return self._manager.clear_profile() def clear_profile(self): diff --git a/aiida/manage/tests/pytest_fixtures.py b/aiida/manage/tests/pytest_fixtures.py index de9c0757ec..d33d76b323 100644 --- a/aiida/manage/tests/pytest_fixtures.py +++ b/aiida/manage/tests/pytest_fixtures.py @@ -21,12 +21,11 @@ import asyncio import shutil import tempfile -import warnings import pytest from aiida.common.log import AIIDA_LOGGER -from aiida.common.warnings import AiidaDeprecationWarning +from aiida.common.warnings import warn_deprecation from aiida.manage.tests import get_test_backend_name, get_test_profile_name, test_manager @@ -77,9 +76,7 @@ def clear_database(clear_database_after_test): @pytest.fixture(scope='function') def clear_database_after_test(aiida_profile): """Clear the database after the test.""" - warnings.warn( - 'the clear_database_after_test fixture is deprecated, use aiida_profile_clean instead', AiidaDeprecationWarning - ) + warn_deprecation('the clear_database_after_test fixture is deprecated, use aiida_profile_clean instead', version=3) yield aiida_profile aiida_profile.clear_profile() @@ -87,9 +84,7 @@ def clear_database_after_test(aiida_profile): @pytest.fixture(scope='function') def clear_database_before_test(aiida_profile): """Clear the database before the test.""" - warnings.warn( - 'the clear_database_before_test fixture deprecated, use aiida_profile_clean instead', AiidaDeprecationWarning - ) + warn_deprecation('the clear_database_before_test fixture deprecated, use aiida_profile_clean instead', version=3) aiida_profile.clear_profile() yield aiida_profile @@ -97,9 +92,8 @@ def clear_database_before_test(aiida_profile): @pytest.fixture(scope='class') def clear_database_before_test_class(aiida_profile): """Clear the database before a test class.""" - warnings.warn( - 'the clear_database_before_test_class is deprecated, use aiida_profile_clean_class instead', - AiidaDeprecationWarning + warn_deprecation( + 'the clear_database_before_test_class is deprecated, use aiida_profile_clean_class instead', version=3 ) aiida_profile.clear_profile() yield diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index dcf7f66104..a20396ea7f 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -72,7 +72,7 @@ 'Node', 'NodeEntityLoader', 'NodeLinksManager', - 'NodeRepositoryMixin', + 'NodeRepository', 'NumericType', 'OrbitalData', 'OrderSpecifier', diff --git a/aiida/orm/implementation/storage_backend.py b/aiida/orm/implementation/storage_backend.py index 02f3c8ba29..fe3f023107 100644 --- a/aiida/orm/implementation/storage_backend.py +++ b/aiida/orm/implementation/storage_backend.py @@ -40,7 +40,7 @@ class StorageBackend(abc.ABC): # pylint: disable=too-many-public-methods - Searchable data, which is stored in the database and can be queried using the QueryBuilder - Non-searchable (binary) data, which is stored in the repository and can be loaded using the RepositoryBackend - The two sources are inter-linked by the ``Node.repository_metadata``. + The two sources are inter-linked by the ``Node.base.repository.metadata``. Once stored, the leaf values of this dictionary must be valid pointers to object keys in the repository. The class methods,`version_profile` and `migrate`, diff --git a/aiida/orm/nodes/__init__.py b/aiida/orm/nodes/__init__.py index 99e32294d6..9cd3c65371 100644 --- a/aiida/orm/nodes/__init__.py +++ b/aiida/orm/nodes/__init__.py @@ -40,7 +40,7 @@ 'KpointsData', 'List', 'Node', - 'NodeRepositoryMixin', + 'NodeRepository', 'NumericType', 'OrbitalData', 'ProcessNode', diff --git a/aiida/orm/nodes/data/array/array.py b/aiida/orm/nodes/data/array/array.py index 1bb97e94b3..43088209a4 100644 --- a/aiida/orm/nodes/data/array/array.py +++ b/aiida/orm/nodes/data/array/array.py @@ -45,11 +45,11 @@ def delete_array(self, name): :param name: The name of the array to delete from the node. """ fname = f'{name}.npy' - if fname not in self.list_object_names(): + if fname not in self.base.repository.list_object_names(): raise KeyError(f"Array with name '{name}' not found in node pk= {self.pk}") # remove both file and attribute - self.delete_object(fname) + self.base.repository.delete_object(fname) try: self.delete_attribute(f'{self.array_prefix}{name}') except (KeyError, AttributeError): @@ -71,7 +71,7 @@ def _arraynames_from_files(self): Return a list of all arrays stored in the node, listing the files (and not relying on the properties). """ - return [i[:-4] for i in self.list_object_names() if i.endswith('.npy')] + return [i[:-4] for i in self.base.repository.list_object_names() if i.endswith('.npy')] def _arraynames_from_properties(self): """ @@ -111,11 +111,11 @@ def get_array_from_file(self, name): """Return the array stored in a .npy file""" filename = f'{name}.npy' - if filename not in self.list_object_names(): + if filename not in self.base.repository.list_object_names(): raise KeyError(f'Array with name `{name}` not found in ArrayData<{self.pk}>') # Open a handle in binary read mode as the arrays are written as binary files as well - with self.open(filename, mode='rb') as handle: + with self.base.repository.open(filename, mode='rb') as handle: return numpy.load(handle, allow_pickle=False) # pylint: disable=unexpected-keyword-arg # Return with proper caching if the node is stored, otherwise always re-read from disk @@ -171,7 +171,7 @@ def set_array(self, name, array): handle.seek(0) # Write the numpy array to the repository, keeping the byte representation - self.put_object_from_filelike(handle, f'{name}.npy') + self.base.repository.put_object_from_filelike(handle, f'{name}.npy') # Store the array name and shape for querying purposes self.set_attribute(f'{self.array_prefix}{name}', list(array.shape)) diff --git a/aiida/orm/nodes/data/code.py b/aiida/orm/nodes/data/code.py index 4306ca5aaf..081d80445b 100644 --- a/aiida/orm/nodes/data/code.py +++ b/aiida/orm/nodes/data/code.py @@ -93,7 +93,7 @@ def set_files(self, files): for filename in files: if os.path.isfile(filename): with open(filename, 'rb') as handle: - self.put_object_from_filelike(handle, os.path.split(filename)[1]) + self.base.repository.put_object_from_filelike(handle, os.path.split(filename)[1]) def __str__(self): local_str = 'Local' if self.is_local() else 'Remote' @@ -282,12 +282,12 @@ def _validate(self): 'You have to set which file is the local executable ' 'using the set_exec_filename() method' ) - if self.get_local_executable() not in self.list_object_names(): + if self.get_local_executable() not in self.base.repository.list_object_names(): raise exceptions.ValidationError( f"The local executable '{self.get_local_executable()}' is not in the list of files of this code" ) else: - if self.list_object_names(): + if self.base.repository.list_object_names(): raise exceptions.ValidationError('The code is remote but it has files inside') if not self.get_remote_computer(): raise exceptions.ValidationError('You did not specify a remote computer') diff --git a/aiida/orm/nodes/data/data.py b/aiida/orm/nodes/data/data.py index 0c80acc188..f204fc2587 100644 --- a/aiida/orm/nodes/data/data.py +++ b/aiida/orm/nodes/data/data.py @@ -69,8 +69,8 @@ def clone(self): backend_clone = self.backend_entity.clone() clone = self.__class__.from_backend_entity(backend_clone) - clone.reset_attributes(copy.deepcopy(self.attributes)) # pylint: disable=no-member - clone._repository.clone(self._repository) # pylint: disable=no-member,protected-access + clone.reset_attributes(copy.deepcopy(self.attributes)) + clone.base.repository._clone(self.base.repository) # pylint: disable=protected-access return clone diff --git a/aiida/orm/nodes/data/folder.py b/aiida/orm/nodes/data/folder.py index e85a4235f6..38c679d9ef 100644 --- a/aiida/orm/nodes/data/folder.py +++ b/aiida/orm/nodes/data/folder.py @@ -37,4 +37,4 @@ def __init__(self, **kwargs): tree = kwargs.pop('tree', None) super().__init__(**kwargs) if tree: - self.put_object_from_tree(tree) + self.base.repository.put_object_from_tree(tree) diff --git a/aiida/orm/nodes/data/singlefile.py b/aiida/orm/nodes/data/singlefile.py index 452e496f86..32e6ae63ca 100644 --- a/aiida/orm/nodes/data/singlefile.py +++ b/aiida/orm/nodes/data/singlefile.py @@ -56,7 +56,7 @@ def open(self, path=None, mode='r'): if path is None: path = self.filename - with super().open(path, mode=mode) as handle: + with self.base.repository.open(path, mode=mode) as handle: yield handle def get_content(self): @@ -94,7 +94,7 @@ def set_file(self, file, filename=None): key = filename or key - existing_object_names = self.list_object_names() + existing_object_names = self.base.repository.list_object_names() try: # Remove the 'key' from the list of currently existing objects such that it is not deleted after storing @@ -103,13 +103,13 @@ def set_file(self, file, filename=None): pass if is_filelike: - self.put_object_from_filelike(file, key) + self.base.repository.put_object_from_filelike(file, key) else: - self.put_object_from_file(file, key) + self.base.repository.put_object_from_file(file, key) # Delete any other existing objects (minus the current `key` which was already removed from the list) for existing_key in existing_object_names: - self.delete_object(existing_key) + self.base.repository.delete_object(existing_key) self.set_attribute('filename', key) @@ -122,7 +122,7 @@ def _validate(self): except AttributeError: raise exceptions.ValidationError('the `filename` attribute is not set.') - objects = self.list_object_names() + objects = self.base.repository.list_object_names() if [filename] != objects: raise exceptions.ValidationError( diff --git a/aiida/orm/nodes/node.py b/aiida/orm/nodes/node.py index 0b53719316..19bbfeee13 100644 --- a/aiida/orm/nodes/node.py +++ b/aiida/orm/nodes/node.py @@ -9,8 +9,8 @@ ########################################################################### # pylint: disable=too-many-lines,too-many-arguments """Package for node ORM classes.""" -import copy import datetime +from functools import cached_property import importlib from logging import Logger from typing import ( @@ -35,6 +35,7 @@ from aiida.common.hashing import make_hash from aiida.common.lang import classproperty, type_check from aiida.common.links import LinkType +from aiida.common.warnings import warn_deprecation from aiida.manage import get_manager from aiida.orm.utils.links import LinkManager, LinkTriple from aiida.orm.utils.node import AbstractNodeMeta @@ -45,7 +46,7 @@ from ..entities import Entity, EntityAttributesMixin, EntityExtrasMixin from ..querybuilder import QueryBuilder from ..users import User -from .repository import NodeRepositoryMixin +from .repository import NodeRepository if TYPE_CHECKING: from ..implementation import BackendNode, StorageBackend @@ -59,7 +60,7 @@ class NodeCollection(EntityCollection[NodeType], Generic[NodeType]): """The collection of nodes.""" @staticmethod - def _entity_base_cls() -> Type['Node']: + def _entity_base_cls() -> Type['Node']: # type: ignore return Node def delete(self, pk: int) -> None: @@ -101,9 +102,20 @@ def iter_repo_keys(self, yield key -class Node( - Entity['BackendNode'], NodeRepositoryMixin, EntityAttributesMixin, EntityExtrasMixin, metaclass=AbstractNodeMeta -): +class NodeBase: + """A namespace for node related functionality, that is not directly related to its user-facing properties.""" + + def __init__(self, node: 'Node') -> None: + """Construct a new instance of the base namespace.""" + self._node: 'Node' = node + + @cached_property + def repository(self) -> 'NodeRepository': + """Return the repository for this node.""" + return NodeRepository(self._node) + + +class Node(Entity['BackendNode'], EntityAttributesMixin, EntityExtrasMixin, metaclass=AbstractNodeMeta): """ Base class for all nodes in AiiDA. @@ -151,7 +163,7 @@ class Node( Collection = NodeCollection @classproperty - def objects(cls: Type[NodeType]) -> NodeCollection[NodeType]: # pylint: disable=no-self-argument + def objects(cls: Type[NodeType]) -> NodeCollection[NodeType]: # type: ignore # pylint: disable=no-self-argument return NodeCollection.get_cached(cls, get_manager().get_profile_storage()) # type: ignore[arg-type] def __init__( @@ -177,6 +189,11 @@ def __init__( ) super().__init__(backend_entity) + @cached_property + def base(self) -> NodeBase: + """Return the node base namespace.""" + return NodeBase(self) + def __eq__(self, other: Any) -> bool: """Fallback equality comparison by uuid (can be overwritten by specific types)""" if isinstance(other, Node) and self.uuid == other.uuid: @@ -325,22 +342,6 @@ def description(self, value: str) -> None: """ self.backend_entity.description = value - @property - def repository_metadata(self) -> Dict[str, Any]: - """Return the node repository metadata. - - :return: the repository metadata - """ - return self.backend_entity.repository_metadata - - @repository_metadata.setter - def repository_metadata(self, value: Dict[str, Any]) -> None: - """Set the repository metadata. - - :param value: the new value to set - """ - self.backend_entity.repository_metadata = value - @property def computer(self) -> Optional[Computer]: """Return the computer of this node.""" @@ -726,18 +727,7 @@ def _store(self, with_transaction: bool = True, clean: bool = True) -> 'Node': :param with_transaction: if False, do not use a transaction because the caller will already have opened one. :param clean: boolean, if True, will clean the attributes and extras before attempting to store """ - from aiida.repository import Repository - from aiida.repository.backend import SandboxRepositoryBackend - - # Only if the backend repository is a sandbox do we have to clone its contents to the permanent repository. - if isinstance(self._repository.backend, SandboxRepositoryBackend): - repository_backend = self.backend.get_repository() - repository = Repository(backend=repository_backend) - repository.clone(self._repository) - # Swap the sandbox repository for the new permanent repository instance which should delete the sandbox - self._repository_instance = repository - - self.repository_metadata = self._repository.serialize() + self.base.repository._store() # pylint: disable=protected-access links = self._incoming_cache self._backend_entity.store(links, with_transaction=with_transaction, clean=clean) @@ -772,7 +762,6 @@ def _store_from_cache(self, cache_node: 'Node', with_transaction: bool) -> None: """ from aiida.orm.utils.mixins import Sealable - from aiida.repository import Repository assert self.node_type == cache_node.node_type # Make sure the node doesn't have any RETURN links @@ -783,7 +772,7 @@ def _store_from_cache(self, cache_node: 'Node', with_transaction: bool) -> None: self.description = cache_node.description # Make sure to reinitialize the repository instance of the clone to that of the source node. - self._repository: Repository = copy.copy(cache_node._repository) # pylint: disable=protected-access + self.base.repository._copy(cache_node.base.repository) # pylint: disable=protected-access for key, value in cache_node.attributes.items(): if key != Sealable.SEALED_KEY: @@ -829,7 +818,6 @@ def _get_hash(self, ignore_errors: bool = True, **kwargs: Any) -> Optional[str]: def _get_objects_to_hash(self) -> List[Any]: """Return a list of objects which should be included in the hash.""" - assert self._repository is not None, 'repository not initialised' top_level_module = self.__module__.split('.', 1)[0] try: version = importlib.import_module(top_level_module).__version__ @@ -842,7 +830,7 @@ def _get_objects_to_hash(self) -> List[Any]: for key, val in self.attributes_items() if key not in self._hash_ignored_attributes and key not in self._updatable_attributes # pylint: disable=unsupported-membership-test }, - self._repository.hash(), + self.base.repository.hash(), self.computer.uuid if self.computer is not None else None ] return objects @@ -946,3 +934,36 @@ def get_description(self) -> str: """ # pylint: disable=no-self-use return '' + + _deprecated_repo_methods = { + 'copy_tree': 'copy_tree', + 'delete_object': 'delete_object', + 'get_object': 'get_object', + 'get_object_content': 'get_object_content', + 'glob': 'glob', + 'list_objects': 'list_objects', + 'list_object_names': 'list_object_names', + 'open': 'open', + 'put_object_from_filelike': 'put_object_from_filelike', + 'put_object_from_file': 'put_object_from_file', + 'put_object_from_tree': 'put_object_from_tree', + 'walk': 'walk', + 'repository_metadata': 'metadata', + } + + def __getattr__(self, name: str) -> Any: + """ + This method is called when an attribute is not found in the instance. + + It allows for the handling of deprecated mixin methods. + """ + if name in self._deprecated_repo_methods: + new_name = self._deprecated_repo_methods[name] + kls = self.__class__.__name__ + warn_deprecation( + f'`{kls}.{name}` is deprecated, use `{kls}.base.repository.{new_name}` instead.', + version=3, + stacklevel=3 + ) + return getattr(self.base.repository, new_name) + raise AttributeError(name) diff --git a/aiida/orm/nodes/process/calculation/calcjob.py b/aiida/orm/nodes/process/calculation/calcjob.py index bf4376e83d..ea06c8871f 100644 --- a/aiida/orm/nodes/process/calculation/calcjob.py +++ b/aiida/orm/nodes/process/calculation/calcjob.py @@ -494,7 +494,7 @@ def get_scheduler_stdout(self) -> Optional[AnyStr]: return None try: - stdout = retrieved_node.get_object_content(filename) + stdout = retrieved_node.base.repository.get_object_content(filename) except IOError: stdout = None @@ -512,7 +512,7 @@ def get_scheduler_stderr(self) -> Optional[AnyStr]: return None try: - stderr = retrieved_node.get_object_content(filename) + stderr = retrieved_node.base.repository.get_object_content(filename) except IOError: stderr = None diff --git a/aiida/orm/nodes/repository.py b/aiida/orm/nodes/repository.py index 67dc7bb3db..faaba4626d 100644 --- a/aiida/orm/nodes/repository.py +++ b/aiida/orm/nodes/repository.py @@ -1,21 +1,25 @@ # -*- coding: utf-8 -*- """Interface to the file repository of a node instance.""" import contextlib +import copy import io import pathlib import tempfile -from typing import BinaryIO, Dict, Iterable, Iterator, List, Tuple, Union +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, Iterator, List, Optional, TextIO, Tuple, Union from aiida.common import exceptions from aiida.repository import File, Repository from aiida.repository.backend import SandboxRepositoryBackend -__all__ = ('NodeRepositoryMixin',) +if TYPE_CHECKING: + from .node import Node + +__all__ = ('NodeRepository',) FilePath = Union[str, pathlib.PurePosixPath] -class NodeRepositoryMixin: +class NodeRepository: """Interface to the file repository of a node instance. This is the compatibility layer between the `Node` class and the `Repository` class. The repository in principle has @@ -26,19 +30,42 @@ class NodeRepositoryMixin: hierarchy if the instance was constructed normally, or from a specific hierarchy if reconstructred through the ``Repository.from_serialized`` classmethod. This is only the case for stored nodes, because unstored nodes do not have any files yet when they are constructed. Once the node get's stored, the repository is asked to serialize its - metadata contents which is then stored in the ``repository_metadata`` attribute of the node in the database. This - layer explicitly does not update the metadata of the node on a mutation action. The reason is that for stored nodes - these actions are anyway forbidden and for unstored nodes, the final metadata will be stored in one go, once the - node is stored, so there is no need to keep updating the node metadata intermediately. Note that this does mean that - ``repository_metadata`` does not give accurate information as long as the node is not yet stored. + metadata contents which is then stored in the ``repository_metadata`` field of the backend node. + This layer explicitly does not update the metadata of the node on a mutation action. + The reason is that for stored nodes these actions are anyway forbidden and for unstored nodes, + the final metadata will be stored in one go, once the node is stored, + so there is no need to keep updating the node metadata intermediately. + Note that this does mean that ``repository_metadata`` does not give accurate information, + as long as the node is not yet stored. """ - _repository_instance = None + def __init__(self, node: 'Node') -> None: + """Construct a new instance of the repository interface.""" + self._node: 'Node' = node + self._repository_instance: Optional[Repository] = None + + @property + def metadata(self) -> Dict[str, Any]: + """Return the repository metadata, representing the virtual file hierarchy. + + Note, this is only accurate if the node is stored. + + :return: the repository metadata + """ + return self._node.backend_entity.repository_metadata def _update_repository_metadata(self): - """Refresh the repository metadata of the node if it is stored and the decorated method returns successfully.""" - if self.is_stored: - self.repository_metadata = self._repository.serialize() + """Refresh the repository metadata of the node if it is stored.""" + if self._node.is_stored: + self._node.backend_entity.repository_metadata = self.serialize() + + def _check_mutability(self): + """Check if the node is mutable. + + :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. + """ + if self._node.is_stored: + raise exceptions.ModificationNotAllowed('the node is stored and therefore the repository is immutable.') @property def _repository(self) -> Repository: @@ -49,10 +76,9 @@ def _repository(self) -> Repository: :return: the file repository instance. """ if self._repository_instance is None: - if self.is_stored: - backend = self.backend.get_repository() - serialized = self.repository_metadata - self._repository_instance = Repository.from_serialized(backend=backend, serialized=serialized) + if self._node.is_stored: + backend = self._node.backend.get_repository() + self._repository_instance = Repository.from_serialized(backend=backend, serialized=self.metadata) else: self._repository_instance = Repository(backend=SandboxRepositoryBackend()) @@ -69,20 +95,49 @@ def _repository(self, repository: Repository) -> None: self._repository_instance = repository - def repository_serialize(self) -> Dict: + def _store(self) -> None: + """Store the repository in the backend.""" + if isinstance(self._repository.backend, SandboxRepositoryBackend): + # Only if the backend repository is a sandbox do we have to clone its contents to the permanent repository. + repository_backend = self._node.backend.get_repository() + repository = Repository(backend=repository_backend) + repository.clone(self._repository) + # Swap the sandbox repository for the new permanent repository instance which should delete the sandbox + self._repository_instance = repository + # update the metadata on the node backend + self._node.backend_entity.repository_metadata = self.serialize() + + def _copy(self, repo: 'NodeRepository') -> None: + """Copy a repository from another instance. + + This is used when storing cached nodes. + + :param repo: the repository to clone. + """ + self._repository = copy.copy(repo._repository) # pylint: disable=protected-access + + def _clone(self, repo: 'NodeRepository') -> None: + """Clone the repository from another instance. + + This is used when cloning a node. + + :param repo: the repository to clone. + """ + self._repository.clone(repo._repository) # pylint: disable=protected-access + + def serialize(self) -> Dict: """Serialize the metadata of the repository content into a JSON-serializable format. :return: dictionary with the content metadata. """ return self._repository.serialize() - def check_mutability(self): - """Check if the node is mutable. + def hash(self) -> str: + """Generate a hash of the repository's contents. - :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. + :return: the hash representing the contents of the repository. """ - if self.is_stored: - raise exceptions.ModificationNotAllowed('the node is stored and therefore the repository is immutable.') + return self._repository.hash() def list_objects(self, path: str = None) -> List[File]: """Return a list of the objects contained in this repository sorted by name, optionally in given sub directory. @@ -107,7 +162,7 @@ def list_object_names(self, path: str = None) -> List[str]: return self._repository.list_object_names(path) @contextlib.contextmanager - def open(self, path: str, mode='r') -> Iterator[BinaryIO]: + def open(self, path: str, mode='r') -> Iterator[Union[BinaryIO, TextIO]]: """Open a file handle to an object stored under the given key. .. note:: this should only be used to open a handle to read an existing file. To write a new file use the method @@ -156,6 +211,18 @@ def get_object_content(self, path: str, mode='r') -> Union[str, bytes]: return self._repository.get_object_content(path) + def put_object_from_bytes(self, content: bytes, path: str) -> None: + """Store the given content in the repository at the given path. + + :param path: the relative path where to store the object in the repository. + :param content: the content to store. + :raises TypeError: if the path is not a string and relative path. + :raises FileExistsError: if an object already exists at the given path. + """ + self._check_mutability() + self._repository.put_object_from_filelike(io.BytesIO(content), path) + self._update_repository_metadata() + def put_object_from_filelike(self, handle: io.BufferedReader, path: str): """Store the byte contents of a file in the repository. @@ -164,7 +231,7 @@ def put_object_from_filelike(self, handle: io.BufferedReader, path: str): :raises TypeError: if the path is not a string and relative path. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() if isinstance(handle, io.StringIO): handle = io.BytesIO(handle.read().encode('utf-8')) @@ -186,7 +253,7 @@ def put_object_from_file(self, filepath: str, path: str): :raises TypeError: if the path is not a string and relative path, or the handle is not a byte stream. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.put_object_from_file(filepath, path) self._update_repository_metadata() @@ -198,7 +265,7 @@ def put_object_from_tree(self, filepath: str, path: str = None): :raises TypeError: if the path is not a string and relative path. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.put_object_from_tree(filepath, path) self._update_repository_metadata() @@ -241,7 +308,7 @@ def delete_object(self, path: str): :raises OSError: if the file could not be deleted. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.delete_object(path) self._update_repository_metadata() @@ -250,6 +317,6 @@ def erase(self): :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.erase() self._update_repository_metadata() diff --git a/aiida/orm/querybuilder.py b/aiida/orm/querybuilder.py index fdbe53c900..ba99983872 100644 --- a/aiida/orm/querybuilder.py +++ b/aiida/orm/querybuilder.py @@ -216,10 +216,8 @@ def as_dict(self, copy: bool = True) -> QueryDictType: @property def queryhelp(self) -> 'QueryDictType': """"Legacy name for ``as_dict`` method.""" - from aiida.common.warnings import AiidaDeprecationWarning - warnings.warn( - '`QueryBuilder.queryhelp` is deprecated, use `QueryBuilder.as_dict()` instead', AiidaDeprecationWarning - ) + from aiida.common.warnings import warn_deprecation + warn_deprecation('`QueryBuilder.queryhelp` is deprecated, use `QueryBuilder.as_dict()` instead', version=3) return self.as_dict() @classmethod diff --git a/aiida/orm/utils/managers.py b/aiida/orm/utils/managers.py index efc4d927dc..1c56f112e4 100644 --- a/aiida/orm/utils/managers.py +++ b/aiida/orm/utils/managers.py @@ -12,12 +12,10 @@ to access members of other classes via TAB-completable attributes (e.g. the class underlying `calculation.inputs` to allow to do `calculation.inputs.