diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4204ac27f..41443d0643 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,8 @@ +ci: + autoupdate_schedule: monthly + autofix_prs: true + skip: [mypy, pylint, dm-generate-all, pyproject, dependencies, verdi-autodocs, version-number] + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.5.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index a24a6db506..b5dc47e78c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## v1.6.5 - 2021-08-13 + +[full changelog](https://github.com/aiidateam/aiida-core/compare/v1.6.4...v1.6.5) + +This patch release contains a number of helpful bug fixes and improvements. + +### Improvements 👌 + +- Add support for the `ProxyJump` SSH config option for seting up an arbitrary number of proxy jumps without additional processes by creating TCP channels over existing SSH connections. + This provides improved control over the lifetime of the different connections. + See [SSH configuration](docs/source/howto/ssh.rst) for further details. [[#4951]](https://github.com/aiidateam/aiida-core/pull/4951) +- Allow numpy arrays to be serialized to a process checkpoint. [[#4730)]](https://github.com/aiidateam/aiida-core/pull/4730)) +- Add the `_merge` method to `ProcessBuilder`, to update the builder with a nested dictionary. [[#4983)]](https://github.com/aiidateam/aiida-core/pull/4983)) +- `verdi setup`: Set the defaut database hostname as `localhost`. [[#4908]](https://github.com/aiidateam/aiida-core/pull/4908) +- Allow `Node.__init__` to be constructed with a specific `User` node. [[#4977]](https://github.com/aiidateam/aiida-core/pull/4977) +- Minimize database logs of failed schema version retrievals. [[#5056]](https://github.com/aiidateam/aiida-core/pull/5056) +- Remove duplicate call of normal `callback` for `InteractiveOption`. [[#5064]](https://github.com/aiidateam/aiida-core/pull/5064) +- Update requirement `pyyaml~=5.4`, which contains critical security fixes. [[#5060]](https://github.com/aiidateam/aiida-core/pull/5060) + +### Bug Fixes 🐛 + +- Fix regression issue with `__contains__` operator in `LinkManager`, when using double underscores, e.g. for `'some__nested__namespace' in calc.inputs`. [#5067](https://github.com/aiidateam/aiida-core/pull/5067) +- Stop deprecation warning being shown when tab-completing incoming and outgoing node links. [[#5011]](https://github.com/aiidateam/aiida-core/pull/5011) +- Stop possible command hints being shown when attempting to tab complete `verdi` commands that do not exist. [[#5012]](https://github.com/aiidateam/aiida-core/pull/5012) +- Do not use `get_detailed_job_info` when retrieving a calculation job, if no job id is set. [[#4967]](https://github.com/aiidateam/aiida-core/pull/4967) +- Race condition when two processes try to create the same `Folder`/`SandboxFolder`, [[#4912]](https://github.com/aiidateam/aiida-core/pull/4912) +- Return the whole nested namespace when using `BaseRestartWorkChain.result`. [[#4961]](https://github.com/aiidateam/aiida-core/pull/4961) +- Use `numpy.nanmin` and `numpy.nanmax` for computing y-limits of `BandsData` matplotlib methods. [[#5024]](https://github.com/aiidateam/aiida-core/pull/5024) +- Use sanitized job title with `SgeScheduler` scheduler. [[#4994]](https://github.com/aiidateam/aiida-core/pull/4994) + ## v1.6.4 - 2021-06-23 [full changelog](https://github.com/aiidateam/aiida-core/compare/v1.6.3...v1.6.4) diff --git a/aiida/__init__.py b/aiida/__init__.py index 8813a864d5..c0292fcd06 100644 --- a/aiida/__init__.py +++ b/aiida/__init__.py @@ -31,7 +31,7 @@ 'For further information please visit http://www.aiida.net/. All rights reserved.' ) __license__ = 'MIT license, see LICENSE.txt file.' -__version__ = '1.6.4' +__version__ = '1.6.5' __authors__ = 'The AiiDA team.' __paper__ = ( 'S. P. Huber et al., "AiiDA 1.0, a scalable computational infrastructure for automated reproducible workflows and ' diff --git a/aiida/backends/djsite/manager.py b/aiida/backends/djsite/manager.py index 81c12c2dfe..edaf636c4b 100644 --- a/aiida/backends/djsite/manager.py +++ b/aiida/backends/djsite/manager.py @@ -87,14 +87,16 @@ def get_schema_generation_database(self): backend = get_manager()._load_backend(schema_check=False) # pylint: disable=protected-access try: - result = backend.execute_raw(r"""SELECT tval FROM db_dbsetting WHERE key = 'schema_generation';""") - except ProgrammingError: result = backend.execute_raw(r"""SELECT val FROM db_dbsetting WHERE key = 'schema_generation';""") - - try: - return str(int(result[0][0])) - except (IndexError, TypeError, ValueError): + except ProgrammingError: + # If this value does not exist, the schema has to correspond to the first generation which didn't actually + # record its value explicitly in the database until ``aiida-core>=1.0.0``. return '1' + else: + try: + return str(int(result[0][0])) + except (IndexError, ValueError, TypeError): + return '1' def get_schema_version_database(self): """Return the database schema version. @@ -107,9 +109,9 @@ def get_schema_version_database(self): backend = get_manager()._load_backend(schema_check=False) # pylint: disable=protected-access try: - result = backend.execute_raw(r"""SELECT tval FROM db_dbsetting WHERE key = 'db|schemaversion';""") - except ProgrammingError: result = backend.execute_raw(r"""SELECT val FROM db_dbsetting WHERE key = 'db|schemaversion';""") + except ProgrammingError: + result = backend.execute_raw(r"""SELECT tval FROM db_dbsetting WHERE key = 'db|schemaversion';""") return result[0][0] def set_schema_version_database(self, version): diff --git a/aiida/cmdline/commands/cmd_verdi.py b/aiida/cmdline/commands/cmd_verdi.py index 6a395b1185..2cd299f164 100644 --- a/aiida/cmdline/commands/cmd_verdi.py +++ b/aiida/cmdline/commands/cmd_verdi.py @@ -8,8 +8,8 @@ # For further information please visit http://www.aiida.net # ########################################################################### """The main `verdi` click group.""" - import difflib + import click from aiida import __version__ @@ -52,23 +52,33 @@ def get_command(self, ctx, cmd_name): """ cmd = click.Group.get_command(self, ctx, cmd_name) - # return the exact match + # If we match an actual command, simply return the match if cmd is not None: return cmd + # If this command is called during tab-completion, we do not want to print an error message if the command can't + # be found, but instead we want to simply return here. However, in a normal command execution, we do want to + # execute the rest of this method to try and match commands that are similar in order to provide the user with + # some hints. The problem is that there is no one canonical way to determine whether the invocation is due to a + # normal command execution or a tab-complete operation. The `resilient_parsing` attribute of the `Context` is + # designed to allow things like tab-completion, however, it is not the only purpose. For now this is our best + # bet though to detect a tab-complete event. When `resilient_parsing` is switched on, we assume a tab-complete + # and do nothing in case the command name does not match an actual command. + if ctx.resilient_parsing: + return + if int(cmd_name.lower().encode('utf-8').hex(), 16) == 0x6769757365707065: import base64 import gzip click.echo(gzip.decompress(base64.b85decode(GIU.encode('utf-8'))).decode('utf-8')) return None - # we might get better results with the Levenshtein distance - # or more advanced methods implemented in FuzzyWuzzy or similar libs, - # but this is an easy win for now + # We might get better results with the Levenshtein distance or more advanced methods implemented in FuzzyWuzzy + # or similar libs, but this is an easy win for now. matches = difflib.get_close_matches(cmd_name, self.list_commands(ctx), cutoff=0.5) if not matches: - # single letters are sometimes not matched, try with a simple startswith + # Single letters are sometimes not matched so also try with a simple startswith matches = [c for c in sorted(self.list_commands(ctx)) if c.startswith(cmd_name)][:3] if matches: diff --git a/aiida/cmdline/params/options/__init__.py b/aiida/cmdline/params/options/__init__.py index 16ae5eb95c..accd78c65f 100644 --- a/aiida/cmdline/params/options/__init__.py +++ b/aiida/cmdline/params/options/__init__.py @@ -249,7 +249,7 @@ def decorator(command): '--db-host', type=types.HostnameType(), help='Database server host. Leave empty for "peer" authentication.', - default=DEFAULT_DBINFO['host'] + default='localhost' ) DB_PORT = OverridableOption( diff --git a/aiida/cmdline/params/options/commands/setup.py b/aiida/cmdline/params/options/commands/setup.py index b5cb9f974d..1ec43c82ed 100644 --- a/aiida/cmdline/params/options/commands/setup.py +++ b/aiida/cmdline/params/options/commands/setup.py @@ -259,7 +259,7 @@ def get_quicksetup_password(ctx, param, value): # pylint: disable=unused-argume SETUP_DATABASE_HOSTNAME = QUICKSETUP_DATABASE_HOSTNAME.clone( prompt='Database host', - contextual_default=functools.partial(get_profile_attribute_default, ('database_hostname', DEFAULT_DBINFO['host'])), + contextual_default=functools.partial(get_profile_attribute_default, ('database_hostname', 'localhost')), cls=options.interactive.InteractiveOption ) diff --git a/aiida/cmdline/params/options/interactive.py b/aiida/cmdline/params/options/interactive.py index beee3a92c2..dfb3a69b3e 100644 --- a/aiida/cmdline/params/options/interactive.py +++ b/aiida/cmdline/params/options/interactive.py @@ -271,11 +271,11 @@ def prompt_callback(self, ctx, param, value): # If we are here, we are in interactive mode and the parameter is not specified # We enter the prompt loop - value = self.prompt_loop(ctx, param, value) - else: - # There is a prompt_fn function and returns False (i.e. should not ask for this value - # We then set the value to None - value = None + return self.prompt_loop(ctx, param, value) + + # There is a prompt_fn function and returns False (i.e. should not ask for this value + # We then set the value to None + value = None # And then we call the callback return self.after_callback(ctx, param, value) diff --git a/aiida/common/folders.py b/aiida/common/folders.py index df185faa79..bedb183929 100644 --- a/aiida/common/folders.py +++ b/aiida/common/folders.py @@ -342,8 +342,7 @@ def create(self): It is always safe to call it, it will do nothing if the folder already exists. """ - if not self.exists(): - os.makedirs(self.abspath, mode=self.mode_dir) + os.makedirs(self.abspath, mode=self.mode_dir, exist_ok=True) def replace_with_folder(self, srcdir, move=False, overwrite=False): """This routine copies or moves the source folder 'srcdir' to the local folder pointed to by this Folder. @@ -370,8 +369,7 @@ def replace_with_folder(self, srcdir, move=False, overwrite=False): # Create parent dir, if needed, with the right mode pardir = os.path.dirname(self.abspath) - if not os.path.exists(pardir): - os.makedirs(pardir, mode=self.mode_dir) + os.makedirs(pardir, mode=self.mode_dir, exist_ok=True) if move: shutil.move(srcdir, self.abspath) @@ -417,8 +415,7 @@ def __init__(self, sandbox_in_repo=True): # First check if the sandbox folder already exists if sandbox_in_repo: sandbox = os.path.join(get_profile().repository_path, 'sandbox') - if not os.path.exists(sandbox): - os.makedirs(sandbox) + os.makedirs(sandbox, exist_ok=True) abspath = tempfile.mkdtemp(dir=sandbox) else: abspath = tempfile.mkdtemp() diff --git a/aiida/engine/persistence.py b/aiida/engine/persistence.py index 2ccdac03c1..5ee9970b14 100644 --- a/aiida/engine/persistence.py +++ b/aiida/engine/persistence.py @@ -121,7 +121,7 @@ def load_checkpoint(self, pid: Hashable, tag: Optional[str] = None) -> plumpy.pe raise PersistenceError(f'Calculation<{calculation.pk}> does not have a saved checkpoint') try: - bundle = serialize.deserialize(checkpoint) + bundle = serialize.deserialize_unsafe(checkpoint) except Exception: raise PersistenceError(f'Failed to load the checkpoint for process<{pid}>: {traceback.format_exc()}') diff --git a/aiida/engine/processes/builder.py b/aiida/engine/processes/builder.py index c7f6939918..3f6eab4271 100644 --- a/aiida/engine/processes/builder.py +++ b/aiida/engine/processes/builder.py @@ -41,7 +41,7 @@ def __init__(self, port_namespace: PortNamespace) -> None: self._valid_fields = [] self._data = {} - # The name and port objects have to be passed to the defined functions as defaults for + # The name and port objects have to be passed to the defined functions as defaults for # their arguments, because this way the content at the time of defining the method is # saved. If they are used directly in the body, it will try to capture the value from # its enclosing scope at the time of being called. @@ -83,16 +83,28 @@ def __setattr__(self, attr: str, value: Any) -> None: else: try: port = self._port_namespace[attr] - except KeyError: + except KeyError as exception: if not self._port_namespace.dynamic: - raise AttributeError(f'Unknown builder parameter: {attr}') + raise AttributeError(f'Unknown builder parameter: {attr}') from exception + port = None # type: ignore[assignment] else: value = port.serialize(value) # type: ignore[union-attr] validation_error = port.validate(value) if validation_error: raise ValueError(f'invalid attribute value {validation_error.message}') - self._data[attr] = value + # If the attribute that is being set corresponds to a port that is a ``PortNamespace`` we need to make sure + # that the nested value remains a ``ProcessBuilderNamespace``. Otherwise, the nested namespaces will become + # plain dictionaries and no longer have the properties of the ``ProcessBuilderNamespace`` that provide all + # the autocompletion and validation when values are being set. Therefore we first construct a new instance + # of a ``ProcessBuilderNamespace`` for the port of the attribute that is being set and than iteratively set + # all the values within the mapping that is being assigned to the attribute. + if isinstance(port, PortNamespace): + self._data[attr] = ProcessBuilderNamespace(port) + for sub_key, sub_value in value.items(): + setattr(self._data[attr], sub_key, sub_value) + else: + self._data[attr] = value def __repr__(self): return self._data.__repr__() @@ -119,19 +131,45 @@ def __delitem__(self, item): def __delattr__(self, item): self._data.__delitem__(item) - def _update(self, *args, **kwds): - """Update the values of the builder namespace passing a mapping as argument or individual keyword value pairs. + def _recursive_merge(self, dictionary, key, value): + """Recursively merge the contents of ``dictionary`` setting its ``key`` to ``value``.""" + if isinstance(value, collections.abc.Mapping): + for inner_key, inner_value in value.items(): + self._recursive_merge(dictionary[key], inner_key, inner_value) + else: + dictionary[key] = value + + def _merge(self, *args, **kwds): + """Merge the content of a dictionary or keyword arguments in . - The method is prefixed with an underscore in order to not reserve the name for a potential port, but in - principle the method functions just as `collections.abc.MutableMapping.update`. + .. note:: This method differs in behavior from ``_update`` in that ``_merge`` will recursively update the + existing dictionary with the one that is specified in the arguments. The ``_update`` method will merge only + the keys on the top level, but any lower lying nested namespace will be replaced entirely. - :param args: a single mapping that should be mapped on the namespace + The method is prefixed with an underscore in order to not reserve the name for a potential port. - :param kwds: keyword value pairs that should be mapped onto the ports + :param args: a single mapping that should be mapped on the namespace. + :param kwds: keyword value pairs that should be mapped onto the ports. """ if len(args) > 1: raise TypeError(f'update expected at most 1 arguments, got {int(len(args))}') + if args: + for key, value in args[0].items(): + self._recursive_merge(self, key, value) + + for key, value in kwds.items(): + self._recursive_merge(self, key, value) + + def _update(self, *args, **kwds): + """Update the values of the builder namespace passing a mapping as argument or individual keyword value pairs. + + The method functions just as `collections.abc.MutableMapping.update` and is merely prefixed with an underscore + in order to not reserve the name for a potential port. + + :param args: a single mapping that should be mapped on the namespace. + :param kwds: keyword value pairs that should be mapped onto the ports. + """ if args: for key, value in args[0].items(): if isinstance(value, collections.abc.Mapping): diff --git a/aiida/engine/processes/calcjobs/tasks.py b/aiida/engine/processes/calcjobs/tasks.py index 95fb4b0f8e..2b2c270015 100644 --- a/aiida/engine/processes/calcjobs/tasks.py +++ b/aiida/engine/processes/calcjobs/tasks.py @@ -256,6 +256,10 @@ async def do_retrieve(): scheduler = node.computer.get_scheduler() # type: ignore[union-attr] scheduler.set_transport(transport) + if node.get_job_id() is None: + logger.warning(f'there is no job id for CalcJobNoe<{node.pk}>: skipping `get_detailed_job_info`') + return execmanager.retrieve_calculation(node, transport, retrieved_temporary_folder) + try: detailed_job_info = scheduler.get_detailed_job_info(node.get_job_id()) except FeatureNotAvailable: diff --git a/aiida/engine/processes/process.py b/aiida/engine/processes/process.py index 12d4d9dc6c..3064bfe75b 100644 --- a/aiida/engine/processes/process.py +++ b/aiida/engine/processes/process.py @@ -604,7 +604,7 @@ def decode_input_args(self, encoded: str) -> Dict[str, Any]: # pylint: disable= :param encoded: encoded (serialized) inputs :return: The decoded input args """ - return serialize.deserialize(encoded) + return serialize.deserialize_unsafe(encoded) def update_node_state(self, state: plumpy.process_states.State) -> None: self.update_outputs() diff --git a/aiida/engine/processes/workchains/restart.py b/aiida/engine/processes/workchains/restart.py index 5719e1496f..12a3a05dc4 100644 --- a/aiida/engine/processes/workchains/restart.py +++ b/aiida/engine/processes/workchains/restart.py @@ -303,11 +303,13 @@ def results(self) -> Optional['ExitCode']: self.report(f'work chain completed after {self.ctx.iteration} iterations') + exposed_outputs = self.exposed_outputs(node, self.process_class) + for name, port in self.spec().outputs.items(): try: - output = node.get_outgoing(link_label_filter=name).one().node - except ValueError: + output = exposed_outputs[name] + except KeyError: if port.required: self.report(f"required output '{name}' was not an output of {self.ctx.process_name}<{node.pk}>") else: diff --git a/aiida/manage/manager.py b/aiida/manage/manager.py index 8f8bdfd1f1..390e62fba3 100644 --- a/aiida/manage/manager.py +++ b/aiida/manage/manager.py @@ -243,7 +243,7 @@ def create_communicator( if with_orm: from aiida.orm.utils import serialize encoder = functools.partial(serialize.serialize, encoding='utf-8') - decoder = serialize.deserialize + decoder = serialize.deserialize_unsafe else: # used by verdi status to get a communicator without needing to load the dbenv from aiida.common import json diff --git a/aiida/manage/tests/pytest_fixtures.py b/aiida/manage/tests/pytest_fixtures.py index 08c203b358..586d0cdac2 100644 --- a/aiida/manage/tests/pytest_fixtures.py +++ b/aiida/manage/tests/pytest_fixtures.py @@ -70,6 +70,13 @@ def clear_database_before_test(aiida_profile): yield +@pytest.fixture(scope='class') +def clear_database_before_test_class(aiida_profile): + """Clear the database before a test class.""" + aiida_profile.reset_db() + yield + + @pytest.fixture(scope='function') def temporary_event_loop(): """Create a temporary loop for independent test case""" diff --git a/aiida/orm/nodes/data/array/bands.py b/aiida/orm/nodes/data/array/bands.py index 21ba303707..0559017a4f 100644 --- a/aiida/orm/nodes/data/array/bands.py +++ b/aiida/orm/nodes/data/array/bands.py @@ -784,9 +784,9 @@ def _matplotlib_get_dict( # axis limits if y_max_lim is None: - y_max_lim = numpy.array(bands).max() + y_max_lim = numpy.nanmax(bands) if y_min_lim is None: - y_min_lim = numpy.array(bands).min() + y_min_lim = numpy.nanmin(bands) x_min_lim = min(x) # this isn't a numpy array, but a list x_max_lim = max(x) all_data['x_min_lim'] = x_min_lim diff --git a/aiida/orm/nodes/node.py b/aiida/orm/nodes/node.py index a30a1d1135..fe45ad92da 100644 --- a/aiida/orm/nodes/node.py +++ b/aiida/orm/nodes/node.py @@ -185,7 +185,7 @@ def __init__( raise ValueError('the computer is not stored') computer = computer.backend_entity if computer else None - user = user.backend_entity if user else User.objects(backend).get_default() + user = user if user else User.objects(backend).get_default() if user is None: raise ValueError('the user cannot be None') diff --git a/aiida/orm/utils/managers.py b/aiida/orm/utils/managers.py index ad53db290c..dd11a4f0d4 100644 --- a/aiida/orm/utils/managers.py +++ b/aiida/orm/utils/managers.py @@ -12,9 +12,12 @@ to access members of other classes via TAB-completable attributes (e.g. the class underlying `calculation.inputs` to allow to do `calculation.inputs.