diff options
| -rw-r--r-- | lib/spack/docs/environments.rst | 52 | ||||
| -rw-r--r-- | lib/spack/spack/environment.py | 70 | ||||
| -rw-r--r-- | lib/spack/spack/schema/env.py | 5 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/env.py | 48 |
4 files changed, 171 insertions, 4 deletions
diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 6ce67f0067..1ce765210f 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -292,19 +292,37 @@ or $ spack -E myenv add python +.. _environments_concretization: + ^^^^^^^^^^^^ Concretizing ^^^^^^^^^^^^ Once some user specs have been added to an environment, they can be -concretized. The following command will concretize all user specs -that have been added and not yet concretized: +concretized. *By default specs are concretized separately*, one after +the other. This mode of operation permits to deploy a full +software stack where multiple configurations of the same package +need to be installed alongside each other. Central installations done +at HPC centers by system administrators or user support groups +are a common case that fits in this behavior. +Environments *can also be configured to concretize all +the root specs in a self-consistent way* to ensure that +each package in the environment comes with a single configuration. This +mode of operation is usually what is required by software developers that +want to deploy their development environment. + +Regardless of which mode of operation has been chosen, the following +command will ensure all the root specs are concretized according to the +constraints that are prescribed in the configuration: .. code-block:: console [myenv]$ spack concretize -This command will re-concretize all specs: +In the case of specs that are not concretized together, the command +above will concretize only the specs that were added and not yet +concretized. Forcing a re-concretization of all the specs can be done +instead with this command: .. code-block:: console @@ -467,6 +485,34 @@ Appending to this list in the yaml is identical to using the ``spack add`` command from the command line. However, there is more power available from the yaml file. +""""""""""""""""""" +Spec concretization +""""""""""""""""""" + +Specs can be concretized separately or together, as already +explained in :ref:`environments_concretization`. The behavior active +under any environment is determined by the ``concretization`` property: + +.. code-block:: yaml + + spack: + specs: + - ncview + - netcdf + - nco + - py-sphinx + concretization: together + +which can currently take either one of the two allowed values ``together`` or ``separately`` +(the default). + +.. admonition:: Re-concretization of user specs + + When concretizing specs together the entire set of specs will be + re-concretized after any addition of new user specs, to ensure that + the environment remains consistent. When instead the specs are concretized + separately only the new specs will be re-concretized after any addition. + """"""""""""" Spec Matrices """"""""""""" diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 84c77df6b0..ab8d7e14f1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections import os import re import sys @@ -19,6 +20,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.tty.color import colorize +import spack.concretize import spack.error import spack.hash_types as ht import spack.repo @@ -514,6 +516,9 @@ class Environment(object): path to the view. """ self.path = os.path.abspath(path) + # This attribute will be set properly from configuration + # during concretization + self.concretization = None self.clear() if init_file: @@ -591,6 +596,9 @@ class Environment(object): for name, values in enable_view.items()) else: self.views = {} + # Retrieve the current concretization strategy + configuration = config_dict(self.yaml) + self.concretization = configuration.get('concretization') @property def user_specs(self): @@ -845,6 +853,59 @@ class Environment(object): self.concretized_order = [] self.specs_by_hash = {} + # Pick the right concretization strategy + if self.concretization == 'together': + return self._concretize_together() + if self.concretization == 'separately': + return self._concretize_separately() + + msg = 'concretization strategy not implemented [{0}]' + raise SpackEnvironmentError(msg.format(self.concretization)) + + def _concretize_together(self): + """Concretization strategy that concretizes all the specs + in the same DAG. + """ + # Exit early if the set of concretized specs is the set of user specs + user_specs_did_not_change = not bool( + set(self.user_specs) - set(self.concretized_user_specs) + ) + if user_specs_did_not_change: + return [] + + # Check that user specs don't have duplicate packages + counter = collections.defaultdict(int) + for user_spec in self.user_specs: + counter[user_spec.name] += 1 + + duplicates = [] + for name, count in counter.items(): + if count > 1: + duplicates.append(name) + + if duplicates: + msg = ('environment that are configured to concretize specs' + ' together cannot contain more than one spec for each' + ' package [{0}]'.format(', '.join(duplicates))) + raise SpackEnvironmentError(msg) + + # Proceed with concretization + self.concretized_user_specs = [] + self.concretized_order = [] + self.specs_by_hash = {} + + concrete_specs = spack.concretize.concretize_specs_together( + *self.user_specs + ) + concretized_specs = [x for x in zip(self.user_specs, concrete_specs)] + for abstract, concrete in concretized_specs: + self._add_concrete_spec(abstract, concrete) + return concretized_specs + + def _concretize_separately(self): + """Concretization strategy that concretizes separately one + user spec after the other. + """ # keep any concretized specs whose user specs are still in the manifest old_concretized_user_specs = self.concretized_user_specs old_concretized_order = self.concretized_order @@ -875,6 +936,13 @@ class Environment(object): This will automatically concretize the single spec, but it won't affect other as-yet unconcretized specs. """ + if self.concretization == 'together': + msg = 'cannot install a single spec in an environment that is ' \ + 'configured to be concretized together. Run instead:\n\n' \ + ' $ spack add <spec>\n' \ + ' $ spack install\n' + raise SpackEnvironmentError(msg) + spec = Spec(user_spec) if self.add(spec): @@ -1292,7 +1360,7 @@ class Environment(object): def __enter__(self): self._previous_active = _active_environment activate(self) - return + return self def __exit__(self, exc_type, exc_val, exc_tb): deactivate() diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 2a3afb0aef..0af877185a 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -119,6 +119,11 @@ schema = { } } ] + }, + 'concretization': { + 'type': 'string', + 'enum': ['together', 'separately'], + 'default': 'separately' } } ) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 45b409df39..b7f3e5e325 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -1694,3 +1694,51 @@ def test_env_activate_csh_prints_shell_output( assert "setenv SPACK_ENV" in out assert "set prompt=" in out assert "alias despacktivate" in out + + +def test_concretize_user_specs_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + # Concretize a first time using 'mpich' as the MPI provider + e.add('mpileaks') + e.add('mpich') + e.concretize() + + assert all('mpich' in spec for _, spec in e.concretized_specs()) + assert all('mpich2' not in spec for _, spec in e.concretized_specs()) + + # Concretize a second time using 'mpich2' as the MPI provider + e.remove('mpich') + e.add('mpich2') + e.concretize() + + assert all('mpich2' in spec for _, spec in e.concretized_specs()) + assert all('mpich' not in spec for _, spec in e.concretized_specs()) + + # Concretize again without changing anything, check everything + # stays the same + e.concretize() + + assert all('mpich2' in spec for _, spec in e.concretized_specs()) + assert all('mpich' not in spec for _, spec in e.concretized_specs()) + + +def test_cant_install_single_spec_when_concretizing_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + with pytest.raises(ev.SpackEnvironmentError, match=r'cannot install'): + e.install('zlib') + + +def test_duplicate_packages_raise_when_concretizing_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + e.add('mpileaks+opt') + e.add('mpileaks~opt') + e.add('mpich') + + with pytest.raises(ev.SpackEnvironmentError, match=r'cannot contain more'): + e.concretize() |