From dc78f4c58abba2bd1bb1e9f481c506f98d060ca2 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Wed, 9 Mar 2022 21:35:26 +0100 Subject: environment.py: allow link:run (#29336) * environment.py: allow link:run Some users want minimal views, excluding run-type dependencies, since those type of dependencies are covered by rpaths and the symlinked libraries in the view aren't used anyways. With this change, an environment like this: ``` spack: specs: ['py-flake8'] view: default: root: view link: run ``` includes python packages and python, but no link type deps of python. --- lib/spack/docs/environments.rst | 26 +++-- lib/spack/llnl/util/lang.py | 19 +++- lib/spack/spack/environment/environment.py | 150 ++++++++++++++--------------- lib/spack/spack/schema/env.py | 2 +- lib/spack/spack/test/cmd/env.py | 53 +++++++--- lib/spack/spack/test/llnl/util/lang.py | 7 +- 6 files changed, 159 insertions(+), 98 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index ed8e36c9ff..4e4a75b88c 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -740,9 +740,10 @@ file snippet we define a view named ``mpis``, rooted at version, and compiler name to determine the path for a given package. This view selects all packages that depend on MPI, and excludes those built with the PGI compiler at version 18.5. -All the dependencies of each root spec in the environment will be linked -in the view due to the command ``link: all`` and the files in the view will -be symlinks to the spack install directories. +The root specs with their (transitive) link and run type dependencies +will be put in the view due to the ``link: all`` option, +and the files in the view will be symlinks to the spack install +directories. .. code-block:: yaml @@ -762,9 +763,22 @@ For more information on using view projections, see the section on :ref:`adding_projections_to_views`. The default for the ``select`` and ``exclude`` values is to select everything and exclude nothing. The default projection is the default view projection (``{}``). The ``link`` -defaults to ``all`` but can also be ``roots`` when only the root specs -in the environment are desired in the view. The ``link_type`` defaults -to ``symlink`` but can also take the value of ``hardlink`` or ``copy``. +attribute allows the following values: + +#. ``link: all`` include root specs with their transitive run and link type + dependencies (default); +#. ``link: run`` include root specs with their transitive run type dependencies; +#. ``link: roots`` include root specs without their dependencies. + +The ``link_type`` defaults to ``symlink`` but can also take the value +of ``hardlink`` or ``copy``. + +.. tip:: + + The option ``link: run`` can be used to create small environment views for + Python packages. Python will be able to import packages *inside* of the view even + when the environment is not activated, and linked libraries will be located + *outside* of the view thanks to rpaths. Any number of views may be defined under the ``view`` heading in a Spack Environment. diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index e474924507..69e0c886fb 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -589,20 +589,31 @@ def match_predicate(*args): return match -def dedupe(sequence): - """Yields a stable de-duplication of an hashable sequence +def dedupe(sequence, key=None): + """Yields a stable de-duplication of an hashable sequence by key Args: sequence: hashable sequence to be de-duplicated + key: callable applied on values before uniqueness test; identity + by default. Returns: stable de-duplication of the sequence + + Examples: + + Dedupe a list of integers: + + [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3] + + [x for x in llnl.util.lang.dedupe([1,-2,1,3,2], key=abs)] == [1, -2, 3] """ seen = set() for x in sequence: - if x not in seen: + x_key = x if key is None else key(x) + if x_key not in seen: yield x - seen.add(x) + seen.add(x_key) def pretty_date(time, now=None): diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index e7f51c750a..4f2a08a107 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -16,6 +16,7 @@ import six import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.util.lang import dedupe import spack.bootstrap import spack.compilers @@ -471,80 +472,80 @@ class ViewDescriptor(object): return True - def specs_for_view(self, all_specs, roots): - specs_for_view = [] - specs = all_specs if self.link == 'all' else roots - - for spec in specs: - # The view does not store build deps, so if we want it to - # recognize environment specs (which do store build deps), - # then they need to be stripped. - if spec.concrete: # Do not link unconcretized roots - # We preserve _hash _normal to avoid recomputing DAG - # hashes (DAG hashes don't consider build deps) - spec_copy = spec.copy(deps=('link', 'run')) - spec_copy._hash = spec._hash - spec_copy._normal = spec._normal - specs_for_view.append(spec_copy) - return specs_for_view - - def regenerate(self, all_specs, roots): - specs_for_view = self.specs_for_view(all_specs, roots) - - # regeneration queries the database quite a bit; this read - # transaction ensures that we don't repeatedly lock/unlock. + def specs_for_view(self, concretized_specs): + """ + From the list of concretized user specs in the environment, flatten + the dags, and filter selected, installed specs, remove duplicates on dag hash. + """ + specs = [] + + for (_, s) in concretized_specs: + if self.link == 'all': + specs.extend(s.traverse(deptype=('link', 'run'))) + elif self.link == 'run': + specs.extend(s.traverse(deptype=('run'))) + else: + specs.append(s) + + # De-dupe by dag hash + specs = dedupe(specs, key=lambda s: s.dag_hash()) + + # Filter selected, installed specs with spack.store.db.read_transaction(): - installed_specs_for_view = set( - s for s in specs_for_view if s in self and s.package.installed) - - # To ensure there are no conflicts with packages being installed - # that cannot be resolved or have repos that have been removed - # we always regenerate the view from scratch. - # We will do this by hashing the view contents and putting the view - # in a directory by hash, and then having a symlink to the real - # view in the root. The real root for a view at /dirname/basename - # will be /dirname/._basename_. - # This allows for atomic swaps when we update the view - - # cache the roots because the way we determine which is which does - # not work while we are updating - new_root = self._next_root(installed_specs_for_view) - old_root = self._current_root - - if new_root == old_root: - tty.debug("View at %s does not need regeneration." % self.root) - return - - # construct view at new_root - tty.msg("Updating view at {0}".format(self.root)) - - view = self.view(new=new_root) - fs.mkdirp(new_root) - view.add_specs(*installed_specs_for_view, - with_dependencies=False) - - # create symlink from tmpname to new_root - root_dirname = os.path.dirname(self.root) - tmp_symlink_name = os.path.join(root_dirname, '._view_link') - if os.path.exists(tmp_symlink_name): - os.unlink(tmp_symlink_name) - os.symlink(new_root, tmp_symlink_name) - - # mv symlink atomically over root symlink to old_root - if os.path.exists(self.root) and not os.path.islink(self.root): - msg = "Cannot create view: " - msg += "file already exists and is not a link: %s" % self.root - raise SpackEnvironmentViewError(msg) - os.rename(tmp_symlink_name, self.root) - - # remove old_root - if old_root and os.path.exists(old_root): - try: - shutil.rmtree(old_root) - except (IOError, OSError) as e: - msg = "Failed to remove old view at %s\n" % old_root - msg += str(e) - tty.warn(msg) + specs = [s for s in specs if s in self and s.package.installed] + + return specs + + def regenerate(self, concretized_specs): + specs = self.specs_for_view(concretized_specs) + + # To ensure there are no conflicts with packages being installed + # that cannot be resolved or have repos that have been removed + # we always regenerate the view from scratch. + # We will do this by hashing the view contents and putting the view + # in a directory by hash, and then having a symlink to the real + # view in the root. The real root for a view at /dirname/basename + # will be /dirname/._basename_. + # This allows for atomic swaps when we update the view + + # cache the roots because the way we determine which is which does + # not work while we are updating + new_root = self._next_root(specs) + old_root = self._current_root + + if new_root == old_root: + tty.debug("View at %s does not need regeneration." % self.root) + return + + # construct view at new_root + tty.msg("Updating view at {0}".format(self.root)) + + view = self.view(new=new_root) + fs.mkdirp(new_root) + view.add_specs(*specs, with_dependencies=False) + + # create symlink from tmpname to new_root + root_dirname = os.path.dirname(self.root) + tmp_symlink_name = os.path.join(root_dirname, '._view_link') + if os.path.exists(tmp_symlink_name): + os.unlink(tmp_symlink_name) + os.symlink(new_root, tmp_symlink_name) + + # mv symlink atomically over root symlink to old_root + if os.path.exists(self.root) and not os.path.islink(self.root): + msg = "Cannot create view: " + msg += "file already exists and is not a link: %s" % self.root + raise SpackEnvironmentViewError(msg) + os.rename(tmp_symlink_name, self.root) + + # remove old_root + if old_root and os.path.exists(old_root): + try: + shutil.rmtree(old_root) + except (IOError, OSError) as e: + msg = "Failed to remove old view at %s\n" % old_root + msg += str(e) + tty.warn(msg) def _create_environment(*args, **kwargs): @@ -1303,9 +1304,8 @@ class Environment(object): " maintain a view") return - specs = self._get_environment_specs() for view in self.views.values(): - view.regenerate(specs, self.roots()) + view.regenerate(self.concretized_specs()) def check_views(self): """Checks if the environments default view can be activated.""" diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index d2ef1bc691..65ef1a76a8 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -124,7 +124,7 @@ schema = { }, 'link': { 'type': 'string', - 'pattern': '(roots|all)', + 'pattern': '(roots|all|run)', }, 'link_type': { 'type': 'string' diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index a240923358..b1ff4291e9 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -386,23 +386,23 @@ def test_environment_status(capsys, tmpdir): def test_env_status_broken_view( mutable_mock_env_path, mock_archive, mock_fetch, mock_packages, - install_mockery + install_mockery, tmpdir ): - with ev.create('test'): + env_dir = str(tmpdir) + with ev.Environment(env_dir): install('trivial-install-test-package') - # switch to a new repo that doesn't include the installed package - # test that Spack detects the missing package and warns the user - new_repo = MockPackageMultiRepo() - with spack.repo.use_repositories(new_repo): + # switch to a new repo that doesn't include the installed package + # test that Spack detects the missing package and warns the user + with spack.repo.use_repositories(MockPackageMultiRepo()): + with ev.Environment(env_dir): output = env('status') - assert 'In environment test' in output - assert 'Environment test includes out of date' in output + assert 'includes out of date packages or repos' in output - # Test that the warning goes away when it's fixed + # Test that the warning goes away when it's fixed + with ev.Environment(env_dir): output = env('status') - assert 'In environment test' in output - assert 'Environment test includes out of date' not in output + assert 'includes out of date packages or repos' not in output def test_env_activate_broken_view( @@ -1962,6 +1962,37 @@ env: (spec.version, spec.compiler.name))) +def test_view_link_run(tmpdir, mock_fetch, mock_packages, mock_archive, + install_mockery): + yaml = str(tmpdir.join('spack.yaml')) + viewdir = str(tmpdir.join('view')) + envdir = str(tmpdir) + with open(yaml, 'w') as f: + f.write(""" +spack: + specs: + - dttop + + view: + combinatorial: + root: %s + link: run + projections: + all: '{name}'""" % viewdir) + + with ev.Environment(envdir): + install() + + # make sure transitive run type deps are in the view + for pkg in ('dtrun1', 'dtrun3'): + assert os.path.exists(os.path.join(viewdir, pkg)) + + # and non-run-type deps are not. + for pkg in ('dtlink1', 'dtlink2', 'dtlink3', 'dtlink4', 'dtlink5' + 'dtbuild1', 'dtbuild2', 'dtbuild3'): + assert not os.path.exists(os.path.join(viewdir, pkg)) + + @pytest.mark.parametrize('link_type', ['hardlink', 'copy', 'symlink']) def test_view_link_type(link_type, tmpdir, mock_fetch, mock_packages, mock_archive, install_mockery): diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py index 3fb6196c3a..c085f30259 100644 --- a/lib/spack/spack/test/llnl/util/lang.py +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta import pytest import llnl.util.lang -from llnl.util.lang import match_predicate, memoized, pretty_date, stable_args +from llnl.util.lang import dedupe, match_predicate, memoized, pretty_date, stable_args @pytest.fixture() @@ -265,3 +265,8 @@ def test_memoized_unhashable(args, kwargs): key = stable_args(*args, **kwargs) assert str(key) in exc_msg assert "function 'f'" in exc_msg + + +def test_dedupe(): + assert [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3] + assert [x for x in dedupe([1, -2, 1, 3, 2], key=abs)] == [1, -2, 3] -- cgit v1.2.3-60-g2f50