From ec6e5b0fd38faed77597424cf173bc462e0bd7e8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 3 Jul 2019 15:17:07 -0700 Subject: stacks: add option to link only roots or all specs, default to all --- lib/spack/spack/environment.py | 57 +++++++++++++-------- lib/spack/spack/schema/env.py | 4 ++ lib/spack/spack/test/cmd/env.py | 110 ++++++++++++++++++++++++++++++++++------ 3 files changed, 134 insertions(+), 37 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index d613c19d99..69bffce0a0 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -78,8 +78,12 @@ lockfile_format_version = 1 env_schema_keys = ('spack', 'env') # Magic names +# The name of the standalone spec list in the manifest yaml user_speclist_name = 'specs' -def_view_name = 'default' +# The name of the default view (the view loaded on env.activate) +default_view_name = 'default' +# Default behavior to link all packages into views (vs. only root packages) +default_view_link = 'all' def valid_env_name(name): @@ -145,7 +149,7 @@ def activate( cmds += 'export SPACK_OLD_PS1="${PS1}"; fi;\n' cmds += 'export PS1="%s ${PS1}";\n' % prompt - if add_view and def_view_name in env.views: + if add_view and default_view_name in env.views: cmds += env.add_default_view_to_shell(shell) return cmds @@ -187,7 +191,7 @@ def deactivate(shell='sh'): cmds += 'unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\n' cmds += 'fi;\n' - if def_view_name in _active_environment.views: + if default_view_name in _active_environment.views: cmds += _active_environment.rm_default_view_from_shell(shell) tty.debug("Deactivated environmennt '%s'" % _active_environment.name) @@ -414,7 +418,8 @@ def _eval_conditional(string): class ViewDescriptor(object): - def __init__(self, root, projections={}, select=[], exclude=[]): + def __init__(self, root, projections={}, select=[], exclude=[], + link=default_view_link): self.root = root self.projections = projections self.select = select @@ -422,6 +427,7 @@ class ViewDescriptor(object): self.exclude = exclude self.exclude_fn = lambda x: not any(x.satisfies(e) for e in self.exclude) + self.link = link def to_dict(self): ret = {'root': self.root} @@ -431,6 +437,8 @@ class ViewDescriptor(object): ret['select'] = self.select if self.exclude: ret['exclude'] = self.exclude + if self.link != default_view_link: + ret['link'] = self.link return ret @staticmethod @@ -438,22 +446,25 @@ class ViewDescriptor(object): return ViewDescriptor(d['root'], d.get('projections', {}), d.get('select', []), - d.get('exclude', [])) + d.get('exclude', []), + d.get('link', default_view_link)) def view(self): return YamlFilesystemView(self.root, spack.store.layout, ignore_conflicts=True, projections=self.projections) - def regenerate(self, specs): + def regenerate(self, all_specs, roots): specs_for_view = [] + specs = all_specs if self.link == 'all' else roots for spec in specs: # The view does not store build deps, so if we want it to # recognize environment specs (which do store build deps), then # they need to be stripped - specs_for_view.append(spack.spec.Spec.from_dict( - spec.to_dict(all_deps=False) - )) + if spec.concrete: # Do not link unconcretized roots + specs_for_view.append(spack.spec.Spec.from_dict( + spec.to_dict(all_deps=False) + )) if self.select: specs_for_view = list(filter(self.select_fn, specs_for_view)) @@ -523,9 +534,9 @@ class Environment(object): self.views = {} elif with_view is True: self.views = { - def_view_name: ViewDescriptor(self.view_path_default)} + default_view_name: ViewDescriptor(self.view_path_default)} elif isinstance(with_view, six.string_types): - self.views = {def_view_name: ViewDescriptor(with_view)} + self.views = {default_view_name: ViewDescriptor(with_view)} # If with_view is None, then defer to the view settings determined by # the manifest file @@ -556,9 +567,9 @@ class Environment(object): # enable_view can be boolean, string, or None if enable_view is True or enable_view is None: self.views = { - def_view_name: ViewDescriptor(self.view_path_default)} + default_view_name: ViewDescriptor(self.view_path_default)} elif isinstance(enable_view, six.string_types): - self.views = {def_view_name: ViewDescriptor(enable_view)} + self.views = {default_view_name: ViewDescriptor(enable_view)} elif enable_view: self.views = dict((name, ViewDescriptor.from_dict(values)) for name, values in enable_view.items()) @@ -881,23 +892,24 @@ class Environment(object): raise SpackEnvironmentError( "{0} does not have a view enabled".format(self.name)) - if def_view_name not in self.views: + if default_view_name not in self.views: raise SpackEnvironmentError( "{0} does not have a default view enabled".format(self.name)) - return self.views[def_view_name] + return self.views[default_view_name] def update_default_view(self, viewpath): - if def_view_name in self.views and self.default_view.root != viewpath: + name = default_view_name + if name in self.views and self.default_view.root != viewpath: shutil.rmtree(self.default_view.root) if viewpath: - if def_view_name in self.views: + if name in self.views: self.default_view.root = viewpath else: - self.views[def_view_name] = ViewDescriptor(viewpath) + self.views[name] = ViewDescriptor(viewpath) else: - self.views.pop(def_view_name, None) + self.views.pop(name, None) def regenerate_views(self): if not self.views: @@ -907,7 +919,7 @@ class Environment(object): specs = self._get_environment_specs() for view in self.views.values(): - view.regenerate(specs) + view.regenerate(specs, self.roots()) def _shell_vars(self): updates = [ @@ -921,7 +933,7 @@ class Environment(object): ('CMAKE_PREFIX_PATH', ['']), ] path_updates = list() - if def_view_name in self.views: + if default_view_name in self.views: for var, subdirs in updates: paths = filter(lambda x: os.path.exists(x), list(os.path.join(self.default_view.root, x) @@ -1198,7 +1210,8 @@ class Environment(object): []) yaml_spec_list[:] = self.user_specs.yaml_list - if self.views and len(self.views) == 1 and def_view_name in self.views: + default_name = default_view_name + if self.views and len(self.views) == 1 and default_name in self.views: path = self.default_view.root if self.default_view == ViewDescriptor(self.view_path_default): view = True diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 435d8f6b3f..5e5b9ed64f 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -96,6 +96,10 @@ schema = { 'root': { 'type': 'string' }, + 'link': { + 'type': 'string', + 'pattern': '(roots|all)', + }, 'select': { 'type': 'array', 'items': { diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 7a5bb053ca..cf7dcb93ff 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -32,10 +32,9 @@ uninstall = SpackCommand('uninstall') find = SpackCommand('find') -def check_mpileaks_install(viewdir): +def check_mpileaks_and_deps_in_view(viewdir): """Check that the expected install directories exist.""" assert os.path.exists(str(viewdir.join('.spack', 'mpileaks'))) - # Check that dependencies got in too assert os.path.exists(str(viewdir.join('.spack', 'libdwarf'))) @@ -634,7 +633,7 @@ def test_env_updates_view_install( add('mpileaks') install('--fake') - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) def test_env_without_view_install( @@ -656,7 +655,7 @@ def test_env_without_view_install( # After enabling the view, the specs should be linked into the environment # view dir - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) def test_env_config_view_default( @@ -686,7 +685,7 @@ def test_env_updates_view_install_package( with ev.read('test'): install('--fake', 'mpileaks') - check_mpileaks_install(view_dir) + assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) def test_env_updates_view_add_concretize( @@ -698,7 +697,7 @@ def test_env_updates_view_add_concretize( add('mpileaks') concretize() - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) def test_env_updates_view_uninstall( @@ -708,7 +707,7 @@ def test_env_updates_view_uninstall( with ev.read('test'): install('--fake', 'mpileaks') - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) with ev.read('test'): uninstall('-ay') @@ -725,7 +724,7 @@ def test_env_updates_view_uninstall_referenced_elsewhere( add('mpileaks') concretize() - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) with ev.read('test'): uninstall('-ay') @@ -742,7 +741,7 @@ def test_env_updates_view_remove_concretize( add('mpileaks') concretize() - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) with ev.read('test'): remove('mpileaks') @@ -758,7 +757,7 @@ def test_env_updates_view_force_remove( with ev.read('test'): install('--fake', 'mpileaks') - check_mpileaks_install(view_dir) + check_mpileaks_and_deps_in_view(view_dir) with ev.read('test'): remove('-f', 'mpileaks') @@ -1123,7 +1122,7 @@ env: install() test = ev.read('test') - for _, spec in test.concretized_specs(): + for spec in test._get_environment_specs(): assert os.path.exists( os.path.join(viewdir, spec.name, '%s-%s' % (spec.version, spec.compiler.name))) @@ -1156,7 +1155,7 @@ env: install() test = ev.read('test') - for _, spec in test.concretized_specs(): + for spec in test._get_environment_specs(): if spec.satisfies('%gcc'): assert os.path.exists( os.path.join(viewdir, spec.name, '%s-%s' % @@ -1194,7 +1193,7 @@ env: install() test = ev.read('test') - for _, spec in test.concretized_specs(): + for spec in test._get_environment_specs(): if not spec.satisfies('callpath'): assert os.path.exists( os.path.join(viewdir, spec.name, '%s-%s' % @@ -1233,7 +1232,88 @@ env: install() test = ev.read('test') - for _, spec in test.concretized_specs(): + for spec in test._get_environment_specs(): + if spec.satisfies('%gcc') and not spec.satisfies('callpath'): + assert os.path.exists( + os.path.join(viewdir, spec.name, '%s-%s' % + (spec.version, spec.compiler.name))) + else: + assert not os.path.exists( + os.path.join(viewdir, spec.name, '%s-%s' % + (spec.version, spec.compiler.name))) + + +def test_view_link_roots(tmpdir, mock_fetch, mock_packages, mock_archive, + install_mockery): + filename = str(tmpdir.join('spack.yaml')) + viewdir = str(tmpdir.join('view')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [mpileaks, callpath] + - compilers: ['%%gcc', '%%clang'] + specs: + - matrix: + - [$packages] + - [$compilers] + + view: + combinatorial: + root: %s + select: ['%%gcc'] + exclude: [callpath] + link: 'roots' + projections: + 'all': '{name}/{version}-{compiler.name}'""" % viewdir) + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + install() + + test = ev.read('test') + for spec in test._get_environment_specs(): + if spec in test.roots() and (spec.satisfies('%gcc') and + not spec.satisfies('callpath')): + assert os.path.exists( + os.path.join(viewdir, spec.name, '%s-%s' % + (spec.version, spec.compiler.name))) + else: + assert not os.path.exists( + os.path.join(viewdir, spec.name, '%s-%s' % + (spec.version, spec.compiler.name))) + + +def test_view_link_all(tmpdir, mock_fetch, mock_packages, mock_archive, + install_mockery): + filename = str(tmpdir.join('spack.yaml')) + viewdir = str(tmpdir.join('view')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [mpileaks, callpath] + - compilers: ['%%gcc', '%%clang'] + specs: + - matrix: + - [$packages] + - [$compilers] + + view: + combinatorial: + root: %s + select: ['%%gcc'] + exclude: [callpath] + link: 'all' + projections: + 'all': '{name}/{version}-{compiler.name}'""" % viewdir) + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + install() + + test = ev.read('test') + for spec in test._get_environment_specs(): if spec.satisfies('%gcc') and not spec.satisfies('callpath'): assert os.path.exists( os.path.join(viewdir, spec.name, '%s-%s' % @@ -1342,7 +1422,7 @@ env: assert os.path.join(default_viewdir, 'bin') in shell test = ev.read('test') - for _, spec in test.concretized_specs(): + for spec in test._get_environment_specs(): if not spec.satisfies('callpath%gcc'): assert os.path.exists( os.path.join(combin_viewdir, spec.name, '%s-%s' % -- cgit v1.2.3-60-g2f50