summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/workflows.rst68
-rw-r--r--lib/spack/spack/build_systems/aspell_dict.py2
-rw-r--r--lib/spack/spack/build_systems/python.py14
-rw-r--r--lib/spack/spack/cmd/__init__.py13
-rw-r--r--lib/spack/spack/cmd/view.py58
-rw-r--r--lib/spack/spack/config.py14
-rw-r--r--lib/spack/spack/directory_layout.py19
-rw-r--r--lib/spack/spack/filesystem_view.py159
-rw-r--r--lib/spack/spack/package.py2
-rw-r--r--lib/spack/spack/schema/projections.py34
-rw-r--r--lib/spack/spack/spec.py6
-rw-r--r--lib/spack/spack/test/cmd/view.py116
-rw-r--r--lib/spack/spack/test/config.py2
-rw-r--r--lib/spack/spack/test/test_activations.py140
14 files changed, 500 insertions, 147 deletions
diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst
index 4fe2b158d7..371fce35a2 100644
--- a/lib/spack/docs/workflows.rst
+++ b/lib/spack/docs/workflows.rst
@@ -437,11 +437,23 @@ Filesystem views offer an alternative to environment modules, another
way to assemble packages in a useful way and load them into a user's
environment.
-A filesystem view is a single directory tree that is the union of the
-directory hierarchies of a number of installed packages; it is similar
-to the directory hiearchy that might exist under ``/usr/local``. The
-files of the view's installed packages are brought into the view by
-symbolic or hard links, referencing the original Spack installation.
+A single-prefix filesystem view is a single directory tree that is the
+union of the directory hierarchies of a number of installed packages;
+it is similar to the directory hiearchy that might exist under
+``/usr/local``. The files of the view's installed packages are
+brought into the view by symbolic or hard links, referencing the
+original Spack installation.
+
+A combinatorial filesystem view can contain more software than a
+single-prefix view. Combinatorial filesystem views are created by
+defining a projection for each spec or set of specs. The syntax for
+this will be discussed in the section for the ``spack view`` command
+under `adding_projections_to_views`_.
+
+The projection for a spec or set of specs specifies the naming scheme
+for the directory structure under the root of the view into which the
+package will be linked. For example, the spec ``zlib@1.2.8%gcc@4.4.7``
+could be projected to ``MYVIEW/zlib-1.2.8-gcc``.
When software is built and installed, absolute paths are frequently
"baked into" the software, making it non-relocatable. This happens
@@ -507,6 +519,51 @@ files in the ``cmake`` package while retaining its dependencies.
When packages are removed from a view, empty directories are
purged.
+.. _adding_projections_to_views:
+
+""""""""""""""""""""""""""""
+Controlling View Projections
+""""""""""""""""""""""""""""
+
+The default projection into a view is to link every package into the
+root of the view. This can be changed by adding a ``projections.yaml``
+configuration file to the view. The projection configuration file for
+a view located at ``/my/view`` is stored in
+``/my/view/.spack/projections.yaml``.
+
+When creating a view, the projection configuration file can also be
+specified from the command line using the ``--projection-file`` option
+to the ``spack view`` command.
+
+The projections configuration file is a mapping of partial specs to
+spec format strings, as shown in the example below.
+
+.. code-block:: yaml
+
+ projections:
+ zlib: ${PACKAGE}-${VERSION}
+ ^mpi: ${PACKAGE}-${VERSION}/${DEP:mpi:PACKAGE}-${DEP:mpi:VERSION}-${COMPILERNAME}-${COMPILERVER}
+ all: ${PACKAGE}-${VERSION}/${COMPILERNAME}-${COMPILERVER}
+
+The entries in the projections configuration file must all be either
+specs or the keyword ``all``. For each spec, the projection used will
+be the first non-``all`` entry that the spec satisfies, or ``all`` if
+there is an entry for ``all`` and no other entry is satisfied by the
+spec. Where the keyword ``all`` appears in the file does not
+matter. Given the example above, any spec satisfying ``zlib@1.2.8``
+will be linked into ``/my/view/zlib-1.2.8/``, any spec satisfying
+``hdf5@1.8.10+mpi %gcc@4.9.3 ^mvapich2@2.2`` will be linked into
+``/my/view/hdf5-1.8.10/mvapich2-2.2-gcc-4.9.3``, and any spec
+satisfying ``hdf5@1.8.10~mpi %gcc@4.9.3`` will be linked into
+``/my/view/hdf5-1.8.10/gcc-4.9.3``.
+
+If the keyword ``all`` does not appear in the projections
+configuration file, any spec that does not satisfy any entry in the
+file will be linked into the root of the view as in a single-prefix
+view. Any entries that appear below the keyword ``all`` in the
+projections configuration file will not be used, as all specs will use
+the projection under ``all`` before reaching those entries.
+
""""""""""""""""""
Fine-Grain Control
""""""""""""""""""
@@ -1437,4 +1494,3 @@ Disadvantages:
2. Although patches of a few lines work OK, large patch files can be
hard to create and maintain.
-
diff --git a/lib/spack/spack/build_systems/aspell_dict.py b/lib/spack/spack/build_systems/aspell_dict.py
index 285a8b18ce..e9019a1948 100644
--- a/lib/spack/spack/build_systems/aspell_dict.py
+++ b/lib/spack/spack/build_systems/aspell_dict.py
@@ -26,7 +26,7 @@ class AspellDictPackage(AutotoolsPackage):
def view_destination(self, view):
aspell_spec = self.spec['aspell']
- if view.root != aspell_spec.prefix:
+ if view.get_projection_for_spec(aspell_spec) != aspell_spec.prefix:
raise ExtensionError(
'aspell does not support non-global extensions')
aspell = aspell_spec.command
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index dd8a9bb1fb..91718f0cae 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -414,7 +414,9 @@ class PythonPackage(PackageBase):
def add_files_to_view(self, view, merge_map):
bin_dir = self.spec.prefix.bin
python_prefix = self.extendee_spec.prefix
- global_view = same_path(python_prefix, view.root)
+ global_view = same_path(python_prefix, view.get_projection_for_spec(
+ self.spec
+ ))
for src, dst in merge_map.items():
if os.path.exists(dst):
continue
@@ -424,7 +426,9 @@ class PythonPackage(PackageBase):
shutil.copy2(src, dst)
if 'script' in get_filetype(src):
filter_file(
- python_prefix, os.path.abspath(view.root), dst)
+ python_prefix, os.path.abspath(
+ view.get_projection_for_spec(self.spec)), dst
+ )
else:
orig_link_target = os.path.realpath(src)
new_link_target = os.path.abspath(merge_map[orig_link_target])
@@ -443,7 +447,11 @@ class PythonPackage(PackageBase):
ignore_namespace = True
bin_dir = self.spec.prefix.bin
- global_view = self.extendee_spec.prefix == view.root
+ global_view = (
+ self.extendee_spec.prefix == view.get_projection_for_spec(
+ self.spec
+ )
+ )
for src, dst in merge_map.items():
if ignore_namespace and namespace_init(dst):
continue
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index a3f669455a..630a6b6e63 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -8,6 +8,7 @@ from __future__ import print_function
import os
import re
import sys
+import argparse
import llnl.util.tty as tty
from llnl.util.lang import attr_setdefault, index_by
@@ -342,3 +343,15 @@ def spack_is_git_repo():
"""Ensure that this instance of Spack is a git clone."""
with working_dir(spack.paths.prefix):
return os.path.isdir('.git')
+
+
+########################################
+# argparse types for argument validation
+########################################
+def extant_file(f):
+ """
+ Argparse type for files that exist.
+ """
+ if not os.path.isfile(f):
+ raise argparse.ArgumentTypeError('%s does not exist' % f)
+ return f
diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py
index 8cdb6ec9b0..6ef455770f 100644
--- a/lib/spack/spack/cmd/view.py
+++ b/lib/spack/spack/cmd/view.py
@@ -37,13 +37,17 @@ import os
import llnl.util.tty as tty
from llnl.util.link_tree import MergeConflictError
+from llnl.util.tty.color import colorize
import spack.environment as ev
import spack.cmd
import spack.store
+import spack.schema.projections
+from spack.config import validate
from spack.filesystem_view import YamlFilesystemView
+from spack.util import spack_yaml as s_yaml
-description = "produce a single-rooted directory view of packages"
+description = "project packages to a compact naming scheme on the filesystem."
section = "environments"
level = "short"
@@ -52,28 +56,28 @@ actions_remove = ["remove", "rm"]
actions_status = ["statlink", "status", "check"]
-def relaxed_disambiguate(specs, view):
+def disambiguate_in_view(specs, view):
"""
- When dealing with querying actions (remove/status) the name of the spec
- is sufficient even though more versions of that name might be in the
- database.
+ When dealing with querying actions (remove/status) we only need to
+ disambiguate among specs in the view
"""
- name_to_spec = dict((s.name, s) for s in view.get_all_specs())
+ view_specs = set(view.get_all_specs())
def squash(matching_specs):
if not matching_specs:
tty.die("Spec matches no installed packages.")
- elif len(matching_specs) == 1:
- return matching_specs[0]
+ matching_in_view = [ms for ms in matching_specs if ms in view_specs]
- elif matching_specs[0].name in name_to_spec:
- return name_to_spec[matching_specs[0].name]
+ if len(matching_in_view) > 1:
+ args = ["Spec matches multiple packages.",
+ "Matching packages:"]
+ args += [colorize(" @K{%s} " % s.dag_hash(7)) +
+ s.cformat('$_$@$%@$=') for s in matching_in_view]
+ args += ["Use a more specific spec."]
+ tty.die(*args)
- else:
- # we just return the first matching spec, the error about the
- # missing spec will be printed later on
- return matching_specs[0]
+ return matching_in_view[0] if matching_in_view else matching_specs[0]
# make function always return a list to keep consistency between py2/3
return list(map(squash, map(spack.store.db.query, specs)))
@@ -120,6 +124,13 @@ def setup_parser(sp):
act.add_argument('path', nargs=1,
help="path to file system view directory")
+ if cmd in ("symlink", "hardlink"):
+ # invalid for remove/statlink, for those commands the view needs to
+ # already know its own projections.
+ help_msg = "Initialize view using projections from file."
+ act.add_argument('--projection-file', dest='projection_file',
+ type=spack.cmd.extant_file, help=help_msg)
+
if cmd == "remove":
grp = act.add_mutually_exclusive_group(required=True)
act.add_argument(
@@ -158,11 +169,20 @@ def view(parser, args):
specs = spack.cmd.parse_specs(args.specs)
path = args.path[0]
+ if args.action in actions_link and args.projection_file:
+ # argparse confirms file exists
+ with open(args.projection_file, 'r') as f:
+ projections_data = s_yaml.load(f)
+ validate(projections_data, spack.schema.projections.schema)
+ ordered_projections = projections_data['projections']
+ else:
+ ordered_projections = {}
+
view = YamlFilesystemView(
path, spack.store.layout,
+ projections=ordered_projections,
ignore_conflicts=getattr(args, "ignore_conflicts", False),
- link=os.link if args.action in ["hardlink", "hard"]
- else os.symlink,
+ link=os.link if args.action in ["hardlink", "hard"] else os.symlink,
verbose=args.verbose)
# Process common args and specs
@@ -181,11 +201,11 @@ def view(parser, args):
if len(specs) == 0:
specs = view.get_all_specs()
else:
- specs = relaxed_disambiguate(specs, view)
+ specs = disambiguate_in_view(specs, view)
else:
- # status and remove can map the name to packages in view
- specs = relaxed_disambiguate(specs, view)
+ # status and remove can map a partial spec to packages in view
+ specs = disambiguate_in_view(specs, view)
with_dependencies = args.dependencies.lower() in ['true', 'yes']
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 7762c59e56..be0cbc4965 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -142,12 +142,12 @@ class ConfigScope(object):
def write_section(self, section):
filename = self.get_section_filename(section)
data = self.get_section(section)
- _validate(data, section_schemas[section])
+ validate(data, section_schemas[section])
try:
mkdirp(self.path)
with open(filename, 'w') as f:
- _validate(data, section_schemas[section])
+ validate(data, section_schemas[section])
syaml.dump(data, stream=f, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError(
@@ -233,7 +233,7 @@ class SingleFileScope(ConfigScope):
return self.sections[section]
def write_section(self, section):
- _validate(self.sections, self.schema)
+ validate(self.sections, self.schema)
try:
parent = os.path.dirname(self.path)
mkdirp(parent)
@@ -277,7 +277,7 @@ class InternalConfigScope(ConfigScope):
if data:
for section in data:
dsec = data[section]
- _validate({section: dsec}, section_schemas[section])
+ validate({section: dsec}, section_schemas[section])
self.sections[section] = _mark_internal(
syaml.syaml_dict({section: dsec}), name)
@@ -295,7 +295,7 @@ class InternalConfigScope(ConfigScope):
"""This only validates, as the data is already in memory."""
data = self.get_section(section)
if data is not None:
- _validate(data, section_schemas[section])
+ validate(data, section_schemas[section])
self.sections[section] = _mark_internal(data, self.name)
def __repr__(self):
@@ -646,7 +646,7 @@ def _validate_section_name(section):
% (section, " ".join(section_schemas.keys())))
-def _validate(data, schema, set_defaults=True):
+def validate(data, schema, set_defaults=True):
"""Validate data read in from a Spack YAML file.
Arguments:
@@ -683,7 +683,7 @@ def _read_config_file(filename, schema):
data = _mark_overrides(syaml.load(f))
if data:
- _validate(data, schema)
+ validate(data, schema)
return data
except MarkedYAMLError as e:
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index f579d5664b..490a073a33 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -108,8 +108,8 @@ class ExtensionsLayout(object):
directly in the installation folder - or extensions activated in
filesystem views.
"""
- def __init__(self, root, **kwargs):
- self.root = root
+ def __init__(self, view, **kwargs):
+ self.view = view
def add_extension(self, spec, ext_spec):
"""Add to the list of currently installed extensions."""
@@ -309,11 +309,11 @@ class YamlDirectoryLayout(DirectoryLayout):
class YamlViewExtensionsLayout(ExtensionsLayout):
"""Maintain extensions within a view.
"""
- def __init__(self, root, layout):
+ def __init__(self, view, layout):
"""layout is the corresponding YamlDirectoryLayout object for which
we implement extensions.
"""
- super(YamlViewExtensionsLayout, self).__init__(root)
+ super(YamlViewExtensionsLayout, self).__init__(view)
self.layout = layout
self.extension_file_name = 'extensions.yaml'
@@ -354,15 +354,18 @@ class YamlViewExtensionsLayout(ExtensionsLayout):
_check_concrete(spec)
normalize_path = lambda p: (
os.path.abspath(p).rstrip(os.path.sep))
- if normalize_path(spec.prefix) == normalize_path(self.root):
- # For backwards compatibility, when the root is the extended
+
+ view_prefix = self.view.get_projection_for_spec(spec)
+ if normalize_path(spec.prefix) == normalize_path(view_prefix):
+ # For backwards compatibility, when the view is the extended
# package's installation directory, do not include the spec name
# as a subdirectory.
- components = [self.root, self.layout.metadata_dir,
+ components = [view_prefix, self.layout.metadata_dir,
self.extension_file_name]
else:
- components = [self.root, self.layout.metadata_dir, spec.name,
+ components = [view_prefix, self.layout.metadata_dir, spec.name,
self.extension_file_name]
+
return os.path.join(*components)
def extension_map(self, spec):
diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py
index bfb8d2a17e..ed69f53df6 100644
--- a/lib/spack/spack/filesystem_view.py
+++ b/lib/spack/spack/filesystem_view.py
@@ -12,13 +12,21 @@ import sys
from llnl.util.link_tree import LinkTree, MergeConflictError
from llnl.util import tty
-from llnl.util.lang import match_predicate
+from llnl.util.lang import match_predicate, index_by
+from llnl.util.tty.color import colorize
+from llnl.util.filesystem import mkdirp
+
+import spack.util.spack_yaml as s_yaml
import spack.spec
import spack.store
+import spack.schema.projections
+import spack.config
+from spack.error import SpackError
from spack.directory_layout import ExtensionAlreadyInstalledError
from spack.directory_layout import YamlViewExtensionsLayout
+
# compatability
if sys.version_info < (3, 0):
from itertools import imap as map
@@ -28,6 +36,9 @@ if sys.version_info < (3, 0):
__all__ = ["FilesystemView", "YamlFilesystemView"]
+_projections_path = '.spack/projections.yaml'
+
+
class FilesystemView(object):
"""
Governs a filesystem view that is located at certain root-directory.
@@ -48,9 +59,11 @@ class FilesystemView(object):
Files are linked by method `link` (os.symlink by default).
"""
- self.root = root
+ self._root = root
self.layout = layout
+ self.projections = kwargs.get('projections', {})
+
self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
self.link = kwargs.get("link", os.symlink)
self.verbose = kwargs.get("verbose", False)
@@ -125,6 +138,12 @@ class FilesystemView(object):
"""
raise NotImplementedError
+ def get_projection_for_spec(self, spec):
+ """
+ Get the projection in this view for a spec.
+ """
+ raise NotImplementedError
+
def get_all_specs(self):
"""
Get all specs currently active in this view.
@@ -166,9 +185,37 @@ class YamlFilesystemView(FilesystemView):
def __init__(self, root, layout, **kwargs):
super(YamlFilesystemView, self).__init__(root, layout, **kwargs)
- self.extensions_layout = YamlViewExtensionsLayout(root, layout)
+ # Super class gets projections from the kwargs
+ # YAML specific to get projections from YAML file
+ projections_path = os.path.join(self._root, _projections_path)
+ if not self.projections:
+ if os.path.exists(projections_path):
+ # Read projections file from view
+ with open(projections_path, 'r') as f:
+ projections_data = s_yaml.load(f)
+ spack.config.validate(projections_data,
+ spack.schema.projections.schema)
+ self.projections = projections_data['projections']
+ else:
+ # Write projections file to new view
+ # Not strictly necessary as the empty file is the empty
+ # projection but it makes sense for consistency
+ mkdirp(os.path.dirname(projections_path))
+ with open(projections_path, 'w') as f:
+ f.write(s_yaml.dump({'projections': self.projections}))
+ elif not os.path.exists(projections_path):
+ # Write projections file to new view
+ mkdirp(os.path.dirname(projections_path))
+ with open(projections_path, 'w') as f:
+ f.write(s_yaml.dump({'projections': self.projections}))
+ else:
+ msg = 'View at %s has projections file' % self._root
+ msg += ' and was passed projections manually.'
+ raise ConflictingProjectionsError(msg)
+
+ self.extensions_layout = YamlViewExtensionsLayout(self, layout)
- self._croot = colorize_root(self.root) + " "
+ self._croot = colorize_root(self._root) + " "
def add_specs(self, *specs, **kwargs):
assert all((s.concrete for s in specs))
@@ -371,8 +418,6 @@ class YamlFilesystemView(FilesystemView):
'Skipping package not linked in view: %s' % spec.name)
return
- # The spec might have been deactivated as depdency of another package
- # already
if spec.package.is_activated(self):
spec.package.do_deactivate(
self,
@@ -395,13 +440,45 @@ class YamlFilesystemView(FilesystemView):
if self.verbose:
tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec))
+ def get_projection_for_spec(self, spec):
+ """
+ Return the projection for a spec in this view.
+
+ Relies on the ordering of projections to avoid ambiguity.
+ """
+ spec = spack.spec.Spec(spec)
+ # Extensions are placed by their extendee, not by their own spec
+ locator_spec = spec
+ if spec.package.extendee_spec:
+ locator_spec = spec.package.extendee_spec
+
+ all_fmt_str = None
+ for spec_like, fmt_str in self.projections.items():
+ if locator_spec.satisfies(spec_like, strict=True):
+ return os.path.join(self._root, locator_spec.format(fmt_str))
+ elif spec_like == 'all':
+ all_fmt_str = fmt_str
+ if all_fmt_str:
+ return os.path.join(self._root, locator_spec.format(all_fmt_str))
+ return self._root
+
def get_all_specs(self):
- dotspack = os.path.join(self.root,
- spack.store.layout.metadata_dir)
- if os.path.exists(dotspack):
- return list(filter(None, map(self.get_spec, os.listdir(dotspack))))
- else:
- return []
+ md_dirs = []
+ for root, dirs, files in os.walk(self._root):
+ if spack.store.layout.metadata_dir in dirs:
+ md_dirs.append(os.path.join(root,
+ spack.store.layout.metadata_dir))
+
+ specs = []
+ for md_dir in md_dirs:
+ if os.path.exists(md_dir):
+ for name_dir in os.listdir(md_dir):
+ filename = os.path.join(md_dir, name_dir,
+ spack.store.layout.spec_file_name)
+ spec = get_spec_from_file(filename)
+ if spec:
+ specs.append(spec)
+ return specs
def get_conflicts(self, *specs):
"""
@@ -414,7 +491,7 @@ class YamlFilesystemView(FilesystemView):
def get_path_meta_folder(self, spec):
"Get path to meta folder for either spec or spec name."
- return os.path.join(self.root,
+ return os.path.join(self.get_projection_for_spec(spec),
spack.store.layout.metadata_dir,
getattr(spec, "name", spec))
@@ -423,11 +500,7 @@ class YamlFilesystemView(FilesystemView):
filename = os.path.join(dotspack,
spack.store.layout.spec_file_name)
- try:
- with open(filename, "r") as f:
- return spack.spec.Spec.from_yaml(f)
- except IOError:
- return None
+ return get_spec_from_file(filename)
def link_meta_folder(self, spec):
src = spack.store.layout.metadata_path(spec)
@@ -466,10 +539,39 @@ class YamlFilesystemView(FilesystemView):
if len(specs) > 0:
tty.msg("Packages linked in %s:" % self._croot[:-1])
- # avoid circular dependency
- import spack.cmd
- spack.cmd.display_specs(in_view, flags=True, variants=True,
- long=self.verbose)
+ # Make a dict with specs keyed by architecture and compiler.
+ index = index_by(specs, ('architecture', 'compiler'))
+
+ # Traverse the index and print out each package
+ for i, (architecture, compiler) in enumerate(sorted(index)):
+ if i > 0:
+ print()
+
+ header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
+ architecture,
+ spack.spec.compiler_color,
+ compiler)
+ tty.hline(colorize(header), char='-')
+
+ specs = index[(architecture, compiler)]
+ specs.sort()
+
+ format_string = '$_$@$%@+$+'
+ abbreviated = [s.cformat(format_string) for s in specs]
+
+ # Print one spec per line along with prefix path
+ width = max(len(s) for s in abbreviated)
+ width += 2
+ format = " %%-%ds%%s" % width
+
+ for abbrv, s in zip(abbreviated, specs):
+ prefix = ''
+ if self.verbose:
+ prefix = colorize('@K{%s}' % s.dag_hash(7))
+ print(
+ prefix + (format % (abbrv,
+ self.get_projection_for_spec(s)))
+ )
else:
tty.warn(self._croot + "No packages found.")
@@ -478,7 +580,7 @@ class YamlFilesystemView(FilesystemView):
Ascend up from the leaves accessible from `path`
and remove empty directories.
"""
- for dirpath, subdirs, files in os.walk(self.root, topdown=False):
+ for dirpath, subdirs, files in os.walk(self._root, topdown=False):
for sd in subdirs:
sdp = os.path.join(dirpath, sd)
try:
@@ -508,6 +610,13 @@ class YamlFilesystemView(FilesystemView):
#####################
# utility functions #
#####################
+def get_spec_from_file(filename):
+ try:
+ with open(filename, "r") as f:
+ return spack.spec.Spec.from_yaml(f)
+ except IOError:
+ return None
+
def colorize_root(root):
colorize = ft.partial(tty.color.colorize, color=sys.stdout.isatty())
@@ -553,3 +662,7 @@ def get_dependencies(specs):
retval = set()
set(map(retval.update, (set(s.traverse()) for s in specs)))
return retval
+
+
+class ConflictingProjectionsError(SpackError):
+ """Raised when a view has a projections file and is given one manually."""
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index d262e71df2..ebdfb4b1f8 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -284,7 +284,7 @@ class PackageViewMixin(object):
"""The target root directory: each file is added relative to this
directory.
"""
- return view.root
+ return view.get_projection_for_spec(self.spec)
def view_file_conflicts(self, view, merge_map):
"""Report any files which prevent adding this package to the view. The
diff --git a/lib/spack/spack/schema/projections.py b/lib/spack/spack/schema/projections.py
new file mode 100644
index 0000000000..b8aadd7379
--- /dev/null
+++ b/lib/spack/spack/schema/projections.py
@@ -0,0 +1,34 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for projections.yaml configuration file.
+
+.. literalinclude:: ../spack/schema/projections.py
+ :lines: 13-
+"""
+
+
+#: Properties for inclusion in other schemas
+properties = {
+ 'projections': {
+ 'type': 'object',
+ 'default': {},
+ 'patternProperties': {
+ r'all|\w[\w-]*': {
+ 'type': 'string'
+ },
+ },
+ },
+}
+
+
+#: Full schema with metadata
+schema = {
+ '$schema': 'http://json-schema.org/schema#',
+ 'title': 'Spack view projection configuration file schema',
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': properties,
+}
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 1ff2fb4677..9706ab215d 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -3025,6 +3025,8 @@ class Spec(object):
${SHA1} Dependencies 8-char sha1 prefix
${HASH:len} DAG hash with optional length specifier
+ ${DEP:name:OPTION} Evaluates as OPTION would for self['name']
+
${SPACK_ROOT} The spack root directory
${SPACK_INSTALL} The default spack install directory,
${SPACK_PREFIX}/opt
@@ -3218,6 +3220,10 @@ class Spec(object):
out.write(fmt % (self.dag_hash(hashlen)))
elif named_str == 'NAMESPACE':
out.write(fmt % transform(self.namespace))
+ elif named_str.startswith('DEP:'):
+ _, dep_name, dep_option = named_str.lower().split(':', 2)
+ dep_spec = self[dep_name]
+ out.write(fmt % (dep_spec.format('${%s}' % dep_option)))
named = False
diff --git a/lib/spack/spack/test/cmd/view.py b/lib/spack/spack/test/cmd/view.py
index 69eac0f926..ea4b0cc4f5 100644
--- a/lib/spack/spack/test/cmd/view.py
+++ b/lib/spack/spack/test/cmd/view.py
@@ -7,12 +7,23 @@ from spack.main import SpackCommand
import os.path
import pytest
+import spack.util.spack_yaml as s_yaml
+
activate = SpackCommand('activate')
extensions = SpackCommand('extensions')
install = SpackCommand('install')
view = SpackCommand('view')
+def create_projection_file(tmpdir, projection):
+ if 'projections' not in projection:
+ projection = {'projections': projection}
+
+ projection_file = tmpdir.mkdir('projection').join('projection.yaml')
+ projection_file.write(s_yaml.dump(projection))
+ return projection_file
+
+
@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
def test_view_link_type(
tmpdir, mock_packages, mock_archive, mock_fetch, config,
@@ -25,6 +36,71 @@ def test_view_link_type(
assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
+@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
+def test_view_projections(
+ tmpdir, mock_packages, mock_archive, mock_fetch, config,
+ install_mockery, cmd):
+ install('libdwarf@20130207')
+
+ viewpath = str(tmpdir.mkdir('view_{0}'.format(cmd)))
+ view_projection = {
+ 'projections': {
+ 'all': '${PACKAGE}-${VERSION}'
+ }
+ }
+ projection_file = create_projection_file(tmpdir, view_projection)
+ view(cmd, viewpath, '--projection-file={0}'.format(projection_file),
+ 'libdwarf')
+
+ package_prefix = os.path.join(viewpath, 'libdwarf-20130207/libdwarf')
+ assert os.path.exists(package_prefix)
+ assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
+
+
+def test_view_multiple_projections(
+ tmpdir, mock_packages, mock_archive, mock_fetch, config,
+ install_mockery):
+ install('libdwarf@20130207')
+ install('extendee@1.0%gcc')
+
+ viewpath = str(tmpdir.mkdir('view'))
+ view_projection = s_yaml.syaml_dict(
+ [('extendee', '${PACKAGE}-${COMPILERNAME}'),
+ ('all', '${PACKAGE}-${VERSION}')]
+ )
+
+ projection_file = create_projection_file(tmpdir, view_projection)
+ view('add', viewpath, '--projection-file={0}'.format(projection_file),
+ 'libdwarf', 'extendee')
+
+ libdwarf_prefix = os.path.join(viewpath, 'libdwarf-20130207/libdwarf')
+ extendee_prefix = os.path.join(viewpath, 'extendee-gcc/bin')
+ assert os.path.exists(libdwarf_prefix)
+ assert os.path.exists(extendee_prefix)
+
+
+def test_view_multiple_projections_all_first(
+ tmpdir, mock_packages, mock_archive, mock_fetch, config,
+ install_mockery):
+ install('libdwarf@20130207')
+ install('extendee@1.0%gcc')
+
+ viewpath = str(tmpdir.mkdir('view'))
+ view_projection = s_yaml.syaml_dict(
+ [('all', '${PACKAGE}-${VERSION}'),
+ ('extendee', '${PACKAGE}-${COMPILERNAME}')]
+ )
+
+ projection_file = create_projection_file(tmpdir, view_projection)
+ view('add', viewpath, '--projection-file={0}'.format(projection_file),
+ 'libdwarf', 'extendee')
+
+ libdwarf_prefix = os.path.join(viewpath, 'libdwarf-20130207/libdwarf')
+ extendee_prefix = os.path.join(viewpath, 'extendee-gcc/bin')
+ assert os.path.exists(libdwarf_prefix)
+ assert os.path.exists(extendee_prefix)
+
+
def test_view_external(
tmpdir, mock_packages, mock_archive, mock_fetch, config,
install_mockery):
@@ -60,6 +136,39 @@ def test_view_extension(
assert os.path.exists(os.path.join(viewpath, 'bin', 'extension1'))
+def test_view_extension_projection(
+ tmpdir, mock_packages, mock_archive, mock_fetch, config,
+ install_mockery):
+ install('extendee@1.0')
+ install('extension1@1.0')
+ install('extension1@2.0')
+ install('extension2@1.0')
+
+ viewpath = str(tmpdir.mkdir('view'))
+ view_projection = {'all': '${PACKAGE}-${VERSION}'}
+ projection_file = create_projection_file(tmpdir, view_projection)
+ view('symlink', viewpath, '--projection-file={0}'.format(projection_file),
+ 'extension1@1.0')
+
+ all_installed = extensions('--show', 'installed', 'extendee')
+ assert 'extension1@1.0' in all_installed
+ assert 'extension1@2.0' in all_installed
+ assert 'extension2@1.0' in all_installed
+ global_activated = extensions('--show', 'activated', 'extendee')
+ assert 'extension1@1.0' not in global_activated
+ assert 'extension1@2.0' not in global_activated
+ assert 'extension2@1.0' not in global_activated
+ view_activated = extensions('--show', 'activated',
+ '-v', viewpath,
+ 'extendee')
+ assert 'extension1@1.0' in view_activated
+ assert 'extension1@2.0' not in view_activated
+ assert 'extension2@1.0' not in view_activated
+
+ assert os.path.exists(os.path.join(viewpath, 'extendee-1.0',
+ 'bin', 'extension1'))
+
+
def test_view_extension_remove(
tmpdir, mock_packages, mock_archive, mock_fetch, config,
install_mockery):
@@ -144,3 +253,10 @@ def test_view_extendee_with_global_activations(
activate('extension1@2.0')
output = view('symlink', viewpath, 'extension1@1.0')
assert 'Error: Globally activated extensions cannot be used' in output
+
+
+def test_view_fails_with_missing_projections_file(tmpdir):
+ viewpath = str(tmpdir.mkdir('view'))
+ projection_file = os.path.join(str(tmpdir), 'nonexistent')
+ with pytest.raises(SystemExit):
+ view('symlink', '--projection-file', projection_file, viewpath, 'foo')
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
index e9c012b856..b89dbc5fe5 100644
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -677,7 +677,7 @@ def check_schema(name, file_contents):
"""Check a Spack YAML schema against some data"""
f = StringIO(file_contents)
data = syaml.load(f)
- spack.config._validate(data, name)
+ spack.config.validate(data, name)
def test_good_env_yaml(tmpdir):
diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py
index 7ed235f81e..092a004d4f 100644
--- a/lib/spack/spack/test/test_activations.py
+++ b/lib/spack/spack/test/test_activations.py
@@ -5,69 +5,33 @@
import os
import pytest
-import sys
import spack.spec
import spack.package
from llnl.util.link_tree import MergeConflictError
-from spack.build_systems.python import PythonPackage
from spack.directory_layout import YamlDirectoryLayout
from spack.filesystem_view import YamlFilesystemView
-from spack.util.prefix import Prefix
+from spack.repo import RepoPath
"""This includes tests for customized activation logic for specific packages
(e.g. python and perl).
"""
-class FakeExtensionPackage(spack.package.PackageViewMixin):
- def __init__(self, name, prefix):
- self.name = name
- self.prefix = Prefix(prefix)
- self.spec = FakeSpec(self)
+def create_ext_pkg(name, prefix, extendee_spec):
+ ext_spec = spack.spec.Spec(name)
+ ext_spec._concrete = True
+ ext_spec.package.spec.prefix = prefix
+ ext_pkg = ext_spec.package
+ ext_pkg.extends_spec = extendee_spec
+ return ext_pkg
-class FakeSpec(object):
- def __init__(self, package):
- self.name = package.name
- self.prefix = package.prefix
- self.hash = self.name
- self.package = package
- self.concrete = True
- def dag_hash(self):
- return self.hash
-
- def __lt__(self, other):
- return self.name < other.name
-
-
-class FakePythonExtensionPackage(FakeExtensionPackage):
- def __init__(self, name, prefix, py_namespace, python_spec):
- self.py_namespace = py_namespace
- self.extendee_spec = python_spec
- super(FakePythonExtensionPackage, self).__init__(name, prefix)
-
- def add_files_to_view(self, view, merge_map):
- if sys.version_info >= (3, 0):
- add_fn = PythonPackage.add_files_to_view
- else:
- add_fn = PythonPackage.add_files_to_view.im_func
- return add_fn(self, view, merge_map)
-
- def view_file_conflicts(self, view, merge_map):
- if sys.version_info >= (3, 0):
- conflicts_fn = PythonPackage.view_file_conflicts
- else:
- conflicts_fn = PythonPackage.view_file_conflicts.im_func
- return conflicts_fn(self, view, merge_map)
-
- def remove_files_from_view(self, view, merge_map):
- if sys.version_info >= (3, 0):
- remove_fn = PythonPackage.remove_files_from_view
- else:
- remove_fn = PythonPackage.remove_files_from_view.im_func
- return remove_fn(self, view, merge_map)
+def create_python_ext_pkg(name, prefix, python_spec, namespace=None):
+ ext_pkg = create_ext_pkg(name, prefix, python_spec)
+ ext_pkg.py_namespace = namespace
+ return ext_pkg
def create_dir_structure(tmpdir, dir_structure):
@@ -78,7 +42,20 @@ def create_dir_structure(tmpdir, dir_structure):
@pytest.fixture()
-def python_and_extension_dirs(tmpdir):
+def builtin_and_mock_packages():
+ # These tests use mock_repo packages to test functionality of builtin
+ # packages for python and perl. To test this we put the mock repo at lower
+ # precedence than the builtin repo, so we test builtin.perl against
+ # builtin.mock.perl-extension.
+ repo_dirs = [spack.paths.packages_path, spack.paths.mock_packages_path]
+ path = RepoPath(*repo_dirs)
+
+ with spack.repo.swap(path):
+ yield
+
+
+@pytest.fixture()
+def python_and_extension_dirs(tmpdir, builtin_and_mock_packages):
python_dirs = {
'bin/': {
'python': None
@@ -105,7 +82,7 @@ def python_and_extension_dirs(tmpdir):
'lib/': {
'python2.7/': {
'site-packages/': {
- 'py-extension/': {
+ 'py-extension1/': {
'sample.py': None
}
}
@@ -113,7 +90,7 @@ def python_and_extension_dirs(tmpdir):
}
}
- ext_name = 'py-extension'
+ ext_name = 'py-extension1'
ext_prefix = tmpdir.join(ext_name)
create_dir_structure(ext_prefix, ext_dirs)
@@ -126,7 +103,7 @@ path/to/setuptools.egg""")
@pytest.fixture()
-def namespace_extensions(tmpdir):
+def namespace_extensions(tmpdir, builtin_and_mock_packages):
ext1_dirs = {
'bin/': {
'py-ext-tool1': None
@@ -170,14 +147,15 @@ def namespace_extensions(tmpdir):
return str(ext1_prefix), str(ext2_prefix), 'examplenamespace'
-def test_python_activation_with_files(tmpdir, python_and_extension_dirs):
+def test_python_activation_with_files(tmpdir, python_and_extension_dirs,
+ builtin_and_mock_packages):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True
python_spec.package.spec.prefix = python_prefix
- ext_pkg = FakeExtensionPackage('py-extension', ext_prefix)
+ ext_pkg = create_python_ext_pkg('py-extension1', ext_prefix, python_spec)
python_pkg = python_spec.package
python_pkg.activate(ext_pkg, python_pkg.view())
@@ -192,14 +170,15 @@ def test_python_activation_with_files(tmpdir, python_and_extension_dirs):
assert 'setuptools.egg' not in easy_install_contents
-def test_python_activation_view(tmpdir, python_and_extension_dirs):
+def test_python_activation_view(tmpdir, python_and_extension_dirs,
+ builtin_and_mock_packages):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True
python_spec.package.spec.prefix = python_prefix
- ext_pkg = FakeExtensionPackage('py-extension', ext_prefix)
+ ext_pkg = create_python_ext_pkg('py-extension1', ext_prefix, python_spec)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
@@ -213,7 +192,8 @@ def test_python_activation_view(tmpdir, python_and_extension_dirs):
assert os.path.exists(os.path.join(view_dir, 'bin/py-ext-tool'))
-def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions):
+def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions,
+ builtin_and_mock_packages):
"""Test the view update logic in PythonPackage ignores conflicting
instances of __init__ for packages which are in the same namespace.
"""
@@ -222,10 +202,10 @@ def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions):
python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True
- ext1_pkg = FakePythonExtensionPackage(
- 'py-extension1', ext1_prefix, py_namespace, python_spec)
- ext2_pkg = FakePythonExtensionPackage(
- 'py-extension2', ext2_prefix, py_namespace, python_spec)
+ ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+ py_namespace)
+ ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+ py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
@@ -246,7 +226,8 @@ def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions):
assert os.path.exists(os.path.join(view_dir, init_file))
-def test_python_keep_namespace_init(tmpdir, namespace_extensions):
+def test_python_keep_namespace_init(tmpdir, namespace_extensions,
+ builtin_and_mock_packages):
"""Test the view update logic in PythonPackage keeps the namespace
__init__ file as long as one package in the namespace still
exists.
@@ -256,10 +237,10 @@ def test_python_keep_namespace_init(tmpdir, namespace_extensions):
python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True
- ext1_pkg = FakePythonExtensionPackage(
- 'py-extension1', ext1_prefix, py_namespace, python_spec)
- ext2_pkg = FakePythonExtensionPackage(
- 'py-extension2', ext2_prefix, py_namespace, python_spec)
+ ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+ py_namespace)
+ ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+ py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
@@ -287,7 +268,8 @@ def test_python_keep_namespace_init(tmpdir, namespace_extensions):
assert not os.path.exists(os.path.join(view_dir, init_file))
-def test_python_namespace_conflict(tmpdir, namespace_extensions):
+def test_python_namespace_conflict(tmpdir, namespace_extensions,
+ builtin_and_mock_packages):
"""Test the view update logic in PythonPackage reports an error when two
python extensions with different namespaces have a conflicting __init__
file.
@@ -298,10 +280,10 @@ def test_python_namespace_conflict(tmpdir, namespace_extensions):
python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True
- ext1_pkg = FakePythonExtensionPackage(
- 'py-extension1', ext1_prefix, py_namespace, python_spec)
- ext2_pkg = FakePythonExtensionPackage(
- 'py-extension2', ext2_prefix, other_namespace, python_spec)
+ ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
+ py_namespace)
+ ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
+ other_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
@@ -315,7 +297,7 @@ def test_python_namespace_conflict(tmpdir, namespace_extensions):
@pytest.fixture()
-def perl_and_extension_dirs(tmpdir):
+def perl_and_extension_dirs(tmpdir, builtin_and_mock_packages):
perl_dirs = {
'bin/': {
'perl': None
@@ -360,7 +342,7 @@ def perl_and_extension_dirs(tmpdir):
return str(perl_prefix), str(ext_prefix)
-def test_perl_activation(tmpdir):
+def test_perl_activation(tmpdir, builtin_and_mock_packages):
# Note the lib directory is based partly on the perl version
perl_spec = spack.spec.Spec('perl@5.24.1')
perl_spec._concrete = True
@@ -375,20 +357,21 @@ def test_perl_activation(tmpdir):
ext_name = 'perl-extension'
tmpdir.ensure(ext_name, dir=True)
- ext_pkg = FakeExtensionPackage(ext_name, str(tmpdir.join(ext_name)))
+ ext_pkg = create_ext_pkg(ext_name, str(tmpdir.join(ext_name)), perl_spec)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, perl_pkg.view())
-def test_perl_activation_with_files(tmpdir, perl_and_extension_dirs):
+def test_perl_activation_with_files(tmpdir, perl_and_extension_dirs,
+ builtin_and_mock_packages):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('perl@5.24.1')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
- ext_pkg = FakeExtensionPackage('perl-extension', ext_prefix)
+ ext_pkg = create_ext_pkg('perl-extension', ext_prefix, perl_spec)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, perl_pkg.view())
@@ -396,14 +379,15 @@ def test_perl_activation_with_files(tmpdir, perl_and_extension_dirs):
assert os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool'))
-def test_perl_activation_view(tmpdir, perl_and_extension_dirs):
+def test_perl_activation_view(tmpdir, perl_and_extension_dirs,
+ builtin_and_mock_packages):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('perl@5.24.1')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
- ext_pkg = FakeExtensionPackage('perl-extension', ext_prefix)
+ ext_pkg = create_ext_pkg('perl-extension', ext_prefix, perl_spec)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-06 06:44:49 +0000