diff options
| -rw-r--r-- | lib/spack/docs/config_yaml.rst | 37 | ||||
| -rw-r--r-- | lib/spack/spack/directory_layout.py | 44 | ||||
| -rw-r--r-- | lib/spack/spack/schema/config.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/store.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/test/directory_layout.py | 43 |
6 files changed, 112 insertions, 22 deletions
diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 56aa6ed0a1..197bf5c530 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -42,6 +42,43 @@ or with braces to distinguish the variable from surrounding characters: The location where Spack will install packages and their dependencies. Default is ``$spack/opt/spack``. +--------------------------------------------------- +``install_hash_length`` and ``install_path_scheme`` +--------------------------------------------------- + +The default Spack installation path can be very long and can create +problems for scripts with hardcoded shebangs. There are two parameters +to help with that. Firstly, the ``install_hash_length`` parameter can +set the length of the hash in the installation path from 1 to 32. The +default path uses the full 32 characters. + +Secondly, it is +also possible to modify the entire installation scheme. By default +Spack uses +``${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}`` +where the tokens that are available for use in this directive are the +same as those understood by the ``Spec.format`` method. Using this parameter it +is possible to use a different package layout or reduce the depth of +the installation paths. For example + + .. code-block:: yaml + + config: + install_path_scheme: '${PACKAGE}/${VERSION}/${HASH:7}' + +would install packages into sub-directories using only the package +name, version and a hash length of 7 characters. + +When using either parameter to set the hash length it only affects the +representation of the hash in the installation directory. You +should be aware that the smaller the hash length the more likely +naming conflicts will occur. These parameters are independent of those +used to configure module names. + +.. warning:: Modifying the installation hash length or path scheme after + packages have been installed will prevent Spack from being + able to find the old installation directories. + -------------------- ``module_roots`` -------------------- diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 28e6584fb2..b84ee3be5b 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -28,6 +28,7 @@ import shutil import glob import tempfile import yaml +import re from llnl.util.filesystem import join_path, mkdirp @@ -150,24 +151,31 @@ class DirectoryLayout(object): class YamlDirectoryLayout(DirectoryLayout): - """Lays out installation directories like this:: + """By default lays out installation directories like this:: <install root>/ <platform-os-target>/ <compiler>-<compiler version>/ - <name>-<version>-<variants>-<hash> + <name>-<version>-<hash> The hash here is a SHA-1 hash for the full DAG plus the build spec. TODO: implement the build spec. - To avoid special characters (like ~) in the directory name, - only enabled variants are included in the install path. - Disabled variants are omitted. + The installation directory scheme can be modified with the + arguments hash_len and path_scheme. """ def __init__(self, root, **kwargs): super(YamlDirectoryLayout, self).__init__(root) self.metadata_dir = kwargs.get('metadata_dir', '.spack') - self.hash_len = kwargs.get('hash_len', None) + self.hash_len = kwargs.get('hash_len') + self.path_scheme = kwargs.get('path_scheme') or ( + "${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}") # NOQA: E501 + if self.hash_len is not None: + if re.search('\${HASH:\d+}', self.path_scheme): + raise InvalidDirectoryLayoutParametersError( + "Conflicting options for installation layout hash length") + self.path_scheme = self.path_scheme.replace( + "${HASH}", "${HASH:%d}" % self.hash_len) self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' @@ -188,16 +196,7 @@ class YamlDirectoryLayout(DirectoryLayout): if spec.external: return spec.external - dir_name = "%s-%s-%s" % ( - spec.name, - spec.version, - spec.dag_hash(self.hash_len)) - - path = join_path( - spec.architecture, - "%s-%s" % (spec.compiler.name, spec.compiler.version), - dir_name) - + path = spec.format(self.path_scheme) return path def write_spec(self, spec, path): @@ -285,8 +284,9 @@ class YamlDirectoryLayout(DirectoryLayout): if not os.path.isdir(self.root): return [] - pattern = join_path( - self.root, '*', '*', '*', self.metadata_dir, self.spec_file_name) + path_elems = ["*"] * len(self.path_scheme.split(os.sep)) + path_elems += [self.metadata_dir, self.spec_file_name] + pattern = join_path(self.root, *path_elems) spec_files = glob.glob(pattern) return [self.read_spec(s) for s in spec_files] @@ -447,6 +447,14 @@ class SpecReadError(DirectoryLayoutError): """Raised when directory layout can't read a spec.""" +class InvalidDirectoryLayoutParametersError(DirectoryLayoutError): + """Raised when a invalid directory layout parameters are supplied""" + + def __init__(self, message, long_msg=None): + super(InvalidDirectoryLayoutParametersError, self).__init__( + message, long_msg) + + class InvalidExtensionSpecError(DirectoryLayoutError): """Raised when an extension file has a bad spec in it.""" diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index e51fa69afe..7a41c0e14a 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -41,6 +41,8 @@ schema = { 'additionalProperties': False, 'properties': { 'install_tree': {'type': 'string'}, + 'install_hash_length': {'type': 'integer', 'minimum': 1}, + 'install_path_scheme': {'type': 'string'}, 'build_stage': { 'oneOf': [ {'type': 'string'}, diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9fc2c99e4a..b2fae9fd8e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2687,7 +2687,7 @@ class Spec(object): write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': if self.architecture and str(self.architecture): - write(fmt % str(self.architecture) + ' ', ' arch=') + write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -2703,7 +2703,7 @@ class Spec(object): hashlen = int(hashlen) else: hashlen = None - out.write('/' + fmt % (self.dag_hash(hashlen))) + out.write(fmt % (self.dag_hash(hashlen))) named = False diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 3f559315d2..4a5c8d18a7 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -72,4 +72,6 @@ db = Database(root) # This controls how spack lays out install prefixes and # stage directories. # -layout = YamlDirectoryLayout(root) +layout = YamlDirectoryLayout(root, + hash_len=config.get('install_hash_length'), + path_scheme=config.get('install_path_scheme')) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 2caadad0fe..1987bb3a44 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -29,7 +29,8 @@ import os import pytest import spack -from spack.directory_layout import YamlDirectoryLayout +from spack.directory_layout import (YamlDirectoryLayout, + InvalidDirectoryLayoutParametersError) from spack.repository import RepoPath from spack.spec import Spec @@ -43,6 +44,46 @@ def layout_and_dir(tmpdir): yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir) +def test_yaml_directory_layout_parameters( + tmpdir, config +): + """This tests the various parameters that can be used to configure + the install location """ + spec = Spec('python') + spec.concretize() + + # Ensure default layout matches expected spec format + layout_default = YamlDirectoryLayout(str(tmpdir)) + path_default = layout_default.relative_path_for_spec(spec) + assert(path_default == + spec.format("${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}")) # NOQA: ignore=E501 + + # Test hash_length parameter works correctly + layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10) + path_10 = layout_10.relative_path_for_spec(spec) + layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7) + path_7 = layout_7.relative_path_for_spec(spec) + + assert(len(path_default) - len(path_10) == 22) + assert(len(path_default) - len(path_7) == 25) + + # Test path_scheme + arch, compiler, package7 = path_7.split('/') + scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}" + + layout_package7 = YamlDirectoryLayout(str(tmpdir), + path_scheme=scheme_package7) + path_package7 = layout_package7.relative_path_for_spec(spec) + + assert(package7 == path_package7) + + # Ensure conflicting parameters caught + with pytest.raises(InvalidDirectoryLayoutParametersError): + YamlDirectoryLayout(str(tmpdir), + hash_len=20, + path_scheme=scheme_package7) + + def test_read_and_write_spec( layout_and_dir, config, builtin_mock ): |