summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Hopkins <paul.hopkins@ligo.org>2016-11-03 07:49:51 +0000
committerPaul Hopkins <paul.hopkins@ligo.org>2017-04-05 12:08:56 +0100
commit95c3cff6be794881980e5c218567f7cc5fbd5ed3 (patch)
treece583266072d018748ec003d2cb1953fc426f076
parentf3ee4ec5bde6b16aed5aaf73ee340e34f47f789c (diff)
downloadspack-95c3cff6be794881980e5c218567f7cc5fbd5ed3.tar.gz
spack-95c3cff6be794881980e5c218567f7cc5fbd5ed3.tar.bz2
spack-95c3cff6be794881980e5c218567f7cc5fbd5ed3.tar.xz
spack-95c3cff6be794881980e5c218567f7cc5fbd5ed3.zip
Allow installation directory layout to be configured using either hash
length parameter or spec formatting.
-rw-r--r--lib/spack/docs/config_yaml.rst37
-rw-r--r--lib/spack/spack/directory_layout.py44
-rw-r--r--lib/spack/spack/schema/config.py2
-rw-r--r--lib/spack/spack/spec.py4
-rw-r--r--lib/spack/spack/store.py4
-rw-r--r--lib/spack/spack/test/directory_layout.py43
6 files changed, 112 insertions, 22 deletions
diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst
index 56aa6ed0a1..197bf5c530 100644
--- a/lib/spack/docs/config_yaml.rst
+++ b/lib/spack/docs/config_yaml.rst
@@ -42,6 +42,43 @@ or with braces to distinguish the variable from surrounding characters:
The location where Spack will install packages and their dependencies.
Default is ``$spack/opt/spack``.
+---------------------------------------------------
+``install_hash_length`` and ``install_path_scheme``
+---------------------------------------------------
+
+The default Spack installation path can be very long and can create
+problems for scripts with hardcoded shebangs. There are two parameters
+to help with that. Firstly, the ``install_hash_length`` parameter can
+set the length of the hash in the installation path from 1 to 32. The
+default path uses the full 32 characters.
+
+Secondly, it is
+also possible to modify the entire installation scheme. By default
+Spack uses
+``${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}``
+where the tokens that are available for use in this directive are the
+same as those understood by the ``Spec.format`` method. Using this parameter it
+is possible to use a different package layout or reduce the depth of
+the installation paths. For example
+
+ .. code-block:: yaml
+
+ config:
+ install_path_scheme: '${PACKAGE}/${VERSION}/${HASH:7}'
+
+would install packages into sub-directories using only the package
+name, version and a hash length of 7 characters.
+
+When using either parameter to set the hash length it only affects the
+representation of the hash in the installation directory. You
+should be aware that the smaller the hash length the more likely
+naming conflicts will occur. These parameters are independent of those
+used to configure module names.
+
+.. warning:: Modifying the installation hash length or path scheme after
+ packages have been installed will prevent Spack from being
+ able to find the old installation directories.
+
--------------------
``module_roots``
--------------------
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 28e6584fb2..b84ee3be5b 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -28,6 +28,7 @@ import shutil
import glob
import tempfile
import yaml
+import re
from llnl.util.filesystem import join_path, mkdirp
@@ -150,24 +151,31 @@ class DirectoryLayout(object):
class YamlDirectoryLayout(DirectoryLayout):
- """Lays out installation directories like this::
+ """By default lays out installation directories like this::
<install root>/
<platform-os-target>/
<compiler>-<compiler version>/
- <name>-<version>-<variants>-<hash>
+ <name>-<version>-<hash>
The hash here is a SHA-1 hash for the full DAG plus the build
spec. TODO: implement the build spec.
- To avoid special characters (like ~) in the directory name,
- only enabled variants are included in the install path.
- Disabled variants are omitted.
+ The installation directory scheme can be modified with the
+ arguments hash_len and path_scheme.
"""
def __init__(self, root, **kwargs):
super(YamlDirectoryLayout, self).__init__(root)
self.metadata_dir = kwargs.get('metadata_dir', '.spack')
- self.hash_len = kwargs.get('hash_len', None)
+ self.hash_len = kwargs.get('hash_len')
+ self.path_scheme = kwargs.get('path_scheme') or (
+ "${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}") # NOQA: E501
+ if self.hash_len is not None:
+ if re.search('\${HASH:\d+}', self.path_scheme):
+ raise InvalidDirectoryLayoutParametersError(
+ "Conflicting options for installation layout hash length")
+ self.path_scheme = self.path_scheme.replace(
+ "${HASH}", "${HASH:%d}" % self.hash_len)
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
@@ -188,16 +196,7 @@ class YamlDirectoryLayout(DirectoryLayout):
if spec.external:
return spec.external
- dir_name = "%s-%s-%s" % (
- spec.name,
- spec.version,
- spec.dag_hash(self.hash_len))
-
- path = join_path(
- spec.architecture,
- "%s-%s" % (spec.compiler.name, spec.compiler.version),
- dir_name)
-
+ path = spec.format(self.path_scheme)
return path
def write_spec(self, spec, path):
@@ -285,8 +284,9 @@ class YamlDirectoryLayout(DirectoryLayout):
if not os.path.isdir(self.root):
return []
- pattern = join_path(
- self.root, '*', '*', '*', self.metadata_dir, self.spec_file_name)
+ path_elems = ["*"] * len(self.path_scheme.split(os.sep))
+ path_elems += [self.metadata_dir, self.spec_file_name]
+ pattern = join_path(self.root, *path_elems)
spec_files = glob.glob(pattern)
return [self.read_spec(s) for s in spec_files]
@@ -447,6 +447,14 @@ class SpecReadError(DirectoryLayoutError):
"""Raised when directory layout can't read a spec."""
+class InvalidDirectoryLayoutParametersError(DirectoryLayoutError):
+ """Raised when a invalid directory layout parameters are supplied"""
+
+ def __init__(self, message, long_msg=None):
+ super(InvalidDirectoryLayoutParametersError, self).__init__(
+ message, long_msg)
+
+
class InvalidExtensionSpecError(DirectoryLayoutError):
"""Raised when an extension file has a bad spec in it."""
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index e51fa69afe..7a41c0e14a 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -41,6 +41,8 @@ schema = {
'additionalProperties': False,
'properties': {
'install_tree': {'type': 'string'},
+ 'install_hash_length': {'type': 'integer', 'minimum': 1},
+ 'install_path_scheme': {'type': 'string'},
'build_stage': {
'oneOf': [
{'type': 'string'},
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 9fc2c99e4a..b2fae9fd8e 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -2687,7 +2687,7 @@ class Spec(object):
write(fmt % str(self.variants), '+')
elif named_str == 'ARCHITECTURE':
if self.architecture and str(self.architecture):
- write(fmt % str(self.architecture) + ' ', ' arch=')
+ write(fmt % str(self.architecture), ' arch=')
elif named_str == 'SHA1':
if self.dependencies:
out.write(fmt % str(self.dag_hash(7)))
@@ -2703,7 +2703,7 @@ class Spec(object):
hashlen = int(hashlen)
else:
hashlen = None
- out.write('/' + fmt % (self.dag_hash(hashlen)))
+ out.write(fmt % (self.dag_hash(hashlen)))
named = False
diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py
index 3f559315d2..4a5c8d18a7 100644
--- a/lib/spack/spack/store.py
+++ b/lib/spack/spack/store.py
@@ -72,4 +72,6 @@ db = Database(root)
# This controls how spack lays out install prefixes and
# stage directories.
#
-layout = YamlDirectoryLayout(root)
+layout = YamlDirectoryLayout(root,
+ hash_len=config.get('install_hash_length'),
+ path_scheme=config.get('install_path_scheme'))
diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py
index 2caadad0fe..1987bb3a44 100644
--- a/lib/spack/spack/test/directory_layout.py
+++ b/lib/spack/spack/test/directory_layout.py
@@ -29,7 +29,8 @@ import os
import pytest
import spack
-from spack.directory_layout import YamlDirectoryLayout
+from spack.directory_layout import (YamlDirectoryLayout,
+ InvalidDirectoryLayoutParametersError)
from spack.repository import RepoPath
from spack.spec import Spec
@@ -43,6 +44,46 @@ def layout_and_dir(tmpdir):
yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir)
+def test_yaml_directory_layout_parameters(
+ tmpdir, config
+):
+ """This tests the various parameters that can be used to configure
+ the install location """
+ spec = Spec('python')
+ spec.concretize()
+
+ # Ensure default layout matches expected spec format
+ layout_default = YamlDirectoryLayout(str(tmpdir))
+ path_default = layout_default.relative_path_for_spec(spec)
+ assert(path_default ==
+ spec.format("${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}")) # NOQA: ignore=E501
+
+ # Test hash_length parameter works correctly
+ layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
+ path_10 = layout_10.relative_path_for_spec(spec)
+ layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
+ path_7 = layout_7.relative_path_for_spec(spec)
+
+ assert(len(path_default) - len(path_10) == 22)
+ assert(len(path_default) - len(path_7) == 25)
+
+ # Test path_scheme
+ arch, compiler, package7 = path_7.split('/')
+ scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
+
+ layout_package7 = YamlDirectoryLayout(str(tmpdir),
+ path_scheme=scheme_package7)
+ path_package7 = layout_package7.relative_path_for_spec(spec)
+
+ assert(package7 == path_package7)
+
+ # Ensure conflicting parameters caught
+ with pytest.raises(InvalidDirectoryLayoutParametersError):
+ YamlDirectoryLayout(str(tmpdir),
+ hash_len=20,
+ path_scheme=scheme_package7)
+
+
def test_read_and_write_spec(
layout_and_dir, config, builtin_mock
):