summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/basic_usage.rst2
-rw-r--r--lib/spack/docs/getting_started.rst2
-rwxr-xr-xlib/spack/env/cc2
-rw-r--r--lib/spack/llnl/util/filesystem.py11
-rw-r--r--lib/spack/spack/cmd/checksum.py13
-rw-r--r--lib/spack/spack/cmd/repo.py48
-rw-r--r--lib/spack/spack/directives.py6
-rw-r--r--lib/spack/spack/directory_layout.py14
-rw-r--r--lib/spack/spack/modules.py2
-rw-r--r--lib/spack/spack/multimethod.py9
-rw-r--r--lib/spack/spack/package.py57
-rw-r--r--lib/spack/spack/repository.py99
-rw-r--r--lib/spack/spack/resource.py2
-rw-r--r--lib/spack/spack/test/namespace_trie.py2
-rw-r--r--lib/spack/spack/test/tally_plugin.py8
-rw-r--r--lib/spack/spack/util/environment.py7
-rw-r--r--lib/spack/spack/util/web.py6
17 files changed, 218 insertions, 72 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 0578f0c8db..f94ac3d2ba 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -896,7 +896,7 @@ Or, similarly with modules, you could type:
$ spack load mpich %gcc@4.4.7
These commands will add appropriate directories to your ``PATH``,
-``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use
+``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use
a package, you can type unload or unuse similarly:
.. code-block:: sh
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index 67ca18e71a..2c5b68ea65 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -22,7 +22,7 @@ go:
$ spack install libelf
For a richer experience, use Spack's `shell support
-<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_:
+<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_:
.. code-block:: sh
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index aacba996b3..a323c48124 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -130,7 +130,7 @@ if [ -z "$mode" ]; then
done
fi
-# Dump the version and exist if we're in testing mode.
+# Dump the version and exit if we're in testing mode.
if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
echo "$mode"
exit
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 015eeb9aa1..366237ef8f 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -152,15 +152,20 @@ def set_install_permissions(path):
def copy_mode(src, dest):
src_mode = os.stat(src).st_mode
dest_mode = os.stat(dest).st_mode
- if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
- if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
- if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+ if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
+ if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
+ if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
os.chmod(dest, dest_mode)
def install(src, dest):
"""Manually install a file to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
+
+ # Expand dsst to its eventual full path if it is a directory.
+ if os.path.isdir(dest):
+ dest = join_path(dest, os.path.basename(src))
+
shutil.copy(src, dest)
set_install_permissions(dest)
copy_mode(src, dest)
diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py
index b1ad89dbb8..c451993233 100644
--- a/lib/spack/spack/cmd/checksum.py
+++ b/lib/spack/spack/cmd/checksum.py
@@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs):
tty.msg("Downloading...")
hashes = []
- for i, (url, version) in enumerate(zip(urls, versions)):
+ i = 0
+ for url, version in zip(urls, versions):
stage = Stage(url)
try:
stage.fetch()
if i == 0 and first_stage_function:
first_stage_function(stage)
- hashes.append(
- spack.util.crypto.checksum(hashlib.md5, stage.archive_file))
+ hashes.append((version,
+ spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
except FailedDownloadError, e:
tty.msg("Failed to fetch %s" % url)
continue
+ except Exception, e:
+ tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e))
+ continue
finally:
if not keep_stage:
stage.destroy()
+ i += 1
- return zip(versions, hashes)
+ return hashes
diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py
index 34c755fb67..c2e352786d 100644
--- a/lib/spack/spack/cmd/repo.py
+++ b/lib/spack/spack/cmd/repo.py
@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -74,51 +74,7 @@ def setup_parser(subparser):
def repo_create(args):
"""Create a new package repository."""
- root = canonicalize_path(args.directory)
- namespace = args.namespace
-
- if not args.namespace:
- namespace = os.path.basename(root)
-
- if not re.match(r'\w[\.\w-]*', namespace):
- tty.die("'%s' is not a valid namespace." % namespace)
-
- existed = False
- if os.path.exists(root):
- if os.path.isfile(root):
- tty.die('File %s already exists and is not a directory' % root)
- elif os.path.isdir(root):
- if not os.access(root, os.R_OK | os.W_OK):
- tty.die('Cannot create new repo in %s: cannot access directory.' % root)
- if os.listdir(root):
- tty.die('Cannot create new repo in %s: directory is not empty.' % root)
- existed = True
-
- full_path = os.path.realpath(root)
- parent = os.path.dirname(full_path)
- if not os.access(parent, os.R_OK | os.W_OK):
- tty.die("Cannot create repository in %s: can't access parent!" % root)
-
- try:
- config_path = os.path.join(root, repo_config_name)
- packages_path = os.path.join(root, packages_dir_name)
-
- mkdirp(packages_path)
- with open(config_path, 'w') as config:
- config.write("repo:\n")
- config.write(" namespace: '%s'\n" % namespace)
-
- except (IOError, OSError) as e:
- tty.die('Failed to create new repository in %s.' % root,
- "Caused by %s: %s" % (type(e), e))
-
- # try to clean up.
- if existed:
- shutil.rmtree(config_path, ignore_errors=True)
- shutil.rmtree(packages_path, ignore_errors=True)
- else:
- shutil.rmtree(root, ignore_errors=True)
-
+ full_path, namespace = create_repo(args.directory, args.namespace)
tty.msg("Created repo with namespace '%s'." % namespace)
tty.msg("To register it with spack, run this command:",
'spack repo add %s' % full_path)
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 5745adce63..c8542f55f0 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs):
def _depends_on(pkg, spec, when=None):
- if when is None:
+ # If when is False do nothing
+ if when is False:
+ return
+ # If when is None or True make sure the condition is always satisfied
+ if when is None or when is True:
when = pkg.name
when_spec = parse_anonymous_spec(when, pkg.name)
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 3e416a6a1f..29d87b65b3 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -173,7 +173,9 @@ class YamlDirectoryLayout(DirectoryLayout):
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
- self.build_log_name = 'build.out' # TODO: use config file.
+ self.build_log_name = 'build.out' # build log.
+ self.build_env_name = 'build.env' # build environment
+ self.packages_dir = 'repos' # archive of package.py files
# Cache of already written/read extension maps.
self._extension_maps = {}
@@ -231,6 +233,16 @@ class YamlDirectoryLayout(DirectoryLayout):
self.build_log_name)
+ def build_env_path(self, spec):
+ return join_path(self.path_for_spec(spec), self.metadata_dir,
+ self.build_env_name)
+
+
+ def build_packages_path(self, spec):
+ return join_path(self.path_for_spec(spec), self.metadata_dir,
+ self.packages_dir)
+
+
def create_install_directory(self, spec):
_check_concrete(spec)
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index c834763564..c27043db8c 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -33,6 +33,7 @@ number of directories to be appended to paths in the user's environment:
* /bin directories to be appended to PATH
* /lib* directories for LD_LIBRARY_PATH
+ * /include directories for CPATH
* /man* and /share/man* directories for MANPATH
* the package prefix for CMAKE_PREFIX_PATH
@@ -121,6 +122,7 @@ class EnvModule(object):
('LIBRARY_PATH', self.spec.prefix.lib64),
('LD_LIBRARY_PATH', self.spec.prefix.lib),
('LD_LIBRARY_PATH', self.spec.prefix.lib64),
+ ('CPATH', self.spec.prefix.include),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:
diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py
index df9b9b2ab1..51c6a8e89d 100644
--- a/lib/spack/spack/multimethod.py
+++ b/lib/spack/spack/multimethod.py
@@ -193,10 +193,11 @@ class when(object):
platform-specific versions. There's not much we can do to get
around this because of the way decorators work.
"""
-class when(object):
def __init__(self, spec):
pkg = get_calling_module_name()
- self.spec = parse_anonymous_spec(spec, pkg)
+ if spec is True:
+ spec = pkg
+ self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None
def __call__(self, method):
# Get the first definition of the method in the calling scope
@@ -207,7 +208,9 @@ class when(object):
if not type(original_method) == SpecMultiMethod:
original_method = SpecMultiMethod(original_method)
- original_method.register(self.spec, method)
+ if self.spec is not None:
+ original_method.register(self.spec, method)
+
return original_method
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 0214dcd771..106c546d5c 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -58,6 +58,7 @@ import spack.compilers
import spack.mirror
import spack.hooks
import spack.directives
+import spack.repository
import spack.build_environment
import spack.url
import spack.util.web
@@ -66,6 +67,7 @@ from spack.version import *
from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
+from spack.util.environment import dump_environment
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -884,10 +886,14 @@ class Package(object):
# Do the real install in the source directory.
self.stage.chdir_to_source()
+ # Save the build environment in a file before building.
+ env_path = join_path(os.getcwd(), 'spack-build.env')
+
# This redirects I/O to a build log (and optionally to the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w')
with log_output(log_file, verbose, sys.stdout.isatty(), True):
+ dump_environment(env_path)
self.install(self.spec, self.prefix)
# Ensure that something was actually installed.
@@ -896,7 +902,12 @@ class Package(object):
# Move build log into install directory on success
if not fake:
log_install_path = spack.install_layout.build_log_path(self.spec)
+ env_install_path = spack.install_layout.build_env_path(self.spec)
install(log_path, log_install_path)
+ install(env_path, env_install_path)
+
+ packages_dir = spack.install_layout.build_packages_path(self.spec)
+ dump_packages(self.spec, packages_dir)
# On successful install, remove the stage.
if not keep_stage:
@@ -1205,6 +1216,52 @@ def validate_package_url(url_string):
tty.die("Invalid file type in URL: '%s'" % url_string)
+def dump_packages(spec, path):
+ """Dump all package information for a spec and its dependencies.
+
+ This creates a package repository within path for every
+ namespace in the spec DAG, and fills the repos wtih package
+ files and patch files for every node in the DAG.
+ """
+ mkdirp(path)
+
+ # Copy in package.py files from any dependencies.
+ # Note that we copy them in as they are in the *install* directory
+ # NOT as they are in the repository, because we want a snapshot of
+ # how *this* particular build was done.
+ for node in spec.traverse():
+ if node is not spec:
+ # Locate the dependency package in the install tree and find
+ # its provenance information.
+ source = spack.install_layout.build_packages_path(node)
+ source_repo_root = join_path(source, node.namespace)
+
+ # There's no provenance installed for the source package. Skip it.
+ # User can always get something current from the builtin repo.
+ if not os.path.isdir(source_repo_root):
+ continue
+
+ # Create a source repo and get the pkg directory out of it.
+ try:
+ source_repo = spack.repository.Repo(source_repo_root)
+ source_pkg_dir = source_repo.dirname_for_package_name(node.name)
+ except RepoError as e:
+ tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
+
+ # Create a destination repository
+ dest_repo_root = join_path(path, node.namespace)
+ if not os.path.exists(dest_repo_root):
+ spack.repository.create_repo(dest_repo_root)
+ repo = spack.repository.Repo(dest_repo_root)
+
+ # Get the location of the package in the dest repo.
+ dest_pkg_dir = repo.dirname_for_package_name(node.name)
+ if node is not spec:
+ install_tree(source_pkg_dir, dest_pkg_dir)
+ else:
+ spack.repo.dump_provenance(node, dest_pkg_dir)
+
+
def print_pkg(message):
"""Outputs a message with a package icon."""
from llnl.util.tty.color import cwrite
diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py
index f58cd52125..8d06fefe7f 100644
--- a/lib/spack/spack/repository.py
+++ b/lib/spack/spack/repository.py
@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -33,7 +33,7 @@ from bisect import bisect_left
from external import yaml
import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import *
import spack.error
import spack.config
@@ -316,6 +316,16 @@ class RepoPath(object):
return self.repo_for_pkg(spec).get(spec)
+ @_autospec
+ def dump_provenance(self, spec, path):
+ """Dump provenance information for a spec to a particular path.
+
+ This dumps the package file and any associated patch files.
+ Raises UnknownPackageError if not found.
+ """
+ return self.repo_for_pkg(spec).dump_provenance(spec, path)
+
+
def dirname_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
@@ -552,6 +562,35 @@ class Repo(object):
return self._instances[key]
+ @_autospec
+ def dump_provenance(self, spec, path):
+ """Dump provenance information for a spec to a particular path.
+
+ This dumps the package file and any associated patch files.
+ Raises UnknownPackageError if not found.
+ """
+ # Some preliminary checks.
+ if spec.virtual:
+ raise UnknownPackageError(spec.name)
+
+ if spec.namespace and spec.namespace != self.namespace:
+ raise UnknownPackageError("Repository %s does not contain package %s."
+ % (self.namespace, spec.fullname))
+
+ # Install any patch files needed by packages.
+ mkdirp(path)
+ for spec, patches in spec.package.patches.items():
+ for patch in patches:
+ if patch.path:
+ if os.path.exists(patch.path):
+ install(patch.path, path)
+ else:
+ tty.warn("Patch file did not exist: %s" % patch.path)
+
+ # Install the package.py file itself.
+ install(self.filename_for_package_name(spec), path)
+
+
def purge(self):
"""Clear entire package instance cache."""
self._instances.clear()
@@ -705,6 +744,58 @@ class Repo(object):
return self.exists(pkg_name)
+def create_repo(root, namespace=None):
+ """Create a new repository in root with the specified namespace.
+
+ If the namespace is not provided, use basename of root.
+ Return the canonicalized path and the namespace of the created repository.
+ """
+ root = canonicalize_path(root)
+ if not namespace:
+ namespace = os.path.basename(root)
+
+ if not re.match(r'\w[\.\w-]*', namespace):
+ raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
+
+ existed = False
+ if os.path.exists(root):
+ if os.path.isfile(root):
+ raise BadRepoError('File %s already exists and is not a directory' % root)
+ elif os.path.isdir(root):
+ if not os.access(root, os.R_OK | os.W_OK):
+ raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
+ if os.listdir(root):
+ raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
+ existed = True
+
+ full_path = os.path.realpath(root)
+ parent = os.path.dirname(full_path)
+ if not os.access(parent, os.R_OK | os.W_OK):
+ raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
+
+ try:
+ config_path = os.path.join(root, repo_config_name)
+ packages_path = os.path.join(root, packages_dir_name)
+
+ mkdirp(packages_path)
+ with open(config_path, 'w') as config:
+ config.write("repo:\n")
+ config.write(" namespace: '%s'\n" % namespace)
+
+ except (IOError, OSError) as e:
+ raise BadRepoError('Failed to create new repository in %s.' % root,
+ "Caused by %s: %s" % (type(e), e))
+
+ # try to clean up.
+ if existed:
+ shutil.rmtree(config_path, ignore_errors=True)
+ shutil.rmtree(packages_path, ignore_errors=True)
+ else:
+ shutil.rmtree(root, ignore_errors=True)
+
+ return full_path, namespace
+
+
class RepoError(spack.error.SpackError):
"""Superclass for repository-related errors."""
@@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError):
"""Raised when there are no repositories configured."""
+class InvalidNamespaceError(RepoError):
+ """Raised when an invalid namespace is encountered."""
+
+
class BadRepoError(RepoError):
"""Raised when repo layout is invalid."""
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
index 2bf92947fd..ddfaaf4cb0 100644
--- a/lib/spack/spack/resource.py
+++ b/lib/spack/spack/resource.py
@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py
index 83fb34bf76..2023ba6d96 100644
--- a/lib/spack/spack/test/namespace_trie.py
+++ b/lib/spack/spack/test/namespace_trie.py
@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py
index eb1e4a3240..4163ab95dd 100644
--- a/lib/spack/spack/test/tally_plugin.py
+++ b/lib/spack/spack/test/tally_plugin.py
@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
-# For details, see https://scalability-llnl.github.io/spack
+# For details, see https://scalability-software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -34,7 +34,7 @@ class Tally(Plugin):
self.successCount = 0
self.failCount = 0
self.errorCount = 0
-
+
@property
def numberOfTestsRun(self):
"""Excludes skipped tests"""
@@ -48,10 +48,10 @@ class Tally(Plugin):
def addSuccess(self, test):
self.successCount += 1
-
+
def addError(self, test, err):
self.errorCount += 1
-
+
def addFailure(self, test, err):
self.failCount += 1
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index cd413dcfbc..ae8e5708be 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
dictionary.pop(key)
+
+
+def dump_environment(path):
+ """Dump the current environment out to a file."""
+ with open(path, 'w') as env_file:
+ for key,val in sorted(os.environ.items()):
+ env_file.write("%s=%s\n" % (key, val))
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index e26daef296..73f4858b02 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -86,12 +86,12 @@ def _spider(args):
if not "Content-type" in resp.headers:
tty.debug("ignoring page " + url)
- return pages
+ return pages, links
if not resp.headers["Content-type"].startswith('text/html'):
tty.debug("ignoring page " + url + " with content type " +
resp.headers["Content-type"])
- return pages
+ return pages, links
# Do the real GET request when we know it's just HTML.
req.get_method = lambda: "GET"
@@ -173,7 +173,7 @@ def spider(root_url, **kwargs):
performance over a sequential fetch.
"""
max_depth = kwargs.setdefault('depth', 1)
- pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
+ pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
return pages, links