Merge branch 'develop' of https://github.com/LLNL/spack into refactoring/stage

Conflicts:
	lib/spack/spack/package.py

17 files changed, 218 insertions, 72 deletions

diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 0578f0c8db..f94ac3d2ba 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -896,7 +896,7 @@ Or, similarly with modules, you could type:
    $ spack load mpich %gcc@4.4.7
 
 These commands will add appropriate directories to your ``PATH``,
-``MANPATH``, and ``LD_LIBRARY_PATH``.  When you no longer want to use
+``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``.  When you no longer want to use
 a package, you can type unload or unuse similarly:
 
 .. code-block:: sh
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index 67ca18e71a..2c5b68ea65 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -22,7 +22,7 @@ go:
    $ spack install libelf
 
 For a richer experience, use Spack's `shell support
-<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_:
+<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_:
 
 .. code-block:: sh
 
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index aacba996b3..a323c48124 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -130,7 +130,7 @@ if [ -z "$mode" ]; then
     done
 fi
 
-# Dump the version and exist if we're in testing mode.
+# Dump the version and exit if we're in testing mode.
 if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
     echo "$mode"
     exit
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 015eeb9aa1..366237ef8f 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -152,15 +152,20 @@ def set_install_permissions(path):
 def copy_mode(src, dest):
     src_mode = os.stat(src).st_mode
     dest_mode = os.stat(dest).st_mode
-    if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
-    if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
-    if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+    if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
+    if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
+    if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
     os.chmod(dest, dest_mode)
 
 
 def install(src, dest):
     """Manually install a file to a particular location."""
     tty.debug("Installing %s to %s" % (src, dest))
+
+    # Expand dsst to its eventual full path if it is a directory.
+    if os.path.isdir(dest):
+        dest = join_path(dest, os.path.basename(src))
+
     shutil.copy(src, dest)
     set_install_permissions(dest)
     copy_mode(src, dest)
diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py
index b1ad89dbb8..c451993233 100644
--- a/lib/spack/spack/cmd/checksum.py
+++ b/lib/spack/spack/cmd/checksum.py
@@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs):
 
     tty.msg("Downloading...")
     hashes = []
-    for i, (url, version) in enumerate(zip(urls, versions)):
+    i = 0
+    for url, version in zip(urls, versions):
         stage = Stage(url)
         try:
             stage.fetch()
             if i == 0 and first_stage_function:
                 first_stage_function(stage)
 
-            hashes.append(
-                spack.util.crypto.checksum(hashlib.md5, stage.archive_file))
+            hashes.append((version,
+                spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
         except FailedDownloadError, e:
             tty.msg("Failed to fetch %s" % url)
             continue
+	except Exception, e:
+            tty.msg('Something failed on %s, skipping.\n    (%s)' % (url, e))
+            continue
 
         finally:
             if not keep_stage:
                 stage.destroy()
+        i += 1
 
-    return zip(versions, hashes)
+    return hashes
 
 
 
diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py
index 34c755fb67..c2e352786d 100644
--- a/lib/spack/spack/cmd/repo.py
+++ b/lib/spack/spack/cmd/repo.py
@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -74,51 +74,7 @@ def setup_parser(subparser):
 
 def repo_create(args):
     """Create a new package repository."""
-    root = canonicalize_path(args.directory)
-    namespace = args.namespace
-
-    if not args.namespace:
-        namespace = os.path.basename(root)
-
-    if not re.match(r'\w[\.\w-]*', namespace):
-        tty.die("'%s' is not a valid namespace." % namespace)
-
-    existed = False
-    if os.path.exists(root):
-        if os.path.isfile(root):
-            tty.die('File %s already exists and is not a directory' % root)
-        elif os.path.isdir(root):
-            if not os.access(root, os.R_OK | os.W_OK):
-                tty.die('Cannot create new repo in %s: cannot access directory.' % root)
-            if os.listdir(root):
-                tty.die('Cannot create new repo in %s: directory is not empty.' % root)
-        existed = True
-
-    full_path = os.path.realpath(root)
-    parent = os.path.dirname(full_path)
-    if not os.access(parent, os.R_OK | os.W_OK):
-        tty.die("Cannot create repository in %s: can't access parent!" % root)
-
-    try:
-        config_path = os.path.join(root, repo_config_name)
-        packages_path = os.path.join(root, packages_dir_name)
-
-        mkdirp(packages_path)
-        with open(config_path, 'w') as config:
-            config.write("repo:\n")
-            config.write("  namespace: '%s'\n" % namespace)
-
-    except (IOError, OSError) as e:
-        tty.die('Failed to create new repository in %s.' % root,
-                "Caused by %s: %s" % (type(e), e))
-
-        # try to clean up.
-        if existed:
-            shutil.rmtree(config_path, ignore_errors=True)
-            shutil.rmtree(packages_path, ignore_errors=True)
-        else:
-            shutil.rmtree(root, ignore_errors=True)
-
+    full_path, namespace = create_repo(args.directory, args.namespace)
     tty.msg("Created repo with namespace '%s'." % namespace)
     tty.msg("To register it with spack, run this command:",
             'spack repo add %s' % full_path)
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 5745adce63..c8542f55f0 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs):
 
 
 def _depends_on(pkg, spec, when=None):
-    if when is None:
+    # If when is False do nothing
+    if when is False:
+        return
+    # If when is None or True make sure the condition is always satisfied
+    if when is None or when is True:
         when = pkg.name
     when_spec = parse_anonymous_spec(when, pkg.name)
 
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 3e416a6a1f..29d87b65b3 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -173,7 +173,9 @@ class YamlDirectoryLayout(DirectoryLayout):
 
         self.spec_file_name      = 'spec.yaml'
         self.extension_file_name = 'extensions.yaml'
-        self.build_log_name      = 'build.out' # TODO: use config file.
+        self.build_log_name      = 'build.out'  # build log.
+        self.build_env_name      = 'build.env'  # build environment
+        self.packages_dir        = 'repos'      # archive of package.py files
 
         # Cache of already written/read extension maps.
         self._extension_maps = {}
@@ -231,6 +233,16 @@ class YamlDirectoryLayout(DirectoryLayout):
                          self.build_log_name)
 
 
+    def build_env_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.build_env_name)
+
+
+    def build_packages_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.packages_dir)
+
+
     def create_install_directory(self, spec):
         _check_concrete(spec)
 
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index c834763564..c27043db8c 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -33,6 +33,7 @@ number of directories to be appended to paths in the user's environment:
 
   * /bin directories to be appended to PATH
   * /lib* directories for LD_LIBRARY_PATH
+  * /include directories for CPATH
   * /man* and /share/man* directories for MANPATH
   * the package prefix for CMAKE_PREFIX_PATH
 
@@ -121,6 +122,7 @@ class EnvModule(object):
                     ('LIBRARY_PATH', self.spec.prefix.lib64),
                     ('LD_LIBRARY_PATH', self.spec.prefix.lib),
                     ('LD_LIBRARY_PATH', self.spec.prefix.lib64),
+                    ('CPATH', self.spec.prefix.include),
                     ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
                     ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:
 
diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py
index df9b9b2ab1..51c6a8e89d 100644
--- a/lib/spack/spack/multimethod.py
+++ b/lib/spack/spack/multimethod.py
@@ -193,10 +193,11 @@ class when(object):
        platform-specific versions.  There's not much we can do to get
        around this because of the way decorators work.
     """
-class when(object):
     def __init__(self, spec):
         pkg = get_calling_module_name()
-        self.spec = parse_anonymous_spec(spec, pkg)
+        if spec is True:
+            spec = pkg
+        self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None
 
     def __call__(self, method):
         # Get the first definition of the method in the calling scope
@@ -207,7 +208,9 @@ class when(object):
         if not type(original_method) == SpecMultiMethod:
             original_method = SpecMultiMethod(original_method)
 
-        original_method.register(self.spec, method)
+        if self.spec is not None:
+            original_method.register(self.spec, method)
+
         return original_method
 
 
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 0214dcd771..106c546d5c 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -58,6 +58,7 @@ import spack.compilers
 import spack.mirror
 import spack.hooks
 import spack.directives
+import spack.repository
 import spack.build_environment
 import spack.url
 import spack.util.web
@@ -66,6 +67,7 @@ from spack.version import *
 from spack.stage import Stage, ResourceStage, StageComposite
 from spack.util.compression import allowed_archive, extension
 from spack.util.executable import ProcessError
+from spack.util.environment import dump_environment
 
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -884,10 +886,14 @@ class Package(object):
                         # Do the real install in the source directory.
                         self.stage.chdir_to_source()
 
+                        # Save the build environment in a file before building.
+                        env_path = join_path(os.getcwd(), 'spack-build.env')
+
                         # This redirects I/O to a build log (and optionally to the terminal)
                         log_path = join_path(os.getcwd(), 'spack-build.out')
                         log_file = open(log_path, 'w')
                         with log_output(log_file, verbose, sys.stdout.isatty(), True):
+                            dump_environment(env_path)
                             self.install(self.spec, self.prefix)
 
                     # Ensure that something was actually installed.
@@ -896,7 +902,12 @@ class Package(object):
                     # Move build log into install directory on success
                     if not fake:
                         log_install_path = spack.install_layout.build_log_path(self.spec)
+                        env_install_path = spack.install_layout.build_env_path(self.spec)
                         install(log_path, log_install_path)
+                        install(env_path, env_install_path)
+
+                    packages_dir = spack.install_layout.build_packages_path(self.spec)
+                    dump_packages(self.spec, packages_dir)
 
                     # On successful install, remove the stage.
                     if not keep_stage:
@@ -1205,6 +1216,52 @@ def validate_package_url(url_string):
         tty.die("Invalid file type in URL: '%s'" % url_string)
 
 
+def dump_packages(spec, path):
+    """Dump all package information for a spec and its dependencies.
+
+       This creates a package repository within path for every
+       namespace in the spec DAG, and fills the repos wtih package
+       files and patch files for every node in the DAG.
+    """
+    mkdirp(path)
+
+    # Copy in package.py files from any dependencies.
+    # Note that we copy them in as they are in the *install* directory
+    # NOT as they are in the repository, because we want a snapshot of
+    # how *this* particular build was done.
+    for node in spec.traverse():
+        if node is not spec:
+            # Locate the dependency package in the install tree and find
+            # its provenance information.
+            source = spack.install_layout.build_packages_path(node)
+            source_repo_root = join_path(source, node.namespace)
+
+            # There's no provenance installed for the source package.  Skip it.
+            # User can always get something current from the builtin repo.
+            if not os.path.isdir(source_repo_root):
+                continue
+
+            # Create a source repo and get the pkg directory out of it.
+            try:
+                source_repo = spack.repository.Repo(source_repo_root)
+                source_pkg_dir = source_repo.dirname_for_package_name(node.name)
+            except RepoError as e:
+                tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
+
+        # Create a destination repository
+        dest_repo_root = join_path(path, node.namespace)
+        if not os.path.exists(dest_repo_root):
+            spack.repository.create_repo(dest_repo_root)
+        repo = spack.repository.Repo(dest_repo_root)
+
+        # Get the location of the package in the dest repo.
+        dest_pkg_dir = repo.dirname_for_package_name(node.name)
+        if node is not spec:
+            install_tree(source_pkg_dir, dest_pkg_dir)
+        else:
+            spack.repo.dump_provenance(node, dest_pkg_dir)
+
+
 def print_pkg(message):
     """Outputs a message with a package icon."""
     from llnl.util.tty.color import cwrite
diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py
index f58cd52125..8d06fefe7f 100644
--- a/lib/spack/spack/repository.py
+++ b/lib/spack/spack/repository.py
@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -33,7 +33,7 @@ from bisect import bisect_left
 from external import yaml
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import *
 
 import spack.error
 import spack.config
@@ -316,6 +316,16 @@ class RepoPath(object):
         return self.repo_for_pkg(spec).get(spec)
 
 
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        return self.repo_for_pkg(spec).dump_provenance(spec, path)
+
+
     def dirname_for_package_name(self, pkg_name):
         return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
 
@@ -552,6 +562,35 @@ class Repo(object):
         return self._instances[key]
 
 
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        # Some preliminary checks.
+        if spec.virtual:
+            raise UnknownPackageError(spec.name)
+
+        if spec.namespace and spec.namespace != self.namespace:
+            raise UnknownPackageError("Repository %s does not contain package %s."
+                                      % (self.namespace, spec.fullname))
+
+        # Install any patch files needed by packages.
+        mkdirp(path)
+        for spec, patches in spec.package.patches.items():
+            for patch in patches:
+                if patch.path:
+                    if os.path.exists(patch.path):
+                        install(patch.path, path)
+                    else:
+                        tty.warn("Patch file did not exist: %s" % patch.path)
+
+        # Install the package.py file itself.
+        install(self.filename_for_package_name(spec), path)
+
+
     def purge(self):
         """Clear entire package instance cache."""
         self._instances.clear()
@@ -705,6 +744,58 @@ class Repo(object):
         return self.exists(pkg_name)
 
 
+def create_repo(root, namespace=None):
+    """Create a new repository in root with the specified namespace.
+
+       If the namespace is not provided, use basename of root.
+       Return the canonicalized path and the namespace of the created repository.
+    """
+    root = canonicalize_path(root)
+    if not namespace:
+        namespace = os.path.basename(root)
+
+    if not re.match(r'\w[\.\w-]*', namespace):
+        raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
+
+    existed = False
+    if os.path.exists(root):
+        if os.path.isfile(root):
+            raise BadRepoError('File %s already exists and is not a directory' % root)
+        elif os.path.isdir(root):
+            if not os.access(root, os.R_OK | os.W_OK):
+                raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
+            if os.listdir(root):
+                raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
+        existed = True
+
+    full_path = os.path.realpath(root)
+    parent = os.path.dirname(full_path)
+    if not os.access(parent, os.R_OK | os.W_OK):
+        raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
+
+    try:
+        config_path = os.path.join(root, repo_config_name)
+        packages_path = os.path.join(root, packages_dir_name)
+
+        mkdirp(packages_path)
+        with open(config_path, 'w') as config:
+            config.write("repo:\n")
+            config.write("  namespace: '%s'\n" % namespace)
+
+    except (IOError, OSError) as e:
+        raise BadRepoError('Failed to create new repository in %s.' % root,
+                           "Caused by %s: %s" % (type(e), e))
+
+        # try to clean up.
+        if existed:
+            shutil.rmtree(config_path, ignore_errors=True)
+            shutil.rmtree(packages_path, ignore_errors=True)
+        else:
+            shutil.rmtree(root, ignore_errors=True)
+
+    return full_path, namespace
+
+
 class RepoError(spack.error.SpackError):
     """Superclass for repository-related errors."""
 
@@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError):
     """Raised when there are no repositories configured."""
 
 
+class InvalidNamespaceError(RepoError):
+    """Raised when an invalid namespace is encountered."""
+
+
 class BadRepoError(RepoError):
     """Raised when repo layout is invalid."""
 
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
index 2bf92947fd..ddfaaf4cb0 100644
--- a/lib/spack/spack/resource.py
+++ b/lib/spack/spack/resource.py
@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py
index 83fb34bf76..2023ba6d96 100644
--- a/lib/spack/spack/test/namespace_trie.py
+++ b/lib/spack/spack/test/namespace_trie.py
@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py
index eb1e4a3240..4163ab95dd 100644
--- a/lib/spack/spack/test/tally_plugin.py
+++ b/lib/spack/spack/test/tally_plugin.py
@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://scalability-llnl.github.io/spack
+# For details, see https://scalability-software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -34,7 +34,7 @@ class Tally(Plugin):
         self.successCount = 0
         self.failCount = 0
         self.errorCount = 0
-    
+
     @property
     def numberOfTestsRun(self):
         """Excludes skipped tests"""
@@ -48,10 +48,10 @@ class Tally(Plugin):
 
     def addSuccess(self, test):
         self.successCount += 1
-        
+
     def addError(self, test, err):
         self.errorCount += 1
-            
+
     def addFailure(self, test, err):
         self.failCount += 1
 
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index cd413dcfbc..ae8e5708be 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
     for key in keys:
         if key in dictionary:
             dictionary.pop(key)
+
+
+def dump_environment(path):
+    """Dump the current environment out to a file."""
+    with open(path, 'w') as env_file:
+        for key,val in sorted(os.environ.items()):
+            env_file.write("%s=%s\n" % (key, val))
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index e26daef296..73f4858b02 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -86,12 +86,12 @@ def _spider(args):
 
         if not "Content-type" in resp.headers:
             tty.debug("ignoring page " + url)
-            return pages
+            return pages, links
 
         if not resp.headers["Content-type"].startswith('text/html'):
             tty.debug("ignoring page " + url + " with content type " +
                       resp.headers["Content-type"])
-            return pages
+            return pages, links
 
         # Do the real GET request when we know it's just HTML.
         req.get_method = lambda: "GET"
@@ -173,7 +173,7 @@ def spider(root_url, **kwargs):
        performance over a sequential fetch.
     """
     max_depth = kwargs.setdefault('depth', 1)
-    pages, links =  _spider((root_url, set(), root_url, None, 1, max_depth, False))
+    pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
     return pages, links