Merge pull request #208 from epfl-scitas/features/resource_directive

resource directive : implementation + clang / llvm use case

6 files changed, 220 insertions, 36 deletions

diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index cc87b7eac2..c84828951d 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
 
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
-       This allows a fucntion to insert variables into its caller's
+       This allows a function to insert variables into its caller's
        scope.  Yes, this is some black magic, and yes it's useful
        for implementing things like depends_on and provides.
     """
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 2a818e8d0c..aa9fbd8d33 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -42,15 +42,19 @@ The available directives are:
   * ``extends``
   * ``patch``
   * ``variant``
+  * ``resource``
 
 """
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
-            'variant' ]
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
+           'variant', 'resource']
 
 import re
 import inspect
+import os.path
+import functools
 
 from llnl.util.lang import *
+from llnl.util.filesystem import join_path
 
 import spack
 import spack.spec
@@ -60,7 +64,8 @@ from spack.version import Version
 from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import from_kwargs
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -79,8 +84,8 @@ class directive(object):
     """Decorator for Spack directives.
 
     Spack directives allow you to modify a package while it is being
-    defined, e.g. to add version or depenency information.  Directives
-    are one of the key pieces of Spack's package "langauge", which is
+    defined, e.g. to add version or dependency information.  Directives
+    are one of the key pieces of Spack's package "language", which is
     embedded in python.
 
     Here's an example directive:
@@ -141,6 +146,7 @@ class directive(object):
     def __call__(self, directive_function):
         directives[directive_function.__name__] = self
 
+        @functools.wraps(directive_function)
         def wrapped(*args, **kwargs):
             pkg = DictWrapper(caller_locals())
             self.ensure_dicts(pkg)
@@ -259,6 +265,43 @@ def variant(pkg, name, default=False, description=""):
     pkg.variants[name] = Variant(default, description)
 
 
+@directive('resources')
+def resource(pkg, **kwargs):
+    """
+    Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+    dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
+    own folder inside spack stage area, and then linked into the stage area of the package that needs them.
+
+    List of recognized keywords:
+
+    * 'when' : (optional) represents the condition upon which the resource is needed
+    * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
+    area.
+    * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
+    area.
+    """
+    when = kwargs.get('when', pkg.name)
+    destination = kwargs.get('destination', "")
+    placement = kwargs.get('placement', None)
+    # Check if the path is relative
+    if os.path.isabs(destination):
+        message = "The destination keyword of a resource directive can't be an absolute path.\n"
+        message += "\tdestination : '{dest}\n'".format(dest=destination)
+        raise RuntimeError(message)
+    # Check if the path falls within the main package stage area
+    test_path = 'stage_folder_root/'
+    normalized_destination = os.path.normpath(join_path(test_path, destination))  # Normalized absolute path
+    if test_path not in normalized_destination:
+        message = "The destination folder of a resource must fall within the main package stage directory.\n"
+        message += "\tdestination : '{dest}'\n".format(dest=destination)
+        raise RuntimeError(message)
+    when_spec = parse_anonymous_spec(when, pkg.name)
+    resources = pkg.resources.setdefault(when_spec, [])
+    fetcher = from_kwargs(**kwargs)
+    name = kwargs.get('name')
+    resources.append(Resource(name, fetcher, destination, placement))
+
+
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
     def __init__(self, directive, message):
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 8c4bd4ed8a..a9374fb34b 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -634,6 +634,22 @@ def from_url(url):
     return URLFetchStrategy(url)
 
 
+def from_kwargs(**kwargs):
+    """
+    Construct the appropriate FetchStrategy from the given keyword arguments.
+
+    :param kwargs: dictionary of keyword arguments
+    :return: fetcher or raise a FetchError exception
+    """
+    for fetcher in all_strategies:
+        if fetcher.matches(kwargs):
+            return fetcher(**kwargs)
+    # Raise an error in case we can't instantiate any known strategy
+    message = "Cannot instantiate any FetchStrategy"
+    long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+    raise FetchError(message, long_message)
+
+
 def args_are_for(args, fetcher):
     fetcher.matches(args)
 
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 6fbf82de14..1d9b0e7ef2 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -26,7 +26,7 @@
 This file contains code for creating spack mirror directories.  A
 mirror is an organized hierarchy containing specially named archive
 files.  This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down.  Or, if the computer
+the main server for a particular package is down.  Or, if the computer
 where spack is run is not connected to the internet, it allows spack
 to download packages directly from a mirror (e.g., on an intranet).
 """
@@ -42,7 +42,7 @@ import spack.fetch_strategy as fs
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
 
 
 def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
+                s.variants = spec.variants.copy()
                 matching.append(s)
 
     return matching
 
 
+def suggest_archive_basename(resource):
+    """
+    Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+    :param fetcher:
+    :return:
+    """
+    basename = os.path.basename(resource.fetcher.url)
+    if not allowed_archive(basename):
+        raise RuntimeError("%s is not an allowed archive tye" % basename)
+    return basename
+
+
+
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
 
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
-         * present:  Package specs that were already prsent.
+         * present:  Package specs that were already present.
          * mirrored: Package specs that were successfully mirrored.
          * error:    Package specs that failed to mirror due to some error.
 
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
     error    = []
 
     # Iterate through packages and download all the safe tarballs for each of them
+    everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
 
@@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
 
             if os.path.exists(archive_path):
                 tty.msg("Already added %s" % spec.format("$_$@"))
+            else:
+                everything_already_exists = False
+                # Set up a stage and a fetcher for the download
+                unique_fetch_name = spec.format("$_$@")
+                fetcher = fs.for_package_version(pkg, pkg.version)
+                stage = Stage(fetcher, name=unique_fetch_name)
+                fetcher.set_stage(stage)
+
+                # Do the fetch and checksum if necessary
+                fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    fetcher.check()
+                    tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+                # Fetchers have to know how to archive their files.  Use
+                # that to move/copy/create an archive in the mirror.
+                fetcher.archive(archive_path)
+                tty.msg("Added %s." % spec.format("$_$@"))
+
+            # Fetch resources if they are associated with the spec
+            resources = pkg._get_resources()
+            for resource in resources:
+                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+                if os.path.exists(resource_archive_path):
+                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                    continue
+                everything_already_exists = False
+                resource_stage_folder = pkg._resource_stage(resource)
+                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+                resource.fetcher.set_stage(resource_stage)
+                resource.fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    resource.fetcher.check()
+                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+                resource.fetcher.archive(resource_archive_path)
+                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+            if everything_already_exists:
                 present.append(spec)
-                continue
-
-            # Set up a stage and a fetcher for the download
-            unique_fetch_name = spec.format("$_$@")
-            fetcher = fs.for_package_version(pkg, pkg.version)
-            stage = Stage(fetcher, name=unique_fetch_name)
-            fetcher.set_stage(stage)
-
-            # Do the fetch and checksum if necessary
-            fetcher.fetch()
-            if not kwargs.get('no_checksum', False):
-                fetcher.check()
-                tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
-            # Fetchers have to know how to archive their files.  Use
-            # that to move/copy/create an archive in the mirror.
-            fetcher.archive(archive_path)
-            tty.msg("Added %s." % spec.format("$_$@"))
-            mirrored.append(spec)
+            else:
+                mirrored.append(spec)
 
         except Exception, e:
             if spack.debug:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 4d75726e06..b95afb073d 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -655,26 +655,62 @@ class Package(object):
                     "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch()
+
+        ##########
+        # Fetch resources
+        resources = self._get_resources()
+        for resource in resources:
+            resource_stage_folder = self._resource_stage(resource)
+            # FIXME : works only for URLFetchStrategy
+            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+            resource.fetcher.set_stage(resource_stage)
+            # Delegate to stage object to trigger mirror logic
+            resource_stage.fetch()
+            resource_stage.check()
+        ##########
+
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
             self.stage.check()
 
-
     def do_stage(self):
         """Unpacks the fetched tarball, then changes into the expanded tarball
            directory."""
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        self.do_fetch()
+        def _expand_archive(stage, name=self.name):
+            archive_dir = stage.source_path
+            if not archive_dir:
+                stage.expand_archive()
+                tty.msg("Created stage in %s." % stage.path)
+            else:
+                tty.msg("Already staged %s in %s." % (name, stage.path))
 
-        archive_dir = self.stage.source_path
-        if not archive_dir:
-            self.stage.expand_archive()
-            tty.msg("Created stage in %s." % self.stage.path)
-        else:
-            tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+
+        self.do_fetch()
+        _expand_archive(self.stage)
+
+        ##########
+        # Stage resources in appropriate path
+        resources = self._get_resources()
+        for resource in resources:
+            stage = resource.fetcher.stage
+            _expand_archive(stage, resource.name)
+            # Turn placement into a dict with relative paths
+            placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
+            if not isinstance(placement, dict):
+                placement = {'': placement}
+            # Make the paths in the dictionary absolute and link
+            for key, value in placement.iteritems():
+                link_path = join_path(self.stage.source_path, resource.destination, value)
+                source_path = join_path(stage.source_path, key)
+                if not os.path.exists(link_path):
+                    # Create a symlink
+                    os.symlink(source_path, link_path)
+        ##########
         self.stage.chdir_to_source()
 
 
@@ -746,6 +782,19 @@ class Package(object):
         mkdirp(self.prefix.man1)
 
 
+    def _get_resources(self):
+        resources = []
+        # Select the resources that are needed for this build
+        for when_spec, resource_list in self.resources.items():
+            if when_spec in self.spec:
+                resources.extend(resource_list)
+        return resources
+
+    def _resource_stage(self, resource):
+        pieces = ['resource', resource.name, self.spec.dag_hash()]
+        resource_stage_folder = '-'.join(pieces)
+        return resource_stage_folder
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
new file mode 100644
index 0000000000..8d081b45c9
--- /dev/null
+++ b/lib/spack/spack/resource.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+
+class Resource(object):
+    """
+    Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement
+    """
+    def __init__(self, name, fetcher, destination, placement):
+        self.name = name
+        self.fetcher = fetcher
+        self.destination = destination
+        self.placement = placement