From 1fe626ec7cd8b8a2bceb9e73dd9597d9f99813cf Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Thu, 26 Nov 2015 17:53:33 +0100
Subject: resource directive : sketch of implementation + clang / llvm use case

---
 lib/spack/llnl/util/lang.py   |  2 +-
 lib/spack/spack/directives.py | 36 +++++++++++++++++++++++++-----
 lib/spack/spack/package.py    | 51 ++++++++++++++++++++++++++++++++++++-------
 lib/spack/spack/resource.py   | 37 +++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+), 14 deletions(-)
 create mode 100644 lib/spack/spack/resource.py

(limited to 'lib')

diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 156ee34c9e..e726368794 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
 
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
-       This allows a fucntion to insert variables into its caller's
+       This allows a function to insert variables into its caller's
        scope.  Yes, this is some black magic, and yes it's useful
        for implementing things like depends_on and provides.
     """
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 78039ac6f9..4d8333fd7d 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -44,11 +44,12 @@ The available directives are:
   * ``variant``
 
 """
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
-            'variant' ]
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
+           'variant', 'resource']
 
 import re
 import inspect
+import functools
 
 from llnl.util.lang import *
 
@@ -60,7 +61,8 @@ from spack.version import Version
 from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import URLFetchStrategy
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -79,8 +81,8 @@ class directive(object):
     """Decorator for Spack directives.
 
     Spack directives allow you to modify a package while it is being
-    defined, e.g. to add version or depenency information.  Directives
-    are one of the key pieces of Spack's package "langauge", which is
+    defined, e.g. to add version or dependency information.  Directives
+    are one of the key pieces of Spack's package "language", which is
     embedded in python.
 
     Here's an example directive:
@@ -141,6 +143,7 @@ class directive(object):
     def __call__(self, directive_function):
         directives[directive_function.__name__] = self
 
+        @functools.wraps(directive_function)
         def wrapped(*args, **kwargs):
             pkg = DictWrapper(caller_locals())
             self.ensure_dicts(pkg)
@@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""):
     pkg.variants[name] = Variant(default, description)
 
 
+@directive('resources')
+def resource(pkg, **kwargs):
+    """
+    Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+    dictionary the appropriate FetchStrategy will be used for the resource.
+
+    List of recognized keywords:
+
+    * 'when' : represents the condition upon which the resource is needed (optional)
+    * 'destination' : path where to extract / checkout the resource (optional)
+
+    """
+    when = kwargs.get('when', pkg.name)
+    # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path)
+    destination = kwargs.get('destination', "")
+    when_spec = parse_anonymous_spec(when, pkg.name)
+    resources = pkg.resources.setdefault(when_spec, [])
+    # FIXME : change URLFetchStrategy with a factory that selects based on kwargs
+    fetcher = URLFetchStrategy(**kwargs)
+    # FIXME : should we infer the name somehow if not passed ?
+    name = kwargs.get('name')
+    resources.append(Resource(name, fetcher, destination))
+
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
     def __init__(self, directive, message):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index c631a35bf3..9fee962df3 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -639,26 +639,53 @@ class Package(object):
                     "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch()
+
+        ##########
+        # Fetch resources
+        resources = self._get_resources()
+        # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ?
+        pieces = [self.name, str(self.version), self.spec.dag_hash()]
+        for resource in resources:
+            resource_stage_folder = '-'.join(pieces + [resource.name])
+            stage = Stage(resource.fetcher, name=resource_stage_folder)
+            resource.fetcher.set_stage(stage)
+            resource.fetcher.fetch()
+        ##########
+
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
             self.stage.check()
 
-
     def do_stage(self):
         """Unpacks the fetched tarball, then changes into the expanded tarball
            directory."""
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        self.do_fetch()
+        def _expand_archive(stage, name=self.name):
+            archive_dir = stage.source_path
+            if not archive_dir:
+                stage.expand_archive()
+                tty.msg("Created stage in %s." % stage.path)
+            else:
+                tty.msg("Already staged %s in %s." % (name, stage.path))
 
-        archive_dir = self.stage.source_path
-        if not archive_dir:
-            self.stage.expand_archive()
-            tty.msg("Created stage in %s." % self.stage.path)
-        else:
-            tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+
+        self.do_fetch()
+        _expand_archive(self.stage)
+
+        ##########
+        # Stage resources in appropriate path
+        resources = self._get_resources()
+        for resource in resources:
+            stage = resource.fetcher.stage
+            _expand_archive(stage, resource.name)
+            link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path))
+            if not os.path.exists(link_path):
+                # Create a symlink
+                os.symlink(stage.source_path, link_path)
+        ##########
         self.stage.chdir_to_source()
 
 
@@ -730,6 +757,14 @@ class Package(object):
         mkdirp(self.prefix.man1)
 
 
+    def _get_resources(self):
+        resources = []
+        # Select the resources that are needed for this build
+        for when_spec, resource_list in self.resources.items():
+            if when_spec in self.spec:
+                resources.extend(resource_list)
+        return resources
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
new file mode 100644
index 0000000000..00ee2d49e4
--- /dev/null
+++ b/lib/spack/spack/resource.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+class Resource(object):
+    """
+    Represents an optional resource. Aggregates a name, a fetcher and a destination.
+    """
+    def __init__(self, name, fetcher, destination):
+        self.name = name
+        self.fetcher = fetcher
+        self.destination = destination
-- 
cgit v1.2.3-60-g2f50


From a075d581eff5e07cd4b703cbcd1d1e5f490b1fb4 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Tue, 1 Dec 2015 12:56:46 +0100
Subject: resource : fetch strategy constructed from kwargs instead or
 hardcoded URLFetchStrategy

---
 lib/spack/spack/directives.py     |  5 ++---
 lib/spack/spack/fetch_strategy.py | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 4d8333fd7d..741df3a31b 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -62,7 +62,7 @@ from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
 from spack.resource import Resource
-from spack.fetch_strategy import URLFetchStrategy
+from spack.fetch_strategy import from_kwargs
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -279,8 +279,7 @@ def resource(pkg, **kwargs):
     destination = kwargs.get('destination', "")
     when_spec = parse_anonymous_spec(when, pkg.name)
     resources = pkg.resources.setdefault(when_spec, [])
-    # FIXME : change URLFetchStrategy with a factory that selects based on kwargs
-    fetcher = URLFetchStrategy(**kwargs)
+    fetcher = from_kwargs(**kwargs)
     # FIXME : should we infer the name somehow if not passed ?
     name = kwargs.get('name')
     resources.append(Resource(name, fetcher, destination))
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 5e6850d14b..1f5ef16caa 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -634,6 +634,22 @@ def from_url(url):
     return URLFetchStrategy(url)
 
 
+def from_kwargs(**kwargs):
+    """
+    Construct the appropriate FetchStrategy from the given keyword arguments.
+
+    :param kwargs: dictionary of keyword arguments
+    :return: fetcher or raise a FetchError exception
+    """
+    for fetcher in all_strategies:
+        if fetcher.matches(kwargs):
+            return fetcher(**kwargs)
+    # Raise an error in case we can't instantiate any known strategy
+    message = "Cannot instantiate any FetchStrategy"
+    long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+    raise FetchError(message, long_message)
+
+
 def args_are_for(args, fetcher):
     fetcher.matches(args)
 
-- 
cgit v1.2.3-60-g2f50


From d3d9b5401b4d265f4de927cd34828a8c8b99e923 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Tue, 1 Dec 2015 13:26:44 +0100
Subject: resources : changed name of stage folder

---
 lib/spack/spack/package.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 9fee962df3..c1dbbafc79 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -643,10 +643,9 @@ class Package(object):
         ##########
         # Fetch resources
         resources = self._get_resources()
-        # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ?
-        pieces = [self.name, str(self.version), self.spec.dag_hash()]
         for resource in resources:
-            resource_stage_folder = '-'.join(pieces + [resource.name])
+            pieces = ['resource', resource.name, self.spec.dag_hash()]
+            resource_stage_folder = '-'.join(pieces)
             stage = Stage(resource.fetcher, name=resource_stage_folder)
             resource.fetcher.set_stage(stage)
             resource.fetcher.fetch()
-- 
cgit v1.2.3-60-g2f50


From b85dccca927bdd2e9c5d30841544ef4328ce9d56 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Tue, 1 Dec 2015 15:12:26 +0100
Subject: resources : added error handling for destination keyword on resource
 directive

---
 lib/spack/spack/directives.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 741df3a31b..48a7df7462 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -42,6 +42,7 @@ The available directives are:
   * ``extends``
   * ``patch``
   * ``variant``
+  * ``resource``
 
 """
 __all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
@@ -49,9 +50,11 @@ __all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
 
 import re
 import inspect
+import os.path
 import functools
 
 from llnl.util.lang import *
+from llnl.util.filesystem import join_path
 
 import spack
 import spack.spec
@@ -271,16 +274,27 @@ def resource(pkg, **kwargs):
     List of recognized keywords:
 
     * 'when' : represents the condition upon which the resource is needed (optional)
-    * 'destination' : path where to extract / checkout the resource (optional)
+    * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path,
+    and it must fall inside the stage area of the main package.
 
     """
     when = kwargs.get('when', pkg.name)
-    # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path)
     destination = kwargs.get('destination', "")
+    # Check if the path is relative
+    if os.path.isabs(destination):
+        message = "The destination keyword of a resource directive can't be an absolute path.\n"
+        message += "\tdestination : '{dest}\n'".format(dest=destination)
+        raise RuntimeError(message)
+    # Check if the path falls within the main package stage area
+    test_path = 'stage_folder_root/'
+    normalized_destination = os.path.normpath(join_path(test_path, destination))  # Normalized absolute path
+    if test_path not in normalized_destination:
+        message = "The destination folder of a resource must fall within the main package stage directory.\n"
+        message += "\tdestination : '{dest}'\n".format(dest=destination)
+        raise RuntimeError(message)
     when_spec = parse_anonymous_spec(when, pkg.name)
     resources = pkg.resources.setdefault(when_spec, [])
     fetcher = from_kwargs(**kwargs)
-    # FIXME : should we infer the name somehow if not passed ?
     name = kwargs.get('name')
     resources.append(Resource(name, fetcher, destination))
 
-- 
cgit v1.2.3-60-g2f50


From 4b2168ab8ef5b5e69545a9bd82ef8804a4108d75 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Wed, 2 Dec 2015 10:13:39 +0100
Subject: resources : added missing check after fetch

---
 lib/spack/spack/package.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index c1dbbafc79..c10283a6ee 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -649,6 +649,7 @@ class Package(object):
             stage = Stage(resource.fetcher, name=resource_stage_folder)
             resource.fetcher.set_stage(stage)
             resource.fetcher.fetch()
+            resource.fetcher.check()
         ##########
 
         self._fetch_time = time.time() - start_time
-- 
cgit v1.2.3-60-g2f50


From 39a3cfd4d9fc8d0981ba118de9a1e049fba4b144 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Wed, 2 Dec 2015 12:24:37 +0100
Subject: reource directive accepts 'basename' keyword llvm : libc++ variant

---
 lib/spack/spack/directives.py      | 5 ++++-
 lib/spack/spack/package.py         | 3 ++-
 lib/spack/spack/resource.py        | 4 +++-
 var/spack/packages/llvm/package.py | 9 ++++++++-
 4 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 48a7df7462..07ef32294b 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -276,10 +276,12 @@ def resource(pkg, **kwargs):
     * 'when' : represents the condition upon which the resource is needed (optional)
     * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path,
     and it must fall inside the stage area of the main package.
+    * 'basename' : basename of the resource source folder within destination (optional).
 
     """
     when = kwargs.get('when', pkg.name)
     destination = kwargs.get('destination', "")
+    basename = kwargs.get('basename', None)
     # Check if the path is relative
     if os.path.isabs(destination):
         message = "The destination keyword of a resource directive can't be an absolute path.\n"
@@ -296,7 +298,8 @@ def resource(pkg, **kwargs):
     resources = pkg.resources.setdefault(when_spec, [])
     fetcher = from_kwargs(**kwargs)
     name = kwargs.get('name')
-    resources.append(Resource(name, fetcher, destination))
+    resources.append(Resource(name, fetcher, destination, basename))
+
 
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index c10283a6ee..3f48231c75 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -681,7 +681,8 @@ class Package(object):
         for resource in resources:
             stage = resource.fetcher.stage
             _expand_archive(stage, resource.name)
-            link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path))
+            basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename
+            link_path = join_path(self.stage.source_path, resource.destination, basename)
             if not os.path.exists(link_path):
                 # Create a symlink
                 os.symlink(stage.source_path, link_path)
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
index 00ee2d49e4..b20612686d 100644
--- a/lib/spack/spack/resource.py
+++ b/lib/spack/spack/resource.py
@@ -27,11 +27,13 @@ Describes an optional resource needed for a build. Typically a bunch of sources
 package to enable optional features.
 """
 
+
 class Resource(object):
     """
     Represents an optional resource. Aggregates a name, a fetcher and a destination.
     """
-    def __init__(self, name, fetcher, destination):
+    def __init__(self, name, fetcher, destination, basename):
         self.name = name
         self.fetcher = fetcher
         self.destination = destination
+        self.basename = basename
diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py
index b68aa82aff..872a6c082b 100644
--- a/var/spack/packages/llvm/package.py
+++ b/var/spack/packages/llvm/package.py
@@ -41,15 +41,22 @@ class Llvm(Package):
 
     depends_on('python@2.7:')
 
+    variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11")
+
     ##########
     # @3.7.0
-    # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant?
     resource(name='compiler-rt',
              url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
              destination='projects', when='@3.7.0')
     resource(name='openmp',
              url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
              destination='projects', when='@3.7.0')
+    resource(name='libcxx',
+             url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
+             destination='projects', basename='libcxx', when='+libcxx@3.7.0')
+    resource(name='libcxxabi',
+             url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
+             destination='projects', basename='libcxxabi', when='+libcxx@3.7.0')
     ##########
 
     def install(self, spec, prefix):
-- 
cgit v1.2.3-60-g2f50


From 50bd4d2e4ed3fed83afa30e7796eaa99989bc3ce Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@googlemail.com>
Date: Wed, 2 Dec 2015 16:18:25 +0100
Subject: mirror : deals correctly with variants that optionally enable
 resources (if they are archive URLs)

---
 lib/spack/spack/mirror.py  | 81 +++++++++++++++++++++++++++++++++-------------
 lib/spack/spack/package.py | 19 +++++++----
 2 files changed, 72 insertions(+), 28 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 306c8085aa..1c38d79164 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -26,7 +26,7 @@
 This file contains code for creating spack mirror directories.  A
 mirror is an organized hierarchy containing specially named archive
 files.  This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down.  Or, if the computer
+the main server for a particular package is down.  Or, if the computer
 where spack is run is not connected to the internet, it allows spack
 to download packages directly from a mirror (e.g., on an intranet).
 """
@@ -42,7 +42,7 @@ import spack.fetch_strategy as fs
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
 
 
 def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
+                s.variants = spec.variants.copy()
                 matching.append(s)
 
     return matching
 
 
+def suggest_archive_basename(resource):
+    """
+    Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+    :param fetcher:
+    :return:
+    """
+    basename = os.path.basename(resource.fetcher.url)
+    if not allowed_archive(basename):
+        raise RuntimeError("%s is not an allowed archive tye" % basename)
+    return basename
+
+
+
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
 
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
-         * present:  Package specs that were already prsent.
+         * present:  Package specs that were already present.
          * mirrored: Package specs that were successfully mirrored.
          * error:    Package specs that failed to mirror due to some error.
 
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
     error    = []
 
     # Iterate through packages and download all the safe tarballs for each of them
+    everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
 
@@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
 
             if os.path.exists(archive_path):
                 tty.msg("Already added %s" % spec.format("$_$@"))
+            else:
+                everything_already_exists = False
+                # Set up a stage and a fetcher for the download
+                unique_fetch_name = spec.format("$_$@")
+                fetcher = fs.for_package_version(pkg, pkg.version)
+                stage = Stage(fetcher, name=unique_fetch_name)
+                fetcher.set_stage(stage)
+
+                # Do the fetch and checksum if necessary
+                fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    fetcher.check()
+                    tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+                # Fetchers have to know how to archive their files.  Use
+                # that to move/copy/create an archive in the mirror.
+                fetcher.archive(archive_path)
+                tty.msg("Added %s." % spec.format("$_$@"))
+
+            # Fetch resources if they are associated with the spec
+            resources = pkg._get_resources()
+            for resource in resources:
+                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+                if os.path.exists(resource_archive_path):
+                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                    continue
+                everything_already_exists = False
+                resource_stage_folder = pkg._resource_stage(resource)
+                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+                resource.fetcher.set_stage(resource_stage)
+                resource.fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    resource.fetcher.check()
+                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+                resource.fetcher.archive(resource_archive_path)
+                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+            if everything_already_exists:
                 present.append(spec)
-                continue
-
-            # Set up a stage and a fetcher for the download
-            unique_fetch_name = spec.format("$_$@")
-            fetcher = fs.for_package_version(pkg, pkg.version)
-            stage = Stage(fetcher, name=unique_fetch_name)
-            fetcher.set_stage(stage)
-
-            # Do the fetch and checksum if necessary
-            fetcher.fetch()
-            if not kwargs.get('no_checksum', False):
-                fetcher.check()
-                tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
-            # Fetchers have to know how to archive their files.  Use
-            # that to move/copy/create an archive in the mirror.
-            fetcher.archive(archive_path)
-            tty.msg("Added %s." % spec.format("$_$@"))
-            mirrored.append(spec)
+            else:
+                mirrored.append(spec)
 
         except Exception, e:
             if spack.debug:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 3f48231c75..dcb514af00 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -644,12 +644,14 @@ class Package(object):
         # Fetch resources
         resources = self._get_resources()
         for resource in resources:
-            pieces = ['resource', resource.name, self.spec.dag_hash()]
-            resource_stage_folder = '-'.join(pieces)
-            stage = Stage(resource.fetcher, name=resource_stage_folder)
-            resource.fetcher.set_stage(stage)
-            resource.fetcher.fetch()
-            resource.fetcher.check()
+            resource_stage_folder = self._resource_stage(resource)
+            # FIXME : works only for URLFetchStrategy
+            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+            resource.fetcher.set_stage(resource_stage)
+            # Delegate to stage object to trigger mirror logic
+            resource_stage.fetch()
+            resource_stage.check()
         ##########
 
         self._fetch_time = time.time() - start_time
@@ -766,6 +768,11 @@ class Package(object):
                 resources.extend(resource_list)
         return resources
 
+    def _resource_stage(self, resource):
+        pieces = ['resource', resource.name, self.spec.dag_hash()]
+        resource_stage_folder = '-'.join(pieces)
+        return resource_stage_folder
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 
-- 
cgit v1.2.3-60-g2f50


From 20e67bc5e6c4a4ac759bb068ff78c13bfc17fb0f Mon Sep 17 00:00:00 2001
From: alalazo <massimiliano.culpo@googlemail.com>
Date: Wed, 9 Dec 2015 13:06:39 +0100
Subject: clang : solve the issue with missing default include paths for OpenMP
 and libc++ resource : support for finer grained linking of resources

---
 lib/spack/spack/directives.py       | 17 +++++++++--------
 lib/spack/spack/package.py          | 16 +++++++++++-----
 lib/spack/spack/resource.py         |  6 +++---
 var/spack/packages/clang/package.py | 31 +++++++++++++++++++++++++++++++
 var/spack/packages/llvm/package.py  |  4 ++--
 5 files changed, 56 insertions(+), 18 deletions(-)

(limited to 'lib')

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 07ef32294b..22b262b57c 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -269,19 +269,20 @@ def variant(pkg, name, default=False, description=""):
 def resource(pkg, **kwargs):
     """
     Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
-    dictionary the appropriate FetchStrategy will be used for the resource.
+    dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
+    own folder inside spack stage area, and then linked into the stage area of the package that needs them.
 
     List of recognized keywords:
 
-    * 'when' : represents the condition upon which the resource is needed (optional)
-    * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path,
-    and it must fall inside the stage area of the main package.
-    * 'basename' : basename of the resource source folder within destination (optional).
-
+    * 'when' : (optional) represents the condition upon which the resource is needed
+    * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
+    area.
+    * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
+    area.
     """
     when = kwargs.get('when', pkg.name)
     destination = kwargs.get('destination', "")
-    basename = kwargs.get('basename', None)
+    placement = kwargs.get('placement', None)
     # Check if the path is relative
     if os.path.isabs(destination):
         message = "The destination keyword of a resource directive can't be an absolute path.\n"
@@ -298,7 +299,7 @@ def resource(pkg, **kwargs):
     resources = pkg.resources.setdefault(when_spec, [])
     fetcher = from_kwargs(**kwargs)
     name = kwargs.get('name')
-    resources.append(Resource(name, fetcher, destination, basename))
+    resources.append(Resource(name, fetcher, destination, placement))
 
 
 class DirectiveError(spack.error.SpackError):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index dcb514af00..b386f8f6a8 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -683,11 +683,17 @@ class Package(object):
         for resource in resources:
             stage = resource.fetcher.stage
             _expand_archive(stage, resource.name)
-            basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename
-            link_path = join_path(self.stage.source_path, resource.destination, basename)
-            if not os.path.exists(link_path):
-                # Create a symlink
-                os.symlink(stage.source_path, link_path)
+            # Turn placement into a dict with relative paths
+            placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
+            if not isinstance(placement, dict):
+                placement = {'': placement}
+            # Make the paths in the dictionary absolute and link
+            for key, value in placement.iteritems():
+                link_path = join_path(self.stage.source_path, resource.destination, value)
+                source_path = join_path(stage.source_path, key)
+                if not os.path.exists(link_path):
+                    # Create a symlink
+                    os.symlink(source_path, link_path)
         ##########
         self.stage.chdir_to_source()
 
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
index b20612686d..8d081b45c9 100644
--- a/lib/spack/spack/resource.py
+++ b/lib/spack/spack/resource.py
@@ -30,10 +30,10 @@ package to enable optional features.
 
 class Resource(object):
     """
-    Represents an optional resource. Aggregates a name, a fetcher and a destination.
+    Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement
     """
-    def __init__(self, name, fetcher, destination, basename):
+    def __init__(self, name, fetcher, destination, placement):
         self.name = name
         self.fetcher = fetcher
         self.destination = destination
-        self.basename = basename
+        self.placement = placement
diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py
index eac1863b97..ca368b3074 100644
--- a/var/spack/packages/clang/package.py
+++ b/var/spack/packages/clang/package.py
@@ -22,8 +22,13 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+
+
 from spack import *
 
+import os
+import os.path
+
 class Clang(Package):
     """The goal of the Clang project is to create a new C, C++,
        Objective C and Objective C++ front-end for the LLVM compiler.
@@ -62,3 +67,29 @@ class Clang(Package):
                   *options)
             make()
             make("install")
+            # CLang doesn't look in llvm folders for system headers...
+            self.link_llvm_directories(spec)
+
+    def link_llvm_directories(self, spec):
+
+        def clang_include_dir_at(root):
+            return join_path(root, 'include')
+
+        def clang_lib_dir_at(root):
+            return join_path(root, 'lib/clang/', str(self.version), 'include')
+
+        def do_link(source_dir, destination_dir):
+            if os.path.exists(source_dir):
+                for name in os.listdir(source_dir):
+                    source = join_path(source_dir, name)
+                    link = join_path(destination_dir, name)
+                    os.symlink(source, link)
+
+        # Link folder and files in include
+        llvm_dir = clang_include_dir_at(spec['llvm'].prefix)
+        clang_dir = clang_include_dir_at(self.prefix)
+        do_link(llvm_dir, clang_dir)
+        # Link folder and files in lib
+        llvm_dir = clang_lib_dir_at(spec['llvm'].prefix)
+        clang_dir = clang_lib_dir_at(self.prefix)
+        do_link(llvm_dir, clang_dir)
\ No newline at end of file
diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py
index 872a6c082b..d7ae3390be 100644
--- a/var/spack/packages/llvm/package.py
+++ b/var/spack/packages/llvm/package.py
@@ -53,10 +53,10 @@ class Llvm(Package):
              destination='projects', when='@3.7.0')
     resource(name='libcxx',
              url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
-             destination='projects', basename='libcxx', when='+libcxx@3.7.0')
+             destination='projects', placement='libcxx', when='+libcxx@3.7.0')
     resource(name='libcxxabi',
              url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
-             destination='projects', basename='libcxxabi', when='+libcxx@3.7.0')
+             destination='projects', placement='libcxxabi', when='+libcxx@3.7.0')
     ##########
 
     def install(self, spec, prefix):
-- 
cgit v1.2.3-60-g2f50