summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/llnl/util/lang.py2
-rw-r--r--lib/spack/spack/directives.py53
-rw-r--r--lib/spack/spack/fetch_strategy.py16
-rw-r--r--lib/spack/spack/mirror.py81
-rw-r--r--lib/spack/spack/package.py65
-rw-r--r--lib/spack/spack/resource.py39
6 files changed, 220 insertions, 36 deletions
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index cc87b7eac2..c84828951d 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
def caller_locals():
"""This will return the locals of the *parent* of the caller.
- This allows a fucntion to insert variables into its caller's
+ This allows a function to insert variables into its caller's
scope. Yes, this is some black magic, and yes it's useful
for implementing things like depends_on and provides.
"""
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 2a818e8d0c..aa9fbd8d33 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -42,15 +42,19 @@ The available directives are:
* ``extends``
* ``patch``
* ``variant``
+ * ``resource``
"""
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
- 'variant' ]
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
+ 'variant', 'resource']
import re
import inspect
+import os.path
+import functools
from llnl.util.lang import *
+from llnl.util.filesystem import join_path
import spack
import spack.spec
@@ -60,7 +64,8 @@ from spack.version import Version
from spack.patch import Patch
from spack.variant import Variant
from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import from_kwargs
#
# This is a list of all directives, built up as they are defined in
@@ -79,8 +84,8 @@ class directive(object):
"""Decorator for Spack directives.
Spack directives allow you to modify a package while it is being
- defined, e.g. to add version or depenency information. Directives
- are one of the key pieces of Spack's package "langauge", which is
+ defined, e.g. to add version or dependency information. Directives
+ are one of the key pieces of Spack's package "language", which is
embedded in python.
Here's an example directive:
@@ -141,6 +146,7 @@ class directive(object):
def __call__(self, directive_function):
directives[directive_function.__name__] = self
+ @functools.wraps(directive_function)
def wrapped(*args, **kwargs):
pkg = DictWrapper(caller_locals())
self.ensure_dicts(pkg)
@@ -259,6 +265,43 @@ def variant(pkg, name, default=False, description=""):
pkg.variants[name] = Variant(default, description)
+@directive('resources')
+def resource(pkg, **kwargs):
+ """
+ Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+ dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
+ own folder inside spack stage area, and then linked into the stage area of the package that needs them.
+
+ List of recognized keywords:
+
+ * 'when' : (optional) represents the condition upon which the resource is needed
+ * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
+ area.
+ * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
+ area.
+ """
+ when = kwargs.get('when', pkg.name)
+ destination = kwargs.get('destination', "")
+ placement = kwargs.get('placement', None)
+ # Check if the path is relative
+ if os.path.isabs(destination):
+ message = "The destination keyword of a resource directive can't be an absolute path.\n"
+ message += "\tdestination : '{dest}\n'".format(dest=destination)
+ raise RuntimeError(message)
+ # Check if the path falls within the main package stage area
+ test_path = 'stage_folder_root/'
+ normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path
+ if test_path not in normalized_destination:
+ message = "The destination folder of a resource must fall within the main package stage directory.\n"
+ message += "\tdestination : '{dest}'\n".format(dest=destination)
+ raise RuntimeError(message)
+ when_spec = parse_anonymous_spec(when, pkg.name)
+ resources = pkg.resources.setdefault(when_spec, [])
+ fetcher = from_kwargs(**kwargs)
+ name = kwargs.get('name')
+ resources.append(Resource(name, fetcher, destination, placement))
+
+
class DirectiveError(spack.error.SpackError):
"""This is raised when something is wrong with a package directive."""
def __init__(self, directive, message):
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 8c4bd4ed8a..a9374fb34b 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -634,6 +634,22 @@ def from_url(url):
return URLFetchStrategy(url)
+def from_kwargs(**kwargs):
+ """
+ Construct the appropriate FetchStrategy from the given keyword arguments.
+
+ :param kwargs: dictionary of keyword arguments
+ :return: fetcher or raise a FetchError exception
+ """
+ for fetcher in all_strategies:
+ if fetcher.matches(kwargs):
+ return fetcher(**kwargs)
+ # Raise an error in case we can't instantiate any known strategy
+ message = "Cannot instantiate any FetchStrategy"
+ long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+ raise FetchError(message, long_message)
+
+
def args_are_for(args, fetcher):
fetcher.matches(args)
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 6fbf82de14..1d9b0e7ef2 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -26,7 +26,7 @@
This file contains code for creating spack mirror directories. A
mirror is an organized hierarchy containing specially named archive
files. This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down. Or, if the computer
+the main server for a particular package is down. Or, if the computer
where spack is run is not connected to the internet, it allows spack
to download packages directly from a mirror (e.g., on an intranet).
"""
@@ -42,7 +42,7 @@ import spack.fetch_strategy as fs
from spack.spec import Spec
from spack.stage import Stage
from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
if v.satisfies(spec.versions):
s = Spec(pkg.name)
s.versions = VersionList([v])
+ s.variants = spec.variants.copy()
matching.append(s)
return matching
+def suggest_archive_basename(resource):
+ """
+ Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+ :param fetcher:
+ :return:
+ """
+ basename = os.path.basename(resource.fetcher.url)
+ if not allowed_archive(basename):
+ raise RuntimeError("%s is not an allowed archive tye" % basename)
+ return basename
+
+
+
def create(path, specs, **kwargs):
"""Create a directory to be used as a spack mirror, and fill it with
package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
Return Value:
Returns a tuple of lists: (present, mirrored, error)
- * present: Package specs that were already prsent.
+ * present: Package specs that were already present.
* mirrored: Package specs that were successfully mirrored.
* error: Package specs that failed to mirror due to some error.
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
error = []
# Iterate through packages and download all the safe tarballs for each of them
+ everything_already_exists = True
for spec in version_specs:
pkg = spec.package
@@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@"))
+ else:
+ everything_already_exists = False
+ # Set up a stage and a fetcher for the download
+ unique_fetch_name = spec.format("$_$@")
+ fetcher = fs.for_package_version(pkg, pkg.version)
+ stage = Stage(fetcher, name=unique_fetch_name)
+ fetcher.set_stage(stage)
+
+ # Do the fetch and checksum if necessary
+ fetcher.fetch()
+ if not kwargs.get('no_checksum', False):
+ fetcher.check()
+ tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+ # Fetchers have to know how to archive their files. Use
+ # that to move/copy/create an archive in the mirror.
+ fetcher.archive(archive_path)
+ tty.msg("Added %s." % spec.format("$_$@"))
+
+ # Fetch resources if they are associated with the spec
+ resources = pkg._get_resources()
+ for resource in resources:
+ resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+ if os.path.exists(resource_archive_path):
+ tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+ continue
+ everything_already_exists = False
+ resource_stage_folder = pkg._resource_stage(resource)
+ resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+ resource.fetcher.set_stage(resource_stage)
+ resource.fetcher.fetch()
+ if not kwargs.get('no_checksum', False):
+ resource.fetcher.check()
+ tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+ resource.fetcher.archive(resource_archive_path)
+ tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+ if everything_already_exists:
present.append(spec)
- continue
-
- # Set up a stage and a fetcher for the download
- unique_fetch_name = spec.format("$_$@")
- fetcher = fs.for_package_version(pkg, pkg.version)
- stage = Stage(fetcher, name=unique_fetch_name)
- fetcher.set_stage(stage)
-
- # Do the fetch and checksum if necessary
- fetcher.fetch()
- if not kwargs.get('no_checksum', False):
- fetcher.check()
- tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
- # Fetchers have to know how to archive their files. Use
- # that to move/copy/create an archive in the mirror.
- fetcher.archive(archive_path)
- tty.msg("Added %s." % spec.format("$_$@"))
- mirrored.append(spec)
+ else:
+ mirrored.append(spec)
except Exception, e:
if spack.debug:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 4d75726e06..b95afb073d 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -655,26 +655,62 @@ class Package(object):
"Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
self.stage.fetch()
+
+ ##########
+ # Fetch resources
+ resources = self._get_resources()
+ for resource in resources:
+ resource_stage_folder = self._resource_stage(resource)
+ # FIXME : works only for URLFetchStrategy
+ resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+ resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+ resource.fetcher.set_stage(resource_stage)
+ # Delegate to stage object to trigger mirror logic
+ resource_stage.fetch()
+ resource_stage.check()
+ ##########
+
self._fetch_time = time.time() - start_time
if spack.do_checksum and self.version in self.versions:
self.stage.check()
-
def do_stage(self):
"""Unpacks the fetched tarball, then changes into the expanded tarball
directory."""
if not self.spec.concrete:
raise ValueError("Can only stage concrete packages.")
- self.do_fetch()
+ def _expand_archive(stage, name=self.name):
+ archive_dir = stage.source_path
+ if not archive_dir:
+ stage.expand_archive()
+ tty.msg("Created stage in %s." % stage.path)
+ else:
+ tty.msg("Already staged %s in %s." % (name, stage.path))
- archive_dir = self.stage.source_path
- if not archive_dir:
- self.stage.expand_archive()
- tty.msg("Created stage in %s." % self.stage.path)
- else:
- tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+
+ self.do_fetch()
+ _expand_archive(self.stage)
+
+ ##########
+ # Stage resources in appropriate path
+ resources = self._get_resources()
+ for resource in resources:
+ stage = resource.fetcher.stage
+ _expand_archive(stage, resource.name)
+ # Turn placement into a dict with relative paths
+ placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
+ if not isinstance(placement, dict):
+ placement = {'': placement}
+ # Make the paths in the dictionary absolute and link
+ for key, value in placement.iteritems():
+ link_path = join_path(self.stage.source_path, resource.destination, value)
+ source_path = join_path(stage.source_path, key)
+ if not os.path.exists(link_path):
+ # Create a symlink
+ os.symlink(source_path, link_path)
+ ##########
self.stage.chdir_to_source()
@@ -746,6 +782,19 @@ class Package(object):
mkdirp(self.prefix.man1)
+ def _get_resources(self):
+ resources = []
+ # Select the resources that are needed for this build
+ for when_spec, resource_list in self.resources.items():
+ if when_spec in self.spec:
+ resources.extend(resource_list)
+ return resources
+
+ def _resource_stage(self, resource):
+ pieces = ['resource', resource.name, self.spec.dag_hash()]
+ resource_stage_folder = '-'.join(pieces)
+ return resource_stage_folder
+
def _build_logger(self, log_path):
"""Create a context manager to log build output."""
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
new file mode 100644
index 0000000000..8d081b45c9
--- /dev/null
+++ b/lib/spack/spack/resource.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+
+class Resource(object):
+ """
+ Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement
+ """
+ def __init__(self, name, fetcher, destination, placement):
+ self.name = name
+ self.fetcher = fetcher
+ self.destination = destination
+ self.placement = placement