Merge pull request #298 from epfl-scitas/refactoring/resource_directive

refactoring proposal : composite for Stage and FetchStrategy 

10 files changed, 441 insertions, 215 deletions

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 0b98211cb9..5745adce63 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -296,8 +296,8 @@ def resource(pkg, **kwargs):
         raise RuntimeError(message)
     when_spec = parse_anonymous_spec(when, pkg.name)
     resources = pkg.resources.setdefault(when_spec, [])
-    fetcher = from_kwargs(**kwargs)
     name = kwargs.get('name')
+    fetcher = from_kwargs(**kwargs)
     resources.append(Resource(name, fetcher, destination, placement))
 
 
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 337dd1e198..83a2dbb59c 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -44,6 +44,7 @@ import os
 import sys
 import re
 import shutil
+import copy
 from functools import wraps
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
@@ -55,53 +56,59 @@ from spack.util.string import *
 from spack.version import Version, ver
 from spack.util.compression import decompressor_for, extension
 
+import spack.util.pattern as pattern
+
 """List of all fetch strategies, created by FetchStrategy metaclass."""
 all_strategies = []
 
+
 def _needs_stage(fun):
     """Many methods on fetch strategies require a stage to be set
        using set_stage().  This decorator adds a check for self.stage."""
+
     @wraps(fun)
     def wrapper(self, *args, **kwargs):
         if not self.stage:
             raise NoStageError(fun)
         return fun(self, *args, **kwargs)
+
     return wrapper
 
 
 class FetchStrategy(object):
     """Superclass of all fetch strategies."""
-    enabled = False             # Non-abstract subclasses should be enabled.
+    enabled = False  # Non-abstract subclasses should be enabled.
     required_attributes = None  # Attributes required in version() args.
 
     class __metaclass__(type):
         """This metaclass registers all fetch strategies in a list."""
+
         def __init__(cls, name, bases, dict):
             type.__init__(cls, name, bases, dict)
             if cls.enabled: all_strategies.append(cls)
 
-
     def __init__(self):
         # The stage is initialized late, so that fetch strategies can be constructed
         # at package construction time.  This is where things will be fetched.
         self.stage = None
 
-
     def set_stage(self, stage):
         """This is called by Stage before any of the fetching
            methods are called on the stage."""
         self.stage = stage
 
-
     # Subclasses need to implement these methods
     def fetch(self):   pass  # Return True on success, False on fail.
+
     def check(self):   pass  # Do checksum.
+
     def expand(self):  pass  # Expand archive.
+
     def reset(self):   pass  # Revert to freshly downloaded state.
 
     def archive(self, destination): pass  # Used to create tarball for mirror.
 
-    def __str__(self):       # Should be human readable URL.
+    def __str__(self):  # Should be human readable URL.
         return "FetchStrategy.__str___"
 
     # This method is used to match fetch strategies to version()
@@ -111,6 +118,15 @@ class FetchStrategy(object):
         return any(k in args for k in cls.required_attributes)
 
 
+@pattern.composite(interface=FetchStrategy)
+class FetchStrategyComposite(object):
+    """
+    Composite for a FetchStrategy object. Implements the GoF composite pattern.
+    """
+    matches = FetchStrategy.matches
+    set_stage = FetchStrategy.set_stage
+
+
 class URLFetchStrategy(FetchStrategy):
     """FetchStrategy that pulls source code from a URL for an archive,
        checks the archive against a checksum,and decompresses the archive.
@@ -142,15 +158,15 @@ class URLFetchStrategy(FetchStrategy):
 
         tty.msg("Trying to fetch from %s" % self.url)
 
-        curl_args = ['-O',        # save file to disk
-                     '-f',        # fail on >400 errors
-                     '-D', '-',   # print out HTML headers
-                     '-L', self.url,]
+        curl_args = ['-O',  # save file to disk
+                     '-f',  # fail on >400 errors
+                     '-D', '-',  # print out HTML headers
+                     '-L', self.url, ]
 
         if sys.stdout.isatty():
             curl_args.append('-#')  # status bar when using a tty
         else:
-            curl_args.append('-sS') # just errors when not.
+            curl_args.append('-sS')  # just errors when not.
 
         # Run curl but grab the mime type from the http headers
         headers = spack.curl(
@@ -164,24 +180,23 @@ class URLFetchStrategy(FetchStrategy):
             if spack.curl.returncode == 22:
                 # This is a 404.  Curl will print the error.
                 raise FailedDownloadError(
-                    self.url, "URL %s was not found!" % self.url)
+                        self.url, "URL %s was not found!" % self.url)
 
             elif spack.curl.returncode == 60:
                 # This is a certificate error.  Suggest spack -k
                 raise FailedDownloadError(
-                    self.url,
-                    "Curl was unable to fetch due to invalid certificate. "
-                    "This is either an attack, or your cluster's SSL configuration "
-                    "is bad.  If you believe your SSL configuration is bad, you "
-                    "can try running spack -k, which will not check SSL certificates."
-                    "Use this at your own risk.")
+                        self.url,
+                        "Curl was unable to fetch due to invalid certificate. "
+                        "This is either an attack, or your cluster's SSL configuration "
+                        "is bad.  If you believe your SSL configuration is bad, you "
+                        "can try running spack -k, which will not check SSL certificates."
+                        "Use this at your own risk.")
 
             else:
                 # This is some other curl error.  Curl will print the
                 # error, but print a spack message too
                 raise FailedDownloadError(
-                    self.url, "Curl failed with error %d" % spack.curl.returncode)
-
+                        self.url, "Curl failed with error %d" % spack.curl.returncode)
 
         # Check if we somehow got an HTML file rather than the archive we
         # asked for.  We only look at the last content type, to handle
@@ -196,7 +211,6 @@ class URLFetchStrategy(FetchStrategy):
         if not self.archive_file:
             raise FailedDownloadError(self.url)
 
-
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""
@@ -209,7 +223,7 @@ class URLFetchStrategy(FetchStrategy):
         self.stage.chdir()
         if not self.archive_file:
             raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
-                                      "Failed on expand() for URL %s" % self.url)
+                                     "Failed on expand() for URL %s" % self.url)
 
         decompress = decompressor_for(self.archive_file)
 
@@ -241,7 +255,6 @@ class URLFetchStrategy(FetchStrategy):
         # Set the wd back to the stage when done.
         self.stage.chdir()
 
-
     def archive(self, destination):
         """Just moves this archive to the destination."""
         if not self.archive_file:
@@ -252,7 +265,6 @@ class URLFetchStrategy(FetchStrategy):
 
         shutil.move(self.archive_file, destination)
 
-
     @_needs_stage
     def check(self):
         """Check the downloaded archive against a checksum digest.
@@ -263,9 +275,8 @@ class URLFetchStrategy(FetchStrategy):
         checker = crypto.Checker(self.digest)
         if not checker.check(self.archive_file):
             raise ChecksumError(
-                "%s checksum failed for %s." % (checker.hash_name, self.archive_file),
-                "Expected %s but got %s." % (self.digest, checker.sum))
-
+                    "%s checksum failed for %s." % (checker.hash_name, self.archive_file),
+                    "Expected %s but got %s." % (self.digest, checker.sum))
 
     @_needs_stage
     def reset(self):
@@ -277,12 +288,10 @@ class URLFetchStrategy(FetchStrategy):
             shutil.rmtree(self.stage.source_path, ignore_errors=True)
         self.expand()
 
-
     def __repr__(self):
         url = self.url if self.url else "no url"
         return "URLFetchStrategy<%s>" % url
 
-
     def __str__(self):
         if self.url:
             return self.url
@@ -298,33 +307,30 @@ class VCSFetchStrategy(FetchStrategy):
         # Set a URL based on the type of fetch strategy.
         self.url = kwargs.get(name, None)
         if not self.url: raise ValueError(
-            "%s requires %s argument." % (self.__class__, name))
+                "%s requires %s argument." % (self.__class__, name))
 
         # Ensure that there's only one of the rev_types
         if sum(k in kwargs for k in rev_types) > 1:
             raise FetchStrategyError(
-                "Supply only one of %s to fetch with %s." % (
-                    comma_or(rev_types), name))
+                    "Supply only one of %s to fetch with %s." % (
+                        comma_or(rev_types), name))
 
         # Set attributes for each rev type.
         for rt in rev_types:
             setattr(self, rt, kwargs.get(rt, None))
 
-
     @_needs_stage
     def check(self):
         tty.msg("No checksum needed when fetching with %s." % self.name)
 
-
     @_needs_stage
     def expand(self):
         tty.debug("Source fetched with %s is already expanded." % self.name)
 
-
     @_needs_stage
     def archive(self, destination, **kwargs):
-        assert(extension(destination) == 'tar.gz')
-        assert(self.stage.source_path.startswith(self.stage.path))
+        assert (extension(destination) == 'tar.gz')
+        assert (self.stage.source_path.startswith(self.stage.path))
 
         tar = which('tar', required=True)
 
@@ -338,16 +344,13 @@ class VCSFetchStrategy(FetchStrategy):
         self.stage.chdir()
         tar('-czf', destination, os.path.basename(self.stage.source_path))
 
-
     def __str__(self):
         return "VCS: %s" % self.url
 
-
     def __repr__(self):
         return "%s<%s>" % (self.__class__, self.url)
 
 
-
 class GitFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a git repository.
        Use like this in a package:
@@ -368,24 +371,25 @@ class GitFetchStrategy(VCSFetchStrategy):
     required_attributes = ('git',)
 
     def __init__(self, **kwargs):
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
+        forwarded_args = copy.copy(kwargs)
+        forwarded_args.pop('name', None)
+
         super(GitFetchStrategy, self).__init__(
-            'git', 'tag', 'branch', 'commit', **kwargs)
+                'git', 'tag', 'branch', 'commit', **forwarded_args)
         self._git = None
 
-
     @property
     def git_version(self):
         vstring = self.git('--version', output=str).lstrip('git version ')
         return Version(vstring)
 
-
     @property
     def git(self):
         if not self._git:
             self._git = which('git', required=True)
         return self._git
 
-
     @_needs_stage
     def fetch(self):
         self.stage.chdir()
@@ -418,7 +422,7 @@ class GitFetchStrategy(VCSFetchStrategy):
             if self.branch:
                 args.extend(['--branch', self.branch])
             elif self.tag and self.git_version >= ver('1.8.5.2'):
-                    args.extend(['--branch', self.tag])
+                args.extend(['--branch', self.tag])
 
             # Try to be efficient if we're using a new enough git.
             # This checks out only one branch's history
@@ -429,7 +433,7 @@ class GitFetchStrategy(VCSFetchStrategy):
             # Yet more efficiency, only download a 1-commit deep tree
             if self.git_version >= ver('1.7.1'):
                 try:
-                    self.git(*(args + ['--depth','1', self.url]))
+                    self.git(*(args + ['--depth', '1', self.url]))
                     cloned = True
                 except spack.error.SpackError:
                     # This will fail with the dumb HTTP transport
@@ -452,18 +456,15 @@ class GitFetchStrategy(VCSFetchStrategy):
                 self.git('pull', '--tags', ignore_errors=1)
                 self.git('checkout', self.tag)
 
-
     def archive(self, destination):
         super(GitFetchStrategy, self).archive(destination, exclude='.git')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir_to_source()
         self.git('checkout', '.')
         self.git('clean', '-f')
 
-
     def __str__(self):
         return "[git] %s" % self.url
 
@@ -483,20 +484,22 @@ class SvnFetchStrategy(VCSFetchStrategy):
     required_attributes = ['svn']
 
     def __init__(self, **kwargs):
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
+        forwarded_args = copy.copy(kwargs)
+        forwarded_args.pop('name', None)
+
         super(SvnFetchStrategy, self).__init__(
-            'svn', 'revision', **kwargs)
+                'svn', 'revision', **forwarded_args)
         self._svn = None
         if self.revision is not None:
             self.revision = str(self.revision)
 
-
     @property
     def svn(self):
         if not self._svn:
             self._svn = which('svn', required=True)
         return self._svn
 
-
     @_needs_stage
     def fetch(self):
         self.stage.chdir()
@@ -515,7 +518,6 @@ class SvnFetchStrategy(VCSFetchStrategy):
         self.svn(*args)
         self.stage.chdir_to_source()
 
-
     def _remove_untracked_files(self):
         """Removes untracked files in an svn repository."""
         status = self.svn('status', '--no-ignore', output=str)
@@ -529,23 +531,19 @@ class SvnFetchStrategy(VCSFetchStrategy):
             elif os.path.isdir(path):
                 shutil.rmtree(path, ignore_errors=True)
 
-
     def archive(self, destination):
         super(SvnFetchStrategy, self).archive(destination, exclude='.svn')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir_to_source()
         self._remove_untracked_files()
         self.svn('revert', '.', '-R')
 
-
     def __str__(self):
         return "[svn] %s" % self.url
 
 
-
 class HgFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a Mercurial repository.
        Use like this in a package:
@@ -567,11 +565,14 @@ class HgFetchStrategy(VCSFetchStrategy):
     required_attributes = ['hg']
 
     def __init__(self, **kwargs):
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
+        forwarded_args = copy.copy(kwargs)
+        forwarded_args.pop('name', None)
+
         super(HgFetchStrategy, self).__init__(
-            'hg', 'revision', **kwargs)
+                'hg', 'revision', **forwarded_args)
         self._hg = None
 
-
     @property
     def hg(self):
         if not self._hg:
@@ -597,11 +598,9 @@ class HgFetchStrategy(VCSFetchStrategy):
 
         self.hg(*args)
 
-
     def archive(self, destination):
         super(HgFetchStrategy, self).archive(destination, exclude='.hg')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir()
@@ -619,7 +618,6 @@ class HgFetchStrategy(VCSFetchStrategy):
         shutil.move(scrubbed, source_path)
         self.stage.chdir_to_source()
 
-
     def __str__(self):
         return "[hg] %s" % self.url
 
@@ -693,9 +691,10 @@ class FetchError(spack.error.SpackError):
 
 class FailedDownloadError(FetchError):
     """Raised wen a download fails."""
+
     def __init__(self, url, msg=""):
         super(FailedDownloadError, self).__init__(
-            "Failed to fetch file from URL: %s" % url, msg)
+                "Failed to fetch file from URL: %s" % url, msg)
         self.url = url
 
 
@@ -718,12 +717,14 @@ class InvalidArgsError(FetchError):
 
 class ChecksumError(FetchError):
     """Raised when archive fails to checksum."""
+
     def __init__(self, message, long_msg=None):
         super(ChecksumError, self).__init__(message, long_msg)
 
 
 class NoStageError(FetchError):
     """Raised when fetch operations are called before set_stage()."""
+
     def __init__(self, method):
         super(NoStageError, self).__init__(
-            "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
+                "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 341cc4cb88..fa29e20803 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -45,12 +45,11 @@ from spack.version import *
 from spack.util.compression import extension, allowed_archive
 
 
-def mirror_archive_filename(spec):
+def mirror_archive_filename(spec, fetcher):
     """Get the name of the spec's archive in the mirror."""
     if not spec.version.concrete:
         raise ValueError("mirror.path requires spec with concrete version.")
 
-    fetcher = spec.package.fetcher
     if isinstance(fetcher, fs.URLFetchStrategy):
         # If we fetch this version with a URLFetchStrategy, use URL's archive type
         ext = url.downloaded_file_extension(fetcher.url)
@@ -61,9 +60,9 @@ def mirror_archive_filename(spec):
     return "%s-%s.%s" % (spec.package.name, spec.version, ext)
 
 
-def mirror_archive_path(spec):
+def mirror_archive_path(spec, fetcher):
     """Get the relative path to the spec's archive within a mirror."""
-    return join_path(spec.name, mirror_archive_filename(spec))
+    return join_path(spec.name, mirror_archive_filename(spec, fetcher))
 
 
 def get_matching_versions(specs, **kwargs):
@@ -167,72 +166,47 @@ def create(path, specs, **kwargs):
     everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
-
-        stage = None
+        tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@")))
         try:
-            # create a subdirectory for the current package@version
-            archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec)))
-            subdir = os.path.dirname(archive_path)
-            try:
+            for ii, stage in enumerate(pkg.stage):
+                fetcher = stage.fetcher
+                if ii == 0:
+                    # create a subdirectory for the current package@version
+                    archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher)))
+                    name = spec.format("$_$@")
+                else:
+                    resource = stage.resource
+                    archive_path = join_path(subdir, suggest_archive_basename(resource))
+                    name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@"))
+                subdir = os.path.dirname(archive_path)
                 mkdirp(subdir)
-            except OSError as e:
-                raise MirrorError(
-                    "Cannot create directory '%s':" % subdir, str(e))
 
-            if os.path.exists(archive_path):
-                tty.msg("Already added %s" % spec.format("$_$@"))
-            else:
-                everything_already_exists = False
-                # Set up a stage and a fetcher for the download
-                unique_fetch_name = spec.format("$_$@")
-                fetcher = fs.for_package_version(pkg, pkg.version)
-                stage = Stage(fetcher, name=unique_fetch_name)
-                fetcher.set_stage(stage)
-
-                # Do the fetch and checksum if necessary
-                fetcher.fetch()
-                if not kwargs.get('no_checksum', False):
-                    fetcher.check()
-                    tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
-                # Fetchers have to know how to archive their files.  Use
-                # that to move/copy/create an archive in the mirror.
-                fetcher.archive(archive_path)
-                tty.msg("Added %s." % spec.format("$_$@"))
-
-            # Fetch resources if they are associated with the spec
-            resources = pkg._get_resources()
-            for resource in resources:
-                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
-                if os.path.exists(resource_archive_path):
-                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
-                    continue
-                everything_already_exists = False
-                resource_stage_folder = pkg._resource_stage(resource)
-                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
-                resource.fetcher.set_stage(resource_stage)
-                resource.fetcher.fetch()
-                if not kwargs.get('no_checksum', False):
-                    resource.fetcher.check()
-                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
-                resource.fetcher.archive(resource_archive_path)
-                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                if os.path.exists(archive_path):
+                    tty.msg("{name} : already added".format(name=name))
+                else:
+                    everything_already_exists = False
+                    fetcher.fetch()
+                    if not kwargs.get('no_checksum', False):
+                        fetcher.check()
+                        tty.msg("{name} : checksum passed".format(name=name))
+
+                    # Fetchers have to know how to archive their files.  Use
+                    # that to move/copy/create an archive in the mirror.
+                    fetcher.archive(archive_path)
+                    tty.msg("{name} : added".format(name=name))
 
             if everything_already_exists:
                 present.append(spec)
             else:
                 mirrored.append(spec)
-
         except Exception, e:
             if spack.debug:
                 sys.excepthook(*sys.exc_info())
             else:
                 tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
             error.append(spec)
-
         finally:
-            if stage:
-                stage.destroy()
+            pkg.stage.destroy()
 
     return (present, mirrored, error)
 
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 8cb947c276..8019b29cba 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -63,7 +63,7 @@ import spack.url
 import spack.util.web
 import spack.fetch_strategy as fs
 from spack.version import *
-from spack.stage import Stage
+from spack.stage import Stage, ResourceStage, StageComposite
 from spack.util.compression import allowed_archive, extension
 from spack.util.executable import ProcessError
 
@@ -433,23 +433,46 @@ class Package(object):
         return spack.url.substitute_version(self.nearest_url(version),
                                             self.url_version(version))
 
+    def _make_resource_stage(self, root_stage, fetcher, resource):
+        resource_stage_folder = self._resource_stage(resource)
+        resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
+        stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
+                              name=resource_stage_folder, mirror_path=resource_mirror)
+        return stage
+
+    def _make_root_stage(self, fetcher):
+        # Construct a mirror path (TODO: get this out of package.py)
+        mp = spack.mirror.mirror_archive_path(self.spec, fetcher)
+        # Construct a path where the stage should build..
+        s = self.spec
+        stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
+        # Build the composite stage
+        stage = Stage(fetcher, mirror_path=mp, name=stage_name)
+        return stage
+
+    def _make_stage(self):
+        # Construct a composite stage on top of the composite FetchStrategy
+        composite_fetcher = self.fetcher
+        composite_stage = StageComposite()
+        resources = self._get_resources()
+        for ii, fetcher in enumerate(composite_fetcher):
+            if ii == 0:
+                # Construct root stage first
+                stage = self._make_root_stage(fetcher)
+            else:
+                # Construct resource stage
+                resource = resources[ii - 1]  # ii == 0 is root!
+                stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
+            # Append the item to the composite
+            composite_stage.append(stage)
+        return composite_stage
 
     @property
     def stage(self):
         if not self.spec.concrete:
             raise ValueError("Can only get a stage for a concrete package.")
-
         if self._stage is None:
-            # Construct a mirror path (TODO: get this out of package.py)
-            mp = spack.mirror.mirror_archive_path(self.spec)
-
-            # Construct a path where the stage should build..
-            s = self.spec
-            stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
-
-            # Build the stage
-            self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name)
-
+            self._stage = self._make_stage()
         return self._stage
 
 
@@ -459,17 +482,25 @@ class Package(object):
         self._stage = stage
 
 
+    def _make_fetcher(self):
+        # Construct a composite fetcher that always contains at least one element (the root package). In case there
+        # are resources associated with the package, append their fetcher to the composite.
+        root_fetcher = fs.for_package_version(self, self.version)
+        fetcher = fs.FetchStrategyComposite()  # Composite fetcher
+        fetcher.append(root_fetcher)  # Root fetcher is always present
+        resources = self._get_resources()
+        for resource in resources:
+            fetcher.append(resource.fetcher)
+        return fetcher
+
     @property
     def fetcher(self):
         if not self.spec.versions.concrete:
-            raise ValueError(
-                "Can only get a fetcher for a package with concrete versions.")
-
+            raise ValueError("Can only get a fetcher for a package with concrete versions.")
         if not self._fetcher:
-            self._fetcher = fs.for_package_version(self, self.version)
+            self._fetcher = self._make_fetcher()
         return self._fetcher
 
-
     @fetcher.setter
     def fetcher(self, f):
         self._fetcher = f
@@ -632,7 +663,7 @@ class Package(object):
 
 
     def do_fetch(self, mirror_only=False):
-        """Creates a stage directory and downloads the taball for this package.
+        """Creates a stage directory and downloads the tarball for this package.
            Working directory will be set to the stage directory.
         """
         if not self.spec.concrete:
@@ -658,20 +689,6 @@ class Package(object):
 
         self.stage.fetch(mirror_only)
 
-        ##########
-        # Fetch resources
-        resources = self._get_resources()
-        for resource in resources:
-            resource_stage_folder = self._resource_stage(resource)
-            # FIXME : works only for URLFetchStrategy
-            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
-            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
-            resource.fetcher.set_stage(resource_stage)
-            # Delegate to stage object to trigger mirror logic
-            resource_stage.fetch()
-            resource_stage.check()
-        ##########
-
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
@@ -684,52 +701,10 @@ class Package(object):
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        def _expand_archive(stage, name=self.name):
-            archive_dir = stage.source_path
-            if not archive_dir:
-                stage.expand_archive()
-                tty.msg("Created stage in %s." % stage.path)
-            else:
-                tty.msg("Already staged %s in %s." % (name, stage.path))
-
         self.do_fetch(mirror_only)
-        _expand_archive(self.stage)
-
-        ##########
-        # Stage resources in appropriate path
-        resources = self._get_resources()
-        # TODO: this is to allow nested resources, a better solution would be
-        # good
-        for resource in sorted(resources, key=lambda res: len(res.destination)):
-            stage = resource.fetcher.stage
-            _expand_archive(stage, resource.name)
-            # Turn placement into a dict with relative paths
-            placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
-            if not isinstance(placement, dict):
-                placement = {'': placement}
-            # Make the paths in the dictionary absolute and link
-            for key, value in placement.iteritems():
-                target_path = join_path(self.stage.source_path, resource.destination)
-                link_path = join_path(target_path, value)
-                source_path = join_path(stage.source_path, key)
-
-                try:
-                    os.makedirs(target_path)
-                except OSError as err:
-                    if err.errno == errno.EEXIST and os.path.isdir(target_path):
-                        pass
-                    else: raise
-
-                # NOTE: a reasonable fix for the TODO above might be to have
-                # these expand in place, but expand_archive does not offer
-                # this
-
-                if not os.path.exists(link_path):
-                    shutil.move(source_path, link_path)
-        ##########
+        self.stage.expand_archive()
         self.stage.chdir_to_source()
 
-
     def do_patch(self):
         """Calls do_stage(), then applied patches to the expanded tarball if they
            haven't been applied already."""
@@ -828,6 +803,9 @@ class Package(object):
         for when_spec, resource_list in self.resources.items():
             if when_spec in self.spec:
                 resources.extend(resource_list)
+        # Sorts the resources by the length of the string representing their destination. Since any nested resource
+        # must contain another resource's name in its path, it seems that should work
+        resources = sorted(resources, key=lambda res: len(res.destination))
         return resources
 
     def _resource_stage(self, resource):
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 79c9030e20..f217450d42 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -23,7 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import re
+import errno
 import shutil
 import tempfile
 from urlparse import urljoin
@@ -31,17 +31,18 @@ from urlparse import urljoin
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
 
+import spack.util.pattern as pattern
+
 import spack
 import spack.config
 import spack.fetch_strategy as fs
 import spack.error
 
-
 STAGE_PREFIX = 'spack-stage-'
 
 
 class Stage(object):
-    """A Stage object manaages a directory where some source code is
+    """A Stage object manages a directory where some source code is
        downloaded and built before being installed.  It handles
        fetching the source code, either as an archive to be expanded
        or by checking it out of a repository.  A stage's lifecycle
@@ -93,7 +94,7 @@ class Stage(object):
             raise ValueError("Can't construct Stage without url or fetch strategy")
         self.fetcher.set_stage(self)
         self.default_fetcher = self.fetcher  # self.fetcher can change with mirrors.
-        self.skip_checksum_for_mirror = True # used for mirrored archives of repositories.
+        self.skip_checksum_for_mirror = True  # used for mirrored archives of repositories.
 
         self.name = kwargs.get('name')
         self.mirror_path = kwargs.get('mirror_path')
@@ -102,7 +103,6 @@ class Stage(object):
         self.path = None
         self._setup()
 
-
     def _cleanup_dead_links(self):
         """Remove any dead links in the stage directory."""
         for file in os.listdir(spack.stage_path):
@@ -112,7 +112,6 @@ class Stage(object):
                 if not os.path.exists(path):
                     os.unlink(path)
 
-
     def _need_to_create_path(self):
         """Makes sure nothing weird has happened since the last time we
            looked at path.  Returns True if path already exists and is ok.
@@ -130,7 +129,7 @@ class Stage(object):
         # Path looks ok, but need to check the target of the link.
         if os.path.islink(self.path):
             real_path = os.path.realpath(self.path)
-            real_tmp  = os.path.realpath(self.tmp_root)
+            real_tmp = os.path.realpath(self.tmp_root)
 
             if spack.use_tmp_stage:
                 # If we're using a tmp dir, it's a link, and it points at the right spot,
@@ -149,7 +148,6 @@ class Stage(object):
 
         return False
 
-
     def _setup(self):
         """Creates the stage directory.
            If spack.use_tmp_stage is False, the stage directory is created
@@ -198,7 +196,6 @@ class Stage(object):
         # Make sure we can actually do something with the stage we made.
         ensure_access(self.path)
 
-
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""
@@ -215,7 +212,6 @@ class Stage(object):
         else:
             return None
 
-
     @property
     def source_path(self):
         """Returns the path to the expanded/checked out source code
@@ -230,7 +226,6 @@ class Stage(object):
                 return p
         return None
 
-
     def chdir(self):
         """Changes directory to the stage path.  Or dies if it is not set up."""
         if os.path.isdir(self.path):
@@ -238,7 +233,6 @@ class Stage(object):
         else:
             tty.die("Setup failed: no such directory: " + self.path)
 
-
     def fetch(self, mirror_only=False):
         """Downloads an archive or checks out code from a repository."""
         self.chdir()
@@ -291,7 +285,6 @@ class Stage(object):
             self.fetcher = self.default_fetcher
             raise fs.FetchError(errMessage, None)
 
-
     def check(self):
         """Check the downloaded archive against a checksum digest.
            No-op if this stage checks code out of a repository."""
@@ -305,14 +298,17 @@ class Stage(object):
         else:
             self.fetcher.check()
 
-
     def expand_archive(self):
         """Changes to the stage directory and attempt to expand the downloaded
            archive.  Fail if the stage is not set up or if the archive is not yet
            downloaded.
         """
-        self.fetcher.expand()
-
+        archive_dir = self.source_path
+        if not archive_dir:
+            self.fetcher.expand()
+            tty.msg("Created stage in %s." % self.path)
+        else:
+            tty.msg("Already staged %s in %s." % (self.name, self.path))
 
     def chdir_to_source(self):
         """Changes directory to the expanded archive directory.
@@ -326,14 +322,12 @@ class Stage(object):
             if not os.listdir(path):
                 tty.die("Archive was empty for %s" % self.name)
 
-
     def restage(self):
         """Removes the expanded archive path if it exists, then re-expands
            the archive.
         """
         self.fetcher.reset()
 
-
     def destroy(self):
         """Remove this stage directory."""
         remove_linked_tree(self.path)
@@ -345,8 +339,63 @@ class Stage(object):
             os.chdir(os.path.dirname(self.path))
 
 
+class ResourceStage(Stage):
+    def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
+        super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
+        self.root_stage = root
+        self.resource = resource
+
+    def expand_archive(self):
+        super(ResourceStage, self).expand_archive()
+        root_stage = self.root_stage
+        resource = self.resource
+        placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
+        if not isinstance(placement, dict):
+            placement = {'': placement}
+        # Make the paths in the dictionary absolute and link
+        for key, value in placement.iteritems():
+            target_path = join_path(root_stage.source_path, resource.destination)
+            destination_path = join_path(target_path, value)
+            source_path = join_path(self.source_path, key)
+
+            try:
+                os.makedirs(target_path)
+            except OSError as err:
+                if err.errno == errno.EEXIST and os.path.isdir(target_path):
+                    pass
+                else:
+                    raise
+
+            if not os.path.exists(destination_path):
+                # Create a symlink
+                tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format(
+                    stage=source_path, destination=destination_path
+                ))
+                shutil.move(source_path, destination_path)
+
+
+@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy'])
+class StageComposite:
+    """
+    Composite for Stage type objects. The first item in this composite is considered to be the root package, and
+    operations that return a value are forwarded to it.
+    """
+
+    @property
+    def source_path(self):
+        return self[0].source_path
+
+    @property
+    def path(self):
+        return self[0].path
+
+    def chdir_to_source(self):
+        return self[0].chdir_to_source()
+
+
 class DIYStage(object):
     """Simple class that allows any directory to be a spack stage."""
+
     def __init__(self, path):
         self.archive_file = None
         self.path = path
@@ -384,7 +433,6 @@ def _get_mirrors():
     return [val for name, val in config.iteritems()]
 
 
-
 def ensure_access(file=spack.stage_path):
     """Ensure we can access a directory and die with an error if we can't."""
     if not can_access(file):
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index a569cbbf35..4b9a361d4b 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -48,6 +48,7 @@ test_names = ['versions',
               'package_sanity',
               'config',
               'directory_layout',
+              'pattern',
               'python_version',
               'git_fetch',
               'svn_fetch',
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index c09bd24c8e..8863d13c42 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -31,7 +31,7 @@ from llnl.util.filesystem import *
 
 import spack
 from spack.stage import Stage
-from spack.fetch_strategy import URLFetchStrategy
+from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
 from spack.directory_layout import YamlDirectoryLayout
 from spack.util.executable import which
 from spack.test.mock_packages_test import *
@@ -79,7 +79,10 @@ class InstallTest(MockPackagesTest):
         pkg = spack.repo.get(spec)
 
         # Fake the URL for the package so it downloads from a file.
-        pkg.fetcher = URLFetchStrategy(self.repo.url)
+
+        fetcher = FetchStrategyComposite()
+        fetcher.append(URLFetchStrategy(self.repo.url))
+        pkg.fetcher = fetcher
 
         try:
             pkg.do_install()
diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py
index 046ec56604..f83cc8090c 100644
--- a/lib/spack/spack/test/mirror.py
+++ b/lib/spack/spack/test/mirror.py
@@ -102,6 +102,7 @@ class MirrorTest(MockPackagesTest):
                 spec = Spec(name).concretized()
                 pkg = spec.package
 
+                pkg._stage = None
                 saved_checksum_setting = spack.do_checksum
                 try:
                     # Stage the archive from the mirror and cd to it.
diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py
new file mode 100644
index 0000000000..6c783c6a5f
--- /dev/null
+++ b/lib/spack/spack/test/pattern.py
@@ -0,0 +1,104 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import unittest
+
+import spack.util.pattern as pattern
+
+
+class CompositeTest(unittest.TestCase):
+
+    def setUp(self):
+        class Base:
+            counter = 0
+
+            def add(self):
+                raise NotImplemented('add not implemented')
+
+            def subtract(self):
+                raise NotImplemented('subtract not implemented')
+
+        class One(Base):
+            def add(self):
+                Base.counter += 1
+
+            def subtract(self):
+                Base.counter -= 1
+
+        class Two(Base):
+            def add(self):
+                Base.counter += 2
+
+            def subtract(self):
+                Base.counter -= 2
+
+        self.Base = Base
+        self.One = One
+        self.Two = Two
+
+    def test_composite_from_method_list(self):
+
+        @pattern.composite(method_list=['add', 'subtract'])
+        class CompositeFromMethodList:
+            pass
+
+        composite = CompositeFromMethodList()
+        composite.append(self.One())
+        composite.append(self.Two())
+        composite.add()
+        self.assertEqual(self.Base.counter, 3)
+        composite.pop()
+        composite.subtract()
+        self.assertEqual(self.Base.counter, 2)
+
+    def test_composite_from_interface(self):
+
+        @pattern.composite(interface=self.Base)
+        class CompositeFromInterface:
+            pass
+
+        composite = CompositeFromInterface()
+        composite.append(self.One())
+        composite.append(self.Two())
+        composite.add()
+        self.assertEqual(self.Base.counter, 3)
+        composite.pop()
+        composite.subtract()
+        self.assertEqual(self.Base.counter, 2)
+
+    def test_error_conditions(self):
+
+        def wrong_container():
+            @pattern.composite(interface=self.Base, container=2)
+            class CompositeFromInterface:
+                pass
+
+        def no_methods():
+            @pattern.composite()
+            class CompositeFromInterface:
+                pass
+
+        self.assertRaises(TypeError, wrong_container)
+        self.assertRaises(TypeError, no_methods)
diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py
new file mode 100644
index 0000000000..17a126498b
--- /dev/null
+++ b/lib/spack/spack/util/pattern.py
@@ -0,0 +1,116 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import inspect
+import collections
+import functools
+
+
+def composite(interface=None, method_list=None, container=list):
+    """
+    Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given
+    interface.
+
+    :param interface: class exposing the interface to which the composite object must conform. Only non-private and
+    non-special methods will be taken into account
+
+    :param method_list: names of methods that should be part of the composite
+
+    :param container: container for the composite object (default = list). Must fulfill the MutableSequence contract.
+    The composite class will expose the container API to manage object composition
+
+    :return: class decorator
+    """
+    # Check if container fulfills the MutableSequence contract and raise an exception if it doesn't
+    # The patched class returned by the decorator will inherit from the container class to expose the
+    # interface needed to manage objects composition
+    if not issubclass(container, collections.MutableSequence):
+        raise TypeError("Container must fulfill the MutableSequence contract")
+
+    # Check if at least one of the 'interface' or the 'method_list' arguments are defined
+    if interface is None and method_list is None:
+        raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite")
+
+    def cls_decorator(cls):
+        # Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden
+        def no_special_no_private(x):
+            return inspect.ismethod(x) and not x.__name__.startswith('_')
+
+        # Patch the behavior of each of the methods in the previous list. This is done associating an instance of the
+        # descriptor below to any method that needs to be patched.
+        class IterateOver(object):
+            """
+            Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the
+            associated attribute and calls for each of them an attribute with the same name
+            """
+            def __init__(self, name, func=None):
+                self.name = name
+                self.func = func
+
+            def __get__(self, instance, owner):
+                def getter(*args, **kwargs):
+                    for item in instance:
+                        getattr(item, self.name)(*args, **kwargs)
+                # If we are using this descriptor to wrap a method from an interface, then we must conditionally
+                # use the `functools.wraps` decorator to set the appropriate fields.
+                if self.func is not None:
+                    getter = functools.wraps(self.func)(getter)
+                return getter
+
+        dictionary_for_type_call = {}
+        # Construct a dictionary with the methods explicitly passed as name
+        if method_list is not None:
+            # python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list}
+            method_list_dict = {}
+            for name in method_list:
+                method_list_dict[name] = IterateOver(name)
+            dictionary_for_type_call.update(method_list_dict)
+        # Construct a dictionary with the methods inspected from the interface
+        if interface is not None:
+            ##########
+            # python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)}
+            interface_methods = {}
+            for name, method in inspect.getmembers(interface, predicate=no_special_no_private):
+                interface_methods[name] = method
+            ##########
+            # python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()}
+            interface_methods_dict = {}
+            for name, method in interface_methods.iteritems():
+                interface_methods_dict[name] = IterateOver(name, method)
+            ##########
+            dictionary_for_type_call.update(interface_methods_dict)
+        # Get the methods that are defined in the scope of the composite class and override any previous definition
+        ##########
+        # python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)}
+        cls_method = {}
+        for name, method in inspect.getmembers(cls, predicate=inspect.ismethod):
+            cls_method[name] = method
+        ##########
+        dictionary_for_type_call.update(cls_method)
+        # Generate the new class on the fly and return it
+        # FIXME : inherit from interface if we start to use ABC classes?
+        wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call)
+        return wrapper_class
+
+    return cls_decorator