mirror : deals correctly with variants that optionally enable resources (if they are archive URLs)

2 files changed, 72 insertions, 28 deletions

diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 306c8085aa..1c38d79164 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -26,7 +26,7 @@
 This file contains code for creating spack mirror directories.  A
 mirror is an organized hierarchy containing specially named archive
 files.  This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down.  Or, if the computer
+the main server for a particular package is down.  Or, if the computer
 where spack is run is not connected to the internet, it allows spack
 to download packages directly from a mirror (e.g., on an intranet).
 """
@@ -42,7 +42,7 @@ import spack.fetch_strategy as fs
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
 
 
 def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
+                s.variants = spec.variants.copy()
                 matching.append(s)
 
     return matching
 
 
+def suggest_archive_basename(resource):
+    """
+    Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+    :param fetcher:
+    :return:
+    """
+    basename = os.path.basename(resource.fetcher.url)
+    if not allowed_archive(basename):
+        raise RuntimeError("%s is not an allowed archive tye" % basename)
+    return basename
+
+
+
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
 
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
-         * present:  Package specs that were already prsent.
+         * present:  Package specs that were already present.
          * mirrored: Package specs that were successfully mirrored.
          * error:    Package specs that failed to mirror due to some error.
 
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
     error    = []
 
     # Iterate through packages and download all the safe tarballs for each of them
+    everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
 
@@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
 
             if os.path.exists(archive_path):
                 tty.msg("Already added %s" % spec.format("$_$@"))
+            else:
+                everything_already_exists = False
+                # Set up a stage and a fetcher for the download
+                unique_fetch_name = spec.format("$_$@")
+                fetcher = fs.for_package_version(pkg, pkg.version)
+                stage = Stage(fetcher, name=unique_fetch_name)
+                fetcher.set_stage(stage)
+
+                # Do the fetch and checksum if necessary
+                fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    fetcher.check()
+                    tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+                # Fetchers have to know how to archive their files.  Use
+                # that to move/copy/create an archive in the mirror.
+                fetcher.archive(archive_path)
+                tty.msg("Added %s." % spec.format("$_$@"))
+
+            # Fetch resources if they are associated with the spec
+            resources = pkg._get_resources()
+            for resource in resources:
+                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+                if os.path.exists(resource_archive_path):
+                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                    continue
+                everything_already_exists = False
+                resource_stage_folder = pkg._resource_stage(resource)
+                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+                resource.fetcher.set_stage(resource_stage)
+                resource.fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    resource.fetcher.check()
+                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+                resource.fetcher.archive(resource_archive_path)
+                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+            if everything_already_exists:
                 present.append(spec)
-                continue
-
-            # Set up a stage and a fetcher for the download
-            unique_fetch_name = spec.format("$_$@")
-            fetcher = fs.for_package_version(pkg, pkg.version)
-            stage = Stage(fetcher, name=unique_fetch_name)
-            fetcher.set_stage(stage)
-
-            # Do the fetch and checksum if necessary
-            fetcher.fetch()
-            if not kwargs.get('no_checksum', False):
-                fetcher.check()
-                tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
-            # Fetchers have to know how to archive their files.  Use
-            # that to move/copy/create an archive in the mirror.
-            fetcher.archive(archive_path)
-            tty.msg("Added %s." % spec.format("$_$@"))
-            mirrored.append(spec)
+            else:
+                mirrored.append(spec)
 
         except Exception, e:
             if spack.debug:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 3f48231c75..dcb514af00 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -644,12 +644,14 @@ class Package(object):
         # Fetch resources
         resources = self._get_resources()
         for resource in resources:
-            pieces = ['resource', resource.name, self.spec.dag_hash()]
-            resource_stage_folder = '-'.join(pieces)
-            stage = Stage(resource.fetcher, name=resource_stage_folder)
-            resource.fetcher.set_stage(stage)
-            resource.fetcher.fetch()
-            resource.fetcher.check()
+            resource_stage_folder = self._resource_stage(resource)
+            # FIXME : works only for URLFetchStrategy
+            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+            resource.fetcher.set_stage(resource_stage)
+            # Delegate to stage object to trigger mirror logic
+            resource_stage.fetch()
+            resource_stage.check()
         ##########
 
         self._fetch_time = time.time() - start_time
@@ -766,6 +768,11 @@ class Package(object):
                 resources.extend(resource_list)
         return resources
 
+    def _resource_stage(self, resource):
+        pieces = ['resource', resource.name, self.spec.dag_hash()]
+        resource_stage_folder = '-'.join(pieces)
+        return resource_stage_folder
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""