Add test cases for mirroring.

9 files changed, 276 insertions, 116 deletions

diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 6a6c2b60c7..6e2e4dfb24 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -54,6 +54,9 @@ def setup_parser(subparser):
         'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
     create_parser.add_argument(
         '-f', '--file', help="File with specs of packages to put in mirror.")
+    create_parser.add_argument(
+        '-o', '--one-version-per-spec', action='store_const', const=1, default=0,
+        help="Only fetch one 'preferred' version per spec, not all known versions.")
 
     add_parser = sp.add_parser('add', help=mirror_add.__doc__)
     add_parser.add_argument('name', help="Mnemonic name for mirror.")
@@ -128,26 +131,29 @@ def mirror_create(args):
     # If nothing is passed, use all packages.
     if not specs:
         specs = [Spec(n) for n in spack.db.all_package_names()]
+        specs.sort(key=lambda s: s.format("$_$@").lower())
 
     # Default name for directory is spack-mirror-<DATESTAMP>
-    if not args.directory:
+    directory = args.directory
+    if not directory:
         timestamp = datetime.now().strftime("%Y-%m-%d")
-        args.directory = 'spack-mirror-' + timestamp
+        directory = 'spack-mirror-' + timestamp
 
     # Make sure nothing is in the way.
     existed = False
-    if os.path.isfile(args.directory):
-        tty.error("%s already exists and is a file." % args.directory)
-    elif os.path.isdir(args.directory):
+    if os.path.isfile(directory):
+        tty.error("%s already exists and is a file." % directory)
+    elif os.path.isdir(directory):
         existed = True
 
     # Actually do the work to create the mirror
-    present, mirrored, error = spack.mirror.create(args.directory, specs)
+    present, mirrored, error = spack.mirror.create(
+        directory, specs, num_versions=args.one_version_per_spec)
     p, m, e = len(present), len(mirrored), len(error)
 
     verb = "updated" if existed else "created"
     tty.msg(
-        "Successfully %s mirror in %s." % (verb, args.directory),
+        "Successfully %s mirror in %s." % (verb, directory),
         "Archive stats:",
         "  %-4d already present"  % p,
         "  %-4d added"            % m,
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 9f9cd1789d..e603806af9 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -73,7 +73,6 @@ class DefaultConcretizer(object):
         if valid_versions:
             spec.versions = ver([valid_versions[-1]])
         else:
-            print spec
             raise NoValidVersionError(spec)
 
 
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 2cff15845b..2b574eaba7 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -93,9 +93,6 @@ class FetchStrategy(object):
     def __str__(self):       # Should be human readable URL.
         return "FetchStrategy.__str___"
 
-    @property
-    def unique_name(self): pass
-
     # This method is used to match fetch strategies to version()
     # arguments in packages.
     @classmethod
@@ -197,7 +194,10 @@ class URLFetchStrategy(FetchStrategy):
         """Just moves this archive to the destination."""
         if not self.archive_file:
             raise NoArchiveFileError("Cannot call archive() before fetching.")
-        assert(extension(destination) == extension(self.archive_file))
+
+        if not extension(destination) == extension(self.archive_file):
+            raise ValueError("Cannot archive without matching extensions.")
+
         shutil.move(self.archive_file, destination)
 
 
@@ -236,10 +236,6 @@ class URLFetchStrategy(FetchStrategy):
         else:
             return "URLFetchStrategy<no url>"
 
-    @property
-    def unique_name(self):
-        return "spack-fetch-url:%s" % self
-
 
 class VCSFetchStrategy(FetchStrategy):
     def __init__(self, name, *rev_types, **kwargs):
@@ -393,17 +389,6 @@ class GitFetchStrategy(VCSFetchStrategy):
         self.git('clean', '-f')
 
 
-    @property
-    def unique_name(self):
-        name = "spack-fetch-git:%s" % self.url
-        if self.commit:
-            name += "@" + self.commit
-        elif self.branch:
-            name += "@" + self.branch
-        elif self.tag:
-            name += "@" + self.tag
-
-
 class SvnFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a subversion repository.
        Use like this in a package:
@@ -477,14 +462,6 @@ class SvnFetchStrategy(VCSFetchStrategy):
         self.svn('revert', '.', '-R')
 
 
-    @property
-    def unique_name(self):
-        name = "spack-fetch-svn:%s" % self.url
-        if self.revision:
-            name += "@" + self.revision
-
-
-
 class HgFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a Mercurial repository.
        Use like this in a package:
@@ -560,14 +537,6 @@ class HgFetchStrategy(VCSFetchStrategy):
         self.stage.chdir_to_source()
 
 
-    @property
-    def unique_name(self):
-        name = "spack-fetch-hg:%s" % self.url
-        if self.revision:
-            name += "@" + self.revision
-
-
-
 def from_url(url):
     """Given a URL, find an appropriate fetch strategy for it.
        Currently just gives you a URLFetchStrategy that uses curl.
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index e116ce83b2..f7bbb3f840 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -49,16 +49,18 @@ def mirror_archive_filename(spec):
     if not spec.version.concrete:
         raise ValueError("mirror.path requires spec with concrete version.")
 
-    url = spec.package.default_url
-    if url is None:
-        ext = 'tar.gz'
+    fetcher = spec.package.fetcher
+    if isinstance(fetcher, fs.URLFetchStrategy):
+        # If we fetch this version with a URLFetchStrategy, use URL's archive type
+        ext = extension(fetcher.url)
     else:
-        ext = extension(url)
+        # Otherwise we'll make a .tar.gz ourselves
+        ext = 'tar.gz'
 
     return "%s-%s.%s" % (spec.package.name, spec.version, ext)
 
 
-def get_matching_versions(specs):
+def get_matching_versions(specs, **kwargs):
     """Get a spec for EACH known version matching any spec in the list."""
     matching = []
     for spec in specs:
@@ -69,11 +71,18 @@ def get_matching_versions(specs):
             tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
             continue
 
-        for v in reversed(sorted(pkg.versions)):
+        num_versions = kwargs.get('num_versions', 0)
+        for i, v in enumerate(reversed(sorted(pkg.versions))):
+            # Generate no more than num_versions versions for each spec.
+            if num_versions and i >= num_versions:
+                break
+
+            # Generate only versions that satisfy the spec.
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
                 matching.append(s)
+
     return matching
 
 
@@ -86,6 +95,11 @@ def create(path, specs, **kwargs):
          specs   Any package versions matching these specs will be added
                  to the mirror.
 
+       Keyword args:
+         no_checksum:  If True, do not checkpoint when fetching (default False)
+         num_versions: Max number of versions to fetch per spec,
+                       if spec is ambiguous (default is 0 for all of them)
+
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
          * present:  Package specs that were already prsent.
@@ -104,13 +118,15 @@ def create(path, specs, **kwargs):
     specs = [s if isinstance(s, Spec) else Spec(s) for s in specs]
 
     # Get concrete specs for each matching version of these specs.
-    version_specs = get_matching_versions(specs)
+    version_specs = get_matching_versions(
+        specs, num_versions=kwargs.get('num_versions', 0))
     for s in version_specs:
         s.concretize()
 
-    # Create a directory if none exists
-    if not os.path.isdir(path):
-        mkdirp(path)
+    # Get the absolute path of the root before we start jumping around.
+    mirror_root = os.path.abspath(path)
+    if not os.path.isdir(mirror_root):
+        mkdirp(mirror_root)
 
     # Things to keep track of while parsing specs.
     present  = []
@@ -124,19 +140,21 @@ def create(path, specs, **kwargs):
         stage = None
         try:
             # create a subdirectory for the current package@version
-            realpath = os.path.realpath(path)
-            subdir = join_path(realpath, pkg.name)
+            subdir = join_path(mirror_root, pkg.name)
             mkdirp(subdir)
 
             archive_file = mirror_archive_filename(spec)
             archive_path = join_path(subdir, archive_file)
-            if os.path.exists(archive_path):
+
+            if os.path.exists(archive_file):
+                tty.msg("%s is already present.  Skipping." % spec.format("$_$@"))
                 present.append(spec)
                 continue
 
             # Set up a stage and a fetcher for the download
+            unique_fetch_name = spec.format("$_$@")
             fetcher = fs.for_package_version(pkg, pkg.version)
-            stage = Stage(fetcher, name=fetcher.unique_name)
+            stage = Stage(fetcher, name=unique_fetch_name)
             fetcher.set_stage(stage)
 
             # Do the fetch and checksum if necessary
@@ -148,7 +166,7 @@ def create(path, specs, **kwargs):
             # Fetchers have to know how to archive their files.  Use
             # that to move/copy/create an archive in the mirror.
             fetcher.archive(archive_path)
-            tty.msg("Added %s to mirror" % archive_path)
+            tty.msg("Added %s." % spec.format("$_$@"))
             mirrored.append(spec)
 
         except Exception, e:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 34efe1bec9..e462562e85 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -598,18 +598,6 @@ class Package(object):
         return str(version)
 
 
-    @property
-    def default_url(self):
-        if self.spec.versions.concrete:
-            return self.url_for_version(self.version)
-        else:
-            url = getattr(self, 'url', None)
-            if url:
-                return url
-
-        return None
-
-
     def remove_prefix(self):
         """Removes the prefix for a package along with any empty parent directories."""
         spack.install_layout.remove_path_for_spec(self.spec)
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index ca4c869e42..be9ac5a560 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -50,7 +50,8 @@ test_names = ['versions',
               'python_version',
               'git_fetch',
               'svn_fetch',
-              'hg_fetch']
+              'hg_fetch',
+              'mirror']
 
 
 def list_tests():
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index 0d53bb45c7..e052f53e77 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -36,39 +36,17 @@ from spack.fetch_strategy import URLFetchStrategy
 from spack.directory_layout import SpecHashDirectoryLayout
 from spack.util.executable import which
 from spack.test.mock_packages_test import *
+from spack.test.mock_repo import MockArchive
 
 
-dir_name = 'trivial-1.0'
-archive_name = 'trivial-1.0.tar.gz'
-install_test_package = 'trivial_install_test_package'
-
 class InstallTest(MockPackagesTest):
     """Tests install and uninstall on a trivial package."""
 
     def setUp(self):
         super(InstallTest, self).setUp()
 
-        self.stage = Stage('not_a_real_url')
-        archive_dir = join_path(self.stage.path, dir_name)
-        dummy_configure = join_path(archive_dir, 'configure')
-
-        mkdirp(archive_dir)
-        with closing(open(dummy_configure, 'w')) as configure:
-            configure.write(
-                "#!/bin/sh\n"
-                "prefix=$(echo $1 | sed 's/--prefix=//')\n"
-                "cat > Makefile <<EOF\n"
-                "all:\n"
-                "\techo Building...\n\n"
-                "install:\n"
-                "\tmkdir -p $prefix\n"
-                "\ttouch $prefix/dummy_file\n"
-                "EOF\n")
-        os.chmod(dummy_configure, 0755)
-
-        with working_dir(self.stage.path):
-            tar = which('tar')
-            tar('-czf', archive_name, dir_name)
+        # create a simple installable package directory and tarball
+        self.repo = MockArchive()
 
         # We use a fake package, so skip the checksum.
         spack.do_checksum = False
@@ -83,8 +61,8 @@ class InstallTest(MockPackagesTest):
     def tearDown(self):
         super(InstallTest, self).tearDown()
 
-        if self.stage is not None:
-            self.stage.destroy()
+        if self.repo.stage is not None:
+            self.repo.stage.destroy()
 
         # Turn checksumming back on
         spack.do_checksum = True
@@ -96,7 +74,7 @@ class InstallTest(MockPackagesTest):
 
     def test_install_and_uninstall(self):
         # Get a basic concrete spec for the trivial install package.
-        spec = Spec(install_test_package)
+        spec = Spec('trivial_install_test_package')
         spec.concretize()
         self.assertTrue(spec.concrete)
 
@@ -104,8 +82,7 @@ class InstallTest(MockPackagesTest):
         pkg = spack.db.get(spec)
 
         # Fake the URL for the package so it downloads from a file.
-        archive_path = join_path(self.stage.path, archive_name)
-        pkg.fetcher = URLFetchStrategy('file://' + archive_path)
+        pkg.fetcher = URLFetchStrategy(self.repo.url)
 
         try:
             pkg.do_install()
diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py
new file mode 100644
index 0000000000..51334198ec
--- /dev/null
+++ b/lib/spack/spack/test/mirror.py
@@ -0,0 +1,156 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+from filecmp import dircmp
+
+import spack
+import spack.mirror
+from spack.util.compression import decompressor_for
+from spack.test.mock_packages_test import *
+from spack.test.mock_repo import *
+
+# paths in repos that shouldn't be in the mirror tarballs.
+exclude = ['.hg', '.git', '.svn']
+
+
+class MirrorTest(MockPackagesTest):
+    def setUp(self):
+        """Sets up a mock package and a mock repo for each fetch strategy, to
+           ensure that the mirror can create archives for each of them.
+        """
+        super(MirrorTest, self).setUp()
+        self.repos = {}
+
+
+    def set_up_package(self, name, mock_repo_class, url_attr):
+        """Use this to set up a mock package to be mirrored.
+           Each package needs us to:
+             1. Set up a mock repo/archive to fetch from.
+             2. Point the package's version args at that repo.
+        """
+        # Set up packages to point at mock repos.
+        spec = Spec(name)
+        spec.concretize()
+
+        # Get the package and fix its fetch args to point to a mock repo
+        pkg = spack.db.get(spec)
+        repo = mock_repo_class()
+        self.repos[name] = repo
+
+        # change the fetch args of the first (only) version.
+        assert(len(pkg.versions) == 1)
+        v = next(iter(pkg.versions))
+        pkg.versions[v][url_attr] = repo.url
+
+
+    def tearDown(self):
+        """Destroy all the stages created by the repos in setup."""
+        super(MirrorTest, self).tearDown()
+
+        for name, repo in self.repos.items():
+            if repo.stage:
+                repo.stage.destroy()
+
+        self.repos.clear()
+
+
+    def check_mirror(self):
+        stage = Stage('spack-mirror-test')
+        mirror_root = join_path(stage.path, 'test-mirror')
+
+        try:
+            os.chdir(stage.path)
+            spack.mirror.create(
+                mirror_root, self.repos, no_checksum=True)
+
+            # Stage directory exists
+            self.assertTrue(os.path.isdir(mirror_root))
+
+            # subdirs for each package
+            for name in self.repos:
+                subdir = join_path(mirror_root, name)
+                self.assertTrue(os.path.isdir(subdir))
+
+                files = os.listdir(subdir)
+                self.assertEqual(len(files), 1)
+
+                # Decompress archive in the mirror
+                archive = files[0]
+                archive_path = join_path(subdir, archive)
+                decomp = decompressor_for(archive_path)
+
+                with working_dir(subdir):
+                    decomp(archive_path)
+
+                    # Find the untarred archive directory.
+                    files = os.listdir(subdir)
+                    self.assertEqual(len(files), 2)
+                    self.assertTrue(archive in files)
+                    files.remove(archive)
+
+                    expanded_archive = join_path(subdir, files[0])
+                    self.assertTrue(os.path.isdir(expanded_archive))
+
+                    # Compare the original repo with the expanded archive
+                    repo = self.repos[name]
+                    if not 'svn' in name:
+                        original_path = repo.path
+                    else:
+                        co = 'checked_out'
+                        svn('checkout', repo.url, co)
+                        original_path = join_path(subdir, co)
+
+                    dcmp = dircmp(original_path, expanded_archive)
+
+                    # make sure there are no new files in the expanded tarball
+                    self.assertFalse(dcmp.right_only)
+                    self.assertTrue(all(l in exclude for l in dcmp.left_only))
+
+        finally:
+            stage.destroy()
+
+
+    def test_git_mirror(self):
+        self.set_up_package('git-test', MockGitRepo, 'git')
+        self.check_mirror()
+
+    def test_svn_mirror(self):
+        self.set_up_package('svn-test', MockSvnRepo, 'svn')
+        self.check_mirror()
+
+    def test_hg_mirror(self):
+        self.set_up_package('hg-test', MockHgRepo, 'hg')
+        self.check_mirror()
+
+    def test_url_mirror(self):
+        self.set_up_package('trivial_install_test_package', MockArchive, 'url')
+        self.check_mirror()
+
+    def test_all_mirror(self):
+        self.set_up_package('git-test', MockGitRepo, 'git')
+        self.set_up_package('svn-test', MockSvnRepo, 'svn')
+        self.set_up_package('hg-test',  MockHgRepo,  'hg')
+        self.set_up_package('trivial_install_test_package', MockArchive, 'url')
+        self.check_mirror()
diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py
index ae28f7224e..659f29067a 100644
--- a/lib/spack/spack/test/mock_repo.py
+++ b/lib/spack/spack/test/mock_repo.py
@@ -22,7 +22,9 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import os
 import shutil
+from contextlib import closing
 
 from llnl.util.filesystem import *
 
@@ -32,31 +34,71 @@ from spack.stage import Stage
 from spack.util.executable import which
 
 
+#
+# VCS Systems used by mock repo code.
+#
+git      = which('git',      required=True)
+svn      = which('svn',      required=True)
+svnadmin = which('svnadmin', required=True)
+hg       = which('hg',       required=True)
+tar      = which('tar',      required=True)
+
+
 class MockRepo(object):
     def __init__(self, stage_name, repo_name):
-        """This creates a stage and a repo directory within the stage."""
+        """This creates a stage where some archive/repo files can be staged
+           for testing spack's fetch strategies."""
         # Stage where this repo has been created
         self.stage = Stage(stage_name)
 
         # Full path to the repo within the stage.
-        self.path = join_path(self.stage.path, 'mock-git-repo')
+        self.path = join_path(self.stage.path, repo_name)
         mkdirp(self.path)
 
+
+class MockArchive(MockRepo):
+    """Creates a very simple archive directory with a configure script and a
+       makefile that installs to a prefix.  Tars it up into an archive."""
+
+    def __init__(self):
+        repo_name = 'mock-archive-repo'
+        super(MockArchive, self).__init__('mock-archive-stage', repo_name)
+
+        with working_dir(self.path):
+            configure = join_path(self.path, 'configure')
+
+            with closing(open(configure, 'w')) as cfg_file:
+                cfg_file.write(
+                    "#!/bin/sh\n"
+                    "prefix=$(echo $1 | sed 's/--prefix=//')\n"
+                    "cat > Makefile <<EOF\n"
+                    "all:\n"
+                    "\techo Building...\n\n"
+                    "install:\n"
+                    "\tmkdir -p $prefix\n"
+                    "\ttouch $prefix/dummy_file\n"
+                    "EOF\n")
+            os.chmod(configure, 0755)
+
+        with working_dir(self.stage.path):
+            archive_name = "%s.tar.gz" % repo_name
+            tar('-czf', archive_name, repo_name)
+
+        self.archive_path = join_path(self.stage.path, archive_name)
+        self.url = 'file://' + self.archive_path
+
+
+class MockVCSRepo(MockRepo):
+    def __init__(self, stage_name, repo_name):
+        """This creates a stage and a repo directory within the stage."""
+        super(MockVCSRepo, self).__init__(stage_name, repo_name)
+
         # Name for rev0 & rev1 files in the repo to be
         self.r0_file = 'r0_file'
         self.r1_file = 'r1_file'
 
 
-#
-# VCS Systems used by mock repo code.
-#
-git      = which('git',      required=True)
-svn      = which('svn',      required=True)
-svnadmin = which('svnadmin', required=True)
-hg       = which('hg',       required=True)
-
-
-class MockGitRepo(MockRepo):
+class MockGitRepo(MockVCSRepo):
     def __init__(self):
         super(MockGitRepo, self).__init__('mock-git-stage', 'mock-git-repo')
 
@@ -97,17 +139,20 @@ class MockGitRepo(MockRepo):
             self.r1      = self.rev_hash(self.branch)
             self.r1_file = self.branch_file
 
+            self.url = self.path
+
     def rev_hash(self, rev):
         return git('rev-parse', rev, return_output=True).strip()
 
 
-class MockSvnRepo(MockRepo):
+class MockSvnRepo(MockVCSRepo):
     def __init__(self):
         super(MockSvnRepo, self).__init__('mock-svn-stage', 'mock-svn-repo')
 
+        self.url = 'file://' + self.path
+
         with working_dir(self.stage.path):
             svnadmin('create', self.path)
-            self.url = 'file://' + self.path
 
             tmp_path = join_path(self.stage.path, 'tmp-path')
             mkdirp(tmp_path)
@@ -129,9 +174,10 @@ class MockSvnRepo(MockRepo):
             self.r1 = '2'
 
 
-class MockHgRepo(MockRepo):
+class MockHgRepo(MockVCSRepo):
     def __init__(self):
         super(MockHgRepo, self).__init__('mock-hg-stage', 'mock-hg-repo')
+        self.url = 'file://' + self.path
 
         with working_dir(self.path):
             hg('init')