New spack mirror command, configuration.

- Mirrors now appear in ~/.spackconfig, can be edited in that file.
- spack mirror command allows adding, listing, removing mirrors
	- Also still creates mirror directories.

6 files changed, 184 insertions, 49 deletions

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index c4a2de0b2a..904c6262c8 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -138,24 +138,6 @@ do_checksum = True
 sys_type = None
 
 #
-# Places to download tarballs from.
-#
-# TODO: move to configuration.
-#
-# Examples:
-#
-# For a local directory:
-#   mirrors = ['file:///Users/gamblin2/spack-mirror']
-#
-# For a website:
-#   mirrors = ['http://spackports.org/spack-mirror/']
-#
-# For no mirrors:
-#   mirrors = []
-#
-mirrors = []
-
-#
 # Extra imports that should be generally usable from package.py files.
 #
 from llnl.util.filesystem import working_dir
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 94e40557ff..494af3f718 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -25,69 +25,188 @@
 import os
 import shutil
 import argparse
+from datetime import datetime
+from contextlib import closing
 
 import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
 from llnl.util.filesystem import mkdirp, join_path
 
 import spack
 import spack.cmd
+import spack.config
+from spack.spec import Spec
+from spack.error import SpackError
 from spack.stage import Stage
+from spack.util.compression import extension
 
 
-description = "Create a directory full of package tarballs that can be used as a spack mirror."
+description = "Manage spack mirrors."
 
 def setup_parser(subparser):
-    subparser.add_argument(
-        'directory', help="Directory in which to create mirror.")
-    subparser.add_argument(
-        'packages', nargs=argparse.REMAINDER, help="names of packages to put in mirror")
-
-
-def mirror(parser, args):
-    if not args.packages:
-        args.packages = [p for p in spack.db.all_package_names()]
-
+    sp = subparser.add_subparsers(
+        metavar='SUBCOMMAND', dest='mirror_command')
+
+    create_parser = sp.add_parser('create', help=mirror_create.__doc__)
+    create_parser.add_argument('-d', '--directory', default=None,
+                               help="Directory in which to create mirror.")
+    create_parser.add_argument(
+        'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
+    create_parser.add_argument(
+        '-f', '--file', help="File with specs of packages to put in mirror.")
+
+    add_parser = sp.add_parser('add', help=mirror_add.__doc__)
+    add_parser.add_argument('name', help="Mnemonic name for mirror.")
+    add_parser.add_argument(
+        'url', help="URL of mirror directory created by 'spack mirror create'.")
+
+    remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__)
+    remove_parser.add_argument('name')
+
+    list_parser = sp.add_parser('list', help=mirror_list.__doc__)
+
+
+def mirror_add(args):
+    """Add a mirror to Spack."""
+    config = spack.config.get_config('user')
+    config.set_value('mirror', args.name, 'url', args.url)
+    config.write()
+
+
+def mirror_remove(args):
+    """Remove a mirror by name."""
+    config = spack.config.get_config('user')
+    name = args.name
+
+    if not config.has_named_section('mirror', name):
+        tty.die("No such mirror: %s" % name)
+    config.remove_named_section('mirror', name)
+    config.write()
+
+
+def mirror_list(args):
+    """Print out available mirrors to the console."""
+    config = spack.config.get_config()
+    sec_names = config.get_section_names('mirror')
+
+    if not sec_names:
+        tty.msg("No mirrors configured.")
+        return
+
+    max_len = max(len(s) for s in sec_names)
+    fmt = "%%-%ds%%s" % (max_len + 4)
+
+    for name in sec_names:
+        val = config.get_value('mirror', name, 'url')
+        print fmt % (name, val)
+
+
+def mirror_create(args):
+    """Create a directory to be used as a spack mirror, and fill it with
+       package archives."""
+    # try to parse specs from the command line first.
+    args.specs = spack.cmd.parse_specs(args.specs)
+
+    # If there is a file, parse each line as a spec and add it to the list.
+    if args.file:
+        with closing(open(args.file, "r")) as stream:
+            for i, string in enumerate(stream):
+                try:
+                    s = Spec(string)
+                    s.package
+                    args.specs.append(s)
+                except SpackError, e:
+                    tty.die("Parse error in %s, line %d:" % (args.file, i+1),
+                            ">>> " + string, str(e))
+
+    if not args.specs:
+        args.specs = spack.db.all_package_names()
+
+    # Default name for directory is spack-mirror-<DATESTAMP>
+    if not args.directory:
+        timestamp = datetime.now().strftime("%Y-%m-%d")
+        args.directory = 'spack-mirror-' + timestamp
+
+    # Make sure nothing is in the way.
     if os.path.isfile(args.directory):
         tty.error("%s already exists and is a file." % args.directory)
 
+    # Create a directory if none exists
     if not os.path.isdir(args.directory):
         mkdirp(args.directory)
+        tty.msg("Created new mirror in %s" % args.directory)
+    else:
+        tty.msg("Adding to existing mirror in %s" % args.directory)
 
-    # save working directory
+    # Things to keep track of while parsing specs.
     working_dir = os.getcwd()
+    num_mirrored = 0
+    num_error = 0
 
     # Iterate through packages and download all the safe tarballs for each of them
-    for pkg_name in args.packages:
-        pkg = spack.db.get(pkg_name)
+    for spec in args.specs:
+        pkg = spec.package
 
         # Skip any package that has no checksummed versions.
         if not pkg.versions:
-            tty.msg("No safe (checksummed) versions for package %s.  Skipping."
-                    % pkg_name)
+            tty.msg("No safe (checksummed) versions for package %s."
+                    % pkg.name)
             continue
 
         # create a subdir for the current package.
-        pkg_path = join_path(args.directory, pkg_name)
+        pkg_path = join_path(args.directory, pkg.name)
         mkdirp(pkg_path)
 
         # Download all the tarballs using Stages, then move them into place
         for version in pkg.versions:
+            # Skip versions that don't match the spec
+            vspec = Spec('%s@%s' % (pkg.name, version))
+            if not vspec.satisfies(spec):
+                continue
+
+            mirror_path = "%s/%s-%s.%s" % (
+                pkg.name, pkg.name, version, extension(pkg.url))
+
+            os.chdir(working_dir)
+            mirror_file = join_path(args.directory, mirror_path)
+            if os.path.exists(mirror_file):
+                tty.msg("Already fetched %s. Skipping." % mirror_file)
+                num_mirrored += 1
+                continue
+
+            # Get the URL for the version and set up a stage to download it.
             url = pkg.url_for_version(version)
             stage = Stage(url)
             try:
+                # fetch changes directory into the stage
                 stage.fetch()
-                basename = os.path.basename(stage.archive_file)
-                final_dst = join_path(pkg_path, basename)
 
+                # change back and move the new archive into place.
                 os.chdir(working_dir)
-                shutil.move(stage.archive_file, final_dst)
-                tty.msg("Added %s to mirror" % final_dst)
+                shutil.move(stage.archive_file, mirror_file)
+                tty.msg("Added %s to mirror" % mirror_file)
+                num_mirrored += 1
 
             except Exception, e:
                  tty.warn("Error while fetching %s.  Skipping." % url, e.message)
+                 num_error += 1
 
             finally:
                 stage.destroy()
 
-    # Success!
-    tty.msg("Created Spack mirror in %s" % args.directory)
+    # If nothing happened, try to say why.
+    if not num_mirrored:
+        if num_error:
+            tty.warn("No packages added to mirror.",
+                     "All packages failed to fetch.")
+        else:
+            tty.warn("No packages added to mirror. No versions matched specs:")
+            colify(args.specs, indent=4)
+
+
+def mirror(parser, args):
+    action = { 'create' : mirror_create,
+               'add'    : mirror_add,
+               'remove' : mirror_remove,
+               'list'   : mirror_list }
+    action[args.mirror_command](args)
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 00ff4313a2..5494adc324 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -225,8 +225,7 @@ class SpackConfigParser(cp.RawConfigParser):
     OPTCRE_NV = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE_NV.pattern)
 
     def __init__(self, file_or_files):
-        cp.RawConfigParser.__init__(
-            self, dict_type=OrderedDict, allow_no_value=True)
+        cp.RawConfigParser.__init__(self, dict_type=OrderedDict)
 
         if isinstance(file_or_files, basestring):
             self.read([file_or_files])
@@ -286,6 +285,16 @@ class SpackConfigParser(cp.RawConfigParser):
         return self.has_option(sn, option)
 
 
+    def has_named_section(self, section, name):
+        sn = _make_section_name(section, name)
+        return self.has_section(sn)
+
+
+    def remove_named_section(self, section, name):
+        sn = _make_section_name(section, name)
+        self.remove_section(sn)
+
+
     def get_section_names(self, sectype):
         """Get all named sections with the specified type.
            A named section looks like this:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index ef8b639adf..eb0625cb7e 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -54,7 +54,7 @@ import spack.util.crypto as crypto
 from spack.version import *
 from spack.stage import Stage
 from spack.util.web import get_pages
-from spack.util.compression import allowed_archive
+from spack.util.compression import allowed_archive, extension
 
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file"]
@@ -399,7 +399,9 @@ class Package(object):
             raise ValueError("Can only get a stage for a concrete package.")
 
         if self._stage is None:
-            mirror_path = "%s/%s" % (self.name, os.path.basename(self.url))
+            # TODO: move this logic into a mirror module.
+            mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % (
+                self.name, self.version, extension(self.url)))
             self._stage = Stage(
                 self.url, mirror_path=mirror_path, name=self.spec.short_spec)
         return self._stage
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 8574df71e9..f679cec282 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -31,6 +31,7 @@ import llnl.util.tty as tty
 from llnl.util.filesystem import *
 
 import spack
+import spack.config
 import spack.error as serr
 from spack.util.compression import decompressor_for
 
@@ -185,9 +186,10 @@ class Stage(object):
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""
-        path = os.path.join(self.path, os.path.basename(self.url))
-        if os.path.exists(path):
-            return path
+        for path in (os.path.join(self.path, os.path.basename(self.url)),
+                     os.path.join(self.path, os.path.basename(self.mirror_path))):
+            if os.path.exists(path):
+                return path
         return None
 
 
@@ -247,6 +249,7 @@ class Stage(object):
                      "'spack clean --dist' to remove the bad archive, then fix",
                      "your internet gateway issue and install again.")
 
+
     def fetch(self):
         """Downloads the file at URL to the stage.  Returns true if it was downloaded,
            false if it already existed."""
@@ -257,7 +260,7 @@ class Stage(object):
         else:
             urls = [self.url]
             if self.mirror_path:
-                urls = ["%s/%s" % (m, self.mirror_path) for m in spack.mirrors] + urls
+                urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls
 
             for url in urls:
                 tty.msg("Trying to fetch from %s" % url)
@@ -320,6 +323,17 @@ class Stage(object):
             os.chdir(os.path.dirname(self.path))
 
 
+def _get_mirrors():
+    """Get mirrors from spack configuration."""
+    config = spack.config.get_config()
+
+    mirrors = []
+    sec_names = config.get_section_names('mirror')
+    for name in sec_names:
+        mirrors.append(config.get_value('mirror', name, 'url'))
+    return mirrors
+
+
 def ensure_access(file=spack.stage_path):
     """Ensure we can access a directory and die with an error if we can't."""
     if not can_access(file):
diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py
index 427a9cf020..7ce8e8c65b 100644
--- a/lib/spack/spack/util/compression.py
+++ b/lib/spack/spack/util/compression.py
@@ -56,3 +56,12 @@ def stem(path):
         if re.search(suffix, path):
             return re.sub(suffix, "", path)
     return path
+
+
+def extension(path):
+    """Get the archive extension for a path."""
+    for type in ALLOWED_ARCHIVE_TYPES:
+        suffix = r'\.%s$' % type
+        if re.search(suffix, path):
+            return type
+    return None