summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2014-06-24 11:15:41 -0700
committerTodd Gamblin <tgamblin@llnl.gov>2014-06-24 11:15:41 -0700
commitb33412e03a57783202cc274c48aa02b787adfbf5 (patch)
treeed7205c5a5950ef03945751308faec89d35f1db0 /lib
parent9f8ef85644883a53578fb000827cb937a16d058e (diff)
downloadspack-b33412e03a57783202cc274c48aa02b787adfbf5.tar.gz
spack-b33412e03a57783202cc274c48aa02b787adfbf5.tar.bz2
spack-b33412e03a57783202cc274c48aa02b787adfbf5.tar.xz
spack-b33412e03a57783202cc274c48aa02b787adfbf5.zip
New spack mirror command, configuration.
- Mirrors now appear in ~/.spackconfig, can be edited in that file. - spack mirror command allows adding, listing, removing mirrors - Also still creates mirror directories.
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/__init__.py18
-rw-r--r--lib/spack/spack/cmd/mirror.py165
-rw-r--r--lib/spack/spack/config.py13
-rw-r--r--lib/spack/spack/package.py6
-rw-r--r--lib/spack/spack/stage.py22
-rw-r--r--lib/spack/spack/util/compression.py9
6 files changed, 184 insertions, 49 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index c4a2de0b2a..904c6262c8 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -138,24 +138,6 @@ do_checksum = True
sys_type = None
#
-# Places to download tarballs from.
-#
-# TODO: move to configuration.
-#
-# Examples:
-#
-# For a local directory:
-# mirrors = ['file:///Users/gamblin2/spack-mirror']
-#
-# For a website:
-# mirrors = ['http://spackports.org/spack-mirror/']
-#
-# For no mirrors:
-# mirrors = []
-#
-mirrors = []
-
-#
# Extra imports that should be generally usable from package.py files.
#
from llnl.util.filesystem import working_dir
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 94e40557ff..494af3f718 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -25,69 +25,188 @@
import os
import shutil
import argparse
+from datetime import datetime
+from contextlib import closing
import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
from llnl.util.filesystem import mkdirp, join_path
import spack
import spack.cmd
+import spack.config
+from spack.spec import Spec
+from spack.error import SpackError
from spack.stage import Stage
+from spack.util.compression import extension
-description = "Create a directory full of package tarballs that can be used as a spack mirror."
+description = "Manage spack mirrors."
def setup_parser(subparser):
- subparser.add_argument(
- 'directory', help="Directory in which to create mirror.")
- subparser.add_argument(
- 'packages', nargs=argparse.REMAINDER, help="names of packages to put in mirror")
-
-
-def mirror(parser, args):
- if not args.packages:
- args.packages = [p for p in spack.db.all_package_names()]
-
+ sp = subparser.add_subparsers(
+ metavar='SUBCOMMAND', dest='mirror_command')
+
+ create_parser = sp.add_parser('create', help=mirror_create.__doc__)
+ create_parser.add_argument('-d', '--directory', default=None,
+ help="Directory in which to create mirror.")
+ create_parser.add_argument(
+ 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
+ create_parser.add_argument(
+ '-f', '--file', help="File with specs of packages to put in mirror.")
+
+ add_parser = sp.add_parser('add', help=mirror_add.__doc__)
+ add_parser.add_argument('name', help="Mnemonic name for mirror.")
+ add_parser.add_argument(
+ 'url', help="URL of mirror directory created by 'spack mirror create'.")
+
+ remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__)
+ remove_parser.add_argument('name')
+
+ list_parser = sp.add_parser('list', help=mirror_list.__doc__)
+
+
+def mirror_add(args):
+ """Add a mirror to Spack."""
+ config = spack.config.get_config('user')
+ config.set_value('mirror', args.name, 'url', args.url)
+ config.write()
+
+
+def mirror_remove(args):
+ """Remove a mirror by name."""
+ config = spack.config.get_config('user')
+ name = args.name
+
+ if not config.has_named_section('mirror', name):
+ tty.die("No such mirror: %s" % name)
+ config.remove_named_section('mirror', name)
+ config.write()
+
+
+def mirror_list(args):
+ """Print out available mirrors to the console."""
+ config = spack.config.get_config()
+ sec_names = config.get_section_names('mirror')
+
+ if not sec_names:
+ tty.msg("No mirrors configured.")
+ return
+
+ max_len = max(len(s) for s in sec_names)
+ fmt = "%%-%ds%%s" % (max_len + 4)
+
+ for name in sec_names:
+ val = config.get_value('mirror', name, 'url')
+ print fmt % (name, val)
+
+
+def mirror_create(args):
+ """Create a directory to be used as a spack mirror, and fill it with
+ package archives."""
+ # try to parse specs from the command line first.
+ args.specs = spack.cmd.parse_specs(args.specs)
+
+ # If there is a file, parse each line as a spec and add it to the list.
+ if args.file:
+ with closing(open(args.file, "r")) as stream:
+ for i, string in enumerate(stream):
+ try:
+ s = Spec(string)
+ s.package
+ args.specs.append(s)
+ except SpackError, e:
+ tty.die("Parse error in %s, line %d:" % (args.file, i+1),
+ ">>> " + string, str(e))
+
+ if not args.specs:
+ args.specs = spack.db.all_package_names()
+
+ # Default name for directory is spack-mirror-<DATESTAMP>
+ if not args.directory:
+ timestamp = datetime.now().strftime("%Y-%m-%d")
+ args.directory = 'spack-mirror-' + timestamp
+
+ # Make sure nothing is in the way.
if os.path.isfile(args.directory):
tty.error("%s already exists and is a file." % args.directory)
+ # Create a directory if none exists
if not os.path.isdir(args.directory):
mkdirp(args.directory)
+ tty.msg("Created new mirror in %s" % args.directory)
+ else:
+ tty.msg("Adding to existing mirror in %s" % args.directory)
- # save working directory
+ # Things to keep track of while parsing specs.
working_dir = os.getcwd()
+ num_mirrored = 0
+ num_error = 0
# Iterate through packages and download all the safe tarballs for each of them
- for pkg_name in args.packages:
- pkg = spack.db.get(pkg_name)
+ for spec in args.specs:
+ pkg = spec.package
# Skip any package that has no checksummed versions.
if not pkg.versions:
- tty.msg("No safe (checksummed) versions for package %s. Skipping."
- % pkg_name)
+ tty.msg("No safe (checksummed) versions for package %s."
+ % pkg.name)
continue
# create a subdir for the current package.
- pkg_path = join_path(args.directory, pkg_name)
+ pkg_path = join_path(args.directory, pkg.name)
mkdirp(pkg_path)
# Download all the tarballs using Stages, then move them into place
for version in pkg.versions:
+ # Skip versions that don't match the spec
+ vspec = Spec('%s@%s' % (pkg.name, version))
+ if not vspec.satisfies(spec):
+ continue
+
+ mirror_path = "%s/%s-%s.%s" % (
+ pkg.name, pkg.name, version, extension(pkg.url))
+
+ os.chdir(working_dir)
+ mirror_file = join_path(args.directory, mirror_path)
+ if os.path.exists(mirror_file):
+ tty.msg("Already fetched %s. Skipping." % mirror_file)
+ num_mirrored += 1
+ continue
+
+ # Get the URL for the version and set up a stage to download it.
url = pkg.url_for_version(version)
stage = Stage(url)
try:
+ # fetch changes directory into the stage
stage.fetch()
- basename = os.path.basename(stage.archive_file)
- final_dst = join_path(pkg_path, basename)
+ # change back and move the new archive into place.
os.chdir(working_dir)
- shutil.move(stage.archive_file, final_dst)
- tty.msg("Added %s to mirror" % final_dst)
+ shutil.move(stage.archive_file, mirror_file)
+ tty.msg("Added %s to mirror" % mirror_file)
+ num_mirrored += 1
except Exception, e:
tty.warn("Error while fetching %s. Skipping." % url, e.message)
+ num_error += 1
finally:
stage.destroy()
- # Success!
- tty.msg("Created Spack mirror in %s" % args.directory)
+ # If nothing happened, try to say why.
+ if not num_mirrored:
+ if num_error:
+ tty.warn("No packages added to mirror.",
+ "All packages failed to fetch.")
+ else:
+ tty.warn("No packages added to mirror. No versions matched specs:")
+ colify(args.specs, indent=4)
+
+
+def mirror(parser, args):
+ action = { 'create' : mirror_create,
+ 'add' : mirror_add,
+ 'remove' : mirror_remove,
+ 'list' : mirror_list }
+ action[args.mirror_command](args)
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 00ff4313a2..5494adc324 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -225,8 +225,7 @@ class SpackConfigParser(cp.RawConfigParser):
OPTCRE_NV = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE_NV.pattern)
def __init__(self, file_or_files):
- cp.RawConfigParser.__init__(
- self, dict_type=OrderedDict, allow_no_value=True)
+ cp.RawConfigParser.__init__(self, dict_type=OrderedDict)
if isinstance(file_or_files, basestring):
self.read([file_or_files])
@@ -286,6 +285,16 @@ class SpackConfigParser(cp.RawConfigParser):
return self.has_option(sn, option)
+ def has_named_section(self, section, name):
+ sn = _make_section_name(section, name)
+ return self.has_section(sn)
+
+
+ def remove_named_section(self, section, name):
+ sn = _make_section_name(section, name)
+ self.remove_section(sn)
+
+
def get_section_names(self, sectype):
"""Get all named sections with the specified type.
A named section looks like this:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index ef8b639adf..eb0625cb7e 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -54,7 +54,7 @@ import spack.util.crypto as crypto
from spack.version import *
from spack.stage import Stage
from spack.util.web import get_pages
-from spack.util.compression import allowed_archive
+from spack.util.compression import allowed_archive, extension
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file"]
@@ -399,7 +399,9 @@ class Package(object):
raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None:
- mirror_path = "%s/%s" % (self.name, os.path.basename(self.url))
+ # TODO: move this logic into a mirror module.
+ mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % (
+ self.name, self.version, extension(self.url)))
self._stage = Stage(
self.url, mirror_path=mirror_path, name=self.spec.short_spec)
return self._stage
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 8574df71e9..f679cec282 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -31,6 +31,7 @@ import llnl.util.tty as tty
from llnl.util.filesystem import *
import spack
+import spack.config
import spack.error as serr
from spack.util.compression import decompressor_for
@@ -185,9 +186,10 @@ class Stage(object):
@property
def archive_file(self):
"""Path to the source archive within this stage directory."""
- path = os.path.join(self.path, os.path.basename(self.url))
- if os.path.exists(path):
- return path
+ for path in (os.path.join(self.path, os.path.basename(self.url)),
+ os.path.join(self.path, os.path.basename(self.mirror_path))):
+ if os.path.exists(path):
+ return path
return None
@@ -247,6 +249,7 @@ class Stage(object):
"'spack clean --dist' to remove the bad archive, then fix",
"your internet gateway issue and install again.")
+
def fetch(self):
"""Downloads the file at URL to the stage. Returns true if it was downloaded,
false if it already existed."""
@@ -257,7 +260,7 @@ class Stage(object):
else:
urls = [self.url]
if self.mirror_path:
- urls = ["%s/%s" % (m, self.mirror_path) for m in spack.mirrors] + urls
+ urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls
for url in urls:
tty.msg("Trying to fetch from %s" % url)
@@ -320,6 +323,17 @@ class Stage(object):
os.chdir(os.path.dirname(self.path))
+def _get_mirrors():
+ """Get mirrors from spack configuration."""
+ config = spack.config.get_config()
+
+ mirrors = []
+ sec_names = config.get_section_names('mirror')
+ for name in sec_names:
+ mirrors.append(config.get_value('mirror', name, 'url'))
+ return mirrors
+
+
def ensure_access(file=spack.stage_path):
"""Ensure we can access a directory and die with an error if we can't."""
if not can_access(file):
diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py
index 427a9cf020..7ce8e8c65b 100644
--- a/lib/spack/spack/util/compression.py
+++ b/lib/spack/spack/util/compression.py
@@ -56,3 +56,12 @@ def stem(path):
if re.search(suffix, path):
return re.sub(suffix, "", path)
return path
+
+
+def extension(path):
+ """Get the archive extension for a path."""
+ for type in ALLOWED_ARCHIVE_TYPES:
+ suffix = r'\.%s$' % type
+ if re.search(suffix, path):
+ return type
+ return None