diff options
| -rw-r--r-- | lib/spack/spack/__init__.py | 18 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/mirror.py | 165 | ||||
| -rw-r--r-- | lib/spack/spack/config.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 22 | ||||
| -rw-r--r-- | lib/spack/spack/util/compression.py | 9 |
6 files changed, 184 insertions, 49 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c4a2de0b2a..904c6262c8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -138,24 +138,6 @@ do_checksum = True sys_type = None # -# Places to download tarballs from. -# -# TODO: move to configuration. -# -# Examples: -# -# For a local directory: -# mirrors = ['file:///Users/gamblin2/spack-mirror'] -# -# For a website: -# mirrors = ['http://spackports.org/spack-mirror/'] -# -# For no mirrors: -# mirrors = [] -# -mirrors = [] - -# # Extra imports that should be generally usable from package.py files. # from llnl.util.filesystem import working_dir diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 94e40557ff..494af3f718 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -25,69 +25,188 @@ import os import shutil import argparse +from datetime import datetime +from contextlib import closing import llnl.util.tty as tty +from llnl.util.tty.colify import colify from llnl.util.filesystem import mkdirp, join_path import spack import spack.cmd +import spack.config +from spack.spec import Spec +from spack.error import SpackError from spack.stage import Stage +from spack.util.compression import extension -description = "Create a directory full of package tarballs that can be used as a spack mirror." +description = "Manage spack mirrors." def setup_parser(subparser): - subparser.add_argument( - 'directory', help="Directory in which to create mirror.") - subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="names of packages to put in mirror") - - -def mirror(parser, args): - if not args.packages: - args.packages = [p for p in spack.db.all_package_names()] - + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='mirror_command') + + create_parser = sp.add_parser('create', help=mirror_create.__doc__) + create_parser.add_argument('-d', '--directory', default=None, + help="Directory in which to create mirror.") + create_parser.add_argument( + 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + create_parser.add_argument( + '-f', '--file', help="File with specs of packages to put in mirror.") + + add_parser = sp.add_parser('add', help=mirror_add.__doc__) + add_parser.add_argument('name', help="Mnemonic name for mirror.") + add_parser.add_argument( + 'url', help="URL of mirror directory created by 'spack mirror create'.") + + remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=mirror_list.__doc__) + + +def mirror_add(args): + """Add a mirror to Spack.""" + config = spack.config.get_config('user') + config.set_value('mirror', args.name, 'url', args.url) + config.write() + + +def mirror_remove(args): + """Remove a mirror by name.""" + config = spack.config.get_config('user') + name = args.name + + if not config.has_named_section('mirror', name): + tty.die("No such mirror: %s" % name) + config.remove_named_section('mirror', name) + config.write() + + +def mirror_list(args): + """Print out available mirrors to the console.""" + config = spack.config.get_config() + sec_names = config.get_section_names('mirror') + + if not sec_names: + tty.msg("No mirrors configured.") + return + + max_len = max(len(s) for s in sec_names) + fmt = "%%-%ds%%s" % (max_len + 4) + + for name in sec_names: + val = config.get_value('mirror', name, 'url') + print fmt % (name, val) + + +def mirror_create(args): + """Create a directory to be used as a spack mirror, and fill it with + package archives.""" + # try to parse specs from the command line first. + args.specs = spack.cmd.parse_specs(args.specs) + + # If there is a file, parse each line as a spec and add it to the list. + if args.file: + with closing(open(args.file, "r")) as stream: + for i, string in enumerate(stream): + try: + s = Spec(string) + s.package + args.specs.append(s) + except SpackError, e: + tty.die("Parse error in %s, line %d:" % (args.file, i+1), + ">>> " + string, str(e)) + + if not args.specs: + args.specs = spack.db.all_package_names() + + # Default name for directory is spack-mirror-<DATESTAMP> + if not args.directory: + timestamp = datetime.now().strftime("%Y-%m-%d") + args.directory = 'spack-mirror-' + timestamp + + # Make sure nothing is in the way. if os.path.isfile(args.directory): tty.error("%s already exists and is a file." % args.directory) + # Create a directory if none exists if not os.path.isdir(args.directory): mkdirp(args.directory) + tty.msg("Created new mirror in %s" % args.directory) + else: + tty.msg("Adding to existing mirror in %s" % args.directory) - # save working directory + # Things to keep track of while parsing specs. working_dir = os.getcwd() + num_mirrored = 0 + num_error = 0 # Iterate through packages and download all the safe tarballs for each of them - for pkg_name in args.packages: - pkg = spack.db.get(pkg_name) + for spec in args.specs: + pkg = spec.package # Skip any package that has no checksummed versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s. Skipping." - % pkg_name) + tty.msg("No safe (checksummed) versions for package %s." + % pkg.name) continue # create a subdir for the current package. - pkg_path = join_path(args.directory, pkg_name) + pkg_path = join_path(args.directory, pkg.name) mkdirp(pkg_path) # Download all the tarballs using Stages, then move them into place for version in pkg.versions: + # Skip versions that don't match the spec + vspec = Spec('%s@%s' % (pkg.name, version)) + if not vspec.satisfies(spec): + continue + + mirror_path = "%s/%s-%s.%s" % ( + pkg.name, pkg.name, version, extension(pkg.url)) + + os.chdir(working_dir) + mirror_file = join_path(args.directory, mirror_path) + if os.path.exists(mirror_file): + tty.msg("Already fetched %s. Skipping." % mirror_file) + num_mirrored += 1 + continue + + # Get the URL for the version and set up a stage to download it. url = pkg.url_for_version(version) stage = Stage(url) try: + # fetch changes directory into the stage stage.fetch() - basename = os.path.basename(stage.archive_file) - final_dst = join_path(pkg_path, basename) + # change back and move the new archive into place. os.chdir(working_dir) - shutil.move(stage.archive_file, final_dst) - tty.msg("Added %s to mirror" % final_dst) + shutil.move(stage.archive_file, mirror_file) + tty.msg("Added %s to mirror" % mirror_file) + num_mirrored += 1 except Exception, e: tty.warn("Error while fetching %s. Skipping." % url, e.message) + num_error += 1 finally: stage.destroy() - # Success! - tty.msg("Created Spack mirror in %s" % args.directory) + # If nothing happened, try to say why. + if not num_mirrored: + if num_error: + tty.warn("No packages added to mirror.", + "All packages failed to fetch.") + else: + tty.warn("No packages added to mirror. No versions matched specs:") + colify(args.specs, indent=4) + + +def mirror(parser, args): + action = { 'create' : mirror_create, + 'add' : mirror_add, + 'remove' : mirror_remove, + 'list' : mirror_list } + action[args.mirror_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 00ff4313a2..5494adc324 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -225,8 +225,7 @@ class SpackConfigParser(cp.RawConfigParser): OPTCRE_NV = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE_NV.pattern) def __init__(self, file_or_files): - cp.RawConfigParser.__init__( - self, dict_type=OrderedDict, allow_no_value=True) + cp.RawConfigParser.__init__(self, dict_type=OrderedDict) if isinstance(file_or_files, basestring): self.read([file_or_files]) @@ -286,6 +285,16 @@ class SpackConfigParser(cp.RawConfigParser): return self.has_option(sn, option) + def has_named_section(self, section, name): + sn = _make_section_name(section, name) + return self.has_section(sn) + + + def remove_named_section(self, section, name): + sn = _make_section_name(section, name) + self.remove_section(sn) + + def get_section_names(self, sectype): """Get all named sections with the specified type. A named section looks like this: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ef8b639adf..eb0625cb7e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -54,7 +54,7 @@ import spack.util.crypto as crypto from spack.version import * from spack.stage import Stage from spack.util.web import get_pages -from spack.util.compression import allowed_archive +from spack.util.compression import allowed_archive, extension """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file"] @@ -399,7 +399,9 @@ class Package(object): raise ValueError("Can only get a stage for a concrete package.") if self._stage is None: - mirror_path = "%s/%s" % (self.name, os.path.basename(self.url)) + # TODO: move this logic into a mirror module. + mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % ( + self.name, self.version, extension(self.url))) self._stage = Stage( self.url, mirror_path=mirror_path, name=self.spec.short_spec) return self._stage diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 8574df71e9..f679cec282 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -31,6 +31,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * import spack +import spack.config import spack.error as serr from spack.util.compression import decompressor_for @@ -185,9 +186,10 @@ class Stage(object): @property def archive_file(self): """Path to the source archive within this stage directory.""" - path = os.path.join(self.path, os.path.basename(self.url)) - if os.path.exists(path): - return path + for path in (os.path.join(self.path, os.path.basename(self.url)), + os.path.join(self.path, os.path.basename(self.mirror_path))): + if os.path.exists(path): + return path return None @@ -247,6 +249,7 @@ class Stage(object): "'spack clean --dist' to remove the bad archive, then fix", "your internet gateway issue and install again.") + def fetch(self): """Downloads the file at URL to the stage. Returns true if it was downloaded, false if it already existed.""" @@ -257,7 +260,7 @@ class Stage(object): else: urls = [self.url] if self.mirror_path: - urls = ["%s/%s" % (m, self.mirror_path) for m in spack.mirrors] + urls + urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls for url in urls: tty.msg("Trying to fetch from %s" % url) @@ -320,6 +323,17 @@ class Stage(object): os.chdir(os.path.dirname(self.path)) +def _get_mirrors(): + """Get mirrors from spack configuration.""" + config = spack.config.get_config() + + mirrors = [] + sec_names = config.get_section_names('mirror') + for name in sec_names: + mirrors.append(config.get_value('mirror', name, 'url')) + return mirrors + + def ensure_access(file=spack.stage_path): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index 427a9cf020..7ce8e8c65b 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -56,3 +56,12 @@ def stem(path): if re.search(suffix, path): return re.sub(suffix, "", path) return path + + +def extension(path): + """Get the archive extension for a path.""" + for type in ALLOWED_ARCHIVE_TYPES: + suffix = r'\.%s$' % type + if re.search(suffix, path): + return type + return None |