From fbd7e966808abf28b04ad9a98d359d425c8d7635 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 14 Oct 2014 23:26:43 -0700 Subject: Add a mirror module that handles new fetch strategies. - Uses new fetchers to get source - Add archive() method to fetch strategies to support this. - Updated mirror command to use new mirror module --- lib/spack/spack/cmd/mirror.py | 130 ++++++++--------------------- lib/spack/spack/fetch_strategy.py | 59 +++++++++++-- lib/spack/spack/mirror.py | 171 ++++++++++++++++++++++++++++++++++++++ lib/spack/spack/package.py | 19 +++-- 4 files changed, 270 insertions(+), 109 deletions(-) create mode 100644 lib/spack/spack/mirror.py diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index b42b329085..6a6c2b60c7 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -23,23 +23,19 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import shutil +import sys from datetime import datetime -from contextlib import closing from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -from llnl.util.filesystem import mkdirp, join_path import spack import spack.cmd import spack.config +import spack.mirror from spack.spec import Spec from spack.error import SpackError -from spack.stage import Stage -from spack.util.compression import extension - description = "Manage mirrors." @@ -105,26 +101,33 @@ def mirror_list(args): print fmt % (name, val) +def _read_specs_from_file(filename): + with closing(open(filename, "r")) as stream: + for i, string in enumerate(stream): + try: + s = Spec(string) + s.package + args.specs.append(s) + except SpackError, e: + tty.die("Parse error in %s, line %d:" % (args.file, i+1), + ">>> " + string, str(e)) + + def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" # try to parse specs from the command line first. - args.specs = spack.cmd.parse_specs(args.specs) + specs = spack.cmd.parse_specs(args.specs) # If there is a file, parse each line as a spec and add it to the list. if args.file: - with closing(open(args.file, "r")) as stream: - for i, string in enumerate(stream): - try: - s = Spec(string) - s.package - args.specs.append(s) - except SpackError, e: - tty.die("Parse error in %s, line %d:" % (args.file, i+1), - ">>> " + string, str(e)) - - if not args.specs: - args.specs = [Spec(n) for n in spack.db.all_package_names()] + if specs: + tty.die("Cannot pass specs on the command line with --file.") + specs = _read_specs_from_file(args.file) + + # If nothing is passed, use all packages. + if not specs: + specs = [Spec(n) for n in spack.db.all_package_names()] # Default name for directory is spack-mirror- if not args.directory: @@ -132,85 +135,23 @@ def mirror_create(args): args.directory = 'spack-mirror-' + timestamp # Make sure nothing is in the way. + existed = False if os.path.isfile(args.directory): tty.error("%s already exists and is a file." % args.directory) + elif os.path.isdir(args.directory): + existed = True - # Create a directory if none exists - if not os.path.isdir(args.directory): - mkdirp(args.directory) - tty.msg("Created new mirror in %s" % args.directory) - else: - tty.msg("Adding to existing mirror in %s" % args.directory) - - # Things to keep track of while parsing specs. - working_dir = os.getcwd() - num_mirrored = 0 - num_error = 0 - - # Iterate through packages and download all the safe tarballs for each of them - for spec in args.specs: - pkg = spec.package - - # Skip any package that has no checksummed versions. - if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s." - % pkg.name) - continue - - # create a subdir for the current package. - pkg_path = join_path(args.directory, pkg.name) - mkdirp(pkg_path) - - # Download all the tarballs using Stages, then move them into place - for version in pkg.versions: - # Skip versions that don't match the spec - vspec = Spec('%s@%s' % (pkg.name, version)) - if not vspec.satisfies(spec): - continue - - mirror_path = "%s/%s-%s.%s" % ( - pkg.name, pkg.name, version, extension(pkg.url)) - - os.chdir(working_dir) - mirror_file = join_path(args.directory, mirror_path) - if os.path.exists(mirror_file): - tty.msg("Already fetched %s." % mirror_file) - num_mirrored += 1 - continue - - # Get the URL for the version and set up a stage to download it. - url = pkg.url_for_version(version) - stage = Stage(url) - try: - # fetch changes directory into the stage - stage.fetch() + # Actually do the work to create the mirror + present, mirrored, error = spack.mirror.create(args.directory, specs) + p, m, e = len(present), len(mirrored), len(error) - if not args.no_checksum and version in pkg.versions: - digest = pkg.versions[version] - stage.check(digest) - tty.msg("Checksum passed for %s@%s" % (pkg.name, version)) - - # change back and move the new archive into place. - os.chdir(working_dir) - shutil.move(stage.archive_file, mirror_file) - tty.msg("Added %s to mirror" % mirror_file) - num_mirrored += 1 - - except Exception, e: - tty.warn("Error while fetching %s." % url, e.message) - num_error += 1 - - finally: - stage.destroy() - - # If nothing happened, try to say why. - if not num_mirrored: - if num_error: - tty.error("No packages added to mirror.", - "All packages failed to fetch.") - else: - tty.error("No packages added to mirror. No versions matched specs:") - colify(args.specs, indent=4) + verb = "updated" if existed else "created" + tty.msg( + "Successfully %s mirror in %s." % (verb, args.directory), + "Archive stats:", + " %-4d already present" % p, + " %-4d added" % m, + " %-4d failed to fetch." % e) def mirror(parser, args): @@ -218,4 +159,5 @@ def mirror(parser, args): 'add' : mirror_add, 'remove' : mirror_remove, 'list' : mirror_list } + action[args.mirror_command](args) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 8905dc7d5f..2cff15845b 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -37,6 +37,8 @@ in order to build it. They need to define the following methods: Restore original state of downloaded code. Used by clean commands. This may just remove the expanded source and re-expand an archive, or it may run something like git reset --hard. + * archive() + Archive a source directory, e.g. for creating a mirror. """ import os import re @@ -91,6 +93,9 @@ class FetchStrategy(object): def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" + @property + def unique_name(self): pass + # This method is used to match fetch strategies to version() # arguments in packages. @classmethod @@ -189,7 +194,7 @@ class URLFetchStrategy(FetchStrategy): def archive(self, destination): - """This archive""" + """Just moves this archive to the destination.""" if not self.archive_file: raise NoArchiveFileError("Cannot call archive() before fetching.") assert(extension(destination) == extension(self.archive_file)) @@ -231,6 +236,10 @@ class URLFetchStrategy(FetchStrategy): else: return "URLFetchStrategy" + @property + def unique_name(self): + return "spack-fetch-url:%s" % self + class VCSFetchStrategy(FetchStrategy): def __init__(self, name, *rev_types, **kwargs): @@ -384,6 +393,17 @@ class GitFetchStrategy(VCSFetchStrategy): self.git('clean', '-f') + @property + def unique_name(self): + name = "spack-fetch-git:%s" % self.url + if self.commit: + name += "@" + self.commit + elif self.branch: + name += "@" + self.branch + elif self.tag: + name += "@" + self.tag + + class SvnFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a subversion repository. Use like this in a package: @@ -457,6 +477,14 @@ class SvnFetchStrategy(VCSFetchStrategy): self.svn('revert', '.', '-R') + @property + def unique_name(self): + name = "spack-fetch-svn:%s" % self.url + if self.revision: + name += "@" + self.revision + + + class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -532,6 +560,14 @@ class HgFetchStrategy(VCSFetchStrategy): self.stage.chdir_to_source() + @property + def unique_name(self): + name = "spack-fetch-hg:%s" % self.url + if self.revision: + name += "@" + self.revision + + + def from_url(url): """Given a URL, find an appropriate fetch strategy for it. Currently just gives you a URLFetchStrategy that uses curl. @@ -546,9 +582,18 @@ def args_are_for(args, fetcher): fetcher.matches(args) -def from_args(args, pkg): +def for_package_version(pkg, version): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" + # If it's not a known version, extrapolate one. + if not version in pkg.versions: + url = pkg.url_for_verison(version) + if not url: + raise InvalidArgsError(pkg, version) + return URLFetchStrategy() + + # Grab a dict of args out of the package version dict + args = pkg.versions[version] # Test all strategies against per-version arguments. for fetcher in all_strategies: @@ -564,9 +609,7 @@ def from_args(args, pkg): if fetcher.matches(attrs): return fetcher(**attrs) - raise InvalidArgsError( - "Could not construct fetch strategy for package %s", - pkg.spec.format("%_%@")) + raise InvalidArgsError(pkg, version) class FetchStrategyError(spack.error.SpackError): @@ -593,5 +636,7 @@ class NoDigestError(FetchStrategyError): class InvalidArgsError(FetchStrategyError): - def __init__(self, msg, long_msg): - super(InvalidArgsError, self).__init__(msg, long_msg) + def __init__(self, pkg, version): + msg = "Could not construct a fetch strategy for package %s at version %s" + msg %= (pkg.name, version) + super(InvalidArgsError, self).__init__(msg) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py new file mode 100644 index 0000000000..e116ce83b2 --- /dev/null +++ b/lib/spack/spack/mirror.py @@ -0,0 +1,171 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +This file contains code for creating spack mirror directories. A +mirror is an organized hierarchy containing specially named archive +files. This enabled spack to know where to find files in a mirror if +the main server for a particualr package is down. Or, if the computer +where spack is run is not connected to the internet, it allows spack +to download packages directly from a mirror (e.g., on an intranet). +""" +import sys +import os +import llnl.util.tty as tty +from llnl.util.filesystem import * + +import spack +import spack.error +import spack.fetch_strategy as fs +from spack.spec import Spec +from spack.stage import Stage +from spack.version import * +from spack.util.compression import extension + + +def mirror_archive_filename(spec): + """Get the path that this spec will live at within a mirror.""" + if not spec.version.concrete: + raise ValueError("mirror.path requires spec with concrete version.") + + url = spec.package.default_url + if url is None: + ext = 'tar.gz' + else: + ext = extension(url) + + return "%s-%s.%s" % (spec.package.name, spec.version, ext) + + +def get_matching_versions(specs): + """Get a spec for EACH known version matching any spec in the list.""" + matching = [] + for spec in specs: + pkg = spec.package + + # Skip any package that has no known versions. + if not pkg.versions: + tty.msg("No safe (checksummed) versions for package %s." % pkg.name) + continue + + for v in reversed(sorted(pkg.versions)): + if v.satisfies(spec.versions): + s = Spec(pkg.name) + s.versions = VersionList([v]) + matching.append(s) + return matching + + +def create(path, specs, **kwargs): + """Create a directory to be used as a spack mirror, and fill it with + package archives. + + Arguments: + path Path to create a mirror directory hierarchy in. + specs Any package versions matching these specs will be added + to the mirror. + + Return Value: + Returns a tuple of lists: (present, mirrored, error) + * present: Package specs that were already prsent. + * mirrored: Package specs that were successfully mirrored. + * error: Package specs that failed to mirror due to some error. + + This routine iterates through all known package versions, and + it creates specs for those versions. If the version satisfies any spec + in the specs list, it is downloaded and added to the mirror. + """ + # Make sure nothing is in the way. + if os.path.isfile(path): + raise MirrorError("%s already exists and is a file." % path) + + # automatically spec-ify anything in the specs array. + specs = [s if isinstance(s, Spec) else Spec(s) for s in specs] + + # Get concrete specs for each matching version of these specs. + version_specs = get_matching_versions(specs) + for s in version_specs: + s.concretize() + + # Create a directory if none exists + if not os.path.isdir(path): + mkdirp(path) + + # Things to keep track of while parsing specs. + present = [] + mirrored = [] + error = [] + + # Iterate through packages and download all the safe tarballs for each of them + for spec in version_specs: + pkg = spec.package + + stage = None + try: + # create a subdirectory for the current package@version + realpath = os.path.realpath(path) + subdir = join_path(realpath, pkg.name) + mkdirp(subdir) + + archive_file = mirror_archive_filename(spec) + archive_path = join_path(subdir, archive_file) + if os.path.exists(archive_path): + present.append(spec) + continue + + # Set up a stage and a fetcher for the download + fetcher = fs.for_package_version(pkg, pkg.version) + stage = Stage(fetcher, name=fetcher.unique_name) + fetcher.set_stage(stage) + + # Do the fetch and checksum if necessary + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("Added %s to mirror" % archive_path) + mirrored.append(spec) + + except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + error.append(spec) + + finally: + if stage: + stage.destroy() + + return (present, mirrored, error) + + +class MirrorError(spack.error.SpackError): + """Superclass of all mirror-creation related errors.""" + def __init__(self, msg, long_msg=None): + super(MirrorError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ee3e73a072..34efe1bec9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -385,8 +385,8 @@ class Package(object): @property def version(self): - if not self.spec.concrete: - raise ValueError("Can only get version of concrete package.") + if not self.spec.versions.concrete: + raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] @@ -451,18 +451,20 @@ class Package(object): raise ValueError("Can only get a stage for a concrete package.") if self._stage is None: - self._stage = Stage( - self.fetcher, mirror_path=self.mirror_path(), name=self.spec.short_spec) + self._stage = Stage(self.fetcher, + mirror_path=self.mirror_path(), + name=self.spec.short_spec) return self._stage @property def fetcher(self): - if not self.spec.concrete: - raise ValueError("Can only get a fetcher for a concrete package.") + if not self.spec.versions.concrete: + raise ValueError( + "Can only get a fetcher for a package with concrete versions.") if not self._fetcher: - self._fetcher = fs.from_args(self.versions[self.version], self) + self._fetcher = fs.for_package_version(self, self.version) return self._fetcher @@ -598,13 +600,14 @@ class Package(object): @property def default_url(self): - if self.spec.version.concrete: + if self.spec.versions.concrete: return self.url_for_version(self.version) else: url = getattr(self, 'url', None) if url: return url + return None def remove_prefix(self): -- cgit v1.2.3-70-g09d2