From 9f8ef85644883a53578fb000827cb937a16d058e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Jun 2014 07:17:30 -0700 Subject: Add more output; don't re-add existing compilers --- lib/spack/spack/cmd/compiler.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 0586484c32..a4cd2df7e2 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -60,8 +60,17 @@ def compiler_add(args): if not paths: paths = get_path('PATH') - compilers = spack.compilers.find_compilers(*args.add_paths) - spack.compilers.add_compilers_to_config('user', *compilers) + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_filename('user'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): -- cgit v1.2.3-70-g09d2 From b33412e03a57783202cc274c48aa02b787adfbf5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Jun 2014 11:15:41 -0700 Subject: New spack mirror command, configuration. - Mirrors now appear in ~/.spackconfig, can be edited in that file. - spack mirror command allows adding, listing, removing mirrors - Also still creates mirror directories. --- lib/spack/spack/__init__.py | 18 ---- lib/spack/spack/cmd/mirror.py | 165 +++++++++++++++++++++++++++++++----- lib/spack/spack/config.py | 13 ++- lib/spack/spack/package.py | 6 +- lib/spack/spack/stage.py | 22 ++++- lib/spack/spack/util/compression.py | 9 ++ 6 files changed, 184 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c4a2de0b2a..904c6262c8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -137,24 +137,6 @@ do_checksum = True # sys_type = None -# -# Places to download tarballs from. -# -# TODO: move to configuration. -# -# Examples: -# -# For a local directory: -# mirrors = ['file:///Users/gamblin2/spack-mirror'] -# -# For a website: -# mirrors = ['http://spackports.org/spack-mirror/'] -# -# For no mirrors: -# mirrors = [] -# -mirrors = [] - # # Extra imports that should be generally usable from package.py files. # diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 94e40557ff..494af3f718 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -25,69 +25,188 @@ import os import shutil import argparse +from datetime import datetime +from contextlib import closing import llnl.util.tty as tty +from llnl.util.tty.colify import colify from llnl.util.filesystem import mkdirp, join_path import spack import spack.cmd +import spack.config +from spack.spec import Spec +from spack.error import SpackError from spack.stage import Stage +from spack.util.compression import extension -description = "Create a directory full of package tarballs that can be used as a spack mirror." +description = "Manage spack mirrors." def setup_parser(subparser): - subparser.add_argument( - 'directory', help="Directory in which to create mirror.") - subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="names of packages to put in mirror") - - -def mirror(parser, args): - if not args.packages: - args.packages = [p for p in spack.db.all_package_names()] - + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='mirror_command') + + create_parser = sp.add_parser('create', help=mirror_create.__doc__) + create_parser.add_argument('-d', '--directory', default=None, + help="Directory in which to create mirror.") + create_parser.add_argument( + 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + create_parser.add_argument( + '-f', '--file', help="File with specs of packages to put in mirror.") + + add_parser = sp.add_parser('add', help=mirror_add.__doc__) + add_parser.add_argument('name', help="Mnemonic name for mirror.") + add_parser.add_argument( + 'url', help="URL of mirror directory created by 'spack mirror create'.") + + remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=mirror_list.__doc__) + + +def mirror_add(args): + """Add a mirror to Spack.""" + config = spack.config.get_config('user') + config.set_value('mirror', args.name, 'url', args.url) + config.write() + + +def mirror_remove(args): + """Remove a mirror by name.""" + config = spack.config.get_config('user') + name = args.name + + if not config.has_named_section('mirror', name): + tty.die("No such mirror: %s" % name) + config.remove_named_section('mirror', name) + config.write() + + +def mirror_list(args): + """Print out available mirrors to the console.""" + config = spack.config.get_config() + sec_names = config.get_section_names('mirror') + + if not sec_names: + tty.msg("No mirrors configured.") + return + + max_len = max(len(s) for s in sec_names) + fmt = "%%-%ds%%s" % (max_len + 4) + + for name in sec_names: + val = config.get_value('mirror', name, 'url') + print fmt % (name, val) + + +def mirror_create(args): + """Create a directory to be used as a spack mirror, and fill it with + package archives.""" + # try to parse specs from the command line first. + args.specs = spack.cmd.parse_specs(args.specs) + + # If there is a file, parse each line as a spec and add it to the list. + if args.file: + with closing(open(args.file, "r")) as stream: + for i, string in enumerate(stream): + try: + s = Spec(string) + s.package + args.specs.append(s) + except SpackError, e: + tty.die("Parse error in %s, line %d:" % (args.file, i+1), + ">>> " + string, str(e)) + + if not args.specs: + args.specs = spack.db.all_package_names() + + # Default name for directory is spack-mirror- + if not args.directory: + timestamp = datetime.now().strftime("%Y-%m-%d") + args.directory = 'spack-mirror-' + timestamp + + # Make sure nothing is in the way. if os.path.isfile(args.directory): tty.error("%s already exists and is a file." % args.directory) + # Create a directory if none exists if not os.path.isdir(args.directory): mkdirp(args.directory) + tty.msg("Created new mirror in %s" % args.directory) + else: + tty.msg("Adding to existing mirror in %s" % args.directory) - # save working directory + # Things to keep track of while parsing specs. working_dir = os.getcwd() + num_mirrored = 0 + num_error = 0 # Iterate through packages and download all the safe tarballs for each of them - for pkg_name in args.packages: - pkg = spack.db.get(pkg_name) + for spec in args.specs: + pkg = spec.package # Skip any package that has no checksummed versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s. Skipping." - % pkg_name) + tty.msg("No safe (checksummed) versions for package %s." + % pkg.name) continue # create a subdir for the current package. - pkg_path = join_path(args.directory, pkg_name) + pkg_path = join_path(args.directory, pkg.name) mkdirp(pkg_path) # Download all the tarballs using Stages, then move them into place for version in pkg.versions: + # Skip versions that don't match the spec + vspec = Spec('%s@%s' % (pkg.name, version)) + if not vspec.satisfies(spec): + continue + + mirror_path = "%s/%s-%s.%s" % ( + pkg.name, pkg.name, version, extension(pkg.url)) + + os.chdir(working_dir) + mirror_file = join_path(args.directory, mirror_path) + if os.path.exists(mirror_file): + tty.msg("Already fetched %s. Skipping." % mirror_file) + num_mirrored += 1 + continue + + # Get the URL for the version and set up a stage to download it. url = pkg.url_for_version(version) stage = Stage(url) try: + # fetch changes directory into the stage stage.fetch() - basename = os.path.basename(stage.archive_file) - final_dst = join_path(pkg_path, basename) + # change back and move the new archive into place. os.chdir(working_dir) - shutil.move(stage.archive_file, final_dst) - tty.msg("Added %s to mirror" % final_dst) + shutil.move(stage.archive_file, mirror_file) + tty.msg("Added %s to mirror" % mirror_file) + num_mirrored += 1 except Exception, e: tty.warn("Error while fetching %s. Skipping." % url, e.message) + num_error += 1 finally: stage.destroy() - # Success! - tty.msg("Created Spack mirror in %s" % args.directory) + # If nothing happened, try to say why. + if not num_mirrored: + if num_error: + tty.warn("No packages added to mirror.", + "All packages failed to fetch.") + else: + tty.warn("No packages added to mirror. No versions matched specs:") + colify(args.specs, indent=4) + + +def mirror(parser, args): + action = { 'create' : mirror_create, + 'add' : mirror_add, + 'remove' : mirror_remove, + 'list' : mirror_list } + action[args.mirror_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 00ff4313a2..5494adc324 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -225,8 +225,7 @@ class SpackConfigParser(cp.RawConfigParser): OPTCRE_NV = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE_NV.pattern) def __init__(self, file_or_files): - cp.RawConfigParser.__init__( - self, dict_type=OrderedDict, allow_no_value=True) + cp.RawConfigParser.__init__(self, dict_type=OrderedDict) if isinstance(file_or_files, basestring): self.read([file_or_files]) @@ -286,6 +285,16 @@ class SpackConfigParser(cp.RawConfigParser): return self.has_option(sn, option) + def has_named_section(self, section, name): + sn = _make_section_name(section, name) + return self.has_section(sn) + + + def remove_named_section(self, section, name): + sn = _make_section_name(section, name) + self.remove_section(sn) + + def get_section_names(self, sectype): """Get all named sections with the specified type. A named section looks like this: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ef8b639adf..eb0625cb7e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -54,7 +54,7 @@ import spack.util.crypto as crypto from spack.version import * from spack.stage import Stage from spack.util.web import get_pages -from spack.util.compression import allowed_archive +from spack.util.compression import allowed_archive, extension """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file"] @@ -399,7 +399,9 @@ class Package(object): raise ValueError("Can only get a stage for a concrete package.") if self._stage is None: - mirror_path = "%s/%s" % (self.name, os.path.basename(self.url)) + # TODO: move this logic into a mirror module. + mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % ( + self.name, self.version, extension(self.url))) self._stage = Stage( self.url, mirror_path=mirror_path, name=self.spec.short_spec) return self._stage diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 8574df71e9..f679cec282 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -31,6 +31,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * import spack +import spack.config import spack.error as serr from spack.util.compression import decompressor_for @@ -185,9 +186,10 @@ class Stage(object): @property def archive_file(self): """Path to the source archive within this stage directory.""" - path = os.path.join(self.path, os.path.basename(self.url)) - if os.path.exists(path): - return path + for path in (os.path.join(self.path, os.path.basename(self.url)), + os.path.join(self.path, os.path.basename(self.mirror_path))): + if os.path.exists(path): + return path return None @@ -247,6 +249,7 @@ class Stage(object): "'spack clean --dist' to remove the bad archive, then fix", "your internet gateway issue and install again.") + def fetch(self): """Downloads the file at URL to the stage. Returns true if it was downloaded, false if it already existed.""" @@ -257,7 +260,7 @@ class Stage(object): else: urls = [self.url] if self.mirror_path: - urls = ["%s/%s" % (m, self.mirror_path) for m in spack.mirrors] + urls + urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls for url in urls: tty.msg("Trying to fetch from %s" % url) @@ -320,6 +323,17 @@ class Stage(object): os.chdir(os.path.dirname(self.path)) +def _get_mirrors(): + """Get mirrors from spack configuration.""" + config = spack.config.get_config() + + mirrors = [] + sec_names = config.get_section_names('mirror') + for name in sec_names: + mirrors.append(config.get_value('mirror', name, 'url')) + return mirrors + + def ensure_access(file=spack.stage_path): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index 427a9cf020..7ce8e8c65b 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -56,3 +56,12 @@ def stem(path): if re.search(suffix, path): return re.sub(suffix, "", path) return path + + +def extension(path): + """Get the archive extension for a path.""" + for type in ALLOWED_ARCHIVE_TYPES: + suffix = r'\.%s$' % type + if re.search(suffix, path): + return type + return None -- cgit v1.2.3-70-g09d2 From 3c3f272280c530553322142d9d836c91b1b01137 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Jun 2014 11:53:44 -0700 Subject: spack mirror now checksums fetched archives. --- bin/spack | 2 +- lib/spack/spack/cmd/mirror.py | 19 ++++++++++++++----- lib/spack/spack/package.py | 18 +++--------------- lib/spack/spack/stage.py | 28 ++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index df517c1f1d..a0ce203d63 100755 --- a/bin/spack +++ b/bin/spack @@ -105,4 +105,4 @@ except SpackError, e: tty.die(e.message) except KeyboardInterrupt: - tty.die("Got a keyboard interrupt from the user.") + tty.die("Keyboard interrupt.") diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 494af3f718..813c58b84d 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -44,6 +44,10 @@ from spack.util.compression import extension description = "Manage spack mirrors." def setup_parser(subparser): + subparser.add_argument( + '-n', '--no-checksum', action='store_true', dest='no_checksum', + help="Do not check fetched packages against checksum") + sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='mirror_command') @@ -170,7 +174,7 @@ def mirror_create(args): os.chdir(working_dir) mirror_file = join_path(args.directory, mirror_path) if os.path.exists(mirror_file): - tty.msg("Already fetched %s. Skipping." % mirror_file) + tty.msg("Already fetched %s." % mirror_file) num_mirrored += 1 continue @@ -181,6 +185,11 @@ def mirror_create(args): # fetch changes directory into the stage stage.fetch() + if not args.no_checksum and version in pkg.versions: + digest = pkg.versions[version] + stage.check(digest) + tty.msg("Checksum passed for %s@%s" % (pkg.name, version)) + # change back and move the new archive into place. os.chdir(working_dir) shutil.move(stage.archive_file, mirror_file) @@ -188,7 +197,7 @@ def mirror_create(args): num_mirrored += 1 except Exception, e: - tty.warn("Error while fetching %s. Skipping." % url, e.message) + tty.warn("Error while fetching %s." % url, e.message) num_error += 1 finally: @@ -197,10 +206,10 @@ def mirror_create(args): # If nothing happened, try to say why. if not num_mirrored: if num_error: - tty.warn("No packages added to mirror.", - "All packages failed to fetch.") + tty.error("No packages added to mirror.", + "All packages failed to fetch.") else: - tty.warn("No packages added to mirror. No versions matched specs:") + tty.error("No packages added to mirror. No versions matched specs:") colify(args.specs, indent=4) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index eb0625cb7e..da599037dd 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -50,7 +50,6 @@ import spack.spec import spack.error import spack.build_environment as build_env import spack.url as url -import spack.util.crypto as crypto from spack.version import * from spack.stage import Stage from spack.util.web import get_pages @@ -539,7 +538,7 @@ class Package(object): raise ValueError("Can only fetch concrete packages.") if spack.do_checksum and not self.version in self.versions: - raise ChecksumError( + raise FetchError( "Cannot fetch %s safely; there is no checksum on file for version %s." % (self.name, self.version), "Add a checksum to the package file, or use --no-checksum to " @@ -549,13 +548,8 @@ class Package(object): if spack.do_checksum and self.version in self.versions: digest = self.versions[self.version] - checker = crypto.Checker(digest) - if checker.check(self.stage.archive_file): - tty.msg("Checksum passed for %s" % self.name) - else: - raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.name), - "Expected %s but got %s." % (digest, checker.sum)) + self.stage.check(digest) + tty.msg("Checksum passed for %s@%s" % (self.name, self.version)) def do_stage(self): @@ -868,12 +862,6 @@ class FetchError(spack.error.SpackError): super(FetchError, self).__init__(message, long_msg) -class ChecksumError(FetchError): - """Raised when archive fails to checksum.""" - def __init__(self, message, long_msg): - super(ChecksumError, self).__init__(message, long_msg) - - class InstallError(spack.error.SpackError): """Raised when something goes wrong during install or uninstall.""" def __init__(self, message, long_msg=None): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f679cec282..839555d630 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -32,9 +32,11 @@ from llnl.util.filesystem import * import spack import spack.config -import spack.error as serr +import spack.error +import spack.util.crypto as crypto from spack.util.compression import decompressor_for + STAGE_PREFIX = 'spack-stage-' @@ -186,8 +188,11 @@ class Stage(object): @property def archive_file(self): """Path to the source archive within this stage directory.""" - for path in (os.path.join(self.path, os.path.basename(self.url)), - os.path.join(self.path, os.path.basename(self.mirror_path))): + paths = [os.path.join(self.path, os.path.basename(self.url))] + if self.mirror_path: + paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + + for path in paths: if os.path.exists(path): return path return None @@ -274,6 +279,15 @@ class Stage(object): return self.archive_file + def check(self, digest): + """Check the downloaded archive against a checksum digest""" + checker = crypto.Checker(digest) + if not checker.check(self.archive_file): + raise ChecksumError( + "%s checksum failed for %s." % (checker.hash_name, self.archive_file), + "Expected %s but got %s." % (digest, checker.sum)) + + def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -380,9 +394,15 @@ def find_tmp_root(): return None -class FailedDownloadError(serr.SpackError): +class FailedDownloadError(spack.error.SpackError): """Raised wen a download fails.""" def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( "Failed to fetch file from URL: %s" % url, msg) self.url = url + + +class ChecksumError(spack.error.SpackError): + """Raised when archive fails to checksum.""" + def __init__(self, message, long_msg): + super(ChecksumError, self).__init__(message, long_msg) -- cgit v1.2.3-70-g09d2