diff options
| -rwxr-xr-x | bin/spack | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/mirror.py | 20 | ||||
| -rw-r--r-- | lib/spack/spack/concretize.py | 1 | ||||
| -rw-r--r-- | lib/spack/spack/fetch_strategy.py | 39 | ||||
| -rw-r--r-- | lib/spack/spack/mirror.py | 48 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/test/__init__.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/test/install.py | 37 | ||||
| -rw-r--r-- | lib/spack/spack/test/mirror.py | 156 | ||||
| -rw-r--r-- | lib/spack/spack/test/mock_repo.py | 76 |
10 files changed, 277 insertions, 117 deletions
@@ -96,7 +96,7 @@ if args.mock: # If the user asked for it, don't check ssl certs. if args.insecure: - tty.warn("You asked for --insecure, which does not check SSL certificates.") + tty.warn("You asked for --insecure, which does not check SSL certificates or checksums.") spack.curl.add_default_arg('-k') # Try to load the particular command asked for and run it diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 6a6c2b60c7..6e2e4dfb24 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -54,6 +54,9 @@ def setup_parser(subparser): 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") + create_parser.add_argument( + '-o', '--one-version-per-spec', action='store_const', const=1, default=0, + help="Only fetch one 'preferred' version per spec, not all known versions.") add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") @@ -128,26 +131,29 @@ def mirror_create(args): # If nothing is passed, use all packages. if not specs: specs = [Spec(n) for n in spack.db.all_package_names()] + specs.sort(key=lambda s: s.format("$_$@").lower()) # Default name for directory is spack-mirror-<DATESTAMP> - if not args.directory: + directory = args.directory + if not directory: timestamp = datetime.now().strftime("%Y-%m-%d") - args.directory = 'spack-mirror-' + timestamp + directory = 'spack-mirror-' + timestamp # Make sure nothing is in the way. existed = False - if os.path.isfile(args.directory): - tty.error("%s already exists and is a file." % args.directory) - elif os.path.isdir(args.directory): + if os.path.isfile(directory): + tty.error("%s already exists and is a file." % directory) + elif os.path.isdir(directory): existed = True # Actually do the work to create the mirror - present, mirrored, error = spack.mirror.create(args.directory, specs) + present, mirrored, error = spack.mirror.create( + directory, specs, num_versions=args.one_version_per_spec) p, m, e = len(present), len(mirrored), len(error) verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, args.directory), + "Successfully %s mirror in %s." % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 9f9cd1789d..e603806af9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -73,7 +73,6 @@ class DefaultConcretizer(object): if valid_versions: spec.versions = ver([valid_versions[-1]]) else: - print spec raise NoValidVersionError(spec) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 2cff15845b..2b574eaba7 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -93,9 +93,6 @@ class FetchStrategy(object): def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" - @property - def unique_name(self): pass - # This method is used to match fetch strategies to version() # arguments in packages. @classmethod @@ -197,7 +194,10 @@ class URLFetchStrategy(FetchStrategy): """Just moves this archive to the destination.""" if not self.archive_file: raise NoArchiveFileError("Cannot call archive() before fetching.") - assert(extension(destination) == extension(self.archive_file)) + + if not extension(destination) == extension(self.archive_file): + raise ValueError("Cannot archive without matching extensions.") + shutil.move(self.archive_file, destination) @@ -236,10 +236,6 @@ class URLFetchStrategy(FetchStrategy): else: return "URLFetchStrategy<no url>" - @property - def unique_name(self): - return "spack-fetch-url:%s" % self - class VCSFetchStrategy(FetchStrategy): def __init__(self, name, *rev_types, **kwargs): @@ -393,17 +389,6 @@ class GitFetchStrategy(VCSFetchStrategy): self.git('clean', '-f') - @property - def unique_name(self): - name = "spack-fetch-git:%s" % self.url - if self.commit: - name += "@" + self.commit - elif self.branch: - name += "@" + self.branch - elif self.tag: - name += "@" + self.tag - - class SvnFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a subversion repository. Use like this in a package: @@ -477,14 +462,6 @@ class SvnFetchStrategy(VCSFetchStrategy): self.svn('revert', '.', '-R') - @property - def unique_name(self): - name = "spack-fetch-svn:%s" % self.url - if self.revision: - name += "@" + self.revision - - - class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -560,14 +537,6 @@ class HgFetchStrategy(VCSFetchStrategy): self.stage.chdir_to_source() - @property - def unique_name(self): - name = "spack-fetch-hg:%s" % self.url - if self.revision: - name += "@" + self.revision - - - def from_url(url): """Given a URL, find an appropriate fetch strategy for it. Currently just gives you a URLFetchStrategy that uses curl. diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index e116ce83b2..f7bbb3f840 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -49,16 +49,18 @@ def mirror_archive_filename(spec): if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") - url = spec.package.default_url - if url is None: - ext = 'tar.gz' + fetcher = spec.package.fetcher + if isinstance(fetcher, fs.URLFetchStrategy): + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = extension(fetcher.url) else: - ext = extension(url) + # Otherwise we'll make a .tar.gz ourselves + ext = 'tar.gz' return "%s-%s.%s" % (spec.package.name, spec.version, ext) -def get_matching_versions(specs): +def get_matching_versions(specs, **kwargs): """Get a spec for EACH known version matching any spec in the list.""" matching = [] for spec in specs: @@ -69,11 +71,18 @@ def get_matching_versions(specs): tty.msg("No safe (checksummed) versions for package %s." % pkg.name) continue - for v in reversed(sorted(pkg.versions)): + num_versions = kwargs.get('num_versions', 0) + for i, v in enumerate(reversed(sorted(pkg.versions))): + # Generate no more than num_versions versions for each spec. + if num_versions and i >= num_versions: + break + + # Generate only versions that satisfy the spec. if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) matching.append(s) + return matching @@ -86,6 +95,11 @@ def create(path, specs, **kwargs): specs Any package versions matching these specs will be added to the mirror. + Keyword args: + no_checksum: If True, do not checkpoint when fetching (default False) + num_versions: Max number of versions to fetch per spec, + if spec is ambiguous (default is 0 for all of them) + Return Value: Returns a tuple of lists: (present, mirrored, error) * present: Package specs that were already prsent. @@ -104,13 +118,15 @@ def create(path, specs, **kwargs): specs = [s if isinstance(s, Spec) else Spec(s) for s in specs] # Get concrete specs for each matching version of these specs. - version_specs = get_matching_versions(specs) + version_specs = get_matching_versions( + specs, num_versions=kwargs.get('num_versions', 0)) for s in version_specs: s.concretize() - # Create a directory if none exists - if not os.path.isdir(path): - mkdirp(path) + # Get the absolute path of the root before we start jumping around. + mirror_root = os.path.abspath(path) + if not os.path.isdir(mirror_root): + mkdirp(mirror_root) # Things to keep track of while parsing specs. present = [] @@ -124,19 +140,21 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - realpath = os.path.realpath(path) - subdir = join_path(realpath, pkg.name) + subdir = join_path(mirror_root, pkg.name) mkdirp(subdir) archive_file = mirror_archive_filename(spec) archive_path = join_path(subdir, archive_file) - if os.path.exists(archive_path): + + if os.path.exists(archive_file): + tty.msg("%s is already present. Skipping." % spec.format("$_$@")) present.append(spec) continue # Set up a stage and a fetcher for the download + unique_fetch_name = spec.format("$_$@") fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=fetcher.unique_name) + stage = Stage(fetcher, name=unique_fetch_name) fetcher.set_stage(stage) # Do the fetch and checksum if necessary @@ -148,7 +166,7 @@ def create(path, specs, **kwargs): # Fetchers have to know how to archive their files. Use # that to move/copy/create an archive in the mirror. fetcher.archive(archive_path) - tty.msg("Added %s to mirror" % archive_path) + tty.msg("Added %s." % spec.format("$_$@")) mirrored.append(spec) except Exception, e: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 34efe1bec9..e462562e85 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -598,18 +598,6 @@ class Package(object): return str(version) - @property - def default_url(self): - if self.spec.versions.concrete: - return self.url_for_version(self.version) - else: - url = getattr(self, 'url', None) - if url: - return url - - return None - - def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" spack.install_layout.remove_path_for_spec(self.spec) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index ca4c869e42..be9ac5a560 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -50,7 +50,8 @@ test_names = ['versions', 'python_version', 'git_fetch', 'svn_fetch', - 'hg_fetch'] + 'hg_fetch', + 'mirror'] def list_tests(): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 0d53bb45c7..e052f53e77 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -36,39 +36,17 @@ from spack.fetch_strategy import URLFetchStrategy from spack.directory_layout import SpecHashDirectoryLayout from spack.util.executable import which from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockArchive -dir_name = 'trivial-1.0' -archive_name = 'trivial-1.0.tar.gz' -install_test_package = 'trivial_install_test_package' - class InstallTest(MockPackagesTest): """Tests install and uninstall on a trivial package.""" def setUp(self): super(InstallTest, self).setUp() - self.stage = Stage('not_a_real_url') - archive_dir = join_path(self.stage.path, dir_name) - dummy_configure = join_path(archive_dir, 'configure') - - mkdirp(archive_dir) - with closing(open(dummy_configure, 'w')) as configure: - configure.write( - "#!/bin/sh\n" - "prefix=$(echo $1 | sed 's/--prefix=//')\n" - "cat > Makefile <<EOF\n" - "all:\n" - "\techo Building...\n\n" - "install:\n" - "\tmkdir -p $prefix\n" - "\ttouch $prefix/dummy_file\n" - "EOF\n") - os.chmod(dummy_configure, 0755) - - with working_dir(self.stage.path): - tar = which('tar') - tar('-czf', archive_name, dir_name) + # create a simple installable package directory and tarball + self.repo = MockArchive() # We use a fake package, so skip the checksum. spack.do_checksum = False @@ -83,8 +61,8 @@ class InstallTest(MockPackagesTest): def tearDown(self): super(InstallTest, self).tearDown() - if self.stage is not None: - self.stage.destroy() + if self.repo.stage is not None: + self.repo.stage.destroy() # Turn checksumming back on spack.do_checksum = True @@ -96,7 +74,7 @@ class InstallTest(MockPackagesTest): def test_install_and_uninstall(self): # Get a basic concrete spec for the trivial install package. - spec = Spec(install_test_package) + spec = Spec('trivial_install_test_package') spec.concretize() self.assertTrue(spec.concrete) @@ -104,8 +82,7 @@ class InstallTest(MockPackagesTest): pkg = spack.db.get(spec) # Fake the URL for the package so it downloads from a file. - archive_path = join_path(self.stage.path, archive_name) - pkg.fetcher = URLFetchStrategy('file://' + archive_path) + pkg.fetcher = URLFetchStrategy(self.repo.url) try: pkg.do_install() diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py new file mode 100644 index 0000000000..51334198ec --- /dev/null +++ b/lib/spack/spack/test/mirror.py @@ -0,0 +1,156 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +from filecmp import dircmp + +import spack +import spack.mirror +from spack.util.compression import decompressor_for +from spack.test.mock_packages_test import * +from spack.test.mock_repo import * + +# paths in repos that shouldn't be in the mirror tarballs. +exclude = ['.hg', '.git', '.svn'] + + +class MirrorTest(MockPackagesTest): + def setUp(self): + """Sets up a mock package and a mock repo for each fetch strategy, to + ensure that the mirror can create archives for each of them. + """ + super(MirrorTest, self).setUp() + self.repos = {} + + + def set_up_package(self, name, mock_repo_class, url_attr): + """Use this to set up a mock package to be mirrored. + Each package needs us to: + 1. Set up a mock repo/archive to fetch from. + 2. Point the package's version args at that repo. + """ + # Set up packages to point at mock repos. + spec = Spec(name) + spec.concretize() + + # Get the package and fix its fetch args to point to a mock repo + pkg = spack.db.get(spec) + repo = mock_repo_class() + self.repos[name] = repo + + # change the fetch args of the first (only) version. + assert(len(pkg.versions) == 1) + v = next(iter(pkg.versions)) + pkg.versions[v][url_attr] = repo.url + + + def tearDown(self): + """Destroy all the stages created by the repos in setup.""" + super(MirrorTest, self).tearDown() + + for name, repo in self.repos.items(): + if repo.stage: + repo.stage.destroy() + + self.repos.clear() + + + def check_mirror(self): + stage = Stage('spack-mirror-test') + mirror_root = join_path(stage.path, 'test-mirror') + + try: + os.chdir(stage.path) + spack.mirror.create( + mirror_root, self.repos, no_checksum=True) + + # Stage directory exists + self.assertTrue(os.path.isdir(mirror_root)) + + # subdirs for each package + for name in self.repos: + subdir = join_path(mirror_root, name) + self.assertTrue(os.path.isdir(subdir)) + + files = os.listdir(subdir) + self.assertEqual(len(files), 1) + + # Decompress archive in the mirror + archive = files[0] + archive_path = join_path(subdir, archive) + decomp = decompressor_for(archive_path) + + with working_dir(subdir): + decomp(archive_path) + + # Find the untarred archive directory. + files = os.listdir(subdir) + self.assertEqual(len(files), 2) + self.assertTrue(archive in files) + files.remove(archive) + + expanded_archive = join_path(subdir, files[0]) + self.assertTrue(os.path.isdir(expanded_archive)) + + # Compare the original repo with the expanded archive + repo = self.repos[name] + if not 'svn' in name: + original_path = repo.path + else: + co = 'checked_out' + svn('checkout', repo.url, co) + original_path = join_path(subdir, co) + + dcmp = dircmp(original_path, expanded_archive) + + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + + finally: + stage.destroy() + + + def test_git_mirror(self): + self.set_up_package('git-test', MockGitRepo, 'git') + self.check_mirror() + + def test_svn_mirror(self): + self.set_up_package('svn-test', MockSvnRepo, 'svn') + self.check_mirror() + + def test_hg_mirror(self): + self.set_up_package('hg-test', MockHgRepo, 'hg') + self.check_mirror() + + def test_url_mirror(self): + self.set_up_package('trivial_install_test_package', MockArchive, 'url') + self.check_mirror() + + def test_all_mirror(self): + self.set_up_package('git-test', MockGitRepo, 'git') + self.set_up_package('svn-test', MockSvnRepo, 'svn') + self.set_up_package('hg-test', MockHgRepo, 'hg') + self.set_up_package('trivial_install_test_package', MockArchive, 'url') + self.check_mirror() diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ae28f7224e..659f29067a 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -22,7 +22,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os import shutil +from contextlib import closing from llnl.util.filesystem import * @@ -32,31 +34,71 @@ from spack.stage import Stage from spack.util.executable import which +# +# VCS Systems used by mock repo code. +# +git = which('git', required=True) +svn = which('svn', required=True) +svnadmin = which('svnadmin', required=True) +hg = which('hg', required=True) +tar = which('tar', required=True) + + class MockRepo(object): def __init__(self, stage_name, repo_name): - """This creates a stage and a repo directory within the stage.""" + """This creates a stage where some archive/repo files can be staged + for testing spack's fetch strategies.""" # Stage where this repo has been created self.stage = Stage(stage_name) # Full path to the repo within the stage. - self.path = join_path(self.stage.path, 'mock-git-repo') + self.path = join_path(self.stage.path, repo_name) mkdirp(self.path) + +class MockArchive(MockRepo): + """Creates a very simple archive directory with a configure script and a + makefile that installs to a prefix. Tars it up into an archive.""" + + def __init__(self): + repo_name = 'mock-archive-repo' + super(MockArchive, self).__init__('mock-archive-stage', repo_name) + + with working_dir(self.path): + configure = join_path(self.path, 'configure') + + with closing(open(configure, 'w')) as cfg_file: + cfg_file.write( + "#!/bin/sh\n" + "prefix=$(echo $1 | sed 's/--prefix=//')\n" + "cat > Makefile <<EOF\n" + "all:\n" + "\techo Building...\n\n" + "install:\n" + "\tmkdir -p $prefix\n" + "\ttouch $prefix/dummy_file\n" + "EOF\n") + os.chmod(configure, 0755) + + with working_dir(self.stage.path): + archive_name = "%s.tar.gz" % repo_name + tar('-czf', archive_name, repo_name) + + self.archive_path = join_path(self.stage.path, archive_name) + self.url = 'file://' + self.archive_path + + +class MockVCSRepo(MockRepo): + def __init__(self, stage_name, repo_name): + """This creates a stage and a repo directory within the stage.""" + super(MockVCSRepo, self).__init__(stage_name, repo_name) + # Name for rev0 & rev1 files in the repo to be self.r0_file = 'r0_file' self.r1_file = 'r1_file' -# -# VCS Systems used by mock repo code. -# -git = which('git', required=True) -svn = which('svn', required=True) -svnadmin = which('svnadmin', required=True) -hg = which('hg', required=True) - - -class MockGitRepo(MockRepo): +class MockGitRepo(MockVCSRepo): def __init__(self): super(MockGitRepo, self).__init__('mock-git-stage', 'mock-git-repo') @@ -97,17 +139,20 @@ class MockGitRepo(MockRepo): self.r1 = self.rev_hash(self.branch) self.r1_file = self.branch_file + self.url = self.path + def rev_hash(self, rev): return git('rev-parse', rev, return_output=True).strip() -class MockSvnRepo(MockRepo): +class MockSvnRepo(MockVCSRepo): def __init__(self): super(MockSvnRepo, self).__init__('mock-svn-stage', 'mock-svn-repo') + self.url = 'file://' + self.path + with working_dir(self.stage.path): svnadmin('create', self.path) - self.url = 'file://' + self.path tmp_path = join_path(self.stage.path, 'tmp-path') mkdirp(tmp_path) @@ -129,9 +174,10 @@ class MockSvnRepo(MockRepo): self.r1 = '2' -class MockHgRepo(MockRepo): +class MockHgRepo(MockVCSRepo): def __init__(self): super(MockHgRepo, self).__init__('mock-hg-stage', 'mock-hg-repo') + self.url = 'file://' + self.path with working_dir(self.path): hg('init') |