summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/cmd/mirror.py20
-rw-r--r--lib/spack/spack/concretize.py1
-rw-r--r--lib/spack/spack/fetch_strategy.py39
-rw-r--r--lib/spack/spack/mirror.py48
-rw-r--r--lib/spack/spack/package.py12
-rw-r--r--lib/spack/spack/test/__init__.py3
-rw-r--r--lib/spack/spack/test/install.py37
-rw-r--r--lib/spack/spack/test/mirror.py156
-rw-r--r--lib/spack/spack/test/mock_repo.py76
9 files changed, 276 insertions, 116 deletions
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 6a6c2b60c7..6e2e4dfb24 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -54,6 +54,9 @@ def setup_parser(subparser):
'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
create_parser.add_argument(
'-f', '--file', help="File with specs of packages to put in mirror.")
+ create_parser.add_argument(
+ '-o', '--one-version-per-spec', action='store_const', const=1, default=0,
+ help="Only fetch one 'preferred' version per spec, not all known versions.")
add_parser = sp.add_parser('add', help=mirror_add.__doc__)
add_parser.add_argument('name', help="Mnemonic name for mirror.")
@@ -128,26 +131,29 @@ def mirror_create(args):
# If nothing is passed, use all packages.
if not specs:
specs = [Spec(n) for n in spack.db.all_package_names()]
+ specs.sort(key=lambda s: s.format("$_$@").lower())
# Default name for directory is spack-mirror-<DATESTAMP>
- if not args.directory:
+ directory = args.directory
+ if not directory:
timestamp = datetime.now().strftime("%Y-%m-%d")
- args.directory = 'spack-mirror-' + timestamp
+ directory = 'spack-mirror-' + timestamp
# Make sure nothing is in the way.
existed = False
- if os.path.isfile(args.directory):
- tty.error("%s already exists and is a file." % args.directory)
- elif os.path.isdir(args.directory):
+ if os.path.isfile(directory):
+ tty.error("%s already exists and is a file." % directory)
+ elif os.path.isdir(directory):
existed = True
# Actually do the work to create the mirror
- present, mirrored, error = spack.mirror.create(args.directory, specs)
+ present, mirrored, error = spack.mirror.create(
+ directory, specs, num_versions=args.one_version_per_spec)
p, m, e = len(present), len(mirrored), len(error)
verb = "updated" if existed else "created"
tty.msg(
- "Successfully %s mirror in %s." % (verb, args.directory),
+ "Successfully %s mirror in %s." % (verb, directory),
"Archive stats:",
" %-4d already present" % p,
" %-4d added" % m,
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 9f9cd1789d..e603806af9 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -73,7 +73,6 @@ class DefaultConcretizer(object):
if valid_versions:
spec.versions = ver([valid_versions[-1]])
else:
- print spec
raise NoValidVersionError(spec)
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 2cff15845b..2b574eaba7 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -93,9 +93,6 @@ class FetchStrategy(object):
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
- @property
- def unique_name(self): pass
-
# This method is used to match fetch strategies to version()
# arguments in packages.
@classmethod
@@ -197,7 +194,10 @@ class URLFetchStrategy(FetchStrategy):
"""Just moves this archive to the destination."""
if not self.archive_file:
raise NoArchiveFileError("Cannot call archive() before fetching.")
- assert(extension(destination) == extension(self.archive_file))
+
+ if not extension(destination) == extension(self.archive_file):
+ raise ValueError("Cannot archive without matching extensions.")
+
shutil.move(self.archive_file, destination)
@@ -236,10 +236,6 @@ class URLFetchStrategy(FetchStrategy):
else:
return "URLFetchStrategy<no url>"
- @property
- def unique_name(self):
- return "spack-fetch-url:%s" % self
-
class VCSFetchStrategy(FetchStrategy):
def __init__(self, name, *rev_types, **kwargs):
@@ -393,17 +389,6 @@ class GitFetchStrategy(VCSFetchStrategy):
self.git('clean', '-f')
- @property
- def unique_name(self):
- name = "spack-fetch-git:%s" % self.url
- if self.commit:
- name += "@" + self.commit
- elif self.branch:
- name += "@" + self.branch
- elif self.tag:
- name += "@" + self.tag
-
-
class SvnFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a subversion repository.
Use like this in a package:
@@ -477,14 +462,6 @@ class SvnFetchStrategy(VCSFetchStrategy):
self.svn('revert', '.', '-R')
- @property
- def unique_name(self):
- name = "spack-fetch-svn:%s" % self.url
- if self.revision:
- name += "@" + self.revision
-
-
-
class HgFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a Mercurial repository.
Use like this in a package:
@@ -560,14 +537,6 @@ class HgFetchStrategy(VCSFetchStrategy):
self.stage.chdir_to_source()
- @property
- def unique_name(self):
- name = "spack-fetch-hg:%s" % self.url
- if self.revision:
- name += "@" + self.revision
-
-
-
def from_url(url):
"""Given a URL, find an appropriate fetch strategy for it.
Currently just gives you a URLFetchStrategy that uses curl.
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index e116ce83b2..f7bbb3f840 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -49,16 +49,18 @@ def mirror_archive_filename(spec):
if not spec.version.concrete:
raise ValueError("mirror.path requires spec with concrete version.")
- url = spec.package.default_url
- if url is None:
- ext = 'tar.gz'
+ fetcher = spec.package.fetcher
+ if isinstance(fetcher, fs.URLFetchStrategy):
+ # If we fetch this version with a URLFetchStrategy, use URL's archive type
+ ext = extension(fetcher.url)
else:
- ext = extension(url)
+ # Otherwise we'll make a .tar.gz ourselves
+ ext = 'tar.gz'
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
-def get_matching_versions(specs):
+def get_matching_versions(specs, **kwargs):
"""Get a spec for EACH known version matching any spec in the list."""
matching = []
for spec in specs:
@@ -69,11 +71,18 @@ def get_matching_versions(specs):
tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
continue
- for v in reversed(sorted(pkg.versions)):
+ num_versions = kwargs.get('num_versions', 0)
+ for i, v in enumerate(reversed(sorted(pkg.versions))):
+ # Generate no more than num_versions versions for each spec.
+ if num_versions and i >= num_versions:
+ break
+
+ # Generate only versions that satisfy the spec.
if v.satisfies(spec.versions):
s = Spec(pkg.name)
s.versions = VersionList([v])
matching.append(s)
+
return matching
@@ -86,6 +95,11 @@ def create(path, specs, **kwargs):
specs Any package versions matching these specs will be added
to the mirror.
+ Keyword args:
+ no_checksum: If True, do not checkpoint when fetching (default False)
+ num_versions: Max number of versions to fetch per spec,
+ if spec is ambiguous (default is 0 for all of them)
+
Return Value:
Returns a tuple of lists: (present, mirrored, error)
* present: Package specs that were already prsent.
@@ -104,13 +118,15 @@ def create(path, specs, **kwargs):
specs = [s if isinstance(s, Spec) else Spec(s) for s in specs]
# Get concrete specs for each matching version of these specs.
- version_specs = get_matching_versions(specs)
+ version_specs = get_matching_versions(
+ specs, num_versions=kwargs.get('num_versions', 0))
for s in version_specs:
s.concretize()
- # Create a directory if none exists
- if not os.path.isdir(path):
- mkdirp(path)
+ # Get the absolute path of the root before we start jumping around.
+ mirror_root = os.path.abspath(path)
+ if not os.path.isdir(mirror_root):
+ mkdirp(mirror_root)
# Things to keep track of while parsing specs.
present = []
@@ -124,19 +140,21 @@ def create(path, specs, **kwargs):
stage = None
try:
# create a subdirectory for the current package@version
- realpath = os.path.realpath(path)
- subdir = join_path(realpath, pkg.name)
+ subdir = join_path(mirror_root, pkg.name)
mkdirp(subdir)
archive_file = mirror_archive_filename(spec)
archive_path = join_path(subdir, archive_file)
- if os.path.exists(archive_path):
+
+ if os.path.exists(archive_file):
+ tty.msg("%s is already present. Skipping." % spec.format("$_$@"))
present.append(spec)
continue
# Set up a stage and a fetcher for the download
+ unique_fetch_name = spec.format("$_$@")
fetcher = fs.for_package_version(pkg, pkg.version)
- stage = Stage(fetcher, name=fetcher.unique_name)
+ stage = Stage(fetcher, name=unique_fetch_name)
fetcher.set_stage(stage)
# Do the fetch and checksum if necessary
@@ -148,7 +166,7 @@ def create(path, specs, **kwargs):
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
- tty.msg("Added %s to mirror" % archive_path)
+ tty.msg("Added %s." % spec.format("$_$@"))
mirrored.append(spec)
except Exception, e:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 34efe1bec9..e462562e85 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -598,18 +598,6 @@ class Package(object):
return str(version)
- @property
- def default_url(self):
- if self.spec.versions.concrete:
- return self.url_for_version(self.version)
- else:
- url = getattr(self, 'url', None)
- if url:
- return url
-
- return None
-
-
def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories."""
spack.install_layout.remove_path_for_spec(self.spec)
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index ca4c869e42..be9ac5a560 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -50,7 +50,8 @@ test_names = ['versions',
'python_version',
'git_fetch',
'svn_fetch',
- 'hg_fetch']
+ 'hg_fetch',
+ 'mirror']
def list_tests():
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index 0d53bb45c7..e052f53e77 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -36,39 +36,17 @@ from spack.fetch_strategy import URLFetchStrategy
from spack.directory_layout import SpecHashDirectoryLayout
from spack.util.executable import which
from spack.test.mock_packages_test import *
+from spack.test.mock_repo import MockArchive
-dir_name = 'trivial-1.0'
-archive_name = 'trivial-1.0.tar.gz'
-install_test_package = 'trivial_install_test_package'
-
class InstallTest(MockPackagesTest):
"""Tests install and uninstall on a trivial package."""
def setUp(self):
super(InstallTest, self).setUp()
- self.stage = Stage('not_a_real_url')
- archive_dir = join_path(self.stage.path, dir_name)
- dummy_configure = join_path(archive_dir, 'configure')
-
- mkdirp(archive_dir)
- with closing(open(dummy_configure, 'w')) as configure:
- configure.write(
- "#!/bin/sh\n"
- "prefix=$(echo $1 | sed 's/--prefix=//')\n"
- "cat > Makefile <<EOF\n"
- "all:\n"
- "\techo Building...\n\n"
- "install:\n"
- "\tmkdir -p $prefix\n"
- "\ttouch $prefix/dummy_file\n"
- "EOF\n")
- os.chmod(dummy_configure, 0755)
-
- with working_dir(self.stage.path):
- tar = which('tar')
- tar('-czf', archive_name, dir_name)
+ # create a simple installable package directory and tarball
+ self.repo = MockArchive()
# We use a fake package, so skip the checksum.
spack.do_checksum = False
@@ -83,8 +61,8 @@ class InstallTest(MockPackagesTest):
def tearDown(self):
super(InstallTest, self).tearDown()
- if self.stage is not None:
- self.stage.destroy()
+ if self.repo.stage is not None:
+ self.repo.stage.destroy()
# Turn checksumming back on
spack.do_checksum = True
@@ -96,7 +74,7 @@ class InstallTest(MockPackagesTest):
def test_install_and_uninstall(self):
# Get a basic concrete spec for the trivial install package.
- spec = Spec(install_test_package)
+ spec = Spec('trivial_install_test_package')
spec.concretize()
self.assertTrue(spec.concrete)
@@ -104,8 +82,7 @@ class InstallTest(MockPackagesTest):
pkg = spack.db.get(spec)
# Fake the URL for the package so it downloads from a file.
- archive_path = join_path(self.stage.path, archive_name)
- pkg.fetcher = URLFetchStrategy('file://' + archive_path)
+ pkg.fetcher = URLFetchStrategy(self.repo.url)
try:
pkg.do_install()
diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py
new file mode 100644
index 0000000000..51334198ec
--- /dev/null
+++ b/lib/spack/spack/test/mirror.py
@@ -0,0 +1,156 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+from filecmp import dircmp
+
+import spack
+import spack.mirror
+from spack.util.compression import decompressor_for
+from spack.test.mock_packages_test import *
+from spack.test.mock_repo import *
+
+# paths in repos that shouldn't be in the mirror tarballs.
+exclude = ['.hg', '.git', '.svn']
+
+
+class MirrorTest(MockPackagesTest):
+ def setUp(self):
+ """Sets up a mock package and a mock repo for each fetch strategy, to
+ ensure that the mirror can create archives for each of them.
+ """
+ super(MirrorTest, self).setUp()
+ self.repos = {}
+
+
+ def set_up_package(self, name, mock_repo_class, url_attr):
+ """Use this to set up a mock package to be mirrored.
+ Each package needs us to:
+ 1. Set up a mock repo/archive to fetch from.
+ 2. Point the package's version args at that repo.
+ """
+ # Set up packages to point at mock repos.
+ spec = Spec(name)
+ spec.concretize()
+
+ # Get the package and fix its fetch args to point to a mock repo
+ pkg = spack.db.get(spec)
+ repo = mock_repo_class()
+ self.repos[name] = repo
+
+ # change the fetch args of the first (only) version.
+ assert(len(pkg.versions) == 1)
+ v = next(iter(pkg.versions))
+ pkg.versions[v][url_attr] = repo.url
+
+
+ def tearDown(self):
+ """Destroy all the stages created by the repos in setup."""
+ super(MirrorTest, self).tearDown()
+
+ for name, repo in self.repos.items():
+ if repo.stage:
+ repo.stage.destroy()
+
+ self.repos.clear()
+
+
+ def check_mirror(self):
+ stage = Stage('spack-mirror-test')
+ mirror_root = join_path(stage.path, 'test-mirror')
+
+ try:
+ os.chdir(stage.path)
+ spack.mirror.create(
+ mirror_root, self.repos, no_checksum=True)
+
+ # Stage directory exists
+ self.assertTrue(os.path.isdir(mirror_root))
+
+ # subdirs for each package
+ for name in self.repos:
+ subdir = join_path(mirror_root, name)
+ self.assertTrue(os.path.isdir(subdir))
+
+ files = os.listdir(subdir)
+ self.assertEqual(len(files), 1)
+
+ # Decompress archive in the mirror
+ archive = files[0]
+ archive_path = join_path(subdir, archive)
+ decomp = decompressor_for(archive_path)
+
+ with working_dir(subdir):
+ decomp(archive_path)
+
+ # Find the untarred archive directory.
+ files = os.listdir(subdir)
+ self.assertEqual(len(files), 2)
+ self.assertTrue(archive in files)
+ files.remove(archive)
+
+ expanded_archive = join_path(subdir, files[0])
+ self.assertTrue(os.path.isdir(expanded_archive))
+
+ # Compare the original repo with the expanded archive
+ repo = self.repos[name]
+ if not 'svn' in name:
+ original_path = repo.path
+ else:
+ co = 'checked_out'
+ svn('checkout', repo.url, co)
+ original_path = join_path(subdir, co)
+
+ dcmp = dircmp(original_path, expanded_archive)
+
+ # make sure there are no new files in the expanded tarball
+ self.assertFalse(dcmp.right_only)
+ self.assertTrue(all(l in exclude for l in dcmp.left_only))
+
+ finally:
+ stage.destroy()
+
+
+ def test_git_mirror(self):
+ self.set_up_package('git-test', MockGitRepo, 'git')
+ self.check_mirror()
+
+ def test_svn_mirror(self):
+ self.set_up_package('svn-test', MockSvnRepo, 'svn')
+ self.check_mirror()
+
+ def test_hg_mirror(self):
+ self.set_up_package('hg-test', MockHgRepo, 'hg')
+ self.check_mirror()
+
+ def test_url_mirror(self):
+ self.set_up_package('trivial_install_test_package', MockArchive, 'url')
+ self.check_mirror()
+
+ def test_all_mirror(self):
+ self.set_up_package('git-test', MockGitRepo, 'git')
+ self.set_up_package('svn-test', MockSvnRepo, 'svn')
+ self.set_up_package('hg-test', MockHgRepo, 'hg')
+ self.set_up_package('trivial_install_test_package', MockArchive, 'url')
+ self.check_mirror()
diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py
index ae28f7224e..659f29067a 100644
--- a/lib/spack/spack/test/mock_repo.py
+++ b/lib/spack/spack/test/mock_repo.py
@@ -22,7 +22,9 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+import os
import shutil
+from contextlib import closing
from llnl.util.filesystem import *
@@ -32,31 +34,71 @@ from spack.stage import Stage
from spack.util.executable import which
+#
+# VCS Systems used by mock repo code.
+#
+git = which('git', required=True)
+svn = which('svn', required=True)
+svnadmin = which('svnadmin', required=True)
+hg = which('hg', required=True)
+tar = which('tar', required=True)
+
+
class MockRepo(object):
def __init__(self, stage_name, repo_name):
- """This creates a stage and a repo directory within the stage."""
+ """This creates a stage where some archive/repo files can be staged
+ for testing spack's fetch strategies."""
# Stage where this repo has been created
self.stage = Stage(stage_name)
# Full path to the repo within the stage.
- self.path = join_path(self.stage.path, 'mock-git-repo')
+ self.path = join_path(self.stage.path, repo_name)
mkdirp(self.path)
+
+class MockArchive(MockRepo):
+ """Creates a very simple archive directory with a configure script and a
+ makefile that installs to a prefix. Tars it up into an archive."""
+
+ def __init__(self):
+ repo_name = 'mock-archive-repo'
+ super(MockArchive, self).__init__('mock-archive-stage', repo_name)
+
+ with working_dir(self.path):
+ configure = join_path(self.path, 'configure')
+
+ with closing(open(configure, 'w')) as cfg_file:
+ cfg_file.write(
+ "#!/bin/sh\n"
+ "prefix=$(echo $1 | sed 's/--prefix=//')\n"
+ "cat > Makefile <<EOF\n"
+ "all:\n"
+ "\techo Building...\n\n"
+ "install:\n"
+ "\tmkdir -p $prefix\n"
+ "\ttouch $prefix/dummy_file\n"
+ "EOF\n")
+ os.chmod(configure, 0755)
+
+ with working_dir(self.stage.path):
+ archive_name = "%s.tar.gz" % repo_name
+ tar('-czf', archive_name, repo_name)
+
+ self.archive_path = join_path(self.stage.path, archive_name)
+ self.url = 'file://' + self.archive_path
+
+
+class MockVCSRepo(MockRepo):
+ def __init__(self, stage_name, repo_name):
+ """This creates a stage and a repo directory within the stage."""
+ super(MockVCSRepo, self).__init__(stage_name, repo_name)
+
# Name for rev0 & rev1 files in the repo to be
self.r0_file = 'r0_file'
self.r1_file = 'r1_file'
-#
-# VCS Systems used by mock repo code.
-#
-git = which('git', required=True)
-svn = which('svn', required=True)
-svnadmin = which('svnadmin', required=True)
-hg = which('hg', required=True)
-
-
-class MockGitRepo(MockRepo):
+class MockGitRepo(MockVCSRepo):
def __init__(self):
super(MockGitRepo, self).__init__('mock-git-stage', 'mock-git-repo')
@@ -97,17 +139,20 @@ class MockGitRepo(MockRepo):
self.r1 = self.rev_hash(self.branch)
self.r1_file = self.branch_file
+ self.url = self.path
+
def rev_hash(self, rev):
return git('rev-parse', rev, return_output=True).strip()
-class MockSvnRepo(MockRepo):
+class MockSvnRepo(MockVCSRepo):
def __init__(self):
super(MockSvnRepo, self).__init__('mock-svn-stage', 'mock-svn-repo')
+ self.url = 'file://' + self.path
+
with working_dir(self.stage.path):
svnadmin('create', self.path)
- self.url = 'file://' + self.path
tmp_path = join_path(self.stage.path, 'tmp-path')
mkdirp(tmp_path)
@@ -129,9 +174,10 @@ class MockSvnRepo(MockRepo):
self.r1 = '2'
-class MockHgRepo(MockRepo):
+class MockHgRepo(MockVCSRepo):
def __init__(self):
super(MockHgRepo, self).__init__('mock-hg-stage', 'mock-hg-repo')
+ self.url = 'file://' + self.path
with working_dir(self.path):
hg('init')
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-10-30 20:01:29 +0000