summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rwxr-xr-xlib/spack/env/cc4
l---------lib/spack/env/nag/nagfor1
-rw-r--r--lib/spack/spack/cmd/bootstrap.py2
-rw-r--r--lib/spack/spack/cmd/create.py2
-rw-r--r--lib/spack/spack/cmd/deactivate.py2
-rw-r--r--lib/spack/spack/cmd/location.py2
-rw-r--r--lib/spack/spack/cmd/mirror.py14
-rw-r--r--lib/spack/spack/cmd/pkg.py2
-rw-r--r--lib/spack/spack/cmd/test-install.py40
-rw-r--r--lib/spack/spack/compiler.py2
-rw-r--r--lib/spack/spack/compilers/__init__.py2
-rw-r--r--lib/spack/spack/compilers/clang.py32
-rw-r--r--lib/spack/spack/compilers/nag.py33
-rw-r--r--lib/spack/spack/database.py2
-rw-r--r--lib/spack/spack/fetch_strategy.py6
-rw-r--r--lib/spack/spack/mirror.py19
-rw-r--r--lib/spack/spack/package.py14
-rw-r--r--lib/spack/spack/repository.py5
-rw-r--r--lib/spack/spack/stage.py18
-rw-r--r--lib/spack/spack/test/cc.py6
-rw-r--r--lib/spack/spack/test/git_fetch.py5
-rw-r--r--lib/spack/spack/test/hg_fetch.py5
-rw-r--r--lib/spack/spack/test/install.py4
-rw-r--r--lib/spack/spack/test/make_executable.py48
-rw-r--r--lib/spack/spack/test/mirror.py72
-rw-r--r--lib/spack/spack/test/mock_repo.py10
-rw-r--r--lib/spack/spack/test/svn_fetch.py7
-rw-r--r--lib/spack/spack/util/executable.py95
28 files changed, 296 insertions, 158 deletions
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index 0966277a91..aacba996b3 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -94,11 +94,11 @@ case "$command" in
command="$SPACK_CXX"
language="C++"
;;
- f90|fc|f95|gfortran|ifort|pgf90|xlf90)
+ f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor)
command="$SPACK_FC"
language="Fortran 90"
;;
- f77|gfortran|ifort|pgf77|xlf)
+ f77|gfortran|ifort|pgf77|xlf|nagfor)
command="$SPACK_F77"
language="Fortran 77"
;;
diff --git a/lib/spack/env/nag/nagfor b/lib/spack/env/nag/nagfor
new file mode 120000
index 0000000000..82c2b8e90a
--- /dev/null
+++ b/lib/spack/env/nag/nagfor
@@ -0,0 +1 @@
+../cc \ No newline at end of file
diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py
index e4ec7da35d..bdbd623b39 100644
--- a/lib/spack/spack/cmd/bootstrap.py
+++ b/lib/spack/spack/cmd/bootstrap.py
@@ -42,7 +42,7 @@ def get_origin_url():
git = which('git', required=True)
origin_url = git(
'--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url',
- return_output=True)
+ output=str)
return origin_url.strip()
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index 7cea39cb55..edcea0718c 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -132,7 +132,7 @@ class ConfigureGuesser(object):
# Peek inside the tarball.
tar = which('tar')
output = tar(
- "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True)
+ "--exclude=*/*/*", "-tf", stage.archive_file, output=str)
lines = output.split("\n")
# Set the configure line to the one that matched.
diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py
index a0c78bf755..d6b23d6a08 100644
--- a/lib/spack/spack/cmd/deactivate.py
+++ b/lib/spack/spack/cmd/deactivate.py
@@ -37,7 +37,7 @@ def setup_parser(subparser):
help="Run deactivation even if spec is NOT currently activated.")
subparser.add_argument(
'-a', '--all', action='store_true',
- help="Deactivate all extensions of an extendable pacakge, or "
+ help="Deactivate all extensions of an extendable package, or "
"deactivate an extension AND its dependencies.")
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.")
diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py
index 39c225e9b2..307ee8982d 100644
--- a/lib/spack/spack/cmd/location.py
+++ b/lib/spack/spack/cmd/location.py
@@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path
import spack
import spack.cmd
-description="Print out locations of various diectories used by Spack"
+description="Print out locations of various directories used by Spack"
def setup_parser(subparser):
global directories
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 885483a840..8e9438c1a3 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -53,11 +53,13 @@ def setup_parser(subparser):
create_parser.add_argument('-d', '--directory', default=None,
help="Directory in which to create mirror.")
create_parser.add_argument(
- 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
+ 'specs', nargs=argparse.REMAINDER,
+ help="Specs of packages to put in mirror")
create_parser.add_argument(
'-f', '--file', help="File with specs of packages to put in mirror.")
create_parser.add_argument(
- '-D', '--dependencies', action='store_true', help="Also fetch all dependencies")
+ '-D', '--dependencies', action='store_true',
+ help="Also fetch all dependencies")
create_parser.add_argument(
'-o', '--one-version-per-spec', action='store_const', const=1, default=0,
help="Only fetch one 'preferred' version per spec, not all known versions.")
@@ -74,7 +76,8 @@ def setup_parser(subparser):
help="Configuration scope to modify.")
# Remove
- remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__)
+ remove_parser = sp.add_parser('remove', aliases=['rm'],
+ help=mirror_remove.__doc__)
remove_parser.add_argument('name')
remove_parser.add_argument(
'--scope', choices=scopes, default=spack.cmd.default_modify_scope,
@@ -141,15 +144,17 @@ def mirror_list(args):
def _read_specs_from_file(filename):
+ specs = []
with open(filename, "r") as stream:
for i, string in enumerate(stream):
try:
s = Spec(string)
s.package
- args.specs.append(s)
+ specs.append(s)
except SpackError, e:
tty.die("Parse error in %s, line %d:" % (args.file, i+1),
">>> " + string, str(e))
+ return specs
def mirror_create(args):
@@ -169,6 +174,7 @@ def mirror_create(args):
specs = [Spec(n) for n in spack.repo.all_package_names()]
specs.sort(key=lambda s: s.format("$_$@").lower())
+ # If the user asked for dependencies, traverse spec DAG get them.
if args.dependencies:
new_specs = set()
for spec in specs:
diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py
index 448f762841..cf478d3763 100644
--- a/lib/spack/spack/cmd/pkg.py
+++ b/lib/spack/spack/cmd/pkg.py
@@ -79,7 +79,7 @@ def list_packages(rev):
git = get_git()
relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep
output = git('ls-tree', '--full-tree', '--name-only', rev, relpath,
- return_output=True)
+ output=str)
return sorted(line[len(relpath):] for line in output.split('\n') if line)
diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py
index 74017f59fb..656873a2f0 100644
--- a/lib/spack/spack/cmd/test-install.py
+++ b/lib/spack/spack/cmd/test-install.py
@@ -37,20 +37,20 @@ from spack.build_environment import InstallError
from spack.fetch_strategy import FetchError
import spack.cmd
-description = "Treat package installations as unit tests and output formatted test results"
+description = "Run package installation as a unit test, output formatted results."
def setup_parser(subparser):
subparser.add_argument(
'-j', '--jobs', action='store', type=int,
help="Explicitly set number of make jobs. Default is #cpus.")
-
+
subparser.add_argument(
'-n', '--no-checksum', action='store_true', dest='no_checksum',
help="Do not check packages against checksum")
-
+
subparser.add_argument(
'-o', '--output', action='store', help="test output goes in this file")
-
+
subparser.add_argument(
'package', nargs=argparse.REMAINDER, help="spec of package to install")
@@ -59,10 +59,10 @@ class JunitResultFormat(object):
def __init__(self):
self.root = ET.Element('testsuite')
self.tests = []
-
+
def add_test(self, buildId, testResult, buildInfo=None):
self.tests.append((buildId, testResult, buildInfo))
-
+
def write_to(self, stream):
self.root.set('tests', '{0}'.format(len(self.tests)))
for buildId, testResult, buildInfo in self.tests:
@@ -84,25 +84,25 @@ class TestResult(object):
PASSED = 0
FAILED = 1
SKIPPED = 2
-
+
class BuildId(object):
def __init__(self, spec):
self.name = spec.name
self.version = spec.version
self.hashId = spec.dag_hash()
-
+
def stringId(self):
return "-".join(str(x) for x in (self.name, self.version, self.hashId))
def __hash__(self):
return hash((self.name, self.version, self.hashId))
-
+
def __eq__(self, other):
if not isinstance(other, BuildId):
return False
-
- return ((self.name, self.version, self.hashId) ==
+
+ return ((self.name, self.version, self.hashId) ==
(other.name, other.version, other.hashId))
@@ -114,12 +114,12 @@ def fetch_log(path):
def failed_dependencies(spec):
- return set(childSpec for childSpec in spec.dependencies.itervalues() if not
+ return set(childSpec for childSpec in spec.dependencies.itervalues() if not
spack.repo.get(childSpec).installed)
def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log):
- # Post-order traversal is not strictly required but it makes sense to output
+ # Post-order traversal is not strictly required but it makes sense to output
# tests for dependencies first.
for spec in topSpec.traverse(order='post'):
if spec not in newInstalls:
@@ -143,12 +143,12 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log):
re.search('error:', line, re.IGNORECASE))
errOutput = errMessages if errMessages else lines[-10:]
errOutput = '\n'.join(itertools.chain(
- [spec.to_yaml(), "Errors:"], errOutput,
+ [spec.to_yaml(), "Errors:"], errOutput,
["Build Log:", package.build_log_path]))
else:
result = TestResult.PASSED
errOutput = None
-
+
bId = BuildId(spec)
output.add_test(bId, result, errOutput)
@@ -163,18 +163,18 @@ def test_install(parser, args):
if args.no_checksum:
spack.do_checksum = False # TODO: remove this global.
-
+
specs = spack.cmd.parse_specs(args.package, concretize=True)
if len(specs) > 1:
tty.die("Only 1 top-level package can be specified")
topSpec = iter(specs).next()
-
+
newInstalls = set()
for spec in topSpec.traverse():
package = spack.repo.get(spec)
if not package.installed:
newInstalls.add(spec)
-
+
if not args.output:
bId = BuildId(topSpec)
outputDir = join_path(os.getcwd(), "test-output")
@@ -183,7 +183,7 @@ def test_install(parser, args):
outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId()))
else:
outputFpath = args.output
-
+
for spec in topSpec.traverse(order='post'):
# Calling do_install for the top-level package would be sufficient but
# this attempts to keep going if any package fails (other packages which
@@ -202,7 +202,7 @@ def test_install(parser, args):
pass
except FetchError:
pass
-
+
jrf = JunitResultFormat()
handled = {}
create_test_output(topSpec, newInstalls, jrf)
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index a665f6062d..12c02e0ea2 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -51,7 +51,7 @@ _version_cache = {}
def get_compiler_version(compiler_path, version_arg, regex='(.*)'):
if not compiler_path in _version_cache:
compiler = Executable(compiler_path)
- output = compiler(version_arg, return_output=True, error=os.devnull)
+ output = compiler(version_arg, output=str, error=str)
match = re.search(regex, output)
_version_cache[compiler_path] = match.group(1) if match else 'unknown'
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index facc9c338b..6159ef576c 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -51,7 +51,7 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
if platform.system() == 'Darwin':
_default_order = ['clang', 'gcc', 'intel']
else:
- _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc']
+ _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag']
def _auto_compiler_spec(function):
diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py
index 340051019c..e406d86a24 100644
--- a/lib/spack/spack/compilers/clang.py
+++ b/lib/spack/spack/compilers/clang.py
@@ -22,7 +22,10 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+import re
+import spack.compiler as cpr
from spack.compiler import *
+from spack.util.executable import *
class Clang(Compiler):
# Subclasses use possible names of C compiler
@@ -47,11 +50,34 @@ class Clang(Compiler):
@classmethod
def default_version(self, comp):
"""The '--version' option works for clang compilers.
- Output looks like this::
+ On most platforms, output looks like this::
clang version 3.1 (trunk 149096)
Target: x86_64-unknown-linux-gnu
Thread model: posix
+
+ On Mac OS X, it looks like this:
+
+ Apple LLVM version 7.0.2 (clang-700.1.81)
+ Target: x86_64-apple-darwin15.2.0
+ Thread model: posix
+
"""
- return get_compiler_version(
- comp, '--version', r'(?:clang version|based on LLVM) ([^ )]+)')
+ if comp not in cpr._version_cache:
+ compiler = Executable(comp)
+ output = compiler('--version', output=str, error=str)
+
+ ver = 'unknown'
+ match = re.search(r'^Apple LLVM version ([^ )]+)', output)
+ if match:
+ # Apple's LLVM compiler has its own versions, so suffix them.
+ ver = match.group(1) + '-apple'
+ else:
+ # Normal clang compiler versions are left as-is
+ match = re.search(r'^clang version ([^ )]+)', output)
+ if match:
+ ver = match.group(1)
+
+ cpr._version_cache[comp] = ver
+
+ return cpr._version_cache[comp]
diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py
new file mode 100644
index 0000000000..527a05a090
--- /dev/null
+++ b/lib/spack/spack/compilers/nag.py
@@ -0,0 +1,33 @@
+from spack.compiler import *
+
+class Nag(Compiler):
+ # Subclasses use possible names of C compiler
+ cc_names = []
+
+ # Subclasses use possible names of C++ compiler
+ cxx_names = []
+
+ # Subclasses use possible names of Fortran 77 compiler
+ f77_names = ['nagfor']
+
+ # Subclasses use possible names of Fortran 90 compiler
+ fc_names = ['nagfor']
+
+ # Named wrapper links within spack.build_env_path
+ link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml
+ 'cc' : 'cc',
+ 'cxx' : 'c++',
+ 'f77' : 'nag/nagfor',
+ 'fc' : 'nag/nagfor' }
+
+ @classmethod
+ def default_version(self, comp):
+ """The '-V' option works for nag compilers.
+ Output looks like this::
+
+ NAG Fortran Compiler Release 6.0(Hibiya) Build 1037
+ Product NPL6A60NA for x86-64 Linux
+ Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K.
+ """
+ return get_compiler_version(
+ comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)')
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
index 0fa18db34b..9cbe7de44a 100644
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -489,7 +489,7 @@ class Database(object):
1. Marks the spec as not installed.
2. Removes the spec if it has no more dependents.
3. If removed, recursively updates dependencies' ref counts
- and remvoes them if they are no longer needed.
+ and removes them if they are no longer needed.
"""
# Take a lock around the entire removal.
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 0657146bf6..337dd1e198 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -154,7 +154,7 @@ class URLFetchStrategy(FetchStrategy):
# Run curl but grab the mime type from the http headers
headers = spack.curl(
- *curl_args, return_output=True, fail_on_error=False)
+ *curl_args, output=str, fail_on_error=False)
if spack.curl.returncode != 0:
# clean up archive on failure.
@@ -375,7 +375,7 @@ class GitFetchStrategy(VCSFetchStrategy):
@property
def git_version(self):
- vstring = self.git('--version', return_output=True).lstrip('git version ')
+ vstring = self.git('--version', output=str).lstrip('git version ')
return Version(vstring)
@@ -518,7 +518,7 @@ class SvnFetchStrategy(VCSFetchStrategy):
def _remove_untracked_files(self):
"""Removes untracked files in an svn repository."""
- status = self.svn('status', '--no-ignore', return_output=True)
+ status = self.svn('status', '--no-ignore', output=str)
self.svn('status', '--no-ignore')
for line in status.split('\n'):
if not re.match('^[I?]', line):
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 1d9b0e7ef2..341cc4cb88 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -78,6 +78,7 @@ def get_matching_versions(specs, **kwargs):
continue
num_versions = kwargs.get('num_versions', 0)
+ matching_spec = []
for i, v in enumerate(reversed(sorted(pkg.versions))):
# Generate no more than num_versions versions for each spec.
if num_versions and i >= num_versions:
@@ -88,7 +89,11 @@ def get_matching_versions(specs, **kwargs):
s = Spec(pkg.name)
s.versions = VersionList([v])
s.variants = spec.variants.copy()
- matching.append(s)
+ matching_spec.append(s)
+
+ if not matching_spec:
+ tty.warn("No known version matches spec: %s" % spec)
+ matching.extend(matching_spec)
return matching
@@ -147,7 +152,11 @@ def create(path, specs, **kwargs):
# Get the absolute path of the root before we start jumping around.
mirror_root = os.path.abspath(path)
if not os.path.isdir(mirror_root):
- mkdirp(mirror_root)
+ try:
+ mkdirp(mirror_root)
+ except OSError as e:
+ raise MirrorError(
+ "Cannot create directory '%s':" % mirror_root, str(e))
# Things to keep track of while parsing specs.
present = []
@@ -164,7 +173,11 @@ def create(path, specs, **kwargs):
# create a subdirectory for the current package@version
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec)))
subdir = os.path.dirname(archive_path)
- mkdirp(subdir)
+ try:
+ mkdirp(subdir)
+ except OSError as e:
+ raise MirrorError(
+ "Cannot create directory '%s':" % subdir, str(e))
if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@"))
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index fba2912b75..8cb947c276 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -631,7 +631,7 @@ class Package(object):
spack.install_layout.remove_install_directory(self.spec)
- def do_fetch(self):
+ def do_fetch(self, mirror_only=False):
"""Creates a stage directory and downloads the taball for this package.
Working directory will be set to the stage directory.
"""
@@ -656,7 +656,7 @@ class Package(object):
raise FetchError(
"Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
- self.stage.fetch()
+ self.stage.fetch(mirror_only)
##########
# Fetch resources
@@ -677,7 +677,8 @@ class Package(object):
if spack.do_checksum and self.version in self.versions:
self.stage.check()
- def do_stage(self):
+
+ def do_stage(self, mirror_only=False):
"""Unpacks the fetched tarball, then changes into the expanded tarball
directory."""
if not self.spec.concrete:
@@ -691,8 +692,7 @@ class Package(object):
else:
tty.msg("Already staged %s in %s." % (name, stage.path))
-
- self.do_fetch()
+ self.do_fetch(mirror_only)
_expand_archive(self.stage)
##########
@@ -835,10 +835,6 @@ class Package(object):
resource_stage_folder = '-'.join(pieces)
return resource_stage_folder
- def _build_logger(self, log_path):
- """Create a context manager to log build output."""
-
-
def do_install(self,
keep_prefix=False, keep_stage=False, ignore_deps=False,
diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py
index 31596cee7a..b5df1168b6 100644
--- a/lib/spack/spack/repository.py
+++ b/lib/spack/spack/repository.py
@@ -233,6 +233,11 @@ class RepoPath(object):
return providers
+ @_autospec
+ def extensions_for(self, extendee_spec):
+ return [p for p in self.all_packages() if p.extends(extendee_spec)]
+
+
def find_module(self, fullname, path=None):
"""Implements precedence for overlaid namespaces.
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 31635a854a..79c9030e20 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -97,7 +97,6 @@ class Stage(object):
self.name = kwargs.get('name')
self.mirror_path = kwargs.get('mirror_path')
-
self.tmp_root = find_tmp_root()
self.path = None
@@ -240,11 +239,13 @@ class Stage(object):
tty.die("Setup failed: no such directory: " + self.path)
- def fetch(self):
+ def fetch(self, mirror_only=False):
"""Downloads an archive or checks out code from a repository."""
self.chdir()
- fetchers = [self.default_fetcher]
+ fetchers = []
+ if not mirror_only:
+ fetchers.append(self.default_fetcher)
# TODO: move mirror logic out of here and clean it up!
# TODO: Or @alalazo may have some ideas about how to use a
@@ -252,7 +253,13 @@ class Stage(object):
self.skip_checksum_for_mirror = True
if self.mirror_path:
mirrors = spack.config.get_config('mirrors')
- urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()]
+
+ # Join URLs of mirror roots with mirror paths. Because
+ # urljoin() will strip everything past the final '/' in
+ # the root, so we add a '/' if it is not present.
+ mirror_roots = [root if root.endswith('/') else root + '/'
+ for root in mirrors.values()]
+ urls = [urljoin(root, self.mirror_path) for root in mirror_roots]
# If this archive is normally fetched from a tarball URL,
# then use the same digest. `spack mirror` ensures that
@@ -261,10 +268,11 @@ class Stage(object):
if isinstance(self.default_fetcher, fs.URLFetchStrategy):
digest = self.default_fetcher.digest
- # Have to skip the checkesum for things archived from
+ # Have to skip the checksum for things archived from
# repositories. How can this be made safer?
self.skip_checksum_for_mirror = not bool(digest)
+ # Add URL strategies for all the mirrors with the digest
for url in urls:
fetchers.insert(0, fs.URLFetchStrategy(url, digest))
diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py
index 4188b8d550..905af28a06 100644
--- a/lib/spack/spack/test/cc.py
+++ b/lib/spack/spack/test/cc.py
@@ -65,17 +65,17 @@ class CompilerTest(unittest.TestCase):
def check_cc(self, command, args, expected):
os.environ['SPACK_TEST_COMMAND'] = command
- self.assertEqual(self.cc(*args, return_output=True).strip(), expected)
+ self.assertEqual(self.cc(*args, output=str).strip(), expected)
def check_ld(self, command, args, expected):
os.environ['SPACK_TEST_COMMAND'] = command
- self.assertEqual(self.ld(*args, return_output=True).strip(), expected)
+ self.assertEqual(self.ld(*args, output=str).strip(), expected)
def check_cpp(self, command, args, expected):
os.environ['SPACK_TEST_COMMAND'] = command
- self.assertEqual(self.cpp(*args, return_output=True).strip(), expected)
+ self.assertEqual(self.cpp(*args, output=str).strip(), expected)
def test_vcheck_mode(self):
diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py
index 3813079065..d84433176a 100644
--- a/lib/spack/spack/test/git_fetch.py
+++ b/lib/spack/spack/test/git_fetch.py
@@ -56,10 +56,7 @@ class GitFetchTest(MockPackagesTest):
def tearDown(self):
"""Destroy the stage space used by this test."""
super(GitFetchTest, self).tearDown()
-
- if self.repo.stage is not None:
- self.repo.stage.destroy()
-
+ self.repo.destroy()
self.pkg.do_clean()
diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py
index ee8327aec8..bbcb64e4c1 100644
--- a/lib/spack/spack/test/hg_fetch.py
+++ b/lib/spack/spack/test/hg_fetch.py
@@ -53,10 +53,7 @@ class HgFetchTest(MockPackagesTest):
def tearDown(self):
"""Destroy the stage space used by this test."""
super(HgFetchTest, self).tearDown()
-
- if self.repo.stage is not None:
- self.repo.stage.destroy()
-
+ self.repo.destroy()
self.pkg.do_clean()
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index 628329a423..c09bd24c8e 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -59,9 +59,7 @@ class InstallTest(MockPackagesTest):
def tearDown(self):
super(InstallTest, self).tearDown()
-
- if self.repo.stage is not None:
- self.repo.stage.destroy()
+ self.repo.destroy()
# Turn checksumming back on
spack.do_checksum = True
diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py
index 09efec8580..d568a28d44 100644
--- a/lib/spack/spack/test/make_executable.py
+++ b/lib/spack/spack/test/make_executable.py
@@ -56,47 +56,47 @@ class MakeExecutableTest(unittest.TestCase):
def test_make_normal(self):
make = MakeExecutable('make', 8)
- self.assertEqual(make(return_output=True).strip(), '-j8')
- self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(output=str).strip(), '-j8')
+ self.assertEqual(make('install', output=str).strip(), '-j8 install')
def test_make_explicit(self):
make = MakeExecutable('make', 8)
- self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
- self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(parallel=True, output=str).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install')
def test_make_one_job(self):
make = MakeExecutable('make', 1)
- self.assertEqual(make(return_output=True).strip(), '')
- self.assertEqual(make('install', return_output=True).strip(), 'install')
+ self.assertEqual(make(output=str).strip(), '')
+ self.assertEqual(make('install', output=str).strip(), 'install')
def test_make_parallel_false(self):
make = MakeExecutable('make', 8)
- self.assertEqual(make(parallel=False, return_output=True).strip(), '')
- self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install')
+ self.assertEqual(make(parallel=False, output=str).strip(), '')
+ self.assertEqual(make('install', parallel=False, output=str).strip(), 'install')
def test_make_parallel_disabled(self):
make = MakeExecutable('make', 8)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
- self.assertEqual(make(return_output=True).strip(), '')
- self.assertEqual(make('install', return_output=True).strip(), 'install')
+ self.assertEqual(make(output=str).strip(), '')
+ self.assertEqual(make('install', output=str).strip(), 'install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
- self.assertEqual(make(return_output=True).strip(), '')
- self.assertEqual(make('install', return_output=True).strip(), 'install')
+ self.assertEqual(make(output=str).strip(), '')
+ self.assertEqual(make('install', output=str).strip(), 'install')
# These don't disable (false and random string)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
- self.assertEqual(make(return_output=True).strip(), '-j8')
- self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(output=str).strip(), '-j8')
+ self.assertEqual(make('install', output=str).strip(), '-j8 install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
- self.assertEqual(make(return_output=True).strip(), '-j8')
- self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(output=str).strip(), '-j8')
+ self.assertEqual(make('install', output=str).strip(), '-j8 install')
del os.environ['SPACK_NO_PARALLEL_MAKE']
@@ -106,20 +106,20 @@ class MakeExecutableTest(unittest.TestCase):
# These should work
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
- self.assertEqual(make(parallel=True, return_output=True).strip(), '')
- self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
+ self.assertEqual(make(parallel=True, output=str).strip(), '')
+ self.assertEqual(make('install', parallel=True, output=str).strip(), 'install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
- self.assertEqual(make(parallel=True, return_output=True).strip(), '')
- self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
+ self.assertEqual(make(parallel=True, output=str).strip(), '')
+ self.assertEqual(make('install', parallel=True, output=str).strip(), 'install')
# These don't disable (false and random string)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
- self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
- self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(parallel=True, output=str).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
- self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
- self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+ self.assertEqual(make(parallel=True, output=str).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install')
del os.environ['SPACK_NO_PARALLEL_MAKE']
diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py
index 04e9e3db2e..046ec56604 100644
--- a/lib/spack/spack/test/mirror.py
+++ b/lib/spack/spack/test/mirror.py
@@ -44,8 +44,16 @@ class MirrorTest(MockPackagesTest):
self.repos = {}
+ def tearDown(self):
+ """Destroy all the stages created by the repos in setup."""
+ super(MirrorTest, self).tearDown()
+ for repo in self.repos.values():
+ repo.destroy()
+ self.repos.clear()
+
+
def set_up_package(self, name, MockRepoClass, url_attr):
- """Use this to set up a mock package to be mirrored.
+ """Set up a mock package to be mirrored.
Each package needs us to:
1. Set up a mock repo/archive to fetch from.
2. Point the package's version args at that repo.
@@ -65,21 +73,14 @@ class MirrorTest(MockPackagesTest):
pkg.versions[v][url_attr] = repo.url
- def tearDown(self):
- """Destroy all the stages created by the repos in setup."""
- super(MirrorTest, self).tearDown()
-
- for name, repo in self.repos.items():
- if repo.stage:
- pass #repo.stage.destroy()
-
- self.repos.clear()
-
-
def check_mirror(self):
stage = Stage('spack-mirror-test')
mirror_root = join_path(stage.path, 'test-mirror')
+ # register mirror with spack config
+ mirrors = { 'spack-mirror-test' : 'file://' + mirror_root }
+ spack.config.update_config('mirrors', mirrors)
+
try:
os.chdir(stage.path)
spack.mirror.create(
@@ -88,7 +89,7 @@ class MirrorTest(MockPackagesTest):
# Stage directory exists
self.assertTrue(os.path.isdir(mirror_root))
- # subdirs for each package
+ # check that there are subdirs for each package
for name in self.repos:
subdir = join_path(mirror_root, name)
self.assertTrue(os.path.isdir(subdir))
@@ -96,40 +97,37 @@ class MirrorTest(MockPackagesTest):
files = os.listdir(subdir)
self.assertEqual(len(files), 1)
- # Decompress archive in the mirror
- archive = files[0]
- archive_path = join_path(subdir, archive)
- decomp = decompressor_for(archive_path)
-
- with working_dir(subdir):
- decomp(archive_path)
+ # Now try to fetch each package.
+ for name, mock_repo in self.repos.items():
+ spec = Spec(name).concretized()
+ pkg = spec.package
- # Find the untarred archive directory.
- files = os.listdir(subdir)
- self.assertEqual(len(files), 2)
- self.assertTrue(archive in files)
- files.remove(archive)
-
- expanded_archive = join_path(subdir, files[0])
- self.assertTrue(os.path.isdir(expanded_archive))
+ saved_checksum_setting = spack.do_checksum
+ try:
+ # Stage the archive from the mirror and cd to it.
+ spack.do_checksum = False
+ pkg.do_stage(mirror_only=True)
# Compare the original repo with the expanded archive
- repo = self.repos[name]
- if not 'svn' in name:
- original_path = repo.path
- else:
- co = 'checked_out'
- svn('checkout', repo.url, co)
- original_path = join_path(subdir, co)
+ original_path = mock_repo.path
+ if 'svn' in name:
+ # have to check out the svn repo to compare.
+ original_path = join_path(mock_repo.path, 'checked_out')
+ svn('checkout', mock_repo.url, original_path)
- dcmp = dircmp(original_path, expanded_archive)
+ dcmp = dircmp(original_path, pkg.stage.source_path)
# make sure there are no new files in the expanded tarball
self.assertFalse(dcmp.right_only)
+
+ # and that all original files are present.
self.assertTrue(all(l in exclude for l in dcmp.left_only))
+ finally:
+ spack.do_checksum = saved_checksum_setting
+ pkg.do_clean()
finally:
- pass #stage.destroy()
+ stage.destroy()
def test_git_mirror(self):
diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py
index c454b1f106..ed94023b0e 100644
--- a/lib/spack/spack/test/mock_repo.py
+++ b/lib/spack/spack/test/mock_repo.py
@@ -55,6 +55,12 @@ class MockRepo(object):
mkdirp(self.path)
+ def destroy(self):
+ """Destroy resources associated with this mock repo."""
+ if self.stage:
+ self.stage.destroy()
+
+
class MockArchive(MockRepo):
"""Creates a very simple archive directory with a configure script and a
makefile that installs to a prefix. Tars it up into an archive."""
@@ -141,7 +147,7 @@ class MockGitRepo(MockVCSRepo):
self.url = self.path
def rev_hash(self, rev):
- return git('rev-parse', rev, return_output=True).strip()
+ return git('rev-parse', rev, output=str).strip()
class MockSvnRepo(MockVCSRepo):
@@ -193,4 +199,4 @@ class MockHgRepo(MockVCSRepo):
def get_rev(self):
"""Get current mercurial revision."""
- return hg('id', '-i', return_output=True).strip()
+ return hg('id', '-i', output=str).strip()
diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py
index 2ee4748fdb..454a7f1d1f 100644
--- a/lib/spack/spack/test/svn_fetch.py
+++ b/lib/spack/spack/test/svn_fetch.py
@@ -55,17 +55,14 @@ class SvnFetchTest(MockPackagesTest):
def tearDown(self):
"""Destroy the stage space used by this test."""
super(SvnFetchTest, self).tearDown()
-
- if self.repo.stage is not None:
- self.repo.stage.destroy()
-
+ self.repo.destroy()
self.pkg.do_clean()
def assert_rev(self, rev):
"""Check that the current revision is equal to the supplied rev."""
def get_rev():
- output = svn('info', return_output=True)
+ output = svn('info', output=str)
self.assertTrue("Revision" in output)
for line in output.split('\n'):
match = re.match(r'Revision: (\d+)', line)
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index ba765eb662..fc27b789d0 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -55,24 +55,80 @@ class Executable(object):
def __call__(self, *args, **kwargs):
- """Run the executable with subprocess.check_output, return output."""
- return_output = kwargs.get("return_output", False)
- fail_on_error = kwargs.get("fail_on_error", True)
- ignore_errors = kwargs.get("ignore_errors", ())
+ """Run this executable in a subprocess.
+
+ Arguments
+ args
+ command line arguments to the executable to run.
+
+ Optional arguments
+
+ fail_on_error
+
+ Raise an exception if the subprocess returns an
+ error. Default is True. When not set, the return code is
+ avaiale as `exe.returncode`.
+
+ ignore_errors
+
+ An optional list/tuple of error codes that can be
+ *ignored*. i.e., if these codes are returned, this will
+ not raise an exception when `fail_on_error` is `True`.
+
+ output, error
+
+ These arguments allow you to specify new stdout and stderr
+ values. They default to `None`, which means the
+ subprocess will inherit the parent's file descriptors.
+
+ You can set these to:
+ - python streams, e.g. open Python file objects, or os.devnull;
+ - filenames, which will be automatically opened for writing; or
+ - `str`, as in the Python string type. If you set these to `str`,
+ output and error will be written to pipes and returned as
+ a string. If both `output` and `error` are set to `str`,
+ then one string is returned containing output concatenated
+ with error.
+
+ input
+
+ Same as output, error, but `str` is not an allowed value.
+
+ Deprecated arguments
+
+ return_output[=False]
+
+ Setting this to True is the same as setting output=str.
+ This argument may be removed in future Spack versions.
+
+ """
+ fail_on_error = kwargs.pop("fail_on_error", True)
+ ignore_errors = kwargs.pop("ignore_errors", ())
+
+ # TODO: This is deprecated. Remove in a future version.
+ return_output = kwargs.pop("return_output", False)
# Default values of None says to keep parent's file descriptors.
- output = kwargs.get("output", None)
- error = kwargs.get("error", None)
- input = kwargs.get("input", None)
+ if return_output:
+ output = str
+ else:
+ output = kwargs.pop("output", None)
+
+ error = kwargs.pop("error", None)
+ input = kwargs.pop("input", None)
+ if input is str:
+ raise ValueError("Cannot use `str` as input stream.")
def streamify(arg, mode):
if isinstance(arg, basestring):
return open(arg, mode), True
+ elif arg is str:
+ return subprocess.PIPE, False
else:
return arg, False
- output, ostream = streamify(output, 'w')
- error, estream = streamify(error, 'w')
- input, istream = streamify(input, 'r')
+ ostream, close_ostream = streamify(output, 'w')
+ estream, close_estream = streamify(error, 'w')
+ istream, close_istream = streamify(input, 'r')
# if they just want to ignore one error code, make it a tuple.
if isinstance(ignore_errors, int):
@@ -92,19 +148,20 @@ class Executable(object):
tty.debug(cmd_line)
try:
- if return_output:
- output = subprocess.PIPE
-
proc = subprocess.Popen(
- cmd, stdin=input, stderr=error, stdout=output)
+ cmd, stdin=istream, stderr=estream, stdout=ostream)
out, err = proc.communicate()
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
raise ProcessError("Command exited with status %d:"
% proc.returncode, cmd_line)
- if return_output:
- return out
+
+ if output is str or error is str:
+ result = ''
+ if output is str: result += out
+ if error is str: result += err
+ return result
except OSError, e:
raise ProcessError(
@@ -119,9 +176,9 @@ class Executable(object):
% (proc.returncode, cmd_line))
finally:
- if ostream: output.close()
- if estream: error.close()
- if istream: input.close()
+ if close_ostream: output.close()
+ if close_estream: error.close()
+ if close_istream: input.close()
def __eq__(self, other):