summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/fetch_strategy.py202
-rw-r--r--lib/spack/spack/util/executable.py92
2 files changed, 168 insertions, 126 deletions
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 0144d26057..7c8cebe0c9 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -57,7 +57,6 @@ from spack.version import Version, ver
from spack.util.compression import decompressor_for, extension
import spack.util.pattern as pattern
-
"""List of all fetch strategies, created by FetchStrategy metaclass."""
all_strategies = []
@@ -82,13 +81,16 @@ class FetchStrategy(object):
class __metaclass__(type):
"""This metaclass registers all fetch strategies in a list."""
+
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
- if cls.enabled: all_strategies.append(cls)
+ if cls.enabled:
+ all_strategies.append(cls)
def __init__(self):
- # The stage is initialized late, so that fetch strategies can be constructed
- # at package construction time. This is where things will be fetched.
+ # The stage is initialized late, so that fetch strategies can be
+ # constructed at package construction time. This is where things
+ # will be fetched.
self.stage = None
def set_stage(self, stage):
@@ -97,15 +99,20 @@ class FetchStrategy(object):
self.stage = stage
# Subclasses need to implement these methods
- def fetch(self): pass # Return True on success, False on fail.
+ def fetch(self):
+ pass # Return True on success, False on fail.
- def check(self): pass # Do checksum.
+ def check(self):
+ pass # Do checksum.
- def expand(self): pass # Expand archive.
+ def expand(self):
+ pass # Expand archive.
- def reset(self): pass # Revert to freshly downloaded state.
+ def reset(self):
+ pass # Revert to freshly downloaded state.
- def archive(self, destination): pass # Used to create tarball for mirror.
+ def archive(self, destination):
+ pass # Used to create tarball for mirror.
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
@@ -139,10 +146,12 @@ class URLFetchStrategy(FetchStrategy):
# If URL or digest are provided in the kwargs, then prefer
# those values.
self.url = kwargs.get('url', None)
- if not self.url: self.url = url
+ if not self.url:
+ self.url = url
self.digest = kwargs.get('md5', None)
- if not self.digest: self.digest = digest
+ if not self.digest:
+ self.digest = digest
self.expand_archive = kwargs.get('expand', True)
@@ -167,16 +176,20 @@ class URLFetchStrategy(FetchStrategy):
tty.msg("Trying to fetch from %s" % self.url)
if partial_file:
- save_args = ['-C', '-', # continue partial downloads
- '-o', partial_file] # use a .part file
+ save_args = ['-C',
+ '-', # continue partial downloads
+ '-o',
+ partial_file] # use a .part file
else:
save_args = ['-O']
curl_args = save_args + [
- '-f', # fail on >400 errors
- '-D', '-', # print out HTML headers
- '-L', # resolve 3xx redirects
- self.url, ]
+ '-f', # fail on >400 errors
+ '-D',
+ '-', # print out HTML headers
+ '-L', # resolve 3xx redirects
+ self.url,
+ ]
if sys.stdout.isatty():
curl_args.append('-#') # status bar when using a tty
@@ -184,8 +197,7 @@ class URLFetchStrategy(FetchStrategy):
curl_args.append('-sS') # just errors when not.
# Run curl but grab the mime type from the http headers
- headers = spack.curl(
- *curl_args, output=str, fail_on_error=False)
+ headers = spack.curl(*curl_args, output=str, fail_on_error=False)
if spack.curl.returncode != 0:
# clean up archive on failure.
@@ -198,33 +210,36 @@ class URLFetchStrategy(FetchStrategy):
if spack.curl.returncode == 22:
# This is a 404. Curl will print the error.
raise FailedDownloadError(
- self.url, "URL %s was not found!" % self.url)
+ self.url, "URL %s was not found!" % self.url)
elif spack.curl.returncode == 60:
# This is a certificate error. Suggest spack -k
raise FailedDownloadError(
- self.url,
- "Curl was unable to fetch due to invalid certificate. "
- "This is either an attack, or your cluster's SSL configuration "
- "is bad. If you believe your SSL configuration is bad, you "
- "can try running spack -k, which will not check SSL certificates."
- "Use this at your own risk.")
+ self.url,
+ "Curl was unable to fetch due to invalid certificate. "
+ "This is either an attack, or your cluster's SSL "
+ "configuration is bad. If you believe your SSL "
+ "configuration is bad, you can try running spack -k, "
+ "which will not check SSL certificates."
+ "Use this at your own risk.")
else:
# This is some other curl error. Curl will print the
# error, but print a spack message too
raise FailedDownloadError(
- self.url, "Curl failed with error %d" % spack.curl.returncode)
+ self.url,
+ "Curl failed with error %d" % spack.curl.returncode)
# Check if we somehow got an HTML file rather than the archive we
# asked for. We only look at the last content type, to handle
# redirects properly.
content_types = re.findall(r'Content-Type:[^\r\n]+', headers)
if content_types and 'text/html' in content_types[-1]:
- tty.warn("The contents of " + self.archive_file + " look like HTML.",
- "The checksum will likely be bad. If it is, you can use",
- "'spack clean <package>' to remove the bad archive, then fix",
- "your internet gateway issue and install again.")
+ tty.warn(
+ "The contents of " + self.archive_file + " look like HTML.",
+ "The checksum will likely be bad. If it is, you can use",
+ "'spack clean <package>' to remove the bad archive, then fix",
+ "your internet gateway issue and install again.")
if save_file:
os.rename(partial_file, save_file)
@@ -247,14 +262,16 @@ class URLFetchStrategy(FetchStrategy):
self.stage.chdir()
if not self.archive_file:
- raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
- "Failed on expand() for URL %s" % self.url)
+ raise NoArchiveFileError(
+ "URLFetchStrategy couldn't find archive file",
+ "Failed on expand() for URL %s" % self.url)
decompress = decompressor_for(self.archive_file)
# Expand all tarballs in their own directory to contain
# exploding tarballs.
- tarball_container = os.path.join(self.stage.path, "spack-expanded-archive")
+ tarball_container = os.path.join(self.stage.path,
+ "spack-expanded-archive")
mkdirp(tarball_container)
os.chdir(tarball_container)
decompress(self.archive_file)
@@ -295,20 +312,25 @@ class URLFetchStrategy(FetchStrategy):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
if not self.digest:
- raise NoDigestError("Attempt to check URLFetchStrategy with no digest.")
+ raise NoDigestError(
+ "Attempt to check URLFetchStrategy with no digest.")
checker = crypto.Checker(self.digest)
if not checker.check(self.archive_file):
raise ChecksumError(
- "%s checksum failed for %s" % (checker.hash_name, self.archive_file),
- "Expected %s but got %s" % (self.digest, checker.sum))
+ "%s checksum failed for %s" %
+ (checker.hash_name, self.archive_file),
+ "Expected %s but got %s" % (self.digest, checker.sum))
@_needs_stage
def reset(self):
- """Removes the source path if it exists, then re-expands the archive."""
+ """
+ Removes the source path if it exists, then re-expands the archive.
+ """
if not self.archive_file:
- raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching",
- "Failed on reset() for URL %s" % self.url)
+ raise NoArchiveFileError(
+ "Tried to reset URLFetchStrategy before fetching",
+ "Failed on reset() for URL %s" % self.url)
# Remove everythigng but the archive from the stage
for filename in os.listdir(self.stage.path):
@@ -337,14 +359,16 @@ class VCSFetchStrategy(FetchStrategy):
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None)
- if not self.url: raise ValueError(
+ if not self.url:
+ raise ValueError(
"%s requires %s argument." % (self.__class__, name))
# Ensure that there's only one of the rev_types
if sum(k in kwargs for k in rev_types) > 1:
raise FetchStrategyError(
- "Supply only one of %s to fetch with %s" % (
- comma_or(rev_types), name))
+ "Supply only one of %s to fetch with %s" % (
+ comma_or(rev_types), name
+ ))
# Set attributes for each rev type.
for rt in rev_types:
@@ -381,19 +405,23 @@ class VCSFetchStrategy(FetchStrategy):
def __repr__(self):
return "%s<%s>" % (self.__class__, self.url)
+
class GoFetchStrategy(VCSFetchStrategy):
- """Fetch strategy that employs the `go get` infrastructure
- Use like this in a package:
+ """
+ Fetch strategy that employs the `go get` infrastructure
+ Use like this in a package:
- version('name', go='github.com/monochromegane/the_platinum_searcher/...')
+ version('name',
+ go='github.com/monochromegane/the_platinum_searcher/...')
- Go get does not natively support versions, they can be faked with git
+ Go get does not natively support versions, they can be faked with git
"""
enabled = True
- required_attributes = ('go',)
+ required_attributes = ('go', )
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next
+ # call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
@@ -422,8 +450,8 @@ class GoFetchStrategy(VCSFetchStrategy):
except OSError:
pass
env = dict(os.environ)
- env['GOPATH'] = os.path.join(os.getcwd(),'go')
- self.go('get', '-v', '-d', self.url, env=env)
+ env['GOPATH'] = os.path.join(os.getcwd(), 'go')
+ self.go('get', '-v', '-d', self.url, env=env)
def archive(self, destination):
super(GoFetchStrategy, self).archive(destination, exclude='.git')
@@ -436,32 +464,35 @@ class GoFetchStrategy(VCSFetchStrategy):
def __str__(self):
return "[go] %s" % self.url
+
class GitFetchStrategy(VCSFetchStrategy):
- """Fetch strategy that gets source code from a git repository.
- Use like this in a package:
+ """
+ Fetch strategy that gets source code from a git repository.
+ Use like this in a package:
- version('name', git='https://github.com/project/repo.git')
+ version('name', git='https://github.com/project/repo.git')
- Optionally, you can provide a branch, or commit to check out, e.g.:
+ Optionally, you can provide a branch, or commit to check out, e.g.:
- version('1.1', git='https://github.com/project/repo.git', tag='v1.1')
+ version('1.1', git='https://github.com/project/repo.git', tag='v1.1')
- You can use these three optional attributes in addition to ``git``:
+ You can use these three optional attributes in addition to ``git``:
- * ``branch``: Particular branch to build from (default is master)
- * ``tag``: Particular tag to check out
- * ``commit``: Particular commit hash in the repo
+ * ``branch``: Particular branch to build from (default is master)
+ * ``tag``: Particular tag to check out
+ * ``commit``: Particular commit hash in the repo
"""
enabled = True
- required_attributes = ('git',)
+ required_attributes = ('git', )
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GitFetchStrategy, self).__init__(
- 'git', 'tag', 'branch', 'commit', **forwarded_args)
+ 'git', 'tag', 'branch', 'commit', **forwarded_args)
self._git = None
@property
@@ -569,12 +600,13 @@ class SvnFetchStrategy(VCSFetchStrategy):
required_attributes = ['svn']
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(SvnFetchStrategy, self).__init__(
- 'svn', 'revision', **forwarded_args)
+ 'svn', 'revision', **forwarded_args)
self._svn = None
if self.revision is not None:
self.revision = str(self.revision)
@@ -630,32 +662,35 @@ class SvnFetchStrategy(VCSFetchStrategy):
class HgFetchStrategy(VCSFetchStrategy):
- """Fetch strategy that gets source code from a Mercurial repository.
- Use like this in a package:
+ """
+ Fetch strategy that gets source code from a Mercurial repository.
+ Use like this in a package:
- version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2')
+ version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2')
- Optionally, you can provide a branch, or revision to check out, e.g.:
+ Optionally, you can provide a branch, or revision to check out, e.g.:
- version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus')
+ version('torus',
+ hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus')
- You can use the optional 'revision' attribute to check out a
- branch, tag, or particular revision in hg. To prevent
- non-reproducible builds, using a moving target like a branch is
- discouraged.
+ You can use the optional 'revision' attribute to check out a
+ branch, tag, or particular revision in hg. To prevent
+ non-reproducible builds, using a moving target like a branch is
+ discouraged.
- * ``revision``: Particular revision, branch, or tag.
+ * ``revision``: Particular revision, branch, or tag.
"""
enabled = True
required_attributes = ['hg']
def __init__(self, **kwargs):
- # Discards the keywords in kwargs that may conflict with the next call to __init__
+ # Discards the keywords in kwargs that may conflict with the next call
+ # to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(HgFetchStrategy, self).__init__(
- 'hg', 'revision', **forwarded_args)
+ 'hg', 'revision', **forwarded_args)
self._hg = None
@property
@@ -729,7 +764,8 @@ def from_kwargs(**kwargs):
return fetcher(**kwargs)
# Raise an error in case we can't instantiate any known strategy
message = "Cannot instantiate any FetchStrategy"
- long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+ long_message = message + " from the given arguments : {arguments}".format(
+ srguments=kwargs)
raise FetchError(message, long_message)
@@ -741,7 +777,7 @@ def for_package_version(pkg, version):
"""Determine a fetch strategy based on the arguments supplied to
version() in the package description."""
# If it's not a known version, extrapolate one.
- if not version in pkg.versions:
+ if version not in pkg.versions:
url = pkg.url_for_version(version)
if not url:
raise InvalidArgsError(pkg, version)
@@ -779,7 +815,7 @@ class FailedDownloadError(FetchError):
def __init__(self, url, msg=""):
super(FailedDownloadError, self).__init__(
- "Failed to fetch file from URL: %s" % url, msg)
+ "Failed to fetch file from URL: %s" % url, msg)
self.url = url
@@ -795,7 +831,8 @@ class NoDigestError(FetchError):
class InvalidArgsError(FetchError):
def __init__(self, pkg, version):
- msg = "Could not construct a fetch strategy for package %s at version %s"
+ msg = ("Could not construct a fetch strategy for package %s at "
+ "version %s")
msg %= (pkg.name, version)
super(InvalidArgsError, self).__init__(msg)
@@ -812,4 +849,5 @@ class NoStageError(FetchError):
def __init__(self, method):
super(NoStageError, self).__init__(
- "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
+ "Must call FetchStrategy.set_stage() before calling %s" %
+ method.__name__)
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 0b91b6c360..38b778fa00 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -22,10 +22,8 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-__all__ = ['Executable', 'which', 'ProcessError']
import os
-import sys
import re
import subprocess
import inspect
@@ -34,9 +32,12 @@ import llnl.util.tty as tty
import spack
import spack.error
+__all__ = ['Executable', 'which', 'ProcessError']
+
class Executable(object):
"""Class representing a program that can be run on the command line."""
+
def __init__(self, name):
self.exe = name.split(' ')
self.returncode = None
@@ -44,16 +45,13 @@ class Executable(object):
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
-
def add_default_arg(self, arg):
self.exe.append(arg)
-
@property
def command(self):
return ' '.join(self.exe)
-
def __call__(self, *args, **kwargs):
"""Run this executable in a subprocess.
@@ -105,7 +103,7 @@ class Executable(object):
fail_on_error = kwargs.pop("fail_on_error", True)
ignore_errors = kwargs.pop("ignore_errors", ())
- env = kwargs.get('env', None)
+ env = kwargs.get('env', None)
# TODO: This is deprecated. Remove in a future version.
return_output = kwargs.pop("return_output", False)
@@ -116,8 +114,8 @@ class Executable(object):
else:
output = kwargs.pop("output", None)
- error = kwargs.pop("error", None)
- input = kwargs.pop("input", None)
+ error = kwargs.pop("error", None)
+ input = kwargs.pop("input", None)
if input is str:
raise ValueError("Cannot use `str` as input stream.")
@@ -128,85 +126,90 @@ class Executable(object):
return subprocess.PIPE, False
else:
return arg, False
+
ostream, close_ostream = streamify(output, 'w')
- estream, close_estream = streamify(error, 'w')
- istream, close_istream = streamify(input, 'r')
+ estream, close_estream = streamify(error, 'w')
+ istream, close_istream = streamify(input, 'r')
# if they just want to ignore one error code, make it a tuple.
if isinstance(ignore_errors, int):
- ignore_errors = (ignore_errors,)
+ ignore_errors = (ignore_errors, )
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
if quoted_args:
- tty.warn("Quotes in command arguments can confuse scripts like configure.",
- "The following arguments may cause problems when executed:",
- str("\n".join([" "+arg for arg in quoted_args])),
- "Quotes aren't needed because spack doesn't use a shell.",
- "Consider removing them")
+ tty.warn(
+ "Quotes in command arguments can confuse scripts like"
+ " configure.",
+ "The following arguments may cause problems when executed:",
+ str("\n".join([" " + arg for arg in quoted_args])),
+ "Quotes aren't needed because spack doesn't use a shell.",
+ "Consider removing them")
cmd = self.exe + list(args)
- cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
+ cmd_line = "'%s'" % "' '".join(
+ map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
tty.debug(cmd_line)
try:
proc = subprocess.Popen(
- cmd, stdin=istream, stderr=estream, stdout=ostream, env=env)
+ cmd,
+ stdin=istream,
+ stderr=estream,
+ stdout=ostream,
+ env=env)
out, err = proc.communicate()
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
- raise ProcessError("Command exited with status %d:"
- % proc.returncode, cmd_line)
+ raise ProcessError("Command exited with status %d:" %
+ proc.returncode, cmd_line)
if output is str or error is str:
result = ''
- if output is str: result += out
- if error is str: result += err
+ if output is str:
+ result += out
+ if error is str:
+ result += err
return result
except OSError, e:
raise ProcessError(
- "%s: %s" % (self.exe[0], e.strerror),
- "Command: " + cmd_line)
+ "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line)
except subprocess.CalledProcessError, e:
if fail_on_error:
raise ProcessError(
- str(e),
- "\nExit status %d when invoking command: %s"
- % (proc.returncode, cmd_line))
+ str(e), "\nExit status %d when invoking command: %s" %
+ (proc.returncode, cmd_line))
finally:
- if close_ostream: output.close()
- if close_estream: error.close()
- if close_istream: input.close()
-
+ if close_ostream:
+ output.close()
+ if close_estream:
+ error.close()
+ if close_istream:
+ input.close()
def __eq__(self, other):
return self.exe == other.exe
-
def __neq__(self, other):
return not (self == other)
-
def __hash__(self):
- return hash((type(self),) + tuple(self.exe))
-
+ return hash((type(self), ) + tuple(self.exe))
def __repr__(self):
return "<exe: %s>" % self.exe
-
def __str__(self):
return ' '.join(self.exe)
-
def which(name, **kwargs):
"""Finds an executable in the path like command-line which."""
- path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
+ path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
required = kwargs.get('required', False)
if not path:
@@ -235,14 +238,16 @@ class ProcessError(spack.error.SpackError):
@property
def long_message(self):
msg = self._long_message
- if msg: msg += "\n\n"
+ if msg:
+ msg += "\n\n"
if self.build_log:
msg += "See build log for details:\n"
msg += " %s" % self.build_log
if self.package_context:
- if msg: msg += "\n\n"
+ if msg:
+ msg += "\n\n"
msg += '\n'.join(self.package_context)
return msg
@@ -269,7 +274,7 @@ def _get_package_context():
frame = f[0]
# Find a frame with 'self' in the local variables.
- if not 'self' in frame.f_locals:
+ if 'self' not in frame.f_locals:
continue
# Look only at a frame in a subclass of spack.Package
@@ -282,9 +287,8 @@ def _get_package_context():
# Build a message showing where in install we failed.
lines.append("%s:%d, in %s:" % (
- inspect.getfile(frame.f_code),
- frame.f_lineno,
- frame.f_code.co_name))
+ inspect.getfile(frame.f_code), frame.f_lineno, frame.f_code.co_name
+ ))
sourcelines, start = inspect.getsourcelines(frame)
for i, line in enumerate(sourcelines):