diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/spack/directives.py | 51 | ||||
-rw-r--r-- | lib/spack/spack/environment.py | 38 | ||||
-rw-r--r-- | lib/spack/spack/fetch_strategy.py | 240 | ||||
-rw-r--r-- | lib/spack/spack/modules.py | 6 | ||||
-rw-r--r-- | lib/spack/spack/spec.py | 5 | ||||
-rw-r--r-- | lib/spack/spack/util/executable.py | 92 |
6 files changed, 272 insertions, 160 deletions
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 74ee7b0add..51b26773e2 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -45,11 +45,8 @@ The available directives are: * ``resource`` """ -__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', - 'variant', 'resource'] import re -import inspect import os.path import functools @@ -67,6 +64,9 @@ from spack.spec import Spec, parse_anonymous_spec from spack.resource import Resource from spack.fetch_strategy import from_kwargs +__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', 'variant', + 'resource'] + # # This is a list of all directives, built up as they are defined in # this file. @@ -122,15 +122,14 @@ class directive(object): def __init__(self, dicts=None): if isinstance(dicts, basestring): - dicts = (dicts,) + dicts = (dicts, ) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s" - % type(dicts)) + "dicts arg must be list, tuple, or string. Found %s" % + type(dicts)) self.dicts = dicts - def ensure_dicts(self, pkg): """Ensure that a package has the dicts required by this directive.""" for d in self.dicts: @@ -142,7 +141,6 @@ class directive(object): raise spack.error.SpackError( "Package %s has non-dict %s attribute!" % (pkg, d)) - def __call__(self, directive_function): directives[directive_function.__name__] = self @@ -259,11 +257,12 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = bool(default) description = str(description).strip() if not re.match(spack.spec.identifier_re, name): - raise DirectiveError("Invalid variant name in %s: '%s'" % (pkg.name, name)) + raise DirectiveError("Invalid variant name in %s: '%s'" % + (pkg.name, name)) pkg.variants[name] = Variant(default, description) @@ -271,31 +270,37 @@ def variant(pkg, name, default=False, description=""): @directive('resources') def resource(pkg, **kwargs): """ - Define an external resource to be fetched and staged when building the package. Based on the keywords present in the - dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their - own folder inside spack stage area, and then linked into the stage area of the package that needs them. + Define an external resource to be fetched and staged when building the + package. Based on the keywords present in the dictionary the appropriate + FetchStrategy will be used for the resource. Resources are fetched and + staged in their own folder inside spack stage area, and then linked into + the stage area of the package that needs them. List of recognized keywords: - * 'when' : (optional) represents the condition upon which the resource is needed - * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage - area. - * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage - area. + * 'when' : (optional) represents the condition upon which the resource is + needed + * 'destination' : (optional) path where to link the resource. This path + must be relative to the main package stage area. + * 'placement' : (optional) gives the possibility to fine tune how the + resource is linked into the main package stage area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") placement = kwargs.get('placement', None) # Check if the path is relative if os.path.isabs(destination): - message = "The destination keyword of a resource directive can't be an absolute path.\n" + message = "The destination keyword of a resource directive can't be" + " an absolute path.\n" message += "\tdestination : '{dest}\n'".format(dest=destination) raise RuntimeError(message) # Check if the path falls within the main package stage area - test_path = 'stage_folder_root/' - normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path + test_path = 'stage_folder_root' + normalized_destination = os.path.normpath(join_path(test_path, destination) + ) # Normalized absolute path if test_path not in normalized_destination: - message = "The destination folder of a resource must fall within the main package stage directory.\n" + message = "The destination folder of a resource must fall within the" + " main package stage directory.\n" message += "\tdestination : '{dest}'\n".format(dest=destination) raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) @@ -307,6 +312,7 @@ def resource(pkg, **kwargs): class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" + def __init__(self, directive, message): super(DirectiveError, self).__init__(message) self.directive = directive @@ -314,6 +320,7 @@ class DirectiveError(spack.error.SpackError): class CircularReferenceError(DirectiveError): """This is raised when something depends on itself.""" + def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 11998ad8d2..af642dcc9b 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -39,7 +39,8 @@ class NameValueModifier(object): def __init__(self, name, value, **kwargs): self.name = name self.value = value - self.args = {'name': name, 'value': value} + self.separator = kwargs.get('separator', ':') + self.args = {'name': name, 'value': value, 'delim': self.separator} self.args.update(kwargs) @@ -56,34 +57,36 @@ class UnsetEnv(NameModifier): class SetPath(NameValueModifier): def execute(self): - string_path = concatenate_paths(self.value) + string_path = concatenate_paths(self.value, separator=self.separator) os.environ[self.name] = string_path class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] + directories = environment_value.split( + self.separator) if environment_value else [] directories.append(os.path.normpath(self.value)) - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class PrependPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] + directories = environment_value.split( + self.separator) if environment_value else [] directories = [os.path.normpath(self.value)] + directories - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') - directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) - for x in directories + directories = environment_value.split( + self.separator) if environment_value else [] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] - os.environ[self.name] = ':'.join(directories) + os.environ[self.name] = self.separator.join(directories) class EnvironmentModifications(object): @@ -238,17 +241,19 @@ class EnvironmentModifications(object): x.execute() -def concatenate_paths(paths): +def concatenate_paths(paths, separator=':'): """ - Concatenates an iterable of paths into a string of column separated paths + Concatenates an iterable of paths into a string of paths separated by + separator, defaulting to colon Args: paths: iterable of paths + separator: the separator to use, default ':' Returns: string """ - return ':'.join(str(item) for item in paths) + return separator.join(str(item) for item in paths) def set_or_unset_not_first(variable, changes, errstream): @@ -256,16 +261,13 @@ def set_or_unset_not_first(variable, changes, errstream): Check if we are going to set or unset something after other modifications have already been requested """ - indexes = [ii - for ii, item in enumerate(changes) + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 - errstream( - message.format( - var=variable)) + errstream(message.format(var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index e05cb13c1e..7c8cebe0c9 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -57,7 +57,6 @@ from spack.version import Version, ver from spack.util.compression import decompressor_for, extension import spack.util.pattern as pattern - """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] @@ -82,13 +81,16 @@ class FetchStrategy(object): class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.enabled: all_strategies.append(cls) + if cls.enabled: + all_strategies.append(cls) def __init__(self): - # The stage is initialized late, so that fetch strategies can be constructed - # at package construction time. This is where things will be fetched. + # The stage is initialized late, so that fetch strategies can be + # constructed at package construction time. This is where things + # will be fetched. self.stage = None def set_stage(self, stage): @@ -97,15 +99,20 @@ class FetchStrategy(object): self.stage = stage # Subclasses need to implement these methods - def fetch(self): pass # Return True on success, False on fail. + def fetch(self): + pass # Return True on success, False on fail. - def check(self): pass # Do checksum. + def check(self): + pass # Do checksum. - def expand(self): pass # Expand archive. + def expand(self): + pass # Expand archive. - def reset(self): pass # Revert to freshly downloaded state. + def reset(self): + pass # Revert to freshly downloaded state. - def archive(self, destination): pass # Used to create tarball for mirror. + def archive(self, destination): + pass # Used to create tarball for mirror. def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -139,10 +146,12 @@ class URLFetchStrategy(FetchStrategy): # If URL or digest are provided in the kwargs, then prefer # those values. self.url = kwargs.get('url', None) - if not self.url: self.url = url + if not self.url: + self.url = url self.digest = kwargs.get('md5', None) - if not self.digest: self.digest = digest + if not self.digest: + self.digest = digest self.expand_archive = kwargs.get('expand', True) @@ -167,16 +176,20 @@ class URLFetchStrategy(FetchStrategy): tty.msg("Trying to fetch from %s" % self.url) if partial_file: - save_args = ['-C', '-', # continue partial downloads - '-o', partial_file] # use a .part file + save_args = ['-C', + '-', # continue partial downloads + '-o', + partial_file] # use a .part file else: save_args = ['-O'] curl_args = save_args + [ - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', # resolve 3xx redirects - self.url, ] + '-f', # fail on >400 errors + '-D', + '-', # print out HTML headers + '-L', # resolve 3xx redirects + self.url, + ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty @@ -184,8 +197,7 @@ class URLFetchStrategy(FetchStrategy): curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers - headers = spack.curl( - *curl_args, output=str, fail_on_error=False) + headers = spack.curl(*curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -198,33 +210,36 @@ class URLFetchStrategy(FetchStrategy): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL " + "configuration is bad. If you believe your SSL " + "configuration is bad, you can try running spack -k, " + "which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) + self.url, + "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: - tty.warn("The contents of " + self.archive_file + " look like HTML.", - "The checksum will likely be bad. If it is, you can use", - "'spack clean <package>' to remove the bad archive, then fix", - "your internet gateway issue and install again.") + tty.warn( + "The contents of " + self.archive_file + " look like HTML.", + "The checksum will likely be bad. If it is, you can use", + "'spack clean <package>' to remove the bad archive, then fix", + "your internet gateway issue and install again.") if save_file: os.rename(partial_file, save_file) @@ -247,14 +262,16 @@ class URLFetchStrategy(FetchStrategy): self.stage.chdir() if not self.archive_file: - raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + raise NoArchiveFileError( + "URLFetchStrategy couldn't find archive file", + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) # Expand all tarballs in their own directory to contain # exploding tarballs. - tarball_container = os.path.join(self.stage.path, "spack-expanded-archive") + tarball_container = os.path.join(self.stage.path, + "spack-expanded-archive") mkdirp(tarball_container) os.chdir(tarball_container) decompress(self.archive_file) @@ -295,20 +312,25 @@ class URLFetchStrategy(FetchStrategy): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" if not self.digest: - raise NoDigestError("Attempt to check URLFetchStrategy with no digest.") + raise NoDigestError( + "Attempt to check URLFetchStrategy with no digest.") checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s" % (checker.hash_name, self.archive_file), - "Expected %s but got %s" % (self.digest, checker.sum)) + "%s checksum failed for %s" % + (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): - """Removes the source path if it exists, then re-expands the archive.""" + """ + Removes the source path if it exists, then re-expands the archive. + """ if not self.archive_file: - raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching", - "Failed on reset() for URL %s" % self.url) + raise NoArchiveFileError( + "Tried to reset URLFetchStrategy before fetching", + "Failed on reset() for URL %s" % self.url) # Remove everythigng but the archive from the stage for filename in os.listdir(self.stage.path): @@ -337,14 +359,16 @@ class VCSFetchStrategy(FetchStrategy): # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) - if not self.url: raise ValueError( + if not self.url: + raise ValueError( "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s" % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s" % ( + comma_or(rev_types), name + )) # Set attributes for each rev type. for rt in rev_types: @@ -382,32 +406,93 @@ class VCSFetchStrategy(FetchStrategy): return "%s<%s>" % (self.__class__, self.url) +class GoFetchStrategy(VCSFetchStrategy): + """ + Fetch strategy that employs the `go get` infrastructure + Use like this in a package: + + version('name', + go='github.com/monochromegane/the_platinum_searcher/...') + + Go get does not natively support versions, they can be faked with git + """ + enabled = True + required_attributes = ('go', ) + + def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next + # call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + + super(GoFetchStrategy, self).__init__('go', **forwarded_args) + self._go = None + + @property + def go_version(self): + vstring = self.go('version', output=str).split(' ')[2] + return Version(vstring) + + @property + def go(self): + if not self._go: + self._go = which('go', required=True) + return self._go + + @_needs_stage + def fetch(self): + self.stage.chdir() + + tty.msg("Trying to get go resource:", self.url) + + try: + os.mkdir('go') + except OSError: + pass + env = dict(os.environ) + env['GOPATH'] = os.path.join(os.getcwd(), 'go') + self.go('get', '-v', '-d', self.url, env=env) + + def archive(self, destination): + super(GoFetchStrategy, self).archive(destination, exclude='.git') + + @_needs_stage + def reset(self): + self.stage.chdir_to_source() + self.go('clean') + + def __str__(self): + return "[go] %s" % self.url + + class GitFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a git repository. - Use like this in a package: + """ + Fetch strategy that gets source code from a git repository. + Use like this in a package: - version('name', git='https://github.com/project/repo.git') + version('name', git='https://github.com/project/repo.git') - Optionally, you can provide a branch, or commit to check out, e.g.: + Optionally, you can provide a branch, or commit to check out, e.g.: - version('1.1', git='https://github.com/project/repo.git', tag='v1.1') + version('1.1', git='https://github.com/project/repo.git', tag='v1.1') - You can use these three optional attributes in addition to ``git``: + You can use these three optional attributes in addition to ``git``: - * ``branch``: Particular branch to build from (default is master) - * ``tag``: Particular tag to check out - * ``commit``: Particular commit hash in the repo + * ``branch``: Particular branch to build from (default is master) + * ``tag``: Particular tag to check out + * ``commit``: Particular commit hash in the repo """ enabled = True - required_attributes = ('git',) + required_attributes = ('git', ) def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **forwarded_args) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None @property @@ -515,12 +600,13 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **forwarded_args) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @@ -576,32 +662,35 @@ class SvnFetchStrategy(VCSFetchStrategy): class HgFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a Mercurial repository. - Use like this in a package: + """ + Fetch strategy that gets source code from a Mercurial repository. + Use like this in a package: - version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') + version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') - Optionally, you can provide a branch, or revision to check out, e.g.: + Optionally, you can provide a branch, or revision to check out, e.g.: - version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') + version('torus', + hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') - You can use the optional 'revision' attribute to check out a - branch, tag, or particular revision in hg. To prevent - non-reproducible builds, using a moving target like a branch is - discouraged. + You can use the optional 'revision' attribute to check out a + branch, tag, or particular revision in hg. To prevent + non-reproducible builds, using a moving target like a branch is + discouraged. - * ``revision``: Particular revision, branch, or tag. + * ``revision``: Particular revision, branch, or tag. """ enabled = True required_attributes = ['hg'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **forwarded_args) + 'hg', 'revision', **forwarded_args) self._hg = None @property @@ -675,7 +764,8 @@ def from_kwargs(**kwargs): return fetcher(**kwargs) # Raise an error in case we can't instantiate any known strategy message = "Cannot instantiate any FetchStrategy" - long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs) + long_message = message + " from the given arguments : {arguments}".format( + srguments=kwargs) raise FetchError(message, long_message) @@ -687,7 +777,7 @@ def for_package_version(pkg, version): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" # If it's not a known version, extrapolate one. - if not version in pkg.versions: + if version not in pkg.versions: url = pkg.url_for_version(version) if not url: raise InvalidArgsError(pkg, version) @@ -725,7 +815,7 @@ class FailedDownloadError(FetchError): def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -741,7 +831,8 @@ class NoDigestError(FetchError): class InvalidArgsError(FetchError): def __init__(self, pkg, version): - msg = "Could not construct a fetch strategy for package %s at version %s" + msg = ("Could not construct a fetch strategy for package %s at " + "version %s") msg %= (pkg.name, version) super(InvalidArgsError, self).__init__(msg) @@ -758,4 +849,5 @@ class NoStageError(FetchError): def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % + method.__name__) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index a35e21c3db..d2b819e80a 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -485,9 +485,9 @@ class TclModule(EnvModule): path = join_path(spack.share_path, "modules") environment_modifications_formats = { - PrependPath: 'prepend-path {name} \"{value}\"\n', - AppendPath: 'append-path {name} \"{value}\"\n', - RemovePath: 'remove-path {name} \"{value}\"\n', + PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n', + AppendPath: 'append-path --delim "{delim}" {name} \"{value}\"\n', + RemovePath: 'remove-path --delim "{delim}" {name} \"{value}\"\n', SetEnv: 'setenv {name} \"{value}\"\n', UnsetEnv: 'unsetenv {name}\n' } diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 89a023a750..470353c4f6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1071,6 +1071,11 @@ class Spec(object): # If there is a provider for the vpkg, then use that instead of # the virtual package. if providers: + # Remove duplicate providers that can concretize to the same result. + for provider in providers: + for spec in providers: + if spec is not provider and provider.satisfies(spec): + providers.remove(spec) # Can't have multiple providers for the same thing in one spec. if len(providers) > 1: raise MultipleProviderError(vdep, providers) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d2ccfde69b..38b778fa00 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -22,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -__all__ = ['Executable', 'which', 'ProcessError'] import os -import sys import re import subprocess import inspect @@ -34,9 +32,12 @@ import llnl.util.tty as tty import spack import spack.error +__all__ = ['Executable', 'which', 'ProcessError'] + class Executable(object): """Class representing a program that can be run on the command line.""" + def __init__(self, name): self.exe = name.split(' ') self.returncode = None @@ -44,16 +45,13 @@ class Executable(object): if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) - def add_default_arg(self, arg): self.exe.append(arg) - @property def command(self): return ' '.join(self.exe) - def __call__(self, *args, **kwargs): """Run this executable in a subprocess. @@ -105,6 +103,8 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) + env = kwargs.get('env', None) + # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -114,8 +114,8 @@ class Executable(object): else: output = kwargs.pop("output", None) - error = kwargs.pop("error", None) - input = kwargs.pop("input", None) + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) if input is str: raise ValueError("Cannot use `str` as input stream.") @@ -126,85 +126,90 @@ class Executable(object): return subprocess.PIPE, False else: return arg, False + ostream, close_ostream = streamify(output, 'w') - estream, close_estream = streamify(error, 'w') - istream, close_istream = streamify(input, 'r') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): - ignore_errors = (ignore_errors,) + ignore_errors = (ignore_errors, ) quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: - tty.warn("Quotes in command arguments can confuse scripts like configure.", - "The following arguments may cause problems when executed:", - str("\n".join([" "+arg for arg in quoted_args])), - "Quotes aren't needed because spack doesn't use a shell.", - "Consider removing them") + tty.warn( + "Quotes in command arguments can confuse scripts like" + " configure.", + "The following arguments may cause problems when executed:", + str("\n".join([" " + arg for arg in quoted_args])), + "Quotes aren't needed because spack doesn't use a shell.", + "Consider removing them") cmd = self.exe + list(args) - cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) + cmd_line = "'%s'" % "' '".join( + map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( - cmd, stdin=istream, stderr=estream, stdout=ostream) + cmd, + stdin=istream, + stderr=estream, + stdout=ostream, + env=env) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): - raise ProcessError("Command exited with status %d:" - % proc.returncode, cmd_line) + raise ProcessError("Command exited with status %d:" % + proc.returncode, cmd_line) if output is str or error is str: result = '' - if output is str: result += out - if error is str: result += err + if output is str: + result += out + if error is str: + result += err return result except OSError, e: raise ProcessError( - "%s: %s" % (self.exe[0], e.strerror), - "Command: " + cmd_line) + "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line) except subprocess.CalledProcessError, e: if fail_on_error: raise ProcessError( - str(e), - "\nExit status %d when invoking command: %s" - % (proc.returncode, cmd_line)) + str(e), "\nExit status %d when invoking command: %s" % + (proc.returncode, cmd_line)) finally: - if close_ostream: output.close() - if close_estream: error.close() - if close_istream: input.close() - + if close_ostream: + output.close() + if close_estream: + error.close() + if close_istream: + input.close() def __eq__(self, other): return self.exe == other.exe - def __neq__(self, other): return not (self == other) - def __hash__(self): - return hash((type(self),) + tuple(self.exe)) - + return hash((type(self), ) + tuple(self.exe)) def __repr__(self): return "<exe: %s>" % self.exe - def __str__(self): return ' '.join(self.exe) - def which(name, **kwargs): """Finds an executable in the path like command-line which.""" - path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) + path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) required = kwargs.get('required', False) if not path: @@ -233,14 +238,16 @@ class ProcessError(spack.error.SpackError): @property def long_message(self): msg = self._long_message - if msg: msg += "\n\n" + if msg: + msg += "\n\n" if self.build_log: msg += "See build log for details:\n" msg += " %s" % self.build_log if self.package_context: - if msg: msg += "\n\n" + if msg: + msg += "\n\n" msg += '\n'.join(self.package_context) return msg @@ -267,7 +274,7 @@ def _get_package_context(): frame = f[0] # Find a frame with 'self' in the local variables. - if not 'self' in frame.f_locals: + if 'self' not in frame.f_locals: continue # Look only at a frame in a subclass of spack.Package @@ -280,9 +287,8 @@ def _get_package_context(): # Build a message showing where in install we failed. lines.append("%s:%d, in %s:" % ( - inspect.getfile(frame.f_code), - frame.f_lineno, - frame.f_code.co_name)) + inspect.getfile(frame.f_code), frame.f_lineno, frame.f_code.co_name + )) sourcelines, start = inspect.getsourcelines(frame) for i, line in enumerate(sourcelines): |