summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGregory L. Lee <lee218@llnl.gov>2015-06-01 08:39:07 -0700
committerGregory L. Lee <lee218@llnl.gov>2015-06-01 08:39:07 -0700
commit37aa88ba2a0da15ec9e75545e7d01b4abbf40233 (patch)
treeb53e7e1720cfa232b5143f6cc6c05f40da2338e9
parent74b0a9c059bc854d94e2160efbe9bd3fa401af0f (diff)
parent16c25886358ca8e89d2cad252be26677c9137e27 (diff)
downloadspack-37aa88ba2a0da15ec9e75545e7d01b4abbf40233.tar.gz
spack-37aa88ba2a0da15ec9e75545e7d01b4abbf40233.tar.bz2
spack-37aa88ba2a0da15ec9e75545e7d01b4abbf40233.tar.xz
spack-37aa88ba2a0da15ec9e75545e7d01b4abbf40233.zip
Merge branch 'develop' of ssh://cz-stash.llnl.gov:7999/scale/spack into develop
-rw-r--r--lib/spack/llnl/util/filesystem.py4
-rw-r--r--lib/spack/llnl/util/tty/__init__.py8
-rw-r--r--lib/spack/llnl/util/tty/color.py8
-rw-r--r--lib/spack/llnl/util/tty/log.py178
-rw-r--r--lib/spack/spack/build_environment.py8
-rw-r--r--lib/spack/spack/cmd/edit.py2
-rw-r--r--lib/spack/spack/cmd/find.py22
-rw-r--r--lib/spack/spack/cmd/install.py4
-rw-r--r--lib/spack/spack/directory_layout.py7
-rw-r--r--lib/spack/spack/error.py7
-rw-r--r--lib/spack/spack/package.py90
-rw-r--r--lib/spack/spack/test/__init__.py3
-rw-r--r--lib/spack/spack/test/make_executable.py125
-rw-r--r--lib/spack/spack/util/environment.py3
-rw-r--r--lib/spack/spack/util/executable.py93
15 files changed, 487 insertions, 75 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 1fdd25f608..3b34e04740 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -152,7 +152,7 @@ def copy_mode(src, dest):
def install(src, dest):
"""Manually install a file to a particular location."""
- tty.info("Installing %s to %s" % (src, dest))
+ tty.debug("Installing %s to %s" % (src, dest))
shutil.copy(src, dest)
set_install_permissions(dest)
copy_mode(src, dest)
@@ -160,7 +160,7 @@ def install(src, dest):
def install_tree(src, dest, **kwargs):
"""Manually install a file to a particular location."""
- tty.info("Installing %s to %s" % (src, dest))
+ tty.debug("Installing %s to %s" % (src, dest))
shutil.copytree(src, dest, **kwargs)
for s, d in traverse_tree(src, dest, follow_nonexisting=False):
diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py
index aba9e61f4f..48368543ff 100644
--- a/lib/spack/llnl/util/tty/__init__.py
+++ b/lib/spack/llnl/util/tty/__init__.py
@@ -36,6 +36,14 @@ _debug = False
_verbose = False
indent = " "
+def is_verbose():
+ return _verbose
+
+
+def is_debug():
+ return _debug
+
+
def set_debug(flag):
global _debug
_debug = flag
diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py
index 81688d7f14..22080a7b37 100644
--- a/lib/spack/llnl/util/tty/color.py
+++ b/lib/spack/llnl/util/tty/color.py
@@ -99,6 +99,10 @@ colors = {'k' : 30, 'K' : 90, # black
color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)'
+# Force color even if stdout is not a tty.
+_force_color = False
+
+
class match_to_ansi(object):
def __init__(self, color=True):
self.color = color
@@ -162,7 +166,7 @@ def cwrite(string, stream=sys.stdout, color=None):
then it will be set based on stream.isatty().
"""
if color is None:
- color = stream.isatty()
+ color = stream.isatty() or _force_color
stream.write(colorize(string, color=color))
@@ -189,7 +193,7 @@ class ColorStream(object):
if raw:
color=True
else:
- color = self._stream.isatty()
+ color = self._stream.isatty() or _force_color
raw_write(colorize(string, color=color))
def writelines(self, sequence, **kwargs):
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py
new file mode 100644
index 0000000000..6ccd0e66d9
--- /dev/null
+++ b/lib/spack/llnl/util/tty/log.py
@@ -0,0 +1,178 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Utility classes for logging the output of blocks of code.
+"""
+import sys
+import os
+import re
+import select
+import inspect
+import llnl.util.tty as tty
+import llnl.util.tty.color as color
+
+# Use this to strip escape sequences
+_escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h')
+
+def _strip(line):
+ """Strip color and control characters from a line."""
+ return _escape.sub('', line)
+
+
+class _SkipWithBlock():
+ """Special exception class used to skip a with block."""
+ pass
+
+
+class log_output(object):
+ """Redirects output and error of enclosed block to a file.
+
+ Usage:
+ with log_output(open('logfile.txt', 'w')):
+ # do things ... output will be logged.
+
+ or:
+ with log_output(open('logfile.txt', 'w'), echo=True):
+ # do things ... output will be logged
+ # and also printed to stdout.
+
+ Closes the provided stream when done with the block.
+ If echo is True, also prints the output to stdout.
+ """
+ def __init__(self, stream, echo=False, force_color=False, debug=False):
+ self.stream = stream
+
+ # various output options
+ self.echo = echo
+ self.force_color = force_color
+ self.debug = debug
+
+ def trace(self, frame, event, arg):
+ """Jumps to __exit__ on the child process."""
+ raise _SkipWithBlock()
+
+
+ def __enter__(self):
+ """Redirect output from the with block to a file.
+
+ This forks the with block as a separate process, with stdout
+ and stderr redirected back to the parent via a pipe. If
+ echo is set, also writes to standard out.
+
+ """
+ # remember these values for later.
+ self._force_color = color._force_color
+ self._debug = tty._debug
+
+ read, write = os.pipe()
+
+ self.pid = os.fork()
+ if self.pid:
+ # Parent: read from child, skip the with block.
+ os.close(write)
+
+ read_file = os.fdopen(read, 'r', 0)
+ with self.stream as log_file:
+ while True:
+ rlist, w, x = select.select([read_file], [], [])
+ if not rlist:
+ break
+
+ line = read_file.readline()
+ if not line:
+ break
+
+ # Echo to stdout if requested.
+ if self.echo:
+ sys.stdout.write(line)
+
+ # Stripped output to log file.
+ log_file.write(_strip(line))
+
+ read_file.flush()
+ read_file.close()
+
+ # Set a trace function to skip the with block.
+ sys.settrace(lambda *args, **keys: None)
+ frame = inspect.currentframe(1)
+ frame.f_trace = self.trace
+
+ else:
+ # Child: redirect output, execute the with block.
+ os.close(read)
+
+ # Save old stdout and stderr
+ self._stdout = os.dup(sys.stdout.fileno())
+ self._stderr = os.dup(sys.stderr.fileno())
+
+ # redirect to the pipe.
+ os.dup2(write, sys.stdout.fileno())
+ os.dup2(write, sys.stderr.fileno())
+
+ if self.force_color:
+ color._force_color = True
+
+ if self.debug:
+ tty._debug = True
+
+
+ def __exit__(self, exc_type, exception, traceback):
+ """Exits on child, handles skipping the with block on parent."""
+ # Child should just exit here.
+ if self.pid == 0:
+ # Flush the log to disk.
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ if exception:
+ # Restore stdout on the child if there's an exception,
+ # and let it be raised normally.
+ #
+ # This assumes that even if the exception is caught,
+ # the child will exit with a nonzero return code. If
+ # it doesn't, the child process will continue running.
+ #
+ # TODO: think about how this works outside install.
+ # TODO: ideally would propagate exception to parent...
+ os.dup2(self._stdout, sys.stdout.fileno())
+ os.dup2(self._stderr, sys.stderr.fileno())
+
+ return False
+
+ else:
+ # Die quietly if there was no exception.
+ os._exit(0)
+
+ else:
+ # If the child exited badly, parent also should exit.
+ pid, returncode = os.waitpid(self.pid, 0)
+ if returncode != 0:
+ os._exit(1)
+
+ # restore output options.
+ color._force_color = self._force_color
+ tty._debug = self._debug
+
+ # Suppresses exception if it's our own.
+ return exc_type is _SkipWithBlock
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 2e72f3c787..f9e795ac42 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -73,14 +73,14 @@ class MakeExecutable(Executable):
self.jobs = jobs
def __call__(self, *args, **kwargs):
- parallel = kwargs.get('parallel', self.jobs > 1)
- disable_parallel = env_flag(SPACK_NO_PARALLEL_MAKE)
+ disable = env_flag(SPACK_NO_PARALLEL_MAKE)
+ parallel = not disable and kwargs.get('parallel', self.jobs > 1)
- if self.jobs > 1 and not disable_parallel:
+ if parallel:
jobs = "-j%d" % self.jobs
args = (jobs,) + args
- super(MakeExecutable, self).__call__(*args, **kwargs)
+ return super(MakeExecutable, self).__call__(*args, **kwargs)
def set_compiler_environment_variables(pkg):
diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py
index c4225a7dd3..b8764ba391 100644
--- a/lib/spack/spack/cmd/edit.py
+++ b/lib/spack/spack/cmd/edit.py
@@ -95,6 +95,7 @@ def edit(parser, args):
path = join_path(spack.cmd.command_path, name + ".py")
if not os.path.exists(path):
tty.die("No command named '%s'." % name)
+ spack.editor(path)
else:
# By default open the directory where packages or commands live.
@@ -103,4 +104,3 @@ def edit(parser, args):
spack.editor(path)
else:
edit_package(name, args.force)
-
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index 15c1cc9196..a0232d2f12 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -50,20 +50,28 @@ def setup_parser(subparser):
subparser.add_argument(
'-l', '--long', action='store_true', dest='long',
help='Show dependency hashes as well as versions.')
+ subparser.add_argument(
+ '-L', '--very-long', action='store_true', dest='very_long',
+ help='Show dependency hashes as well as versions.')
subparser.add_argument(
'query_specs', nargs=argparse.REMAINDER,
help='optional specs to filter results')
-def gray_hash(spec):
- return colorize('@K{[%s]}' % spec.dag_hash(7))
+def gray_hash(spec, length):
+ return colorize('@K{%s}' % spec.dag_hash(length))
def display_specs(specs, **kwargs):
mode = kwargs.get('mode', 'short')
hashes = kwargs.get('long', False)
+ hlen = 7
+ if kwargs.get('very_long', False):
+ hashes = True
+ hlen = None
+
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
@@ -87,6 +95,8 @@ def display_specs(specs, **kwargs):
format = " %-{}s%s".format(width)
for abbrv, spec in zip(abbreviated, specs):
+ if hashes:
+ print gray_hash(spec, hlen),
print format % (abbrv, spec.prefix)
elif mode == 'deps':
@@ -95,13 +105,13 @@ def display_specs(specs, **kwargs):
format='$_$@$+',
color=True,
indent=4,
- prefix=(lambda s: gray_hash(s)) if hashes else None)
+ prefix=(lambda s: gray_hash(s, hlen)) if hashes else None)
elif mode == 'short':
def fmt(s):
string = ""
if hashes:
- string += gray_hash(s) + ' '
+ string += gray_hash(s, hlen) + ' '
string += s.format('$-_$@$+', color=True)
return string
colify(fmt(s) for s in specs)
@@ -138,4 +148,6 @@ def find(parser, args):
if sys.stdout.isatty():
tty.msg("%d installed packages." % len(specs))
- display_specs(specs, mode=args.mode, long=args.long)
+ display_specs(specs, mode=args.mode,
+ long=args.long,
+ very_long=args.very_long)
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index 820444a404..acb688a092 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -48,6 +48,9 @@ def setup_parser(subparser):
'-n', '--no-checksum', action='store_true', dest='no_checksum',
help="Do not check packages against checksum")
subparser.add_argument(
+ '-v', '--verbose', action='store_true', dest='verbose',
+ help="Display verbose build output while installing.")
+ subparser.add_argument(
'--fake', action='store_true', dest='fake',
help="Fake install. Just remove the prefix and touch a fake file in it.")
subparser.add_argument(
@@ -73,4 +76,5 @@ def install(parser, args):
keep_stage=args.keep_stage,
ignore_deps=args.ignore_deps,
make_jobs=args.jobs,
+ verbose=args.verbose,
fake=args.fake)
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index fe02fff3b0..e61929d8fd 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -35,7 +35,6 @@ import llnl.util.tty as tty
from llnl.util.lang import memoized
from llnl.util.filesystem import join_path, mkdirp
-import spack
from spack.spec import Spec
from spack.error import SpackError
@@ -175,6 +174,7 @@ class YamlDirectoryLayout(DirectoryLayout):
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
+ self.build_log_name = 'build.out' # TODO: use config file.
# Cache of already written/read extension maps.
self._extension_maps = {}
@@ -233,6 +233,11 @@ class YamlDirectoryLayout(DirectoryLayout):
return join_path(self.path_for_spec(spec), self.metadata_dir)
+ def build_log_path(self, spec):
+ return join_path(self.path_for_spec(spec), self.metadata_dir,
+ self.build_log_name)
+
+
def create_install_directory(self, spec):
_check_concrete(spec)
diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py
index e483ea613b..e8fa756682 100644
--- a/lib/spack/spack/error.py
+++ b/lib/spack/spack/error.py
@@ -30,7 +30,12 @@ class SpackError(Exception):
def __init__(self, message, long_message=None):
super(SpackError, self).__init__()
self.message = message
- self.long_message = long_message
+ self._long_message = long_message
+
+
+ @property
+ def long_message(self):
+ return self._long_message
def __str__(self):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 452544be49..5dd410d0e4 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -36,7 +36,7 @@ README.
import os
import re
import time
-import inspect
+import itertools
import subprocess
import platform as py_platform
import multiprocessing
@@ -45,6 +45,7 @@ import textwrap
from StringIO import StringIO
import llnl.util.tty as tty
+from llnl.util.tty.log import log_output
from llnl.util.link_tree import LinkTree
from llnl.util.filesystem import *
from llnl.util.lang import *
@@ -55,12 +56,12 @@ import spack.compilers
import spack.mirror
import spack.hooks
import spack.directives
-import spack.build_environment as build_env
-import spack.url as url
+import spack.build_environment
+import spack.url
+import spack.util.web
import spack.fetch_strategy as fs
from spack.version import *
from spack.stage import Stage
-from spack.util.web import get_pages
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
@@ -427,8 +428,8 @@ class Package(object):
return version_urls[version]
# If we have no idea, try to substitute the version.
- return url.substitute_version(self.nearest_url(version),
- self.url_version(version))
+ return spack.url.substitute_version(self.nearest_url(version),
+ self.url_version(version))
@property
@@ -711,20 +712,28 @@ class Package(object):
mkdirp(self.prefix.man1)
- def do_install(self, **kwargs):
- """This class should call this version of the install method.
- Package implementations should override install().
- """
- # whether to keep the prefix on failure. Default is to destroy it.
- keep_prefix = kwargs.get('keep_prefix', False)
- keep_stage = kwargs.get('keep_stage', False)
- ignore_deps = kwargs.get('ignore_deps', False)
- fake_install = kwargs.get('fake', False)
- skip_patch = kwargs.get('skip_patch', False)
+ def _build_logger(self, log_path):
+ """Create a context manager to log build output."""
+
+
- # Override builtin number of make jobs.
- self.make_jobs = kwargs.get('make_jobs', None)
+ def do_install(self,
+ keep_prefix=False, keep_stage=False, ignore_deps=False,
+ skip_patch=False, verbose=False, make_jobs=None, fake=False):
+ """Called by commands to install a package and its dependencies.
+ Package implementations should override install() to describe
+ their build process.
+
+ Args:
+ keep_prefix -- Keep install prefix on failure. By default, destroys it.
+ keep_stage -- Keep stage on successful build. By default, destroys it.
+ ignore_deps -- Do not install dependencies before installing this package.
+ fake -- Don't really build -- install fake stub files instead.
+ skip_patch -- Skip patch stage of build if True.
+ verbose -- Display verbose build output (by default, suppresses it)
+ make_jobs -- Number of make jobs to use for install. Default is ncpus.
+ """
if not self.spec.concrete:
raise ValueError("Can only install concrete packages.")
@@ -735,10 +744,13 @@ class Package(object):
tty.msg("Installing %s" % self.name)
if not ignore_deps:
- self.do_install_dependencies(**kwargs)
+ self.do_install_dependencies(
+ keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps,
+ fake=fake, skip_patch=skip_patch, verbose=verbose,
+ make_jobs=make_jobs)
start_time = time.time()
- if not fake_install:
+ if not fake:
if not skip_patch:
self.do_patch()
else:
@@ -768,16 +780,26 @@ class Package(object):
spack.hooks.pre_install(self)
# Set up process's build environment before running install.
- if fake_install:
+ if fake:
self.do_fake_install()
else:
- # Subclasses implement install() to do the real work.
+ # Do the real install in the source directory.
self.stage.chdir_to_source()
- self.install(self.spec, self.prefix)
+
+ # This redirects I/O to a build log (and optionally to the terminal)
+ log_path = join_path(os.getcwd(), 'spack-build.out')
+ log_file = open(log_path, 'w')
+ with log_output(log_file, verbose, sys.stdout.isatty(), True):
+ self.install(self.spec, self.prefix)
# Ensure that something was actually installed.
self._sanity_check_install()
+ # Move build log into install directory on success
+ if not fake:
+ log_install_path = spack.install_layout.build_log_path(self.spec)
+ install(log_path, log_install_path)
+
# On successful install, remove the stage.
if not keep_stage:
self.stage.destroy()
@@ -792,6 +814,9 @@ class Package(object):
print_pkg(self.prefix)
except ProcessError, e:
+ # Annotate with location of build log.
+ e.build_log = log_path
+
# One of the processes returned an error code.
# Suppress detailed stack trace here unless in debug mode
if spack.debug:
@@ -801,13 +826,14 @@ class Package(object):
# Still need to clean up b/c there was an error.
cleanup()
+ os._exit(1)
except:
# other exceptions just clean up and raise.
cleanup()
raise
- build_env.fork(self, real_work)
+ spack.build_environment.fork(self, real_work)
# Once everything else is done, run post install hooks
spack.hooks.post_install(self)
@@ -867,9 +893,7 @@ class Package(object):
raise InstallError("Package %s provides no install method!" % self.name)
- def do_uninstall(self, **kwargs):
- force = kwargs.get('force', False)
-
+ def do_uninstall(self, force=False):
if not self.installed:
raise InstallError(str(self.spec) + " is not installed.")
@@ -912,14 +936,13 @@ class Package(object):
raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name))
- def do_activate(self, **kwargs):
+ def do_activate(self, force=False):
"""Called on an etension to invoke the extendee's activate method.
Commands should call this routine, and should not call
activate() directly.
"""
self._sanity_check_extension()
- force = kwargs.get('force', False)
spack.install_layout.check_extension_conflict(
self.extendee_spec, self.spec)
@@ -929,7 +952,7 @@ class Package(object):
for spec in self.spec.traverse(root=False):
if spec.package.extends(self.extendee_spec):
if not spec.package.activated:
- spec.package.do_activate(**kwargs)
+ spec.package.do_activate(force=force)
self.extendee_spec.package.activate(self, **self.extendee_args)
@@ -1083,12 +1106,13 @@ def find_versions_of_archive(*archive_urls, **kwargs):
if list_url:
list_urls.add(list_url)
for aurl in archive_urls:
- list_urls.add(url.find_list_url(aurl))
+ list_urls.add(spack.url.find_list_url(aurl))
# Grab some web pages to scrape.
page_map = {}
for lurl in list_urls:
- page_map.update(get_pages(lurl, depth=list_depth))
+ pages = spack.util.web.get_pages(lurl, depth=list_depth)
+ page_map.update(pages)
# Scrape them for archive URLs
regexes = []
@@ -1097,7 +1121,7 @@ def find_versions_of_archive(*archive_urls, **kwargs):
# the version part of the URL. The capture group is converted
# to a generic wildcard, so we can use this to extract things
# on a page that look like archive URLs.
- url_regex = url.wildcard_version(aurl)
+ url_regex = spack.url.wildcard_version(aurl)
# We'll be a bit more liberal and just look for the archive
# part, not the full path.
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 7ff512c370..8d4b0c6cde 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -54,7 +54,8 @@ test_names = ['versions',
'cc',
'link_tree',
'spec_yaml',
- 'optional_deps']
+ 'optional_deps',
+ 'make_executable']
def list_tests():
diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py
new file mode 100644
index 0000000000..c4bfeb2a03
--- /dev/null
+++ b/lib/spack/spack/test/make_executable.py
@@ -0,0 +1,125 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Tests for Spack's built-in parallel make support.
+
+This just tests whether the right args are getting passed to make.
+"""
+import os
+import unittest
+import tempfile
+import shutil
+
+from llnl.util.filesystem import *
+from spack.util.environment import path_put_first
+from spack.build_environment import MakeExecutable
+
+
+class MakeExecutableTest(unittest.TestCase):
+ def setUp(self):
+ self.tmpdir = tempfile.mkdtemp()
+
+ make_exe = join_path(self.tmpdir, 'make')
+ with open(make_exe, 'w') as f:
+ f.write('#!/bin/sh\n')
+ f.write('echo "$@"')
+ os.chmod(make_exe, 0700)
+
+ path_put_first('PATH', [self.tmpdir])
+
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
+
+ def test_make_normal(self):
+ make = MakeExecutable('make', 8)
+ self.assertEqual(make(return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+
+
+ def test_make_explicit(self):
+ make = MakeExecutable('make', 8)
+ self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+
+
+ def test_make_one_job(self):
+ make = MakeExecutable('make', 1)
+ self.assertEqual(make(return_output=True).strip(), '')
+ self.assertEqual(make('install', return_output=True).strip(), 'install')
+
+
+ def test_make_parallel_false(self):
+ make = MakeExecutable('make', 8)
+ self.assertEqual(make(parallel=False, return_output=True).strip(), '')
+ self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install')
+
+
+ def test_make_parallel_disabled(self):
+ make = MakeExecutable('make', 8)
+
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
+ self.assertEqual(make(return_output=True).strip(), '')
+ self.assertEqual(make('install', return_output=True).strip(), 'install')
+
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
+ self.assertEqual(make(return_output=True).strip(), '')
+ self.assertEqual(make('install', return_output=True).strip(), 'install')
+
+ # These don't disable (false and random string)
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
+ self.assertEqual(make(return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
+ self.assertEqual(make(return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
+
+ del os.environ['SPACK_NO_PARALLEL_MAKE']
+
+
+ def test_make_parallel_precedence(self):
+ make = MakeExecutable('make', 8)
+
+ # These should work
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
+ self.assertEqual(make(parallel=True, return_output=True).strip(), '')
+ self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
+
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
+ self.assertEqual(make(parallel=True, return_output=True).strip(), '')
+ self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
+
+ # These don't disable (false and random string)
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
+ self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+
+ os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
+ self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
+ self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
+
+ del os.environ['SPACK_NO_PARALLEL_MAKE']
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index afdf51c707..7a4ff919ad 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -35,7 +35,8 @@ def get_path(name):
def env_flag(name):
if name in os.environ:
- return os.environ[name].lower() == "true"
+ value = os.environ[name].lower()
+ return value == "true" or value == "1"
return False
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 81759d3053..6eede0f78e 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -143,27 +143,72 @@ def which(name, **kwargs):
class ProcessError(spack.error.SpackError):
- def __init__(self, msg, long_msg=None):
- # Friendlier exception trace info for failed executables
- long_msg = long_msg + "\n" if long_msg else ""
- for f in inspect.stack():
- frame = f[0]
- loc = frame.f_locals
- if 'self' in loc:
- obj = loc['self']
- if isinstance(obj, spack.Package):
- long_msg += "---\n"
- long_msg += "Context:\n"
- long_msg += " %s:%d, in %s:\n" % (
- inspect.getfile(frame.f_code),
- frame.f_lineno,
- frame.f_code.co_name)
-
- lines, start = inspect.getsourcelines(frame)
- for i, line in enumerate(lines):
- mark = ">> " if start + i == frame.f_lineno else " "
- long_msg += " %s%-5d%s\n" % (
- mark, start + i, line.rstrip())
- break
-
- super(ProcessError, self).__init__(msg, long_msg)
+ def __init__(self, msg, long_message=None):
+ # These are used for detailed debugging information for
+ # package builds. They're built up gradually as the exception
+ # propagates.
+ self.package_context = _get_package_context()
+ self.build_log = None
+
+ super(ProcessError, self).__init__(msg, long_message)
+
+ @property
+ def long_message(self):
+ msg = self._long_message
+ if msg: msg += "\n\n"
+
+ if self.build_log:
+ msg += "See build log for details:\n"
+ msg += " %s" % self.build_log
+
+ if self.package_context:
+ if msg: msg += "\n\n"
+ msg += '\n'.join(self.package_context)
+
+ return msg
+
+
+def _get_package_context():
+ """Return some context for an error message when the build fails.
+
+ This should be called within a ProcessError when the exception is
+ thrown.
+
+ Args:
+ process_error -- A ProcessError raised during install()
+
+ This function inspects the stack to find where we failed in the
+ package file, and it adds detailed context to the long_message
+ from there.
+
+ """
+ lines = []
+
+ # Walk up the stack
+ for f in inspect.stack():
+ frame = f[0]
+
+ # Find a frame with 'self' in the local variables.
+ if not 'self' in frame.f_locals:
+ continue
+
+ # Look only at a frame in a subclass of spack.Package
+ obj = frame.f_locals['self']
+ if type(obj) != spack.Package and isinstance(obj, spack.Package):
+ break
+ else:
+ # Didn't find anything
+ return lines
+
+ # Build a message showing where in install we failed.
+ lines.append("%s:%d, in %s:" % (
+ inspect.getfile(frame.f_code),
+ frame.f_lineno,
+ frame.f_code.co_name))
+
+ sourcelines, start = inspect.getsourcelines(frame)
+ for i, line in enumerate(sourcelines):
+ mark = ">> " if start + i == frame.f_lineno else " "
+ lines.append(" %s%-5d%s" % (mark, start + i, line.rstrip()))
+
+ return lines