diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/llnl/util/tty/log.py | 204 | ||||
| -rw-r--r-- | lib/spack/spack/__init__.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/build_environment.py | 63 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/install.py | 8 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/setup.py | 87 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 707 |
6 files changed, 636 insertions, 446 deletions
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index b67edcf9cc..a4ba2a9bdf 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -24,11 +24,11 @@ ############################################################################## """Utility classes for logging the output of blocks of code. """ -import sys +import multiprocessing import os import re import select -import inspect +import sys import llnl.util.tty as tty import llnl.util.tty.color as color @@ -100,25 +100,29 @@ class keyboard_input(object): class log_output(object): - """Redirects output and error of enclosed block to a file. + """Spawns a daemon that reads from a pipe and writes to a file Usage: - with log_output(open('logfile.txt', 'w')): - # do things ... output will be logged. + # Spawns the daemon + with log_output('logfile.txt', 'w') as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged or: - with log_output(open('logfile.txt', 'w'), echo=True): - # do things ... output will be logged - # and also printed to stdout. - - Closes the provided stream when done with the block. - If echo is True, also prints the output to stdout. + with log_output('logfile.txt', echo=True) as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged + # and also printed to stdout. + + Opens a stream in 'w' mode at daemon spawning and closes it at + daemon joining. If echo is True, also prints the output to stdout. """ - def __init__(self, stream, echo=False, force_color=False, debug=False): - self.stream = stream - - # various output options + def __init__(self, filename, echo=False, force_color=False, debug=False): + self.filename = filename + # Various output options self.echo = echo self.force_color = force_color self.debug = debug @@ -126,70 +130,81 @@ class log_output(object): # Default is to try file-descriptor reassignment unless the system # out/err streams do not have an associated file descriptor self.directAssignment = False - - def trace(self, frame, event, arg): - """Jumps to __exit__ on the child process.""" - raise _SkipWithBlock() + self.read, self.write = os.pipe() + + # Sets a daemon that writes to file what it reads from a pipe + self.p = multiprocessing.Process( + target=self._spawn_writing_daemon, + args=(self.read,), + name='logger_daemon' + ) + self.p.daemon = True + # Needed to un-summon the daemon + self.parent_pipe, self.child_pipe = multiprocessing.Pipe() def __enter__(self): - """Redirect output from the with block to a file. - - This forks the with block as a separate process, with stdout - and stderr redirected back to the parent via a pipe. If - echo is set, also writes to standard out. - - """ - # remember these values for later. - self._force_color = color._force_color - self._debug = tty._debug - - read, write = os.pipe() - - self.pid = os.fork() - if self.pid: - # Parent: read from child, skip the with block. - os.close(write) - - read_file = os.fdopen(read, 'r', 0) - with self.stream as log_file: - with keyboard_input(sys.stdin): - while True: - rlist, w, x = select.select( - [read_file, sys.stdin], [], []) - if not rlist: + self.p.start() + return log_output.OutputRedirection(self) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.parent_pipe.send(True) + self.p.join(60.0) # 1 minute to join the child + + def _spawn_writing_daemon(self, read): + # Parent: read from child, skip the with block. + read_file = os.fdopen(read, 'r', 0) + with open(self.filename, 'w') as log_file: + with keyboard_input(sys.stdin): + while True: + rlist, _, _ = select.select([read_file, sys.stdin], [], []) + if not rlist: + break + + # Allow user to toggle echo with 'v' key. + # Currently ignores other chars. + if sys.stdin in rlist: + if sys.stdin.read(1) == 'v': + self.echo = not self.echo + + # Handle output from the with block process. + if read_file in rlist: + line = read_file.readline() + if not line: + # For some reason we never reach this point... break - # Allow user to toggle echo with 'v' key. - # Currently ignores other chars. - if sys.stdin in rlist: - if sys.stdin.read(1) == 'v': - self.echo = not self.echo + # Echo to stdout if requested. + if self.echo: + sys.stdout.write(line) - # handle output from the with block process. - if read_file in rlist: - line = read_file.readline() - if not line: - break + # Stripped output to log file. + log_file.write(_strip(line)) + log_file.flush() - # Echo to stdout if requested. - if self.echo: - sys.stdout.write(line) + if self.child_pipe.poll(): + break - # Stripped output to log file. - log_file.write(_strip(line)) + def __del__(self): + """Closes the pipes""" + os.close(self.write) + os.close(self.read) - read_file.flush() - read_file.close() + class OutputRedirection(object): - # Set a trace function to skip the with block. - sys.settrace(lambda *args, **keys: None) - frame = inspect.currentframe(1) - frame.f_trace = self.trace + def __init__(self, other): + self.__dict__.update(other.__dict__) - else: - # Child: redirect output, execute the with block. - os.close(read) + def __enter__(self): + """Redirect output from the with block to a file. + Hijacks stdout / stderr and writes to the pipe + connected to the logger daemon + """ + # remember these values for later. + self._force_color = color._force_color + self._debug = tty._debug + # Redirect this output to a pipe + write = self.write try: # Save old stdout and stderr self._stdout = os.dup(sys.stdout.fileno()) @@ -205,53 +220,26 @@ class log_output(object): output_redirect = os.fdopen(write, 'w') sys.stdout = output_redirect sys.stderr = output_redirect - if self.force_color: color._force_color = True - if self.debug: tty._debug = True - def __exit__(self, exc_type, exception, traceback): - """Exits on child, handles skipping the with block on parent.""" - # Child should just exit here. - if self.pid == 0: + def __exit__(self, exc_type, exception, traceback): + """Plugs back the original file descriptors + for stdout and stderr + """ # Flush the log to disk. sys.stdout.flush() sys.stderr.flush() - - if exception: - # Restore stdout on the child if there's an exception, - # and let it be raised normally. - # - # This assumes that even if the exception is caught, - # the child will exit with a nonzero return code. If - # it doesn't, the child process will continue running. - # - # TODO: think about how this works outside install. - # TODO: ideally would propagate exception to parent... - if self.directAssignment: - sys.stdout = self._stdout - sys.stderr = self._stderr - else: - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) - - return False - + if self.directAssignment: + # We seem to need this only to pass test/install.py + sys.stdout = self._stdout + sys.stderr = self._stderr else: - # Die quietly if there was no exception. - os._exit(0) - - else: - # If the child exited badly, parent also should exit. - pid, returncode = os.waitpid(self.pid, 0) - if returncode != 0: - os._exit(1) - - # restore output options. - color._force_color = self._force_color - tty._debug = self._debug + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) - # Suppresses exception if it's our own. - return exc_type is _SkipWithBlock + # restore output options. + color._force_color = self._force_color + tty._debug = self._debug diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 3d508d0fde..c9c29a32cc 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -185,10 +185,17 @@ sys_type = None # packages should live. This file is overloaded for spack core vs. # for packages. # -__all__ = ['Package', 'StagedPackage', 'CMakePackage', - 'Version', 'when', 'ver', 'alldeps', 'nolink'] +__all__ = ['Package', + 'CMakePackage', + 'AutotoolsPackage', + 'EditableMakefile', + 'Version', + 'when', + 'ver', + 'alldeps', + 'nolink'] from spack.package import Package, ExtensionConflictError -from spack.package import StagedPackage, CMakePackage +from spack.package import CMakePackage, AutotoolsPackage, EditableMakefile from spack.version import Version, ver from spack.spec import DependencySpec, alldeps, nolink from spack.multimethod import when diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 15fb943ca4..98fb3b6917 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -51,16 +51,14 @@ There are two parts to the build environment: Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ +import multiprocessing import os -import sys import shutil -import multiprocessing -import platform +import sys import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -349,8 +347,8 @@ def set_module_variables_for_package(pkg, module): m.cmake = Executable('cmake') m.ctest = Executable('ctest') - # standard CMake arguments - m.std_cmake_args = get_std_cmake_args(pkg) + # Standard CMake arguments + m.std_cmake_args = spack.CMakePackage._std_args(pkg) # Put spack compiler paths in module scope. link_dir = spack.build_env_path @@ -520,41 +518,26 @@ def fork(pkg, function, dirty=False): carries on. """ - try: - pid = os.fork() - except OSError as e: - raise InstallError("Unable to fork build process: %s" % e) - - if pid == 0: - # Give the child process the package's build environment. - setup_package(pkg, dirty=dirty) - + def child_execution(child_connection): try: - # call the forked function. + setup_package(pkg, dirty=dirty) function() - - # Use os._exit here to avoid raising a SystemExit exception, - # which interferes with unit tests. - os._exit(0) - - except spack.error.SpackError as e: - e.die() - - except: - # Child doesn't raise or return to main spack code. - # Just runs default exception handler and exits. - sys.excepthook(*sys.exc_info()) - os._exit(1) - - else: - # Parent process just waits for the child to complete. If the - # child exited badly, assume it already printed an appropriate - # message. Just make the parent exit with an error code. - pid, returncode = os.waitpid(pid, 0) - if returncode != 0: - message = "Installation process had nonzero exit code : {code}" - strcode = str(returncode) - raise InstallError(message.format(code=strcode)) + child_connection.send([None, None, None]) + except Exception as e: + child_connection.send([type(e), e, None]) + finally: + child_connection.close() + + parent_connection, child_connection = multiprocessing.Pipe() + p = multiprocessing.Process( + target=child_execution, + args=(child_connection,) + ) + p.start() + exc_type, exception, traceback = parent_connection.recv() + p.join() + if exception is not None: + raise exception class InstallError(spack.error.SpackError): diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 70abe1dd00..cc9b6c73ee 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -22,7 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from __future__ import print_function import argparse import llnl.util.tty as tty @@ -63,6 +62,9 @@ def setup_parser(subparser): '--dirty', action='store_true', dest='dirty', help="Install a package *without* cleaning the environment.") subparser.add_argument( + '--stop-at', help="Stop at a particular phase of installation" + ) + subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") subparser.add_argument( @@ -94,4 +96,6 @@ def install(parser, args): verbose=args.verbose, fake=args.fake, dirty=args.dirty, - explicit=True) + explicit=True, + stop_at=args.stop_at + ) diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index c63c566338..f39a827a8d 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -22,16 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys -import os import argparse +import os +import string +import sys import llnl.util.tty as tty - import spack import spack.cmd +from spack import which from spack.cmd.edit import edit_package from spack.stage import DIYStage +from llnl.util.filesystem import set_executable description = "Create a configuration script and module, but don't build." @@ -51,6 +53,72 @@ def setup_parser(subparser): help="Install a package *without* cleaning the environment.") +def spack_transitive_include_path(): + return ';'.join( + os.path.join(dep, 'include') + for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) + ) + + +def write_spconfig(package): + # Set-up the environment + spack.build_environment.setup_package(package) + + cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args() + + env = dict() + + paths = os.environ['PATH'].split(':') + paths = [item for item in paths if 'spack/env' not in item] + env['PATH'] = ':'.join(paths) + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path() + env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] + env['CC'] = os.environ['SPACK_CC'] + env['CXX'] = os.environ['SPACK_CXX'] + env['FC'] = os.environ['SPACK_FC'] + + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: + fout.write( + r"""#!%s +# + +import sys +import os +import subprocess + +def cmdlist(str): + return list(x.strip().replace("'",'') for x in str.split('\n') if x) +env = dict(os.environ) +""" % sys.executable) + + env_vars = sorted(list(env.keys())) + for name in env_vars: + val = env[name] + if string.find(name, 'PATH') < 0: + fout.write('env[%s] = %s\n' % (repr(name), repr(val))) + else: + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': + sep = ';' + else: + sep = ':' + + fout.write( + 'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep)) + for part in string.split(val, sep): + fout.write(' %s\n' % part) + fout.write('"""))\n') + + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") # NOQA: ignore=E501 + fout.write('\ncmd = cmdlist("""\n') + fout.write('%s\n' % cmd[0]) + for arg in cmd[1:]: + fout.write(' %s\n' % arg) + fout.write('""") + sys.argv[1:]\n') + fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + set_executable(setup_fname) + + def setup(self, args): if not args.spec: tty.die("spack setup requires a package spec argument.") @@ -89,11 +157,8 @@ def setup(self, args): # TODO: make this an argument, not a global. spack.do_checksum = False - package.do_install( - keep_prefix=True, # Don't remove install directory - install_deps=not args.ignore_deps, - install_self=True, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SETUP. - install_phases=set(['setup', 'provenance']), - dirty=args.dirty) + if not isinstance(package, spack.CMakePackage): + raise RuntimeError( + 'Support for {0} not yet implemented'.format(type(package))) + + write_spconfig(package) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 768605294f..a9b5b2069a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -33,22 +33,20 @@ Homebrew makes it very easy to create packages. For a complete rundown on spack and how it differs from homebrew, look at the README. """ +import contextlib +import copy +import functools +import inspect import os -import sys +import platform import re +import sys import textwrap import time -import string -import contextlib from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty -from llnl.util.filesystem import * -from llnl.util.lang import * -from llnl.util.link_tree import LinkTree -from llnl.util.tty.log import log_output - import spack import spack.build_environment import spack.compilers @@ -60,20 +58,190 @@ import spack.mirror import spack.repository import spack.url import spack.util.web - +from llnl.util.filesystem import * +from llnl.util.filesystem import * +from llnl.util.lang import * +from llnl.util.lang import * +from llnl.util.link_tree import LinkTree +from llnl.util.tty.log import log_output +from spack import directory_layout from spack.stage import Stage, ResourceStage, StageComposite from spack.util.crypto import bit_length from spack.util.environment import dump_environment -from spack.util.executable import ProcessError, which +from spack.util.executable import ProcessError from spack.version import * -from spack import directory_layout - """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -class Package(object): +class InstallPhase(object): + """Manages a single phase of the installation + + This descriptor stores at creation time the name of the method it should + search for execution. The method is retrieved at __get__ time, so that + it can be overridden by subclasses of whatever class declared the phases. + + It also provides hooks to execute prerequisite and sanity checks. + """ + + def __init__(self, name): + self.name = name + self.preconditions = [] + self.sanity_checks = [] + + def __get__(self, instance, owner): + # The caller is a class that is trying to customize + # my behavior adding something + if instance is None: + return self + # If instance is there the caller wants to execute the + # install phase, thus return a properly set wrapper + phase = getattr(instance, self.name) + + @functools.wraps(phase) + def phase_wrapper(spec, prefix): + # Check instance attributes at the beginning of a phase + self._on_phase_start(instance) + # Execute phase pre-conditions, + # and give them the chance to fail + for check in self.preconditions: + # Do something sensible at some point + check(instance) + phase(spec, prefix) + # Execute phase sanity_checks, + # and give them the chance to fail + for check in self.sanity_checks: + check(instance) + # Check instance attributes at the end of a phase + self._on_phase_exit(instance) + return phase_wrapper + + def _on_phase_start(self, instance): + pass + + def _on_phase_exit(self, instance): + # If a phase has a matching last_phase attribute, + # stop the installation process raising a StopIteration + if getattr(instance, 'last_phase', None) == self.name: + raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + + def copy(self): + try: + return copy.deepcopy(self) + except TypeError: + # This bug-fix was not back-ported in Python 2.6 + # http://bugs.python.org/issue1515 + other = InstallPhase(self.name) + other.preconditions.extend(self.preconditions) + other.sanity_checks.extend(self.sanity_checks) + return other + + +class PackageMeta(type): + """Conveniently transforms attributes to permit extensible phases + + Iterates over the attribute 'phases' and creates / updates private + InstallPhase attributes in the class that is being initialized + """ + phase_fmt = '_InstallPhase_{0}' + + _InstallPhase_sanity_checks = {} + _InstallPhase_preconditions = {} + + def __new__(meta, name, bases, attr_dict): + # Check if phases is in attr dict, then set + # install phases wrappers + if 'phases' in attr_dict: + _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] # NOQA: ignore=E501 + for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): # NOQA: ignore=E501 + attr_dict[phase_name] = InstallPhase(callback_name) + attr_dict['_InstallPhase_phases'] = _InstallPhase_phases + + def _append_checks(check_name): + # Name of the attribute I am going to check it exists + attr_name = PackageMeta.phase_fmt.format(check_name) + checks = getattr(meta, attr_name) + if checks: + for phase_name, funcs in checks.items(): + try: + # Search for the phase in the attribute dictionary + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] + except KeyError: + # If it is not there it's in the bases + # and we added a check. We need to copy + # and extend + for base in bases: + phase = getattr( + base, + PackageMeta.phase_fmt.format(phase_name), + None + ) + attr_dict[PackageMeta.phase_fmt.format( + phase_name)] = phase.copy() + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] + getattr(phase, check_name).extend(funcs) + # Clear the attribute for the next class + setattr(meta, attr_name, {}) + + @classmethod + def _register_checks(cls, check_type, *args): + def _register_sanity_checks(func): + attr_name = PackageMeta.phase_fmt.format(check_type) + check_list = getattr(meta, attr_name) + for item in args: + checks = check_list.setdefault(item, []) + checks.append(func) + setattr(meta, attr_name, check_list) + return func + return _register_sanity_checks + + @staticmethod + def on_package_attributes(**attrs): + def _execute_under_condition(func): + @functools.wraps(func) + def _wrapper(instance): + # If all the attributes have the value we require, then + # execute + if all([getattr(instance, key, None) == value for key, value in attrs.items()]): # NOQA: ignore=E501 + func(instance) + return _wrapper + return _execute_under_condition + + @classmethod + def precondition(cls, *args): + return cls._register_checks('preconditions', *args) + + @classmethod + def sanity_check(cls, *args): + return cls._register_checks('sanity_checks', *args) + + if all([not hasattr(x, '_register_checks') for x in bases]): + attr_dict['_register_checks'] = _register_checks + + if all([not hasattr(x, 'sanity_check') for x in bases]): + attr_dict['sanity_check'] = sanity_check + + if all([not hasattr(x, 'precondition') for x in bases]): + attr_dict['precondition'] = precondition + + if all([not hasattr(x, 'on_package_attributes') for x in bases]): + attr_dict['on_package_attributes'] = on_package_attributes + + # Preconditions + _append_checks('preconditions') + # Sanity checks + _append_checks('sanity_checks') + return super(PackageMeta, meta).__new__(meta, name, bases, attr_dict) + + def __init__(cls, name, bases, dict): + type.__init__(cls, name, bases, dict) + spack.directives.ensure_dicts(cls) + + +class PackageBase(object): """This is the superclass for all spack packages. ***The Package class*** @@ -309,7 +477,7 @@ class Package(object): Package creators override functions like install() (all of them do this), clean() (some of them do this), and others to provide custom behavior. """ - + __metaclass__ = PackageMeta # # These are default values for instance variables. # @@ -344,12 +512,6 @@ class Package(object): """Per-process lock objects for each install prefix.""" prefix_locks = {} - class __metaclass__(type): - """Ensure attributes required by Spack directives are present.""" - def __init__(cls, name, bases, dict): - type.__init__(cls, name, bases, dict) - spack.directives.ensure_dicts(cls) - def __init__(self, spec): # this determines how the package should be built. self.spec = spec @@ -429,6 +591,8 @@ class Package(object): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() + self.extra_args = {} + def possible_dependencies(self, visited=None): """Return set of possible transitive dependencies of this package.""" if visited is None: @@ -886,7 +1050,8 @@ class Package(object): return namespace def do_fake_install(self): - """Make a fake install directory contaiing a 'fake' file in bin.""" + """Make a fake install directory containing a 'fake' file in bin.""" + # FIXME : Make this part of the 'install' behavior ? mkdirp(self.prefix.bin) touch(join_path(self.prefix.bin, 'fake')) mkdirp(self.prefix.lib) @@ -939,7 +1104,7 @@ class Package(object): fake=False, explicit=False, dirty=False, - install_phases=install_phases): + **kwargs): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -977,9 +1142,7 @@ class Package(object): # Ensure package is not already installed layout = spack.install_layout with self._prefix_read_lock(): - if ('install' in install_phases and - layout.check_installed(self.spec)): - + if layout.check_installed(self.spec): tty.msg( "%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) @@ -989,6 +1152,8 @@ class Package(object): rec.explicit = True return + self._do_install_pop_kwargs(kwargs) + tty.msg("Installing %s" % self.name) # First, install dependencies recursively. @@ -1030,112 +1195,134 @@ class Package(object): else: self.do_stage() - tty.msg("Building %s" % self.name) + tty.msg("Building {0} [{1}]".format( + self.name, type(self).__base__)) self.stage.keep = keep_stage - self.install_phases = install_phases self.build_directory = join_path(self.stage.path, 'spack-build') self.source_directory = self.stage.source_path - with contextlib.nested(self.stage, self._prefix_write_lock()): - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') - - try: + try: + with contextlib.nested(self.stage, self._prefix_write_lock()): + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') # Redirect I/O to a build log (and optionally to # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), - True): - dump_environment(env_path) - self.install(self.spec, self.prefix) - - except ProcessError as e: - # Annotate ProcessErrors with the location of - # the build log - e.build_log = log_path - raise e - - # Ensure that something was actually installed. - if 'install' in self.install_phases: - self.sanity_check_prefix() - - # Copy provenance into the install directory on success - if 'provenance' in self.install_phases: - log_install_path = layout.build_log_path(self.spec) - env_install_path = layout.build_env_path(self.spec) - packages_dir = layout.build_packages_path(self.spec) - - # Remove first if we're overwriting another build - # (can happen with spack setup) - try: - # log_install_path and env_install_path are here - shutil.rmtree(packages_dir) - except: - pass - - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) - - # Run post install hooks before build stage is removed. - spack.hooks.post_install(self) - - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time - - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." % - (_hms(self._fetch_time), _hms(build_time), - _hms(self._total_time))) - print_pkg(self.prefix) - # ------------------- END def build_process() + # FIXME : refactor this assignment + self.log_path = log_path + self.env_path = env_path + dump_environment(env_path) + # Spawn a daemon that reads from a pipe and redirects + # everything to log_path + redirection_context = log_output( + log_path, verbose, + sys.stdout.isatty(), + True + ) + with redirection_context as log_redirection: + for phase_name, phase in zip(self.phases, self._InstallPhase_phases): # NOQA: ignore=E501 + tty.msg( + 'Executing phase : \'{0}\''.format(phase_name) # NOQA: ignore=E501 + ) + # Redirect stdout and stderr to daemon pipe + with log_redirection: + getattr(self, phase)( + self.spec, self.prefix) + self.log() + # Run post install hooks before build stage is removed. + spack.hooks.post_install(self) + + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time + + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." % + (_hms(self._fetch_time), _hms(build_time), + _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError as e: + # Annotate ProcessErrors with the location of + # the build log + e.build_log = log_path + raise e try: # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) - except directory_layout.InstallDirectoryAlreadyExistsError: - if 'install' in install_phases: - # Abort install if install directory exists. - # But do NOT remove it (you'd be overwriting someone's data) - tty.warn("Keeping existing install prefix in place.") - raise - else: - # We're not installing anyway, so don't worry if someone - # else has already written in the install directory - pass - - try: + # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) - except: - # remove the install prefix if anything went wrong during install. + # If we installed then we should keep the prefix + keep_prefix = True if self.last_phase is None else keep_prefix + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add( + self.spec, spack.install_layout, explicit=explicit + ) + except directory_layout.InstallDirectoryAlreadyExistsError: + # Abort install if install directory exists. + # But do NOT remove it (you'd be overwriting someone else's stuff) + tty.warn("Keeping existing install prefix in place.") + raise + except StopIteration as e: + # A StopIteration exception means that do_install + # was asked to stop early from clients + tty.msg(e.message) + except Exception: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, + wrap=False) + raise + finally: + # Remove the install prefix if anything went wrong during install. if not keep_prefix: self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, - wrap=False) - raise - # Parent of the build process adds the new package to - # the database, so that we don't need to re-read from file. - # NOTE: add() implicitly acquires a write-lock - spack.installed_db.add( - self.spec, spack.install_layout, explicit=explicit) + def _do_install_pop_kwargs(self, kwargs): + """Pops kwargs from do_install before starting the installation + + Args: + kwargs: + 'stop_at': last installation phase to be executed (or None) + + """ + self.last_phase = kwargs.pop('stop_at', None) + if self.last_phase is not None and self.last_phase not in self.phases: + tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501 + + def log(self): + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path( + self.spec) + env_install_path = spack.install_layout.build_env_path( + self.spec) + packages_dir = spack.install_layout.build_packages_path( + self.spec) + + # Remove first if we're overwriting another build + # (can happen with spack setup) + try: + # log_install_path and env_install_path are inside this + shutil.rmtree(packages_dir) + except Exception: + # FIXME : this potentially catches too many things... + pass + + install(self.log_path, log_install_path) + install(self.env_path, env_install_path) + dump_packages(self.spec, packages_dir) def sanity_check_prefix(self): """This function checks whether install succeeded.""" @@ -1289,13 +1476,6 @@ class Package(object): """ pass - def install(self, spec, prefix): - """ - Package implementations override this with their own configuration - """ - raise InstallError("Package %s provides no install method!" % - self.name) - def do_uninstall(self, force=False): if not self.installed: # prefix may not exist, but DB may be inconsistent. Try to fix by @@ -1506,6 +1686,129 @@ class Package(object): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +class Package(PackageBase): + phases = ['install'] + # This will be used as a registration decorator in user + # packages, if need be + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + +class EditableMakefile(PackageBase): + phases = ['edit', 'build', 'install'] + + def wdir(self): + return self.stage.source_path + + def build_args(self): + return list() + + def install_args(self): + return list() + + def edit(self, spec, prefix): + raise NotImplementedError('\'edit\' function not implemented') + + def build(self, spec, prefix): + args = self.build_args() + with working_dir(self.wdir()): + inspect.getmodule(self).make(*args) + + def install(self, spec, prefix): + args = self.install_args() + ['install'] + with working_dir(self.wdir()): + inspect.getmodule(self).make(*args) + + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + +class AutotoolsPackage(PackageBase): + phases = ['autoreconf', 'configure', 'build', 'install'] + + def autoreconf(self, spec, prefix): + """Not needed usually, configure should be already there""" + pass + + @PackageBase.sanity_check('autoreconf') + def is_configure_or_die(self): + if not os.path.exists('configure'): + raise RuntimeError( + 'configure script not found in {0}'.format(os.getcwd())) + + def configure_args(self): + return list() + + def configure(self, spec, prefix): + options = ['--prefix={0}'.format(prefix)] + self.configure_args() + inspect.getmodule(self).configure(*options) + + def build(self, spec, prefix): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + inspect.getmodule(self).make('install') + + # This will be used as a registration decorator in user + # packages, if need be + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + +class CMakePackage(PackageBase): + phases = ['cmake', 'build', 'install'] + + def build_type(self): + return 'RelWithDebInfo' + + def root_cmakelists_dir(self): + return self.source_directory + + @property + def std_cmake_args(self): + # standard CMake arguments + return CMakePackage._std_args(self) + + @staticmethod + def _std_args(pkg): + try: + build_type = pkg.build_type() + except AttributeError: + build_type = 'RelWithDebInfo' + + args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), + '-DCMAKE_VERBOSE_MAKEFILE=ON'] + if platform.mac_ver()[0]: + args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') + + # Set up CMake rpath + args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') + rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) + args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) + return args + + def wdir(self): + return join_path(self.stage.source_path, 'spack-build') + + def cmake_args(self): + return list() + + def cmake(self, spec, prefix): + options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ + self.cmake_args() + create = not os.path.exists(self.wdir()) + with working_dir(self.wdir(), create=create): + inspect.getmodule(self).cmake(*options) + + def build(self, spec, prefix): + with working_dir(self.wdir()): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + with working_dir(self.wdir()): + inspect.getmodule(self).make('install') + + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) @@ -1606,166 +1909,6 @@ def _hms(seconds): return ' '.join(parts) -class StagedPackage(Package): - """A Package subclass where the install() is split up into stages.""" - - def install_setup(self): - """Creates a spack_setup.py script to configure the package later.""" - raise InstallError( - "Package %s provides no install_setup() method!" % self.name) - - def install_configure(self): - """Runs the configure process.""" - raise InstallError( - "Package %s provides no install_configure() method!" % self.name) - - def install_build(self): - """Runs the build process.""" - raise InstallError( - "Package %s provides no install_build() method!" % self.name) - - def install_install(self): - """Runs the install process.""" - raise InstallError( - "Package %s provides no install_install() method!" % self.name) - - def install(self, spec, prefix): - if 'setup' in self.install_phases: - self.install_setup() - - if 'configure' in self.install_phases: - self.install_configure() - - if 'build' in self.install_phases: - self.install_build() - - if 'install' in self.install_phases: - self.install_install() - else: - # Create a dummy file so the build doesn't fail. - # That way, the module file will also be created. - with open(os.path.join(prefix, 'dummy'), 'w'): - pass - - -# stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python -def make_executable(path): - mode = os.stat(path).st_mode - mode |= (mode & 0o444) >> 2 # copy R bits to X - os.chmod(path, mode) - - -class CMakePackage(StagedPackage): - - def make_make(self): - import multiprocessing - # number of jobs spack will to build with. - jobs = multiprocessing.cpu_count() - if not self.parallel: - jobs = 1 - elif self.make_jobs: - jobs = self.make_jobs - - make = spack.build_environment.MakeExecutable('make', jobs) - return make - - def configure_args(self): - """Returns package-specific arguments to be provided to - the configure command. - """ - return list() - - def configure_env(self): - """Returns package-specific environment under which the - configure command should be run. - """ - return dict() - - def transitive_inc_path(self): - return ';'.join( - os.path.join(dep, 'include') - for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) - ) - - def install_setup(self): - cmd = [str(which('cmake'))] - cmd += spack.build_environment.get_std_cmake_args(self) - cmd += ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], - '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], - '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], - '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] - cmd += self.configure_args() - - env = { - 'PATH': os.environ['PATH'], - 'SPACK_TRANSITIVE_INCLUDE_PATH': self.transitive_inc_path(), - 'CMAKE_PREFIX_PATH': os.environ['CMAKE_PREFIX_PATH'] - } - - setup_fname = 'spconfig.py' - with open(setup_fname, 'w') as fout: - fout.write(r"""#!%s -# - -import sys -import os -import subprocess - -def cmdlist(str): - return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict(os.environ) -""" % sys.executable) - - env_vars = sorted(list(env.keys())) - for name in env_vars: - val = env[name] - if string.find(name, 'PATH') < 0: - fout.write('env[%s] = %s\n' % (repr(name), repr(val))) - else: - if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': - sep = ';' - else: - sep = ':' - - fout.write('env[%s] = "%s".join(cmdlist("""\n' - % (repr(name), sep)) - for part in string.split(val, sep): - fout.write(' %s\n' % part) - fout.write('"""))\n') - - fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = " - "env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") - fout.write('\ncmd = cmdlist("""\n') - fout.write('%s\n' % cmd[0]) - for arg in cmd[1:]: - fout.write(' %s\n' % arg) - fout.write('""") + sys.argv[1:]\n') - fout.write('\nproc = subprocess.Popen(cmd, env=env)\n') - fout.write('proc.wait()\n') - make_executable(setup_fname) - - def install_configure(self): - cmake = which('cmake') - with working_dir(self.build_directory, create=True): - env = os.environ - env.update(self.configure_env()) - env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.transitive_inc_path() - - options = self.configure_args() - options += spack.build_environment.get_std_cmake_args(self) - cmake(self.source_directory, *options) - - def install_build(self): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make() - - def install_install(self): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make('install') - - class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" |