diff options
author | Todd Gamblin <tgamblin@llnl.gov> | 2016-10-24 17:13:49 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-10-24 17:13:49 -0700 |
commit | 0f6a5cd38538e8969d11bd2167f11060b1f53b43 (patch) | |
tree | 529d9838df1417a87b2dbfb75f5c0384edbf87ef /lib | |
parent | 7dd14870ce963b5ec6672e8378abb586ca3eb13a (diff) | |
parent | c1ad4bde28a09f39dbae0f6488dc7b7182d11f93 (diff) | |
download | spack-0f6a5cd38538e8969d11bd2167f11060b1f53b43.tar.gz spack-0f6a5cd38538e8969d11bd2167f11060b1f53b43.tar.bz2 spack-0f6a5cd38538e8969d11bd2167f11060b1f53b43.tar.xz spack-0f6a5cd38538e8969d11bd2167f11060b1f53b43.zip |
Merge pull request #1186 from epfl-scitas/features/install_with_phases
do_install : allow for an arbitrary number of phases
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/llnl/util/tty/log.py | 204 | ||||
-rw-r--r-- | lib/spack/spack/__init__.py | 15 | ||||
-rw-r--r-- | lib/spack/spack/build_environment.py | 63 | ||||
-rw-r--r-- | lib/spack/spack/build_systems/__init__.py | 0 | ||||
-rw-r--r-- | lib/spack/spack/build_systems/autotools.py | 106 | ||||
-rw-r--r-- | lib/spack/spack/build_systems/cmake.py | 143 | ||||
-rw-r--r-- | lib/spack/spack/build_systems/makefile.py | 77 | ||||
-rw-r--r-- | lib/spack/spack/cmd/build.py | 43 | ||||
-rw-r--r-- | lib/spack/spack/cmd/configure.py | 90 | ||||
-rw-r--r-- | lib/spack/spack/cmd/create.py | 247 | ||||
-rw-r--r-- | lib/spack/spack/cmd/info.py | 16 | ||||
-rw-r--r-- | lib/spack/spack/cmd/install.py | 7 | ||||
-rw-r--r-- | lib/spack/spack/cmd/setup.py | 88 | ||||
-rw-r--r-- | lib/spack/spack/error.py | 13 | ||||
-rw-r--r-- | lib/spack/spack/package.py | 616 |
15 files changed, 1177 insertions, 551 deletions
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index b67edcf9cc..a4ba2a9bdf 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -24,11 +24,11 @@ ############################################################################## """Utility classes for logging the output of blocks of code. """ -import sys +import multiprocessing import os import re import select -import inspect +import sys import llnl.util.tty as tty import llnl.util.tty.color as color @@ -100,25 +100,29 @@ class keyboard_input(object): class log_output(object): - """Redirects output and error of enclosed block to a file. + """Spawns a daemon that reads from a pipe and writes to a file Usage: - with log_output(open('logfile.txt', 'w')): - # do things ... output will be logged. + # Spawns the daemon + with log_output('logfile.txt', 'w') as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged or: - with log_output(open('logfile.txt', 'w'), echo=True): - # do things ... output will be logged - # and also printed to stdout. - - Closes the provided stream when done with the block. - If echo is True, also prints the output to stdout. + with log_output('logfile.txt', echo=True) as log_redirection: + # do things ... output is not redirected + with log_redirection: + # do things ... output will be logged + # and also printed to stdout. + + Opens a stream in 'w' mode at daemon spawning and closes it at + daemon joining. If echo is True, also prints the output to stdout. """ - def __init__(self, stream, echo=False, force_color=False, debug=False): - self.stream = stream - - # various output options + def __init__(self, filename, echo=False, force_color=False, debug=False): + self.filename = filename + # Various output options self.echo = echo self.force_color = force_color self.debug = debug @@ -126,70 +130,81 @@ class log_output(object): # Default is to try file-descriptor reassignment unless the system # out/err streams do not have an associated file descriptor self.directAssignment = False - - def trace(self, frame, event, arg): - """Jumps to __exit__ on the child process.""" - raise _SkipWithBlock() + self.read, self.write = os.pipe() + + # Sets a daemon that writes to file what it reads from a pipe + self.p = multiprocessing.Process( + target=self._spawn_writing_daemon, + args=(self.read,), + name='logger_daemon' + ) + self.p.daemon = True + # Needed to un-summon the daemon + self.parent_pipe, self.child_pipe = multiprocessing.Pipe() def __enter__(self): - """Redirect output from the with block to a file. - - This forks the with block as a separate process, with stdout - and stderr redirected back to the parent via a pipe. If - echo is set, also writes to standard out. - - """ - # remember these values for later. - self._force_color = color._force_color - self._debug = tty._debug - - read, write = os.pipe() - - self.pid = os.fork() - if self.pid: - # Parent: read from child, skip the with block. - os.close(write) - - read_file = os.fdopen(read, 'r', 0) - with self.stream as log_file: - with keyboard_input(sys.stdin): - while True: - rlist, w, x = select.select( - [read_file, sys.stdin], [], []) - if not rlist: + self.p.start() + return log_output.OutputRedirection(self) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.parent_pipe.send(True) + self.p.join(60.0) # 1 minute to join the child + + def _spawn_writing_daemon(self, read): + # Parent: read from child, skip the with block. + read_file = os.fdopen(read, 'r', 0) + with open(self.filename, 'w') as log_file: + with keyboard_input(sys.stdin): + while True: + rlist, _, _ = select.select([read_file, sys.stdin], [], []) + if not rlist: + break + + # Allow user to toggle echo with 'v' key. + # Currently ignores other chars. + if sys.stdin in rlist: + if sys.stdin.read(1) == 'v': + self.echo = not self.echo + + # Handle output from the with block process. + if read_file in rlist: + line = read_file.readline() + if not line: + # For some reason we never reach this point... break - # Allow user to toggle echo with 'v' key. - # Currently ignores other chars. - if sys.stdin in rlist: - if sys.stdin.read(1) == 'v': - self.echo = not self.echo + # Echo to stdout if requested. + if self.echo: + sys.stdout.write(line) - # handle output from the with block process. - if read_file in rlist: - line = read_file.readline() - if not line: - break + # Stripped output to log file. + log_file.write(_strip(line)) + log_file.flush() - # Echo to stdout if requested. - if self.echo: - sys.stdout.write(line) + if self.child_pipe.poll(): + break - # Stripped output to log file. - log_file.write(_strip(line)) + def __del__(self): + """Closes the pipes""" + os.close(self.write) + os.close(self.read) - read_file.flush() - read_file.close() + class OutputRedirection(object): - # Set a trace function to skip the with block. - sys.settrace(lambda *args, **keys: None) - frame = inspect.currentframe(1) - frame.f_trace = self.trace + def __init__(self, other): + self.__dict__.update(other.__dict__) - else: - # Child: redirect output, execute the with block. - os.close(read) + def __enter__(self): + """Redirect output from the with block to a file. + Hijacks stdout / stderr and writes to the pipe + connected to the logger daemon + """ + # remember these values for later. + self._force_color = color._force_color + self._debug = tty._debug + # Redirect this output to a pipe + write = self.write try: # Save old stdout and stderr self._stdout = os.dup(sys.stdout.fileno()) @@ -205,53 +220,26 @@ class log_output(object): output_redirect = os.fdopen(write, 'w') sys.stdout = output_redirect sys.stderr = output_redirect - if self.force_color: color._force_color = True - if self.debug: tty._debug = True - def __exit__(self, exc_type, exception, traceback): - """Exits on child, handles skipping the with block on parent.""" - # Child should just exit here. - if self.pid == 0: + def __exit__(self, exc_type, exception, traceback): + """Plugs back the original file descriptors + for stdout and stderr + """ # Flush the log to disk. sys.stdout.flush() sys.stderr.flush() - - if exception: - # Restore stdout on the child if there's an exception, - # and let it be raised normally. - # - # This assumes that even if the exception is caught, - # the child will exit with a nonzero return code. If - # it doesn't, the child process will continue running. - # - # TODO: think about how this works outside install. - # TODO: ideally would propagate exception to parent... - if self.directAssignment: - sys.stdout = self._stdout - sys.stderr = self._stderr - else: - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) - - return False - + if self.directAssignment: + # We seem to need this only to pass test/install.py + sys.stdout = self._stdout + sys.stderr = self._stderr else: - # Die quietly if there was no exception. - os._exit(0) - - else: - # If the child exited badly, parent also should exit. - pid, returncode = os.waitpid(self.pid, 0) - if returncode != 0: - os._exit(1) - - # restore output options. - color._force_color = self._force_color - tty._debug = self._debug + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) - # Suppresses exception if it's our own. - return exc_type is _SkipWithBlock + # restore output options. + color._force_color = self._force_color + tty._debug = self._debug diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e284a58194..67c64276ee 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -186,10 +186,19 @@ sys_type = None # packages should live. This file is overloaded for spack core vs. # for packages. # -__all__ = ['Package', 'StagedPackage', 'CMakePackage', - 'Version', 'when', 'ver', 'alldeps', 'nolink'] +__all__ = ['Package', + 'CMakePackage', + 'AutotoolsPackage', + 'MakefilePackage', + 'Version', + 'when', + 'ver', + 'alldeps', + 'nolink'] from spack.package import Package, ExtensionConflictError -from spack.package import StagedPackage, CMakePackage +from spack.build_systems.makefile import MakefilePackage +from spack.build_systems.autotools import AutotoolsPackage +from spack.build_systems.cmake import CMakePackage from spack.version import Version, ver from spack.spec import DependencySpec, alldeps, nolink from spack.multimethod import when diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 792cd09eb8..7579f9adc6 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -51,16 +51,14 @@ There are two parts to the build environment: Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ +import multiprocessing import os -import sys import shutil -import multiprocessing -import platform +import sys import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -351,8 +349,8 @@ def set_module_variables_for_package(pkg, module): m.cmake = Executable('cmake') m.ctest = Executable('ctest') - # standard CMake arguments - m.std_cmake_args = get_std_cmake_args(pkg) + # Standard CMake arguments + m.std_cmake_args = spack.CMakePackage._std_args(pkg) # Put spack compiler paths in module scope. link_dir = spack.build_env_path @@ -522,41 +520,26 @@ def fork(pkg, function, dirty=False): carries on. """ - try: - pid = os.fork() - except OSError as e: - raise InstallError("Unable to fork build process: %s" % e) - - if pid == 0: - # Give the child process the package's build environment. - setup_package(pkg, dirty=dirty) - + def child_execution(child_connection): try: - # call the forked function. + setup_package(pkg, dirty=dirty) function() - - # Use os._exit here to avoid raising a SystemExit exception, - # which interferes with unit tests. - os._exit(0) - - except spack.error.SpackError as e: - e.die() - - except: - # Child doesn't raise or return to main spack code. - # Just runs default exception handler and exits. - sys.excepthook(*sys.exc_info()) - os._exit(1) - - else: - # Parent process just waits for the child to complete. If the - # child exited badly, assume it already printed an appropriate - # message. Just make the parent exit with an error code. - pid, returncode = os.waitpid(pid, 0) - if returncode != 0: - message = "Installation process had nonzero exit code : {code}" - strcode = str(returncode) - raise InstallError(message.format(code=strcode)) + child_connection.send([None, None, None]) + except Exception as e: + child_connection.send([type(e), e, None]) + finally: + child_connection.close() + + parent_connection, child_connection = multiprocessing.Pipe() + p = multiprocessing.Process( + target=child_execution, + args=(child_connection,) + ) + p.start() + exc_type, exception, traceback = parent_connection.recv() + p.join() + if exception is not None: + raise exception class InstallError(spack.error.SpackError): diff --git a/lib/spack/spack/build_systems/__init__.py b/lib/spack/spack/build_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/build_systems/__init__.py diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py new file mode 100644 index 0000000000..0bb5576708 --- /dev/null +++ b/lib/spack/spack/build_systems/autotools.py @@ -0,0 +1,106 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os.path + +import llnl.util.tty as tty +from spack.package import PackageBase + + +class AutotoolsPackage(PackageBase): + """Specialized class for packages that are built using GNU Autotools + + This class provides four phases that can be overridden: + - autoreconf + - configure + - build + - install + + They all have sensible defaults and for many packages the only thing + necessary will be to override `configure_args` + """ + phases = ['autoreconf', 'configure', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'AutotoolsPackage' + + def autoreconf(self, spec, prefix): + """Not needed usually, configure should be already there""" + pass + + @PackageBase.sanity_check('autoreconf') + def is_configure_or_die(self): + """Checks the presence of a `configure` file after the + autoreconf phase""" + if not os.path.exists('configure'): + raise RuntimeError( + 'configure script not found in {0}'.format(os.getcwd())) + + def configure_args(self): + """Method to be overridden. Should return an iterable containing + all the arguments that must be passed to configure, except --prefix + """ + return [] + + def configure(self, spec, prefix): + """Runs configure with the arguments specified in `configure_args` + and an appropriately set prefix + """ + options = ['--prefix={0}'.format(prefix)] + self.configure_args() + inspect.getmodule(self).configure(*options) + + def build(self, spec, prefix): + """The usual `make` after configure""" + inspect.getmodule(self).make() + + def install(self, spec, prefix): + """...and the final `make install` after configure""" + inspect.getmodule(self).make('install') + + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + """This function is run after build if self.run_tests == True + + It will search for a method named `check` and run it. A sensible + default is provided in the base class. + """ + try: + fn = getattr(self, 'check') + tty.msg('Trying default sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + + def check(self): + """Default test : search the Makefile for targets `test` and `check` + and run them if found. + """ + self._if_make_target_execute('test') + self._if_make_target_execute('check') + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py new file mode 100644 index 0000000000..cb1076d7b7 --- /dev/null +++ b/lib/spack/spack/build_systems/cmake.py @@ -0,0 +1,143 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os +import platform + +import llnl.util.tty as tty +import spack.build_environment +from llnl.util.filesystem import working_dir, join_path +from spack.package import PackageBase + + +class CMakePackage(PackageBase): + """Specialized class for packages that are built using cmake + + This class provides three phases that can be overridden: + - cmake + - build + - install + + They all have sensible defaults and for many packages the only thing + necessary will be to override `cmake_args` + """ + phases = ['cmake', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'CMakePackage' + + def build_type(self): + """Override to provide the correct build_type in case a complex + logic is needed + """ + return 'RelWithDebInfo' + + def root_cmakelists_dir(self): + """Directory where to find the root CMakeLists.txt""" + return self.stage.source_path + + @property + def std_cmake_args(self): + """Standard cmake arguments provided as a property for + convenience of package writers + """ + # standard CMake arguments + return CMakePackage._std_args(self) + + @staticmethod + def _std_args(pkg): + """Computes the standard cmake arguments for a generic package""" + try: + build_type = pkg.build_type() + except AttributeError: + build_type = 'RelWithDebInfo' + + args = ['-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'] + if platform.mac_ver()[0]: + args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') + + # Set up CMake rpath + args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') + rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) + args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) + return args + + def build_directory(self): + """Override to provide another place to build the package""" + return join_path(self.stage.source_path, 'spack-build') + + def cmake_args(self): + """Method to be overridden. Should return an iterable containing + all the arguments that must be passed to configure, except: + - CMAKE_INSTALL_PREFIX + - CMAKE_BUILD_TYPE + """ + return [] + + def cmake(self, spec, prefix): + """Run cmake in the build directory""" + options = [self.root_cmakelists_dir()] + self.std_cmake_args + \ + self.cmake_args() + create = not os.path.exists(self.build_directory()) + with working_dir(self.build_directory(), create=create): + inspect.getmodule(self).cmake(*options) + + def build(self, spec, prefix): + """The usual `make` after cmake""" + with working_dir(self.build_directory()): + inspect.getmodule(self).make() + + def install(self, spec, prefix): + """...and the final `make install` after cmake""" + with working_dir(self.build_directory()): + inspect.getmodule(self).make('install') + + @PackageBase.sanity_check('build') + @PackageBase.on_package_attributes(run_tests=True) + def _run_default_function(self): + """This function is run after build if self.run_tests == True + + It will search for a method named `check` and run it. A sensible + default is provided in the base class. + """ + try: + fn = getattr(self, 'check') + tty.msg('Trying default build sanity checks [check]') + fn() + except AttributeError: + tty.msg('Skipping default build sanity checks [method `check` not implemented]') # NOQA: ignore=E501 + + def check(self): + """Default test : search the Makefile for the target `test` + and run them if found. + """ + with working_dir(self.build_directory()): + self._if_make_target_execute('test') + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/build_systems/makefile.py b/lib/spack/spack/build_systems/makefile.py new file mode 100644 index 0000000000..dcddadeedc --- /dev/null +++ b/lib/spack/spack/build_systems/makefile.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect + +from llnl.util.filesystem import working_dir +from spack.package import PackageBase + + +class MakefilePackage(PackageBase): + """Specialized class for packages that are built using editable Makefiles + + This class provides three phases that can be overridden: + - edit + - build + - install + + It is necessary to override the 'edit' phase, while 'build' and 'install' + have sensible defaults. + """ + phases = ['edit', 'build', 'install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'MakefilePackage' + + def build_directory(self): + """Directory where the main Makefile is located""" + return self.stage.source_path + + def build_args(self): + """List of arguments that should be passed to make at build time""" + return [] + + def install_args(self): + """List of arguments that should be passed to make at install time""" + return [] + + def edit(self, spec, prefix): + """This phase cannot be defaulted for obvious reasons...""" + raise NotImplementedError('\'edit\' function not implemented') + + def build(self, spec, prefix): + """Default build phase : call make passing build_args""" + args = self.build_args() + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + def install(self, spec, prefix): + """Default install phase : call make passing install_args""" + args = self.install_args() + ['install'] + with working_dir(self.build_directory()): + inspect.getmodule(self).make(*args) + + # Check that self.prefix is there after installation + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/cmd/build.py b/lib/spack/spack/cmd/build.py new file mode 100644 index 0000000000..1c43acc2b3 --- /dev/null +++ b/lib/spack/spack/cmd/build.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack.cmd.configure as cfg + +from spack import * + +description = 'Stops at build stage when installing a package, if possible' + +build_system_to_phase = { + CMakePackage: 'build', + AutotoolsPackage: 'build' +} + + +def setup_parser(subparser): + cfg.setup_parser(subparser) + + +def build(parser, args): + cfg._stop_at_phase_during_install(args, build, build_system_to_phase) diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py new file mode 100644 index 0000000000..3eebe2584b --- /dev/null +++ b/lib/spack/spack/cmd/configure.py @@ -0,0 +1,90 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import argparse + +import llnl.util.tty as tty +import spack.cmd +import spack.cmd.install as inst + +from spack import * + +description = 'Stops at configuration stage when installing a package, if possible' # NOQA: ignore=E501 + + +build_system_to_phase = { + CMakePackage: 'cmake', + AutotoolsPackage: 'configure' +} + + +def setup_parser(subparser): + subparser.add_argument( + 'package', + nargs=argparse.REMAINDER, + help="spec of the package to install" + ) + subparser.add_argument( + '-v', '--verbose', + action='store_true', + help="Print additional output during builds" + ) + + +def _stop_at_phase_during_install(args, calling_fn, phase_mapping): + if not args.package: + tty.die("configure requires at least one package argument") + + # TODO: to be refactored with code in install + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) != 1: + tty.error('only one spec can be installed at a time.') + spec = specs.pop() + pkg = spec.package + try: + key = [cls for cls in phase_mapping if isinstance(pkg, cls)].pop() + phase = phase_mapping[key] + # Install package dependencies if needed + parser = argparse.ArgumentParser() + inst.setup_parser(parser) + tty.msg('Checking dependencies for {0}'.format(args.package)) + cli_args = ['-v'] if args.verbose else [] + install_args = parser.parse_args(cli_args + ['--only=dependencies']) + install_args.package = args.package + inst.install(parser, install_args) + # Install package and stop at the given phase + cli_args = ['-v'] if args.verbose else [] + install_args = parser.parse_args(cli_args + ['--only=package']) + install_args.package = args.package + inst.install(parser, install_args, stop_at=phase) + except IndexError: + tty.error( + 'Package {0} has no {1} phase, or its {1} phase is not separated from install'.format( # NOQA: ignore=E501 + spec.name, calling_fn.__name__) + ) + + +def configure(parser, args): + _stop_at_phase_during_install(args, configure, build_system_to_phase) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 741a320ea7..5db0601d44 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -22,25 +22,24 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import string +from __future__ import print_function + import os import re +import string -from ordereddict_backport import OrderedDict import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp - import spack import spack.cmd import spack.cmd.checksum import spack.url import spack.util.web -from spack.spec import Spec -from spack.util.naming import * +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import Repo, RepoError - +from spack.spec import Spec from spack.util.executable import which - +from spack.util.naming import * description = "Create a new package file from an archive URL" @@ -87,7 +86,7 @@ package_template = string.Template("""\ from spack import * -class ${class_name}(Package): +class ${class_name}(${base_class_name}): ""\"FIXME: Put a proper description of your package here.""\" # FIXME: Add a proper url for your package's homepage here. @@ -98,109 +97,160 @@ ${versions} ${dependencies} - def install(self, spec, prefix): -${install} +${body} """) -# Build dependencies and extensions -dependencies_dict = { - 'autotools': """\ + +class DefaultGuess(object): + """Provides the default values to be used for the package file template""" + base_class_name = 'Package' + + dependencies = """\ # FIXME: Add dependencies if required. - # depends_on('foo')""", + # depends_on('foo')""" - 'cmake': """\ - # FIXME: Add additional dependencies if required. - depends_on('cmake', type='build')""", + body = """\ + def install(self, spec, prefix): + # FIXME: Unknown build system + make() + make('install')""" - 'scons': """\ - # FIXME: Add additional dependencies if required. - depends_on('scons', type='build')""", + def __init__(self, name, url, version_hash_tuples): + self.name = name + self.class_name = mod_to_class(name) + self.url = url + self.version_hash_tuples = version_hash_tuples - 'bazel': """\ - # FIXME: Add additional dependencies if required. - depends_on('bazel', type='build')""", + @property + def versions(self): + """Adds a version() call to the package for each version found.""" + max_len = max(len(str(v)) for v, h in self.version_hash_tuples) + format = " version(%%-%ds, '%%s')" % (max_len + 2) + return '\n'.join( + format % ("'%s'" % v, h) for v, h in self.version_hash_tuples + ) - 'python': """\ - extends('python') - # FIXME: Add additional dependencies if required. - # depends_on('py-setuptools', type='build') - # depends_on('py-foo', type=nolink)""", +class AutotoolsGuess(DefaultGuess): + """Provides appropriate overrides for autotools-based packages""" + base_class_name = 'AutotoolsPackage' - 'R': """\ - extends('R') + dependencies = """\ + # FIXME: Add dependencies if required. + # depends_on('m4', type='build') + # depends_on('autoconf', type='build') + # depends_on('automake', type='build') + # depends_on('libtool', type='build') + # depends_on('foo')""" - # FIXME: Add additional dependencies if required. - # depends_on('r-foo', type=nolink)""", + body = """\ + def configure_args(self): + # FIXME: Add arguments other than --prefix + # FIXME: If not needed delete the function + args = [] + return args""" - 'octave': """\ - extends('octave') +class CMakeGuess(DefaultGuess): + """Provides appropriate overrides for cmake-based packages""" + base_class_name = 'CMakePackage' + + dependencies = """\ # FIXME: Add additional dependencies if required. - # depends_on('octave-foo', type=nolink)""", + depends_on('cmake', type='build')""" + + body = """\ + def cmake_args(self): + # FIXME: Add arguments other than + # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE + # FIXME: If not needed delete the function + args = [] + return args""" - 'unknown': """\ - # FIXME: Add dependencies if required. - # depends_on('foo')""" -} -# Default installation instructions -install_dict = { - 'autotools': """\ - # FIXME: Modify the configure line to suit your build system here. - configure('--prefix={0}'.format(prefix)) +class SconsGuess(DefaultGuess): + """Provides appropriate overrides for scons-based packages""" + dependencies = """\ + # FIXME: Add additional dependencies if required. + depends_on('scons', type='build')""" + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. - make() - make('install')""", + scons('prefix={0}'.format(prefix)) + scons('install')""" - 'cmake': """\ - with working_dir('spack-build', create=True): - # FIXME: Modify the cmake line to suit your build system here. - cmake('..', *std_cmake_args) - # FIXME: Add logic to build and install here. - make() - make('install')""", +class BazelGuess(DefaultGuess): + """Provides appropriate overrides for bazel-based packages""" + dependencies = """\ + # FIXME: Add additional dependencies if required. + depends_on('bazel', type='build')""" - 'scons': """\ + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. - scons('prefix={0}'.format(prefix)) - scons('install')""", + bazel()""" - 'bazel': """\ - # FIXME: Add logic to build and install here. - bazel()""", - 'python': """\ +class PythonGuess(DefaultGuess): + """Provides appropriate overrides for python extensions""" + dependencies = """\ + extends('python') + + # FIXME: Add additional dependencies if required. + # depends_on('py-setuptools', type='build') + # depends_on('py-foo', type=nolink)""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. - setup_py('install', '--prefix={0}'.format(prefix))""", + setup_py('install', '--prefix={0}'.format(prefix))""" + + def __init__(self, name, *args): + name = 'py-{0}'.format(name) + super(PythonGuess, self).__init__(name, *args) + - 'R': """\ +class RGuess(DefaultGuess): + """Provides appropriate overrides for R extensions""" + dependencies = """\ + extends('R') + + # FIXME: Add additional dependencies if required. + # depends_on('r-foo', type=nolink)""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), - self.stage.source_path)""", + self.stage.source_path)""" - 'octave': """\ + def __init__(self, name, *args): + name = 'r-{0}'.format(name) + super(RGuess, self).__init__(name, *args) + + +class OctaveGuess(DefaultGuess): + """Provides appropriate overrides for octave packages""" + dependencies = """\ + extends('octave') + + # FIXME: Add additional dependencies if required. + # depends_on('octave-foo', type=nolink)""" + + body = """\ + def install(self, spec, prefix): # FIXME: Add logic to build and install here. octave('--quiet', '--norc', '--built-in-docstrings-file=/dev/null', '--texi-macros-file=/dev/null', '--eval', 'pkg prefix {0}; pkg install {1}'.format( - prefix, self.stage.archive_file))""", + prefix, self.stage.archive_file))""" - 'unknown': """\ - # FIXME: Unknown build system - make() - make('install')""" -} - - -def make_version_calls(ver_hash_tuples): - """Adds a version() call to the package for each version found.""" - max_len = max(len(str(v)) for v, h in ver_hash_tuples) - format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) + def __init__(self, name, *args): + name = 'octave-{0}'.format(name) + super(OctaveGuess, self).__init__(name, *args) def setup_parser(subparser): @@ -227,6 +277,16 @@ def setup_parser(subparser): class BuildSystemGuesser(object): + _choices = { + 'autotools': AutotoolsGuess, + 'cmake': CMakeGuess, + 'scons': SconsGuess, + 'bazel': BazelGuess, + 'python': PythonGuess, + 'R': RGuess, + 'octave': OctaveGuess + } + def __call__(self, stage, url): """Try to guess the type of build system used by a project based on the contents of its archive or the URL it was downloaded from.""" @@ -275,6 +335,10 @@ class BuildSystemGuesser(object): self.build_system = build_system + def make_guess(self, name, url, ver_hash_tuples): + cls = self._choiches.get(self.build_system, DefaultGuess) + return cls(name, url, ver_hash_tuples) + def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -348,7 +412,7 @@ def fetch_tarballs(url, name, version): tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( ["%-10s%s" % (v, u) for v, u in versions.iteritems()])) - print + print('') archives_to_fetch = tty.get_number( "Include how many checksums in the package file?", default=5, abort='q') @@ -389,16 +453,10 @@ def create(parser, args): if not ver_hash_tuples: tty.die("Could not fetch any tarballs for %s" % name) - # Add prefix to package name if it is an extension. - if guesser.build_system == 'python': - name = 'py-{0}'.format(name) - if guesser.build_system == 'R': - name = 'r-{0}'.format(name) - if guesser.build_system == 'octave': - name = 'octave-{0}'.format(name) + guess = guesser.make_guess(name, url, ver_hash_tuples) # Create a directory for the new package. - pkg_path = repo.filename_for_package_name(name) + pkg_path = repo.filename_for_package_name(guess.name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: @@ -408,12 +466,15 @@ def create(parser, args): with open(pkg_path, "w") as pkg_file: pkg_file.write( package_template.substitute( - name=name, - class_name=mod_to_class(name), - url=url, - versions=make_version_calls(ver_hash_tuples), - dependencies=dependencies_dict[guesser.build_system], - install=install_dict[guesser.build_system])) + name=guess.name, + class_name=guess.class_name, + base_class_name=guess.base_class_name, + url=guess.url, + versions=guess.versions, + dependencies=guess.dependencies, + body=guess.body + ) + ) # If everything checks out, go ahead and edit. spack.editor(pkg_path) diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 2fa3a07525..5366ad4aa8 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -48,8 +48,11 @@ def setup_parser(subparser): def print_text_info(pkg): """Print out a plain text description of a package.""" - print "Package: ", pkg.name - print "Homepage: ", pkg.homepage + header = "{0}: ".format(pkg.build_system_class) + + print header, pkg.name + whitespaces = ''.join([' '] * (len(header) - len("Homepage: "))) + print "Homepage:", whitespaces, pkg.homepage print print "Safe versions: " @@ -84,6 +87,13 @@ def print_text_info(pkg): print " " + fmt % (name, default, desc) + print + print "Installation Phases:" + phase_str = '' + for phase in pkg.phases: + phase_str += " {0}".format(phase) + print phase_str + for deptype in ('build', 'link', 'run'): print print "%s Dependencies:" % deptype.capitalize() @@ -94,7 +104,7 @@ def print_text_info(pkg): print " None" print - print "Virtual packages: " + print "Virtual Packages: " if pkg.provided: for spec, when in pkg.provided.items(): print " %s provides %s" % (when, spec) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 8cc7f40efc..aab7c0abc7 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -22,7 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from __future__ import print_function import argparse import llnl.util.tty as tty @@ -75,7 +74,7 @@ the dependencies.""" help="Run tests during installation of a package.") -def install(parser, args): +def install(parser, args, **kwargs): if not args.package: tty.die("install requires at least one package argument") @@ -88,7 +87,7 @@ def install(parser, args): # Parse cli arguments and construct a dictionary # that will be passed to Package.do_install API - kwargs = { + kwargs.update({ 'keep_prefix': args.keep_prefix, 'keep_stage': args.keep_stage, 'install_deps': 'dependencies' in args.things_to_install, @@ -97,7 +96,7 @@ def install(parser, args): 'verbose': args.verbose, 'fake': args.fake, 'dirty': args.dirty - } + }) # Spec from cli specs = spack.cmd.parse_specs(args.package, concretize=True) diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index c393378a8d..50bc031330 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -22,16 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys -import os import argparse +import os +import string +import sys import llnl.util.tty as tty - import spack import spack.cmd +from spack import which from spack.cmd.edit import edit_package from spack.stage import DIYStage +from llnl.util.filesystem import set_executable description = "Create a configuration script and module, but don't build." @@ -51,6 +53,72 @@ def setup_parser(subparser): help="Install a package *without* cleaning the environment.") +def spack_transitive_include_path(): + return ';'.join( + os.path.join(dep, 'include') + for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) + ) + + +def write_spconfig(package): + # Set-up the environment + spack.build_environment.setup_package(package) + + cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args() + + env = dict() + + paths = os.environ['PATH'].split(':') + paths = [item for item in paths if 'spack/env' not in item] + env['PATH'] = ':'.join(paths) + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path() + env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] + env['CC'] = os.environ['SPACK_CC'] + env['CXX'] = os.environ['SPACK_CXX'] + env['FC'] = os.environ['SPACK_FC'] + + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: + fout.write( + r"""#!%s +# + +import sys +import os +import subprocess + +def cmdlist(str): + return list(x.strip().replace("'",'') for x in str.split('\n') if x) +env = dict(os.environ) +""" % sys.executable) + + env_vars = sorted(list(env.keys())) + for name in env_vars: + val = env[name] + if string.find(name, 'PATH') < 0: + fout.write('env[%s] = %s\n' % (repr(name), repr(val))) + else: + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': + sep = ';' + else: + sep = ':' + + fout.write( + 'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep)) + for part in string.split(val, sep): + fout.write(' %s\n' % part) + fout.write('"""))\n') + + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") # NOQA: ignore=E501 + fout.write('\ncmd = cmdlist("""\n') + fout.write('%s\n' % cmd[0]) + for arg in cmd[1:]: + fout.write(' %s\n' % arg) + fout.write('""") + sys.argv[1:]\n') + fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + set_executable(setup_fname) + + def setup(self, args): if not args.spec: tty.die("spack setup requires a package spec argument.") @@ -80,6 +148,12 @@ def setup(self, args): spec.concretize() package = spack.repo.get(spec) + if not isinstance(package, spack.CMakePackage): + tty.die( + 'Support for {0} derived packages not yet implemented'.format( + package.build_system_class + ) + ) # It's OK if the package is already installed. @@ -89,10 +163,4 @@ def setup(self, args): # TODO: make this an argument, not a global. spack.do_checksum = False - package.do_install( - keep_prefix=True, # Don't remove install directory - install_deps=not args.ignore_deps, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SETUP. - install_phases=set(['setup', 'provenance']), - dirty=args.dirty) + write_spconfig(package) diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index c94875e91a..5e5c1b1c7e 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -26,6 +26,7 @@ import os import sys import llnl.util.tty as tty import spack +import inspect class SpackError(Exception): @@ -49,7 +50,7 @@ class SpackError(Exception): else: tty.error(self.message) if self.long_message: - print self.long_message + print(self.long_message) os._exit(1) def __str__(self): @@ -58,6 +59,16 @@ class SpackError(Exception): msg += "\n %s" % self._long_message return msg + def __repr__(self): + args = [repr(self.message), repr(self.long_message)] + args = ','.join(args) + qualified_name = inspect.getmodule( + self).__name__ + '.' + type(self).__name__ + return qualified_name + '(' + args + ')' + + def __reduce__(self): + return type(self), (self.message, self.long_message) + class UnsupportedPlatformError(SpackError): """Raised by packages when a platform is not supported""" diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 7387fbed58..52dbd40f6f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -33,24 +33,20 @@ Homebrew makes it very easy to create packages. For a complete rundown on spack and how it differs from homebrew, look at the README. """ +import contextlib +import copy +import functools +import inspect import os -import sys import re +import sys import textwrap import time -import string -import contextlib from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty -from llnl.util.filesystem import * -from llnl.util.lang import * -from llnl.util.link_tree import LinkTree -from llnl.util.tty.log import log_output - import spack -import spack.build_environment import spack.compilers import spack.directives import spack.error @@ -60,20 +56,188 @@ import spack.mirror import spack.repository import spack.url import spack.util.web - +from llnl.util.filesystem import * +from llnl.util.lang import * +from llnl.util.link_tree import LinkTree +from llnl.util.tty.log import log_output +from spack import directory_layout from spack.stage import Stage, ResourceStage, StageComposite from spack.util.crypto import bit_length from spack.util.environment import dump_environment -from spack.util.executable import ProcessError, which +from spack.util.executable import ProcessError from spack.version import * -from spack import directory_layout - """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -class Package(object): +class InstallPhase(object): + """Manages a single phase of the installation + + This descriptor stores at creation time the name of the method it should + search for execution. The method is retrieved at __get__ time, so that + it can be overridden by subclasses of whatever class declared the phases. + + It also provides hooks to execute prerequisite and sanity checks. + """ + + def __init__(self, name): + self.name = name + self.preconditions = [] + self.sanity_checks = [] + + def __get__(self, instance, owner): + # The caller is a class that is trying to customize + # my behavior adding something + if instance is None: + return self + # If instance is there the caller wants to execute the + # install phase, thus return a properly set wrapper + phase = getattr(instance, self.name) + + @functools.wraps(phase) + def phase_wrapper(spec, prefix): + # Check instance attributes at the beginning of a phase + self._on_phase_start(instance) + # Execute phase pre-conditions, + # and give them the chance to fail + for check in self.preconditions: + # Do something sensible at some point + check(instance) + phase(spec, prefix) + # Execute phase sanity_checks, + # and give them the chance to fail + for check in self.sanity_checks: + check(instance) + # Check instance attributes at the end of a phase + self._on_phase_exit(instance) + return phase_wrapper + + def _on_phase_start(self, instance): + pass + + def _on_phase_exit(self, instance): + # If a phase has a matching last_phase attribute, + # stop the installation process raising a StopIteration + if getattr(instance, 'last_phase', None) == self.name: + raise StopIteration('Stopping at \'{0}\' phase'.format(self.name)) + + def copy(self): + try: + return copy.deepcopy(self) + except TypeError: + # This bug-fix was not back-ported in Python 2.6 + # http://bugs.python.org/issue1515 + other = InstallPhase(self.name) + other.preconditions.extend(self.preconditions) + other.sanity_checks.extend(self.sanity_checks) + return other + + +class PackageMeta(type): + """Conveniently transforms attributes to permit extensible phases + + Iterates over the attribute 'phases' and creates / updates private + InstallPhase attributes in the class that is being initialized + """ + phase_fmt = '_InstallPhase_{0}' + + _InstallPhase_sanity_checks = {} + _InstallPhase_preconditions = {} + + def __new__(meta, name, bases, attr_dict): + # Check if phases is in attr dict, then set + # install phases wrappers + if 'phases' in attr_dict: + _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] # NOQA: ignore=E501 + for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): # NOQA: ignore=E501 + attr_dict[phase_name] = InstallPhase(callback_name) + attr_dict['_InstallPhase_phases'] = _InstallPhase_phases + + def _append_checks(check_name): + # Name of the attribute I am going to check it exists + attr_name = PackageMeta.phase_fmt.format(check_name) + checks = getattr(meta, attr_name) + if checks: + for phase_name, funcs in checks.items(): + try: + # Search for the phase in the attribute dictionary + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] + except KeyError: + # If it is not there it's in the bases + # and we added a check. We need to copy + # and extend + for base in bases: + phase = getattr( + base, + PackageMeta.phase_fmt.format(phase_name), + None + ) + attr_dict[PackageMeta.phase_fmt.format( + phase_name)] = phase.copy() + phase = attr_dict[ + PackageMeta.phase_fmt.format(phase_name)] + getattr(phase, check_name).extend(funcs) + # Clear the attribute for the next class + setattr(meta, attr_name, {}) + + @classmethod + def _register_checks(cls, check_type, *args): + def _register_sanity_checks(func): + attr_name = PackageMeta.phase_fmt.format(check_type) + check_list = getattr(meta, attr_name) + for item in args: + checks = check_list.setdefault(item, []) + checks.append(func) + setattr(meta, attr_name, check_list) + return func + return _register_sanity_checks + + @staticmethod + def on_package_attributes(**attrs): + def _execute_under_condition(func): + @functools.wraps(func) + def _wrapper(instance): + # If all the attributes have the value we require, then + # execute + if all([getattr(instance, key, None) == value for key, value in attrs.items()]): # NOQA: ignore=E501 + func(instance) + return _wrapper + return _execute_under_condition + + @classmethod + def precondition(cls, *args): + return cls._register_checks('preconditions', *args) + + @classmethod + def sanity_check(cls, *args): + return cls._register_checks('sanity_checks', *args) + + if all([not hasattr(x, '_register_checks') for x in bases]): + attr_dict['_register_checks'] = _register_checks + + if all([not hasattr(x, 'sanity_check') for x in bases]): + attr_dict['sanity_check'] = sanity_check + + if all([not hasattr(x, 'precondition') for x in bases]): + attr_dict['precondition'] = precondition + + if all([not hasattr(x, 'on_package_attributes') for x in bases]): + attr_dict['on_package_attributes'] = on_package_attributes + + # Preconditions + _append_checks('preconditions') + # Sanity checks + _append_checks('sanity_checks') + return super(PackageMeta, meta).__new__(meta, name, bases, attr_dict) + + def __init__(cls, name, bases, dict): + type.__init__(cls, name, bases, dict) + spack.directives.ensure_dicts(cls) + + +class PackageBase(object): """This is the superclass for all spack packages. ***The Package class*** @@ -309,7 +473,7 @@ class Package(object): Package creators override functions like install() (all of them do this), clean() (some of them do this), and others to provide custom behavior. """ - + __metaclass__ = PackageMeta # # These are default values for instance variables. # @@ -344,12 +508,6 @@ class Package(object): """Per-process lock objects for each install prefix.""" prefix_locks = {} - class __metaclass__(type): - """Ensure attributes required by Spack directives are present.""" - def __init__(cls, name, bases, dict): - type.__init__(cls, name, bases, dict) - spack.directives.ensure_dicts(cls) - def __init__(self, spec): # this determines how the package should be built. self.spec = spec @@ -429,6 +587,8 @@ class Package(object): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() + self.extra_args = {} + def possible_dependencies(self, visited=None): """Return set of possible transitive dependencies of this package.""" if visited is None: @@ -886,12 +1046,34 @@ class Package(object): return namespace def do_fake_install(self): - """Make a fake install directory contaiing a 'fake' file in bin.""" + """Make a fake install directory containing a 'fake' file in bin.""" + # FIXME : Make this part of the 'install' behavior ? mkdirp(self.prefix.bin) touch(join_path(self.prefix.bin, 'fake')) mkdirp(self.prefix.lib) mkdirp(self.prefix.man1) + def _if_make_target_execute(self, target): + try: + # Check if we have a makefile + file = [x for x in ('Makefile', 'makefile') if os.path.exists(x)] + file = file.pop() + except IndexError: + tty.msg('No Makefile found in the build directory') + return + + # Check if 'target' is in the makefile + regex = re.compile('^' + target + ':') + with open(file, 'r') as f: + matches = [line for line in f.readlines() if regex.match(line)] + + if not matches: + tty.msg('Target \'' + target + ':\' not found in Makefile') + return + + # Execute target + inspect.getmodule(self).make(target) + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build @@ -925,13 +1107,10 @@ class Package(object): finally: self.prefix_lock.release_write() - install_phases = set(['configure', 'build', 'install', 'provenance']) - def do_install(self, keep_prefix=False, keep_stage=False, install_deps=True, - install_self=True, skip_patch=False, verbose=False, make_jobs=None, @@ -939,7 +1118,7 @@ class Package(object): fake=False, explicit=False, dirty=False, - install_phases=install_phases): + **kwargs): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -975,9 +1154,7 @@ class Package(object): # Ensure package is not already installed layout = spack.install_layout with self._prefix_read_lock(): - if ('install' in install_phases and - layout.check_installed(self.spec)): - + if layout.check_installed(self.spec): tty.msg( "%s is already installed in %s" % (self.name, self.prefix)) rec = spack.installed_db.get_record(self.spec) @@ -987,6 +1164,8 @@ class Package(object): rec.explicit = True return + self._do_install_pop_kwargs(kwargs) + tty.msg("Installing %s" % self.name) # First, install dependencies recursively. @@ -1009,7 +1188,6 @@ class Package(object): # Set parallelism before starting build. self.make_jobs = make_jobs - # ------------------- BEGIN def build_process() # Then install the package itself. def build_process(): """Forked for each build. Has its own process and python @@ -1022,112 +1200,129 @@ class Package(object): else: self.do_stage() - tty.msg("Building %s" % self.name) + tty.msg( + 'Building {0} [{1}]'.format(self.name, self.build_system_class) + ) self.stage.keep = keep_stage - self.install_phases = install_phases - self.build_directory = join_path(self.stage.path, 'spack-build') - self.source_directory = self.stage.source_path - with contextlib.nested(self.stage, self._prefix_write_lock()): - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') - - try: + try: + with contextlib.nested(self.stage, self._prefix_write_lock()): + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') # Redirect I/O to a build log (and optionally to # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), - True): - dump_environment(env_path) - self.install(self.spec, self.prefix) - - except ProcessError as e: - # Annotate ProcessErrors with the location of - # the build log - e.build_log = log_path - raise e - - # Ensure that something was actually installed. - if 'install' in self.install_phases: - self.sanity_check_prefix() - - # Copy provenance into the install directory on success - if 'provenance' in self.install_phases: - log_install_path = layout.build_log_path(self.spec) - env_install_path = layout.build_env_path(self.spec) - packages_dir = layout.build_packages_path(self.spec) - - # Remove first if we're overwriting another build - # (can happen with spack setup) - try: - # log_install_path and env_install_path are here - shutil.rmtree(packages_dir) - except: - pass - - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) - - # Run post install hooks before build stage is removed. - spack.hooks.post_install(self) - - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time - - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." % - (_hms(self._fetch_time), _hms(build_time), - _hms(self._total_time))) - print_pkg(self.prefix) - # ------------------- END def build_process() + # FIXME : refactor this assignment + self.log_path = log_path + self.env_path = env_path + dump_environment(env_path) + # Spawn a daemon that reads from a pipe and redirects + # everything to log_path + redirection_context = log_output( + log_path, verbose, + sys.stdout.isatty(), + True + ) + with redirection_context as log_redirection: + for phase_name, phase in zip(self.phases, self._InstallPhase_phases): # NOQA: ignore=E501 + tty.msg( + 'Executing phase : \'{0}\''.format(phase_name) # NOQA: ignore=E501 + ) + # Redirect stdout and stderr to daemon pipe + with log_redirection: + getattr(self, phase)( + self.spec, self.prefix) + self.log() + # Run post install hooks before build stage is removed. + spack.hooks.post_install(self) + + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time + + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." % + (_hms(self._fetch_time), _hms(build_time), + _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError as e: + # Annotate ProcessErrors with the location of + # the build log + e.build_log = log_path + raise e try: # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) - except directory_layout.InstallDirectoryAlreadyExistsError: - if 'install' in install_phases: - # Abort install if install directory exists. - # But do NOT remove it (you'd be overwriting someone's data) - tty.warn("Keeping existing install prefix in place.") - raise - else: - # We're not installing anyway, so don't worry if someone - # else has already written in the install directory - pass - - try: + # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) - except: - # remove the install prefix if anything went wrong during install. + # If we installed then we should keep the prefix + keep_prefix = True if self.last_phase is None else keep_prefix + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add( + self.spec, spack.install_layout, explicit=explicit + ) + except directory_layout.InstallDirectoryAlreadyExistsError: + # Abort install if install directory exists. + # But do NOT remove it (you'd be overwriting someone else's stuff) + tty.warn("Keeping existing install prefix in place.") + raise + except StopIteration as e: + # A StopIteration exception means that do_install + # was asked to stop early from clients + tty.msg(e.message) + tty.msg( + 'Package stage directory : {0}'.format(self.stage.source_path) + ) + finally: + # Remove the install prefix if anything went wrong during install. if not keep_prefix: self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, - wrap=False) - raise - # Parent of the build process adds the new package to - # the database, so that we don't need to re-read from file. - # NOTE: add() implicitly acquires a write-lock - spack.installed_db.add( - self.spec, spack.install_layout, explicit=explicit) + def _do_install_pop_kwargs(self, kwargs): + """Pops kwargs from do_install before starting the installation + + Args: + kwargs: + 'stop_at': last installation phase to be executed (or None) + + """ + self.last_phase = kwargs.pop('stop_at', None) + if self.last_phase is not None and self.last_phase not in self.phases: + tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501 + + def log(self): + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path( + self.spec) + env_install_path = spack.install_layout.build_env_path( + self.spec) + packages_dir = spack.install_layout.build_packages_path( + self.spec) + + # Remove first if we're overwriting another build + # (can happen with spack setup) + try: + # log_install_path and env_install_path are inside this + shutil.rmtree(packages_dir) + except Exception: + # FIXME : this potentially catches too many things... + pass + + install(self.log_path, log_install_path) + install(self.env_path, env_install_path) + dump_packages(self.spec, packages_dir) def sanity_check_prefix(self): """This function checks whether install succeeded.""" @@ -1281,13 +1476,6 @@ class Package(object): """ pass - def install(self, spec, prefix): - """ - Package implementations override this with their own configuration - """ - raise InstallError("Package %s provides no install method!" % - self.name) - def do_uninstall(self, force=False): if not self.installed: # prefix may not exist, but DB may be inconsistent. Try to fix by @@ -1498,6 +1686,16 @@ class Package(object): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +class Package(PackageBase): + phases = ['install'] + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'Package' + # This will be used as a registration decorator in user + # packages, if need be + PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix) + + def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) @@ -1598,166 +1796,6 @@ def _hms(seconds): return ' '.join(parts) -class StagedPackage(Package): - """A Package subclass where the install() is split up into stages.""" - - def install_setup(self): - """Creates a spack_setup.py script to configure the package later.""" - raise InstallError( - "Package %s provides no install_setup() method!" % self.name) - - def install_configure(self): - """Runs the configure process.""" - raise InstallError( - "Package %s provides no install_configure() method!" % self.name) - - def install_build(self): - """Runs the build process.""" - raise InstallError( - "Package %s provides no install_build() method!" % self.name) - - def install_install(self): - """Runs the install process.""" - raise InstallError( - "Package %s provides no install_install() method!" % self.name) - - def install(self, spec, prefix): - if 'setup' in self.install_phases: - self.install_setup() - - if 'configure' in self.install_phases: - self.install_configure() - - if 'build' in self.install_phases: - self.install_build() - - if 'install' in self.install_phases: - self.install_install() - else: - # Create a dummy file so the build doesn't fail. - # That way, the module file will also be created. - with open(os.path.join(prefix, 'dummy'), 'w'): - pass - - -# stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python -def make_executable(path): - mode = os.stat(path).st_mode - mode |= (mode & 0o444) >> 2 # copy R bits to X - os.chmod(path, mode) - - -class CMakePackage(StagedPackage): - - def make_make(self): - import multiprocessing - # number of jobs spack will to build with. - jobs = multiprocessing.cpu_count() - if not self.parallel: - jobs = 1 - elif self.make_jobs: - jobs = self.make_jobs - - make = spack.build_environment.MakeExecutable('make', jobs) - return make - - def configure_args(self): - """Returns package-specific arguments to be provided to - the configure command. - """ - return list() - - def configure_env(self): - """Returns package-specific environment under which the - configure command should be run. - """ - return dict() - - def transitive_inc_path(self): - return ';'.join( - os.path.join(dep, 'include') - for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) - ) - - def install_setup(self): - cmd = [str(which('cmake'))] - cmd += spack.build_environment.get_std_cmake_args(self) - cmd += ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], - '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], - '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], - '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] - cmd += self.configure_args() - - env = { - 'PATH': os.environ['PATH'], - 'SPACK_TRANSITIVE_INCLUDE_PATH': self.transitive_inc_path(), - 'CMAKE_PREFIX_PATH': os.environ['CMAKE_PREFIX_PATH'] - } - - setup_fname = 'spconfig.py' - with open(setup_fname, 'w') as fout: - fout.write(r"""#!%s -# - -import sys -import os -import subprocess - -def cmdlist(str): - return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict(os.environ) -""" % sys.executable) - - env_vars = sorted(list(env.keys())) - for name in env_vars: - val = env[name] - if string.find(name, 'PATH') < 0: - fout.write('env[%s] = %s\n' % (repr(name), repr(val))) - else: - if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': - sep = ';' - else: - sep = ':' - - fout.write('env[%s] = "%s".join(cmdlist("""\n' - % (repr(name), sep)) - for part in string.split(val, sep): - fout.write(' %s\n' % part) - fout.write('"""))\n') - - fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = " - "env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") - fout.write('\ncmd = cmdlist("""\n') - fout.write('%s\n' % cmd[0]) - for arg in cmd[1:]: - fout.write(' %s\n' % arg) - fout.write('""") + sys.argv[1:]\n') - fout.write('\nproc = subprocess.Popen(cmd, env=env)\n') - fout.write('proc.wait()\n') - make_executable(setup_fname) - - def install_configure(self): - cmake = which('cmake') - with working_dir(self.build_directory, create=True): - env = os.environ - env.update(self.configure_env()) - env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.transitive_inc_path() - - options = self.configure_args() - options += spack.build_environment.get_std_cmake_args(self) - cmake(self.source_directory, *options) - - def install_build(self): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make() - - def install_install(self): - make = self.make_make() - with working_dir(self.build_directory, create=False): - make('install') - - class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" |