diff options
18 files changed, 2539 insertions, 3 deletions
diff --git a/lib/spack/docs/tutorial.rst b/lib/spack/docs/tutorial.rst index 47dc17a407..6974fe5e1a 100644 --- a/lib/spack/docs/tutorial.rst +++ b/lib/spack/docs/tutorial.rst @@ -39,7 +39,10 @@ correspond to sections in the slides above. 1. :ref:`basics-tutorial` 2. :ref:`configs-tutorial` 3. :ref:`packaging-tutorial` - 4. :ref:`modules-tutorial` + 4. :ref:`build-systems-tutorial` + 5. :ref:`advanced-packaging-tutorial` + 6. :ref:`modules-tutorial` + 7. :ref:`modules-tutorial` Full contents: @@ -47,5 +50,6 @@ Full contents: tutorial_basics tutorial_configuration tutorial_packaging + tutorial_buildsystems tutorial_advanced_packaging tutorial_modules diff --git a/lib/spack/docs/tutorial/examples/1.package.py b/lib/spack/docs/tutorial/examples/1.package.py index 359a457567..308779d016 100644 --- a/lib/spack/docs/tutorial/examples/1.package.py +++ b/lib/spack/docs/tutorial/examples/1.package.py @@ -30,8 +30,7 @@ class Mpileaks(Package): MPI_Datatypes.""" homepage = "https://github.com/hpc/mpileaks" - url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" - + url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" # NOQA version('1.0', '8838c574b39202a57d7c2d68692718aa') # FIXME: Add dependencies if required. diff --git a/lib/spack/docs/tutorial/examples/Autotools/0.package.py b/lib/spack/docs/tutorial/examples/Autotools/0.package.py new file mode 100644 index 0000000000..44a157b36a --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Autotools/0.package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Mpileaks(AutoToolsPackage): + """Tool to detect and report leaked MPI objects like MPI_Requests and + MPI_Datatypes.""" + + homepage = "https://github.com/hpc/mpileaks" + url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" + + version('1.0', '8838c574b39202a57d7c2d68692718aa') + + depends_on("mpi") + depends_on("adept-utils") + depends_on("callpath") + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--with-adept-utils=" + spec['adept-utils'].prefix, + "--with-callpath=" + spec['callpath'].prefix) + make() + make("install") diff --git a/lib/spack/docs/tutorial/examples/Autotools/1.package.py b/lib/spack/docs/tutorial/examples/Autotools/1.package.py new file mode 100644 index 0000000000..fba2720329 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Autotools/1.package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Mpileaks(AutoToolsPackage): + """Tool to detect and report leaked MPI objects like MPI_Requests and + MPI_Datatypes.""" + + homepage = "https://github.com/hpc/mpileaks" + url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" + + version('1.0', '8838c574b39202a57d7c2d68692718aa') + + depends_on("mpi") + depends_on("adept-utils") + depends_on("callpath") + + def configure_args(self): + args = ["--with-adept-utils=" + spec['adept-utils'].prefix, + "--with-callpath=" + spec['callpath'].prefix] + return args diff --git a/lib/spack/docs/tutorial/examples/Autotools/autotools_class.py b/lib/spack/docs/tutorial/examples/Autotools/autotools_class.py new file mode 100644 index 0000000000..90ff8540bd --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Autotools/autotools_class.py @@ -0,0 +1,460 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os +import os.path +import shutil +import stat +from subprocess import PIPE +from subprocess import check_call + +import llnl.util.tty as tty +from llnl.util.filesystem import working_dir, join_path, force_remove +from spack.package import PackageBase, run_after, run_before +from spack.util.executable import Executable + + +class AutotoolsPackage(PackageBase): + """Specialized class for packages built using GNU Autotools. + + This class provides four phases that can be overridden: + + 1. :py:meth:`~.AutotoolsPackage.autoreconf` + 2. :py:meth:`~.AutotoolsPackage.configure` + 3. :py:meth:`~.AutotoolsPackage.build` + 4. :py:meth:`~.AutotoolsPackage.install` + + They all have sensible defaults and for many packages the only thing + necessary will be to override the helper method + :py:meth:`~.AutotoolsPackage.configure_args`. + For a finer tuning you may also override: + + +-----------------------------------------------+--------------------+ + | **Method** | **Purpose** | + +===============================================+====================+ + | :py:attr:`~.AutotoolsPackage.build_targets` | Specify ``make`` | + | | targets for the | + | | build phase | + +-----------------------------------------------+--------------------+ + | :py:attr:`~.AutotoolsPackage.install_targets` | Specify ``make`` | + | | targets for the | + | | install phase | + +-----------------------------------------------+--------------------+ + | :py:meth:`~.AutotoolsPackage.check` | Run build time | + | | tests if required | + +-----------------------------------------------+--------------------+ + + """ + #: Phases of a GNU Autotools package + phases = ['autoreconf', 'configure', 'build', 'install'] + #: This attribute is used in UI queries that need to know the build + #: system base class + build_system_class = 'AutotoolsPackage' + #: Whether or not to update ``config.guess`` on old architectures + patch_config_guess = True + + #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.build` + #: phase + build_targets = [] + #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.install` + #: phase + install_targets = ['install'] + + #: Callback names for build-time test + build_time_test_callbacks = ['check'] + + #: Callback names for install-time test + install_time_test_callbacks = ['installcheck'] + + #: Set to true to force the autoreconf step even if configure is present + force_autoreconf = False + #: Options to be passed to autoreconf when using the default implementation + autoreconf_extra_args = [] + + @run_after('autoreconf') + def _do_patch_config_guess(self): + """Some packages ship with an older config.guess and need to have + this updated when installed on a newer architecture. In particular, + config.guess fails for PPC64LE for version prior to a 2013-06-10 + build date (automake 1.13.4).""" + + if not self.patch_config_guess or not self.spec.satisfies( + 'target=ppc64le' + ): + return + my_config_guess = None + config_guess = None + if os.path.exists('config.guess'): + # First search the top-level source directory + my_config_guess = 'config.guess' + else: + # Then search in all sub directories. + # We would like to use AC_CONFIG_AUX_DIR, but not all packages + # ship with their configure.in or configure.ac. + d = '.' + dirs = [os.path.join(d, o) for o in os.listdir(d) + if os.path.isdir(os.path.join(d, o))] + for dirname in dirs: + path = os.path.join(dirname, 'config.guess') + if os.path.exists(path): + my_config_guess = path + + if my_config_guess is not None: + try: + check_call([my_config_guess], stdout=PIPE, stderr=PIPE) + # The package's config.guess already runs OK, so just use it + return + except Exception: + pass + else: + return + + # Look for a spack-installed automake package + if 'automake' in self.spec: + automake_path = os.path.join(self.spec['automake'].prefix, 'share', + 'automake-' + + str(self.spec['automake'].version)) + path = os.path.join(automake_path, 'config.guess') + if os.path.exists(path): + config_guess = path + # Look for the system's config.guess + if config_guess is None and os.path.exists('/usr/share'): + automake_dir = [s for s in os.listdir('/usr/share') if + "automake" in s] + if automake_dir: + automake_path = os.path.join('/usr/share', automake_dir[0]) + path = os.path.join(automake_path, 'config.guess') + if os.path.exists(path): + config_guess = path + if config_guess is not None: + try: + check_call([config_guess], stdout=PIPE, stderr=PIPE) + mod = os.stat(my_config_guess).st_mode & 0o777 | stat.S_IWUSR + os.chmod(my_config_guess, mod) + shutil.copyfile(config_guess, my_config_guess) + return + except Exception: + pass + + raise RuntimeError('Failed to find suitable config.guess') + + @property + def configure_directory(self): + """Returns the directory where 'configure' resides. + + :return: directory where to find configure + """ + return self.stage.source_path + + @property + def configure_abs_path(self): + # Absolute path to configure + configure_abs_path = join_path( + os.path.abspath(self.configure_directory), 'configure' + ) + return configure_abs_path + + @property + def build_directory(self): + """Override to provide another place to build the package""" + return self.configure_directory + + def default_flag_handler(self, spack_env, flag_val): + # Relies on being the first thing that can affect the spack_env + # EnvironmentModification after it is instantiated or no other + # method trying to affect these variables. Currently both are true + # flag_val is a tuple (flag, value_list). + spack_env.set(flag_val[0].upper(), + ' '.join(flag_val[1])) + return [] + + @run_before('autoreconf') + def delete_configure_to_force_update(self): + if self.force_autoreconf: + force_remove(self.configure_abs_path) + + def autoreconf(self, spec, prefix): + """Not needed usually, configure should be already there""" + # If configure exists nothing needs to be done + if os.path.exists(self.configure_abs_path): + return + # Else try to regenerate it + autotools = ['m4', 'autoconf', 'automake', 'libtool'] + missing = [x for x in autotools if x not in spec] + if missing: + msg = 'Cannot generate configure: missing dependencies {0}' + raise RuntimeError(msg.format(missing)) + tty.msg('Configure script not found: trying to generate it') + tty.warn('*********************************************************') + tty.warn('* If the default procedure fails, consider implementing *') + tty.warn('* a custom AUTORECONF phase in the package *') + tty.warn('*********************************************************') + with working_dir(self.configure_directory): + m = inspect.getmodule(self) + # This part should be redundant in principle, but + # won't hurt + m.libtoolize() + m.aclocal() + # This line is what is needed most of the time + # --install, --verbose, --force + autoreconf_args = ['-ivf'] + if 'pkg-config' in spec: + autoreconf_args += [ + '-I', + join_path(spec['pkg-config'].prefix, 'share', 'aclocal'), + ] + autoreconf_args += self.autoreconf_extra_args + m.autoreconf(*autoreconf_args) + + @run_after('autoreconf') + def set_configure_or_die(self): + """Checks the presence of a ``configure`` file after the + autoreconf phase. If it is found sets a module attribute + appropriately, otherwise raises an error. + + :raises RuntimeError: if a configure script is not found in + :py:meth:`~AutotoolsPackage.configure_directory` + """ + # Check if a configure script is there. If not raise a RuntimeError. + if not os.path.exists(self.configure_abs_path): + msg = 'configure script not found in {0}' + raise RuntimeError(msg.format(self.configure_directory)) + + # Monkey-patch the configure script in the corresponding module + inspect.getmodule(self).configure = Executable( + self.configure_abs_path + ) + + def configure_args(self): + """Produces a list containing all the arguments that must be passed to + configure, except ``--prefix`` which will be pre-pended to the list. + + :return: list of arguments for configure + """ + return [] + + def configure(self, spec, prefix): + """Runs configure with the arguments specified in + :py:meth:`~.AutotoolsPackage.configure_args` + and an appropriately set prefix. + """ + options = ['--prefix={0}'.format(prefix)] + self.configure_args() + + with working_dir(self.build_directory, create=True): + inspect.getmodule(self).configure(*options) + + def build(self, spec, prefix): + """Makes the build targets specified by + :py:attr:``~.AutotoolsPackage.build_targets`` + """ + with working_dir(self.build_directory): + inspect.getmodule(self).make(*self.build_targets) + + def install(self, spec, prefix): + """Makes the install targets specified by + :py:attr:``~.AutotoolsPackage.install_targets`` + """ + with working_dir(self.build_directory): + inspect.getmodule(self).make(*self.install_targets) + + run_after('build')(PackageBase._run_default_build_time_test_callbacks) + + def check(self): + """Searches the Makefile for targets ``test`` and ``check`` + and runs them if found. + """ + with working_dir(self.build_directory): + self._if_make_target_execute('test') + self._if_make_target_execute('check') + + def _activate_or_not( + self, + name, + activation_word, + deactivation_word, + activation_value=None + ): + """This function contains the current implementation details of + :py:meth:`~.AutotoolsPackage.with_or_without` and + :py:meth:`~.AutotoolsPackage.enable_or_disable`. + + Args: + name (str): name of the variant that is being processed + activation_word (str): the default activation word ('with' in the + case of ``with_or_without``) + deactivation_word (str): the default deactivation word ('without' + in the case of ``with_or_without``) + activation_value (callable): callable that accepts a single + value. This value is either one of the allowed values for a + multi-valued variant or the name of a bool-valued variant. + Returns the parameter to be used when the value is activated. + + The special value 'prefix' can also be assigned and will return + ``spec[name].prefix`` as activation parameter. + + Examples: + + Given a package with: + + .. code-block:: python + + variant('foo', values=('x', 'y'), description='') + variant('bar', default=True, description='') + + calling this function like: + + .. code-block:: python + + _activate_or_not( + 'foo', 'with', 'without', activation_value='prefix' + ) + _activate_or_not('bar', 'with', 'without') + + will generate the following configuration options: + + .. code-block:: console + + --with-x=<prefix-to-x> --without-y --with-bar + + for ``<spec-name> foo=x +bar`` + + Returns: + list of strings that corresponds to the activation/deactivation + of the variant that has been processed + + Raises: + KeyError: if name is not among known variants + """ + spec = self.spec + args = [] + + if activation_value == 'prefix': + activation_value = lambda x: spec[x].prefix + + # Defensively look that the name passed as argument is among + # variants + if name not in self.variants: + msg = '"{0}" is not a variant of "{1}"' + raise KeyError(msg.format(name, self.name)) + + # Create a list of pairs. Each pair includes a configuration + # option and whether or not that option is activated + if set(self.variants[name].values) == set((True, False)): + # BoolValuedVariant carry information about a single option. + # Nonetheless, for uniformity of treatment we'll package them + # in an iterable of one element. + condition = '+{name}'.format(name=name) + options = [(name, condition in spec)] + else: + condition = '{name}={value}' + options = [ + (value, condition.format(name=name, value=value) in spec) + for value in self.variants[name].values + ] + + # For each allowed value in the list of values + for option_value, activated in options: + # Search for an override in the package for this value + override_name = '{0}_or_{1}_{2}'.format( + activation_word, deactivation_word, option_value + ) + line_generator = getattr(self, override_name, None) + # If not available use a sensible default + if line_generator is None: + def _default_generator(is_activated): + if is_activated: + line = '--{0}-{1}'.format( + activation_word, option_value + ) + if activation_value is not None and activation_value(option_value): # NOQA=ignore=E501 + line += '={0}'.format( + activation_value(option_value) + ) + return line + return '--{0}-{1}'.format(deactivation_word, option_value) + line_generator = _default_generator + args.append(line_generator(activated)) + return args + + def with_or_without(self, name, activation_value=None): + """Inspects a variant and returns the arguments that activate + or deactivate the selected feature(s) for the configure options. + + This function works on all type of variants. For bool-valued variants + it will return by default ``--with-{name}`` or ``--without-{name}``. + For other kinds of variants it will cycle over the allowed values and + return either ``--with-{value}`` or ``--without-{value}``. + + If activation_value is given, then for each possible value of the + variant, the option ``--with-{value}=activation_value(value)`` or + ``--without-{value}`` will be added depending on whether or not + ``variant=value`` is in the spec. + + Args: + name (str): name of a valid multi-valued variant + activation_value (callable): callable that accepts a single + value and returns the parameter to be used leading to an entry + of the type ``--with-{name}={parameter}``. + + The special value 'prefix' can also be assigned and will return + ``spec[name].prefix`` as activation parameter. + + Returns: + list of arguments to configure + """ + return self._activate_or_not(name, 'with', 'without', activation_value) + + def enable_or_disable(self, name, activation_value=None): + """Same as :py:meth:`~.AutotoolsPackage.with_or_without` but substitute + ``with`` with ``enable`` and ``without`` with ``disable``. + + Args: + name (str): name of a valid multi-valued variant + activation_value (callable): if present accepts a single value + and returns the parameter to be used leading to an entry of the + type ``--enable-{name}={parameter}`` + + The special value 'prefix' can also be assigned and will return + ``spec[name].prefix`` as activation parameter. + + Returns: + list of arguments to configure + """ + return self._activate_or_not( + name, 'enable', 'disable', activation_value + ) + + run_after('install')(PackageBase._run_default_install_time_test_callbacks) + + def installcheck(self): + """Searches the Makefile for an ``installcheck`` target + and runs it if found. + """ + with working_dir(self.build_directory): + self._if_make_target_execute('installcheck') + + # Check that self.prefix is there after installation + run_after('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/docs/tutorial/examples/Cmake/0.package.py b/lib/spack/docs/tutorial/examples/Cmake/0.package.py new file mode 100644 index 0000000000..724b3fdf94 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Cmake/0.package.py @@ -0,0 +1,60 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install callpath +# +# You can edit this file again by typing: +# +# spack edit callpath +# +# See the Spack documentation for more information on packaging. +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +from spack import * + + +class Callpath(CMakePackage): + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz" + + version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + + # FIXME: Add dependencies if required. + # depends_on('foo') + + def cmake_args(self): + # FIXME: Add arguments other than + # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE + # FIXME: If not needed delete this function + args = [] + return args diff --git a/lib/spack/docs/tutorial/examples/Cmake/1.package.py b/lib/spack/docs/tutorial/examples/Cmake/1.package.py new file mode 100644 index 0000000000..dcb44260d2 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Cmake/1.package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Callpath(CMakePackage): + """Library for representing callpaths consistently in + distributed-memory performance tools.""" + + homepage = "https://github.com/llnl/callpath" + url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" + + version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + + depends_on("elf", type="link") + depends_on("libdwarf") + depends_on("dyninst") + depends_on("adept-utils") + depends_on("mpi") + depends_on("cmake@2.8:", type="build") diff --git a/lib/spack/docs/tutorial/examples/Cmake/2.package.py b/lib/spack/docs/tutorial/examples/Cmake/2.package.py new file mode 100644 index 0000000000..126a841ce9 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Cmake/2.package.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Callpath(CMakePackage): + """Library for representing callpaths consistently in + distributed-memory performance tools.""" + + homepage = "https://github.com/llnl/callpath" + url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" + + version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + + depends_on("elf", type="link") + depends_on("libdwarf") + depends_on("dyninst") + depends_on("adept-utils") + depends_on("mpi") + depends_on("cmake@2.8:", type="build") + + def cmake_args(self): + args = ["-DCALLPATH_WALKER=dyninst"] + + if self.spec.satisfies("^dyninst@9.3.0:"): + std.flag = self.compiler.cxx_flag + args.append("-DCMAKE_CXX_FLAGS='{0}' -fpermissive'".format( + std_flag)) + + return args diff --git a/lib/spack/docs/tutorial/examples/Cmake/cmake_class.py b/lib/spack/docs/tutorial/examples/Cmake/cmake_class.py new file mode 100644 index 0000000000..5b0f5526c9 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Cmake/cmake_class.py @@ -0,0 +1,224 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os +import platform + +import spack.build_environment +from llnl.util.filesystem import working_dir, join_path +from spack.util.environment import filter_system_paths +from spack.directives import depends_on, variant +from spack.package import PackageBase, InstallError, run_after + + +class CMakePackage(PackageBase): + """Specialized class for packages built using CMake + + For more information on the CMake build system, see: + https://cmake.org/cmake/help/latest/ + + This class provides three phases that can be overridden: + + 1. :py:meth:`~.CMakePackage.cmake` + 2. :py:meth:`~.CMakePackage.build` + 3. :py:meth:`~.CMakePackage.install` + + They all have sensible defaults and for many packages the only thing + necessary will be to override :py:meth:`~.CMakePackage.cmake_args`. + For a finer tuning you may also override: + + +-----------------------------------------------+--------------------+ + | **Method** | **Purpose** | + +===============================================+====================+ + | :py:meth:`~.CMakePackage.root_cmakelists_dir` | Location of the | + | | root CMakeLists.txt| + +-----------------------------------------------+--------------------+ + | :py:meth:`~.CMakePackage.build_directory` | Directory where to | + | | build the package | + +-----------------------------------------------+--------------------+ + + + """ + #: Phases of a CMake package + phases = ['cmake', 'build', 'install'] + #: This attribute is used in UI queries that need to know the build + #: system base class + build_system_class = 'CMakePackage' + + build_targets = [] + install_targets = ['install'] + + build_time_test_callbacks = ['check'] + + #: The build system generator to use. + #: + #: See ``cmake --help`` for a list of valid generators. + #: Currently, "Unix Makefiles" and "Ninja" are the only generators + #: that Spack supports. Defaults to "Unix Makefiles". + #: + #: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html + #: for more information. + generator = 'Unix Makefiles' + + # https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html + variant('build_type', default='RelWithDebInfo', + description='CMake build type', + values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel')) + + depends_on('cmake', type='build') + + @property + def root_cmakelists_dir(self): + """The relative path to the directory containing CMakeLists.txt + + This path is relative to the root of the extracted tarball, + not to the ``build_directory``. Defaults to the current directory. + + :return: directory containing CMakeLists.txt + """ + return self.stage.source_path + + @property + def std_cmake_args(self): + """Standard cmake arguments provided as a property for + convenience of package writers + + :return: standard cmake arguments + """ + # standard CMake arguments + return CMakePackage._std_args(self) + + @staticmethod + def _std_args(pkg): + """Computes the standard cmake arguments for a generic package""" + try: + generator = pkg.generator + except AttributeError: + generator = 'Unix Makefiles' + + # Make sure a valid generator was chosen + valid_generators = ['Unix Makefiles', 'Ninja'] + if generator not in valid_generators: + msg = "Invalid CMake generator: '{0}'\n".format(generator) + msg += "CMakePackage currently supports the following " + msg += "generators: '{0}'".format("', '".join(valid_generators)) + raise InstallError(msg) + + try: + build_type = pkg.spec.variants['build_type'].value + except KeyError: + build_type = 'RelWithDebInfo' + + args = [ + '-G', generator, + '-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix), + '-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type), + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' + ] + + if platform.mac_ver()[0]: + args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST') + + # Set up CMake rpath + args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE') + rpaths = ':'.join(spack.build_environment.get_rpaths(pkg)) + args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths)) + # CMake's find_package() looks in CMAKE_PREFIX_PATH first, help CMake + # to find immediate link dependencies in right places: + deps = [d.prefix for d in + pkg.spec.dependencies(deptype=('build', 'link'))] + deps = filter_system_paths(deps) + args.append('-DCMAKE_PREFIX_PATH:STRING={0}'.format(';'.join(deps))) + return args + + @property + def build_directory(self): + """Returns the directory to use when building the package + + :return: directory where to build the package + """ + return join_path(self.stage.source_path, 'spack-build') + + def default_flag_handler(self, spack_env, flag_val): + # Relies on being the first thing that can affect the spack_env + # EnvironmentModification after it is instantiated or no other + # method trying to affect these variables. Currently both are true + # flag_val is a tuple (flag, value_list) + spack_env.set(flag_val[0].upper(), + ' '.join(flag_val[1])) + return [] + + def cmake_args(self): + """Produces a list containing all the arguments that must be passed to + cmake, except: + + * CMAKE_INSTALL_PREFIX + * CMAKE_BUILD_TYPE + + which will be set automatically. + + :return: list of arguments for cmake + """ + return [] + + def cmake(self, spec, prefix): + """Runs ``cmake`` in the build directory""" + options = [os.path.abspath(self.root_cmakelists_dir)] + options += self.std_cmake_args + options += self.cmake_args() + with working_dir(self.build_directory, create=True): + inspect.getmodule(self).cmake(*options) + + def build(self, spec, prefix): + """Make the build targets""" + with working_dir(self.build_directory): + if self.generator == 'Unix Makefiles': + inspect.getmodule(self).make(*self.build_targets) + elif self.generator == 'Ninja': + inspect.getmodule(self).ninja(*self.build_targets) + + def install(self, spec, prefix): + """Make the install targets""" + with working_dir(self.build_directory): + if self.generator == 'Unix Makefiles': + inspect.getmodule(self).make(*self.install_targets) + elif self.generator == 'Ninja': + inspect.getmodule(self).ninja(*self.install_targets) + + run_after('build')(PackageBase._run_default_build_time_test_callbacks) + + def check(self): + """Searches the CMake-generated Makefile for the target ``test`` + and runs it if found. + """ + with working_dir(self.build_directory): + if self.generator == 'Unix Makefiles': + self._if_make_target_execute('test') + elif self.generator == 'Ninja': + self._if_ninja_target_execute('test') + + # Check that self.prefix is there after installation + run_after('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/docs/tutorial/examples/Makefile/0.package.py b/lib/spack/docs/tutorial/examples/Makefile/0.package.py new file mode 100644 index 0000000000..641b6ccb8f --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Makefile/0.package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Bowtie(MakefilePackage): + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip" + + version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf') + + # FIXME: Add dependencies if required. + # depends_on('foo') + + def edit(self, spec, prefix): + # FIXME: Edit the Makefile if necessary + # FIXME: If not needed delete this function + # makefile = FileFilter('Makefile') + # makefile.filter('CC = .*', 'CC = cc') + return diff --git a/lib/spack/docs/tutorial/examples/Makefile/1.package.py b/lib/spack/docs/tutorial/examples/Makefile/1.package.py new file mode 100644 index 0000000000..709bc71afa --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Makefile/1.package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Bowtie(MakefilePackage): + """Bowtie is an ultrafast, memory efficient short read aligner + for short DNA sequences (reads) from next-gen sequencers.""" + + homepage = "https://sourceforge.net/projects/bowtie-bio/" + url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip" + + version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf') + + variant("tbb", default=False, description="Use Intel thread building block") + + depends_on("tbb", when="+tbb") + + def edit(self, spec, prefix): + # FIXME: Edit the Makefile if necessary + # FIXME: If not needed delete this function + # makefile = FileFilter('Makefile') + # makefile.filter('CC = .*', 'CC = cc') + return diff --git a/lib/spack/docs/tutorial/examples/Makefile/2.package.py b/lib/spack/docs/tutorial/examples/Makefile/2.package.py new file mode 100644 index 0000000000..10aba473e3 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Makefile/2.package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Bowtie(MakefilePackage): + """Bowtie is an ultrafast, memory efficient short read aligner + for short DNA sequences (reads) from next-gen sequencers.""" + + homepage = "https://sourceforge.net/projects/bowtie-bio/" + url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip" + + version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf') + + variant("tbb", default=False, description="Use Intel thread building block") + + depends_on("tbb", when="+tbb") + + def edit(self, spec, prefix): + makefile = FileFilter("Makefile") + makefile.filter('CC= .*', 'CC = ' + env['CC']) + makefile.filter('CXX = .*', 'CXX = ' + env['CXX']) diff --git a/lib/spack/docs/tutorial/examples/Makefile/3.package.py b/lib/spack/docs/tutorial/examples/Makefile/3.package.py new file mode 100644 index 0000000000..269ab2c454 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Makefile/3.package.py @@ -0,0 +1,53 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Bowtie(MakefilePackage): + """Bowtie is an ultrafast, memory efficient short read aligner + for short DNA sequences (reads) from next-gen sequencers.""" + + homepage = "https://sourceforge.net/projects/bowtie-bio/" + url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip" + + version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf') + + variant("tbb", default=False, description="Use Intel thread building block") + + depends_on("tbb", when="+tbb") + + def edit(self, spec, prefix): + makefile = FileFilter("Makefile") + makefile.filter('CC= .*', 'CC = ' + env['CC']) + makefile.filter('CXX = .*', 'CXX = ' + env['CXX']) + + def build(self, spec, prefix): + if "+tbb" in spec: + make() + else: + make("NO_TBB=1") + + def install(self, spec, prefix): + make('prefix={0}'.format(self.prefix), 'install') diff --git a/lib/spack/docs/tutorial/examples/Makefile/makefile_class.py b/lib/spack/docs/tutorial/examples/Makefile/makefile_class.py new file mode 100644 index 0000000000..5ffb88f43d --- /dev/null +++ b/lib/spack/docs/tutorial/examples/Makefile/makefile_class.py @@ -0,0 +1,129 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect + +import llnl.util.tty as tty +from llnl.util.filesystem import working_dir +from spack.package import PackageBase, run_after + + +class MakefilePackage(PackageBase): + """Specialized class for packages that are built using editable Makefiles + + This class provides three phases that can be overridden: + + 1. :py:meth:`~.MakefilePackage.edit` + 2. :py:meth:`~.MakefilePackage.build` + 3. :py:meth:`~.MakefilePackage.install` + + It is usually necessary to override the :py:meth:`~.MakefilePackage.edit` + phase, while :py:meth:`~.MakefilePackage.build` and + :py:meth:`~.MakefilePackage.install` have sensible defaults. + For a finer tuning you may override: + + +-----------------------------------------------+--------------------+ + | **Method** | **Purpose** | + +===============================================+====================+ + | :py:attr:`~.MakefilePackage.build_targets` | Specify ``make`` | + | | targets for the | + | | build phase | + +-----------------------------------------------+--------------------+ + | :py:attr:`~.MakefilePackage.install_targets` | Specify ``make`` | + | | targets for the | + | | install phase | + +-----------------------------------------------+--------------------+ + | :py:meth:`~.MakefilePackage.build_directory` | Directory where the| + | | Makefile is located| + +-----------------------------------------------+--------------------+ + """ + #: Phases of a package that is built with an hand-written Makefile + phases = ['edit', 'build', 'install'] + #: This attribute is used in UI queries that need to know the build + #: system base class + build_system_class = 'MakefilePackage' + + #: Targets for ``make`` during the :py:meth:`~.MakefilePackage.build` + #: phase + build_targets = [] + #: Targets for ``make`` during the :py:meth:`~.MakefilePackage.install` + #: phase + install_targets = ['install'] + + #: Callback names for build-time test + build_time_test_callbacks = ['check'] + + #: Callback names for install-time test + install_time_test_callbacks = ['installcheck'] + + @property + def build_directory(self): + """Returns the directory containing the main Makefile + + :return: build directory + """ + return self.stage.source_path + + def edit(self, spec, prefix): + """Edits the Makefile before calling make. This phase cannot + be defaulted. + """ + tty.msg('Using default implementation: skipping edit phase.') + + def build(self, spec, prefix): + """Calls make, passing :py:attr:`~.MakefilePackage.build_targets` + as targets. + """ + with working_dir(self.build_directory): + inspect.getmodule(self).make(*self.build_targets) + + def install(self, spec, prefix): + """Calls make, passing :py:attr:`~.MakefilePackage.install_targets` + as targets. + """ + with working_dir(self.build_directory): + inspect.getmodule(self).make(*self.install_targets) + + run_after('build')(PackageBase._run_default_build_time_test_callbacks) + + def check(self): + """Searches the Makefile for targets ``test`` and ``check`` + and runs them if found. + """ + with working_dir(self.build_directory): + self._if_make_target_execute('test') + self._if_make_target_execute('check') + + run_after('install')(PackageBase._run_default_install_time_test_callbacks) + + def installcheck(self): + """Searches the Makefile for an ``installcheck`` target + and runs it if found. + """ + with working_dir(self.build_directory): + self._if_make_target_execute('installcheck') + + # Check that self.prefix is there after installation + run_after('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/docs/tutorial/examples/PyPackage/0.package.py b/lib/spack/docs/tutorial/examples/PyPackage/0.package.py new file mode 100644 index 0000000000..48114075a7 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/PyPackage/0.package.py @@ -0,0 +1,60 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install py-pandas +# +# You can edit this file again by typing: +# +# spack edit py-pandas +# +# See the Spack documentation for more information on packaging. +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +from spack import * + + +class PyPandas(PythonPackage): + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz" + + version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3') + + # FIXME: Add dependencies if required. + # depends_on('py-setuptools', type='build') + # depends_on('py-foo', type=('build', 'run')) + + def build_args(self, spec, prefix): + # FIXME: Add arguments other than --prefix + # FIXME: If not needed delete this function + args = [] + return args diff --git a/lib/spack/docs/tutorial/examples/PyPackage/1.package.py b/lib/spack/docs/tutorial/examples/PyPackage/1.package.py new file mode 100644 index 0000000000..a9cbff00e2 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/PyPackage/1.package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyPandas(PythonPackage): + """pandas is a Python package providing fast, flexible, and expressive + data structures designed to make working with relational or + labeled data both easy and intuitive. It aims to be the + fundamental high-level building block for doing practical, real + world data analysis in Python. Additionally, it has the broader + goal of becoming the most powerful and flexible open source data + analysis / manipulation tool available in any language. + """ + homepage = "http://pandas.pydata.org/" + url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz" + + version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3') + version('0.18.0', 'f143762cd7a59815e348adf4308d2cf6') + version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') + version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') + + depends_on('py-dateutil', type=('build', 'run')) + depends_on('py-numpy', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-cython', type='build') + depends_on('py-pytz', type=('build', 'run')) + depends_on('py-numexpr', type=('build', 'run')) + depends_on('py-bottleneck', type=('build', 'run')) diff --git a/lib/spack/docs/tutorial/examples/PyPackage/python_package_class.py b/lib/spack/docs/tutorial/examples/PyPackage/python_package_class.py new file mode 100644 index 0000000000..190620d7a2 --- /dev/null +++ b/lib/spack/docs/tutorial/examples/PyPackage/python_package_class.py @@ -0,0 +1,399 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import inspect +import os + +from spack.directives import depends_on, extends +from spack.package import PackageBase, run_after + +from llnl.util.filesystem import working_dir + + +class PythonPackage(PackageBase): + """Specialized class for packages that are built using Python + setup.py files + + This class provides the following phases that can be overridden: + + * build + * build_py + * build_ext + * build_clib + * build_scripts + * clean + * install + * install_lib + * install_headers + * install_scripts + * install_data + * sdist + * register + * bdist + * bdist_dumb + * bdist_rpm + * bdist_wininst + * upload + * check + + These are all standard setup.py commands and can be found by running: + + .. code-block:: console + + $ python setup.py --help-commands + + By default, only the 'build' and 'install' phases are run, but if you + need to run more phases, simply modify your ``phases`` list like so: + + .. code-block:: python + + phases = ['build_ext', 'install', 'bdist'] + + Each phase provides a function <phase> that runs: + + .. code-block:: console + + $ python setup.py --no-user-cfg <phase> + + Each phase also has a <phase_args> function that can pass arguments to + this call. All of these functions are empty except for the ``install_args`` + function, which passes ``--prefix=/path/to/installation/directory``. + + If you need to run a phase which is not a standard setup.py command, + you'll need to define a function for it like so: + + .. code-block:: python + + def configure(self, spec, prefix): + self.setup_py('configure') + """ + # Default phases + phases = ['build', 'install'] + + # Name of modules that the Python package provides + # This is used to test whether or not the installation succeeded + # These names generally come from running: + # + # >>> import setuptools + # >>> setuptools.find_packages() + # + # in the source tarball directory + import_modules = [] + + # To be used in UI queries that require to know which + # build-system class we are using + build_system_class = 'PythonPackage' + + #: Callback names for build-time test + build_time_test_callbacks = ['test'] + + #: Callback names for install-time test + install_time_test_callbacks = ['import_module_test'] + + extends('python') + + depends_on('python', type=('build', 'run')) + + def setup_file(self): + """Returns the name of the setup file to use.""" + return 'setup.py' + + @property + def build_directory(self): + """The directory containing the ``setup.py`` file.""" + return self.stage.source_path + + def python(self, *args, **kwargs): + inspect.getmodule(self).python(*args, **kwargs) + + def setup_py(self, *args, **kwargs): + setup = self.setup_file() + + with working_dir(self.build_directory): + self.python(setup, '--no-user-cfg', *args, **kwargs) + + def _setup_command_available(self, command): + """Determines whether or not a setup.py command exists. + + Args: + command (str): The command to look for + + Returns: + bool: True if the command is found, else False + """ + kwargs = { + 'output': os.devnull, + 'error': os.devnull, + 'fail_on_error': False + } + + python = inspect.getmodule(self).python + setup = self.setup_file() + + python(setup, '--no-user-cfg', command, '--help', **kwargs) + return python.returncode == 0 + + # The following phases and their descriptions come from: + # $ python setup.py --help-commands + + # Standard commands + + def build(self, spec, prefix): + """Build everything needed to install.""" + args = self.build_args(spec, prefix) + + self.setup_py('build', *args) + + def build_args(self, spec, prefix): + """Arguments to pass to build.""" + return [] + + def build_py(self, spec, prefix): + '''"Build" pure Python modules (copy to build directory).''' + args = self.build_py_args(spec, prefix) + + self.setup_py('build_py', *args) + + def build_py_args(self, spec, prefix): + """Arguments to pass to build_py.""" + return [] + + def build_ext(self, spec, prefix): + """Build C/C++ extensions (compile/link to build directory).""" + args = self.build_ext_args(spec, prefix) + + self.setup_py('build_ext', *args) + + def build_ext_args(self, spec, prefix): + """Arguments to pass to build_ext.""" + return [] + + def build_clib(self, spec, prefix): + """Build C/C++ libraries used by Python extensions.""" + args = self.build_clib_args(spec, prefix) + + self.setup_py('build_clib', *args) + + def build_clib_args(self, spec, prefix): + """Arguments to pass to build_clib.""" + return [] + + def build_scripts(self, spec, prefix): + '''"Build" scripts (copy and fixup #! line).''' + args = self.build_scripts_args(spec, prefix) + + self.setup_py('build_scripts', *args) + + def clean(self, spec, prefix): + """Clean up temporary files from 'build' command.""" + args = self.clean_args(spec, prefix) + + self.setup_py('clean', *args) + + def clean_args(self, spec, prefix): + """Arguments to pass to clean.""" + return [] + + def install(self, spec, prefix): + """Install everything from build directory.""" + args = self.install_args(spec, prefix) + + self.setup_py('install', *args) + + def install_args(self, spec, prefix): + """Arguments to pass to install.""" + args = ['--prefix={0}'.format(prefix)] + + # This option causes python packages (including setuptools) NOT + # to create eggs or easy-install.pth files. Instead, they + # install naturally into $prefix/pythonX.Y/site-packages. + # + # Eggs add an extra level of indirection to sys.path, slowing + # down large HPC runs. They are also deprecated in favor of + # wheels, which use a normal layout when installed. + # + # Spack manages the package directory on its own by symlinking + # extensions into the site-packages directory, so we don't really + # need the .pth files or egg directories, anyway. + if ('py-setuptools' == spec.name or # this is setuptools, or + 'py-setuptools' in spec._dependencies): # it's an immediate dep + args += ['--single-version-externally-managed', '--root=/'] + + return args + + def install_lib(self, spec, prefix): + """Install all Python modules (extensions and pure Python).""" + args = self.install_lib_args(spec, prefix) + + self.setup_py('install_lib', *args) + + def install_lib_args(self, spec, prefix): + """Arguments to pass to install_lib.""" + return [] + + def install_headers(self, spec, prefix): + """Install C/C++ header files.""" + args = self.install_headers_args(spec, prefix) + + self.setup_py('install_headers', *args) + + def install_headers_args(self, spec, prefix): + """Arguments to pass to install_headers.""" + return [] + + def install_scripts(self, spec, prefix): + """Install scripts (Python or otherwise).""" + args = self.install_scripts_args(spec, prefix) + + self.setup_py('install_scripts', *args) + + def install_scripts_args(self, spec, prefix): + """Arguments to pass to install_scripts.""" + return [] + + def install_data(self, spec, prefix): + """Install data files.""" + args = self.install_data_args(spec, prefix) + + self.setup_py('install_data', *args) + + def install_data_args(self, spec, prefix): + """Arguments to pass to install_data.""" + return [] + + def sdist(self, spec, prefix): + """Create a source distribution (tarball, zip file, etc.).""" + args = self.sdist_args(spec, prefix) + + self.setup_py('sdist', *args) + + def sdist_args(self, spec, prefix): + """Arguments to pass to sdist.""" + return [] + + def register(self, spec, prefix): + """Register the distribution with the Python package index.""" + args = self.register_args(spec, prefix) + + self.setup_py('register', *args) + + def register_args(self, spec, prefix): + """Arguments to pass to register.""" + return [] + + def bdist(self, spec, prefix): + """Create a built (binary) distribution.""" + args = self.bdist_args(spec, prefix) + + self.setup_py('bdist', *args) + + def bdist_args(self, spec, prefix): + """Arguments to pass to bdist.""" + return [] + + def bdist_dumb(self, spec, prefix): + '''Create a "dumb" built distribution.''' + args = self.bdist_dumb_args(spec, prefix) + + self.setup_py('bdist_dumb', *args) + + def bdist_dumb_args(self, spec, prefix): + """Arguments to pass to bdist_dumb.""" + return [] + + def bdist_rpm(self, spec, prefix): + """Create an RPM distribution.""" + args = self.bdist_rpm(spec, prefix) + + self.setup_py('bdist_rpm', *args) + + def bdist_rpm_args(self, spec, prefix): + """Arguments to pass to bdist_rpm.""" + return [] + + def bdist_wininst(self, spec, prefix): + """Create an executable installer for MS Windows.""" + args = self.bdist_wininst_args(spec, prefix) + + self.setup_py('bdist_wininst', *args) + + def bdist_wininst_args(self, spec, prefix): + """Arguments to pass to bdist_wininst.""" + return [] + + def upload(self, spec, prefix): + """Upload binary package to PyPI.""" + args = self.upload_args(spec, prefix) + + self.setup_py('upload', *args) + + def upload_args(self, spec, prefix): + """Arguments to pass to upload.""" + return [] + + def check(self, spec, prefix): + """Perform some checks on the package.""" + args = self.check_args(spec, prefix) + + self.setup_py('check', *args) + + def check_args(self, spec, prefix): + """Arguments to pass to check.""" + return [] + + # Testing + + def test(self): + """Run unit tests after in-place build. + + These tests are only run if the package actually has a 'test' command. + """ + if self._setup_command_available('test'): + args = self.test_args(self.spec, self.prefix) + + self.setup_py('test', *args) + + def test_args(self, spec, prefix): + """Arguments to pass to test.""" + return [] + + run_after('build')(PackageBase._run_default_build_time_test_callbacks) + + def import_module_test(self): + """Attempts to import the module that was just installed. + + This test is only run if the package overrides + :py:attr:`import_modules` with a list of module names.""" + + # Make sure we are importing the installed modules, + # not the ones in the current directory + with working_dir('..'): + for module in self.import_modules: + self.python('-c', 'import {0}'.format(module)) + + run_after('install')(PackageBase._run_default_install_time_test_callbacks) + + # Check that self.prefix is there after installation + run_after('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/docs/tutorial_buildsystems.rst b/lib/spack/docs/tutorial_buildsystems.rst new file mode 100644 index 0000000000..cdc8aeed7c --- /dev/null +++ b/lib/spack/docs/tutorial_buildsystems.rst @@ -0,0 +1,778 @@ +.. _build-systems-tutorial: + +============================== +Spack Package Build Systems +============================== + +You may begin to notice after writing a couple of package template files a +pattern emerge for some packages. For example, you may find yourself writing +an :code:`install()` method that invokes: :code:`configure`, :code:`cmake`, +:code:`make`, :code:`make install`. You may also find yourself writing +:code:`"prefix=" + prefix` as an argument to configure or cmake. Rather than +having you repeat these lines for all packages, Spack has classes that can +take care of these patterns. In addition, these package files allow for finer +grained control of these build systems. In this section, we will describe +each build system and give examples on how these can be manipulated to +install a package. + +----------------------- +Package class Hierarchy +----------------------- +.. graphviz:: + + digraph { + {Package, MakefilePackage, AutotoolsPackage, CMakePackage, PythonPackage, RPackage} -> PackageBaseClass + } + +The above diagram gives a high level view of the class hierarchy and how each +package relates. Each subclass inherits from the :code:`PackageBaseClass` +super class. The bulk of the work is done in this super class which includes +fetching, extracting to a staging directory and installing. Each subclass +then adds additional build-system-specific functionality. In the following +sections, we will go over examples of how to utilize each subclass and to see +how powerful these abstractions are when packaging. + +----------------- +Package +----------------- + +We've already seen examples of a :code:`Package` class in our walkthrough for writing +package files, so we won't be spending much time with them here. Briefly, +the Package class allows for abitrary control over the build process, whereas +subclasses rely on certain patterns (e.g. :code:`configure` :code:`make` +:code:`make install`) to be useful. :code:`Package` classes are particularly useful +for packages that have a non-conventional way of being built since the packager +can utilize some of Spack's helper functions to customize the building and +installing of a package. + +------------------- +Autotools +------------------- + +As we have seen earlier, packages using :code:`Autotools` use :code:`configure`, +:code:`make` and :code:`make install` commands to execute the build and +install process. In our :code:`Package` class, your typical build incantation will +consist of the following: + +.. code-block:: python + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +You'll see that this looks similar to what we wrote in our packaging tutorial. + +The :code:`Autotools` subclass aims to simplify writing package files and provides +convenience methods to manipulate each of the different phases for a :code:`Autotools` +build system. + +:code:`Autotools` packages consist of four phases: + +1. :code:`autoreconf()` +2. :code:`configure()` +3. :code:`build()` +4. :code:`install()` + + +Each of these phases have sensible defaults. Let's take a quick look at some +the internals of the :code:`Autotools` class: + +.. code-block:: console + + $ spack edit --build-system autotools + + +This will open the :code:`AutotoolsPackage` file in your text editor. + +.. note:: + The examples showing code for these classes is abridged to avoid having + long examples. We only show what is relevant to the packager. + + +.. literalinclude:: tutorial/examples/Autotools/autotools_class.py + :language: python + :emphasize-lines: 42,45,62 + :lines: 40-95,259-267 + :linenos: + + +Important to note are the highlighted lines. These properties allow the +packager to set what build targets and install targets they want for their +package. If, for example, we wanted to add as our build target :code:`foo` +then we can append to our :code:`build_targets` property: + +.. code-block:: python + + build_targets = ["foo"] + +Which is similiar to invoking make in our Package + +.. code-block:: python + + make("foo") + +This is useful if we have packages that ignore environment variables and need +a command-line argument. + +Another thing to take note of is in the :code:`configure()` method. +Here we see that the :code:`prefix` argument is already included since it is a +common pattern amongst packages using :code:`Autotools`. We then only have to +override :code:`configure_args()`, which will then return it's output to +to :code:`configure()`. Then, :code:`configure()` will append the common +arguments + +Let's look at the :code:`mpileaks` package.py file that we worked on earlier: + +.. code-block:: console + + $ spack edit mpileaks + +Notice that mpileaks is a :code:`Package` class but uses the :code:`Autotools` +build system. Although this package is acceptable let's make this into an +:code:`AutotoolsPackage` class and simplify it further. + +.. literalinclude:: tutorial/examples/Autotools/0.package.py + :language: python + :emphasize-lines: 28 + :linenos: + +We first inherit from the :code:`AutotoolsPackage` class. + + +Although we could keep the :code:`install()` method, most of it can be handled +by the :code:`AutotoolsPackage` base class. In fact, the only thing that needs +to be overridden is :code:`configure_args()`. + +.. literalinclude:: tutorial/examples/Autotools/1.package.py + :language: python + :emphasize-lines: 42,43 + :linenos: + +Since Spack takes care of setting the prefix for us we can exclude that as +an argument to :code:`configure`. Our packages look simpler, and the packager +does not need to worry about whether they have properly included :code:`configure` +and :code:`make`. + +This version of the :code:`mpileaks` package installs the same as the previous, +but the :code:`AutotoolsPackage` class lets us do it with a cleaner looking +package file. + +----------------- +Makefile +----------------- + +Packages that utilize :code:`GNU Make` or a :code:`Makefile` usually require you +to edit a :code:`Makefile` to set up platform and compiler specific variables. +These packages are handled by the :code:`Makefile` subclass which provides +convenience methods to help write these types of packages. + +A :code:`MakefilePackage` class has three phases that can be overridden. These include: + + 1. :code:`edit()` + 2. :code:`build()` + 3. :code:`install()` + +Packagers then have the ability to control how a :code:`Makefile` is edited, and +what targets to include for the build phase or install phase. + +Let's also take a look inside the :code:`MakefilePackage` class: + +.. code-block:: console + + $ spack edit --build-system makefile + +Take note of the following: + + +.. literalinclude:: tutorial/examples/Makefile/makefile_class.py + :language: python + :lines: 33-79,89-107 + :emphasize-lines: 48,54,61 + :linenos: + +Similar to :code:`Autotools`, :code:`MakefilePackage` class has properties +that can be set by the packager. We can also override the different +methods highlighted. + + +Let's try to recreate the Bowtie_ package: + +.. _Bowtie: http://bowtie-bio.sourceforge.net/index.shtml + + +.. code-block:: console + + $ spack create -f https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip + ==> This looks like a URL for bowtie + ==> Found 1 version of bowtie: + + 1.2.1.1 https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip + + ==> How many would you like to checksum? (default is 1, q to abort) 1 + ==> Downloading... + ==> Fetching https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip + ######################################################################## 100.0% + ==> Checksummed 1 version of bowtie + ==> This package looks like it uses the makefile build system + ==> Created template for bowtie package + ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/bowtie/package.py + +Once the fetching is completed, Spack will open up your text editor in the +usual fashion and create a template of a :code:`MakefilePackage` package.py. + +.. literalinclude:: tutorial/examples/Makefile/0.package.py + :language: python + :linenos: + +Spack was successfully able to detect that :code:`Bowtie` uses :code:`GNU Make`. +Let's add in the rest of our details for our package: + +.. literalinclude:: tutorial/examples/Makefile/1.package.py + :language: python + :emphasize-lines: 29,30,32,33,37,39 + :linenos: + +As we mentioned earlier, most packages using a :code:`Makefile` have hard-coded +variables that must be edited. These variables are fine if you happen to not +care about setup or types of compilers used but Spack is designed to work with +any compiler. The :code:`MakefilePackage` subclass makes it easy to edit +these :code:`Makefiles` by having an :code:`edit()` method that +can be overridden. + +Let's take a look at the default :code:`Makefile` that :code:`Bowtie` provides. +If we look inside, we see that :code:`CC` and :code:`CXX` point to our GNU +compiler: + +.. code-block:: console + + $ spack stage bowtie + +.. note:: + As usual make sure you have shell support activated with spack: + :code:`source /path/to/spack_root/spack/share/spack/setup-env.sh` + +.. code-block:: console + + $ spack cd -s bowtie + $ cd bowtie-1.2 + $ vim Makefile + + +.. code-block:: make + + CPP = g++ -w + CXX = $(CPP) + CC = gcc + LIBS = $(LDFLAGS) -lz + HEADERS = $(wildcard *.h) + +To fix this, we need to use the :code:`edit()` method to write our custom +:code:`Makefile`. + +.. literalinclude:: tutorial/examples/Makefile/2.package.py + :language: python + :emphasize-lines: 42,43,44 + :linenos: + +Here we use a :code:`FileFilter` object to edit our :code:`Makefile`. It takes +in a regular expression and then replaces :code:`CC` and :code:`CXX` to whatever +Spack sets :code:`CC` and :code:`CXX` environment variables to. This allows us to +build :code:`Bowtie` with whatever compiler we specify through Spack's +:code:`spec` syntax. + +Let's change the build and install phases of our package: + +.. literalinclude:: tutorial/examples/Makefile/3.package.py + :language: python + :emphasize-lines: 46, 52 + :linenos: + +Here demonstrate another strategy that we can use to manipulate our package +We can provide command-line arguments to :code:`make()`. Since :code:`Bowtie` +can use :code:`tbb` we can either add :code:`NO_TBB=1` as a argument to prevent +:code:`tbb` support or we can just invoke :code:`make` with no arguments. + +:code:`Bowtie` requires our :code:`install_target` to provide a path to +the install directory. We can do this by providing :code:`prefix=` as a command +line argument to :code:`make()`. + +Let's look at a couple of other examples and go through them: + +.. code-block:: console + + $ spack edit cbench + +Some packages allow environment variables to be set and will honor them. +Packages that use :code:`?=` for assignment in their :code:`Makefile` +can be set using environment variables. In our :code:`cbench` example we +set two environment variables in our :code:`edit()` method: + +.. code-block:: python + + def edit(self, spec, prefix): + # The location of the Cbench source tree + env['CBENCHHOME'] = self.stage.source_path + + # The location that will contain all your tests and your results + env['CBENCHTEST'] = prefix + + # ... more code + +As you may have noticed, we didn't really write anything to the :code:`Makefile` +but rather we set environment variables that will override variables set in +the :code:`Makefile`. + +Some packages include a configuration file that sets certain compiler variables, +platform specific variables, and the location of dependencies or libraries. +If the file is simple and only requires a couple of changes, we can overwrite +those entries with our :code:`FileFilter` object. If the configuration involves +complex changes, we can write a new configuration file from scratch. + +Let's look at an example of this in the :code:`elk` package: + +.. code-block:: console + + $ spack edit elk + +.. code-block:: python + + def edit(self, spec, prefix): + # Dictionary of configuration options + config = { + 'MAKE': 'make', + 'AR': 'ar' + } + + # Compiler-specific flags + flags = '' + if self.compiler.name == 'intel': + flags = '-O3 -ip -unroll -no-prec-div' + elif self.compiler.name == 'gcc': + flags = '-O3 -ffast-math -funroll-loops' + elif self.compiler.name == 'pgi': + flags = '-O3 -lpthread' + elif self.compiler.name == 'g95': + flags = '-O3 -fno-second-underscore' + elif self.compiler.name == 'nag': + flags = '-O4 -kind=byte -dusty -dcfuns' + elif self.compiler.name == 'xl': + flags = '-O3' + config['F90_OPTS'] = flags + config['F77_OPTS'] = flags + + # BLAS/LAPACK support + # Note: BLAS/LAPACK must be compiled with OpenMP support + # if the +openmp variant is chosen + blas = 'blas.a' + lapack = 'lapack.a' + if '+blas' in spec: + blas = spec['blas'].libs.joined() + if '+lapack' in spec: + lapack = spec['lapack'].libs.joined() + # lapack must come before blas + config['LIB_LPK'] = ' '.join([lapack, blas]) + + # FFT support + if '+fft' in spec: + config['LIB_FFT'] = join_path(spec['fftw'].prefix.lib, + 'libfftw3.so') + config['SRC_FFT'] = 'zfftifc_fftw.f90' + else: + config['LIB_FFT'] = 'fftlib.a' + config['SRC_FFT'] = 'zfftifc.f90' + + # MPI support + if '+mpi' in spec: + config['F90'] = spec['mpi'].mpifc + config['F77'] = spec['mpi'].mpif77 + else: + config['F90'] = spack_fc + config['F77'] = spack_f77 + config['SRC_MPI'] = 'mpi_stub.f90' + + # OpenMP support + if '+openmp' in spec: + config['F90_OPTS'] += ' ' + self.compiler.openmp_flag + config['F77_OPTS'] += ' ' + self.compiler.openmp_flag + else: + config['SRC_OMP'] = 'omp_stub.f90' + + # Libxc support + if '+libxc' in spec: + config['LIB_libxc'] = ' '.join([ + join_path(spec['libxc'].prefix.lib, 'libxcf90.so'), + join_path(spec['libxc'].prefix.lib, 'libxc.so') + ]) + config['SRC_libxc'] = ' '.join([ + 'libxc_funcs.f90', + 'libxc.f90', + 'libxcifc.f90' + ]) + else: + config['SRC_libxc'] = 'libxcifc_stub.f90' + + # Write configuration options to include file + with open('make.inc', 'w') as inc: + for key in config: + inc.write('{0} = {1}\n'.format(key, config[key])) + +:code:`config` is just a dictionary that we can add key-value pairs to. By the +end of the :code:`edit()` method we write the contents of our dictionary to +:code:`make.inc`. + +--------------- +CMake +--------------- + +CMake_ is another common build system that has been gaining popularity. It works +in a similar manner to :code:`Autotools` but with differences in variable names, +the number of configuration options available, and the handling of shared libraries. +Typical build incantations look like this: + +.. _CMake: https://cmake.org + +.. code-block:: python + + def install(self, spec, prefix): + cmake("-DCMAKE_INSTALL_PREFIX:PATH=/path/to/install_dir ..") + make() + make("install") + +As you can see from the example above, it's very similar to invoking +:code:`configure` and :code:`make` in an :code:`Autotools` build system. However, +the variable names and options differ. Most options in CMake are prefixed +with a :code:`'-D'` flag to indicate a configuration setting. + +In the :code:`CMakePackage` class we can override the following phases: + +1. :code:`cmake()` +2. :code:`build()` +3. :code:`install()` + +The :code:`CMakePackage` class also provides sensible defaults so we only need to +override :code:`cmake_args()`. + +Let's look at these defaults in the :code:`CMakePackage` class: + +.. code-block:: console + + $ spack edit --build-system cmake + + +And go into a bit of detail on the highlighted sections: + + +.. literalinclude:: tutorial/examples/Cmake/cmake_class.py + :language: python + :lines: 37-92, 94-155, 174-211 + :emphasize-lines: 57,68,86,94,96,99,100,101,102,111,117,135,136 + :linenos: + +Some :code:`CMake` packages use different generators. Spack is able to support +Unix-Makefile_ generators as well as Ninja_ generators. + +.. _Unix-Makefile: https://cmake.org/cmake/help/v3.4/generator/Unix%20Makefiles.html +.. _Ninja: https://cmake.org/cmake/help/v3.4/generator/Ninja.html + +Default generator is :code:`Unix Makefile`. + +Next we setup the build type. In :code:`CMake` you can specify the build type +that you want. Options include: + +1. empty +2. Debug +3. Release +4. RelWithDebInfo +5. MinSizeRel + +With these options you can specify whether you want your executable to have +the debug version only, release version or the release with debug information. +Release executables tend to be more optimized than Debug. In Spack, we set +the default as RelWithDebInfo unless otherwise specified through a variant. + +Spack then automatically sets up the :code:`-DCMAKE_INSTALL_PREFIX` path, +appends the build type (RelDebInfo default), and then specifies a verbose +:code:`Makefile`. + +Next we add the :code:`rpaths` to :code:`-DCMAKE_INSTALL_RPATH:STRING`. + + +Finally we add to :code:`-DCMAKE_PREFIX_PATH:STRING` the locations of all our +dependencies so that :code:`CMake` can find them. + +In the end our :code:`cmake` line will look like this (example is :code:`xrootd`): + +.. code-block:: console + + $ cmake $HOME/spack/var/spack/stage/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/xrootd-4.6.0 -G Unix Makefiles -DCMAKE_INSTALL_PREFIX:PATH=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_FIND_FRAMEWORK:STRING=LAST -DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE -DCMAKE_INSTALL_RPATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib:$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib64 -DCMAKE_PREFIX_PATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/cmake-3.9.4-hally3vnbzydiwl3skxcxcbzsscaasx5 + + +Saves a lot of typing doesn't it? + + +Let's try to recreate callpath_: + +.. _callpath: https://github.com/LLNL/callpath.git + +.. code-block:: console + + $ spack create -f https://github.com/llnl/callpath/archive/v1.0.3.tar.gz + ==> This looks like a URL for callpath + ==> Found 4 versions of callpath: + + 1.0.3 https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz + 1.0.2 https://github.com/LLNL/callpath/archive/v1.0.2.tar.gz + 1.0.1 https://github.com/LLNL/callpath/archive/v1.0.1.tar.gz + 1.0 https://github.com/LLNL/callpath/archive/v1.0.tar.gz + + ==> How many would you like to checksum? (default is 1, q to abort) 1 + ==> Downloading... + ==> Fetching https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz + ######################################################################## 100.0% + ==> Checksummed 1 version of callpath + ==> This package looks like it uses the cmake build system + ==> Created template for callpath package + ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/callpath/package.py + + +which then produces the following template: + +.. literalinclude:: tutorial/examples/Cmake/0.package.py + :language: python + :linenos: + +Again we fill in the details: + +.. literalinclude:: tutorial/examples/Cmake/1.package.py + :language: python + :linenos: + :emphasize-lines: 28,32,33,37,38,39,40,41,42 + +As mentioned earlier, Spack will use sensible defaults to prevent repeated code +and to make writing :code:`CMake` package files simpler. + +In callpath, we want to add options to :code:`CALLPATH_WALKER` as well as add +compiler flags. We add the following options like so: + +.. literalinclude:: tutorial/examples/Cmake/2.package.py + :language: python + :linenos: + :emphasize-lines: 45,49,50 + +Now we can control our build options using :code:`cmake_args()`. If defaults are +sufficient enough for the package, we can leave this method out. + +:code:`CMakePackage` classes allow for control of other features in the +build system. For example, you can specify the path to the "out of source" +build directory and also point to the root of the :code:`CMakeLists.txt` file if it +is placed in a non-standard location. + +A good example of a package that has its :code:`CMakeLists.txt` file located at a +different location is found in :code:`spades`. + +.. code-block:: console + + $ spack edit spade. + +.. code-block:: python + + root_cmakelists_dir = "src" + +Here :code:`root_cmakelists_dir` will tell Spack where to find the location +of :code:`CMakeLists.txt`. In this example, it is located a directory level below in +the :code:`src` directory. + +Some :code:`CMake` packages also require the :code:`install` phase to be +overridden. For example, let's take a look at :code:`sniffles`. + +.. code-block:: console + + $ spack edit sniffles + +In the :code:`install()` method, we have to manually install our targets +so we override the :code:`install()` method to do it for us: + +.. code-block:: python + + # the build process doesn't actually install anything, do it by hand + def install(self, spec, prefix): + mkdir(prefix.bin) + src = "bin/sniffles-core-{0}".format(spec.version.dotted) + binaries = ['sniffles', 'sniffles-debug'] + for b in binaries: + install(join_path(src, b), join_path(prefix.bin, b)) + + +-------------- +PythonPackage +-------------- + +Python extensions and modules are built differently from source than most +applications. Python uses a :code:`setup.py` script to install Python modules. +The script consists of a call to :code:`setup()` which provides the information +required to build a module to Distutils. If you're familiar with pip or +easy_install, setup.py does the same thing. + +These modules are usually installed using the following line: + +.. code-block:: console + + $ python setup.py install + +There are also a list of commands and phases that you can call. To see the full +list you can run: + +.. code-block:: console + + $ python setup.py --help-commands + Standard commands: + build build everything needed to install + build_py "build" pure Python modules (copy to build directory) + build_ext build C/C++ extensions (compile/link to build directory) + build_clib build C/C++ libraries used by Python extensions + build_scripts "build" scripts (copy and fixup #! line) + clean (no description available) + install install everything from build directory + install_lib install all Python modules (extensions and pure Python) + install_headers install C/C++ header files + install_scripts install scripts (Python or otherwise) + install_data install data files + sdist create a source distribution (tarball, zip file, etc.) + register register the distribution with the Python package index + bdist create a built (binary) distribution + bdist_dumb create a "dumb" built distribution + bdist_rpm create an RPM distribution + bdist_wininst create an executable installer for MS Windows + upload upload binary package to PyPI + check perform some checks on the package + + +To see the defaults that Spack has for each a methods, we will take a look +at the :code:`PythonPackage` class: + +.. code-block:: console + + $ spack edit --build-system python + +We see the following: + + +.. literalinclude:: tutorial/examples/PyPackage/python_package_class.py + :language: python + :lines: 35, 161-364 + :linenos: + +Each of these methods have sensible defaults or they can be overridden. + +We can write package files for Python packages using the :code:`Package` class, +but the class brings with it a lot of methods that are useless for Python packages. +Instead, Spack has a :code: `PythonPackage` subclass that allows packagers +of Python modules to be able to invoke :code:`setup.py` and use :code:`Distutils`, +which is much more familiar to a typical python user. + + +We will write a package file for Pandas_: + +.. _pandas: https://pandas.pydata.org + +.. code-block:: console + + $ spack create -f https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz + ==> This looks like a URL for pandas + ==> Warning: Spack was unable to fetch url list due to a certificate verification problem. You can try running spack -k, which will not check SSL certificates. Use this at your own risk. + ==> Found 1 version of pandas: + + 0.19.0 https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz + + ==> How many would you like to checksum? (default is 1, q to abort) 1 + ==> Downloading... + ==> Fetching https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz + ######################################################################## 100.0% + ==> Checksummed 1 version of pandas + ==> This package looks like it uses the python build system + ==> Changing package name from pandas to py-pandas + ==> Created template for py-pandas package + ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/py-pandas/package.py + +And we are left with the following template: + +.. literalinclude:: tutorial/examples/PyPackage/0.package.py + :language: python + :linenos: + +As you can see this is not any different than any package template that we have +written. We have the choice of providing build options or using the sensible +defaults + +Luckily for us, there is no need to provide build args. + +Next we need to find the dependencies of a package. Dependencies are usually +listed in :code:`setup.py`. You can find the dependencies by searching for +:code:`install_requires` keyword in that file. Here it is for :code:`Pandas`: + +.. code-block:: python + + # ... code + if sys.version_info[0] >= 3: + + setuptools_kwargs = { + 'zip_safe': False, + 'install_requires': ['python-dateutil >= 2', + 'pytz >= 2011k', + 'numpy >= %s' % min_numpy_ver], + 'setup_requires': ['numpy >= %s' % min_numpy_ver], + } + if not _have_setuptools: + sys.exit("need setuptools/distribute for Py3k" + "\n$ pip install distribute") + + # ... more code + +You can find a more comprehensive list at the Pandas documentation_. + +.. _documentation: https://pandas.pydata.org/pandas-docs/stable/install.html + + +By reading the documentation and :code:`setup.py` we found that :code:`Pandas` +depends on :code:`python-dateutil`, :code:`pytz`, and :code:`numpy`, :code:`numexpr`, +and finally :code:`bottleneck`. + +Here is the completed :code:`Pandas` script: + +.. literalinclude:: tutorial/examples/PyPackage/1.package.py + :language: python + :linenos: + +It is quite important to declare all the dependencies of a Python package. +Spack can "activate" Python packages to prevent the user from having to +load each dependency module explictly. If a dependency is missed, Spack will +be unable to properly activate the package and it will cause an issue. To +learn more about extensions go to :ref:`cmd-spack-extensions`. + +From this example, you can see that building Python modules is made easy +through the :code:`PythonPackage` class. + +------------------- +Other Build Systems +------------------- + +Although we won't get in depth with any of the other build systems that Spack +supports, it is worth mentioning that Spack does provide subclasses +for the following build systems: + +1. :code:`IntelPackage` +2. :code:`SconsPackage` +3. :code:`WafPackage` +4. :code:`RPackage` +5. :code:`PerlPackage` +6. :code:`QMake` + + +Each of these classes have their own abstractions to help assist in writing +package files. For whatever doesn't fit nicely into the other build-systems, +you can use the :code:`Package` class. + +Hopefully by now you can see how we aim to make packaging simple and +robust through these classes. If you want to learn more about these build +systems, check out :ref:`installation_procedure` in the Packaging Guide. |