From 90bb855ffa2d39d4cd861dd65f51acfa72d11ea4 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:30:38 -0500 Subject: A new subclass StagedPackage(Package) is introduced. This PR should not change the behavior for existing packages that subclass from spack.Package. If a package subclasses spack.StagedPackage instead of spack.Package, the install() phase (run inside a forked process) is now separated into sub-stages: a) spconfig: Generate a script spconfig.py that will configure the package (eg by running CMake or ./configure) This is for use if the user wishes to build externally from Spack. Therefore, the Spack compiler wrappers are NOT used here. Currently, that means that RPATH support is up to the user. b) configure: Configure the project (eg: runs configure, CMake, etc). This will configure it for use within Spack, using the Spack wrapper. c) build: eg: "make" d) install: eg: "install" If one chooses to use StagedPackage instead of Package, then one must implement each of the install sub-stages as a separate method. StagedPackage.install() then calls each of the sub-stages as appropriate. StagedPackage can be configured to only run certain sub-stages. This is done by setting the optional kwarg install_phases when calling do_install(). Setting install_phases() ONLY has an effect on StagedPackage, not on any existing packages. By default, install_phases is set to make StagedPackage run the same stages that are normally run for any package: configure, build, install (and NOT spconfig). The ability for Spack to run stages selectively for StagedPackage instances will enable new functionality. For example, explicit CMake/Autotools helpers that allow Spack to help configure a user's project without fetching, building or installing it. ------------- One implementation of StagedPackage is provided, CMakePackage. This has the following advantage for CMake-based projects over using the standard Package class: a) By separating out the phases, it enables future new functionality for packages that use it. b) It provides an implementation of intall_spconfig(), which will help users configure their CMake-based projects. CMakePackage expects users to implement configure_args() and configure_env(). These methods provide the package-specific arguments and environment needed to properly configure the package. They are placed in separated functions because they are used in both the spconfig and configure stages. TODO: 1. Generate spconfig.py scripts that are more readable. This allows the users to understand how their project is configured. 2. Provide a practical way for the user to ACCESS the spconfig stage without building the project through Spack. 3. The CMAKE_TRANSITIVE_INCLUDE_PATH stuff needs to be reworked; it should be considered provisional for now. 4. User of Autotools might wish to make a similar ConfigurePackage subclass of StagedPackage. --------------- One package using CMakePackage is introduced. See ibmisc/package.py. --- lib/spack/spack/package.py | 100 ++++++++++++++++++++- var/spack/repos/builtin/packages/ibmisc/package.py | 47 ++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 var/spack/repos/builtin/packages/ibmisc/package.py diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf896..d02a80bcad 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -66,7 +66,7 @@ import spack.fetch_strategy as fs from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension -from spack.util.executable import ProcessError +from spack.util.executable import ProcessError, which from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" @@ -826,7 +826,8 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, - skip_patch=False, verbose=False, make_jobs=None, fake=False): + skip_patch=False, verbose=False, make_jobs=None, fake=False, + install_phases = {'spconfig', 'configure', 'build', 'install'}): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -881,6 +882,10 @@ class Package(object): tty.msg("Building %s" % self.name) self.stage.keep = keep_stage + self.install_phases = install_phases + self.build_directory = join_path(self.stage.path, 'spack-build') + self.source_directory = self.stage.source_path + with self.stage: # Run the pre-install hook in the child process after # the directory is created. @@ -1291,6 +1296,97 @@ def _hms(seconds): if s: parts.append("%.2fs" % s) return ' '.join(parts) +class StagedPackage(Package): + """A Package subclass where the install() is split up into stages.""" + + def install_spconfig(self): + """Creates an spconfig.py script to configure the package later if we like.""" + raise InstallError("Package %s provides no install_spconfig() method!" % self.name) + + def install_configure(self): + """Runs the configure process.""" + raise InstallError("Package %s provides no install_configure() method!" % self.name) + + def install_build(self): + """Runs the build process.""" + raise InstallError("Package %s provides no install_build() method!" % self.name) + + def install_install(self): + """Runs the install process.""" + raise InstallError("Package %s provides no install_install() method!" % self.name) + + def install(self, spec, prefix): + if 'spconfig' in self.install_phases: + self.install_spconfig() + + if 'configure' in self.install_phases: + self.install_configure() + + if 'build' in self.install_phases: + self.install_build() + + if 'install' in self.install_phases: + self.install_install() + else: + # Create a dummy file so the build doesn't fail. + # That way, the module file will also be created. + with open(os.path.join(prefix, 'dummy'), 'w') as fout: + pass + + +class CMakePackage(StagedPackage): + + def configure_args(self): + """Returns package-specific arguments to be provided to the configure command.""" + return list() + + def configure_env(self): + """Returns package-specific environment under which the configure command should be run.""" + return dict() + + def cmake_transitive_include_path(self): + return ';'.join( + os.path.join(dep, 'include') + for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) + ) + + def install_spconfig(self): + cmd = [str(which('cmake'))] + \ + spack.build_environment.get_std_cmake_args(self) + \ + ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], + '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], + '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], + '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] + \ + self.configure_args() + + env = dict() + env['PATH'] = os.environ['PATH'] + env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] + + with open('spconfig.py', 'w') as fout: + fout.write('import sys\nimport os\nimport subprocess\n') + fout.write('env = {}\n'.format(repr(env))) + fout.write('cmd = {} + sys.argv[1:]\n'.format(repr(cmd))) + fout.write('proc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + + + def install_configure(self): + cmake = which('cmake') + with working_dir(self.build_directory, create=True): + os.environ.update(self.configure_env()) + os.environ['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) + cmake(self.source_directory, *options) + + def install_build(self): + with working_dir(self.build_directory, create=False): + make() + + def install_install(self): + with working_dir(self.build_directory, create=False): + make('install') + class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" diff --git a/var/spack/repos/builtin/packages/ibmisc/package.py b/var/spack/repos/builtin/packages/ibmisc/package.py new file mode 100644 index 0000000000..9fadee7239 --- /dev/null +++ b/var/spack/repos/builtin/packages/ibmisc/package.py @@ -0,0 +1,47 @@ +from spack import * + +class Ibmisc(CMakePackage): + """Misc. reusable utilities used by IceBin.""" + + homepage = "https://github.com/citibeth/ibmisc" + url = "https://github.com/citibeth/ibmisc/tarball/123" + + version('0.1.0', '12f2a32432a11db48e00217df18e59fa') + + variant('everytrace', default=False, description='Report errors through Everytrace') + variant('proj', default=True, description='Compile utilities for PROJ.4 library') + variant('blitz', default=True, description='Compile utilities for Blitz library') + variant('netcdf', default=True, description='Compile utilities for NetCDF library') + variant('boost', default=True, description='Compile utilities for Boost library') + variant('udunits2', default=True, description='Compile utilities for UDUNITS2 library') + variant('googletest', default=True, description='Compile utilities for Google Test library') + variant('python', default=True, description='Compile utilities for use with Python/Cython') + + extends('python') + + depends_on('eigen') + depends_on('everytrace', when='+everytrace') + depends_on('proj', when='+proj') + depends_on('blitz', when='+blitz') + depends_on('netcdf-cxx4', when='+netcdf') + depends_on('udunits2', when='+udunits2') + depends_on('googletest', when='+googletest') + depends_on('py-cython', when='+python') + depends_on('py-numpy', when='+python') + depends_on('boost', when='+boost') + + # Build dependencies + depends_on('cmake') + depends_on('doxygen') + + def configure_args(self): + spec = self.spec + return [ + '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), + '-DUSE_PROJ4=%s' % ('YES' if '+proj' in spec else 'NO'), + '-DUSE_BLITZ=%s' % ('YES' if '+blitz' in spec else 'NO'), + '-DUSE_NETCDF=%s' % ('YES' if '+netcdf' in spec else 'NO'), + '-DUSE_BOOST=%s' % ('YES' if '+boost' in spec else 'NO'), + '-DUSE_UDUNITS2=%s' % ('YES' if '+udunits2' in spec else 'NO'), + '-DUSE_GTEST=%s' % ('YES' if '+googletest' in spec else 'NO')] + -- cgit v1.2.3-70-g09d2 From 42361578237e4e96ae497107a47e277a1eeb69e9 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:13:00 -0500 Subject: (1) Added "spack spconfig" command. (2) Neatened up the spconfig.py auto-generated file. --- lib/spack/spack/cmd/spconfig.py | 97 +++++++++++++++++++++++++++++ lib/spack/spack/package.py | 133 ++++++++++++++++++++++++++++++---------- 2 files changed, 197 insertions(+), 33 deletions(-) create mode 100644 lib/spack/spack/cmd/spconfig.py diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py new file mode 100644 index 0000000000..a89e5f99e7 --- /dev/null +++ b/lib/spack/spack/cmd/spconfig.py @@ -0,0 +1,97 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Elizabeth Fischer +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +from spack.cmd.edit import edit_package +from spack.stage import DIYStage + +description = "Create a configuration script and module, but don't build." + +def setup_parser(subparser): + subparser.add_argument( + '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + help="Do not try to install dependencies of requested packages.") + subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, + help="specs to use for install. Must contain package AND verison.") + + +def spconfig(self, args): + if not args.spec: + tty.die("spack spconfig requires a package spec argument.") + + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack spconfig only takes one spec.") + + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.repo.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, spack.repo.first_repo(), None, True) + return + + print('spec', spec) + + if not spec.version.concrete: + tty.die("spack spconfig spec must have a single, concrete version.") + + spec.concretize() + package = spack.repo.get(spec) + + # It's OK if the package is already installed. + #if package.installed: + # tty.error("Already installed in %s" % package.prefix) + # tty.msg("Uninstall or try adding a version suffix for this SPCONFIG build.") + # sys.exit(1) + + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) + + # TODO: make this an argument, not a global. + spack.do_checksum = False + + package.do_install( + keep_prefix=True, # Don't remove install directory, even if you think you should + ignore_deps=args.ignore_deps, + verbose=not args.quiet, + keep_stage=True, # don't remove source dir for SPCONFIG. + install_phases = {'spconfig', 'provenance'}) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d02a80bcad..3d8e098346 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -45,6 +45,9 @@ import multiprocessing from urlparse import urlparse, urljoin import textwrap from StringIO import StringIO +import shutil +import sys +import string import llnl.util.tty as tty from llnl.util.tty.log import log_output @@ -68,6 +71,7 @@ from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError, which from spack.util.environment import dump_environment +from spack import directory_layout """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -827,7 +831,7 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False, - install_phases = {'spconfig', 'configure', 'build', 'install'}): + install_phases = {'configure', 'build', 'install', 'provenance'}): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -853,7 +857,7 @@ class Package(object): return # Ensure package is not already installed - if spack.install_layout.check_installed(self.spec): + if 'install' in install_phases and spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return @@ -895,35 +899,46 @@ class Package(object): self.do_fake_install() else: # Do the real install in the source directory. - self.stage.chdir_to_source() + self.stage.chdir_to_source() + + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + + try: + # Redirect I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) + + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Ensure that something was actually installed. + if 'install' in self.install_phases: + self._sanity_check_install() - try: - # Redirect I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) - except ProcessError as e: - # Annotate ProcessErrors with the location of the build log. - e.build_log = log_path - raise e + # Copy provenance into the install directory on success + if 'provenance' in self.install_phases: - # Ensure that something was actually installed. - self._sanity_check_install() + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Remove first if we're overwriting another build + # (can happen with spack spconfig) + try: + shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this + except: + pass - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) # Stop timer. self._total_time = time.time() - start_time @@ -937,16 +952,29 @@ class Package(object): try: # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) + except directory_layout.InstallDirectoryAlreadyExistsError: + if 'install' in install_phases: + # Abort install if install directory exists. + # But do NOT remove it (you'd be overwriting someon else's stuff) + tty.warn("Keeping existing install prefix in place.") + raise + else: + # We're not installing anyway, so don't worry if someone + # else has already written in the install directory + pass + + try: spack.build_environment.fork(self, build_process) except: # remove the install prefix if anything went wrong during install. - if not keep_prefix: - self.remove_prefix() - else: + if keep_prefix: tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed. " + "Manually remove this directory to fix:", self.prefix, wrap=True) + else: + self.remove_prefix() + raise # note: PARENT of the build process adds the new package to @@ -1333,6 +1361,12 @@ class StagedPackage(Package): with open(os.path.join(prefix, 'dummy'), 'w') as fout: pass +# stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python +def make_executable(path): + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + class CMakePackage(StagedPackage): @@ -1364,11 +1398,44 @@ class CMakePackage(StagedPackage): env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - with open('spconfig.py', 'w') as fout: - fout.write('import sys\nimport os\nimport subprocess\n') - fout.write('env = {}\n'.format(repr(env))) - fout.write('cmd = {} + sys.argv[1:]\n'.format(repr(cmd))) - fout.write('proc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + spconfig_fname = 'spconfig.py' + with open(spconfig_fname, 'w') as fout: + fout.write(\ +r"""#!{} +# + +import sys +import os +import subprocess + +def cmdlist(str): + return list(x.strip().replace("'",'') for x in str.split('\n') if x) +env = dict() +""".format(sys.executable)) + + env_vars = sorted(list(env.keys())) + for name in env_vars: + val = env[name] + if string.find(name, 'PATH') < 0: + fout.write('env[{}] = {}\n'.format(repr(name),repr(val))) + else: + if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': + sep = ';' + else: + sep = ':' + + fout.write('env[{}] = "{}".join(cmdlist("""\n'.format(repr(name),sep)) + for part in string.split(val, sep): + fout.write(' {}\n'.format(part)) + fout.write('"""))\n') + + fout.write('\ncmd = cmdlist("""\n') + fout.write('{}\n'.format(cmd[0])) + for arg in cmd[1:]: + fout.write(' {}\n'.format(arg)) + fout.write('""") + sys.argv[1:]\n') + fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + make_executable(spconfig_fname) def install_configure(self): -- cgit v1.2.3-70-g09d2 From c1a8574d8f67d6b4fdce109ecf4dd3a1cbe18488 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 00:29:40 -0500 Subject: Fixed for Python 2.6 --- lib/spack/spack/package.py | 2 +- var/spack/repos/builtin/packages/ibmisc/package.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3d8e098346..f815628bba 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -831,7 +831,7 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False, - install_phases = {'configure', 'build', 'install', 'provenance'}): + install_phases = set(['configure', 'build', 'install', 'provenance'])): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe diff --git a/var/spack/repos/builtin/packages/ibmisc/package.py b/var/spack/repos/builtin/packages/ibmisc/package.py index 9fadee7239..a3bd680655 100644 --- a/var/spack/repos/builtin/packages/ibmisc/package.py +++ b/var/spack/repos/builtin/packages/ibmisc/package.py @@ -1,4 +1,5 @@ from spack import * +import llnl.util.tty as tty class Ibmisc(CMakePackage): """Misc. reusable utilities used by IceBin.""" @@ -15,7 +16,7 @@ class Ibmisc(CMakePackage): variant('boost', default=True, description='Compile utilities for Boost library') variant('udunits2', default=True, description='Compile utilities for UDUNITS2 library') variant('googletest', default=True, description='Compile utilities for Google Test library') - variant('python', default=True, description='Compile utilities for use with Python/Cython') + variant('python', default=True, description='Compile utilities fro use with Python/Cython') extends('python') @@ -26,16 +27,18 @@ class Ibmisc(CMakePackage): depends_on('netcdf-cxx4', when='+netcdf') depends_on('udunits2', when='+udunits2') depends_on('googletest', when='+googletest') +# depends_on('python', when='+python') depends_on('py-cython', when='+python') depends_on('py-numpy', when='+python') depends_on('boost', when='+boost') + + # Build dependencies depends_on('cmake') depends_on('doxygen') - def configure_args(self): - spec = self.spec + def config_args(self, spec, prefix): return [ '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), '-DUSE_PROJ4=%s' % ('YES' if '+proj' in spec else 'NO'), -- cgit v1.2.3-70-g09d2 From 003957a689b8d3d6e6da5408d61c45e73e01b575 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:33:13 -0500 Subject: Reverted bad change --- var/spack/repos/builtin/packages/ibmisc/package.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/ibmisc/package.py b/var/spack/repos/builtin/packages/ibmisc/package.py index a3bd680655..8e6cf429a7 100644 --- a/var/spack/repos/builtin/packages/ibmisc/package.py +++ b/var/spack/repos/builtin/packages/ibmisc/package.py @@ -1,5 +1,4 @@ from spack import * -import llnl.util.tty as tty class Ibmisc(CMakePackage): """Misc. reusable utilities used by IceBin.""" @@ -16,7 +15,7 @@ class Ibmisc(CMakePackage): variant('boost', default=True, description='Compile utilities for Boost library') variant('udunits2', default=True, description='Compile utilities for UDUNITS2 library') variant('googletest', default=True, description='Compile utilities for Google Test library') - variant('python', default=True, description='Compile utilities fro use with Python/Cython') + variant('python', default=True, description='Compile utilities for use with Python/Cython') extends('python') @@ -27,18 +26,16 @@ class Ibmisc(CMakePackage): depends_on('netcdf-cxx4', when='+netcdf') depends_on('udunits2', when='+udunits2') depends_on('googletest', when='+googletest') -# depends_on('python', when='+python') depends_on('py-cython', when='+python') depends_on('py-numpy', when='+python') depends_on('boost', when='+boost') - - # Build dependencies depends_on('cmake') depends_on('doxygen') - def config_args(self, spec, prefix): + def configure_args(self): + spec = self.spec return [ '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), '-DUSE_PROJ4=%s' % ('YES' if '+proj' in spec else 'NO'), @@ -47,4 +44,3 @@ class Ibmisc(CMakePackage): '-DUSE_BOOST=%s' % ('YES' if '+boost' in spec else 'NO'), '-DUSE_UDUNITS2=%s' % ('YES' if '+udunits2' in spec else 'NO'), '-DUSE_GTEST=%s' % ('YES' if '+googletest' in spec else 'NO')] - -- cgit v1.2.3-70-g09d2 From 9885f1a19ec5c81121e20cd201007c89cc7b1b1f Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:34:46 -0500 Subject: Fix for Python 2.6 --- lib/spack/spack/cmd/spconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index a89e5f99e7..6035a61841 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -94,4 +94,4 @@ def spconfig(self, args): ignore_deps=args.ignore_deps, verbose=not args.quiet, keep_stage=True, # don't remove source dir for SPCONFIG. - install_phases = {'spconfig', 'provenance'}) + install_phases = set(['spconfig', 'provenance'])) -- cgit v1.2.3-70-g09d2 From 857f791286a041d527481a35931cb0249624a53a Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:40:50 -0500 Subject: Add missing import. --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 3051d3f742..920e22d672 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -173,7 +173,7 @@ sys_type = None # should live. This file is overloaded for spack core vs. for packages. # __all__ = ['Package', 'Version', 'when', 'ver'] -from spack.package import Package, ExtensionConflictError +from spack.package import Package, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 4c9a52044a89ef8133ab2ac1759da9d4caefa3eb Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 15:18:24 -0400 Subject: Fixed CMakePackage import --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 920e22d672..c31687cafc 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -172,7 +172,7 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'Version', 'when', 'ver'] +__all__ = ['Package', 'CMakePackage', 'Version', 'when', 'ver'] from spack.package import Package, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 030e8dd1ac7c23cfdd9bf7983caec6f8ca41a556 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 18:04:23 -0400 Subject: Removed Python 2.7-style string formatting --- lib/spack/spack/package.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index f815628bba..10f0c4e761 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1401,7 +1401,7 @@ class CMakePackage(StagedPackage): spconfig_fname = 'spconfig.py' with open(spconfig_fname, 'w') as fout: fout.write(\ -r"""#!{} +r"""#!%s # import sys @@ -1411,28 +1411,28 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) env = dict() -""".format(sys.executable)) +""" % sys.executable) env_vars = sorted(list(env.keys())) for name in env_vars: val = env[name] if string.find(name, 'PATH') < 0: - fout.write('env[{}] = {}\n'.format(repr(name),repr(val))) + fout.write('env[%s] = %s\n'. % (repr(name),repr(val))) else: if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': sep = ';' else: sep = ':' - fout.write('env[{}] = "{}".join(cmdlist("""\n'.format(repr(name),sep)) + fout.write('env[%s] = "%s".join(cmdlist("""\n' % (repr(name),sep)) for part in string.split(val, sep): - fout.write(' {}\n'.format(part)) + fout.write(' %s\n' % part) fout.write('"""))\n') fout.write('\ncmd = cmdlist("""\n') - fout.write('{}\n'.format(cmd[0])) + fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: - fout.write(' {}\n'.format(arg)) + fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') make_executable(spconfig_fname) -- cgit v1.2.3-70-g09d2 From 0426796d9f6e064f062f4c3b0d27098dfe685d77 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 18:14:38 -0400 Subject: Fixed typo --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 10f0c4e761..18c4afcf95 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1417,7 +1417,7 @@ env = dict() for name in env_vars: val = env[name] if string.find(name, 'PATH') < 0: - fout.write('env[%s] = %s\n'. % (repr(name),repr(val))) + fout.write('env[%s] = %s\n' % (repr(name),repr(val))) else: if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': sep = ';' -- cgit v1.2.3-70-g09d2 From 6f26c45143e63ee339f9aabc654684056b60654f Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:38:19 -0400 Subject: Fixed typo bug. Made error comment more explicit --- lib/spack/spack/cmd/spconfig.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index 6035a61841..70db48e172 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -69,10 +69,8 @@ def spconfig(self, args): edit_package(spec.name, spack.repo.first_repo(), None, True) return - print('spec', spec) - - if not spec.version.concrete: - tty.die("spack spconfig spec must have a single, concrete version.") + if not spec.versions.concrete: + tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) -- cgit v1.2.3-70-g09d2 From 8f675b0ee8d546a6b7908a26db189eb6583e1d98 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 25 Mar 2016 17:12:16 -0400 Subject: Made StagedPackage user-visible. --- lib/spack/spack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 2cb189b4a2..fce7481b5b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -172,8 +172,8 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'CMakePackage', 'Version', 'when', 'ver'] -from spack.package import Package, CMakePackage, ExtensionConflictError +__all__ = ['Package', 'StagedPackage', 'CMakePackage', 'Version', 'when', 'ver'] +from spack.package import Package, StagedPackage, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 9889b1a5febc2aedec6401ae701b55961cf0d559 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 25 Mar 2016 17:12:32 -0400 Subject: Added documentation for StagedPackage, etc. --- lib/spack/docs/packaging_guide.rst | 101 +++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 519c0da232..bc2834c713 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2707,3 +2707,104 @@ might write: DWARF_PREFIX = $(spack location -i libdwarf) CXXFLAGS += -I$DWARF_PREFIX/include CXXFLAGS += -L$DWARF_PREFIX/lib + +Build System Configuration Support +---------------------------------- + +Imagine a developer creating a CMake-based (or Autotools) project in a local +directory, which depends on libraries A-Z. Once Spack has installed +those dependencies, one would like to run ``cmake`` with appropriate +command line and environment so CMake can find them. The ``spack +spconfig`` command does this conveniently, producing a CMake +configuration that is essentially the same as how Spack *would have* +configured the project. This can be demonstrated with a usage +example: + +.. code-block:: bash + + > cd myproject + > spack spconfig myproject@local + > mkdir build; cd build + > ../spconfig.py .. + > make + > make install + +Notes: + * Spack must have ``myproject/package.py`` in its repository for + this to work. + * ``spack spconfig`` produces the executable script ``spconfig.py`` + in the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the top level of + the source tree. + + * The version number given to ``spack spconfig`` is arbitrary (just + like ``spack diy``). ``myproject/package.py`` does not need to + have any valid downloadable versions listed (typical when a + project is new). + * spconfig.py produces a CMake configuration that *does not* use the + Spack wrappers. Any resulting binaries *will not* use RPATH, + unless the user has enabled it. This is recommended for + development purposes, not production. + * spconfig.py is easily legible, and can serve as a developer + reference of what dependencies are being used. + * ``make install`` installs the package into the Spack repository, + where it may be used by other Spack packages. + +CMakePackage +~~~~~~~~~~~~ + +In order ot enable ``spack spconfig`` functionality, the author of +``myproject/package.py`` must subclass from ``CMakePackage`` instead +of the standard ``Package`` superclass. Because CMake is +standardized, the packager does not need to tell Spack how to run +``cmake; make; make install``. Instead the packager only needs to +create (optional) methods ``configure_args()`` and ``configure_env()``, which +provide the arguments (as a list) and extra environment variables (as +a dict) to provide to the ``cmake`` command. Usually, these will +translate variant flags into CMake definitions. For example: + +.. code-block:: python + def configure_args(self): + spec = self.spec + return [ + '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), + '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), + '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), + '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), + '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO')] + +If needed, a packager may also override methods defined in +``StagedPackage`` (see below). + + +StagedPackage +~~~~~~~~~~~~~ + +``CMakePackage`` is implemented by subclassing the ``StagedPackage`` +superclass, which breaks down the standard ``Package.install()`` +method into several sub-stages: ``spconfig``, ``configure``, ``build`` +and ``install``. Details: + + * Instead of implementing the standard ``install()`` method, package + authors implement the methods for the sub-stages + ``install_spconfig()``, ``install_configure()``, + ``install_build()``, and ``install_install()``. + * The ``spack install`` command runs the sub-stages ``configure``, + ``build`` and ``install`` in order. (The ``spconfig`` stage is + not run by default; see below). + * The ``spack spconfig`` command runs the sub-stages ``spconfig`` + and a dummy install (to create the module file). + * The sub-stage install methods take no arguments (other than + ``self``). The arguments ``spec`` and ``prefix`` to the standard + ``install()`` method may be accessed via ``self.spec`` and + ``self.prefix``. + +GNU Autotools +~~~~~~~~~~~~~ + +The ``spconfig`` functionality is currently only available for +CMake-based packages. Extending this functionality to GNU +Autotools-based packages would be easy (and should be done by a +developer who actively uses Autotools). Packages that use +non-standard build systems can gain ``spconfig`` functionality by +subclassing ``StagedPackage`` directly. -- cgit v1.2.3-70-g09d2 From a1c965d70d7074dcedd88379c636f834f6c88fd2 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 26 Mar 2016 20:21:27 -0400 Subject: Fixed imports to make ``git spconfig`` work after recent merge from develop. --- lib/spack/spack/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2d7cb765a0..51ee3d7d8b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -38,6 +38,7 @@ import re import textwrap import time import glob +import string import llnl.util.tty as tty import spack @@ -51,6 +52,8 @@ import spack.mirror import spack.repository import spack.url import spack.util.web + +from urlparse import urlparse from StringIO import StringIO from llnl.util.filesystem import * from llnl.util.lang import * @@ -59,9 +62,10 @@ from llnl.util.tty.log import log_output from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment -from spack.util.executable import ProcessError +from spack.util.executable import ProcessError, Executable, which from spack.version import * -from urlparse import urlparse +from spack import directory_layout + """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -- cgit v1.2.3-70-g09d2 From 4a6b5d52475095a865d1bb98db1dd136a24607b1 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 27 Mar 2016 00:14:10 -0400 Subject: Update to documentation formatting. --- lib/spack/docs/packaging_guide.rst | 45 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bc2834c713..d8e7cdfa80 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1773,7 +1773,7 @@ discover its dependencies. If you want to see the environment that a package will build with, or if you want to run commands in that environment to test them out, you -can use the :ref:```spack env`` ` command, documented +can use the :ref:`spack env ` command, documented below. .. _compiler-wrappers: @@ -1812,7 +1812,7 @@ An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, ``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` -inserted on the command line. This is done transparently to the +‰ command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do not** have to insert extra ``-I``, ``-L``, etc. on the command line. @@ -2722,12 +2722,12 @@ example: .. code-block:: bash - > cd myproject - > spack spconfig myproject@local - > mkdir build; cd build - > ../spconfig.py .. - > make - > make install + cd myproject + spack spconfig myproject@local + mkdir build; cd build + ../spconfig.py .. + make + make install Notes: * Spack must have ``myproject/package.py`` in its repository for @@ -2749,6 +2749,7 @@ Notes: reference of what dependencies are being used. * ``make install`` installs the package into the Spack repository, where it may be used by other Spack packages. + * CMake-generated makefiles re-run CMake in some circumstances. Use of ``spconfig.py`` breaks this behavior, requiring the developer to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. CMakePackage ~~~~~~~~~~~~ @@ -2764,6 +2765,7 @@ a dict) to provide to the ``cmake`` command. Usually, these will translate variant flags into CMake definitions. For example: .. code-block:: python + def configure_args(self): spec = self.spec return [ @@ -2785,19 +2787,20 @@ superclass, which breaks down the standard ``Package.install()`` method into several sub-stages: ``spconfig``, ``configure``, ``build`` and ``install``. Details: - * Instead of implementing the standard ``install()`` method, package - authors implement the methods for the sub-stages - ``install_spconfig()``, ``install_configure()``, - ``install_build()``, and ``install_install()``. - * The ``spack install`` command runs the sub-stages ``configure``, - ``build`` and ``install`` in order. (The ``spconfig`` stage is - not run by default; see below). - * The ``spack spconfig`` command runs the sub-stages ``spconfig`` - and a dummy install (to create the module file). - * The sub-stage install methods take no arguments (other than - ``self``). The arguments ``spec`` and ``prefix`` to the standard - ``install()`` method may be accessed via ``self.spec`` and - ``self.prefix``. +* Instead of implementing the standard ``install()`` method, package + authors implement the methods for the sub-stages + ``install_spconfig()``, ``install_configure()``, + ``install_build()``, and ``install_install()``. + +* The ``spack install`` command runs the sub-stages ``configure``, + ``build`` and ``install`` in order. (The ``spconfig`` stage is + not run by default; see below). +* The ``spack spconfig`` command runs the sub-stages ``spconfig`` + and a dummy install (to create the module file). +* The sub-stage install methods take no arguments (other than + ``self``). The arguments ``spec`` and ``prefix`` to the standard + ``install()`` method may be accessed via ``self.spec`` and + ``self.prefix``. GNU Autotools ~~~~~~~~~~~~~ -- cgit v1.2.3-70-g09d2 From 4d466fe879820e18f18e96809f6825d393792de3 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 27 Apr 2016 20:57:04 -0400 Subject: spack setup: Fix broken module convenience settings. --- lib/spack/spack/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 51ee3d7d8b..ceb1a499a5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1497,8 +1497,21 @@ def make_executable(path): os.chmod(path, mode) + class CMakePackage(StagedPackage): + def make_make(self): + import multiprocessing + # number of jobs spack will to build with. + jobs = multiprocessing.cpu_count() + if not self.parallel: + jobs = 1 + elif self.make_jobs: + jobs = self.make_jobs + + make = spack.build_environment.MakeExecutable('make', jobs) + return make + def configure_args(self): """Returns package-specific arguments to be provided to the configure command.""" return list() @@ -1576,10 +1589,12 @@ env = dict() cmake(self.source_directory, *options) def install_build(self): + make = self.make_make() with working_dir(self.build_directory, create=False): make() def install_install(self): + make = self.make_make() with working_dir(self.build_directory, create=False): make('install') -- cgit v1.2.3-70-g09d2 From 2243de9e2f6e81aaccdeef8d4ccfebd5caa2be04 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 4 May 2016 19:45:30 -0400 Subject: Make quiet mode default for spack spconfig --- lib/spack/spack/cmd/spconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index 70db48e172..fcd9bdd63d 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -40,8 +40,8 @@ def setup_parser(subparser): '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="Do not try to install dependencies of requested packages.") subparser.add_argument( - '-q', '--quiet', action='store_true', dest='quiet', - help="Do not display verbose build output while installing.") + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -90,6 +90,6 @@ def spconfig(self, args): package.do_install( keep_prefix=True, # Don't remove install directory, even if you think you should ignore_deps=args.ignore_deps, - verbose=not args.quiet, + verbose=args.verbose, keep_stage=True, # don't remove source dir for SPCONFIG. install_phases = set(['spconfig', 'provenance'])) -- cgit v1.2.3-70-g09d2 From 6a48385111a383b2bb96f225e6eb1174376402dd Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Thu, 5 May 2016 17:47:41 -0400 Subject: Keep users environment in the spack setup script (spconfig.py). This is important to avoid breaking things that require module loads to work; for example, non-activate Python packages. --- lib/spack/spack/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ceb1a499a5..959e618005 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1552,7 +1552,8 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict() +#env = dict() +env = dict(os.environ) """ % sys.executable) env_vars = sorted(list(env.keys())) -- cgit v1.2.3-70-g09d2 From efa506b235523c8a9a925c1ecf91690266da4fd4 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Sat, 14 May 2016 17:09:11 -0400 Subject: Preparing spack setup command for merge. Try this out for a while... --- lib/spack/docs/packaging_guide.rst | 123 +++++++++++++++++++++++++++++++------ lib/spack/spack/cmd/setup.py | 91 +++++++++++++++++++++++++++ lib/spack/spack/cmd/spconfig.py | 95 ---------------------------- lib/spack/spack/package.py | 30 ++++----- 4 files changed, 209 insertions(+), 130 deletions(-) create mode 100644 lib/spack/spack/cmd/setup.py delete mode 100644 lib/spack/spack/cmd/spconfig.py diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index d8e7cdfa80..438d787a12 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -377,6 +377,8 @@ add a line like this in the package class: version('8.2.1', '4136d7b4c04df68b686570afa26988ac') ... +Versions should be listed with the newest version first. + Version URLs ~~~~~~~~~~~~~~~~~ @@ -385,8 +387,21 @@ in the package. For example, Spack is smart enough to download version ``8.2.1.`` of the ``Foo`` package above from ``http://example.com/foo-8.2.1.tar.gz``. -If spack *cannot* extrapolate the URL from the ``url`` field, or if -the package doesn't have a ``url`` field, you can add a URL explicitly +If spack *cannot* extrapolate the URL from the ``url`` field by +default, you can write your own URL generation algorithm in place of +the ``url`` declaration. For example: + +.. code-block:: python + :linenos: + + class Foo(Package): + def url_for_version(self, version): + return 'http://example.com/version_%s/foo-%s.tar.gz' \ + % (version, version) + version('8.2.1', '4136d7b4c04df68b686570afa26988ac') + ... + +If a URL cannot be derived systematically, you can add an explicit URL for a particular version: .. code-block:: python @@ -1216,6 +1231,19 @@ Now, the ``py-numpy`` package can be used as an argument to ``spack activate``. When it is activated, all the files in its prefix will be symbolically linked into the prefix of the python package. +Many packages produce Python extensions for *some* variants, but not +others: they should extend ``python`` only if the apropriate +variant(s) are selected. This may be accomplished with conditional +``extends()`` declarations: + +.. code-block:: python + + class FooLib(Package): + variant('python', default=True, description= \ + 'Build the Python extension Module') + extends('python', when='+python') + ... + Sometimes, certain files in one package will conflict with those in another, which means they cannot both be activated (symlinked) at the same time. In this case, you can tell Spack to ignore those files @@ -2392,6 +2420,59 @@ File functions .. _package-lifecycle: +Coding Style Guidelines +--------------------------- + +The following guidelines are provided, in the interests of making +Spack packages work in a consistent manner: + + +Variant Names +~~~~~~~~~~~~~~ + +Spack packages with variants similar to already-existing Spack +packages should use the same name for their variants. Standard +variant names are: + +======= ======== ======================== +Name Default Description +------- -------- ------------------------ +shared True Build shared libraries +static Build static libraries +mpi Use MPI +python Build Python extension +------- -------- ------------------------ + +If specified in this table, the corresponding default should be used +when declaring a variant. + + +Version Lists +~~~~~~~~~~~~~~ + +Spack packges should list supported versions with the newest first. + +Special Versions +~~~~~~~~~~~~~~~~~ + +The following *special* version names may be used when building a package: + +* *@system*: Indicates a hook to the OS-installed version of the + package. This is useful, for example, to tell Spack to use the + OS-installed version in ``packages.yaml``:: + + openssl: + paths: + openssl@system: /usr + buildable: False + + Certain Spack internals look for the *@system* version and do + appropriate things in that case. + +* *@local*: Indicates the version was built manually from some source + tree of unknown provenance (see ``spack setup``). + + Packaging workflow commands --------------------------------- @@ -2715,7 +2796,7 @@ Imagine a developer creating a CMake-based (or Autotools) project in a local directory, which depends on libraries A-Z. Once Spack has installed those dependencies, one would like to run ``cmake`` with appropriate command line and environment so CMake can find them. The ``spack -spconfig`` command does this conveniently, producing a CMake +setup`` command does this conveniently, producing a CMake configuration that is essentially the same as how Spack *would have* configured the project. This can be demonstrated with a usage example: @@ -2723,7 +2804,7 @@ example: .. code-block:: bash cd myproject - spack spconfig myproject@local + spack setup myproject@local mkdir build; cd build ../spconfig.py .. make @@ -2732,29 +2813,31 @@ example: Notes: * Spack must have ``myproject/package.py`` in its repository for this to work. - * ``spack spconfig`` produces the executable script ``spconfig.py`` - in the local directory, and also creates the module file for the - package. ``spconfig.py`` is normally run from the top level of - the source tree. - - * The version number given to ``spack spconfig`` is arbitrary (just - like ``spack diy``). ``myproject/package.py`` does not need to + * ``spack setup`` produces the executable script ``spconfig.py`` in + the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the user's + out-of-source build directory. + * The version number given to ``spack setup`` is arbitrary, just + like ``spack diy``. ``myproject/package.py`` does not need to have any valid downloadable versions listed (typical when a project is new). * spconfig.py produces a CMake configuration that *does not* use the Spack wrappers. Any resulting binaries *will not* use RPATH, unless the user has enabled it. This is recommended for development purposes, not production. - * spconfig.py is easily legible, and can serve as a developer + * ``spconfig.py`` is human readable, and can serve as a developer reference of what dependencies are being used. * ``make install`` installs the package into the Spack repository, where it may be used by other Spack packages. - * CMake-generated makefiles re-run CMake in some circumstances. Use of ``spconfig.py`` breaks this behavior, requiring the developer to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. + * CMake-generated makefiles re-run CMake in some circumstances. Use + of ``spconfig.py`` breaks this behavior, requiring the developer + to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file + has changed. CMakePackage ~~~~~~~~~~~~ -In order ot enable ``spack spconfig`` functionality, the author of +In order ot enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead of the standard ``Package`` superclass. Because CMake is standardized, the packager does not need to tell Spack how to run @@ -2784,18 +2867,18 @@ StagedPackage ``CMakePackage`` is implemented by subclassing the ``StagedPackage`` superclass, which breaks down the standard ``Package.install()`` -method into several sub-stages: ``spconfig``, ``configure``, ``build`` +method into several sub-stages: ``setup``, ``configure``, ``build`` and ``install``. Details: * Instead of implementing the standard ``install()`` method, package authors implement the methods for the sub-stages - ``install_spconfig()``, ``install_configure()``, + ``install_setup()``, ``install_configure()``, ``install_build()``, and ``install_install()``. * The ``spack install`` command runs the sub-stages ``configure``, - ``build`` and ``install`` in order. (The ``spconfig`` stage is + ``build`` and ``install`` in order. (The ``setup`` stage is not run by default; see below). -* The ``spack spconfig`` command runs the sub-stages ``spconfig`` +* The ``spack setup`` command runs the sub-stages ``setup`` and a dummy install (to create the module file). * The sub-stage install methods take no arguments (other than ``self``). The arguments ``spec`` and ``prefix`` to the standard @@ -2805,9 +2888,9 @@ and ``install``. Details: GNU Autotools ~~~~~~~~~~~~~ -The ``spconfig`` functionality is currently only available for +The ``setup`` functionality is currently only available for CMake-based packages. Extending this functionality to GNU Autotools-based packages would be easy (and should be done by a developer who actively uses Autotools). Packages that use -non-standard build systems can gain ``spconfig`` functionality by +non-standard build systems can gain ``setup`` functionality by subclassing ``StagedPackage`` directly. diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py new file mode 100644 index 0000000000..02e9bfd281 --- /dev/null +++ b/lib/spack/spack/cmd/setup.py @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Elizabeth Fischer +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +from spack.cmd.edit import edit_package +from spack.stage import DIYStage + +description = "Create a configuration script and module, but don't build." + +def setup_parser(subparser): + subparser.add_argument( + '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + help="Do not try to install dependencies of requested packages.") + subparser.add_argument( + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, + help="specs to use for install. Must contain package AND verison.") + + +def setup(self, args): + if not args.spec: + tty.die("spack setup requires a package spec argument.") + + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack setup only takes one spec.") + + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.repo.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, spack.repo.first_repo(), None, True) + return + + if not spec.versions.concrete: + tty.die("spack setup spec must have a single, concrete version. Did you forget a package version number?") + + spec.concretize() + package = spack.repo.get(spec) + + # It's OK if the package is already installed. + + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) + + # TODO: make this an argument, not a global. + spack.do_checksum = False + + package.do_install( + keep_prefix=True, # Don't remove install directory, even if you think you should + ignore_deps=args.ignore_deps, + verbose=args.verbose, + keep_stage=True, # don't remove source dir for SETUP. + install_phases = set(['setup', 'provenance'])) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py deleted file mode 100644 index fcd9bdd63d..0000000000 --- a/lib/spack/spack/cmd/spconfig.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Elizabeth Fischer -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import sys -import os -import argparse - -import llnl.util.tty as tty - -import spack -import spack.cmd -from spack.cmd.edit import edit_package -from spack.stage import DIYStage - -description = "Create a configuration script and module, but don't build." - -def setup_parser(subparser): - subparser.add_argument( - '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - help="Do not try to install dependencies of requested packages.") - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="Display verbose build output while installing.") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. Must contain package AND verison.") - - -def spconfig(self, args): - if not args.spec: - tty.die("spack spconfig requires a package spec argument.") - - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack spconfig only takes one spec.") - - # Take a write lock before checking for existence. - with spack.installed_db.write_transaction(): - spec = specs[0] - if not spack.repo.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, spack.repo.first_repo(), None, True) - return - - if not spec.versions.concrete: - tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") - - spec.concretize() - package = spack.repo.get(spec) - - # It's OK if the package is already installed. - #if package.installed: - # tty.error("Already installed in %s" % package.prefix) - # tty.msg("Uninstall or try adding a version suffix for this SPCONFIG build.") - # sys.exit(1) - - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) - - # TODO: make this an argument, not a global. - spack.do_checksum = False - - package.do_install( - keep_prefix=True, # Don't remove install directory, even if you think you should - ignore_deps=args.ignore_deps, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SPCONFIG. - install_phases = set(['spconfig', 'provenance'])) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 959e618005..19198c3b28 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -935,7 +935,7 @@ class Package(object): packages_dir = spack.install_layout.build_packages_path(self.spec) # Remove first if we're overwriting another build - # (can happen with spack spconfig) + # (can happen with spack setup) try: shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this except: @@ -1456,9 +1456,9 @@ def _hms(seconds): class StagedPackage(Package): """A Package subclass where the install() is split up into stages.""" - def install_spconfig(self): - """Creates an spconfig.py script to configure the package later if we like.""" - raise InstallError("Package %s provides no install_spconfig() method!" % self.name) + def install_setup(self): + """Creates an spack_setup.py script to configure the package later if we like.""" + raise InstallError("Package %s provides no install_setup() method!" % self.name) def install_configure(self): """Runs the configure process.""" @@ -1473,8 +1473,8 @@ class StagedPackage(Package): raise InstallError("Package %s provides no install_install() method!" % self.name) def install(self, spec, prefix): - if 'spconfig' in self.install_phases: - self.install_spconfig() + if 'setup' in self.install_phases: + self.install_setup() if 'configure' in self.install_phases: self.install_configure() @@ -1520,13 +1520,13 @@ class CMakePackage(StagedPackage): """Returns package-specific environment under which the configure command should be run.""" return dict() - def cmake_transitive_include_path(self): + def spack_transitive_include_path(self): return ';'.join( os.path.join(dep, 'include') for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) ) - def install_spconfig(self): + def install_setup(self): cmd = [str(which('cmake'))] + \ spack.build_environment.get_std_cmake_args(self) + \ ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], @@ -1537,11 +1537,11 @@ class CMakePackage(StagedPackage): env = dict() env['PATH'] = os.environ['PATH'] - env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - spconfig_fname = 'spconfig.py' - with open(spconfig_fname, 'w') as fout: + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: fout.write(\ r"""#!%s # @@ -1552,7 +1552,6 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) -#env = dict() env = dict(os.environ) """ % sys.executable) @@ -1562,7 +1561,7 @@ env = dict(os.environ) if string.find(name, 'PATH') < 0: fout.write('env[%s] = %s\n' % (repr(name),repr(val))) else: - if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': sep = ';' else: sep = ':' @@ -1572,20 +1571,21 @@ env = dict(os.environ) fout.write(' %s\n' % part) fout.write('"""))\n') + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") fout.write('\ncmd = cmdlist("""\n') fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - make_executable(spconfig_fname) + make_executable(setup_fname) def install_configure(self): cmake = which('cmake') with working_dir(self.build_directory, create=True): os.environ.update(self.configure_env()) - os.environ['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) cmake(self.source_directory, *options) -- cgit v1.2.3-70-g09d2 From 6327877a6f9d2c30e4d644ec145a6f3f323ead81 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Thu, 30 Jun 2016 09:19:57 -0400 Subject: PEP-8 --- lib/spack/spack/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6f74f8ff10..20c9934704 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -176,8 +176,10 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'StagedPackage', 'CMakePackage', 'Version', 'when', 'ver'] -from spack.package import Package, StagedPackage, CMakePackage, ExtensionConflictError +__all__ = ['Package', 'StagedPackage', 'CMakePackage', \ + 'Version', 'when', 'ver'] +from spack.package import Package, ExtensionConflictError +from spack.package import StagedPackage, CMakePackage from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2