From 90bb855ffa2d39d4cd861dd65f51acfa72d11ea4 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:30:38 -0500 Subject: A new subclass StagedPackage(Package) is introduced. This PR should not change the behavior for existing packages that subclass from spack.Package. If a package subclasses spack.StagedPackage instead of spack.Package, the install() phase (run inside a forked process) is now separated into sub-stages: a) spconfig: Generate a script spconfig.py that will configure the package (eg by running CMake or ./configure) This is for use if the user wishes to build externally from Spack. Therefore, the Spack compiler wrappers are NOT used here. Currently, that means that RPATH support is up to the user. b) configure: Configure the project (eg: runs configure, CMake, etc). This will configure it for use within Spack, using the Spack wrapper. c) build: eg: "make" d) install: eg: "install" If one chooses to use StagedPackage instead of Package, then one must implement each of the install sub-stages as a separate method. StagedPackage.install() then calls each of the sub-stages as appropriate. StagedPackage can be configured to only run certain sub-stages. This is done by setting the optional kwarg install_phases when calling do_install(). Setting install_phases() ONLY has an effect on StagedPackage, not on any existing packages. By default, install_phases is set to make StagedPackage run the same stages that are normally run for any package: configure, build, install (and NOT spconfig). The ability for Spack to run stages selectively for StagedPackage instances will enable new functionality. For example, explicit CMake/Autotools helpers that allow Spack to help configure a user's project without fetching, building or installing it. ------------- One implementation of StagedPackage is provided, CMakePackage. This has the following advantage for CMake-based projects over using the standard Package class: a) By separating out the phases, it enables future new functionality for packages that use it. b) It provides an implementation of intall_spconfig(), which will help users configure their CMake-based projects. CMakePackage expects users to implement configure_args() and configure_env(). These methods provide the package-specific arguments and environment needed to properly configure the package. They are placed in separated functions because they are used in both the spconfig and configure stages. TODO: 1. Generate spconfig.py scripts that are more readable. This allows the users to understand how their project is configured. 2. Provide a practical way for the user to ACCESS the spconfig stage without building the project through Spack. 3. The CMAKE_TRANSITIVE_INCLUDE_PATH stuff needs to be reworked; it should be considered provisional for now. 4. User of Autotools might wish to make a similar ConfigurePackage subclass of StagedPackage. --------------- One package using CMakePackage is introduced. See ibmisc/package.py. --- lib/spack/spack/package.py | 100 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf896..d02a80bcad 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -66,7 +66,7 @@ import spack.fetch_strategy as fs from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension -from spack.util.executable import ProcessError +from spack.util.executable import ProcessError, which from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" @@ -826,7 +826,8 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, - skip_patch=False, verbose=False, make_jobs=None, fake=False): + skip_patch=False, verbose=False, make_jobs=None, fake=False, + install_phases = {'spconfig', 'configure', 'build', 'install'}): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -881,6 +882,10 @@ class Package(object): tty.msg("Building %s" % self.name) self.stage.keep = keep_stage + self.install_phases = install_phases + self.build_directory = join_path(self.stage.path, 'spack-build') + self.source_directory = self.stage.source_path + with self.stage: # Run the pre-install hook in the child process after # the directory is created. @@ -1291,6 +1296,97 @@ def _hms(seconds): if s: parts.append("%.2fs" % s) return ' '.join(parts) +class StagedPackage(Package): + """A Package subclass where the install() is split up into stages.""" + + def install_spconfig(self): + """Creates an spconfig.py script to configure the package later if we like.""" + raise InstallError("Package %s provides no install_spconfig() method!" % self.name) + + def install_configure(self): + """Runs the configure process.""" + raise InstallError("Package %s provides no install_configure() method!" % self.name) + + def install_build(self): + """Runs the build process.""" + raise InstallError("Package %s provides no install_build() method!" % self.name) + + def install_install(self): + """Runs the install process.""" + raise InstallError("Package %s provides no install_install() method!" % self.name) + + def install(self, spec, prefix): + if 'spconfig' in self.install_phases: + self.install_spconfig() + + if 'configure' in self.install_phases: + self.install_configure() + + if 'build' in self.install_phases: + self.install_build() + + if 'install' in self.install_phases: + self.install_install() + else: + # Create a dummy file so the build doesn't fail. + # That way, the module file will also be created. + with open(os.path.join(prefix, 'dummy'), 'w') as fout: + pass + + +class CMakePackage(StagedPackage): + + def configure_args(self): + """Returns package-specific arguments to be provided to the configure command.""" + return list() + + def configure_env(self): + """Returns package-specific environment under which the configure command should be run.""" + return dict() + + def cmake_transitive_include_path(self): + return ';'.join( + os.path.join(dep, 'include') + for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) + ) + + def install_spconfig(self): + cmd = [str(which('cmake'))] + \ + spack.build_environment.get_std_cmake_args(self) + \ + ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], + '-DCMAKE_C_COMPILER=%s' % os.environ['SPACK_CC'], + '-DCMAKE_CXX_COMPILER=%s' % os.environ['SPACK_CXX'], + '-DCMAKE_Fortran_COMPILER=%s' % os.environ['SPACK_FC']] + \ + self.configure_args() + + env = dict() + env['PATH'] = os.environ['PATH'] + env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] + + with open('spconfig.py', 'w') as fout: + fout.write('import sys\nimport os\nimport subprocess\n') + fout.write('env = {}\n'.format(repr(env))) + fout.write('cmd = {} + sys.argv[1:]\n'.format(repr(cmd))) + fout.write('proc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + + + def install_configure(self): + cmake = which('cmake') + with working_dir(self.build_directory, create=True): + os.environ.update(self.configure_env()) + os.environ['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) + cmake(self.source_directory, *options) + + def install_build(self): + with working_dir(self.build_directory, create=False): + make() + + def install_install(self): + with working_dir(self.build_directory, create=False): + make('install') + class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" -- cgit v1.2.3-70-g09d2 From 42361578237e4e96ae497107a47e277a1eeb69e9 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:13:00 -0500 Subject: (1) Added "spack spconfig" command. (2) Neatened up the spconfig.py auto-generated file. --- lib/spack/spack/cmd/spconfig.py | 97 +++++++++++++++++++++++++++++ lib/spack/spack/package.py | 133 ++++++++++++++++++++++++++++++---------- 2 files changed, 197 insertions(+), 33 deletions(-) create mode 100644 lib/spack/spack/cmd/spconfig.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py new file mode 100644 index 0000000000..a89e5f99e7 --- /dev/null +++ b/lib/spack/spack/cmd/spconfig.py @@ -0,0 +1,97 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Elizabeth Fischer +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +from spack.cmd.edit import edit_package +from spack.stage import DIYStage + +description = "Create a configuration script and module, but don't build." + +def setup_parser(subparser): + subparser.add_argument( + '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + help="Do not try to install dependencies of requested packages.") + subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, + help="specs to use for install. Must contain package AND verison.") + + +def spconfig(self, args): + if not args.spec: + tty.die("spack spconfig requires a package spec argument.") + + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack spconfig only takes one spec.") + + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.repo.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, spack.repo.first_repo(), None, True) + return + + print('spec', spec) + + if not spec.version.concrete: + tty.die("spack spconfig spec must have a single, concrete version.") + + spec.concretize() + package = spack.repo.get(spec) + + # It's OK if the package is already installed. + #if package.installed: + # tty.error("Already installed in %s" % package.prefix) + # tty.msg("Uninstall or try adding a version suffix for this SPCONFIG build.") + # sys.exit(1) + + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) + + # TODO: make this an argument, not a global. + spack.do_checksum = False + + package.do_install( + keep_prefix=True, # Don't remove install directory, even if you think you should + ignore_deps=args.ignore_deps, + verbose=not args.quiet, + keep_stage=True, # don't remove source dir for SPCONFIG. + install_phases = {'spconfig', 'provenance'}) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d02a80bcad..3d8e098346 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -45,6 +45,9 @@ import multiprocessing from urlparse import urlparse, urljoin import textwrap from StringIO import StringIO +import shutil +import sys +import string import llnl.util.tty as tty from llnl.util.tty.log import log_output @@ -68,6 +71,7 @@ from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError, which from spack.util.environment import dump_environment +from spack import directory_layout """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -827,7 +831,7 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False, - install_phases = {'spconfig', 'configure', 'build', 'install'}): + install_phases = {'configure', 'build', 'install', 'provenance'}): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -853,7 +857,7 @@ class Package(object): return # Ensure package is not already installed - if spack.install_layout.check_installed(self.spec): + if 'install' in install_phases and spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return @@ -895,35 +899,46 @@ class Package(object): self.do_fake_install() else: # Do the real install in the source directory. - self.stage.chdir_to_source() + self.stage.chdir_to_source() + + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + + try: + # Redirect I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) + + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Ensure that something was actually installed. + if 'install' in self.install_phases: + self._sanity_check_install() - try: - # Redirect I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) - except ProcessError as e: - # Annotate ProcessErrors with the location of the build log. - e.build_log = log_path - raise e + # Copy provenance into the install directory on success + if 'provenance' in self.install_phases: - # Ensure that something was actually installed. - self._sanity_check_install() + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Remove first if we're overwriting another build + # (can happen with spack spconfig) + try: + shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this + except: + pass - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) # Stop timer. self._total_time = time.time() - start_time @@ -937,16 +952,29 @@ class Package(object): try: # Create the install prefix and fork the build process. spack.install_layout.create_install_directory(self.spec) + except directory_layout.InstallDirectoryAlreadyExistsError: + if 'install' in install_phases: + # Abort install if install directory exists. + # But do NOT remove it (you'd be overwriting someon else's stuff) + tty.warn("Keeping existing install prefix in place.") + raise + else: + # We're not installing anyway, so don't worry if someone + # else has already written in the install directory + pass + + try: spack.build_environment.fork(self, build_process) except: # remove the install prefix if anything went wrong during install. - if not keep_prefix: - self.remove_prefix() - else: + if keep_prefix: tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed. " + "Manually remove this directory to fix:", self.prefix, wrap=True) + else: + self.remove_prefix() + raise # note: PARENT of the build process adds the new package to @@ -1333,6 +1361,12 @@ class StagedPackage(Package): with open(os.path.join(prefix, 'dummy'), 'w') as fout: pass +# stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python +def make_executable(path): + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + class CMakePackage(StagedPackage): @@ -1364,11 +1398,44 @@ class CMakePackage(StagedPackage): env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - with open('spconfig.py', 'w') as fout: - fout.write('import sys\nimport os\nimport subprocess\n') - fout.write('env = {}\n'.format(repr(env))) - fout.write('cmd = {} + sys.argv[1:]\n'.format(repr(cmd))) - fout.write('proc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + spconfig_fname = 'spconfig.py' + with open(spconfig_fname, 'w') as fout: + fout.write(\ +r"""#!{} +# + +import sys +import os +import subprocess + +def cmdlist(str): + return list(x.strip().replace("'",'') for x in str.split('\n') if x) +env = dict() +""".format(sys.executable)) + + env_vars = sorted(list(env.keys())) + for name in env_vars: + val = env[name] + if string.find(name, 'PATH') < 0: + fout.write('env[{}] = {}\n'.format(repr(name),repr(val))) + else: + if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': + sep = ';' + else: + sep = ':' + + fout.write('env[{}] = "{}".join(cmdlist("""\n'.format(repr(name),sep)) + for part in string.split(val, sep): + fout.write(' {}\n'.format(part)) + fout.write('"""))\n') + + fout.write('\ncmd = cmdlist("""\n') + fout.write('{}\n'.format(cmd[0])) + for arg in cmd[1:]: + fout.write(' {}\n'.format(arg)) + fout.write('""") + sys.argv[1:]\n') + fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') + make_executable(spconfig_fname) def install_configure(self): -- cgit v1.2.3-70-g09d2 From c1a8574d8f67d6b4fdce109ecf4dd3a1cbe18488 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 00:29:40 -0500 Subject: Fixed for Python 2.6 --- lib/spack/spack/package.py | 2 +- var/spack/repos/builtin/packages/ibmisc/package.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3d8e098346..f815628bba 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -831,7 +831,7 @@ class Package(object): def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False, - install_phases = {'configure', 'build', 'install', 'provenance'}): + install_phases = set(['configure', 'build', 'install', 'provenance'])): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe diff --git a/var/spack/repos/builtin/packages/ibmisc/package.py b/var/spack/repos/builtin/packages/ibmisc/package.py index 9fadee7239..a3bd680655 100644 --- a/var/spack/repos/builtin/packages/ibmisc/package.py +++ b/var/spack/repos/builtin/packages/ibmisc/package.py @@ -1,4 +1,5 @@ from spack import * +import llnl.util.tty as tty class Ibmisc(CMakePackage): """Misc. reusable utilities used by IceBin.""" @@ -15,7 +16,7 @@ class Ibmisc(CMakePackage): variant('boost', default=True, description='Compile utilities for Boost library') variant('udunits2', default=True, description='Compile utilities for UDUNITS2 library') variant('googletest', default=True, description='Compile utilities for Google Test library') - variant('python', default=True, description='Compile utilities for use with Python/Cython') + variant('python', default=True, description='Compile utilities fro use with Python/Cython') extends('python') @@ -26,16 +27,18 @@ class Ibmisc(CMakePackage): depends_on('netcdf-cxx4', when='+netcdf') depends_on('udunits2', when='+udunits2') depends_on('googletest', when='+googletest') +# depends_on('python', when='+python') depends_on('py-cython', when='+python') depends_on('py-numpy', when='+python') depends_on('boost', when='+boost') + + # Build dependencies depends_on('cmake') depends_on('doxygen') - def configure_args(self): - spec = self.spec + def config_args(self, spec, prefix): return [ '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), '-DUSE_PROJ4=%s' % ('YES' if '+proj' in spec else 'NO'), -- cgit v1.2.3-70-g09d2 From 9885f1a19ec5c81121e20cd201007c89cc7b1b1f Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:34:46 -0500 Subject: Fix for Python 2.6 --- lib/spack/spack/cmd/spconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index a89e5f99e7..6035a61841 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -94,4 +94,4 @@ def spconfig(self, args): ignore_deps=args.ignore_deps, verbose=not args.quiet, keep_stage=True, # don't remove source dir for SPCONFIG. - install_phases = {'spconfig', 'provenance'}) + install_phases = set(['spconfig', 'provenance'])) -- cgit v1.2.3-70-g09d2 From 857f791286a041d527481a35931cb0249624a53a Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 13 Mar 2016 00:40:50 -0500 Subject: Add missing import. --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 3051d3f742..920e22d672 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -173,7 +173,7 @@ sys_type = None # should live. This file is overloaded for spack core vs. for packages. # __all__ = ['Package', 'Version', 'when', 'ver'] -from spack.package import Package, ExtensionConflictError +from spack.package import Package, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 4c9a52044a89ef8133ab2ac1759da9d4caefa3eb Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 15:18:24 -0400 Subject: Fixed CMakePackage import --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 920e22d672..c31687cafc 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -172,7 +172,7 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'Version', 'when', 'ver'] +__all__ = ['Package', 'CMakePackage', 'Version', 'when', 'ver'] from spack.package import Package, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 030e8dd1ac7c23cfdd9bf7983caec6f8ca41a556 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 18:04:23 -0400 Subject: Removed Python 2.7-style string formatting --- lib/spack/spack/package.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index f815628bba..10f0c4e761 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1401,7 +1401,7 @@ class CMakePackage(StagedPackage): spconfig_fname = 'spconfig.py' with open(spconfig_fname, 'w') as fout: fout.write(\ -r"""#!{} +r"""#!%s # import sys @@ -1411,28 +1411,28 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) env = dict() -""".format(sys.executable)) +""" % sys.executable) env_vars = sorted(list(env.keys())) for name in env_vars: val = env[name] if string.find(name, 'PATH') < 0: - fout.write('env[{}] = {}\n'.format(repr(name),repr(val))) + fout.write('env[%s] = %s\n'. % (repr(name),repr(val))) else: if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': sep = ';' else: sep = ':' - fout.write('env[{}] = "{}".join(cmdlist("""\n'.format(repr(name),sep)) + fout.write('env[%s] = "%s".join(cmdlist("""\n' % (repr(name),sep)) for part in string.split(val, sep): - fout.write(' {}\n'.format(part)) + fout.write(' %s\n' % part) fout.write('"""))\n') fout.write('\ncmd = cmdlist("""\n') - fout.write('{}\n'.format(cmd[0])) + fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: - fout.write(' {}\n'.format(arg)) + fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') make_executable(spconfig_fname) -- cgit v1.2.3-70-g09d2 From 0426796d9f6e064f062f4c3b0d27098dfe685d77 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 18:14:38 -0400 Subject: Fixed typo --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 10f0c4e761..18c4afcf95 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1417,7 +1417,7 @@ env = dict() for name in env_vars: val = env[name] if string.find(name, 'PATH') < 0: - fout.write('env[%s] = %s\n'. % (repr(name),repr(val))) + fout.write('env[%s] = %s\n' % (repr(name),repr(val))) else: if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': sep = ';' -- cgit v1.2.3-70-g09d2 From 6f26c45143e63ee339f9aabc654684056b60654f Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:38:19 -0400 Subject: Fixed typo bug. Made error comment more explicit --- lib/spack/spack/cmd/spconfig.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index 6035a61841..70db48e172 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -69,10 +69,8 @@ def spconfig(self, args): edit_package(spec.name, spack.repo.first_repo(), None, True) return - print('spec', spec) - - if not spec.version.concrete: - tty.die("spack spconfig spec must have a single, concrete version.") + if not spec.versions.concrete: + tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) -- cgit v1.2.3-70-g09d2 From 8f675b0ee8d546a6b7908a26db189eb6583e1d98 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 25 Mar 2016 17:12:16 -0400 Subject: Made StagedPackage user-visible. --- lib/spack/spack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 2cb189b4a2..fce7481b5b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -172,8 +172,8 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'CMakePackage', 'Version', 'when', 'ver'] -from spack.package import Package, CMakePackage, ExtensionConflictError +__all__ = ['Package', 'StagedPackage', 'CMakePackage', 'Version', 'when', 'ver'] +from spack.package import Package, StagedPackage, CMakePackage, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From 9889b1a5febc2aedec6401ae701b55961cf0d559 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 25 Mar 2016 17:12:32 -0400 Subject: Added documentation for StagedPackage, etc. --- lib/spack/docs/packaging_guide.rst | 101 +++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 519c0da232..bc2834c713 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2707,3 +2707,104 @@ might write: DWARF_PREFIX = $(spack location -i libdwarf) CXXFLAGS += -I$DWARF_PREFIX/include CXXFLAGS += -L$DWARF_PREFIX/lib + +Build System Configuration Support +---------------------------------- + +Imagine a developer creating a CMake-based (or Autotools) project in a local +directory, which depends on libraries A-Z. Once Spack has installed +those dependencies, one would like to run ``cmake`` with appropriate +command line and environment so CMake can find them. The ``spack +spconfig`` command does this conveniently, producing a CMake +configuration that is essentially the same as how Spack *would have* +configured the project. This can be demonstrated with a usage +example: + +.. code-block:: bash + + > cd myproject + > spack spconfig myproject@local + > mkdir build; cd build + > ../spconfig.py .. + > make + > make install + +Notes: + * Spack must have ``myproject/package.py`` in its repository for + this to work. + * ``spack spconfig`` produces the executable script ``spconfig.py`` + in the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the top level of + the source tree. + + * The version number given to ``spack spconfig`` is arbitrary (just + like ``spack diy``). ``myproject/package.py`` does not need to + have any valid downloadable versions listed (typical when a + project is new). + * spconfig.py produces a CMake configuration that *does not* use the + Spack wrappers. Any resulting binaries *will not* use RPATH, + unless the user has enabled it. This is recommended for + development purposes, not production. + * spconfig.py is easily legible, and can serve as a developer + reference of what dependencies are being used. + * ``make install`` installs the package into the Spack repository, + where it may be used by other Spack packages. + +CMakePackage +~~~~~~~~~~~~ + +In order ot enable ``spack spconfig`` functionality, the author of +``myproject/package.py`` must subclass from ``CMakePackage`` instead +of the standard ``Package`` superclass. Because CMake is +standardized, the packager does not need to tell Spack how to run +``cmake; make; make install``. Instead the packager only needs to +create (optional) methods ``configure_args()`` and ``configure_env()``, which +provide the arguments (as a list) and extra environment variables (as +a dict) to provide to the ``cmake`` command. Usually, these will +translate variant flags into CMake definitions. For example: + +.. code-block:: python + def configure_args(self): + spec = self.spec + return [ + '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), + '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), + '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), + '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), + '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO')] + +If needed, a packager may also override methods defined in +``StagedPackage`` (see below). + + +StagedPackage +~~~~~~~~~~~~~ + +``CMakePackage`` is implemented by subclassing the ``StagedPackage`` +superclass, which breaks down the standard ``Package.install()`` +method into several sub-stages: ``spconfig``, ``configure``, ``build`` +and ``install``. Details: + + * Instead of implementing the standard ``install()`` method, package + authors implement the methods for the sub-stages + ``install_spconfig()``, ``install_configure()``, + ``install_build()``, and ``install_install()``. + * The ``spack install`` command runs the sub-stages ``configure``, + ``build`` and ``install`` in order. (The ``spconfig`` stage is + not run by default; see below). + * The ``spack spconfig`` command runs the sub-stages ``spconfig`` + and a dummy install (to create the module file). + * The sub-stage install methods take no arguments (other than + ``self``). The arguments ``spec`` and ``prefix`` to the standard + ``install()`` method may be accessed via ``self.spec`` and + ``self.prefix``. + +GNU Autotools +~~~~~~~~~~~~~ + +The ``spconfig`` functionality is currently only available for +CMake-based packages. Extending this functionality to GNU +Autotools-based packages would be easy (and should be done by a +developer who actively uses Autotools). Packages that use +non-standard build systems can gain ``spconfig`` functionality by +subclassing ``StagedPackage`` directly. -- cgit v1.2.3-70-g09d2 From a1c965d70d7074dcedd88379c636f834f6c88fd2 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 26 Mar 2016 20:21:27 -0400 Subject: Fixed imports to make ``git spconfig`` work after recent merge from develop. --- lib/spack/spack/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2d7cb765a0..51ee3d7d8b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -38,6 +38,7 @@ import re import textwrap import time import glob +import string import llnl.util.tty as tty import spack @@ -51,6 +52,8 @@ import spack.mirror import spack.repository import spack.url import spack.util.web + +from urlparse import urlparse from StringIO import StringIO from llnl.util.filesystem import * from llnl.util.lang import * @@ -59,9 +62,10 @@ from llnl.util.tty.log import log_output from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment -from spack.util.executable import ProcessError +from spack.util.executable import ProcessError, Executable, which from spack.version import * -from urlparse import urlparse +from spack import directory_layout + """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -- cgit v1.2.3-70-g09d2 From 4a6b5d52475095a865d1bb98db1dd136a24607b1 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 27 Mar 2016 00:14:10 -0400 Subject: Update to documentation formatting. --- lib/spack/docs/packaging_guide.rst | 45 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bc2834c713..d8e7cdfa80 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1773,7 +1773,7 @@ discover its dependencies. If you want to see the environment that a package will build with, or if you want to run commands in that environment to test them out, you -can use the :ref:```spack env`` ` command, documented +can use the :ref:`spack env ` command, documented below. .. _compiler-wrappers: @@ -1812,7 +1812,7 @@ An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, ``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` -inserted on the command line. This is done transparently to the +‰ command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do not** have to insert extra ``-I``, ``-L``, etc. on the command line. @@ -2722,12 +2722,12 @@ example: .. code-block:: bash - > cd myproject - > spack spconfig myproject@local - > mkdir build; cd build - > ../spconfig.py .. - > make - > make install + cd myproject + spack spconfig myproject@local + mkdir build; cd build + ../spconfig.py .. + make + make install Notes: * Spack must have ``myproject/package.py`` in its repository for @@ -2749,6 +2749,7 @@ Notes: reference of what dependencies are being used. * ``make install`` installs the package into the Spack repository, where it may be used by other Spack packages. + * CMake-generated makefiles re-run CMake in some circumstances. Use of ``spconfig.py`` breaks this behavior, requiring the developer to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. CMakePackage ~~~~~~~~~~~~ @@ -2764,6 +2765,7 @@ a dict) to provide to the ``cmake`` command. Usually, these will translate variant flags into CMake definitions. For example: .. code-block:: python + def configure_args(self): spec = self.spec return [ @@ -2785,19 +2787,20 @@ superclass, which breaks down the standard ``Package.install()`` method into several sub-stages: ``spconfig``, ``configure``, ``build`` and ``install``. Details: - * Instead of implementing the standard ``install()`` method, package - authors implement the methods for the sub-stages - ``install_spconfig()``, ``install_configure()``, - ``install_build()``, and ``install_install()``. - * The ``spack install`` command runs the sub-stages ``configure``, - ``build`` and ``install`` in order. (The ``spconfig`` stage is - not run by default; see below). - * The ``spack spconfig`` command runs the sub-stages ``spconfig`` - and a dummy install (to create the module file). - * The sub-stage install methods take no arguments (other than - ``self``). The arguments ``spec`` and ``prefix`` to the standard - ``install()`` method may be accessed via ``self.spec`` and - ``self.prefix``. +* Instead of implementing the standard ``install()`` method, package + authors implement the methods for the sub-stages + ``install_spconfig()``, ``install_configure()``, + ``install_build()``, and ``install_install()``. + +* The ``spack install`` command runs the sub-stages ``configure``, + ``build`` and ``install`` in order. (The ``spconfig`` stage is + not run by default; see below). +* The ``spack spconfig`` command runs the sub-stages ``spconfig`` + and a dummy install (to create the module file). +* The sub-stage install methods take no arguments (other than + ``self``). The arguments ``spec`` and ``prefix`` to the standard + ``install()`` method may be accessed via ``self.spec`` and + ``self.prefix``. GNU Autotools ~~~~~~~~~~~~~ -- cgit v1.2.3-70-g09d2 From 4d466fe879820e18f18e96809f6825d393792de3 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 27 Apr 2016 20:57:04 -0400 Subject: spack setup: Fix broken module convenience settings. --- lib/spack/spack/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 51ee3d7d8b..ceb1a499a5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1497,8 +1497,21 @@ def make_executable(path): os.chmod(path, mode) + class CMakePackage(StagedPackage): + def make_make(self): + import multiprocessing + # number of jobs spack will to build with. + jobs = multiprocessing.cpu_count() + if not self.parallel: + jobs = 1 + elif self.make_jobs: + jobs = self.make_jobs + + make = spack.build_environment.MakeExecutable('make', jobs) + return make + def configure_args(self): """Returns package-specific arguments to be provided to the configure command.""" return list() @@ -1576,10 +1589,12 @@ env = dict() cmake(self.source_directory, *options) def install_build(self): + make = self.make_make() with working_dir(self.build_directory, create=False): make() def install_install(self): + make = self.make_make() with working_dir(self.build_directory, create=False): make('install') -- cgit v1.2.3-70-g09d2 From 2243de9e2f6e81aaccdeef8d4ccfebd5caa2be04 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 4 May 2016 19:45:30 -0400 Subject: Make quiet mode default for spack spconfig --- lib/spack/spack/cmd/spconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py index 70db48e172..fcd9bdd63d 100644 --- a/lib/spack/spack/cmd/spconfig.py +++ b/lib/spack/spack/cmd/spconfig.py @@ -40,8 +40,8 @@ def setup_parser(subparser): '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="Do not try to install dependencies of requested packages.") subparser.add_argument( - '-q', '--quiet', action='store_true', dest='quiet', - help="Do not display verbose build output while installing.") + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -90,6 +90,6 @@ def spconfig(self, args): package.do_install( keep_prefix=True, # Don't remove install directory, even if you think you should ignore_deps=args.ignore_deps, - verbose=not args.quiet, + verbose=args.verbose, keep_stage=True, # don't remove source dir for SPCONFIG. install_phases = set(['spconfig', 'provenance'])) -- cgit v1.2.3-70-g09d2 From 6a48385111a383b2bb96f225e6eb1174376402dd Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Thu, 5 May 2016 17:47:41 -0400 Subject: Keep users environment in the spack setup script (spconfig.py). This is important to avoid breaking things that require module loads to work; for example, non-activate Python packages. --- lib/spack/spack/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ceb1a499a5..959e618005 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1552,7 +1552,8 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict() +#env = dict() +env = dict(os.environ) """ % sys.executable) env_vars = sorted(list(env.keys())) -- cgit v1.2.3-70-g09d2 From efa506b235523c8a9a925c1ecf91690266da4fd4 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Sat, 14 May 2016 17:09:11 -0400 Subject: Preparing spack setup command for merge. Try this out for a while... --- lib/spack/docs/packaging_guide.rst | 123 +++++++++++++++++++++++++++++++------ lib/spack/spack/cmd/setup.py | 91 +++++++++++++++++++++++++++ lib/spack/spack/cmd/spconfig.py | 95 ---------------------------- lib/spack/spack/package.py | 30 ++++----- 4 files changed, 209 insertions(+), 130 deletions(-) create mode 100644 lib/spack/spack/cmd/setup.py delete mode 100644 lib/spack/spack/cmd/spconfig.py (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index d8e7cdfa80..438d787a12 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -377,6 +377,8 @@ add a line like this in the package class: version('8.2.1', '4136d7b4c04df68b686570afa26988ac') ... +Versions should be listed with the newest version first. + Version URLs ~~~~~~~~~~~~~~~~~ @@ -385,8 +387,21 @@ in the package. For example, Spack is smart enough to download version ``8.2.1.`` of the ``Foo`` package above from ``http://example.com/foo-8.2.1.tar.gz``. -If spack *cannot* extrapolate the URL from the ``url`` field, or if -the package doesn't have a ``url`` field, you can add a URL explicitly +If spack *cannot* extrapolate the URL from the ``url`` field by +default, you can write your own URL generation algorithm in place of +the ``url`` declaration. For example: + +.. code-block:: python + :linenos: + + class Foo(Package): + def url_for_version(self, version): + return 'http://example.com/version_%s/foo-%s.tar.gz' \ + % (version, version) + version('8.2.1', '4136d7b4c04df68b686570afa26988ac') + ... + +If a URL cannot be derived systematically, you can add an explicit URL for a particular version: .. code-block:: python @@ -1216,6 +1231,19 @@ Now, the ``py-numpy`` package can be used as an argument to ``spack activate``. When it is activated, all the files in its prefix will be symbolically linked into the prefix of the python package. +Many packages produce Python extensions for *some* variants, but not +others: they should extend ``python`` only if the apropriate +variant(s) are selected. This may be accomplished with conditional +``extends()`` declarations: + +.. code-block:: python + + class FooLib(Package): + variant('python', default=True, description= \ + 'Build the Python extension Module') + extends('python', when='+python') + ... + Sometimes, certain files in one package will conflict with those in another, which means they cannot both be activated (symlinked) at the same time. In this case, you can tell Spack to ignore those files @@ -2392,6 +2420,59 @@ File functions .. _package-lifecycle: +Coding Style Guidelines +--------------------------- + +The following guidelines are provided, in the interests of making +Spack packages work in a consistent manner: + + +Variant Names +~~~~~~~~~~~~~~ + +Spack packages with variants similar to already-existing Spack +packages should use the same name for their variants. Standard +variant names are: + +======= ======== ======================== +Name Default Description +------- -------- ------------------------ +shared True Build shared libraries +static Build static libraries +mpi Use MPI +python Build Python extension +------- -------- ------------------------ + +If specified in this table, the corresponding default should be used +when declaring a variant. + + +Version Lists +~~~~~~~~~~~~~~ + +Spack packges should list supported versions with the newest first. + +Special Versions +~~~~~~~~~~~~~~~~~ + +The following *special* version names may be used when building a package: + +* *@system*: Indicates a hook to the OS-installed version of the + package. This is useful, for example, to tell Spack to use the + OS-installed version in ``packages.yaml``:: + + openssl: + paths: + openssl@system: /usr + buildable: False + + Certain Spack internals look for the *@system* version and do + appropriate things in that case. + +* *@local*: Indicates the version was built manually from some source + tree of unknown provenance (see ``spack setup``). + + Packaging workflow commands --------------------------------- @@ -2715,7 +2796,7 @@ Imagine a developer creating a CMake-based (or Autotools) project in a local directory, which depends on libraries A-Z. Once Spack has installed those dependencies, one would like to run ``cmake`` with appropriate command line and environment so CMake can find them. The ``spack -spconfig`` command does this conveniently, producing a CMake +setup`` command does this conveniently, producing a CMake configuration that is essentially the same as how Spack *would have* configured the project. This can be demonstrated with a usage example: @@ -2723,7 +2804,7 @@ example: .. code-block:: bash cd myproject - spack spconfig myproject@local + spack setup myproject@local mkdir build; cd build ../spconfig.py .. make @@ -2732,29 +2813,31 @@ example: Notes: * Spack must have ``myproject/package.py`` in its repository for this to work. - * ``spack spconfig`` produces the executable script ``spconfig.py`` - in the local directory, and also creates the module file for the - package. ``spconfig.py`` is normally run from the top level of - the source tree. - - * The version number given to ``spack spconfig`` is arbitrary (just - like ``spack diy``). ``myproject/package.py`` does not need to + * ``spack setup`` produces the executable script ``spconfig.py`` in + the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the user's + out-of-source build directory. + * The version number given to ``spack setup`` is arbitrary, just + like ``spack diy``. ``myproject/package.py`` does not need to have any valid downloadable versions listed (typical when a project is new). * spconfig.py produces a CMake configuration that *does not* use the Spack wrappers. Any resulting binaries *will not* use RPATH, unless the user has enabled it. This is recommended for development purposes, not production. - * spconfig.py is easily legible, and can serve as a developer + * ``spconfig.py`` is human readable, and can serve as a developer reference of what dependencies are being used. * ``make install`` installs the package into the Spack repository, where it may be used by other Spack packages. - * CMake-generated makefiles re-run CMake in some circumstances. Use of ``spconfig.py`` breaks this behavior, requiring the developer to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. + * CMake-generated makefiles re-run CMake in some circumstances. Use + of ``spconfig.py`` breaks this behavior, requiring the developer + to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file + has changed. CMakePackage ~~~~~~~~~~~~ -In order ot enable ``spack spconfig`` functionality, the author of +In order ot enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead of the standard ``Package`` superclass. Because CMake is standardized, the packager does not need to tell Spack how to run @@ -2784,18 +2867,18 @@ StagedPackage ``CMakePackage`` is implemented by subclassing the ``StagedPackage`` superclass, which breaks down the standard ``Package.install()`` -method into several sub-stages: ``spconfig``, ``configure``, ``build`` +method into several sub-stages: ``setup``, ``configure``, ``build`` and ``install``. Details: * Instead of implementing the standard ``install()`` method, package authors implement the methods for the sub-stages - ``install_spconfig()``, ``install_configure()``, + ``install_setup()``, ``install_configure()``, ``install_build()``, and ``install_install()``. * The ``spack install`` command runs the sub-stages ``configure``, - ``build`` and ``install`` in order. (The ``spconfig`` stage is + ``build`` and ``install`` in order. (The ``setup`` stage is not run by default; see below). -* The ``spack spconfig`` command runs the sub-stages ``spconfig`` +* The ``spack setup`` command runs the sub-stages ``setup`` and a dummy install (to create the module file). * The sub-stage install methods take no arguments (other than ``self``). The arguments ``spec`` and ``prefix`` to the standard @@ -2805,9 +2888,9 @@ and ``install``. Details: GNU Autotools ~~~~~~~~~~~~~ -The ``spconfig`` functionality is currently only available for +The ``setup`` functionality is currently only available for CMake-based packages. Extending this functionality to GNU Autotools-based packages would be easy (and should be done by a developer who actively uses Autotools). Packages that use -non-standard build systems can gain ``spconfig`` functionality by +non-standard build systems can gain ``setup`` functionality by subclassing ``StagedPackage`` directly. diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py new file mode 100644 index 0000000000..02e9bfd281 --- /dev/null +++ b/lib/spack/spack/cmd/setup.py @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Elizabeth Fischer +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +from spack.cmd.edit import edit_package +from spack.stage import DIYStage + +description = "Create a configuration script and module, but don't build." + +def setup_parser(subparser): + subparser.add_argument( + '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + help="Do not try to install dependencies of requested packages.") + subparser.add_argument( + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, + help="specs to use for install. Must contain package AND verison.") + + +def setup(self, args): + if not args.spec: + tty.die("spack setup requires a package spec argument.") + + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack setup only takes one spec.") + + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.repo.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, spack.repo.first_repo(), None, True) + return + + if not spec.versions.concrete: + tty.die("spack setup spec must have a single, concrete version. Did you forget a package version number?") + + spec.concretize() + package = spack.repo.get(spec) + + # It's OK if the package is already installed. + + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) + + # TODO: make this an argument, not a global. + spack.do_checksum = False + + package.do_install( + keep_prefix=True, # Don't remove install directory, even if you think you should + ignore_deps=args.ignore_deps, + verbose=args.verbose, + keep_stage=True, # don't remove source dir for SETUP. + install_phases = set(['setup', 'provenance'])) diff --git a/lib/spack/spack/cmd/spconfig.py b/lib/spack/spack/cmd/spconfig.py deleted file mode 100644 index fcd9bdd63d..0000000000 --- a/lib/spack/spack/cmd/spconfig.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Elizabeth Fischer -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import sys -import os -import argparse - -import llnl.util.tty as tty - -import spack -import spack.cmd -from spack.cmd.edit import edit_package -from spack.stage import DIYStage - -description = "Create a configuration script and module, but don't build." - -def setup_parser(subparser): - subparser.add_argument( - '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - help="Do not try to install dependencies of requested packages.") - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="Display verbose build output while installing.") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. Must contain package AND verison.") - - -def spconfig(self, args): - if not args.spec: - tty.die("spack spconfig requires a package spec argument.") - - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack spconfig only takes one spec.") - - # Take a write lock before checking for existence. - with spack.installed_db.write_transaction(): - spec = specs[0] - if not spack.repo.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, spack.repo.first_repo(), None, True) - return - - if not spec.versions.concrete: - tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") - - spec.concretize() - package = spack.repo.get(spec) - - # It's OK if the package is already installed. - #if package.installed: - # tty.error("Already installed in %s" % package.prefix) - # tty.msg("Uninstall or try adding a version suffix for this SPCONFIG build.") - # sys.exit(1) - - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) - - # TODO: make this an argument, not a global. - spack.do_checksum = False - - package.do_install( - keep_prefix=True, # Don't remove install directory, even if you think you should - ignore_deps=args.ignore_deps, - verbose=args.verbose, - keep_stage=True, # don't remove source dir for SPCONFIG. - install_phases = set(['spconfig', 'provenance'])) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 959e618005..19198c3b28 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -935,7 +935,7 @@ class Package(object): packages_dir = spack.install_layout.build_packages_path(self.spec) # Remove first if we're overwriting another build - # (can happen with spack spconfig) + # (can happen with spack setup) try: shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this except: @@ -1456,9 +1456,9 @@ def _hms(seconds): class StagedPackage(Package): """A Package subclass where the install() is split up into stages.""" - def install_spconfig(self): - """Creates an spconfig.py script to configure the package later if we like.""" - raise InstallError("Package %s provides no install_spconfig() method!" % self.name) + def install_setup(self): + """Creates an spack_setup.py script to configure the package later if we like.""" + raise InstallError("Package %s provides no install_setup() method!" % self.name) def install_configure(self): """Runs the configure process.""" @@ -1473,8 +1473,8 @@ class StagedPackage(Package): raise InstallError("Package %s provides no install_install() method!" % self.name) def install(self, spec, prefix): - if 'spconfig' in self.install_phases: - self.install_spconfig() + if 'setup' in self.install_phases: + self.install_setup() if 'configure' in self.install_phases: self.install_configure() @@ -1520,13 +1520,13 @@ class CMakePackage(StagedPackage): """Returns package-specific environment under which the configure command should be run.""" return dict() - def cmake_transitive_include_path(self): + def spack_transitive_include_path(self): return ';'.join( os.path.join(dep, 'include') for dep in os.environ['SPACK_DEPENDENCIES'].split(os.pathsep) ) - def install_spconfig(self): + def install_setup(self): cmd = [str(which('cmake'))] + \ spack.build_environment.get_std_cmake_args(self) + \ ['-DCMAKE_INSTALL_PREFIX=%s' % os.environ['SPACK_PREFIX'], @@ -1537,11 +1537,11 @@ class CMakePackage(StagedPackage): env = dict() env['PATH'] = os.environ['PATH'] - env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + env['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - spconfig_fname = 'spconfig.py' - with open(spconfig_fname, 'w') as fout: + setup_fname = 'spconfig.py' + with open(setup_fname, 'w') as fout: fout.write(\ r"""#!%s # @@ -1552,7 +1552,6 @@ import subprocess def cmdlist(str): return list(x.strip().replace("'",'') for x in str.split('\n') if x) -#env = dict() env = dict(os.environ) """ % sys.executable) @@ -1562,7 +1561,7 @@ env = dict(os.environ) if string.find(name, 'PATH') < 0: fout.write('env[%s] = %s\n' % (repr(name),repr(val))) else: - if name == 'CMAKE_TRANSITIVE_INCLUDE_PATH': + if name == 'SPACK_TRANSITIVE_INCLUDE_PATH': sep = ';' else: sep = ':' @@ -1572,20 +1571,21 @@ env = dict(os.environ) fout.write(' %s\n' % part) fout.write('"""))\n') + fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") fout.write('\ncmd = cmdlist("""\n') fout.write('%s\n' % cmd[0]) for arg in cmd[1:]: fout.write(' %s\n' % arg) fout.write('""") + sys.argv[1:]\n') fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - make_executable(spconfig_fname) + make_executable(setup_fname) def install_configure(self): cmake = which('cmake') with working_dir(self.build_directory, create=True): os.environ.update(self.configure_env()) - os.environ['CMAKE_TRANSITIVE_INCLUDE_PATH'] = self.cmake_transitive_include_path() + os.environ['SPACK_TRANSITIVE_INCLUDE_PATH'] = self.spack_transitive_include_path() options = self.configure_args() + spack.build_environment.get_std_cmake_args(self) cmake(self.source_directory, *options) -- cgit v1.2.3-70-g09d2 From 7fb45e4bfd64a8d78dcb42dcf7a3feb722ac047e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 27 Jun 2016 13:52:48 -0700 Subject: Update documentation for use on Cray Updating how to use compilers.yaml on Cray as well as setting up external packages. Also includes what needs to be set up for usage on Cray and explains what needs to be done to get Spack properly working. Also explain the architecture spec and what it does. --- lib/spack/docs/basic_usage.rst | 162 ++++++++++++++++++++++++--------------- lib/spack/docs/configuration.rst | 60 ++++++++++++--- 2 files changed, 152 insertions(+), 70 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index ec193e767d..199a7ef386 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -114,13 +114,13 @@ that the packages is installed: $ spack install mpileaks ==> Installing mpileaks - ==> mpich is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/mpich@3.0.4. - ==> callpath is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/callpath@1.0.2-5dce4318. - ==> adept-utils is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da. + ==> mpich is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpich@3.0.4. + ==> callpath is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/callpath@1.0.2-5dce4318. + ==> adept-utils is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/adept-utils@1.0-5adef8da. ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz ######################################################################## 100.0% - ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz - ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23. + ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-x86_64-debian7-59f6ad23/mpileaks-1.0.tar.gz + ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-x86_64-debian7-59f6ad23. ==> No patches needed for mpileaks. ==> Building mpileaks. @@ -128,7 +128,7 @@ that the packages is installed: ==> Successfully installed mpileaks. Fetch: 2.16s. Build: 9.82s. Total: 11.98s. - [+] /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/mpileaks@1.0-59f6ad23 + [+] /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpileaks@1.0-59f6ad23 The last line, with the ``[+]``, indicates where the package is installed. @@ -230,7 +230,7 @@ Running ``spack find`` with no arguments lists installed packages: $ spack find ==> 74 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 libdwarf@20130729 py-dateutil@2.4.0 adept-utils@1.0 libdwarf@20130729 py-ipython@2.3.1 atk@2.14.0 libelf@0.8.12 py-matplotlib@1.4.2 @@ -256,7 +256,7 @@ Running ``spack find`` with no arguments lists installed packages: lcms@2.6 pixman@0.32.6 xz@5.2.0 libdrm@2.4.33 py-dateutil@2.4.0 zlib@1.2.8 - -- chaos_5_x86_64_ib / gcc@4.9.2 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.9.2 -------------------------------- libelf@0.8.10 mpich@3.0.4 Packages are divided into groups according to their architecture and @@ -279,7 +279,7 @@ in more detail using ``spack find -d``, and by asking only to show $ spack find --deps libdwarf ==> 2 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 ^libelf@0.8.12 libdwarf@20130729-b52fac98 @@ -295,7 +295,7 @@ want to know whether two packages' dependencies differ, you can use $ spack find -l libdwarf ==> 2 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 Now the ``libwarf`` installs have hashes after their names. These are @@ -309,14 +309,14 @@ use ``spack find -p``: $ spack find -p ==> 74 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- - ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd - adept-utils@1.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da - atk@2.14.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/atk@2.14.0-3d09ac09 - boost@1.55.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/boost@1.55.0 - bzip2@1.0.6 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/bzip2@1.0.6 - cairo@1.14.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/cairo@1.14.0-fcc2ab44 - callpath@1.0.2 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/callpath@1.0.2-5dce4318 + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd + adept-utils@1.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/adept-utils@1.0-5adef8da + atk@2.14.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/atk@2.14.0-3d09ac09 + boost@1.55.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/boost@1.55.0 + bzip2@1.0.6 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/bzip2@1.0.6 + cairo@1.14.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/cairo@1.14.0-fcc2ab44 + callpath@1.0.2 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/callpath@1.0.2-5dce4318 ... And, finally, you can restrict your search to a particular package @@ -325,10 +325,10 @@ by supplying its name: .. code-block:: sh $ spack find -p libelf - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- - libelf@0.8.11 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.11 - libelf@0.8.12 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.12 - libelf@0.8.13 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.13 + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + libelf@0.8.11 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.11 + libelf@0.8.12 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.12 + libelf@0.8.13 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.13 ``spack find`` actually does a lot more than this. You can use *specs* to query for specific configurations and builds of each @@ -338,7 +338,7 @@ package. If you want to find only libelf versions greater than version .. code-block:: sh $ spack find libelf@0.8.12: - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- libelf@0.8.12 libelf@0.8.13 Finding just the versions of libdwarf built with a particular version @@ -348,7 +348,7 @@ of libelf would look like this: $ spack find -l libdwarf ^libelf@0.8.12 ==> 1 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. For example, @@ -460,19 +460,38 @@ editing your ``~/.spack/compilers.yaml`` file. You can do this by running Each compiler configuration in the file looks like this:: ... - chaos_5_x86_64_ib: - ... - intel@15.0.0: + compilers: + - compiler: + modules = [] + operating_system: OS + paths: cc: /usr/local/bin/icc-15.0.024-beta cxx: /usr/local/bin/icpc-15.0.024-beta f77: /usr/local/bin/ifort-15.0.024-beta fc: /usr/local/bin/ifort-15.0.024-beta - ... -The chaos_5_x86_64_ib string is an architecture string, and multiple -compilers can be listed underneath an architecture. The architecture -string may be replaced with the string 'all' to signify compilers that -work on all architectures. + spec: intel@15.0.0: + +If you're on a Cray system, the modules array will hold the names of the +compiler module as well as the corresponding PrgEnv. For example, on Edison +at NERSC the intel compiler looks just like this:: + ... + - compiler: + modules: + - PrEnv-intel + - intel/15.0.109 + ... + +The compiler paths will also look different on a Cray system. Since most +compilers are invoked using cc, CC and ftn, the paths for each compiler are +replaced with their respective Cray compiler wrapper names:: + ... + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + ... For compilers, like ``clang``, that do not support Fortran, put ``None`` for ``f77`` and ``fc``:: @@ -488,10 +507,11 @@ list displayed by ``spack compilers``. You can also add compiler flags to manually configured compilers. The valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, -``ldflags``, and ``ldlibs``. For example,:: +``ldflags``, and ``ldlibs``. For example:: ... - chaos_5_x86_64_ib: + compilers: + - compiler: ... intel@15.0.0: cc: /usr/local/bin/icc-15.0.024-beta @@ -546,8 +566,8 @@ More formally, a spec consists of the following pieces: boolean variants * ``name=`` Optional compiler flag specifiers. Valid flag names are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``. -* ``arch=`` Optional architecture specifier (``arch=bgq_os``) -* ``^`` Dependency specs (``^callpath@1.1``) +* ``target= os=`` Optional architecture specifier +(``target=haswell os=CNL10``) * ``^`` Dependency specs (``^callpath@1.1``) There are two things to notice here. The first is that specs are recursively defined. That is, each dependency after ``^`` is a spec @@ -626,7 +646,7 @@ compilers, variants, and architectures just like any other spec. Specifiers are associated with the nearest package name to their left. For example, above, ``@1.1`` and ``%gcc@4.7.2`` associates with the ``callpath`` package, while ``@1.2:1.4``, ``%gcc@4.7.5``, ``+debug``, -``-qt``, and ``arch=bgq_os`` all associate with the ``mpileaks`` package. +``-qt``, and ``target=haswell os=CNL10`` all associate with the ``mpileaks`` package. In the diagram above, ``mpileaks`` depends on ``mpich`` with an unspecified version, but packages can depend on other packages with @@ -758,14 +778,20 @@ in gnu autotools. If all flags are set, the order is Architecture specifiers ~~~~~~~~~~~~~~~~~~~~~~~ -.. Note:: - - Architecture specifiers are part of specs but are not yet - functional. They will be in Spack version 1.0, due in Q3 2015. - The architecture specifier looks identical to a variant specifier for a -non-boolean variant. The architecture can be specified only using the -reserved name ``arch`` (``arch=bgq_os``). +non-boolean variant. The architecture can be specified by using the reserved +words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). + +If you are on a Cray system, you can specify which target processor to +build with. For example, if you want to build against a compute node processor +with the compute node operating system, you would specify +``target=haswell os=CNL10``. Spack will then load the appropriate module for +the target. Additionally, Spack can also intepret the following values: +``be, backend, fe, frontend``. Backend is used for specifying the compute-node +processor and operating sytem, and frontend is used for login nodes. +If you decide to leave this field empty, Spack will use the +default architecture (compute nodes). The architecture spec is displayed as a +triplet of platform-target-operating_system. (``arch=linux-x86_64-debian7``) .. _sec-virtual-dependencies: @@ -985,7 +1011,7 @@ of installed packages. $ module avail - ------- /home/gamblin2/spack/share/spack/modules/chaos_5_x86_64_ib -------- + ------- /home/gamblin2/spack/share/spack/modules/linux-x86_64-debian7 -------- adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 @@ -1056,7 +1082,7 @@ Spack. For example, this will add the ``mpich`` package built with $ spack use mpich %gcc@4.4.7 Prepending: mpich@3.0.4%gcc@4.4.7 (ok) $ which mpicc - ~/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/mpich@3.0.4/bin/mpicc + ~/src/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpich@3.0.4/bin/mpicc Or, similarly with modules, you could type: @@ -1089,8 +1115,8 @@ than one installed package matches it), then Spack will warn you: $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7 arch=chaos_5_x86_64_ib - libelf@0.8.13%intel@15.0.0 arch=chaos_5_x86_64_ib + libelf@0.8.13%gcc@4.4.7 arch=linux-x86_64-debian7 + libelf@0.8.13%intel@15.0.0 arch=linux-x86_64-debian7 You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -1470,7 +1496,7 @@ an *extension*. Suppose you have Python installed like so: $ spack find python ==> 1 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- python@2.7.8 .. _spack-extensions: @@ -1483,7 +1509,7 @@ You can find extensions for your Python installation like this: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=linux-x86_64-debian7-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1495,7 +1521,7 @@ You can find extensions for your Python installation like this: py-h5py py-numpy py-pyqt py-shiboken ==> 12 installed: - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 @@ -1511,8 +1537,8 @@ prefixes, and you can see this with ``spack find -p``: $ spack find -p py-numpy ==> 1 installed packages. - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- - py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/py-numpy@1.9.1-66733244 + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/linux-x86_64-debian7/gcc@4.4.7/py-numpy@1.9.1-66733244 However, even though this package is installed, you cannot use it directly when you run ``python``: @@ -1573,9 +1599,9 @@ installation: .. code-block:: sh $ spack activate py-numpy - ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-x86_64-debian7-3c74eb69 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=linux-x86_64-debian7-5f70f816 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-x86_64-debian7-66733244 for python@2.7.8%gcc@4.4.7. Several things have happened here. The user requested that ``py-numpy`` be activated in the ``python`` installation it was built @@ -1590,7 +1616,7 @@ packages listed as activated: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=linux-x86_64-debian7-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1602,14 +1628,14 @@ packages listed as activated: py-h5py py-numpy py-pyqt py-shiboken ==> 12 installed: - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0 ==> 3 currently activated: - -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 @@ -1670,7 +1696,7 @@ Spack currently needs to be run from a filesystem that supports ``flock`` locking semantics. Nearly all local filesystems and recent versions of NFS support this, but parallel filesystems may be mounted without ``flock`` support enabled. You can determine how your -filesystems are mounted with ``mount -p``. The output for a Lustre + filesystems are mounted with ``mount -p``. The output for a Lustre filesystem might look like this: .. code-block:: sh @@ -1691,7 +1717,7 @@ This issue typically manifests with the error below: Traceback (most recent call last): File "./spack", line 176, in main() - File "./spack", line 154, in main + File "./spack", line 154,' in main return_val = command(parser, args) File "./spack/lib/spack/spack/cmd/find.py", line 170, in find specs = set(spack.installed_db.query(**q_args)) @@ -1709,6 +1735,20 @@ This issue typically manifests with the error below: A nicer error message is TBD in future versions of Spack. + +Spack on Cray +----------------------------- + +Spack is able to detect compilers through the module avail command. Once it +detects the compiler it writes the appropriate PrgEnv and compiler module +name to compilers.yaml and sets the paths to each compiler with Cray\'s +compiler wrapper names ie (cc, CC, ftn). During build time, Spack will +load the correct PrgEnv and compiler module and will call either cc, CC +or ftn. If you want to use default compilers for each PrgEnv and also be able +to link to cray external modules, you will need to set up a packages.yaml +:ref:`Exernal Packages` + + Getting Help ----------------------- diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index a6f876b2aa..b2e81e4334 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -70,20 +70,31 @@ directory. Here's an example of an external configuration: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in -the given directory. +the given directory. Packages.yaml can also be used to specify modules + +Here's an example of an external configuration for cray modules: + +.. code-block:: yaml + packages: + mpich: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich + + Each ``packages.yaml`` begins with a ``packages:`` token, followed -by a list of package names. To specify externals, add a ``paths`` +by a list of package names. To specify externals, add a ``paths`` or ``modules`` token under the package name, which lists externals in a -``spec : /path`` format. Each spec should be as +``spec: /path`` or ``spec: module-name`` format. Each spec should be as well-defined as reasonably possible. If a package lacks a spec component, such as missing a compiler or package version, then Spack will guess the missing component based @@ -108,9 +119,9 @@ be: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel buildable: False The addition of the ``buildable`` flag tells Spack that it should never build @@ -118,6 +129,9 @@ its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under a package name. +If an external module is specified as not buildable, then Spack will load the +external module into the build environment which can be used for linking. + The ``buildable`` does not need to be paired with external packages. It could also be used alone to forbid packages that may be buggy or otherwise undesirable. @@ -180,7 +194,35 @@ concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. +For Cray users, you can specify the default compiler that Spack will +concretize too. If you want to use the Cray defaults, then set them +under all: in packages.yaml. You can also specify concretization preferences +to cray-mpich. Just set the cray-mpich external module as a preferred package, +set buildable to False and set the preference of mpi to mpich. + +Here is an example of a full packages.yaml used at NERSC +.. code-block:: sh + packages: + mpich: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich + buildable: False + netcdf: + modules: + netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf + netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf + buildable: False + hdf5: + paths: + hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + buildable: False + all: + compiler: [gcc@5.2.0, intel@16.0.0.109] + providers: + mpi: [mpich, openmpi, intelmpi]ackages: Profiling ------------------ -- cgit v1.2.3-70-g09d2 From 26325fe812518f42978cb2471e4c74b095b66e15 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 27 Jun 2016 14:35:56 -0700 Subject: Add missing link to external packages Adding missing _sec-external_packages to link with external packages page --- lib/spack/docs/configuration.rst | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index b2e81e4334..22eff77655 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -53,6 +53,7 @@ in the first directory it finds to which it has write access. Add more elements to the list to indicate where your own site's temporary directory is. +.. _sec-external_packages: External Packages ---------------------------- -- cgit v1.2.3-70-g09d2 From 0384794912c6dac64a64181d591a55ea31ad7469 Mon Sep 17 00:00:00 2001 From: mwilliammyers Date: Mon, 27 Jun 2016 16:00:38 -0600 Subject: Fix #1019 - cannot create mirror for git spec --- lib/spack/spack/fetch_strategy.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 1953d7c1b3..afd293e4ec 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -307,9 +307,6 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise NoArchiveFileError("Cannot call archive() before fetching.") - if not extension(destination) == extension(self.archive_file): - raise ValueError("Cannot archive without matching extensions.") - shutil.move(self.archive_file, destination) @_needs_stage -- cgit v1.2.3-70-g09d2 From 6327877a6f9d2c30e4d644ec145a6f3f323ead81 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Thu, 30 Jun 2016 09:19:57 -0400 Subject: PEP-8 --- lib/spack/spack/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6f74f8ff10..20c9934704 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -176,8 +176,10 @@ sys_type = None # TODO: it's not clear where all the stuff that needs to be included in packages # should live. This file is overloaded for spack core vs. for packages. # -__all__ = ['Package', 'StagedPackage', 'CMakePackage', 'Version', 'when', 'ver'] -from spack.package import Package, StagedPackage, CMakePackage, ExtensionConflictError +__all__ = ['Package', 'StagedPackage', 'CMakePackage', \ + 'Version', 'when', 'ver'] +from spack.package import Package, ExtensionConflictError +from spack.package import StagedPackage, CMakePackage from spack.version import Version, ver from spack.multimethod import when -- cgit v1.2.3-70-g09d2 From d784d561fc334666f6f387abc4c7ac8beb391f8c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 30 Jun 2016 16:59:36 -0700 Subject: Add more documentation fix arch spec Added more documentation for Spack on Cray. Also fixed the architecture spec to be linux-debian7-x86_64 since it was reversed in the previous commit. --- lib/spack/docs/basic_usage.rst | 270 ++++++++++++++++++++++++++++------------- 1 file changed, 185 insertions(+), 85 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 199a7ef386..65688dec46 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -114,13 +114,13 @@ that the packages is installed: $ spack install mpileaks ==> Installing mpileaks - ==> mpich is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpich@3.0.4. - ==> callpath is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/callpath@1.0.2-5dce4318. - ==> adept-utils is already installed in /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/adept-utils@1.0-5adef8da. + ==> mpich is already installed in /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4. + ==> callpath is already installed in /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/callpath@1.0.2-5dce4318. + ==> adept-utils is already installed in /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/adept-utils@1.0-5adef8da. ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz ######################################################################## 100.0% - ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-x86_64-debian7-59f6ad23/mpileaks-1.0.tar.gz - ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-x86_64-debian7-59f6ad23. + ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-debian7-x86_64-59f6ad23/mpileaks-1.0.tar.gz + ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=linux-debian7-x86_64-59f6ad23. ==> No patches needed for mpileaks. ==> Building mpileaks. @@ -128,7 +128,7 @@ that the packages is installed: ==> Successfully installed mpileaks. Fetch: 2.16s. Build: 9.82s. Total: 11.98s. - [+] /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpileaks@1.0-59f6ad23 + [+] /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpileaks@1.0-59f6ad23 The last line, with the ``[+]``, indicates where the package is installed. @@ -230,7 +230,7 @@ Running ``spack find`` with no arguments lists installed packages: $ spack find ==> 74 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 libdwarf@20130729 py-dateutil@2.4.0 adept-utils@1.0 libdwarf@20130729 py-ipython@2.3.1 atk@2.14.0 libelf@0.8.12 py-matplotlib@1.4.2 @@ -256,7 +256,7 @@ Running ``spack find`` with no arguments lists installed packages: lcms@2.6 pixman@0.32.6 xz@5.2.0 libdrm@2.4.33 py-dateutil@2.4.0 zlib@1.2.8 - -- linux-x86_64-debian7 / gcc@4.9.2 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.9.2 -------------------------------- libelf@0.8.10 mpich@3.0.4 Packages are divided into groups according to their architecture and @@ -279,7 +279,7 @@ in more detail using ``spack find -d``, and by asking only to show $ spack find --deps libdwarf ==> 2 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 ^libelf@0.8.12 libdwarf@20130729-b52fac98 @@ -295,7 +295,7 @@ want to know whether two packages' dependencies differ, you can use $ spack find -l libdwarf ==> 2 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 Now the ``libwarf`` installs have hashes after their names. These are @@ -309,14 +309,14 @@ use ``spack find -p``: $ spack find -p ==> 74 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- - ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd - adept-utils@1.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/adept-utils@1.0-5adef8da - atk@2.14.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/atk@2.14.0-3d09ac09 - boost@1.55.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/boost@1.55.0 - bzip2@1.0.6 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/bzip2@1.0.6 - cairo@1.14.0 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/cairo@1.14.0-fcc2ab44 - callpath@1.0.2 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/callpath@1.0.2-5dce4318 + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- + ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd + adept-utils@1.0 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/adept-utils@1.0-5adef8da + atk@2.14.0 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/atk@2.14.0-3d09ac09 + boost@1.55.0 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/boost@1.55.0 + bzip2@1.0.6 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/bzip2@1.0.6 + cairo@1.14.0 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/cairo@1.14.0-fcc2ab44 + callpath@1.0.2 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/callpath@1.0.2-5dce4318 ... And, finally, you can restrict your search to a particular package @@ -325,10 +325,10 @@ by supplying its name: .. code-block:: sh $ spack find -p libelf - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- - libelf@0.8.11 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.11 - libelf@0.8.12 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.12 - libelf@0.8.13 /home/gamblin2/spack/opt/linux-x86_64-debian7/gcc@4.4.7/libelf@0.8.13 + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- + libelf@0.8.11 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.11 + libelf@0.8.12 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.12 + libelf@0.8.13 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.13 ``spack find`` actually does a lot more than this. You can use *specs* to query for specific configurations and builds of each @@ -338,7 +338,7 @@ package. If you want to find only libelf versions greater than version .. code-block:: sh $ spack find libelf@0.8.12: - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.12 libelf@0.8.13 Finding just the versions of libdwarf built with a particular version @@ -348,7 +348,7 @@ of libelf would look like this: $ spack find -l libdwarf ^libelf@0.8.12 ==> 1 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. For example, @@ -359,6 +359,7 @@ will find every installed package with a 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. +.. _compiler-config: Compiler configuration ----------------------------------- @@ -445,7 +446,7 @@ If you want to see specifics on a particular compiler, you can run fc = /usr/local/bin/ifort-15.0.090 This shows which C, C++, and Fortran compilers were detected by Spack. -Notice also that we didn't have to be too specific about the +Notice also that we didn\'t have to be too specific about the version. We just said ``intel@15``, and information about the only matching Intel compiler was displayed. @@ -472,27 +473,6 @@ Each compiler configuration in the file looks like this:: spec: intel@15.0.0: -If you're on a Cray system, the modules array will hold the names of the -compiler module as well as the corresponding PrgEnv. For example, on Edison -at NERSC the intel compiler looks just like this:: - ... - - compiler: - modules: - - PrEnv-intel - - intel/15.0.109 - ... - -The compiler paths will also look different on a Cray system. Since most -compilers are invoked using cc, CC and ftn, the paths for each compiler are -replaced with their respective Cray compiler wrapper names:: - ... - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - ... - For compilers, like ``clang``, that do not support Fortran, put ``None`` for ``f77`` and ``fc``:: @@ -538,10 +518,10 @@ Spack, that descriptor is called a *spec*. Spack uses specs to refer to a particular build configuration (or configurations) of a package. Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, -and dependency options for a build. In this section, we'll go over +and dependency options for a build. In this section, we\'ll go over the full syntax of specs. -Here is an example of a much longer spec than we've seen thus far:: +Here is an example of a much longer spec than we\'ve seen thus far:: mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2 @@ -778,20 +758,18 @@ in gnu autotools. If all flags are set, the order is Architecture specifiers ~~~~~~~~~~~~~~~~~~~~~~~ -The architecture specifier looks identical to a variant specifier for a -non-boolean variant. The architecture can be specified by using the reserved -words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). +The architecture can be specified by using the reserved +words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). You can also +use the triplet form of platform, operating system and processor. -If you are on a Cray system, you can specify which target processor to -build with. For example, if you want to build against a compute node processor -with the compute node operating system, you would specify -``target=haswell os=CNL10``. Spack will then load the appropriate module for -the target. Additionally, Spack can also intepret the following values: -``be, backend, fe, frontend``. Backend is used for specifying the compute-node -processor and operating sytem, and frontend is used for login nodes. -If you decide to leave this field empty, Spack will use the -default architecture (compute nodes). The architecture spec is displayed as a -triplet of platform-target-operating_system. (``arch=linux-x86_64-debian7``) +.. code-block:: sh + + spack install libelf arch=cray_xc-CNL10-haswell + +Users on non-Cray systems won't have to worry about specifying the architecture. +Spack will autodetect what kind of operating system is on your machine as well +as the processor. For more information on how the architecture can be +used on Cray machines, check here :ref:`spack-cray` .. _sec-virtual-dependencies: @@ -1011,7 +989,7 @@ of installed packages. $ module avail - ------- /home/gamblin2/spack/share/spack/modules/linux-x86_64-debian7 -------- + ------- /home/gamblin2/spack/share/spack/modules/linux-debian7-x86_64 -------- adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 @@ -1082,7 +1060,7 @@ Spack. For example, this will add the ``mpich`` package built with $ spack use mpich %gcc@4.4.7 Prepending: mpich@3.0.4%gcc@4.4.7 (ok) $ which mpicc - ~/src/spack/opt/linux-x86_64-debian7/gcc@4.4.7/mpich@3.0.4/bin/mpicc + ~/src/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc Or, similarly with modules, you could type: @@ -1115,8 +1093,8 @@ than one installed package matches it), then Spack will warn you: $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7 arch=linux-x86_64-debian7 - libelf@0.8.13%intel@15.0.0 arch=linux-x86_64-debian7 + libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 + libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -1496,7 +1474,7 @@ an *extension*. Suppose you have Python installed like so: $ spack find python ==> 1 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- python@2.7.8 .. _spack-extensions: @@ -1509,7 +1487,7 @@ You can find extensions for your Python installation like this: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7 arch=linux-x86_64-debian7-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1521,7 +1499,7 @@ You can find extensions for your Python installation like this: py-h5py py-numpy py-pyqt py-shiboken ==> 12 installed: - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 @@ -1537,8 +1515,8 @@ prefixes, and you can see this with ``spack find -p``: $ spack find -p py-numpy ==> 1 installed packages. - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- - py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/linux-x86_64-debian7/gcc@4.4.7/py-numpy@1.9.1-66733244 + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- + py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/linux-debian7-x86_64/gcc@4.4.7/py-numpy@1.9.1-66733244 However, even though this package is installed, you cannot use it directly when you run ``python``: @@ -1599,9 +1577,9 @@ installation: .. code-block:: sh $ spack activate py-numpy - ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-x86_64-debian7-3c74eb69 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=linux-x86_64-debian7-5f70f816 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-x86_64-debian7-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-debian7-x86_64-3c74eb69 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=linux-debian7-x86_64-5f70f816 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7. Several things have happened here. The user requested that ``py-numpy`` be activated in the ``python`` installation it was built @@ -1616,7 +1594,7 @@ packages listed as activated: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7 arch=linux-x86_64-debian7-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1628,14 +1606,14 @@ packages listed as activated: py-h5py py-numpy py-pyqt py-shiboken ==> 12 installed: - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0 ==> 3 currently activated: - -- linux-x86_64-debian7 / gcc@4.4.7 -------------------------------- + -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 @@ -1664,7 +1642,7 @@ dependencies, you can use ``spack activate -f``: .. code-block:: sh $ spack activate -f py-numpy - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7. .. _spack-deactivate: @@ -1736,18 +1714,140 @@ This issue typically manifests with the error below: A nicer error message is TBD in future versions of Spack. +.. _spack-cray: + Spack on Cray ----------------------------- -Spack is able to detect compilers through the module avail command. Once it -detects the compiler it writes the appropriate PrgEnv and compiler module -name to compilers.yaml and sets the paths to each compiler with Cray\'s -compiler wrapper names ie (cc, CC, ftn). During build time, Spack will -load the correct PrgEnv and compiler module and will call either cc, CC -or ftn. If you want to use default compilers for each PrgEnv and also be able -to link to cray external modules, you will need to set up a packages.yaml -:ref:`Exernal Packages` +Spack differs slightly when used on a Cray system. The architecture spec +can differentiate between the front-end and back-end processor and operating system. +For example, on Edison at NERSC, the back-end target processor +is \"Ivy Bridge\", so you can specify to use the back-end this way: + +.. code-block:: sh + + spack install zlib target=ivybridge + +You can also use the operating system to build against the back-end: + +.. code-block:: sh + + spack install zlib os=CNL10 + +Notice that the name includes both the operating system name and the major +version number concatenated together. + +Alternatively, if you want to build something for the front-end, +you can specify the front-end target processor. The processor for a login node +on Edison is \"Sandy bridge\" so we specify on the command line like so: + +.. code-block:: sh + + spack install zlib target=sandybridge + +And the front-end operating system is: + +.. code-block:: sh + + spack install zlib os=SuSE11 + + + +Cray compiler detection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack can detect compilers using two methods. For the front-end, we treat +everything the same. The difference lies in back-end compiler detection. +Back-end compiler detection is made via the Tcl module avail command. +Once it detects the compiler it writes the appropriate PrgEnv and compiler +module name to compilers.yaml and sets the paths to each compiler with Cray\'s +compiler wrapper names (i.e. cc, CC, ftn). During build time, Spack will load +the correct PrgEnv and compiler module and will call appropriate wrapper. + +The compilers.yaml config file will also differ. There is a +modules section that is filled with the compiler\'s Programming Environment +and module name. On other systems, this field is empty []:: + + ... + - compiler: + modules: + - PrgEnv-intel + - intel/15.0.109 + ... + +As mentioned earlier, the compiler paths will look different on a Cray system. +Since most compilers are invoked using cc, CC and ftn, the paths for each +compiler are replaced with their respective Cray compiler wrapper names:: + + ... + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + ... + +As opposed to an explicit path to the compiler executable. This allows Spack +to call the Cray compiler wrappers during build time. + +For more on compiler configuration, check out :ref:`compiler-config`. + +Setting defaults and using Cray modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want to use default compilers for each PrgEnv and also be able +to load cray external modules, you will need to set up a packages.yaml. + +Here\'s an example of an external configuration for cray modules: + +.. code-block:: yaml + + packages: + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich + +This tells Spack that for whatever package that depends on mpi, load the +cray-mpich module into the environment. You can then be able to use whatever +environment variables, libraries, etc, that are brought into the environment +via module load. + +You can set the default compiler that Spack can use for each compiler type. +If you want to use the Cray defaults, then set them under *all:* in packages.yaml. +In the compiler field, set the compiler specs in your order of preference. +Whenever you build with that compiler type, Spack will concretize to that version. + +Here is an example of a full packages.yaml used at NERSC + +.. code-block:: yaml + + packages: + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich + buildable: False + netcdf: + modules: + netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf + netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf + buildable: False + hdf5: + modules: + hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + buildable: False + all: + compiler: [gcc@5.2.0, intel@16.0.0.109] + +Here we tell spack that whenever we want to build with gcc use version 5.2.0 or +if we want to build with intel compilers, use version 16.0.0.109. We add a spec +for each compiler type for each cray modules. This ensures that for each +compiler on our system we can use that external module. + +For more on external packages check out the section :ref:`sec-external_packages`. Getting Help ----------------------- -- cgit v1.2.3-70-g09d2 From 16a4c49f98237193bf1612342e6bcdc1e6cdaf74 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 30 Jun 2016 17:01:18 -0700 Subject: Remove Cray examples of config files Removed examples of Cray config files and added them under Spack on Cray. Any users of Cray can just look at the Spack on Cray section to get an idea of what's used. If they want more information they can click links to the sections. --- lib/spack/docs/configuration.rst | 40 ---------------------------------------- 1 file changed, 40 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 22eff77655..f2ffa07264 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -81,17 +81,6 @@ If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in the given directory. Packages.yaml can also be used to specify modules -Here's an example of an external configuration for cray modules: - -.. code-block:: yaml - packages: - mpich: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich - - - Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` or ``modules`` token under the package name, which lists externals in a @@ -195,35 +184,6 @@ concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. -For Cray users, you can specify the default compiler that Spack will -concretize too. If you want to use the Cray defaults, then set them -under all: in packages.yaml. You can also specify concretization preferences -to cray-mpich. Just set the cray-mpich external module as a preferred package, -set buildable to False and set the preference of mpi to mpich. - -Here is an example of a full packages.yaml used at NERSC - -.. code-block:: sh - packages: - mpich: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich - buildable: False - netcdf: - modules: - netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf - netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf - buildable: False - hdf5: - paths: - hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - buildable: False - all: - compiler: [gcc@5.2.0, intel@16.0.0.109] - providers: - mpi: [mpich, openmpi, intelmpi]ackages: Profiling ------------------ -- cgit v1.2.3-70-g09d2 From 56adb5d9a5262c5d2582d4610b2396e0a8053001 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 30 Jun 2016 17:03:07 -0700 Subject: Change arch spec and fix typo Changed old arch specs to new arch specs. Changed intallation to installation --- lib/spack/docs/packaging_guide.rst | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 54b886310a..c09e8524b0 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1625,21 +1625,21 @@ the user runs ``spack install`` and the time the ``install()`` method is called. The concretized version of the spec above might look like this:: - mpileaks@2.3%gcc@4.7.3 arch=linux-ppc64 - ^callpath@1.0%gcc@4.7.3+debug arch=linux-ppc64 - ^dyninst@8.1.2%gcc@4.7.3 arch=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 - ^libelf@0.8.11%gcc@4.7.3 arch=linux-ppc64 - ^mpich@3.0.4%gcc@4.7.3 arch=linux-ppc64 + mpileaks@2.3%gcc@4.7.3 arch=linux-debian7-x86_64 + ^callpath@1.0%gcc@4.7.3+debug arch=linux-debian7-x86_64 + ^dyninst@8.1.2%gcc@4.7.3 arch=linux-debian7-x86_64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-debian7-x86_64 + ^libelf@0.8.11%gcc@4.7.3 arch=linux-debian7-x86_64 + ^mpich@3.0.4%gcc@4.7.3 arch=linux-debian7-x86_64 .. graphviz:: digraph { - "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" - "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" - "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-debian7-x86_64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-debian7-x86_64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-debian7-x86_64" -> "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-debian7-x86_64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-debian7-x86_64" + "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-debian7-x86_64" -> "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-debian7-x86_64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-debian7-x86_64" -> "libdwarf@20130729\n%gcc@4.7.3\n arch=linux-debian7-x86_64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-debian7-x86_64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-debian7-x86_64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-debian7-x86_64" } Here, all versions, compilers, and platforms are filled in, and there @@ -1668,9 +1668,9 @@ running ``spack spec``. For example: ^libdwarf ^libelf - dyninst@8.0.1%gcc@4.7.3 arch=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 - ^libelf@0.8.13%gcc@4.7.3 arch=linux-ppc64 + dyninst@8.0.1%gcc@4.7.3 arch=linux-debian7-x86_64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-debian7-x86_64 + ^libelf@0.8.13%gcc@4.7.3 arch=linux-debian7-x86_64 This is useful when you want to know exactly what Spack will do when you ask for a particular spec. @@ -2175,12 +2175,12 @@ example: def install(self, prefix): # Do default install - @when('arch=chaos_5_x86_64_ib') + @when('arch=linux-debian7-x86_64') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('arch=bgqos_0") + @when('arch=linux-debian7-x86_64") def install(self, prefix): # This will be executed if the package's sys_type is bgqos_0 @@ -2308,7 +2308,7 @@ build system. .. _sanity-checks: -Sanity checking an intallation +Sanity checking an installation -------------------------------- By default, Spack assumes that a build has failed if nothing is @@ -2770,11 +2770,11 @@ build it: $ spack stage libelf ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz ######################################################################## 100.0% - ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13.tar.gz - ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64. + ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-debian7-x86_64/libelf-0.8.13.tar.gz + ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-debian7-x86_64. $ spack cd libelf $ pwd - /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13 + /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-debian7-x86_64/libelf-0.8.13 ``spack cd`` here changed he current working directory to the directory containing the expanded ``libelf`` source code. There are a -- cgit v1.2.3-70-g09d2 From 31e9ded768b56d076eb452b56d1b5c94ac341761 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Fri, 1 Jul 2016 20:59:14 -0500 Subject: Remove more variables from build environment Remove the LIBRARY_PATH and CPATH environment variables before building a package. --- lib/spack/spack/build_environment.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 99c4cae020..7220539886 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -266,6 +266,8 @@ def set_build_environment_variables(pkg, env): # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') + env.unset('LIBRARY_PATH') + env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') -- cgit v1.2.3-70-g09d2 From 690937f9533765f6b5c2b815d9c4798f15456918 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 4 Jul 2016 22:58:01 -0700 Subject: Add `--dirty` option to `spack install`. - Allow install to be run without cleaning the environment. --- lib/spack/spack/build_environment.py | 35 +++++++++++++++++++++-------------- lib/spack/spack/cmd/install.py | 4 ++++ lib/spack/spack/package.py | 12 +++++++----- 3 files changed, 32 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7220539886..ce0b91b718 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -224,9 +224,12 @@ def set_compiler_environment_variables(pkg, env): return env -def set_build_environment_variables(pkg, env): +def set_build_environment_variables(pkg, env, dirty=False): """ - This ensures a clean install environment when we build packages + This ensures a clean install environment when we build packages. + + Arguments: + dirty -- skip unsetting the user's environment settings. """ # Add spack build environment path with compiler wrappers first in # the path. We add both spack.env_path, which includes default @@ -262,14 +265,17 @@ def set_build_environment_variables(pkg, env): # Install root prefix env.set(SPACK_INSTALL, spack.install_path) - # Remove these vars from the environment during build because they - # can affect how some packages find libraries. We want to make - # sure that builds never pull in unintended external dependencies. - env.unset('LD_LIBRARY_PATH') - env.unset('LIBRARY_PATH') - env.unset('CPATH') - env.unset('LD_RUN_PATH') - env.unset('DYLD_LIBRARY_PATH') + # Stuff in here sanitizes the build environemnt to eliminate + # anything the user has set that may interfere. + if not dirty: + # Remove these vars from the environment during build because they + # can affect how some packages find libraries. We want to make + # sure that builds never pull in unintended external dependencies. + env.unset('LD_LIBRARY_PATH') + env.unset('LIBRARY_PATH') + env.unset('CPATH') + env.unset('LD_RUN_PATH') + env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. bin_dirs = reversed( @@ -407,7 +413,7 @@ def load_external_modules(pkg): load_module(dep.external_module) -def setup_package(pkg): +def setup_package(pkg, dirty=False): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() @@ -430,7 +436,7 @@ def setup_package(pkg): s.package.spec = s set_compiler_environment_variables(pkg, spack_env) - set_build_environment_variables(pkg, spack_env) + set_build_environment_variables(pkg, spack_env, dirty) load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec @@ -459,7 +465,7 @@ def setup_package(pkg): spack_env.apply_modifications() -def fork(pkg, function): +def fork(pkg, function, dirty=False): """Fork a child process to do part of a spack build. Arguments: @@ -467,6 +473,7 @@ def fork(pkg, function): pkg -- pkg whose environemnt we should set up the forked process for. function -- arg-less function to run in the child process. + dirty -- If True, do NOT clean the environment before building. Usage: def child_fun(): @@ -490,7 +497,7 @@ def fork(pkg, function): if pid == 0: # Give the child process the package's build environment. - setup_package(pkg) + setup_package(pkg, dirty=dirty) try: # call the forked function. diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 9d3175786b..3133e080d7 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -53,6 +53,9 @@ def setup_parser(subparser): subparser.add_argument( '--fake', action='store_true', dest='fake', help="Fake install. Just remove the prefix and touch a fake file in it.") + subparser.add_argument( + '--dirty', action='store_true', dest='dirty', + help="Install a package *without* cleaning the environment.") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") @@ -79,4 +82,5 @@ def install(parser, args): make_jobs=args.jobs, verbose=args.verbose, fake=args.fake, + dirty=args.dirty, explicit=True) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 53c521b776..84bd99df54 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -755,7 +755,7 @@ class Package(object): self.stage.check() self.stage.cache_local() - + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball @@ -883,6 +883,7 @@ class Package(object): make_jobs=None, fake=False, explicit=False, + dirty=False, install_phases = install_phases): """Called by commands to install a package and its dependencies. @@ -899,6 +900,7 @@ class Package(object): fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. verbose -- Display verbose build output (by default, suppresses it) + dirty -- Don't clean the build environment before installing. make_jobs -- Number of make jobs to use for install. Default is ncpus """ if not self.spec.concrete: @@ -1037,7 +1039,7 @@ class Package(object): pass try: - spack.build_environment.fork(self, build_process) + spack.build_environment.fork(self, build_process, dirty=dirty) except: # remove the install prefix if anything went wrong during install. if not keep_prefix: @@ -1527,15 +1529,15 @@ class StagedPackage(Package): raise InstallError("Package %s provides no install_setup() method!" % self.name) def install_configure(self): - """Runs the configure process.""" + """Runs the configure process.""" raise InstallError("Package %s provides no install_configure() method!" % self.name) def install_build(self): - """Runs the build process.""" + """Runs the build process.""" raise InstallError("Package %s provides no install_build() method!" % self.name) def install_install(self): - """Runs the install process.""" + """Runs the install process.""" raise InstallError("Package %s provides no install_install() method!" % self.name) def install(self, spec, prefix): -- cgit v1.2.3-70-g09d2 From d687e332ad41cd7c84bf0007f8c331e56470da18 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 4 Jul 2016 22:59:02 -0700 Subject: Fix compile bugs for gcc on Mac OS X with macports. - add macports to things that are cleaned out of the environment. - linker incompatibilities cause issues with packages like OpenSSL. - also clean up NOQA stuff in OpenSSL --- lib/spack/spack/build_environment.py | 9 ++++++ .../repos/builtin/packages/openssl/package.py | 36 ++++++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index ce0b91b718..fe5186a7d7 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -277,6 +277,15 @@ def set_build_environment_variables(pkg, env, dirty=False): env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') + # Remove any macports installs from the PATH. The macports ld can + # cause conflicts with the built-in linker on el capitan. Solves + # assembler issues, e.g.: + # suffix or operands invalid for `movq'" + path = get_path('PATH') + for p in path: + if '/macports/' in p: + env.remove_path('PATH', p) + # Add bin directories from dependencies to the PATH for the build. bin_dirs = reversed( filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index a0747f229d..e7c105d5f5 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -63,7 +63,8 @@ class Openssl(Package): # case return a fake url and exit openssl_url = '@system (reserved version for system openssl)' if not warnings_given_to_user.get(version, False): - tty.msg('Using openssl@system : the version @system is reserved for system openssl') # NOQA: ignore=E501 + tty.msg('Using openssl@system: ' + 'the version @system is reserved for system openssl') warnings_given_to_user[version] = True else: openssl_url = self.check_for_outdated_release( @@ -87,37 +88,46 @@ class Openssl(Package): openssl_url = latest.format(version=version) urllib.urlopen(openssl_url) except IOError: - openssl_url = older.format(version_number=version_number, version_full=version) # NOQA:ignore=E501 + openssl_url = older.format( + version_number=version_number, version_full=version) # Checks if we already warned the user for this particular # version of OpenSSL. If not we display a warning message # and mark this version if not warnings_given_to_user.get(version, False): - tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') # NOQA: ignore=E501 - tty.warn('Consider updating to the latest version of this package.') # NOQA: ignore=E501 - tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) # NOQA: ignore=E501 + tty.warn( + 'This installation depends on an old version of OpenSSL, ' + 'which may have known security issues. ') + tty.warn( + 'Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format( + homepage=Openssl.homepage)) warnings_given_to_user[version] = True return openssl_url + def install(self, spec, prefix): # OpenSSL uses a variable APPS in its Makefile. If it happens to be set # in the environment, then this will override what is set in the # Makefile, leading to build errors. env.pop('APPS', None) - if spec.satisfies("target=x86_64") or spec.satisfies("target=ppc64"): + + if spec.satisfies('target=x86_64') or spec.satisfies('target=ppc64'): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) env['KERNEL_BITS'] = '64' - config = Executable("./config") - config("--prefix=%s" % prefix, - "--openssldir=%s" % join_path(prefix, 'etc', 'openssl'), - "zlib", - "no-krb5", - "shared") + + options = ['zlib', 'no-krb5', 'shared'] + + config = Executable('./config') + config('--prefix=%s' % prefix, + '--openssldir=%s' % join_path(prefix, 'etc', 'openssl'), + *options) + # Remove non-standard compiler options if present. These options are # present e.g. on Darwin. They are non-standard, i.e. most compilers # (e.g. gcc) will not accept them. filter_file(r'-arch x86_64', '', 'Makefile') make() - make("install") + make('install') -- cgit v1.2.3-70-g09d2 From 1f64f08cb33b2b0160e7b1666f1c0d129919aecb Mon Sep 17 00:00:00 2001 From: William Killian Date: Tue, 5 Jul 2016 13:31:40 -0700 Subject: (bugfix) fixed typo of spec --> raw_spec --- lib/spack/spack/cmd/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 3cefb512c2..70da689b67 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -108,7 +108,7 @@ def module_find(mtype, flags, spec_array): tty.die("No installed packages match spec %s" % raw_spec) if len(specs) > 1: - tty.error("Multiple matches for spec %s. Choose one:" % spec) + tty.error("Multiple matches for spec %s. Choose one:" % raw_spec) for s in specs: sys.stderr.write(s.tree(color=True)) sys.exit(1) -- cgit v1.2.3-70-g09d2 From 32e086f44adb88bfe11fe73a9c4b69a1dff862cf Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Jul 2016 15:24:10 +0200 Subject: version : modified __repr__ to return a string representation of the object + unit tests --- lib/spack/spack/test/versions.py | 8 ++++++++ lib/spack/spack/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 4624f901c8..5f7d57b4e4 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -389,3 +389,11 @@ class VersionsTest(unittest.TestCase): self.assertEqual(v.dotted, '1.2.3') self.assertEqual(v.dashed, '1-2-3') self.assertEqual(v.underscored, '1_2_3') + + def test_repr_and_str(self): + a = Version('1.2.3') + self.assertEqual(repr(a), 'Version(\'1.2.3\')') + b = eval(repr(a)) + self.assertEqual(a, b) + self.assertEqual(str(a), '1.2.3') + self.assertEqual(str(a), str(b)) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 858d581472..6f95e8a2f3 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -197,7 +197,7 @@ class Version(object): return tuple(self.version[idx]) def __repr__(self): - return self.string + return 'Version(' + repr(self.string) + ')' def __str__(self): return self.string -- cgit v1.2.3-70-g09d2 From 780a57367dd41f77d45f3e229c0a018c7695821f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Jul 2016 17:00:48 +0200 Subject: version : modified __getitem__ to return either an item or a Version instance + unit tests --- bin/spack | 2 +- lib/spack/spack/test/versions.py | 40 ++++++++++++++++++++++++++++++++++------ lib/spack/spack/version.py | 17 ++++++++++++++++- 3 files changed, 51 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index e9307d1485..9b1276a866 100755 --- a/bin/spack +++ b/bin/spack @@ -77,7 +77,7 @@ import llnl.util.tty as tty from llnl.util.tty.color import * import spack from spack.error import SpackError -import argparse +from external import argparse # Command parsing parser = argparse.ArgumentParser( diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 5f7d57b4e4..c051895ca3 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -391,9 +391,37 @@ class VersionsTest(unittest.TestCase): self.assertEqual(v.underscored, '1_2_3') def test_repr_and_str(self): - a = Version('1.2.3') - self.assertEqual(repr(a), 'Version(\'1.2.3\')') - b = eval(repr(a)) - self.assertEqual(a, b) - self.assertEqual(str(a), '1.2.3') - self.assertEqual(str(a), str(b)) + + def check_repr_and_str(vrs): + a = Version(vrs) + self.assertEqual(repr(a), 'Version(\'' + vrs + '\')') + b = eval(repr(a)) + self.assertEqual(a, b) + self.assertEqual(str(a), vrs) + self.assertEqual(str(a), str(b)) + + check_repr_and_str('1.2.3') + check_repr_and_str('R2016a') + check_repr_and_str('R2016a.2-3_4') + + def test_get_item(self): + a = Version('0.1_2-3') + self.assertTrue(isinstance(a[1], int)) + # Test slicing + b = a[0:2] + self.assertTrue(isinstance(b, Version)) + self.assertEqual(b, Version('0.1')) + self.assertEqual(repr(b), 'Version(\'0.1\')') + self.assertEqual(str(b), '0.1') + b = a[0:3] + self.assertTrue(isinstance(b, Version)) + self.assertEqual(b, Version('0.1_2')) + self.assertEqual(repr(b), 'Version(\'0.1_2\')') + self.assertEqual(str(b), '0.1_2') + b = a[1:] + self.assertTrue(isinstance(b, Version)) + self.assertEqual(b, Version('1_2-3')) + self.assertEqual(repr(b), 'Version(\'1_2-3\')') + self.assertEqual(str(b), '1_2-3') + # Raise TypeError on tuples + self.assertRaises(TypeError, b.__getitem__, 1, 2) \ No newline at end of file diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 6f95e8a2f3..7e5878383d 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -44,6 +44,7 @@ be called on any of the types:: concrete """ import re +import numbers from bisect import bisect_left from functools import wraps @@ -194,7 +195,21 @@ class Version(object): return iter(self.version) def __getitem__(self, idx): - return tuple(self.version[idx]) + cls = type(self) + if isinstance(idx, numbers.Integral): + return self.version[idx] + elif isinstance(idx, slice): + # Currently len(self.separators) == len(self.version) - 1 + extendend_separators = self.separators + ('',) + string_arg = [] + for token, separator in zip(self.version, extendend_separators)[idx]: + string_arg.append(str(token)) + string_arg.append(str(separator)) + string_arg.pop() # We don't need the last separator + string_arg = ''.join(string_arg) + return cls(string_arg) + message = '{cls.__name__} indices must be integers' + raise TypeError(message.format(cls=cls)) def __repr__(self): return 'Version(' + repr(self.string) + ')' -- cgit v1.2.3-70-g09d2 From 9b926a480f84c62cf4ed23c26b950dd7c89fb1ad Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Jul 2016 17:03:32 +0200 Subject: qa : flake8 issues --- bin/spack | 2 +- lib/spack/spack/test/versions.py | 2 +- lib/spack/spack/version.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index 9b1276a866..e9307d1485 100755 --- a/bin/spack +++ b/bin/spack @@ -77,7 +77,7 @@ import llnl.util.tty as tty from llnl.util.tty.color import * import spack from spack.error import SpackError -from external import argparse +import argparse # Command parsing parser = argparse.ArgumentParser( diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index c051895ca3..a3a328fb14 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -424,4 +424,4 @@ class VersionsTest(unittest.TestCase): self.assertEqual(repr(b), 'Version(\'1_2-3\')') self.assertEqual(str(b), '1_2-3') # Raise TypeError on tuples - self.assertRaises(TypeError, b.__getitem__, 1, 2) \ No newline at end of file + self.assertRaises(TypeError, b.__getitem__, 1, 2) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 7e5878383d..6839643941 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -202,9 +202,9 @@ class Version(object): # Currently len(self.separators) == len(self.version) - 1 extendend_separators = self.separators + ('',) string_arg = [] - for token, separator in zip(self.version, extendend_separators)[idx]: + for token, sep in zip(self.version, extendend_separators)[idx]: string_arg.append(str(token)) - string_arg.append(str(separator)) + string_arg.append(str(sep)) string_arg.pop() # We don't need the last separator string_arg = ''.join(string_arg) return cls(string_arg) -- cgit v1.2.3-70-g09d2 From 081918d71a9d58108b8617d8324fd51542161a1e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 6 Jul 2016 22:45:30 +0200 Subject: add --run-tests argument for install() --- lib/spack/spack/cmd/install.py | 5 +++++ lib/spack/spack/package.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 9d3175786b..fadfa7f7c3 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -55,6 +55,10 @@ def setup_parser(subparser): help="Fake install. Just remove the prefix and touch a fake file in it.") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") + subparser.add_argument( + '--run-tests', action='store_true', dest='run_tests', + help="Run tests during installation of a package.") + def install(parser, args): @@ -77,6 +81,7 @@ def install(parser, args): keep_stage=args.keep_stage, ignore_deps=args.ignore_deps, make_jobs=args.jobs, + run_tests=args.run_tests, verbose=args.verbose, fake=args.fake, explicit=True) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 53c521b776..bce6af9c02 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -311,6 +311,8 @@ class Package(object): parallel = True """# jobs to use for parallel make. If set, overrides default of ncpus.""" make_jobs = None + """By default do not run tests within package's install()""" + run_tests = False """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False """List of prefix-relative file paths (or a single path). If these do @@ -755,7 +757,7 @@ class Package(object): self.stage.check() self.stage.cache_local() - + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball @@ -881,6 +883,7 @@ class Package(object): skip_patch=False, verbose=False, make_jobs=None, + run_tests=False, fake=False, explicit=False, install_phases = install_phases): @@ -900,6 +903,7 @@ class Package(object): skip_patch -- Skip patch stage of build if True. verbose -- Display verbose build output (by default, suppresses it) make_jobs -- Number of make jobs to use for install. Default is ncpus + run_tests -- Runn tests within the package's install() """ if not self.spec.concrete: raise ValueError("Can only install concrete packages.") @@ -930,7 +934,11 @@ class Package(object): fake=fake, skip_patch=skip_patch, verbose=verbose, - make_jobs=make_jobs) + make_jobs=make_jobs, + run_tests=run_tests) + + # Set run_tests flag before starting build. + self.run_tests = run_tests # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1527,15 +1535,15 @@ class StagedPackage(Package): raise InstallError("Package %s provides no install_setup() method!" % self.name) def install_configure(self): - """Runs the configure process.""" + """Runs the configure process.""" raise InstallError("Package %s provides no install_configure() method!" % self.name) def install_build(self): - """Runs the build process.""" + """Runs the build process.""" raise InstallError("Package %s provides no install_build() method!" % self.name) def install_install(self): - """Runs the install process.""" + """Runs the install process.""" raise InstallError("Package %s provides no install_install() method!" % self.name) def install(self, spec, prefix): -- cgit v1.2.3-70-g09d2 From 663b30b3e89f2025a2a4746a61efd98416aeecea Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jul 2016 12:02:08 -0700 Subject: Update target autodetection for linux platform --- lib/spack/spack/platforms/linux.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 4d8adac384..4d3f59c320 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,16 +1,23 @@ import subprocess +import platform from spack.architecture import Platform, Target from spack.operating_systems.linux_distro import LinuxDistro class Linux(Platform): priority = 90 - front_end = 'x86_64' - back_end = 'x86_64' - default = 'x86_64' def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default)) + self.add_target('x86_64', Target('x86_64')) + self.add_target('ppc64le', Target('ppc64le')) + + self.default = platform.machine() + self.front_end = platform.machine() + self.back_end = platform.machine() + + if self.default not in self.targets: + self.add_target(self.default, Target(self.default)) + linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.front_os = self.default_os -- cgit v1.2.3-70-g09d2 From 403a55afb2655045e9d7752df3d2dec7c1bbf395 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jul 2016 13:33:37 -0700 Subject: Minor bug fix --- lib/spack/spack/test/mock_packages_test.py | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e1acc32cb7..c8b06cd7d7 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -82,15 +82,6 @@ compilers: f77: None fc: None modules: 'None' -- compiler: - spec: clang@3.3 - operating_system: redhat6 - paths: - cc: /path/to/clang - cxx: /path/to/clang++ - f77: None - fc: None - modules: 'None' - compiler: spec: clang@3.3 operating_system: yosemite @@ -118,15 +109,6 @@ compilers: operating_system: SuSE11 spec: gcc@4.5.0 modules: 'None' -- compiler: - paths: - cc: /path/to/gcc - cxx: /path/to/g++ - f77: /path/to/gfortran - fc: /path/to/gfortran - operating_system: redhat6 - spec: gcc@4.5.0 - modules: 'None' - compiler: paths: cc: /path/to/gcc -- cgit v1.2.3-70-g09d2 From 1b9e8a0e65e927c74ea44a881c932a33cfb1745b Mon Sep 17 00:00:00 2001 From: George Hartzell Date: Fri, 8 Jul 2016 11:01:08 -0700 Subject: Fix typo Comamnds -> Commands --- lib/spack/docs/basic_usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 8fad96010e..dbad0a9f6d 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1154,7 +1154,7 @@ Modules may be loaded recursively with the command: More than one spec may be placed on the command line here. -Module Comamnds for Shell Scripts +Module Commands for Shell Scripts `````````````````````````````````` Although Spack is flexbile, the ``module`` command is much faster. -- cgit v1.2.3-70-g09d2 From 33d0660abc19d33c6e1b5fa21f97f9292a2008d7 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Fri, 8 Jul 2016 15:11:37 -0500 Subject: Have ``spack create`` default to lower case name This will create package directories with lower case names when using `spack create`. The documentation was modified to reflect this. --- lib/spack/docs/packaging_guide.rst | 32 +++++++++++++++++++------------- lib/spack/spack/cmd/create.py | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 6cea4da14f..ae83dd8089 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -36,10 +36,11 @@ Creating & editing packages ``spack create`` ~~~~~~~~~~~~~~~~~~~~~ -The ``spack create`` command generates a boilerplate package template -from a URL. The URL should point to a tarball or other software -archive. In most cases, ``spack create`` plus a few modifications is -all you need to get a package working. +The ``spack create`` command creates a directory with the package name and +generates a ``package.py`` file with a boilerplate package template from a URL. +The URL should point to a tarball or other software archive. In most cases, +``spack create`` plus a few modifications is all you need to get a package +working. Here's an example: @@ -47,12 +48,16 @@ Here's an example: $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz -Spack examines the tarball URL and tries to figure out the name of the -package to be created. It also tries to determine what version strings -look like for this package. Using this information, it will try to -find *additional* versions by spidering the package's webpage. If it -finds multiple versions, Spack prompts you to tell it how many -versions you want to download and checksum: +Spack examines the tarball URL and tries to figure out the name of the package +to be created. Once the name is determined a directory in the appropriate +repository is created with that name. Spack prefers, but does not require, that +names be lower case so the directory name will be lower case when ``spack +create`` generates it. In cases where it is desired to have mixed case or upper +case simply rename the directory. Spack also tries to determine what version +strings look like for this package. Using this information, it will try to find +*additional* versions by spidering the package's webpage. If it finds multiple +versions, Spack prompts you to tell it how many versions you want to download +and checksum: .. code-block:: sh @@ -297,9 +302,10 @@ directories or files (like patches) that it needs to build. Package Names ~~~~~~~~~~~~~~~~~~ -Packages are named after the directory containing ``package.py``. So, -``libelf``'s ``package.py`` lives in a directory called ``libelf``. -The ``package.py`` file defines a class called ``Libelf``, which +Packages are named after the directory containing ``package.py``. It is +preferred, but not required, that the directory, and thus the package name, are +lower case. So, ``libelf``'s ``package.py`` lives in a directory called +``libelf``. The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. for example, here is ``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index f5f234d7a7..18c748e483 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -325,7 +325,7 @@ def create(parser, args): # Figure out a name and repo for the package. name, version = guess_name_and_version(url, args) spec = Spec(name) - name = spec.name # factors out namespace, if any + name = spec.name.lower() # factors out namespace, if any repo = find_repository(spec, args) tty.msg("This looks like a URL for %s version %s" % (name, version)) -- cgit v1.2.3-70-g09d2 From b29d5e603209fe869dc5737d7d403c6f9536597f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 7 Jul 2016 19:18:10 +0200 Subject: add special treatment of develop version --- lib/spack/spack/concretize.py | 6 +++++- lib/spack/spack/test/versions.py | 3 +++ lib/spack/spack/version.py | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f792008c49..d9992a5680 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -166,7 +166,11 @@ class DefaultConcretizer(object): valid_versions.sort(key=prefer_key, reverse=True) if valid_versions: - spec.versions = ver([valid_versions[0]]) + # Disregard @develop and take the next valid version + if ver(valid_versions[0]) == ver('develop') and len(valid_versions) > 1: + spec.versions = ver([valid_versions[1]]) + else: + spec.versions = ver([valid_versions[0]]) else: # We don't know of any SAFE versions that match the given # spec. Grab the spec's versions and grab the highest diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index a3a328fb14..41d72e7c34 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -92,6 +92,9 @@ class VersionsTest(unittest.TestCase): self.assert_ver_eq('1.0', '1.0') self.assert_ver_lt('1.0', '2.0') self.assert_ver_gt('2.0', '1.0') + self.assert_ver_eq('develop', 'develop') + self.assert_ver_lt('1.0', 'develop') + self.assert_ver_gt('develop', '1.0') def test_three_segments(self): self.assert_ver_eq('2.0.1', '2.0.1') diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 6839643941..6f6c83b3d8 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -236,6 +236,14 @@ class Version(object): if self.version == other.version: return False + # dev is __gt__ than anything but itself. + if other.string == 'develop': + return True + + # If lhs is dev then it can't be < than anything + if self.string == 'develop': + return False + for a, b in zip(self.version, other.version): if a == b: continue -- cgit v1.2.3-70-g09d2 From 1682257b35b0c1089a723fac7996063e9c3d2729 Mon Sep 17 00:00:00 2001 From: George Hartzell Date: Mon, 11 Jul 2016 09:42:49 -0700 Subject: Typo: verison -> version --- lib/spack/spack/cmd/diy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 39faf59a17..643e6374b2 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -50,7 +50,7 @@ def setup_parser(subparser): help="Do not display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. Must contain package AND verison.") + help="specs to use for install. Must contain package AND version.") def diy(self, args): -- cgit v1.2.3-70-g09d2 From 306e601d4e0e86090d8876ad50ddd6f6fcc768b7 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 11 Jul 2016 19:50:33 +0200 Subject: document special treatment of development version --- lib/spack/docs/packaging_guide.rst | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 72559e47ce..e616581ca4 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -568,7 +568,7 @@ The package author is responsible for coming up with a sensible name for each version to be fetched from a repository. For example, if you're fetching from a tag like ``v1.0``, you might call that ``1.0``. If you're fetching a nameless git commit or an older subversion -revision, you might give the commit an intuitive name, like ``dev`` +revision, you might give the commit an intuitive name, like ``develop`` for a development version, or ``some-fancy-new-feature`` if you want to be more specific. @@ -578,6 +578,17 @@ branches move forward over time and you aren't guaranteed to get the same thing every time you fetch a particular version. Life isn't always simple, though, so this is not strictly enforced. +When fetching from from the branch corresponding to the development version +(often called ``master``,``trunk`` or ``dev``), it is recommended to +call this version ``develop``. Spack has special treatment for this version so + that ``@develop`` will satisfy dependencies like +``depends_on(abc, when="@x.y.z:")``. In other words, ``@develop`` is +greater than any other version. The rationale is that certain features or +options first appear in the development branch. Therefore if a package author +wants to keep the package on the bleeding edge and provide support for new +features, it is advised to use ``develop`` for such a version which will +greatly simplify writing dependencies and version-related conditionals. + In some future release, Spack may support extrapolating repository versions as it does for tarball URLs, but currently this is not supported. @@ -603,7 +614,7 @@ Default branch class Example(Package): ... - version('dev', git='https://github.com/example-project/example.git') + version('develop', git='https://github.com/example-project/example.git') This is not recommended, as the contents of the default branch change over time. @@ -676,7 +687,7 @@ Default .. code-block:: python - version('hg-head', hg='https://jay.grs.rwth-aachen.de/hg/example') + version('develop', hg='https://jay.grs.rwth-aachen.de/hg/example') Note that this is not recommended; try to fetch a particular revision instead. @@ -708,7 +719,7 @@ Fetching the head .. code-block:: python - version('svn-head', svn='https://outreach.scidac.gov/svn/libmonitor/trunk') + version('develop', svn='https://outreach.scidac.gov/svn/libmonitor/trunk') This is not recommended, as the head will move forward over time. @@ -718,7 +729,7 @@ Fetching a revision .. code-block:: python - version('svn-head', svn='https://outreach.scidac.gov/svn/libmonitor/trunk', + version('develop', svn='https://outreach.scidac.gov/svn/libmonitor/trunk', revision=128) Subversion branches are handled as part of the directory structure, so @@ -2712,15 +2723,15 @@ build process will start from scratch. ``spack purge`` ~~~~~~~~~~~~~~~~~ -Cleans up all of Spack's temporary and cached files. This can be used to -recover disk space if temporary files from interrupted or failed installs -accumulate in the staging area. +Cleans up all of Spack's temporary and cached files. This can be used to +recover disk space if temporary files from interrupted or failed installs +accumulate in the staging area. When called with ``--stage`` or ``--all`` (or without arguments, in which case -the default is ``--all``) this removes all staged files; this is equivalent to +the default is ``--all``) this removes all staged files; this is equivalent to running ``spack clean`` for every package you have fetched or staged. -When called with ``--cache`` or ``--all`` this will clear all resources +When called with ``--cache`` or ``--all`` this will clear all resources :ref:`cached ` during installs. Keeping the stage directory on success -- cgit v1.2.3-70-g09d2