diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 9 | ||||
| -rw-r--r-- | lib/spack/docs/case_studies.rst | 167 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 1 | ||||
| -rw-r--r-- | lib/spack/docs/packaging_guide.rst | 5 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/info.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/compilers/clang.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/modules.py | 57 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/test/operating_system.py | 75 |
9 files changed, 231 insertions, 93 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index a42d941791..a5478d10c2 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -6,6 +6,15 @@ Basic usage The ``spack`` command has many *subcommands*. You'll only need a small subset of them for typical usage. +Note that Spack colorizes output. ``less -R`` should be used with +Spack to maintian this colorization. Eg:: + + spack find | less -R + +It is recommend that the following be put in your ``.bashrc`` file:: + + alias less='less -R' + Listing available packages ------------------------------ diff --git a/lib/spack/docs/case_studies.rst b/lib/spack/docs/case_studies.rst new file mode 100644 index 0000000000..bcd754fdcd --- /dev/null +++ b/lib/spack/docs/case_studies.rst @@ -0,0 +1,167 @@ +Using Spack for CMake-based Development +========================================== + +These are instructions on how to use Spack to aid in the development +of a CMake-based project. Spack is used to help find the dependencies +for the project, configure it at development time, and then package it +it in a way that others can install. Using Spack for CMake-based +development consists of three parts: + +1. Setting up the CMake build in your software +2. Writing the Spack Package +3. Using it from Spack. + + +Setting Up the CMake Build +--------------------------------------- + +You should follow standard CMake conventions in setting up your +software, your CMake build should NOT depend on or require Spack to +build. See here for an example: + https://github.com/citibeth/icebin + +Note that there's one exception here to the rule I mentioned above. +In ``CMakeLists.txt``, I have the following line:: + + include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) + + +This is a hook into Spack, and it ensures that all transitive +dependencies are included in the include path. It's not needed if +everything is in one tree, but it is (sometimes) in the Spack world; +when running without Spack, it has no effect. + +Note that this "feature" is controversial, could break with future +versions of GNU ld, and probably not the best to use. The best +practice is that you make sure that anything you #include is listed as +a dependency in your CMakeLists.txt. + +To be more specific: if you #inlcude something from package A and an +installed HEADER FILE in A #includes something from package B, then +you should also list B as a dependency in your CMake build. If you +depend on A but header files exported by A do NOT #include things from +B, then you do NOT need to list B as a dependency --- even if linking +to A links in libB.so as well. + +I also recommend that you set up your CMake build to use RPATHs +correctly. Not only is this a good idea and nice, but it also ensures +that your package will build the same with or without ``spack +install``. + +Writing the Spack Package +--------------------------------------- + +Now that you have a CMake build, you want to tell Spack how to +configure it. This is done by writing a Spack package for your +software. See here for example: + https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py + +You need to subclass ``CMakePackage``, as is done in this example. +This enables advanced features of Spack for helping you in configuring +your software (keep reading...). Instead of an ``install()`` method +used when subclassing ``Package``, you write ``configure_args()``. +See here for more info on how this works: + https://github.com/LLNL/spack/pull/543/files + +NOTE: if your software is not publicly available, you do not need to +set the URL or version. Or you can set up bogus URLs and +versions... whatever causes Spack to not crash. + + +Using it from Spack +-------------------------------- + +Now that you have a Spack package, you can get Spack to setup your +CMake project for you. Use the following to setup, configure and +build your project:: + + cd myproject + spack spconfig myproject@local + mkdir build; cd build + ../spconfig.py .. + make + make install + + +Everything here should look pretty familiar here from a CMake +perspective, except that ``spack spconfig`` creates the file +``spconfig.py``, which calls CMake with arguments appropriate for your +Spack configuration. Think of it as the equivalent to running a bunch +of ``spack location -i`` commands. You will run ``spconfig.py`` +instead of running CMake directly. + +If your project is publicly available (eg on GitHub), then you can +ALSO use this setup to "just install" a release version without going +through the manual configuration/build step. Just do: + +1. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. + +2. Set the ``url`` in your ``package.py`` to download a tarball for + the appropriate version. (GitHub will give you a tarball for any + version in the repo, if you tickle it the right way). For example:: + + https://github.com/citibeth/icebin/tarball/v0.1.0 + + Set up versions as appropriate in your ``package.py``. (Manually + download the tarball and run ``md5sum`` to determine the + appropriate checksum for it). + +3. Now you should be able to say ``spack install myproject@version`` + and things "just work." + +NOTE... in order to use the features outlined in this post, you +currently need to use the following branch of Spack: + https://github.com/citibeth/spack/tree/efischer/develop + +There is a pull request open on this branch ( +https://github.com/LLNL/spack/pull/543 ) and we are working to get it +integrated into the main ``develop`` branch. + + +Activating your Software +------------------------------------- + +Once you've built your software, you will want to load it up. You can +use ``spack load mypackage@local`` for that in your ``.bashrc``, but +that is slow. Try stuff like the following instead: + +The following command will load the Spack-installed packages needed +for basic Python use of IceBin:: + + module load `spack module find tcl icebin netcdf cmake@3.5.1` + module load `spack module find --dependencies tcl py-basemap py-giss` + + +You can speed up shell startup by turning these into ``module load`` commands. + +1. Cut-n-paste the script ``make_spackenv``:: + + #!/bin/sh + # + # Generate commands to load the Spack environment + + SPACKENV=$HOME/spackenv.sh + + spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 >$SPACKENV + spack module find --dependencies --shell tcl py-basemap py-giss >>$SPACKENV + +2. Add the following to your ``.bashrc`` file:: + + source $HOME/spackenv.sh + # Preferentially use your checked-out Python source + export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH + +3. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). + + +Giving Back +------------------- + +If your software is publicly available, you should submit the +``package.py`` for it as a pull request to the main Spack GitHub +project. This will ensure that anyone can install your software +(almost) painlessly with a simple ``spack install`` command. See here +for how that has turned into detailed instructions that have +successfully enabled collaborators to install complex software: + + https://github.com/citibeth/icebin/blob/develop/README.rst diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 98ed9ff0fe..a5bbd4e23b 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -49,6 +49,7 @@ Table of Contents mirrors configuration developer_guide + case_studies command_index package_list API Docs <spack> diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index a082b85efa..879beb2476 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -401,10 +401,11 @@ the ``url`` declaration. For example: :linenos: class Foo(Package): + version('8.2.1', '4136d7b4c04df68b686570afa26988ac') + ... def url_for_version(self, version): return 'http://example.com/version_%s/foo-%s.tar.gz' \ % (version, version) - version('8.2.1', '4136d7b4c04df68b686570afa26988ac') ... If a URL cannot be derived systematically, you can add an explicit URL @@ -433,7 +434,7 @@ executables and other custom archive types), you can add .. code-block:: python version('8.2.1', '4136d7b4c04df68b686570afa26988ac', - url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False') + url='http://example.com/foo-8.2.1-special-version.tar.gz', expand=False) When ``expand`` is set to ``False``, Spack sets the current working directory to the directory containing the downloaded archive before it diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 498518057b..2fa3a07525 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -87,7 +87,7 @@ def print_text_info(pkg): for deptype in ('build', 'link', 'run'): print print "%s Dependencies:" % deptype.capitalize() - deps = pkg.dependencies_of_type(deptype) + deps = sorted(pkg.dependencies_of_type(deptype)) if deps: colify(deps, indent=4) else: diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 4cf65222ae..36c91f6670 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -101,7 +101,7 @@ class Clang(Compiler): ver = match.group(1) + '-apple' else: # Normal clang compiler versions are left as-is - match = re.search(r'^clang version ([^ )]+)', output) + match = re.search(r'clang version ([^ )]+)', output) if match: ver = match.group(1) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index debc6752b4..70c3c35d8c 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -530,13 +530,6 @@ class Dotkit(EnvModule): class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") - environment_modifications_formats = { - PrependPath: 'prepend-path --delim "{separator}" {name} \"{value}\"\n', - AppendPath: 'append-path --delim "{separator}" {name} \"{value}\"\n', - RemovePath: 'remove-path --delim "{separator}" {name} \"{value}\"\n', - SetEnv: 'setenv {name} \"{value}\"\n', - UnsetEnv: 'unsetenv {name}\n' - } autoload_format = ('if ![ is-loaded {module_file} ] {{\n' ' puts stderr "Autoloading {module_file}"\n' @@ -556,11 +549,13 @@ class TclModule(EnvModule): def header(self): timestamp = datetime.datetime.now() # TCL Modulefile header - header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp - header += '##\n' - header += '## %s\n' % self.spec.short_spec - header += '##\n' + header = """\ +#%%Module1.0 +## Module file created by spack (https://github.com/LLNL/spack) on %s +## +## %s +## +""" % (timestamp, self.spec.short_spec) # TODO : category ? # Short description @@ -575,6 +570,44 @@ class TclModule(EnvModule): header += '}\n\n' return header + def process_environment_command(self, env): + environment_modifications_formats_colon = { + PrependPath: 'prepend-path {name} \"{value}\"\n', + AppendPath: 'append-path {name} \"{value}\"\n', + RemovePath: 'remove-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' + } + environment_modifications_formats_general = { + PrependPath: + 'prepend-path --delim "{separator}" {name} \"{value}\"\n', + AppendPath: + 'append-path --delim "{separator}" {name} \"{value}\"\n', + RemovePath: + 'remove-path --delim "{separator}" {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' + } + for command in env: + # Token expansion from configuration file + name = command.args.get('name', '').format(**self.upper_tokens) + value = str(command.args.get('value', '')).format(**self.tokens) + command.update_args(name=name, value=value) + # Format the line int the module file + try: + if command.args.get('separator', ':') == ':': + yield environment_modifications_formats_colon[type( + command)].format(**command.args) + else: + yield environment_modifications_formats_general[type( + command)].format(**command.args) + except KeyError: + message = ('Cannot handle command of type {command}: ' + 'skipping request') + details = '{context} at {filename}:{lineno}' + tty.warn(message.format(command=type(command))) + tty.warn(details.format(**command.args)) + def module_specific_content(self, configuration): naming_tokens = self.tokens # Conflict diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ff8c8e96bc..8c1204402a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -143,8 +143,10 @@ class Package(object): informational URL, so that users know what they're installing. - url - URL of the source archive that spack will fetch. + url or url_for_version(self, version) + If url, then the URL of the source archive that spack will fetch. + If url_for_version(), then a method returning the URL required + to fetch a particular version. install() This function tells spack how to build and install the diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py deleted file mode 100644 index 8723f7244d..0000000000 --- a/lib/spack/spack/test/operating_system.py +++ /dev/null @@ -1,75 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -""" Test checks if the operating_system class is created correctly and that -the functions are using the correct operating_system. Also checks whether -the operating_system correctly uses the compiler_strategy -""" -import unittest -from spack.platforms.cray_xc import CrayXc -from spack.platforms.linux import Linux -from spack.platforms.darwin import Darwin -from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import ComputeNodeLinux - - -class TestOperatingSystem(unittest.TestCase): - - def setUp(self): - cray_xc = CrayXc() - linux = Linux() - darwin = Darwin() - self.cray_operating_sys = cray_xc.operating_system('front_os') - self.cray_default_os = cray_xc.operating_system('default_os') - self.cray_back_os = cray_xc.operating_system('back_os') - self.darwin_operating_sys = darwin.operating_system('default_os') - self.linux_operating_sys = linux.operating_system('default_os') - - def test_cray_front_end_operating_system(self): - self.assertIsInstance(self.cray_operating_sys, LinuxDistro) - - def test_cray_front_end_compiler_strategy(self): - self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") - - def test_cray_back_end_operating_system(self): - self.assertIsInstance(self.cray_back_os, ComputeNodeLinux) - - def test_cray_back_end_compiler_strategy(self): - self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") - - def test_linux_operating_system(self): - self.assertIsInstance(self.linux_operating_sys, LinuxDistro) - - def test_linux_compiler_strategy(self): - self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - - def test_cray_front_end_compiler_list(self): - """ Operating systems will now be in charge of finding compilers. - So, depending on which operating system you want to build for - or which operating system you are on, then you could detect - compilers in a certain way. Cray linux environment on the front - end is just a regular linux distro whereas the Cray linux compute - node is a stripped down version which modules are important - """ - self.assertEquals(True, False) |