diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 9 | ||||
| -rw-r--r-- | lib/spack/docs/case_studies.rst | 167 | ||||
| -rw-r--r-- | lib/spack/docs/conf.py | 3 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 1 | ||||
| -rw-r--r-- | lib/spack/docs/packaging_guide.rst | 24 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/info.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/md5.py | 5 | ||||
| -rw-r--r-- | lib/spack/spack/modules.py | 57 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/util/web.py | 2 |
10 files changed, 251 insertions, 25 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index a42d941791..a5478d10c2 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -6,6 +6,15 @@ Basic usage The ``spack`` command has many *subcommands*. You'll only need a small subset of them for typical usage. +Note that Spack colorizes output. ``less -R`` should be used with +Spack to maintian this colorization. Eg:: + + spack find | less -R + +It is recommend that the following be put in your ``.bashrc`` file:: + + alias less='less -R' + Listing available packages ------------------------------ diff --git a/lib/spack/docs/case_studies.rst b/lib/spack/docs/case_studies.rst new file mode 100644 index 0000000000..bcd754fdcd --- /dev/null +++ b/lib/spack/docs/case_studies.rst @@ -0,0 +1,167 @@ +Using Spack for CMake-based Development +========================================== + +These are instructions on how to use Spack to aid in the development +of a CMake-based project. Spack is used to help find the dependencies +for the project, configure it at development time, and then package it +it in a way that others can install. Using Spack for CMake-based +development consists of three parts: + +1. Setting up the CMake build in your software +2. Writing the Spack Package +3. Using it from Spack. + + +Setting Up the CMake Build +--------------------------------------- + +You should follow standard CMake conventions in setting up your +software, your CMake build should NOT depend on or require Spack to +build. See here for an example: + https://github.com/citibeth/icebin + +Note that there's one exception here to the rule I mentioned above. +In ``CMakeLists.txt``, I have the following line:: + + include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) + + +This is a hook into Spack, and it ensures that all transitive +dependencies are included in the include path. It's not needed if +everything is in one tree, but it is (sometimes) in the Spack world; +when running without Spack, it has no effect. + +Note that this "feature" is controversial, could break with future +versions of GNU ld, and probably not the best to use. The best +practice is that you make sure that anything you #include is listed as +a dependency in your CMakeLists.txt. + +To be more specific: if you #inlcude something from package A and an +installed HEADER FILE in A #includes something from package B, then +you should also list B as a dependency in your CMake build. If you +depend on A but header files exported by A do NOT #include things from +B, then you do NOT need to list B as a dependency --- even if linking +to A links in libB.so as well. + +I also recommend that you set up your CMake build to use RPATHs +correctly. Not only is this a good idea and nice, but it also ensures +that your package will build the same with or without ``spack +install``. + +Writing the Spack Package +--------------------------------------- + +Now that you have a CMake build, you want to tell Spack how to +configure it. This is done by writing a Spack package for your +software. See here for example: + https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py + +You need to subclass ``CMakePackage``, as is done in this example. +This enables advanced features of Spack for helping you in configuring +your software (keep reading...). Instead of an ``install()`` method +used when subclassing ``Package``, you write ``configure_args()``. +See here for more info on how this works: + https://github.com/LLNL/spack/pull/543/files + +NOTE: if your software is not publicly available, you do not need to +set the URL or version. Or you can set up bogus URLs and +versions... whatever causes Spack to not crash. + + +Using it from Spack +-------------------------------- + +Now that you have a Spack package, you can get Spack to setup your +CMake project for you. Use the following to setup, configure and +build your project:: + + cd myproject + spack spconfig myproject@local + mkdir build; cd build + ../spconfig.py .. + make + make install + + +Everything here should look pretty familiar here from a CMake +perspective, except that ``spack spconfig`` creates the file +``spconfig.py``, which calls CMake with arguments appropriate for your +Spack configuration. Think of it as the equivalent to running a bunch +of ``spack location -i`` commands. You will run ``spconfig.py`` +instead of running CMake directly. + +If your project is publicly available (eg on GitHub), then you can +ALSO use this setup to "just install" a release version without going +through the manual configuration/build step. Just do: + +1. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. + +2. Set the ``url`` in your ``package.py`` to download a tarball for + the appropriate version. (GitHub will give you a tarball for any + version in the repo, if you tickle it the right way). For example:: + + https://github.com/citibeth/icebin/tarball/v0.1.0 + + Set up versions as appropriate in your ``package.py``. (Manually + download the tarball and run ``md5sum`` to determine the + appropriate checksum for it). + +3. Now you should be able to say ``spack install myproject@version`` + and things "just work." + +NOTE... in order to use the features outlined in this post, you +currently need to use the following branch of Spack: + https://github.com/citibeth/spack/tree/efischer/develop + +There is a pull request open on this branch ( +https://github.com/LLNL/spack/pull/543 ) and we are working to get it +integrated into the main ``develop`` branch. + + +Activating your Software +------------------------------------- + +Once you've built your software, you will want to load it up. You can +use ``spack load mypackage@local`` for that in your ``.bashrc``, but +that is slow. Try stuff like the following instead: + +The following command will load the Spack-installed packages needed +for basic Python use of IceBin:: + + module load `spack module find tcl icebin netcdf cmake@3.5.1` + module load `spack module find --dependencies tcl py-basemap py-giss` + + +You can speed up shell startup by turning these into ``module load`` commands. + +1. Cut-n-paste the script ``make_spackenv``:: + + #!/bin/sh + # + # Generate commands to load the Spack environment + + SPACKENV=$HOME/spackenv.sh + + spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 >$SPACKENV + spack module find --dependencies --shell tcl py-basemap py-giss >>$SPACKENV + +2. Add the following to your ``.bashrc`` file:: + + source $HOME/spackenv.sh + # Preferentially use your checked-out Python source + export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH + +3. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). + + +Giving Back +------------------- + +If your software is publicly available, you should submit the +``package.py`` for it as a pull request to the main Spack GitHub +project. This will ensure that anyone can install your software +(almost) painlessly with a simple ``spack install`` command. See here +for how that has turned into detailed instructions that have +successfully enabled collaborators to install complex software: + + https://github.com/citibeth/icebin/blob/develop/README.rst diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 3d2a8251aa..f3cb268177 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -51,7 +51,8 @@ os.environ['SPACK_ROOT'] = spack_root os.environ['PATH'] += os.pathsep + '$SPACK_ROOT/bin' spack_version = subprocess.Popen( - ['spack', '-V'], stderr=subprocess.PIPE).communicate()[1].strip().split('.') + [spack_root + '/bin/spack', '-V'], + stderr=subprocess.PIPE).communicate()[1].strip().split('.') # Set an environment variable so that colify will print output like it would to # a terminal. diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 98ed9ff0fe..a5bbd4e23b 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -49,6 +49,7 @@ Table of Contents mirrors configuration developer_guide + case_studies command_index package_list API Docs <spack> diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 70def5c39a..3dc83f9b92 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -401,10 +401,11 @@ the ``url`` declaration. For example: :linenos: class Foo(Package): + version('8.2.1', '4136d7b4c04df68b686570afa26988ac') + ... def url_for_version(self, version): return 'http://example.com/version_%s/foo-%s.tar.gz' \ % (version, version) - version('8.2.1', '4136d7b4c04df68b686570afa26988ac') ... If a URL cannot be derived systematically, you can add an explicit URL @@ -467,14 +468,25 @@ to use based on the hash length. ``spack md5`` ^^^^^^^^^^^^^^^^^^^^^^ -If you have a single file to checksum, you can use the ``spack md5`` -command to do it. Here's how you might download an archive and get a -checksum for it: +If you have one or more files to checksum, you can use the ``spack md5`` +command to do it: .. code-block:: sh - $ curl -O http://exmaple.com/foo-8.2.1.tar.gz' - $ spack md5 foo-8.2.1.tar.gz + $ spack md5 foo-8.2.1.tar.gz foo-8.2.2.tar.gz + ==> 2 MD5 checksums: + 4136d7b4c04df68b686570afa26988ac foo-8.2.1.tar.gz + 1586b70a49dfe05da5fcc29ef239dce0 foo-8.2.2.tar.gz + +``spack md5`` also accepts one or more URLs and automatically downloads +the files for you: + +.. code-block:: sh + + $ spack md5 http://example.com/foo-8.2.1.tar.gz + ==> Trying to fetch from http://example.com/foo-8.2.1.tar.gz + ######################################################################## 100.0% + ==> 1 MD5 checksum: 4136d7b4c04df68b686570afa26988ac foo-8.2.1.tar.gz Doing this for lots of files, or whenever a new package version is diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 498518057b..2fa3a07525 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -87,7 +87,7 @@ def print_text_info(pkg): for deptype in ('build', 'link', 'run'): print print "%s Dependencies:" % deptype.capitalize() - deps = pkg.dependencies_of_type(deptype) + deps = sorted(pkg.dependencies_of_type(deptype)) if deps: colify(deps, indent=4) else: diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 3ba3c71562..506cf0913f 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -36,7 +36,7 @@ description = "Calculate md5 checksums for files/urls." def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, - help="Files to checksum.") + help="Files/urls to checksum.") def compute_md5_checksum(url): @@ -67,6 +67,7 @@ def md5(parser, args): tty.warn("%s" % e) # Dump the MD5s at last without interleaving them with downloads - tty.msg("%d MD5 checksums:" % len(results)) + checksum = 'checksum' if len(results) == 1 else 'checksums' + tty.msg("%d MD5 %s:" % (len(results), checksum)) for checksum, url in results: print "%s %s" % (checksum, url) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index debc6752b4..70c3c35d8c 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -530,13 +530,6 @@ class Dotkit(EnvModule): class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") - environment_modifications_formats = { - PrependPath: 'prepend-path --delim "{separator}" {name} \"{value}\"\n', - AppendPath: 'append-path --delim "{separator}" {name} \"{value}\"\n', - RemovePath: 'remove-path --delim "{separator}" {name} \"{value}\"\n', - SetEnv: 'setenv {name} \"{value}\"\n', - UnsetEnv: 'unsetenv {name}\n' - } autoload_format = ('if ![ is-loaded {module_file} ] {{\n' ' puts stderr "Autoloading {module_file}"\n' @@ -556,11 +549,13 @@ class TclModule(EnvModule): def header(self): timestamp = datetime.datetime.now() # TCL Modulefile header - header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp - header += '##\n' - header += '## %s\n' % self.spec.short_spec - header += '##\n' + header = """\ +#%%Module1.0 +## Module file created by spack (https://github.com/LLNL/spack) on %s +## +## %s +## +""" % (timestamp, self.spec.short_spec) # TODO : category ? # Short description @@ -575,6 +570,44 @@ class TclModule(EnvModule): header += '}\n\n' return header + def process_environment_command(self, env): + environment_modifications_formats_colon = { + PrependPath: 'prepend-path {name} \"{value}\"\n', + AppendPath: 'append-path {name} \"{value}\"\n', + RemovePath: 'remove-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' + } + environment_modifications_formats_general = { + PrependPath: + 'prepend-path --delim "{separator}" {name} \"{value}\"\n', + AppendPath: + 'append-path --delim "{separator}" {name} \"{value}\"\n', + RemovePath: + 'remove-path --delim "{separator}" {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' + } + for command in env: + # Token expansion from configuration file + name = command.args.get('name', '').format(**self.upper_tokens) + value = str(command.args.get('value', '')).format(**self.tokens) + command.update_args(name=name, value=value) + # Format the line int the module file + try: + if command.args.get('separator', ':') == ':': + yield environment_modifications_formats_colon[type( + command)].format(**command.args) + else: + yield environment_modifications_formats_general[type( + command)].format(**command.args) + except KeyError: + message = ('Cannot handle command of type {command}: ' + 'skipping request') + details = '{context} at {filename}:{lineno}' + tty.warn(message.format(command=type(command))) + tty.warn(details.format(**command.args)) + def module_specific_content(self, configuration): naming_tokens = self.tokens # Conflict diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ff8c8e96bc..8c1204402a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -143,8 +143,10 @@ class Package(object): informational URL, so that users know what they're installing. - url - URL of the source archive that spack will fetch. + url or url_for_version(self, version) + If url, then the URL of the source archive that spack will fetch. + If url_for_version(), then a method returning the URL required + to fetch a particular version. install() This function tells spack how to build and install the diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 25f1e605d6..29ed6e0d32 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -109,7 +109,7 @@ def _spider(args): while link_parser.links: raw_link = link_parser.links.pop() - abs_link = urlparse.urljoin(response_url, raw_link) + abs_link = urlparse.urljoin(response_url, raw_link.strip()) links.add(abs_link) |