From daa38d2ff4acd75b3b592bac5abd9ef642ee801f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Feb 2015 02:33:29 -0800 Subject: SPACK-59: Documentation updates, bugfix in fetching. --- lib/spack/docs/.gitignore | 1 + lib/spack/docs/Makefile | 16 +- lib/spack/docs/basic_usage.rst | 37 +++- lib/spack/docs/command_index.in | 10 ++ lib/spack/docs/features.rst | 2 +- lib/spack/docs/index.rst | 2 + lib/spack/docs/mirrors.rst | 217 +++++++++++++++++++++++ lib/spack/docs/packaging_guide.rst | 312 ++++++++++++++++++++++++++++++++-- lib/spack/docs/site_configuration.rst | 193 --------------------- lib/spack/spack/mirror.py | 2 +- lib/spack/spack/package.py | 5 +- lib/spack/spack/test/git_fetch.py | 4 +- lib/spack/spack/test/hg_fetch.py | 4 +- lib/spack/spack/test/svn_fetch.py | 4 +- 14 files changed, 585 insertions(+), 224 deletions(-) create mode 100644 lib/spack/docs/command_index.in create mode 100644 lib/spack/docs/mirrors.rst diff --git a/lib/spack/docs/.gitignore b/lib/spack/docs/.gitignore index 7701dd9f12..26c343d3eb 100644 --- a/lib/spack/docs/.gitignore +++ b/lib/spack/docs/.gitignore @@ -1,3 +1,4 @@ package_list.rst +command_index.rst spack*.rst _build diff --git a/lib/spack/docs/Makefile b/lib/spack/docs/Makefile index a660e1255d..00203b5b61 100644 --- a/lib/spack/docs/Makefile +++ b/lib/spack/docs/Makefile @@ -27,6 +27,18 @@ all: html package_list: spack package-list > package_list.rst +# +# Generate a command index +# +command_index: + cp command_index.in command_index.rst + echo >> command_index.rst + grep -ho '.. _spack-.*:' *rst \ + | perl -pe 's/.. _([^:]*):/ * :ref:`\1`/' \ + | sort >> command_index.rst + +custom_targets: package_list command_index + # # This creates a git repository and commits generated html docs. # It them pushes the new branch into THIS repository as gh-pages. @@ -77,10 +89,10 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: - -rm -f package_list.rst + -rm -f package_list.rst command_index.rst -rm -rf $(BUILDDIR)/* $(APIDOC_FILES) -html: apidoc package_list +html: apidoc custom_targets $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index bd25d739ea..3d808708e1 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -16,6 +16,8 @@ software. Before that, you need to know what's available. You can see avaialble package names either using the :ref:`package-list`, or using the commands below. +.. _spack-list: + ``spack list`` ~~~~~~~~~~~~~~~~ @@ -31,6 +33,7 @@ do wildcard searches using ``*``: .. command-output:: spack list *util* +.. _spack-info: ``spack info`` ~~~~~~~~~~~~~~~~ @@ -47,6 +50,8 @@ attacks. :ref:`Dependencies ` and :ref:`virtual dependencies `, are described in more detail later. +.. _spack-versions: + ``spack versions`` ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -69,6 +74,8 @@ Installing and uninstalling Now that you know how to list avaiable packages and versions, you're ready to start installing things. +.. _spack-install: + ``spack install`` ~~~~~~~~~~~~~~~~~~~~~ @@ -138,6 +145,7 @@ configuration a **spec**. In the command lines above, both ``mpileaks`` and ``mpileaks@3.0.4`` are specs. Specs are described in detail in :ref:`sec-specs`. +.. _spack-uninstall: ``spack uninstall`` ~~~~~~~~~~~~~~~~~~~~~ @@ -170,6 +178,7 @@ Seeing installed packages We know that ``spack list`` shows you the names of available packages, but how do you figure out which are installed? +.. _spack-find: ``spack find`` ~~~~~~~~~~~~~~~~~~~~~~ @@ -303,7 +312,7 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. -Compiler Configuration +Compiler configuration ----------------------------------- Spack has the ability to build packages with multiple compilers and @@ -311,6 +320,8 @@ compiler versions. Spack searches for compilers on your machine automatically the first time it is run. It does this by inspecting your path. +.. _spack-compilers: + ``spack compilers`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -337,6 +348,8 @@ compilers`` or ``spack compiler list``:: Any of these compilers can be used to build Spack packages. More on how this is done is in :ref:`sec-specs`. +.. _spack-compiler-add: + ``spack compiler add`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -361,6 +374,7 @@ installed, but you know that new compilers have been added to your This loads the environment module for gcc-4.9.0 to get it into the ``PATH``, and then it adds the compiler to Spack. +.. _spack-compiler-info: ``spack compiler info`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -382,7 +396,7 @@ matching Intel compiler was displayed. Manual compiler configuration -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If autodetection fails, you can manually conigure a compiler by editing your ``~/.spackconfig`` file. You can do this by running @@ -413,7 +427,7 @@ list displayed by ``spack compilers``. .. _sec-specs: -Specs & Dependencies +Specs & dependencies ------------------------- We know that ``spack install``, ``spack uninstall``, and other @@ -720,6 +734,8 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +.. _spack-providers: + ``spack providers`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -739,7 +755,7 @@ versions are now filtered out. .. _shell-support: -Environment Modules +Environment modules ------------------------------- .. note:: @@ -787,6 +803,7 @@ The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell support `_. + Using Modules & Dotkits ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -934,6 +951,8 @@ if newer, fancier module support is added to Spack at some later date, you may want to regenerate all the modules to take advantage of these new features. +.. _spack-module: + ``spack module refresh`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -950,7 +969,7 @@ regenerate all module and dotkit files from scratch: .. _extensions: -Extensions & Python Support +Extensions & Python support ------------------------------------ Spack's installation model assumes that each package will live in its @@ -971,6 +990,8 @@ an *extension*. Suppose you have Python installed like so: -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- python@2.7.8 +.. _spack-extensions: + ``spack extensions`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1004,6 +1025,7 @@ they are packages like any ohter. They are installed into their own prefixes, and you can see this with ``spack find -p``: .. code-block:: sh + $ spack find -p py-numpy ==> 1 installed packages. -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- @@ -1060,6 +1082,8 @@ for this case. Instead of requiring users to load particular environment modules, you can *activate* the package within the Python installation: +.. _spack-activate: + ``spack activate`` ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1133,6 +1157,7 @@ dependencies, you can use ``spack activate -f``: $ spack activate -f py-numpy ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. +.. _spack-deactivate: ``spack deactivate`` ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1159,6 +1184,8 @@ several variants: Getting Help ----------------------- +.. _spack-help: + ``spack help`` ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/docs/command_index.in b/lib/spack/docs/command_index.in new file mode 100644 index 0000000000..94cdf38109 --- /dev/null +++ b/lib/spack/docs/command_index.in @@ -0,0 +1,10 @@ +.. _command_index: + +Command index +================= + +This is an alphabetical list of commands with links to the places they +appear in the documentation. + +.. hlist:: + :columns: 3 diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index b39dcd3390..fcb810086d 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,4 +1,4 @@ -Feature Overview +Feature overview ================== This is a high-level overview of features that make Spack different diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 73eff43ab7..2382678cc3 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -46,8 +46,10 @@ Table of Contents getting_started basic_usage packaging_guide + mirrors site_configuration developer_guide + command_index package_list API Docs diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst new file mode 100644 index 0000000000..57ca1af068 --- /dev/null +++ b/lib/spack/docs/mirrors.rst @@ -0,0 +1,217 @@ +.. _mirrors: + +Mirrors +============================ + +Some sites may not have access to the internet for fetching packages. +These sites will need a local repository of tarballs from which they +can get their files. Spack has support for this with *mirrors*. A +mirror is a URL that points to a directory, either on the local +filesystem or on some server, containing tarballs for all of Spack's +packages. + +Here's an example of a mirror's directory structure:: + + mirror/ + cmake/ + cmake-2.8.10.2.tar.gz + dyninst/ + dyninst-8.1.1.tgz + dyninst-8.1.2.tgz + libdwarf/ + libdwarf-20130126.tar.gz + libdwarf-20130207.tar.gz + libdwarf-20130729.tar.gz + libelf/ + libelf-0.8.12.tar.gz + libelf-0.8.13.tar.gz + libunwind/ + libunwind-1.1.tar.gz + mpich/ + mpich-3.0.4.tar.gz + mvapich2/ + mvapich2-1.9.tgz + +The structure is very simple. There is a top-level directory. The +second level directories are named after packages, and the third level +contains tarballs for each package, named after each package. + +.. note:: + + Archives are **not** named exactly they were in the package's fetch + URL. They have the form ``-.``, where + ```` is Spack's name for the package, ```` is the + version of the tarball, and ```` is whatever format the + package's fetch URL contains. + + In order to make mirror creation reasonably fast, we copy the + tarball in its original format to the mirror directory, but we do + not standardize on a particular compression algorithm, because this + would potentially require expanding and recompressing each archive. + +.. _spack-mirror: + +``spack mirror`` +---------------------------- + +Mirrors are managed with the ``spack mirror`` command. The help for +``spack mirror`` looks like this:: + + $ spack mirror -h + usage: spack mirror [-h] SUBCOMMAND ... + + positional arguments: + SUBCOMMAND + create Create a directory to be used as a spack mirror, and fill + it with package archives. + add Add a mirror to Spack. + remove Remove a mirror by name. + list Print out available mirrors to the console. + + optional arguments: + -h, --help show this help message and exit + +The ``create`` command actually builds a mirror by fetching all of its +packages from the internet and checksumming them. + +The other three commands are for managing mirror configuration. They +control the URL(s) from which Spack downloads its packages. + +.. _spack-mirror-create: + +``spack mirror create`` +---------------------------- + +You can create a mirror using the ``spack mirror create`` command, assuming +you're on a machine where you can access the internet. + +The command will iterate through all of Spack's packages and download +the safe ones into a directory structure like the one above. Here is +what it looks like: + + +.. code-block:: bash + + $ spack mirror create libelf libdwarf + ==> Created new mirror in spack-mirror-2014-06-24 + ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz + ########################################################## 81.6% + ==> Checksum passed for libelf@0.8.13 + ==> Added libelf@0.8.13 + ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz + ###################################################################### 98.6% + ==> Checksum passed for libelf@0.8.12 + ==> Added libelf@0.8.12 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz + ###################################################################### 97.3% + ==> Checksum passed for libdwarf@20130207 + ==> Added libdwarf@20130207 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz + ######################################################## 78.9% + ==> Checksum passed for libdwarf@20130126 + ==> Added libdwarf@20130126 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz + ############################################################# 84.7% + ==> Added libdwarf@20130729 + ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror + ==> Added python@2.7.8. + ==> Successfully updated mirror in spack-mirror-2015-02-24. + Archive stats: + 0 already present + 5 added + 0 failed to fetch. + +Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and +copy it over to the machine you want it hosted on. + +Custom package sets +~~~~~~~~~~~~~~~~~~~~~~~ + +Normally, ``spack mirror create`` downloads all the archives it has +checksums for. If you want to only create a mirror for a subset of +packages, you can do that by supplying a list of package specs on the +command line after ``spack mirror create``. For example, this +command:: + + $ spack mirror create libelf@0.8.12: boost@1.44: + +Will create a mirror for libelf versions greater than or equal to +0.8.12 and boost versions greater than or equal to 1.44. + +Mirror files +~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a *very* large number of packages you want to mirror, you +can supply a file with specs in it, one per line:: + + $ cat specs.txt + libdwarf + libelf@0.8.12: + boost@1.44: + boost@1.39.0 + ... + $ spack mirror create -f specs.txt + ... + +This is useful if there is a specific suite of software managed by +your site. + +.. _spack-mirror-add: + +``spack mirror add`` +---------------------------- + +Once you have a mirrror, you need to let spack know about it. This is +relatively simple. First, figure out the URL for the mirror. If it's +a file, you can use a file URL like this one:: + + file:///Users/gamblin2/spack-mirror-2014-06-24 + +That points to the directory on the local filesystem. If it were on a +web server, you could use a URL like this one: + + https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + +Spack will use the URL as the root for all of the packages it fetches. +You can tell your Spack installation to use that mirror like this: + +.. code-block:: bash + + $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 + +Each mirror has a name so that you can refer to it again later. + +.. _spack-mirror-list: + +``spack mirror list`` +---------------------------- + +If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: + + $ spack mirror list + local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 + +.. _spack-mirror-remove: + +``spack mirror remove`` +---------------------------- + +And, if you want to remove a mirror, just remove it by name:: + + $ spack mirror remove local_filesystem + $ spack mirror list + ==> No mirrors configured. + +Mirror precedence +---------------------------- + +Adding a mirror really just adds a section in ``~/.spackconfig``:: + + [mirror "local_filesystem"] + url = file:///Users/gamblin2/spack-mirror-2014-06-24 + [mirror "remote_server"] + url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + +If you want to change the order in which mirrors are searched for +packages, you can edit this file and reorder the sections. Spack will +search the topmost mirror first and the bottom-most mirror last. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bc3bacf8f2..8b4c0a4ce1 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -28,7 +28,7 @@ ubiquitous in the scientific software community. Second, it's a modern language and has many powerful features to help make package writing easy. -Creating & Editing Packages +Creating & editing packages ---------------------------------- .. _spack-create: @@ -254,7 +254,7 @@ This is useful when ``spack create`` cannot figure out the name and version of your package from the archive URL. -Naming & Directory Structure +Naming & directory structure -------------------------------------- .. note:: @@ -497,7 +497,7 @@ versions. See the documentation on `attribute_list_url`_ and .. _vcs-fetch: -Fetching from VCS Repositories +Fetching from VCS repositories -------------------------------------- For some packages, source code is provided in a Version Control System @@ -774,6 +774,8 @@ to patch a package's source. For example, the ``py-pyside`` package contains some custom code for tweaking the way the PySide build handles ``RPATH``: +.. _pyside-patch: + .. code-block:: python :linenos: @@ -810,14 +812,59 @@ You could put this logic in ``install()``, but putting it in a patch function gives you some benefits. First, spack ensures that the ``patch()`` function is run once per code checkout. That means that if you run install, hit ctrl-C, and run install again, the code in the -patch function is only run once. Also, you can tell Spack to run only the patching part of the build using the .. +patch function is only run once. Also, you can tell Spack to run only +the patching part of the build using the :ref:`spack-patch` command. + +Handling RPATHs +---------------------------- + +Spack installs each package in a way that ensures that all of its +dependencies are found when it runs. It does this using `RPATHs +`_. An RPATH is a search +path, stored in a binary (an executable or library), that tells the +dynamic loader where to find its dependencies at runtime. You may be +familiar with ```LD_LIBRARY_PATH`` +`_ +on Linux or ```DYLD_LIBRARY_PATH`` +` +on Mac OS X. RPATH is similar to these paths, in that it tells +the loader where to find libraries. Unlike them, it is embedded in +the binary and not set in each user's environment. + +RPATHs in Spack are handled in one of three ways: + + 1. For most packages, RPATHs are handled automatically using Spack's + :ref:`compiler wrappers `. These wrappers are + set in standard variables like ``CC``, ``CXX``, and ``FC``, so + most build systems (autotools and many gmake systems) pick them + up and use them. + 2. CMake also respects Spack's compiler wrappers, but many CMake + builds have logic to overwrite RPATHs when binaries are + installed. Spack provides the ``std_cmake_args`` variable, which + includes parameters necessary for CMake build use the right + installation RPATH. It can be used like this when ``cmake`` is + invoked: + + .. code-block:: python + + class MyPackage(Package): + ... + def install(self, spec, prefix): + cmake('..', *std_cmake_args) + make() + make('install') + 3. If you need to modify the build to add your own RPATHs, you can + use the ``self.rpath`` property of your package, which will + return a list of all the RPATHs that Spack will use when it + links. You can see this how this is used in the :ref:`PySide + example ` above. -Finding Package Downloads +Finding new versions ---------------------------- -We've already seen the ``homepage`` and ``url`` package attributes: +You've already seen the ``homepage`` and ``url`` package attributes: .. code-block:: python :linenos: @@ -853,7 +900,7 @@ url is: url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" -Spack spiders ``http://www.mr511.de/software/`` to find similar +Here, Spack spiders ``http://www.mr511.de/software/`` to find similar tarball links and ultimately to make a list of available versions of ``libelf``. @@ -907,7 +954,7 @@ when spidering the page. .. _attribute_parallel: -Parallel Builds +Parallel builds ------------------ By default, Spack will invoke ``make()`` with a ``-j `` @@ -1036,6 +1083,203 @@ command line to find installed packages or to install packages with particular constraints, and package authors can use specs to describe relationships between packages. +.. _setup-dependent-environment: + +``setup_dependent_environment()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack provides a mechanism for dependencies to provide variables that +can be used in their dependents' build. Any package can declare a +``setup_dependent_environment()`` function, and this function will be +called before the ``install()`` method of any dependent packages. +This allows dependencies to set up environment variables and other +properties to be used by dependents. + +The funciton declaration should look like this: + +.. code-block:: python + + class Qt(Package): + ... + def setup_dependent_environment(self, module, spec, dep_spec): + """Dependencies of Qt find it using the QTDIR environment variable.""" + os.environ['QTDIR'] = self.prefix + +Here, the Qt package sets the ``QTDIR`` environment variable so that +packages that depend on a particular Qt installation will find it. + +The arguments to this function are: + + * **module**: the module of the dependent package, where global + properties can be assigned. + * **spec**: the spec of the *dependency package* (the one the function is called on). + * **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). + +A goo example of using these is in the Python packge: + +.. code-block:: python + + def setup_dependent_environment(self, module, spec, dep_spec): + # Python extension builds can have a global python executable function + module.python = Executable(join_path(spec.prefix.bin, 'python')) + + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. + module.python_lib_dir = os.path.join(dep_spec.prefix, self.python_lib_dir) + module.python_include_dir = os.path.join(dep_spec.prefix, self.python_include_dir) + module.site_packages_dir = os.path.join(dep_spec.prefix, self.site_packages_dir) + + # Make the site packages directory if it does not exist already. + mkdirp(module.site_packages_dir) + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in dep_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + os.environ['PYTHONPATH'] = ':'.join(python_paths) + +The first thing that happens here is that the ``python`` command is +inserted into module scope of the dependent. This allows most python +packages to have a very simple install method, like this: + +.. code-block:: python + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + +Python's ``setup_dependent_environment`` method also sets up smoe +other variables, creates a directory, and sets up the ``PYTHONPATH`` +so that dependent packages can find their dependencies at build time. + + +.. _packaging_extensions: + +Extensions +------------------------- + +Spack's support for package extensions is documented extensively in +:ref:`extensions`. This section documents how to make your own +extendable packages and extensions. + +To support extensions, a package needs to set its ``extendable`` +property to ``True``, e.g.: + +.. code-block:: python + + class Python(Package): + ... + extendable = True + ... + +To make a package into an extension, simply add simply add an +``extends`` call in the package definition, and pass it the name of an +extendable package: + +.. code-block:: python + + class PyNumpy(Package): + ... + extends('python') + ... + +Now, the ``py-numpy`` package can be used as an argument to ``spack +activate``. When it is activated, all the files in its prefix will be +symbolically linked into the prefix of the python package. + +Sometimes, certain files in one package will conflict with those in +another, which means they cannot both be activated (symlinked) at the +same time. In this case, you can tell Spack to ignore those files +when it does the activation: + +.. code-block:: python + + class PyNose(Package): + ... + extends('python', ignore=r'bin/nosetests.*$') + ... + +The code above will prevent ``$prefix/bin/nosetests`` from being +linked in at activation time. + +.. note:: + + You can call *either* ``depends_on`` or ``extends`` on any one + package, but not both. For example you cannot both + ``depends_on('python')`` and ``extends(python)`` in the same + package. ``extends`` implies ``depends_on``. + + + +Activation & deactivation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack's ``Package`` class has default ``activate`` and ``deactivate`` +implementations that handle symbolically linking extensions' prefixes +into the directory of the parent package. However, extendable +packages can override these methdos to add custom activate/deactivate +logic of their own. For example, the ``activate`` and ``deactivate`` +methods in the Python class use the symbolic linking, but they also +handle details surrounding Python's ``.pth`` files, and other aspects +of Python packaging. + +Spack's extensions mechanism is designed to be extensible, so that +other packages (like Ruby, R, Perl, etc.) can provide their own +custom extension management logic, as they may not handle modules the +same way that Python does. + +Let's look at Python's activate function: + +.. code-block:: python + + def activate(self, ext_pkg, **kwargs): + kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) + super(Python, self).activate(ext_pkg, **kwargs) + + exts = spack.install_layout.extension_map(self.spec) + exts[ext_pkg.name] = ext_pkg.spec + self.write_easy_install_pth(exts) + +This function is called on the *extendee* (Python). It first calls +``activate`` in the superclass, which handles symlinking the +extension package's prefix into this package's prefix. It then does +some special handling of the ``easy-install.pth`` file, part of +Python's setuptools. + +Deactivate behaves similarly to activate, but it unlinks files: + +.. code-block:: python + + def deactivate(self, ext_pkg, **kwargs): + kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) + super(Python, self).deactivate(ext_pkg, **kwargs) + + exts = spack.install_layout.extension_map(self.spec) + if ext_pkg.name in exts: # Make deactivate idempotent. + del exts[ext_pkg.name] + self.write_easy_install_pth(exts) + +Both of these methods call some custom functions in the Python +package. See the source for Spack's Python package for details. + + +Activation arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You may have noticed that the ``activate`` function defined above +takes keyword arguments. These are the keyword arguments from +``extends()``, and they are passed to both activate and deactivate. + +This capability allows an extension to customize its own activation by +passing arguments to the extendee. Extendees can likewise implement +custom ``activate()`` and ``deactivate()`` functions to suit their +needs. + +The only keyword argument supported by default is the ``ignore`` +argument, which can take a regex, list of regexes, or a predicate to +determine which files *not* to symlink during activation. + + .. _virtual-dependencies: Virtual dependencies @@ -1257,6 +1501,7 @@ explicitly. Concretization policies are discussed in more detail in :ref:`site-configuration`. Sites using Spack can customize them to match the preferences of their own users. +.. _spack-spec: ``spack spec`` ~~~~~~~~~~~~~~~~~~~~ @@ -1354,7 +1599,7 @@ information. .. _install-environment: -The Install environment +The install environment -------------------------- In general, you should not have to do much differently in your install @@ -1414,6 +1659,7 @@ easily: ``PATH`` Set to point to ``/bin`` directories of dpeendencies ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies + ``PYTHONPATH`` Path to site-packages dir of any python dependencies ======================= ============================= ``PATH`` is set up to point to dependencies ``/bin`` directories so @@ -1433,6 +1679,12 @@ dependencies using the GNU ``pkg-config`` tool. It is similar to ``CMAKE_PREFIX_PATH`` in that it allows a build to automatically discover its dependencies. +If you want to see the environment that a package will build with, or +if you want to run commands in that environment to test them out, you +can use the :ref:```spack env`` ` command, documented +below. + +.. _compiler-wrappers: Compiler interceptors ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1498,7 +1750,6 @@ process runs. Packages are free to change the environment or to modify Spack internals, because each ``install()`` call has its own dedicated process. - .. _prefix-objects: Prefix objects @@ -1970,7 +2221,7 @@ File functions .. _pacakge-lifecycle: -Package Workflow Commands +Packaging workflow commands --------------------------------- When you are building packages, you will likely not get things @@ -2036,6 +2287,8 @@ this step if they have been. If Spack discovers that patches didn't apply cleanly on some previous run, then it will restage the entire package before patching. +.. _spack-restage: + ``spack restage`` ~~~~~~~~~~~~~~~~~ Restores the source code to pristine state, as it was before building. @@ -2048,6 +2301,7 @@ Does this in one of two ways: 2. If the source was checked out from a repository, this deletes the build directory and checks it out again. +.. _spack-clean: ``spack clean`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2057,6 +2311,8 @@ expanded/checked out source code *and* any downloaded archive. If build process will start from scratch. +.. _spack-purge: + ``spack purge`` ~~~~~~~~~~~~~~~~~ Cleans up all of Spack's temporary files. Use this to recover disk @@ -2102,7 +2358,7 @@ to get rid of the install prefix before you build again: spack uninstall -f -Graphing Dependencies +Graphing dependencies -------------------------- .. _spack-graph: @@ -2181,7 +2437,7 @@ example:: This graph can be provided as input to other graphing tools, such as those in `Graphviz `_. -Interactive Shell Support +Interactive shell support -------------------------- Spack provides some limited shell support to make life easier for @@ -2197,6 +2453,7 @@ For ``csh`` and ``tcsh`` run: ``spack cd`` will then be available. +.. _spack-cd: ``spack cd`` ~~~~~~~~~~~~~~~~~ @@ -2227,6 +2484,35 @@ directory, install directory, package directory) and others change to core spack locations. For example, ``spack cd -m`` will take you to the main python source directory of your spack install. +.. _spack-env: + +``spack env`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack env`` functions much like the standard unix ``env`` command, +but it takes a spec as an argument. You can use it to see the +environment variables that will be set when a particular build runs, +for example: + +.. code-block:: sh + + $ spack env mpileaks@1.1%intel + +This will display the entire environment that will be set when the +``mpileaks@1.1%intel`` build runs. + +To run commands in a package's build environment, you can simply provided them after the spec argument to ``spack env``: + +.. code-block:: sh + + $ spack cd mpileaks@1.1%intel + $ spack env mpileaks@1.1%intel ./configure + +This will cd to the build directory and then run ``configure`` in the +package's build environment. + + +.. _spack-location: ``spack location`` ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 4936e3052c..a3e19cc800 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -3,199 +3,6 @@ Site-specific configuration =================================== -.. _mirrors: - -Mirrors ----------------------------- - -Some sites may not have access to the internet for fetching packages. -These sites will need a local repository of tarballs from which they -can get their files. Spack has support for this with *mirrors*. A -mirror is a URL that points to a directory, either on the local -filesystem or on some server, containing tarballs for all of Spack's -packages. - -Here's an example of a mirror's directory structure:: - - mirror/ - cmake/ - cmake-2.8.10.2.tar.gz - dyninst/ - DyninstAPI-8.1.1.tgz - DyninstAPI-8.1.2.tgz - libdwarf/ - libdwarf-20130126.tar.gz - libdwarf-20130207.tar.gz - libdwarf-20130729.tar.gz - libelf/ - libelf-0.8.12.tar.gz - libelf-0.8.13.tar.gz - libunwind/ - libunwind-1.1.tar.gz - mpich/ - mpich-3.0.4.tar.gz - mvapich2/ - mvapich2-1.9.tgz - -The structure is very simple. There is a top-level directory. The -second level directories are named after packages, and the third level -contains tarballs for each package, named as they were in the -package's fetch URL. - -``spack mirror`` -~~~~~~~~~~~~~~~~~~~~~~~ - -Mirrors are managed with the ``spack mirror`` command. The help for -``spack mirror`` looks like this:: - - $ spack mirror -h - usage: spack mirror [-h] SUBCOMMAND ... - - positional arguments: - SUBCOMMAND - create Create a directory to be used as a spack mirror, and fill - it with package archives. - add Add a mirror to Spack. - remove Remove a mirror by name. - list Print out available mirrors to the console. - - optional arguments: - -h, --help show this help message and exit - -The ``create`` command actually builds a mirror by fetching all of its -packages from the internet and checksumming them. - -The other three commands are for managing mirror configuration. They -control the URL(s) from which Spack downloads its packages. - - -``spack mirror create`` -~~~~~~~~~~~~~~~~~~~~~~~ - -You can create a mirror using the ``spack mirror create`` command, assuming -you're on a machine where you can access the internet. - -The command will iterate through all of Spack's packages and download -the safe ones into a directory structure like the one above. Here is -what it looks like: - - -.. code-block:: bash - - $ spack mirror create libelf libdwarf - ==> Created new mirror in spack-mirror-2014-06-24 - ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz - ########################################################## 81.6% - ==> Checksum passed for libelf@0.8.13 - ==> Added spack-mirror-2014-06-24/libelf/libelf-0.8.13.tar.gz to mirror - ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz - ###################################################################### 98.6% - ==> Checksum passed for libelf@0.8.12 - ==> Added spack-mirror-2014-06-24/libelf/libelf-0.8.12.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz - ###################################################################### 97.3% - ==> Checksum passed for libdwarf@20130207 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130207.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz - ######################################################## 78.9% - ==> Checksum passed for libdwarf@20130126 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130126.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz - ############################################################# 84.7% - ==> Checksum passed for libdwarf@20130729 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror - -Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and -copy it over to the machine you want it hosted on. - -Custom package sets -^^^^^^^^^^^^^^^^^^^^^^^^ - -Normally, ``spack mirror create`` downloads all the archives it has -checksums for. If you want to only create a mirror for a subset of -packages, you can do that by supplying a list of package specs on the -command line after ``spack mirror create``. For example, this -command:: - - $ spack mirror create libelf@0.8.12: boost@1.44: - -Will create a mirror for libelf versions greater than or equal to -0.8.12 and boost versions greater than or equal to 1.44. - -Mirror files -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you have a *very* large number of packages you want to mirror, you -can supply a file with specs in it, one per line:: - - $ cat specs.txt - libdwarf - libelf@0.8.12: - boost@1.44: - boost@1.39.0 - ... - $ spack mirror create -f specs.txt - ... - -This is useful if there is a specific suite of software managed by -your site. - - -``spack mirror add`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Once you have a mirrror, you need to let spack know about it. This is -relatively simple. First, figure out the URL for the mirror. If it's -a file, you can use a file URL like this one:: - - file:///Users/gamblin2/spack-mirror-2014-06-24 - -That points to the directory on the local filesystem. If it were on a -web server, you could use a URL like this one: - - https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 - -Spack will use the URL as the root for all of the packages it fetches. -You can tell your Spack installation to use that mirror like this: - -.. code-block:: bash - - $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 - -Each mirror has a name so that you can refer to it again later. - -``spack mirror list`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: - - $ spack mirror list - local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 - -``spack mirror remove`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -And, if you want to remove a mirror, just remove it by name:: - - $ spack mirror remove local_filesystem - $ spack mirror list - ==> No mirrors configured. - -Mirror precedence -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Adding a mirror really just adds a section in ``~/.spackconfig``:: - - [mirror "local_filesystem"] - url = file:///Users/gamblin2/spack-mirror-2014-06-24 - [mirror "remote_server"] - url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 - -If you want to change the order in which mirrors are searched for -packages, you can edit this file and reorder the sections. Spack will -search the topmost mirror first and the bottom-most mirror last. - - .. _temp-space: Temporary space diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 114c7b6a35..306c8085aa 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -146,7 +146,7 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - archive_path = join_path(path, mirror_archive_path(spec)) + archive_path = os.path.abspath(join_path(path, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) mkdirp(subdir) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 137e8f8837..75e6142a9d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -287,10 +287,9 @@ class Package(object): .. code-block:: python - p.do_clean() # runs make clean - p.do_clean_work() # removes the build directory and + p.do_clean() # removes the stage directory entirely + p.do_restage() # removes the build directory and # re-expands the archive. - p.do_clean_dist() # removes the stage directory entirely The convention used here is that a do_* function is intended to be called internally by Spack commands (in spack.cmd). These aren't for package diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index f6d9bfcf05..04422adb57 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -61,7 +61,7 @@ class GitFetchTest(MockPackagesTest): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def assert_rev(self, rev): @@ -93,7 +93,7 @@ class GitFetchTest(MockPackagesTest): untracked_file = 'foobarbaz' touch(untracked_file) self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked_file)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 97c5b665e7..e1ab2cffe6 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -60,7 +60,7 @@ class HgFetchTest(MockPackagesTest): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def try_fetch(self, rev, test_file, args): @@ -87,7 +87,7 @@ class HgFetchTest(MockPackagesTest): untracked = 'foobarbaz' touch(untracked) self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index a48a86dcc3..0159fb087f 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -60,7 +60,7 @@ class SvnFetchTest(MockPackagesTest): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def assert_rev(self, rev): @@ -99,7 +99,7 @@ class SvnFetchTest(MockPackagesTest): untracked = 'foobarbaz' touch(untracked) self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) -- cgit v1.2.3-70-g09d2