From 7be511a352d2bd84a7dbceb7a9a94c43ad77683b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Dec 2013 11:02:31 -0800 Subject: Finished basic usage. --- lib/spack/docs/basic_usage.rst | 538 +++++++++++++++++++++++++++-------- lib/spack/docs/conf.py | 18 ++ lib/spack/docs/features.rst | 58 ++-- lib/spack/docs/index.rst | 21 +- lib/spack/spack/arch.py | 67 ----- lib/spack/spack/architecture.py | 67 +++++ lib/spack/spack/cmd/arch.py | 11 + lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/providers.py | 17 ++ lib/spack/spack/cmd/sys_type.py | 11 - lib/spack/spack/cmd/versions.py | 3 - lib/spack/spack/colify.py | 7 +- lib/spack/spack/concretize.py | 4 +- lib/spack/spack/globals.py | 1 - lib/spack/spack/multi_function.py | 2 +- lib/spack/spack/packages/__init__.py | 1 - lib/spack/spack/spec.py | 1 - 17 files changed, 587 insertions(+), 242 deletions(-) delete mode 100644 lib/spack/spack/arch.py create mode 100644 lib/spack/spack/architecture.py create mode 100644 lib/spack/spack/cmd/arch.py create mode 100644 lib/spack/spack/cmd/providers.py delete mode 100644 lib/spack/spack/cmd/sys_type.py diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 425fe0c288..ae591920db 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1,25 +1,26 @@ Basic usage ===================== -Nearly everything you do wtih spack will involve the ``spack`` -command. Like many well-known tools (``git``, ``cvs``, ``svn``, -``yum``, ``port``, ``apt-get``, etc.), ``spack`` is generally called -with a *subcommand* indicating the action you want to perform. +Spack is implemented as a single command (``spack``) with many +*subcommands*, much like ``git``, ``svn``, ``yum``, or ``apt-get``. +Only a small subset of commands are needed for typical usage. + +This section covers a small set of subcommands that should cover most +general use cases for Spack. Getting Help ----------------------- - ``spack help`` ~~~~~~~~~~~~~~~~~~~~~~ -The first subcommand you should know is ``spack help``. Run with no -arguments, it will give a list of all spack options and subcommands: +The ``help`` subcommand will print out out a list of all of +``spack``'s options and subcommands: .. command-output:: spack help -If you want help on the usage of a particular subcommand, you can pass -it as an argument to ``spack help``: +Adding an argument, e.g. ``spack help ``, will print out +usage information for a particular subcommand: .. command-output:: spack help install @@ -38,29 +39,54 @@ this: ``spack list`` and ``spack info``. ``spack list`` ~~~~~~~~~~~~~~~~ -The ``spack list`` command does what you might expect. it prints out a -list of all the available packages you can install. Use it like -this: +The ``spack list`` command prints out a list of all of the packages +Spack can install: .. command-output:: spack list -The packages are listed by name in alphabetical order. If you just -want to see *installed* packages, you should use ``spack list -i`` +The packages are listed by name in alphabetical order. To see a list of +only the *installed* packages, use ``spack list -i``. ``spack info`` ~~~~~~~~~~~~~~~~ -To get information on a particular package from the full list, you can -run ``spack info ``. e.g., for ``mpich``: +To get information on a particular package from the full list, run +``spack info ``. For example, for ``mpich`` the output +looks like this: .. command-output:: spack info mpich -This gives basic information about the package, such as where it can -be downloaded, what other packages it depends on, virtual package -information, and a text description, if one is available. We'll give -more details on dependencies and virtual dependencies later in this -guide. +This includes basic information about the package: where to download +it, its dependencies, virtual packages it provides (e.g. an MPI +implementation will provide the MPI interface), and a text +description, if one is available. :ref:`Dependencies +` and :ref:`virtual dependencies +` are described in more detail later. + + +``spack versions`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +To see available versions of a package, run ``spack versions``, for +example: + +.. command-output:: spack versions libelf + +Since it has to manage many different software packages, Spack doesn't +place many restrictions on what a package version has to look like. +Packages like ``mpich`` use traditional version numbers like +``3.0.4``. Other packages, like ``libdwarf`` use date-stamp versions +like ``20130729``. Versions can contain numbers, letters, dashes, +underscores, and periods. + +``spack compilers`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +You can see supported compilers by running ``spack compilers``. The +output will depend on the platform you run it on. + +.. command-output:: spack compilers Installing and uninstalling @@ -69,165 +95,433 @@ Installing and uninstalling ``spack install`` ~~~~~~~~~~~~~~~~~~~~~ -You can install any package from ``spack list``, using ``spack -install``. In the simplest case, if you just want the latest version -and you don't care about any configuration, you can just run ``spack -install ``: +``spack install`` will install any package that appears in the output +of ``spack list``. To install the latest version of a pacakge and all +of its dependencies, simply run ``spack install ``: .. code-block:: sh spack install mpileaks -This will fetch the tarball for ``mpileaks``, expand it, verify that -it was donwloaded without errors, build the package, and install it in -its own directory in ``$SPACK_HOME/opt``. If the requested packages -depends on other packages in order to build, then they will also be -fetched and installed. +Spack will fetch the tarball for ``mpileaks``, expand it, verify that +it was donwloaded without errors, build it, and install it in its own +directory under ``$SPACK_HOME/opt``. If the requested package depends +on other packages in order to build, Spack fetches them as well, and +installs them before it installs the requested package. Like the main +package, each dependency is also installed in its own directory. -Spack also allows you to ask for *specific* configurations of a -package. For example, if you want to install something with a -specific version, you can add ``@`` after the package name, followed -by the version you want: +Spack can also build *specific* configurations of a package. For +example, to install something with a specific version, add ``@`` after +the package name, followed by a version string: .. code-block:: sh spack install mpich@3.0.4 -You can install as many versions of the same pacakge as you want, and -they will not interfere with each other. Spack installs each package -into its own unique prefix. If you or another user links a library -against soething you install using Spack, it will continue to work -until you explicitly uninstall it. - -The version isn't all that you can customize on a spack command line. -Spack can install many configurations, with different versions, -compilers, compiler versions, compile-time options (variants), and -even architectures (e.g., on a machine that requires cross-compiling). -Spack is also unique in that it lets you customize the *dependencies* -you build a package with. That is, you could have two configurations -of the same version of a package: one built with boost 1.39.0, and the -other version built with version 1.43.0. - -Spack calls the descriptor used to refer to a particular package +Any number of configurations of the same package can be installed at +once without interfering with each other. This is good for multi-user +sites, as installing a version that one user needs will not disrupt +existing installations for other users. + +In addition to version configuraitons, Spack can customize the +compiler, compile-time options (variants), and platform (for cross +compiles) of an installation. Spack is unique in that it can also +configure the *dependencies* a package is built with. For example, +two configurations of the same version of a package, one built with +boost 1.39.0, and the other version built with version 1.43.0, can +coexist. + +This can all be done on the command line using special syntax. Spack +calls the descriptor used to refer to a particular package configuration a **spec**. In the command lines above, both -``mpileaks`` and ``mpileaks@3.0.4`` are specs. Specs and their syntax -are covered in more detail in :ref:`sec-specs`. - - +``mpileaks`` and ``mpileaks@3.0.4`` are specs. To customize +additional properties, simply add more attributes to the spec. Specs +and their syntax are covered in more detail in :ref:`sec-specs`. ``spack uninstall`` ~~~~~~~~~~~~~~~~~~~~~ -To uninstall a package, just type ``spack uninstall ``. This -will completely remove the directory in which the package was installed. +To uninstall a package, type ``spack uninstall ``. This will +completely remove the directory in which the package was installed. .. code-block:: sh spack uninstall mpich -If there are other installed packages depend on the package you're -uninstalling, spack will issue a warning to this effect. In general, -you should remove the other packages *before* removing the package -they depend on, or you risk breaking packages on your system. If you -still want to remove the package without regard for its dependencies, -you can run ``spack uninstall -f `` to override Spack's -warning. +If there are other installed packages depend on the package to be +uninstalled, spack will issue a warning. In general, it is safer to +remove dependent packages *before* removing their dependencies. Not +doing so risks breaking packages on your system. To remove a package +without regard for its dependencies, run ``spack uninstall -f +`` to override the warning. -If you have more than one version of the same package installed, spack -may not be able to figure out which on eyou want uninstalled. For -example, if you have both ``mpich@3.0.2`` and ``mpich@3.1`` installed, -and you type ``spack uninstall mpich``, then Spack will not know which -one you're referring to, and it will ask you to be more specific by -providing a version to differentiate, For example, ``spack uninstall -mpich@3.1`` is unambiguous. +A line like ``spack uninstall mpich`` may be ambiguous, if multiple +``mpich`` configurations are installed. For example, if both +``mpich@3.0.2`` and ``mpich@3.1`` are installed, it could refer to +either one, and Spack cannot determine which one to uninstall. Spack +will ask you to provide a version number to remove any ambiguity. For +example, ``spack uninstall mpich@3.1`` is unambiguous in the +above scenario. .. _sec-specs: -Specs +Specs & Dependencies ------------------------- -Dependencies -------------------------- +We now know that ``spack install`` and ``spack uninstall`` both take a +package name with an optional version specifier. In Spack, that +descriptor is called a *spec*. Spack uses specs to refer to a +particular build configuration (or configurations) of a package. +Specs are more than a package name and a version; you can use them to +specify the compiler, compiler version, architecture, compile options, +and dependency options for a build. In this section, we'll go over +the full syntax of specs. + +Here is an example of a much longer spec than we've seen thus far:: + + mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt =bgqos_0 ^callpath @1.1 %gcc@4.7.2 + +If provided to ``spack install``, this will install the ``mpileaks`` +library at some version between ``1.2`` and ``1.4`` (inclusive),, +built using ``gcc`` at version 4.7.5 for the Blue Gene/Q architecture, +with debug options enabled, and without Qt support. Additionally, it +says to link it with the ``callpath`` library (which it depends on), +and to build callpath with ``gcc`` 4.7.2. Most specs will not be as +complicated as this one, but this is a good example of what is +possible with specs. + +More formally, a spec consists of the following pieces: + +* Package name identifier (``mpileaks`` above) +* ``@`` Optional version specifier (``@1.2:1.4``) +* ``%`` Optional compiler specifier, with an optional compiler version + (``gcc`` or ``gcc@4.7.3``) +* ``+`` or ``-`` or ``~`` Optional variant specifiers (``+debug``, + ``-qt``, or ``~qt``) +* ``=`` Optional architecture specifier (``bgqos_0``) +* ``^`` Dependency specs (``^callpath@1.1``) + +There are two things to notice here. One is that specs are +recursively defined. That is, each dependency after ``^`` is a spec +itself. The second is that Notice that everything is optional +*except* for the initial package name identifier. Users can be as +vague or as specific as they want about the details of building +packages, and this makes spack good for beginners and experts alike. + +To really understand what's going on above, we need to think about how +software is structured. An executable or a library (these are +generally the artifacts produced by building software) depends on +other libraries in order to run. We can represent the relationship +between a package and its dependencies as a graph. Here is the full +dependency graph for ``mpileaks``: + +.. graphviz:: + + digraph { + mpileaks -> mpich + mpileaks -> callpath -> mpich + callpath -> dyninst + dyninst -> libdwarf -> libelf + dyninst -> libelf + } + +Each box above is a package and each arrow represents a dependency on +some other package. For example, we say that the package ``mpileaks`` +*depends on* ``callpath`` and ``mpich``. ``mpileaks`` also depends +*indirectly* on ``dyninst``, ``libdwarf``, and ``libelf``, in that +these libraries are dependencies of ``callpath``. To install +``mpileaks``, Spack has to build all of these packages. Dependency +graphs in Spack have to be acyclic, and the *depends on* relationship +is directional, so this is a *directed, acyclic graph* or *DAG*. + +The package name identifier in the spec is the root of some dependency +DAG, and the DAG itself is implicit. Spack knows the precise +dependencies among packages, but users do not need to know the full +DAG structure. Each ``^`` in the full spec refers to some dependency +of the root package. Spack will raise an error if you supply a name +after ``^`` that the root does not actually depend on (e.g. ``mpileaks +^emacs@23.3``). + +Spack further simplifies things by only allowing one configuration of +each package within any single build. Above, both ``mpileaks`` and +``callpath`` depend on ``mpich``, but ``mpich`` appears only once in +the DAG. You cannot build an ``mpileaks`` version that depends on one +version of ``mpich`` *and* on a ``callpath`` version that depends on +some *other* version of ``mpich``. In general, such a configuration +would likely behave unexpectedly at runtime, and Spack enforces this +to ensure a consistent runtime environment. + + +The point of specs is to abstract this full DAG from Spack users. If +a user does not care about the DAG at all, she can refer to mpileaks +by simply writing ``mpileaks``. If she knows that ``mpileaks`` +indirectly uses ``dyninst`` and she wants a particular version of +``dyninst``, then she can refer to ``mpileaks ^dyninst@8.1``. Spack +will fill in the rest when it parses the spec; the user only needs to +know package names and minimal details about their relationship. + +When spack prints out specs, it sorts package names alphabetically to +normalize the way they are displayed, but users do not need to worry +about this when they write specs. The only restriction on the order +of dependencies within a spec is that they appear *after* the root +package. For example, these two specs represent exactly the same +configuration: + +.. code-block:: sh + + mpileaks ^callpath@1.0 ^libelf@0.8.3 + mpileaks ^libelf@0.8.3 ^callpath@1.0 + +You can put all the same modifiers on dependency specs that you would +put on the root spec. That is, you can specify their versions, +compilers, variants, and architectures just like any other spec. +Specifiers are associated with the nearest package name to their left. +For example, above, ``@1.1`` and ``%gcc@4.7.2`` associates with the +``callpath`` package, while ``@1.2:1.4``, ``%gcc@4.7.5``, ``+debug``, +``-qt``, and ``=bgqos_0`` all associate with the ``mpileaks`` package. + +In the diagram above, ``mpileaks`` depends on ``mpich`` with an +unspecified version, but packages can depend on other packages with +*constraints* by adding more specifiers. For example, ``mpileaks`` +could depend on ``mpich@1.2:`` if it can only build with version +``1.2`` or higher of ``mpich``. + +Below are more details about the specifiers that you can add to specs. + +Version specifier +~~~~~~~~~~~~~~~~~~~~~~~ + +A version specifier comes somewhere after a package name and starts +with ``@``. It can be a single version, e.g. ``@1.0``, ``@3``, or +``@1.2a7``. Or, it can be a range of versions, such as ``@1.0:1.5`` +(all versions between ``1.0`` and ``1.5``, inclusive). Version ranges +can be open, e.g. ``:3`` means any version up to and including ``3``. +This would include ``3.4`` and ``3.4.2``. ``4.2:`` means any version +above and including ``4.2``. Finally, a version specifier can be a +set of arbitrary versions, such as ``@1.0,1.5,1.7`` (``1.0``, ``1.5``, +or ``1.7``). When you supply such a specifier to ``spack install``, +it constrains the set of versions that Spack will install. + +If the version spec is not provided, then Spack will choose one +according to policies set for the particular spack installation. If +the spec is ambiguous, i.e. it could match multiple versions, Spack +will choose a version within the spec's constraints according to +policies set for the particular Spack installation. + +Details about how versions are compared and how Spack determines if +one version is less than another are discussed in the developer guide. + + +Compiler specifier +~~~~~~~~~~~~~~~~~~~~~~~ + +A compiler specifier comes somewhere after a package name and starts +with ``%``. It tells Spack what compiler(s) a particular package +should be built with. After the ``%`` should come the name of some +registered Spack compiler. This might include ``gcc``, or ``intel``, +but the specific compilers available depend on the site. You can run +``spack compilers`` to get a list; more on this below. + +The compiler spec can be followed by an optional *compiler version*. +A compiler version specifier looks exactly like a package version +specifier. Version specifiers will associate with the nearest package +name or compiler specifier to their left in the spec. + +If the compiler spec is omitted, Spack will choose a default compiler +based on site policies. + + +Variants +~~~~~~~~~~~~~~~~~~~~~~~ + +Variants are named options associated with a particular package, and +they can be turned on or off. For example, above, supplying +``+debug`` causes ``mpileaks`` to be built with debug flags. The +names of particular variants available for a package depend on what +was provided by the package author. ``spack info `` will +provide information on what build variants are available. + +Depending on the package a variant may be on or off by default. For +``mpileaks`` here, ``debug`` is off by default, and we turned it on +with ``+debug``. If a package is on by default you can turn it off by +either adding ``-name`` or ``~name`` to the spec. + +There are two syntaxes here because, depending on context, ``~`` and +``-`` may mean different things. In most shells, the following will +result in the shell performing home directory substitution: + +.. code-block:: sh + + mpileaks ~debug # shell may try to substitute this! + mpileaks~debug # use this instead + +If there is a user called ``debug``, the ``~`` will be incorrectly +expanded. In this situation, you would want to write ``mpileaks +-debug``. However, ``-`` can be ambiguous when included after a +package name without spaces: + +.. code-block:: sh + + mpileaks-debug # wrong! + mpileaks -debug # right + +Spack allows the ``-`` character to be part of package names, so the +above will be interpreted as a request for the ``mpileaks-debug`` +package, not a request for ``mpileaks`` built without ``debug`` +options. In this scenario, you should write ``mpileaks~debug`` to +avoid ambiguity. + +When spack normalizes specs, it prints them out with no spaces and +uses only ``~`` for disabled variants. We allow ``-`` and spaces on +the command line is provided for convenience and legibility. + + +Architecture specifier +~~~~~~~~~~~~~~~~~~~~~~~ + +The architecture specifier starts with a ``=`` and also comes after +some package name within a spec. It allows a user to specify a +particular architecture for the package to be built. This is mostly +used for architectures that need cross-compilation, and in most cases, +users will not need to specify the architecture when they install a +package. + + +.. _sec-virtual-dependencies: Virtual dependencies ------------------------- -Versions, compilers, and architectures ----------------------------------------- +The dependence graph for ``mpileaks`` we saw above wasn't *quite* +accurate. ``mpileaks`` uses MPI, which is an interface that has many +different implementations. Above, we showed ``mpileaks`` and +``callpath`` depending on ``mpich``, which is one *particular* +implementation of MPI. However, we could build either with another +implementation, such as ``openmpi`` or ``mvapich``. -``spack versions`` -~~~~~~~~~~~~~~~~~~~~~~~~ +Spack represents interfaces like this using *virtual dependencies*. +The real dependency DAG for ``mpileaks`` looks like this: -``spack compilers`` -~~~~~~~~~~~~~~~~~~~~~~~~ +.. graphviz:: -Architectures -~~~~~~~~~~~~~~~~~~~~~~~~ + digraph { + mpi [color=red] + mpileaks -> mpi + mpileaks -> callpath -> mpi + callpath -> dyninst + dyninst -> libdwarf -> libelf + dyninst -> libelf + } + +Notice that ``mpich`` has now been replaced with ``mpi``. There is no +*real* MPI package, but some packages *provide* the MPI interface, and +these packages can be substituted in for ``mpi`` when ``mpileaks`` is +built. + +You can see what virtual packages a particular package provides by +getting info on it: + +.. command-output:: spack info mpich + :ellipsis: 10 + +Spack is unique in that its virtual packages can be versioned, just +like regular packages. A particular version of a package may provide +a particular version of a virtual package, and we can see above that +``mpich`` versions ``1`` and above provide all interfaces up to ``1``, +and ``mpich`` versions ``3`` and above provide ``mpi`` versions up to +``3``. A package can *depend on* a particular version of a virtual +package, e.g. if an application needs MPI-2 functions, it can depend +on ``mpi@2:`` to indicate that it needs some implementation that +provides MPI-2 functions. + +``spack providers`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can see what packages provide a particular virtual package using +``spack providers``. If you wanted to see what packages provide +``mpi``, you would just run: + +.. command-output:: spack providers mpi + +And if you *only* wanted to see packages that provide MPI-2, you would +add a version specifier to the spec: + +.. command-output:: spack providers mpi@2 -Spack's specs allow insatllations for multiple architectures to coexist -within the same prefix. It is also intended to support multiple -architecutres for cross-compilation. +Notice that the package versions that provide insufficient MPI +versions are now filtered out. Package lifecycle ------------------------------ -The ``spack install`` command performs a number of tasks before it -finally installs each package. It downloads an archive, expands it in -a temporary directory, and only then performs the installation. Spack -has several commands that allow finer-grained control over each of -these stages of the build process. +``spack install`` command performs a number of tasks before it finally +installs each package. It downloads an archive, expands it in a +temporary directory, and then performs the installation. Spack has +several commands that allow finer-grained control over each stage of +the build process. ``spack fetch`` ~~~~~~~~~~~~~~~~~ -This is the first step of ``spack install``. It takes a spec and -determines the correct download URL to use for the requested package -version. It then downloads the archive, checks it against an MD5 -checksum, and stores it in a staging directory if the check was -successful. The staging directory will be located under -``$SPACK_HOME/var/spack``. +The first step of ``spack install``. Takes a spec and determines the +correct download URL to use for the requested package version, then +downloads the archive, checks it against an MD5 checksum, and stores +it in a staging directory if the check was successful. The staging +directory will be located under ``$SPACK_HOME/var/spack``. -If run after the archive has already been downloaded, ``spack fetch`` -is idempotent and will not download the archive again. +When run after the archive has already been downloaded, ``spack +fetch`` is idempotent and will not download the archive again. ``spack stage`` ~~~~~~~~~~~~~~~~~ -This is the second step in installation after ``spack fetch``. It -expands the downloaded archive in its temporary directory, where it -will be built by ``spack install``. If the archive has already been -expanded, then this command does nothing. +The second step in ``spack install`` after ``spack fetch``. Expands +the downloaded archive in its temporary directory, where it will be +built by ``spack install``. Similar to ``fetch``, if the archive has +already been expanded, ``stage`` is idempotent. ``spack clean`` ~~~~~~~~~~~~~~~~~ -This command has several variations, each undoing one of the -installation tasks. They are: +There are several variations of ``spack clean``. With no arguments, +``spack clean`` runs ``make clean`` in the expanded archive directory. +This is useful if an attempted build failed, and something needs to be +changed to get a package to build. If a particular package does not +have a ``make clean`` target, this will do nothing. + +``spack clean -w / --work`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deletes the entire build directory and re-expands it from the downloaded +archive. This is useful if a package does not support a proper ``make clean`` +target. + +``spack clean -d / --dist`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deletes the build directory *and* the downloaded archive. If +``fetch``, ``stage``, or ``install`` are run again after this, the +process will start from scratch, and the archive archive will be +downloaded again. Useful if somehow a bad archive is downloaded +accidentally and needs to be cleaned out of the staging area. -``spack clean`` - Runs ``make clean`` in the expanded archive directory. This is useful - if an attempted build failed, and something needs to be changed to get - a package to build. If a particular package does not have a ``make clean`` - target, this will do nothing. - -``spack clean -w`` or ``spack clean --work`` - This deletes the entire build directory and re-expands it from the downloaded - archive. This is useful if a package does not support a proper ``make clean`` - target. - -``spack clean -d`` or ``spack clean --dist`` - This deletes the build directory *and* the downloaded archive. If - ``fetch``, ``stage``, or ``install`` are run again after this, the - process will start from scratch, and the archive archive will be - downloaded again. Useful if somehow a bad archive is downloaded - accidentally and needs to be cleaned out of the staging area. +``spack purge`` +~~~~~~~~~~~~~~~~~ +Cleans up *everything* in the build directory. You can use this to +recover disk space if temporary files from interrupted or failed +installs accumulate in the staging area. +Dirty Installs +-------------------- -``spack purge`` -~~~~~~~~~~~~~~~~~ +By default, ``spack install`` will delete the staging area once a +pacakge has been successfully built and installed, *or* if an error +occurs during the build. Use ``spack install --dirty`` or ``spack +install -d`` to leave the build directory intact. This allows you to +inspect the build directory and potentially fix the build. You can +use ``purge`` or ``clean`` later to get rid of the unwanted temporary +files. diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 68b70f359f..1303503bd6 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -17,8 +17,17 @@ import sys, os # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) + +# Add the Spack bin directory to the path so that we can use its output in docs. os.environ['PATH'] += os.pathsep + '../../../bin' +# Set an environment variable so that colify will print output like it would to +# a terminal. +os.environ['COLIFY_TTY'] = 'true' +os.environ['COLUMNS'] = '80' +os.environ['LINES'] = '25' + +# Enable todo items todo_include_todos = True # -- General configuration ----------------------------------------------------- @@ -33,6 +42,15 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxcontrib.programoutput'] +# Set default graphviz options +graphviz_dot_args = [ + '-Grankdir=LR', '-Gbgcolor=transparent', + '-Nshape=box', '-Nfontname=monaco', '-Nfontsize=10'] + +# Get nice vector graphics +graphviz_output_format = "svg" + + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index 7039563698..27e821d3b2 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,16 +1,16 @@ Feature Overview ================== -This is an overview of features that make Spack different from other -`package managers `_ -and `port systems `_. +This is a high-level overview of features that make Spack different +from other `package managers +`_ and `port +systems `_. Simple package installation ---------------------------- -Installing packages is easy with Spack when you just want the default -version. This installs the latest version of mpileaks and all of its -dependencies: +Installing the default version of a package is simple. This will install +the latest version of the ``mpileaks`` package and all of its dependencies: .. code-block:: sh @@ -19,29 +19,35 @@ dependencies: Custom versions & configurations ------------------------------------------- -If there's some aspect of your package that you want to customize, you -can do that too. +Spack allows installation to be customized. Users can specify the +version, build compiler, compile-time options, and cross-compile +platform, all on the command line. .. code-block:: sh # Install a particular version by appending @ $ spack install mpileaks@1.1.2 - # Or your favorite compiler (and its version), with % + # Specify a compiler (and its version), with % $ spack install mpileaks@1.1.2 %gcc@4.7.3 - # Add some special compile-time options with + + # Add special compile-time options with + $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug # Cross-compile for a different architecture with = $ spack install mpileaks@1.1.2 =bgqos_0 +Users can specify as many or few options as they care about. Spack +will fill in the unspecified values with sensible defaults. + + Customize dependencies ------------------------------------- -You can customize package dependencies with ``^``. Suppose that -``mpileaks`` depends indirectly on ``libelf`` and ``libdwarf``. Using -``^``, you can add custom configurations for the dependencies, too. +Spack is unique in that it allows *dependencies* of a particualr +installation to be customized. Suppose that ``mpileaks`` depends +indirectly on ``libelf`` and ``libdwarf``. Using ``^``, users can add +custom configurations for the dependencies, as well: .. code-block:: sh @@ -52,24 +58,24 @@ You can customize package dependencies with ``^``. Suppose that Non-destructive installs ------------------------------------- -Spack installs every unique package configuration in its own prefix, -so you can install as many different versions and configurations as -you want. New installs will not break existing ones. - +Spack installs every unique package/dependency configuration into its +own prefix, so new installs will not break existing ones. Packages can peacefully coexist ------------------------------------- -Spack uses ``RPATH`` everywhere, so users do not need to customize -``LD_LIBRARY_PATH``. If you use a library or run a program, it will -run the way you built it. - +Spack avoids library misconfiguration by using ``RPATH`` to link +dependencies. When a user links a library or runs a program, it is +tied to the dependencies it was built with, so there is no need to +manipulate ``LD_LIBRARY_PATH`` at runtime. Creating packages is easy ------------------------------------- -To create your own packages, give spack the tarball URL. Spack -creates all the boilerplate for you. +To create a new packages, all Spack needs is a URL for the source +archive. The ``spack create`` command will create a boilerplate +package file, and the package authors can fill in specific build steps +in pure Python. .. code-block:: sh @@ -91,6 +97,6 @@ Creates ``mpileaks.py``: make() make("install") -Packages are pure python, so you have complete freedom when writing -build code. Spack also provides a number of feature that make it -easier to write packages. +Spack also provides wrapper functions around common commands like +``configure``, ``make``, and ``cmake`` to make writing packages +simple. diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 092d696983..b6c1e537fb 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -6,11 +6,22 @@ Spack Documentation ================================= -Spack builds and installs software the way you want it. Other tools -let you install the latest version once. Spack lets you install the -versions you want, built with the compilers, libraries, and options -you want. Spack is non-destructive; installing a new version does not -break your old installs. See the :doc:`features` for more highlights. +Spack is a package management tool designed to support multiple +versions and configurations of software on a wide variety of platforms +and environments. It was designed for large supercomputing centers, +where many users and application teams share common installations of +software on clusters with exotic architectures, using libraries that +do not have a standard ABI. Spack is non-destructive: installing a +new version does not break existing installations, so many +configurations can coexist on the same system. + +Most importantly, Spack is *simple*. It offers a simple *spec* syntax +so that users can specify versions and configuration options +concisely. Spack is also simple for package authors: package files +are writtin in pure Python, and specs allow package authors to write a +single build script for many different builds of the same package. + +See the :doc:`features` for examples and highlights. Get spack and install your first package: diff --git a/lib/spack/spack/arch.py b/lib/spack/spack/arch.py deleted file mode 100644 index c4914de1a0..0000000000 --- a/lib/spack/spack/arch.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import platform as py_platform - -import spack -import spack.error as serr -from spack.version import Version -from spack.util.lang import memoized - - -class InvalidSysTypeError(serr.SpackError): - def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) - - -class NoSysTypeError(serr.SpackError): - def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() - else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - - return "macosx_{}_{}".format( - Version(mac_ver).up_to(2), py_platform.machine()) - - -@memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_mac_sys_type] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type == None: - raise NoSysTypeError() - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - - return sys_type diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py new file mode 100644 index 0000000000..c4914de1a0 --- /dev/null +++ b/lib/spack/spack/architecture.py @@ -0,0 +1,67 @@ +import os +import platform as py_platform + +import spack +import spack.error as serr +from spack.version import Version +from spack.util.lang import memoized + + +class InvalidSysTypeError(serr.SpackError): + def __init__(self, sys_type): + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) + + +class NoSysTypeError(serr.SpackError): + def __init__(self): + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() + else: + return spack.sys_type + + +def get_sys_type_from_environment(): + """Return $SYS_TYPE or None if it's not defined.""" + return os.environ.get('SYS_TYPE') + + +def get_mac_sys_type(): + """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + mac_ver = py_platform.mac_ver()[0] + if not mac_ver: + return None + + return "macosx_{}_{}".format( + Version(mac_ver).up_to(2), py_platform.machine()) + + +@memoized +def sys_type(): + """Returns a SysType for the current machine.""" + methods = [get_sys_type_from_spack_globals, + get_sys_type_from_environment, + get_mac_sys_type] + + # search for a method that doesn't return None + sys_type = None + for method in methods: + sys_type = method() + if sys_type: break + + # Couldn't determine the sys_type for this machine. + if sys_type == None: + raise NoSysTypeError() + + if not isinstance(sys_type, basestring): + raise InvalidSysTypeError(sys_type) + + return sys_type diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py new file mode 100644 index 0000000000..a6d4879fdc --- /dev/null +++ b/lib/spack/spack/cmd/arch.py @@ -0,0 +1,11 @@ +import spack +import spack.architecture as architecture + +description = "Print the architecture for this machine" + +def arch(parser, args): + configured_sys_type = arch.get_sys_type_from_spack_globals() + if not configured_sys_type: + configured_sys_type = "autodetect" + print "Configured sys_type: %s" % configured_sys_type + print "Autodetected default sys_type: %s" % arch.sys_type() diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 1af74e0f60..b3ad463988 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -12,7 +12,7 @@ def setup_parser(subparser): action='store_true', dest='ignore_dependencies', help="Do not try to install dependencies of requested packages.") subparser.add_argument('-d', '--dirty', action='store_true', dest='dirty', - help="Don't clean up partially completed build/installation on error.") + help="Don't clean up staging area when install completes.") subparser.add_argument('packages', nargs=argparse.REMAINDER, help="specs of packages to install") diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py new file mode 100644 index 0000000000..5546c7f312 --- /dev/null +++ b/lib/spack/spack/cmd/providers.py @@ -0,0 +1,17 @@ +import os +import argparse + +import spack.cmd +import spack.packages +from spack.colify import colify + +description ="List packages that provide a particular virtual package" + +def setup_parser(subparser): + subparser.add_argument('vpkg_spec', metavar='VPACKAGE_SPEC', nargs=argparse.REMAINDER, + help='Find packages that provide this virtual package') + + +def providers(parser, args): + for spec in spack.cmd.parse_specs(args.vpkg_spec): + colify(sorted(spack.packages.providers_for(spec)), indent=4) diff --git a/lib/spack/spack/cmd/sys_type.py b/lib/spack/spack/cmd/sys_type.py deleted file mode 100644 index 7f57fb16ff..0000000000 --- a/lib/spack/spack/cmd/sys_type.py +++ /dev/null @@ -1,11 +0,0 @@ -import spack -import spack.arch as arch - -description = "Print the spack sys_type for this machine" - -def sys_type(parser, args): - configured_sys_type = arch.get_sys_type_from_spack_globals() - if not configured_sys_type: - configured_sys_type = "autodetect" - print "Configured sys_type: %s" % configured_sys_type - print "Autodetected default sys_type: %s" % arch.sys_type() diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index d18b1c1265..3ceee0eca7 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -1,7 +1,4 @@ import os -import re -from subprocess import CalledProcessError - import spack.packages as packages from spack.colify import colify diff --git a/lib/spack/spack/colify.py b/lib/spack/spack/colify.py index 0ab2159197..02c4003681 100644 --- a/lib/spack/spack/colify.py +++ b/lib/spack/spack/colify.py @@ -88,6 +88,11 @@ def config_uniform_cols(elts, console_cols, padding): return config +def isatty(ostream): + force = os.environ.get('COLIFY_TTY', 'false').lower() != 'false' + return force or ostream.isatty() + + def colify(elts, **options): # Get keyword arguments or set defaults output = options.get("output", sys.stdout) @@ -99,7 +104,7 @@ def colify(elts, **options): if not elts: return - if not output.isatty(): + if not isatty(output): for elt in elts: output.write("%s\n" % elt) return diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f29f33f144..029b6d0d03 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -9,7 +9,7 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ -import spack.arch +import spack.architecture import spack.compilers import spack.packages import spack.spec @@ -62,7 +62,7 @@ class DefaultConcretizer(object): if spec.root.architecture: spec.architecture = spec.root.architecture else: - spec.architecture = spack.arch.sys_type() + spec.architecture = spack.architecture.sys_type() def concretize_compiler(self, spec): diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index a2a14cfad1..166c463cdf 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -1,6 +1,5 @@ import os -import spack.arch as arch from spack.version import Version from spack.util.filesystem import * from spack.util.executable import * diff --git a/lib/spack/spack/multi_function.py b/lib/spack/spack/multi_function.py index abceb156c0..30146b2139 100644 --- a/lib/spack/spack/multi_function.py +++ b/lib/spack/spack/multi_function.py @@ -12,7 +12,7 @@ of the dependencies of the package. import sys import functools -import arch +import spack.architecture import spack.error as serr class NoSuchVersionError(serr.SpackError): diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index 68a5f7978b..0fd22fa480 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -11,7 +11,6 @@ import spack.spec import spack.tty as tty from spack.util.filesystem import new_path from spack.util.lang import list_modules -import spack.arch as arch # Valid package names can contain '-' but can't start with it. valid_package_re = r'^\w[\w-]*$' diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 17ef523f57..748fa9b36a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -75,7 +75,6 @@ import spack.error import spack.compilers import spack.compilers.gcc import spack.packages as packages -import spack.arch as arch from spack.version import * from spack.color import * -- cgit v1.2.3-60-g2f50