rework spack help (#3033)

- Full help is now only generated lazily, when needed.
  - Executing specific commands doesn't require loading all of them.
  - All commands are only loaded if we need them for help.

- There is now short and long help:
  - short help (spack help) shows only basic spack options
  - long help (spack help -a) shows all spack options
  - Both divide help on commands into high-level sections

- Commands now specify attributes from which help is auto-generated:
  - description: used in help to describe the command.
  - section: help section
  - level: short or long

- Clean up command descriptions

- Add a `spack docs` command to open full documentation
  in the browser.

- move `spack doc` command to `spack pydoc` for clarity

- Add a `spack --spec` command to show documentation on 
  the spec syntax.

55 files changed, 711 insertions, 17 deletions

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 73963b848c..27283d10a9 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -217,5 +217,5 @@ __all__ += [
 
 # Add default values for attributes that would otherwise be modified from
 # Spack main script
-debug = True
+debug = False
 spack_working_dir = None
diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py
index f21799753b..f7e826efd6 100644
--- a/lib/spack/spack/cmd/activate.py
+++ b/lib/spack/spack/cmd/activate.py
@@ -28,6 +28,8 @@ import spack
 import spack.cmd
 
 description = "activate a package extension"
+section = "extensions"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py
index 1079e7f215..d4241dcae9 100644
--- a/lib/spack/spack/cmd/arch.py
+++ b/lib/spack/spack/cmd/arch.py
@@ -27,6 +27,8 @@ from __future__ import print_function
 import spack.architecture as architecture
 
 description = "print architecture information about this machine"
+section = "system"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py
index a804086a38..b6daf4f09b 100644
--- a/lib/spack/spack/cmd/bootstrap.py
+++ b/lib/spack/spack/cmd/bootstrap.py
@@ -33,6 +33,8 @@ from spack.util.executable import ProcessError, which
 _SPACK_UPSTREAM = 'https://github.com/llnl/spack'
 
 description = "create a new installation of spack in another prefix"
+section = "admin"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/build.py b/lib/spack/spack/cmd/build.py
index 877f2ce0cf..cc63c6593b 100644
--- a/lib/spack/spack/cmd/build.py
+++ b/lib/spack/spack/cmd/build.py
@@ -27,6 +27,9 @@ import spack.cmd.configure as cfg
 from spack import *
 
 description = 'stops at build stage when installing a package, if possible'
+section = "build"
+level = "long"
+
 
 build_system_to_phase = {
     AutotoolsPackage: 'build',
diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py
index 784ad4ac83..531f3c59fd 100644
--- a/lib/spack/spack/cmd/cd.py
+++ b/lib/spack/spack/cmd/cd.py
@@ -26,6 +26,8 @@ import spack.cmd.location
 import spack.modules
 
 description = "cd to spack directories in the shell"
+section = "environment"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py
index fda9beed27..d8a17fd383 100644
--- a/lib/spack/spack/cmd/checksum.py
+++ b/lib/spack/spack/cmd/checksum.py
@@ -36,6 +36,8 @@ from spack.util.naming import *
 from spack.version import *
 
 description = "checksum available versions of a package"
+section = "packaging"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index 6c70b5bd38..23507822ef 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -30,6 +30,8 @@ import spack
 import spack.cmd
 
 description = "remove build stage and source tarball for packages"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index 6067d44c5e..f2eeca20ab 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -39,6 +39,8 @@ from spack.spec import CompilerSpec, ArchSpec
 from spack.util.environment import get_path
 
 description = "manage compilers"
+section = "system"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py
index 934fc6cf06..f0e21f987e 100644
--- a/lib/spack/spack/cmd/compilers.py
+++ b/lib/spack/spack/cmd/compilers.py
@@ -25,7 +25,9 @@
 import spack
 from spack.cmd.compiler import compiler_list
 
-description = "list available compilers, same as 'spack compiler list'"
+description = "list available compilers"
+section = "system"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py
index a647e3ed6e..61c2c6f0e8 100644
--- a/lib/spack/spack/cmd/config.py
+++ b/lib/spack/spack/cmd/config.py
@@ -25,6 +25,8 @@
 import spack.config
 
 description = "get and set configuration options"
+section = "config"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py
index 7f6c07c34b..7cab566052 100644
--- a/lib/spack/spack/cmd/configure.py
+++ b/lib/spack/spack/cmd/configure.py
@@ -30,7 +30,9 @@ import spack.cmd.install as inst
 
 from spack import *
 
-description = 'stops at configuration stage when installing a package, if possible'  # NOQA: ignore=E501
+description = 'stage and configure a package but do not install'
+section = "build"
+level = "long"
 
 
 build_system_to_phase = {
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index adaf388387..89ba050a53 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -40,6 +40,9 @@ from spack.util.naming import *
 from spack.url import *
 
 description = "create a new package file"
+section = "packaging"
+level = "short"
+
 
 package_template = '''\
 ##############################################################################
diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py
index 7ea2039236..3d8020d064 100644
--- a/lib/spack/spack/cmd/deactivate.py
+++ b/lib/spack/spack/cmd/deactivate.py
@@ -31,6 +31,8 @@ import spack.store
 from spack.graph import topological_sort
 
 description = "deactivate a package extension"
+section = "extensions"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py
index 06dea9ea70..ba5f783839 100644
--- a/lib/spack/spack/cmd/debug.py
+++ b/lib/spack/spack/cmd/debug.py
@@ -34,6 +34,8 @@ import spack
 from spack.util.executable import which
 
 description = "debugging commands for troubleshooting Spack"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py
index c752ffb943..6c481548d3 100644
--- a/lib/spack/spack/cmd/dependents.py
+++ b/lib/spack/spack/cmd/dependents.py
@@ -31,6 +31,8 @@ import spack.store
 import spack.cmd
 
 description = "show installed packages that depend on another"
+section = "basic"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py
index c67e189f73..14d28bb3f4 100644
--- a/lib/spack/spack/cmd/diy.py
+++ b/lib/spack/spack/cmd/diy.py
@@ -34,6 +34,8 @@ import spack.cmd.common.arguments as arguments
 from spack.stage import DIYStage
 
 description = "do-it-yourself: build from an existing source directory"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/docs.py b/lib/spack/spack/cmd/docs.py
new file mode 100644
index 0000000000..fe026da4a7
--- /dev/null
+++ b/lib/spack/spack/cmd/docs.py
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import webbrowser
+
+description = 'open spack documentation in a web browser'
+section = 'help'
+level = 'short'
+
+
+def docs(parser, args):
+    webbrowser.open('https://spack.readthedocs.io')
diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py
index 01f2b61887..0287b8cd67 100644
--- a/lib/spack/spack/cmd/edit.py
+++ b/lib/spack/spack/cmd/edit.py
@@ -33,6 +33,8 @@ from spack.spec import Spec
 from spack.repository import Repo
 
 description = "open package files in $EDITOR"
+section = "packaging"
+level = "short"
 
 
 def edit_package(name, repo_path, namespace):
diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py
index ed18940ac0..034b710b85 100644
--- a/lib/spack/spack/cmd/env.py
+++ b/lib/spack/spack/cmd/env.py
@@ -31,7 +31,9 @@ import llnl.util.tty as tty
 import spack.cmd
 import spack.build_environment as build_env
 
-description = "run a command with the install environment for a spec"
+description = "show install environment for a spec, and run commands"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py
index 94a3e8288f..d073d42cc3 100644
--- a/lib/spack/spack/cmd/extensions.py
+++ b/lib/spack/spack/cmd/extensions.py
@@ -33,6 +33,8 @@ import spack.cmd.find
 import spack.store
 
 description = "list extensions for package"
+section = "extensions"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py
index 35cc23a963..cf39d4a40b 100644
--- a/lib/spack/spack/cmd/fetch.py
+++ b/lib/spack/spack/cmd/fetch.py
@@ -28,6 +28,8 @@ import spack
 import spack.cmd
 
 description = "fetch archives for packages"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index 3a6d8270fb..0142155fe1 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -29,7 +29,9 @@ import spack.cmd.common.arguments as arguments
 
 from spack.cmd import display_specs
 
-description = "find installed spack packages"
+description = "list and search installed packages"
+section = "basic"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/flake8.py b/lib/spack/spack/cmd/flake8.py
index 42d36a3beb..9753b20e78 100644
--- a/lib/spack/spack/cmd/flake8.py
+++ b/lib/spack/spack/cmd/flake8.py
@@ -36,7 +36,11 @@ from llnl.util.filesystem import *
 import spack
 from spack.util.executable import *
 
+
 description = "runs source code style checks on Spack. requires flake8"
+section = "developer"
+level = "long"
+
 
 """List of directories to exclude from checks."""
 exclude_directories = [spack.external_path]
diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py
index ee401d8fb7..3a82f39ce5 100644
--- a/lib/spack/spack/cmd/graph.py
+++ b/lib/spack/spack/cmd/graph.py
@@ -34,6 +34,8 @@ from spack.spec import *
 from spack.graph import *
 
 description = "generate graphs of package dependency relationships"
+section = "basic"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py
index e867ca1295..313b082e2f 100644
--- a/lib/spack/spack/cmd/help.py
+++ b/lib/spack/spack/cmd/help.py
@@ -22,16 +22,100 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import sys
+from llnl.util.tty import colorize
+
 description = "get help on spack and its commands"
+section = "help"
+level = "short"
+
+#
+# These are longer guides on particular aspects of Spack. Currently there
+# is only one on spec syntax.
+#
+spec_guide = """\
+spec expression syntax:
+
+  package [constraints] [^dependency [constraints] ...]
+
+  package                           any package from 'spack list'
+
+  constraints:
+    versions:
+      @c{@version}                      single version
+      @c{@min:max}                      version range (inclusive)
+      @c{@min:}                         version <min> or higher
+      @c{@:max}                         up to version <max> (inclusive)
+
+    compilers:
+      @g{%compiler}                     build with <compiler>
+      @g{%compiler@version}             build with specific compiler version
+      @g{%compiler@min:max}             specific version range (see above)
+
+    variants:
+      @B{+variant}                      enable <variant>
+      @r{-variant} or @r{~variant}          disable <variant>
+      @B{variant=value}                 set non-boolean <variant> to <value>
+      @B{variant=value1,value2,value3}  set multi-value <variant> values
+
+    architecture variants:
+      @m{target=target}                 specific <target> processor
+      @m{os=operating_system}           specific <operating_system>
+      @m{platform=platform}             linux, darwin, cray, bgq, etc.
+      @m{arch=platform-os-target}       shortcut for all three above
+
+    cross-compiling:
+      @m{os=backend} or @m{os=be}           build for compute node (backend)
+      @m{os=frontend} or @m{os=fe}          build for login node (frontend)
+
+    dependencies:
+      ^dependency [constraints]     specify constraints on dependencies
+
+  examples:
+      hdf5                          any hdf5 configuration
+      hdf5 @c{@1.10.1}                  hdf5 version 1.10.1
+      hdf5 @c{@1.8:}                    hdf5 1.8 or higher
+      hdf5 @c{@1.8:} @g{%gcc}               hdf5 1.8 or higher built with gcc
+      hdf5 @B{+mpi}                     hdf5 with mpi enabled
+      hdf5 @r{~mpi}                     hdf5 with mpi disabled
+      hdf5 @B{+mpi} ^mpich              hdf5 with mpi, using mpich
+      hdf5 @B{+mpi} ^openmpi@c{@1.7}        hdf5 wtih mpi, using openmpi 1.7
+      boxlib @B{dim=2}                  boxlib built for 2 dimensions
+      libdwarf @g{%intel} ^libelf@g{%gcc}
+          libdwarf, built with intel compiler, linked to libelf built with gcc
+      mvapich2 @g{%pgi} @B{fabrics=psm,mrail,sock}
+          mvapich2, built with pgi compiler, with support for multiple fabrics
+"""
+
+
+guides = {
+    'spec': spec_guide,
+}
 
 
 def setup_parser(subparser):
-    subparser.add_argument('help_command', nargs='?', default=None,
-                           help='command to get help on')
+    help_cmd_group = subparser.add_mutually_exclusive_group()
+    help_cmd_group.add_argument('help_command', nargs='?', default=None,
+                                help='command to get help on')
+
+    help_all_group = subparser.add_mutually_exclusive_group()
+    help_all_group.add_argument(
+        '-a', '--all', action='store_const', const='long', default='short',
+        help='print all available commands')
+
+    help_spec_group = subparser.add_mutually_exclusive_group()
+    help_spec_group.add_argument(
+        '--spec', action='store_const', dest='guide', const='spec',
+        default=None, help='print all available commands')
 
 
 def help(parser, args):
+    if args.guide:
+        print(colorize(guides[args.guide]))
+        return 0
+
     if args.help_command:
+        parser.add_command(args.help_command)
         parser.parse_args([args.help_command, '-h'])
     else:
-        parser.print_help()
+        sys.stdout.write(parser.format_help(level=args.all))
diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py
index 86ec839b90..62de5484af 100644
--- a/lib/spack/spack/cmd/info.py
+++ b/lib/spack/spack/cmd/info.py
@@ -31,6 +31,8 @@ import spack
 import spack.fetch_strategy as fs
 
 description = "get detailed information on a particular package"
+section = "basic"
+level = "short"
 
 
 def padder(str_list, extra=0):
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index fb01fc2d5e..87fad76181 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -41,6 +41,8 @@ from spack.fetch_strategy import FetchError
 from spack.package import PackageBase
 
 description = "build and install packages"
+section = "build"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py
index bcfb092945..72be99d260 100644
--- a/lib/spack/spack/cmd/list.py
+++ b/lib/spack/spack/cmd/list.py
@@ -35,7 +35,10 @@ import llnl.util.tty as tty
 import spack
 from llnl.util.tty.colify import colify
 
-description = "print available spack packages to stdout in different formats"
+description = "list and search available packages"
+section = "basic"
+level = "short"
+
 
 formatters = {}
 
diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py
index cdc3a741ae..106a95c9c2 100644
--- a/lib/spack/spack/cmd/load.py
+++ b/lib/spack/spack/cmd/load.py
@@ -25,7 +25,9 @@
 import argparse
 import spack.modules
 
-description = "add package to environment using modules"
+description = "add package to environment using `module load`"
+section = "environment"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py
index d1a7825630..e713d028d2 100644
--- a/lib/spack/spack/cmd/location.py
+++ b/lib/spack/spack/cmd/location.py
@@ -31,6 +31,8 @@ import spack
 import spack.cmd
 
 description = "print out locations of various directories used by Spack"
+section = "environment"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py
index fc205cc693..1d121f0120 100644
--- a/lib/spack/spack/cmd/md5.py
+++ b/lib/spack/spack/cmd/md5.py
@@ -32,6 +32,8 @@ import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 
 description = "calculate md5 checksums for files/urls"
+section = "packaging"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 528fcbfc3f..e5b3b492b4 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -38,6 +38,8 @@ from spack.error import SpackError
 from spack.util.spack_yaml import syaml_dict
 
 description = "manage mirrors"
+section = "config"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py
index 37c79a358b..f8253aad6f 100644
--- a/lib/spack/spack/cmd/module.py
+++ b/lib/spack/spack/cmd/module.py
@@ -36,6 +36,9 @@ from spack.cmd.common import arguments
 from spack.modules import module_types
 
 description = "manipulate module files"
+section = "environment"
+level = "short"
+
 
 # Dictionary that will be populated with the list of sub-commands
 # Each sub-command must be callable and accept 3 arguments :
diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py
index 2e332554ad..dfe45b0494 100644
--- a/lib/spack/spack/cmd/patch.py
+++ b/lib/spack/spack/cmd/patch.py
@@ -30,6 +30,8 @@ import spack
 
 
 description = "patch expanded archive sources in preparation for install"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py
index 12dcb81792..aca69e9c99 100644
--- a/lib/spack/spack/cmd/pkg.py
+++ b/lib/spack/spack/cmd/pkg.py
@@ -34,6 +34,8 @@ import spack
 from spack.util.executable import *
 
 description = "query packages associated with particular git revisions"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py
index 470e3e5ed2..f30c28a951 100644
--- a/lib/spack/spack/cmd/providers.py
+++ b/lib/spack/spack/cmd/providers.py
@@ -30,6 +30,8 @@ import spack
 import spack.cmd
 
 description = "list packages that provide a particular virtual package"
+section = "basic"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py
index 56165d5d97..b7ebb0fc69 100644
--- a/lib/spack/spack/cmd/purge.py
+++ b/lib/spack/spack/cmd/purge.py
@@ -26,6 +26,8 @@ import spack
 import spack.stage as stage
 
 description = "remove temporary build files and/or downloaded archives"
+section = "admin"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/doc.py b/lib/spack/spack/cmd/pydoc.py
index 12ae6b4973..c9003184c4 100644
--- a/lib/spack/spack/cmd/doc.py
+++ b/lib/spack/spack/cmd/pydoc.py
@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory.
 #
 # This file is part of Spack.
@@ -24,11 +24,13 @@
 ##############################################################################
 
 description = "run pydoc from within spack"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
     subparser.add_argument('entity', help="run pydoc help on entity")
 
 
-def doc(parser, args):
+def pydoc(parser, args):
     help(args.entity)
diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py
index 6df9507580..3c4fbf9e87 100644
--- a/lib/spack/spack/cmd/python.py
+++ b/lib/spack/spack/cmd/python.py
@@ -32,6 +32,8 @@ import spack
 
 
 description = "launch an interpreter as spack would launch a command"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py
index 0bbd85069f..dff127bc06 100644
--- a/lib/spack/spack/cmd/reindex.py
+++ b/lib/spack/spack/cmd/reindex.py
@@ -26,6 +26,9 @@ import spack
 import spack.store
 description = "rebuild Spack's package database"
 
+section = "admin"
+level = "long"
+
 
 def reindex(parser, args):
     spack.store.db.reindex(spack.store.layout)
diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py
index dd75f148c2..5beb0083e2 100644
--- a/lib/spack/spack/cmd/repo.py
+++ b/lib/spack/spack/cmd/repo.py
@@ -33,6 +33,8 @@ import spack.config
 from spack.repository import *
 
 description = "manage package source repositories"
+section = "config"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py
index 36fee9237b..4cecf4b42e 100644
--- a/lib/spack/spack/cmd/restage.py
+++ b/lib/spack/spack/cmd/restage.py
@@ -30,6 +30,8 @@ import spack
 import spack.cmd
 
 description = "revert checked out package source code"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py
index 82d00f4e11..79e7bca1ab 100644
--- a/lib/spack/spack/cmd/setup.py
+++ b/lib/spack/spack/cmd/setup.py
@@ -39,6 +39,8 @@ from spack import which
 from spack.stage import DIYStage
 
 description = "create a configuration script and module, but don't build"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py
index 2e917d2ee3..f4105900cb 100644
--- a/lib/spack/spack/cmd/spec.py
+++ b/lib/spack/spack/cmd/spec.py
@@ -29,7 +29,9 @@ import spack
 import spack.cmd
 import spack.cmd.common.arguments as arguments
 
-description = "print out abstract and concrete versions of a spec"
+description = "show what would be installed, given a spec"
+section = "build"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py
index e0023b7254..a469cd896d 100644
--- a/lib/spack/spack/cmd/stage.py
+++ b/lib/spack/spack/cmd/stage.py
@@ -29,6 +29,8 @@ import spack
 import spack.cmd
 
 description = "expand downloaded archive in preparation for install"
+section = "build"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 9384e3a9e6..f7ec6a10e0 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -36,7 +36,9 @@ from llnl.util.tty.colify import colify
 
 import spack
 
-description = "a thin wrapper around the pytest command"
+description = "run spack's unit tests"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py
index f3eaddf88a..6880409c56 100644
--- a/lib/spack/spack/cmd/uninstall.py
+++ b/lib/spack/spack/cmd/uninstall.py
@@ -33,7 +33,9 @@ import spack.repository
 
 from llnl.util import tty
 
-description = "remove an installed package"
+description = "remove installed packages"
+section = "build"
+level = "short"
 
 error_message = """You can either:
     a) use a more specific spec, or
diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py
index 5da6f5daa5..8a0511f64c 100644
--- a/lib/spack/spack/cmd/unload.py
+++ b/lib/spack/spack/cmd/unload.py
@@ -25,7 +25,9 @@
 import argparse
 import spack.modules
 
-description = "remove package from environment using module"
+description = "remove package from environment using `module unload`"
+section = "environment"
+level = "short"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py
index e479749457..77312d1204 100644
--- a/lib/spack/spack/cmd/unuse.py
+++ b/lib/spack/spack/cmd/unuse.py
@@ -26,6 +26,8 @@ import argparse
 import spack.modules
 
 description = "remove package from environment using dotkit"
+section = "environment"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py
index e5cfce0de3..28118a178a 100644
--- a/lib/spack/spack/cmd/url.py
+++ b/lib/spack/spack/cmd/url.py
@@ -34,6 +34,8 @@ from spack.util.web import find_versions_of_archive
 from spack.util.naming import simplify_name
 
 description = "debugging tool for url parsing"
+section = "developer"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py
index c9714d9de0..e67de3a8b3 100644
--- a/lib/spack/spack/cmd/use.py
+++ b/lib/spack/spack/cmd/use.py
@@ -26,6 +26,8 @@ import argparse
 import spack.modules
 
 description = "add package to environment using dotkit"
+section = "environment"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py
index a6f6805fb0..446f0a876d 100644
--- a/lib/spack/spack/cmd/versions.py
+++ b/lib/spack/spack/cmd/versions.py
@@ -29,6 +29,8 @@ import llnl.util.tty as tty
 import spack
 
 description = "list available versions of a package"
+section = "packaging"
+level = "long"
 
 
 def setup_parser(subparser):
diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py
index 72e139d123..8fb94d3f37 100644
--- a/lib/spack/spack/cmd/view.py
+++ b/lib/spack/spack/cmd/view.py
@@ -69,7 +69,9 @@ import spack
 import spack.cmd
 import llnl.util.tty as tty
 
-description = "produce a single-rooted directory view of a spec"
+description = "produce a single-rooted directory view of packages"
+section = "environment"
+level = "short"
 
 
 def setup_parser(sp):
diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py
new file mode 100644
index 0000000000..39c64b3ce0
--- /dev/null
+++ b/lib/spack/spack/main.py
@@ -0,0 +1,468 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""This is the implementation of the Spack command line executable.
+
+In a normal Spack installation, this is invoked from the bin/spack script
+after the system path is set up.
+"""
+from __future__ import print_function
+
+import sys
+import os
+import inspect
+from argparse import _ArgumentGroup, ArgumentParser, RawTextHelpFormatter
+import pstats
+
+import llnl.util.tty as tty
+from llnl.util.tty.color import *
+
+import spack
+import spack.cmd
+from spack.error import SpackError
+
+
+# names of profile statistics
+stat_names = pstats.Stats.sort_arg_dict_default
+
+# help levels in order of detail (i.e., number of commands shown)
+levels = ['short', 'long']
+
+# intro text for help at different levels
+intro_by_level = {
+    'short': 'These are common spack commands:',
+    'long':  'Complete list of spack commands:',
+}
+
+# control top-level spack options shown in basic vs. advanced help
+options_by_level = {
+    'short': 'hkV',
+    'long': 'all'
+}
+
+# Longer text for each section, to show in help
+section_descriptions = {
+    'admin':       'administration',
+    'basic':       'query packages',
+    'build':       'build packages',
+    'config':      'configuration',
+    'developer':   'developer',
+    'environment': 'environment',
+    'extensions':  'extensions',
+    'help':        'more help',
+    'packaging':   'create packages',
+    'system':      'system',
+}
+
+# preferential command order for some sections (e.g., build pipeline is
+# in execution order, not alphabetical)
+section_order = {
+    'basic': ['list', 'info', 'find'],
+    'build': ['fetch', 'stage', 'patch', 'configure', 'build', 'restage',
+              'install', 'uninstall', 'clean']
+}
+
+# Properties that commands are required to set.
+required_command_properties = ['level', 'section', 'description']
+
+
+def set_working_dir():
+    """Change the working directory to getcwd, or spack prefix if no cwd."""
+    try:
+        spack.spack_working_dir = os.getcwd()
+    except OSError:
+        os.chdir(spack_prefix)
+        spack.spack_working_dir = spack_prefix
+
+
+def add_all_commands(parser):
+    """Add all spack subcommands to the parser."""
+    for cmd in spack.cmd.commands:
+        parser.add_command(cmd)
+
+
+def index_commands():
+    """create an index of commands by section for this help level"""
+    index = {}
+    for command in spack.cmd.commands:
+        cmd_module = spack.cmd.get_module(command)
+
+        # make sure command modules have required properties
+        for p in required_command_properties:
+            prop = getattr(cmd_module, p, None)
+            if not prop:
+                tty.die("Command doesn't define a property '%s': %s"
+                        % (p, command))
+
+        # add commands to lists for their level and higher levels
+        for level in reversed(levels):
+            level_sections = index.setdefault(level, {})
+            commands = level_sections.setdefault(cmd_module.section, [])
+            commands.append(command)
+            if level == cmd_module.level:
+                break
+
+    return index
+
+
+class SpackArgumentParser(ArgumentParser):
+    def format_help_sections(self, level):
+        """Format help on sections for a particular verbosity level.
+
+        Args:
+            level (str): 'short' or 'long' (more commands shown for long)
+        """
+        if level not in levels:
+            raise ValueError("level must be one of: %s" % levels)
+
+        # lazily add all commands to the parser when needed.
+        add_all_commands(self)
+
+        """Print help on subcommands in neatly formatted sections."""
+        formatter = self._get_formatter()
+
+        # Create a list of subcommand actions. Argparse internals are nasty!
+        # Note: you can only call _get_subactions() once.  Even nastier!
+        if not hasattr(self, 'actions'):
+            self.actions = self._subparsers._actions[-1]._get_subactions()
+
+        # make a set of commands not yet added.
+        remaining = set(spack.cmd.commands)
+
+        def add_group(group):
+            formatter.start_section(group.title)
+            formatter.add_text(group.description)
+            formatter.add_arguments(group._group_actions)
+            formatter.end_section()
+
+        def add_subcommand_group(title, commands):
+            """Add informational help group for a specific subcommand set."""
+            cmd_set = set(commands)
+
+            # make a dict of commands of interest
+            cmds = dict((action.metavar, action) for action in self.actions
+                        if action.metavar in cmd_set)
+
+            # add commands to a group in order, and add the group
+            group = _ArgumentGroup(self, title=title)
+            for name in commands:
+                group._add_action(cmds[name])
+                if name in remaining:
+                    remaining.remove(name)
+            add_group(group)
+
+        # select only the options for the particular level we're showing.
+        show_options = options_by_level[level]
+        if show_options != 'all':
+            opts = dict((opt.option_strings[0].strip('-'), opt)
+                        for opt in self._optionals._group_actions)
+
+            new_actions = [opts[letter] for letter in show_options]
+            self._optionals._group_actions = new_actions
+
+        options = ''.join(opt.option_strings[0].strip('-')
+                          for opt in self._optionals._group_actions)
+
+        index = index_commands()
+
+        # usage
+        formatter.add_text(
+            "usage: %s [-%s] <command> [...]" % (self.prog, options))
+
+        # description
+        formatter.add_text(self.description)
+
+        # start subcommands
+        formatter.add_text(intro_by_level[level])
+
+        # add argument groups based on metadata in commands
+        sections = index[level]
+        for section in sorted(sections):
+            if section == 'help':
+                continue   # Cover help in the epilog.
+
+            group_description = section_descriptions.get(section, section)
+
+            to_display = sections[section]
+            commands = []
+
+            # add commands whose order we care about first.
+            if section in section_order:
+                commands.extend(cmd for cmd in section_order[section]
+                                if cmd in to_display)
+
+            # add rest in alphabetical order.
+            commands.extend(cmd for cmd in sorted(sections[section])
+                            if cmd not in commands)
+
+            # add the group to the parser
+            add_subcommand_group(group_description, commands)
+
+        # optionals
+        add_group(self._optionals)
+
+        # epilog
+        formatter.add_text("""\
+{help}:
+  spack help -a          list all available commands
+  spack help <command>   help on a specific command
+  spack help --spec      help on the spec syntax
+  spack docs             open http://spack.rtfd.io/ in a browser"""
+.format(help=section_descriptions['help']))
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def add_command(self, name):
+        """Add one subcommand to this parser."""
+        # lazily initialize any subparsers
+        if not hasattr(self, 'subparsers'):
+            # remove the dummy "command" argument.
+            self._remove_action(self._actions[-1])
+            self.subparsers = self.add_subparsers(metavar='COMMAND',
+                                                  dest="command")
+
+        # each command module implements a parser() function, to which we
+        # pass its subparser for setup.
+        module = spack.cmd.get_module(name)
+        cmd_name = name.replace('_', '-')
+        subparser = self.subparsers.add_parser(
+            cmd_name, help=module.description, description=module.description)
+        module.setup_parser(subparser)
+        return module
+
+    def format_help(self, level='short'):
+        if self.prog == 'spack':
+            # use format_help_sections for the main spack parser, but not
+            # for subparsers
+            return self.format_help_sections(level)
+        else:
+            # in subparsers, self.prog is, e.g., 'spack install'
+            return super(SpackArgumentParser, self).format_help()
+
+
+def make_argument_parser():
+    """Create an basic argument parser without any subcommands added."""
+    parser = SpackArgumentParser(
+        formatter_class=RawTextHelpFormatter, add_help=False,
+        description=(
+            "A flexible package manager that supports multiple versions,\n"
+            "configurations, platforms, and compilers."))
+
+    # stat names in groups of 7, for nice wrapping.
+    stat_lines = list(zip(*(iter(stat_names),) * 7))
+
+    parser.add_argument('-h', '--help', action='store_true',
+                        help="show this help message and exit")
+    parser.add_argument('-d', '--debug', action='store_true',
+                        help="write out debug logs during compile")
+    parser.add_argument('-D', '--pdb', action='store_true',
+                        help="run spack under the pdb debugger")
+    parser.add_argument('-k', '--insecure', action='store_true',
+                        help="do not check ssl certificates when downloading")
+    parser.add_argument('-m', '--mock', action='store_true',
+                        help="use mock packages instead of real ones")
+    parser.add_argument('-p', '--profile', action='store_true',
+                        help="profile execution using cProfile")
+    parser.add_argument('-P', '--sorted-profile', default=None, metavar="STAT",
+                        help="profile and sort by one or more of:\n[%s]" %
+                        ',\n '.join([', '.join(line) for line in stat_lines]))
+    parser.add_argument('--lines', default=20, action='store',
+                        help="lines of profile output; default 20; or 'all'")
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help="print additional output during builds")
+    parser.add_argument('-s', '--stacktrace', action='store_true',
+                        help="add stacktraces to all printed statements")
+    parser.add_argument('-V', '--version', action='store_true',
+                        help='show version number and exit')
+    return parser
+
+
+def setup_main_options(args):
+    """Configure spack globals based on the basic options."""
+    # Set up environment based on args.
+    tty.set_verbose(args.verbose)
+    tty.set_debug(args.debug)
+    tty.set_stacktrace(args.stacktrace)
+    spack.debug = args.debug
+
+    if spack.debug:
+        import spack.util.debug as debug
+        debug.register_interrupt_handler()
+
+    if args.mock:
+        from spack.repository import RepoPath
+        spack.repo.swap(RepoPath(spack.mock_packages_path))
+
+    # If the user asked for it, don't check ssl certs.
+    if args.insecure:
+        tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
+        spack.insecure = True
+
+
+def allows_unknown_args(command):
+    """This is a basic argument injection test.
+
+    Commands may add an optional argument called "unknown args" to
+    indicate they can handle unknonwn args, and we'll pass the unknown
+    args in.
+    """
+    info = dict(inspect.getmembers(command))
+    varnames = info['__code__'].co_varnames
+    argcount = info['__code__'].co_argcount
+    return (argcount == 3 and varnames[2] == 'unknown_args')
+
+
+def _main(command, parser, args, unknown_args):
+    # many operations will fail without a working directory.
+    set_working_dir()
+
+    # only setup main options in here, after the real parse (we'll get it
+    # wrong if we do it after the initial, partial parse)
+    setup_main_options(args)
+    spack.hooks.pre_run()
+
+    # Now actually execute the command
+    try:
+        if allows_unknown_args(command):
+            return_val = command(parser, args, unknown_args)
+        else:
+            if unknown_args:
+                tty.die('unrecognized arguments: %s' % ' '.join(unknown_args))
+            return_val = command(parser, args)
+    except SpackError as e:
+        e.die()  # gracefully die on any SpackErrors
+    except Exception as e:
+        if spack.debug:
+            raise
+        tty.die(str(e))
+    except KeyboardInterrupt:
+        sys.stderr.write('\n')
+        tty.die("Keyboard interrupt.")
+
+    # Allow commands to return and error code if they want
+    return 0 if return_val is None else return_val
+
+
+def _profile_wrapper(command, parser, args, unknown_args):
+    import cProfile
+
+    try:
+        nlines = int(args.lines)
+    except ValueError:
+        if args.lines != 'all':
+            tty.die('Invalid number for --lines: %s' % args.lines)
+        nlines = -1
+
+    # allow comma-separated list of fields
+    sortby = ['time']
+    if args.sorted_profile:
+        sortby = args.sorted_profile.split(',')
+        for stat in sortby:
+            if stat not in stat_names:
+                tty.die("Invalid sort field: %s" % stat)
+
+    try:
+        # make a profiler and run the code.
+        pr = cProfile.Profile()
+        pr.enable()
+        return _main(command, parser, args, unknown_args)
+
+    finally:
+        pr.disable()
+
+        # print out profile stats.
+        stats = pstats.Stats(pr)
+        stats.sort_stats(*sortby)
+        stats.print_stats(nlines)
+
+
+def main(argv=None):
+    """This is the entry point for the Spack command.
+
+    Args:
+        argv (list of str or None): command line arguments, NOT including
+            the executable name. If None, parses from sys.argv.
+    """
+    # Create a parser with a simple positional argument first.  We'll
+    # lazily load the subcommand(s) we need later. This allows us to
+    # avoid loading all the modules from spack.cmd when we don't need
+    # them, which reduces startup latency.
+    parser = make_argument_parser()
+    parser.add_argument(
+        'command', metavar='COMMAND', nargs='?', action='store')
+    args, unknown = parser.parse_known_args(argv)
+
+    # Just print help and exit if run with no arguments at all
+    no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0)
+    if no_args:
+        parser.print_help()
+        return 1
+
+    # -h and -V are special as they do not require a command, but all the
+    # other options do nothing without a command.
+    if not args.command:
+        if args.version:
+            print(spack.spack_version)
+            return 0
+        else:
+            parser.print_help()
+            return 0 if args.help else 1
+
+    # Try to load the particular command the caller asked for.  If there
+    # is no module for it, just die.
+    command_name = args.command.replace('-', '_')
+    try:
+        parser.add_command(command_name)
+    except ImportError:
+        if spack.debug:
+            raise
+        tty.die("Unknown command: %s" % args.command)
+
+    # Re-parse with the proper sub-parser added.
+    args, unknown = parser.parse_known_args()
+
+    # we now know whether options go with spack or the command
+    if args.version:
+        print(spack.spack_version)
+        return 0
+    elif args.help:
+        parser.print_help()
+        return 0
+
+    # now we can actually execute the command.
+    command = spack.cmd.get_command(command_name)
+    try:
+        if args.profile or args.sorted_profile:
+            _profile_wrapper(command, parser, args, unknown)
+        elif args.pdb:
+            import pdb
+            pdb.runctx('_main(command, parser, args, unknown)',
+                       globals(), locals())
+            return 0
+        else:
+            return _main(command, parser, args, unknown)
+
+    except SystemExit as e:
+        return e.code