Sphinx no longer supports Python 2.6 (#4266)

* Sphinx no longer supports Python 2.6

* Update vendored sphinxcontrib.programoutput from 0.9.0 to 0.10.0

* Documentation cannot be built in parallel

* Let Travis install programoutput for us

* Remove vendored sphinxcontrib-programoutput

Recent updates to the sphinx package prevent the vendored version
from being found in sys.path. We don't vendor sphinx, so it doesn't
make sense to vendor sphinxcontrib-programoutput either.

8 files changed, 6 insertions, 303 deletions

diff --git a/.travis.yml b/.travis.yml
index 2bf21d0e57..c4bfe17640 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,7 +86,8 @@ install:
   - pip install --upgrade setuptools
   - pip install --upgrade codecov
   - pip install --upgrade flake8
-  - pip install --upgrade sphinx
+  - if [[ "$TEST_SUITE" == "doc" ]]; then pip install --upgrade sphinx; fi
+  - if [[ "$TEST_SUITE" == "doc" ]]; then pip install --upgrade sphinxcontrib-programoutput; fi
 
 before_script:
   # Need this for the git tests to succeed.
diff --git a/lib/spack/docs/Makefile b/lib/spack/docs/Makefile
index 1054d91a50..3503794021 100644
--- a/lib/spack/docs/Makefile
+++ b/lib/spack/docs/Makefile
@@ -3,8 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    = -E
-JOBS          ?= $(shell python -c 'import multiprocessing; print multiprocessing.cpu_count()')
-SPHINXBUILD   = sphinx-build -j $(JOBS)
+SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
 
diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py
index d124848542..ee2b314aa3 100644
--- a/lib/spack/docs/conf.py
+++ b/lib/spack/docs/conf.py
@@ -49,7 +49,6 @@ from sphinx.apidoc import main as sphinx_apidoc
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('exts'))
 sys.path.insert(0, os.path.abspath('../external'))
 if sys.version_info[0] < 3:
     sys.path.insert(0, os.path.abspath('../external/yaml/lib'))
diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst
index a3b3197181..c0e07f7340 100644
--- a/lib/spack/docs/contribution_guide.rst
+++ b/lib/spack/docs/contribution_guide.rst
@@ -189,6 +189,7 @@ Building the documentation requires several dependencies, all of which can be
 installed with Spack:
 
 * sphinx
+* sphinxcontrib-programoutput
 * graphviz
 * git
 * mercurial
@@ -227,7 +228,7 @@ your PR is accepted.
    There is also a ``run-doc-tests`` script in the Quality Assurance directory.
    The only difference between running this script and running ``make`` by hand
    is that the script will exit immediately if it encounters an error or warning.
-   This is necessary for Travis CI. If you made a lot of documentation tests, it
+   This is necessary for Travis CI. If you made a lot of documentation changes, it
    is much quicker to run ``make`` by hand so that you can see all of the warnings
    at once.
 
diff --git a/lib/spack/docs/exts/sphinxcontrib/LICENSE b/lib/spack/docs/exts/sphinxcontrib/LICENSE
deleted file mode 100644
index 43b87a5992..0000000000
--- a/lib/spack/docs/exts/sphinxcontrib/LICENSE
+++ /dev/null
@@ -1,25 +0,0 @@
-Copyright (c) 2010, 2011, 2012 Sebastian Wiesner <lunaryorn@googlemail.com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright
-  notice, this list of conditions and the following disclaimer in the
-  documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/spack/docs/exts/sphinxcontrib/__init__.py b/lib/spack/docs/exts/sphinxcontrib/__init__.py
deleted file mode 100644
index 591cf0e16e..0000000000
--- a/lib/spack/docs/exts/sphinxcontrib/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    sphinxcontrib
-    ~~~~~~~~~~~~~
-
-    Contains 3rd party Sphinx extensions.
-"""
-
-__import__('pkg_resources').declare_namespace(__name__)
diff --git a/lib/spack/docs/exts/sphinxcontrib/programoutput.py b/lib/spack/docs/exts/sphinxcontrib/programoutput.py
deleted file mode 100644
index 3f6a4f1595..0000000000
--- a/lib/spack/docs/exts/sphinxcontrib/programoutput.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2010, 2011, 2012, Sebastian Wiesner <lunaryorn@gmail.com>
-# All rights reserved.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-
-# 1. Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""
-    sphinxcontrib.programoutput
-    ===========================
-
-    This extension provides a directive to include the output of commands as
-    literal block while building the docs.
-
-    .. moduleauthor::  Sebastian Wiesner  <lunaryorn@gmail.com>
-"""
-
-from __future__ import (print_function, division, unicode_literals,
-                        absolute_import)
-
-import sys
-import os
-import shlex
-from subprocess import Popen, PIPE, STDOUT
-from collections import defaultdict, namedtuple
-
-from docutils import nodes
-from docutils.parsers import rst
-from docutils.parsers.rst.directives import flag, unchanged, nonnegative_int
-
-
-__version__ = '0.9'
-
-
-class program_output(nodes.Element):
-    pass
-
-
-def _slice(value):
-    parts = [int(v.strip()) for v in value.split(',')]
-    if len(parts) > 2:
-        raise ValueError('too many slice parts')
-    return tuple((parts + [None] * 2)[:2])
-
-
-class ProgramOutputDirective(rst.Directive):
-    has_content = False
-    final_argument_whitespace = True
-    required_arguments = 1
-
-    option_spec = dict(shell=flag, prompt=flag, nostderr=flag,
-                       ellipsis=_slice, extraargs=unchanged,
-                       returncode=nonnegative_int, cwd=unchanged)
-
-    def run(self):
-        env = self.state.document.settings.env
-
-        node = program_output()
-        node.line = self.lineno
-        node['command'] = self.arguments[0]
-
-        if self.name == 'command-output':
-            node['show_prompt'] = True
-        else:
-            node['show_prompt'] = 'prompt' in self.options
-
-        node['hide_standard_error'] = 'nostderr' in self.options
-        node['extraargs'] = self.options.get('extraargs', '')
-        _, cwd = env.relfn2path(self.options.get('cwd', '/'))
-        node['working_directory'] = cwd
-        node['use_shell'] = 'shell' in self.options
-        node['returncode'] = self.options.get('returncode', 0)
-        if 'ellipsis' in self.options:
-            node['strip_lines'] = self.options['ellipsis']
-        return [node]
-
-
-_Command = namedtuple(
-    'Command', 'command shell hide_standard_error working_directory')
-
-
-class Command(_Command):
-    """
-    A command to be executed.
-    """
-
-    def __new__(cls, command, shell=False, hide_standard_error=False,
-                working_directory='/'):
-        if isinstance(command, list):
-            command = tuple(command)
-        # `chdir()` resolves symlinks, so we need to resolve them too for
-        # caching to make sure that different symlinks to the same directory
-        # don't result in different cache keys.  Also normalize paths to make
-        # sure that identical paths are also equal as strings.
-        working_directory = os.path.normpath(os.path.realpath(
-            working_directory))
-        return _Command.__new__(cls, command, shell, hide_standard_error,
-                                working_directory)
-
-    @classmethod
-    def from_program_output_node(cls, node):
-        """
-        Create a command from a :class:`program_output` node.
-        """
-        extraargs = node.get('extraargs', '')
-        command = (node['command'] + ' ' + extraargs).strip()
-        return cls(command, node['use_shell'],
-                   node['hide_standard_error'], node['working_directory'])
-
-    def execute(self):
-        """
-        Execute this command.
-
-        Return the :class:`~subprocess.Popen` object representing the running
-        command.
-        """
-        if self.shell:
-            if sys.version_info[0] < 3 and isinstance(self.command, unicode):
-                command = self.command.encode(sys.getfilesystemencoding())
-            else:
-                command = self.command
-        else:
-            if sys.version_info[0] < 3 and isinstance(self.command, unicode):
-                command = shlex.split(self.command.encode(
-                    sys.getfilesystemencoding()))
-            elif isinstance(self.command, str):
-                command = shlex.split(self.command)
-            else:
-                command = self.command
-        return Popen(command, shell=self.shell, stdout=PIPE,
-                     stderr=PIPE if self.hide_standard_error else STDOUT,
-                     cwd=self.working_directory)
-
-    def get_output(self):
-        """
-        Get the output of this command.
-
-        Return a tuple ``(returncode, output)``.  ``returncode`` is the
-        integral return code of the process, ``output`` is the output as
-        unicode string, with final trailing spaces and new lines stripped.
-        """
-        process = self.execute()
-        output = process.communicate()[0].decode(
-            sys.getfilesystemencoding(), 'replace').rstrip()
-        return process.returncode, output
-
-    def __str__(self):
-        if isinstance(self.command, tuple):
-            return repr(list(self.command))
-        return repr(self.command)
-
-
-class ProgramOutputCache(defaultdict):
-    """
-    Execute command and cache their output.
-
-    This class is a mapping.  Its keys are :class:`Command` objects represeting
-    command invocations.  Its values are tuples of the form ``(returncode,
-    output)``, where ``returncode`` is the integral return code of the command,
-    and ``output`` is the output as unicode string.
-
-    The first time, a key is retrieved from this object, the command is
-    invoked, and its result is cached.  Subsequent access to the same key
-    returns the cached value.
-    """
-
-    def __missing__(self, command):
-        """
-        Called, if a command was not found in the cache.
-
-        ``command`` is an instance of :class:`Command`.
-        """
-        result = command.get_output()
-        self[command] = result
-        return result
-
-
-def run_programs(app, doctree):
-    """
-    Execute all programs represented by ``program_output`` nodes in
-    ``doctree``.  Each ``program_output`` node in ``doctree`` is then
-    replaced with a node, that represents the output of this program.
-
-    The program output is retrieved from the cache in
-    ``app.env.programoutput_cache``.
-    """
-    if app.config.programoutput_use_ansi:
-        # enable ANSI support, if requested by config
-        from sphinxcontrib.ansi import ansi_literal_block
-        node_class = ansi_literal_block
-    else:
-        node_class = nodes.literal_block
-
-    cache = app.env.programoutput_cache
-
-    for node in doctree.traverse(program_output):
-        command = Command.from_program_output_node(node)
-        try:
-            returncode, output = cache[command]
-        except EnvironmentError as error:
-            error_message = 'Command {0} failed: {1}'.format(command, error)
-            error_node = doctree.reporter.error(error_message, base_node=node)
-            node.replace_self(error_node)
-        else:
-            if returncode != node['returncode']:
-                app.warn('Unexpected return code {0} from command {1}'.format(
-                    returncode, command))
-
-            # replace lines with ..., if ellipsis is specified
-            if 'strip_lines' in node:
-                lines = output.splitlines()
-                start, stop = node['strip_lines']
-                lines[start:stop] = ['...']
-                output = '\n'.join(lines)
-
-            if node['show_prompt']:
-                tmpl = app.config.programoutput_prompt_template
-                output = tmpl.format(command=node['command'], output=output,
-                                     returncode=returncode)
-
-            new_node = node_class(output, output)
-            new_node['language'] = 'text'
-            node.replace_self(new_node)
-
-
-def init_cache(app):
-    """
-    Initialize the cache for program output at
-    ``app.env.programoutput_cache``, if not already present (e.g. being
-    loaded from a pickled environment).
-
-    The cache is of type :class:`ProgramOutputCache`.
-    """
-    if not hasattr(app.env, 'programoutput_cache'):
-        app.env.programoutput_cache = ProgramOutputCache()
-
-
-def setup(app):
-    app.add_config_value('programoutput_use_ansi', False, 'env')
-    app.add_config_value('programoutput_prompt_template',
-                         '$ {command}\n{output}', 'env')
-    app.add_directive('program-output', ProgramOutputDirective)
-    app.add_directive('command-output', ProgramOutputDirective)
-    app.connect(str('builder-inited'), init_cache)
-    app.connect(str('doctree-read'), run_programs)
diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests
index b9a05aa3c8..c43779fcaf 100755
--- a/share/spack/qa/run-doc-tests
+++ b/share/spack/qa/run-doc-tests
@@ -17,4 +17,4 @@ cd "$SPACK_ROOT/lib/spack/docs"
 
 # Treat warnings as fatal errors
 make clean --silent
-make SPHINXOPTS=-W JOBS=1
+make SPHINXOPTS=-W