Rework Spack's Mercurial support (#3834)

* Add tests to mercurial package

* Add support for --insecure with mercurial fetching

* Install man pages and tab-completion scripts

* Add tests and latest version for all deps

* Flake8 fix

* Use certifi module to find CA certificate

* Flake8 fix

* Unset PYTHONPATH when running hg

* svn_fetch should use to svn-test, not hg-test

* Drop Python 3 support in Mercurial

Python 3 support is a work in progress and isn't currently
recommended:

https://www.mercurial-scm.org/wiki/SupportedPythonVersions

* Test both secure and insecure hg fetching

* Test both secure and insecure git and svn fetching

15 files changed, 176 insertions, 52 deletions

diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 38752b3fc1..855b2f9379 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -808,8 +808,17 @@ class HgFetchStrategy(VCSFetchStrategy):
 
     @property
     def hg(self):
+        """:returns: The hg executable
+        :rtype: Executable
+        """
         if not self._hg:
             self._hg = which('hg', required=True)
+
+            # When building PythonPackages, Spack automatically sets
+            # PYTHONPATH. This can interfere with hg, which is a Python
+            # script. Unset PYTHONPATH while running hg.
+            self._hg.add_default_env('PYTHONPATH', '')
+
         return self._hg
 
     @property
@@ -829,9 +838,15 @@ class HgFetchStrategy(VCSFetchStrategy):
             args.append('at revision %s' % self.revision)
         tty.msg("Trying to clone Mercurial repository:", self.url, *args)
 
-        args = ['clone', self.url]
+        args = ['clone']
+
+        if spack.insecure:
+            args.append('--insecure')
+
+        args.append(self.url)
+
         if self.revision:
-            args += ['-r', self.revision]
+            args.extend(['-r', self.revision])
 
         self.hg(*args)
 
diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py
index 3bd998c5c2..ef531ef65f 100644
--- a/lib/spack/spack/test/git_fetch.py
+++ b/lib/spack/spack/test/git_fetch.py
@@ -37,8 +37,15 @@ def type_of_test(request):
     return request.param
 
 
+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
+    return request.param
+
+
 def test_fetch(
         type_of_test,
+        secure,
         mock_git_repository,
         config,
         refresh_builtin_mock
@@ -62,7 +69,12 @@ def test_fetch(
     pkg.versions[ver('git')] = t.args
     # Enter the stage directory and check some properties
     with pkg.stage:
-        pkg.do_stage()
+        try:
+            spack.insecure = secure
+            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
         assert h('HEAD') == h(t.revision)
 
         file_path = join_path(pkg.stage.source_path, t.file)
diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py
index 71e4693c56..29a6eef561 100644
--- a/lib/spack/spack/test/hg_fetch.py
+++ b/lib/spack/spack/test/hg_fetch.py
@@ -37,8 +37,15 @@ def type_of_test(request):
     return request.param
 
 
+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
+    return request.param
+
+
 def test_fetch(
         type_of_test,
+        secure,
         mock_hg_repository,
         config,
         refresh_builtin_mock
@@ -62,7 +69,12 @@ def test_fetch(
     pkg.versions[ver('hg')] = t.args
     # Enter the stage directory and check some properties
     with pkg.stage:
-        pkg.do_stage()
+        try:
+            spack.insecure = secure
+            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
         assert h() == t.revision
 
         file_path = join_path(pkg.stage.source_path, t.file)
diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py
index 962a150909..69d675fe3c 100644
--- a/lib/spack/spack/test/svn_fetch.py
+++ b/lib/spack/spack/test/svn_fetch.py
@@ -33,12 +33,19 @@ from spack.version import ver
 
 @pytest.fixture(params=['default', 'rev0'])
 def type_of_test(request):
-    """Returns one of the test type available for the mock_hg_repository"""
+    """Returns one of the test type available for the mock_svn_repository"""
+    return request.param
+
+
+@pytest.fixture(params=[True, False])
+def secure(request):
+    """Attempt both secure and insecure fetching"""
     return request.param
 
 
 def test_fetch(
         type_of_test,
+        secure,
         mock_svn_repository,
         config,
         refresh_builtin_mock
@@ -56,13 +63,18 @@ def test_fetch(
     t = mock_svn_repository.checks[type_of_test]
     h = mock_svn_repository.hash
     # Construct the package under test
-    spec = Spec('hg-test')
+    spec = Spec('svn-test')
     spec.concretize()
     pkg = spack.repo.get(spec, new=True)
-    pkg.versions[ver('hg')] = t.args
+    pkg.versions[ver('svn')] = t.args
     # Enter the stage directory and check some properties
     with pkg.stage:
-        pkg.do_stage()
+        try:
+            spack.insecure = secure
+            pkg.do_stage()
+        finally:
+            spack.insecure = False
+
         assert h() == t.revision
 
         file_path = join_path(pkg.stage.source_path, t.file)
diff --git a/var/spack/repos/builtin/packages/mercurial/package.py b/var/spack/repos/builtin/packages/mercurial/package.py
index ea77953f15..b383560cfe 100644
--- a/var/spack/repos/builtin/packages/mercurial/package.py
+++ b/var/spack/repos/builtin/packages/mercurial/package.py
@@ -23,16 +23,23 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
-import llnl.util.tty as tty
-import os
+from llnl.util import tty
 
 
-class Mercurial(Package):
+class Mercurial(PythonPackage):
     """Mercurial is a free, distributed source control management tool."""
 
     homepage = "https://www.mercurial-scm.org"
-    url      = "https://www.mercurial-scm.org/release/mercurial-3.9.tar.gz"
+    url      = "https://www.mercurial-scm.org/release/mercurial-4.1.2.tar.gz"
 
+    import_modules = [
+        'hgext', 'hgext3rd', 'mercurial', 'hgext.convert', 'hgext.fsmonitor',
+        'hgext.highlight', 'hgext.largefiles', 'hgext.zeroconf',
+        'hgext.fsmonitor.pywatchman', 'mercurial.hgweb',
+        'mercurial.httpclient', 'mercurial.pure'
+    ]
+
+    version('4.1.2', '934c99808bdc8385e074b902d59b0d93')
     version('3.9.1', '3759dd10edb8c1a6dfb8ff0ce82658ce')
     version('3.9',   'e2b355da744e94747daae3a5339d28a0')
     version('3.8.4', 'cec2c3db688cb87142809089c6ae13e9')
@@ -40,30 +47,62 @@ class Mercurial(Package):
     version('3.8.2', 'c38daa0cbe264fc621dc3bb05933b0b3')
     version('3.8.1', '172a8c588adca12308c2aca16608d7f4')
 
-    extends('python')
     depends_on('python@2.6:2.8')
     depends_on('py-docutils', type='build')
+    depends_on('py-pygments', type=('build', 'run'))
+    depends_on('py-certifi',  type=('build', 'run'))
+
+    @run_after('install')
+    def post_install(self):
+        prefix = self.prefix
+
+        # Install man pages
+        mkdirp(prefix.man1)
+        mkdirp(prefix.man5)
+        mkdirp(prefix.man8)
+        with working_dir('doc'):
+            install('hg.1', prefix.man1)
+            install('hgignore.5', prefix.man5)
+            install('hgrc.5', prefix.man5)
+            install('hg-ssh.8', prefix.man8)
+
+        # Install completion scripts
+        contrib = join_path(prefix, 'contrib')
+        mkdir(contrib)
+        with working_dir('contrib'):
+            install('bash_completion', join_path(contrib, 'bash_completion'))
+            install('zsh_completion',  join_path(contrib, 'zsh_completion'))
+
+    @run_after('install')
+    def configure_certificates(self):
+        """Configuration of HTTPS certificate authorities
+        https://www.mercurial-scm.org/wiki/CACertificates"""
 
-    def install(self, spec, prefix):
-        make('install', 'PREFIX={0}'.format(prefix))
+        etc_dir = join_path(self.prefix.etc, 'mercurial')
+        mkdirp(etc_dir)
 
-        # Configuration of HTTPS certificate authorities
-        # https://www.mercurial-scm.org/wiki/CACertificates
-        hgrc_filename = join_path(prefix.etc, 'mercurial', 'hgrc')
-        mkdirp(os.path.dirname(hgrc_filename))
+        hgrc_filename = join_path(etc_dir, 'hgrc')
 
+        # Use certifi to find the location of the CA certificate
+        certificate = python('-c', 'import certifi; print certifi.where()',
+                             output=str)
+
+        if not certificate:
+            tty.warn('CA certificate not found. You may not be able to '
+                     'connect to an HTTPS server. If your CA certificate '
+                     'is in a non-standard location, you should add it to '
+                     '{0}.'.format(hgrc_filename))
+
+        # Write the global mercurial configuration file
         with open(hgrc_filename, 'w') as hgrc:
-            if os.path.exists('/etc/ssl/certs/ca-certificates.crt'):
-                # Debian/Ubuntu/Gentoo/Arch Linux
-                hgrc.write('[web]\ncacerts = /etc/ssl/certs/ca-certificates.crt')  # noqa
-            elif os.path.exists('/etc/pki/tls/certs/ca-bundle.crt'):
-                # Fedora/RHEL/CentOS
-                hgrc.write('[web]\ncacerts = /etc/pki/tls/certs/ca-bundle.crt')
-            elif os.path.exists('/etc/ssl/ca-bundle.pem'):
-                # openSUSE/SLE
-                hgrc.write('[web]\ncacerts = /etc/ssl/ca-bundle.pem')
-            else:
-                tty.warn('CA certificate not found. You may not be able to '
-                         'connect to an HTTPS server. If your CA certificate '
-                         'is in a non-standard location, you should add it to '
-                         '{0}'.format(hgrc_filename))
+            hgrc.write('[web]\ncacerts = {0}'.format(certificate))
+
+    @run_after('install')
+    @on_package_attributes(run_tests=True)
+    def check_install(self):
+        """Sanity-check setup."""
+
+        hg = Executable(join_path(self.prefix.bin, 'hg'))
+
+        hg('debuginstall')
+        hg('version')
diff --git a/var/spack/repos/builtin/packages/py-appdirs/package.py b/var/spack/repos/builtin/packages/py-appdirs/package.py
index 360f56d987..802d59b99b 100644
--- a/var/spack/repos/builtin/packages/py-appdirs/package.py
+++ b/var/spack/repos/builtin/packages/py-appdirs/package.py
@@ -30,8 +30,11 @@ class PyAppdirs(PythonPackage):
     dirs, e.g. a "user data dir"."""
 
     homepage = "https://github.com/ActiveState/appdirs"
-    url      = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.0.tar.gz"
+    url      = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.3.tar.gz"
 
+    import_modules = ['appdirs']
+
+    version('1.4.3', '44c679904082a2133f5566c8a0d3ab42')
     version('1.4.0', '1d17b4c9694ab84794e228f28dc3275b')
 
     patch('setuptools-import.patch', when='@:1.4.0')
diff --git a/var/spack/repos/builtin/packages/py-certifi/package.py b/var/spack/repos/builtin/packages/py-certifi/package.py
index 959c0221ed..7d8095f3f9 100644
--- a/var/spack/repos/builtin/packages/py-certifi/package.py
+++ b/var/spack/repos/builtin/packages/py-certifi/package.py
@@ -29,9 +29,12 @@ class PyCertifi(PythonPackage):
     """Certifi: A carefully curated collection of Root Certificates for validating
     the trustworthiness of SSL certificates while verifying the identity of TLS
     hosts."""
-    homepage = "https://github.com/certifi/python-certifi"
-    url      = "https://github.com/certifi/python-certifi/archive/2016.02.28.tar.gz"
+    homepage = "http://certifi.io/"
+    url      = "https://pypi.io/packages/source/c/certifi/certifi-2017.1.23.tar.gz"
 
+    import_modules = ['certifi']
+
+    version('2017.1.23',  'b72521a8badff5e89a8eabea586d79ab')
     version('2016.02.28', '5ccfc23bd5e931863f0b01ef3e9d2dbd3bef0e1b')
 
     depends_on('py-setuptools', type='build')
diff --git a/var/spack/repos/builtin/packages/py-cffi/package.py b/var/spack/repos/builtin/packages/py-cffi/package.py
index c0fbae639b..4d1f6ee5ad 100644
--- a/var/spack/repos/builtin/packages/py-cffi/package.py
+++ b/var/spack/repos/builtin/packages/py-cffi/package.py
@@ -23,15 +23,18 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
+import sys
 
 
 class PyCffi(PythonPackage):
     """Foreign Function Interface for Python calling C code"""
     homepage = "http://cffi.readthedocs.org/en/latest/"
-    # base https://pypi.python.org/pypi/cffi
-    url      = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz"
+    url      = "https://pypi.io/packages/source/c/cffi/cffi-1.10.0.tar.gz"
 
-    version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea')
+    import_modules = ['cffi']
+
+    version('1.10.0', '2b5fa41182ed0edaf929a789e602a070')
+    version('1.1.2',  'ca6e6c45b45caa87aee9adc7c796eaea')
 
     depends_on('py-setuptools', type='build')
     depends_on('py-pycparser', type=('build', 'run'))
@@ -44,4 +47,5 @@ class PyCffi(PythonPackage):
         # other compilation.  We are setting the 'LDSHARED" to the
         # spack compiler wrapper plus a few extra flags necessary for
         # building the shared library.
-        spack_env.set('LDSHARED', "{0} -shared -pthread".format(spack_cc))
+        if not sys.platform == 'darwin':
+            spack_env.set('LDSHARED', "{0} -shared -pthread".format(spack_cc))
diff --git a/var/spack/repos/builtin/packages/py-docutils/package.py b/var/spack/repos/builtin/packages/py-docutils/package.py
index 00741284df..76a723e34d 100644
--- a/var/spack/repos/builtin/packages/py-docutils/package.py
+++ b/var/spack/repos/builtin/packages/py-docutils/package.py
@@ -33,8 +33,18 @@ class PyDocutils(PythonPackage):
     markup language."""
 
     homepage = "http://docutils.sourceforge.net/"
-    url      = "https://pypi.python.org/packages/source/d/docutils/docutils-0.12.tar.gz"
+    url      = "https://pypi.io/packages/source/d/docutils/docutils-0.13.1.tar.gz"
 
-    version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53',
-            url="https://pypi.python.org/packages/05/25/7b5484aca5d46915493f1fd4ecb63c38c333bd32aa9ad6e19da8d08895ae/docutils-0.13.1.tar.gz")
+    import_modules = [
+        'docutils', 'docutils.languages', 'docutils.parsers',
+        'docutils.readers', 'docutils.transforms', 'docutils.utils',
+        'docutils.writers', 'docutils.parsers.rst',
+        'docutils.parsers.rst.directives', 'docutils.parsers.rst.languages',
+        'docutils.utils.math', 'docutils.writers.html4css1',
+        'docutils.writers.html5_polyglot', 'docutils.writers.latex2e',
+        'docutils.writers.odf_odt', 'docutils.writers.pep_html',
+        'docutils.writers.s5_html', 'docutils.writers.xetex'
+    ]
+
+    version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53')
     version('0.12',   '4622263b62c5c771c03502afa3157768')
diff --git a/var/spack/repos/builtin/packages/py-packaging/package.py b/var/spack/repos/builtin/packages/py-packaging/package.py
index 506bb6f976..4ee97a507a 100644
--- a/var/spack/repos/builtin/packages/py-packaging/package.py
+++ b/var/spack/repos/builtin/packages/py-packaging/package.py
@@ -31,6 +31,8 @@ class PyPackaging(PythonPackage):
     homepage = "https://github.com/pypa/packaging"
     url      = "https://pypi.io/packages/source/p/packaging/packaging-16.8.tar.gz"
 
+    import_modules = ['packaging']
+
     version('16.8', '53895cdca04ecff80b54128e475b5d3b')
 
     # Not needed for the installation, but used at runtime
diff --git a/var/spack/repos/builtin/packages/py-pycparser/package.py b/var/spack/repos/builtin/packages/py-pycparser/package.py
index 8de5c39d32..a2c00fe33a 100644
--- a/var/spack/repos/builtin/packages/py-pycparser/package.py
+++ b/var/spack/repos/builtin/packages/py-pycparser/package.py
@@ -28,8 +28,11 @@ from spack import *
 class PyPycparser(PythonPackage):
     """A complete parser of the C language, written in pure python."""
     homepage = "https://github.com/eliben/pycparser"
-    url      = "https://pypi.python.org/packages/source/p/pycparser/pycparser-2.13.tar.gz"
+    url      = "https://pypi.io/packages/source/p/pycparser/pycparser-2.17.tar.gz"
 
+    import_modules = ['pycparser', 'pycparser.ply']
+
+    version('2.17', 'ca98dcb50bc1276f230118f6af5a40c7')
     version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f')
 
     depends_on('py-setuptools', type='build')
diff --git a/var/spack/repos/builtin/packages/py-pygments/package.py b/var/spack/repos/builtin/packages/py-pygments/package.py
index 42e3366cdf..0312a4f9aa 100644
--- a/var/spack/repos/builtin/packages/py-pygments/package.py
+++ b/var/spack/repos/builtin/packages/py-pygments/package.py
@@ -29,8 +29,14 @@ class PyPygments(PythonPackage):
     """Pygments is a syntax highlighting package written in Python."""
 
     homepage = "https://pypi.python.org/pypi/pygments"
-    url      = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz"
+    url      = "https://pypi.io/packages/source/P/Pygments/Pygments-2.2.0.tar.gz"
 
+    import_modules = [
+        'pygments', 'pygments.filters', 'pygments.formatters',
+        'pygments.lexers', 'pygments.styles'
+    ]
+
+    version('2.2.0', '13037baca42f16917cbd5ad2fab50844')
     version('2.1.3', 'ed3fba2467c8afcda4d317e4ef2c6150')
     version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c')
     version('2.0.2', '238587a1370d62405edabd0794b3ec4a')
diff --git a/var/spack/repos/builtin/packages/py-pyparsing/package.py b/var/spack/repos/builtin/packages/py-pyparsing/package.py
index 936295a132..1d0c5a3bf0 100644
--- a/var/spack/repos/builtin/packages/py-pyparsing/package.py
+++ b/var/spack/repos/builtin/packages/py-pyparsing/package.py
@@ -28,8 +28,11 @@ from spack import *
 class PyPyparsing(PythonPackage):
     """A Python Parsing Module."""
     homepage = "https://pypi.python.org/pypi/pyparsing"
-    url      = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz"
+    url      = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.2.0.tar.gz"
 
+    import_modules = ['pyparsing']
+
+    version('2.2.0',  '0214e42d63af850256962b6744c948d9')
     version('2.1.10', '065908b92904e0d3634eb156f44cc80e')
     version('2.0.3',  '0fe479be09fc2cf005f753d3acc35939')
 
diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py
index 94ee8a7fc4..6caccd63a4 100644
--- a/var/spack/repos/builtin/packages/py-setuptools/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools/package.py
@@ -32,12 +32,10 @@ class PySetuptools(PythonPackage):
     homepage = "https://pypi.python.org/pypi/setuptools"
     url      = "https://pypi.io/packages/source/s/setuptools/setuptools-25.2.0.tar.gz"
 
-    import_modules = [
-        'pkg_resources', 'setuptools', 'pkg_resources.extern',
-        'pkg_resources._vendor', 'pkg_resources._vendor.packaging',
-        'setuptools.extern', 'setuptools.command'
-    ]
+    import_modules = ['pkg_resources', 'setuptools', 'setuptools.command']
 
+    version('34.4.1', '5f9b07aeaafd29eac2548fc0b89a4934',
+            url="https://pypi.io/packages/source/s/setuptools/setuptools-34.4.1.zip")
     version('34.2.0', '41b630da4ea6cfa5894d9eb3142922be',
             url="https://pypi.io/packages/source/s/setuptools/setuptools-34.2.0.zip")
     version('25.2.0', 'a0dbb65889c46214c691f6c516cf959c')
diff --git a/var/spack/repos/builtin/packages/py-six/package.py b/var/spack/repos/builtin/packages/py-six/package.py
index b2faad3819..47a53438b5 100644
--- a/var/spack/repos/builtin/packages/py-six/package.py
+++ b/var/spack/repos/builtin/packages/py-six/package.py
@@ -29,7 +29,9 @@ class PySix(PythonPackage):
     """Python 2 and 3 compatibility utilities."""
 
     homepage = "https://pypi.python.org/pypi/six"
-    url      = "https://pypi.io/packages/source/s/six/six-1.9.0.tar.gz"
+    url      = "https://pypi.io/packages/source/s/six/six-1.10.0.tar.gz"
+
+    import_modules = ['six']
 
     version('1.10.0', '34eed507548117b2ab523ab14b2f8b55')
     version('1.9.0',  '476881ef4012262dfc8adc645ee786c4')