Add better tests for web.py; fix some bugs found with spidering.

- _spider in web.py was actually failing to spider deeper than a certain
  point.
- Fixed multiprocessing pools to not use daemons and to allow recursive
  spawning.
- Added detailed tests for spidering and for finding archive versions.
- left some xfail URL finding exercises for the reader.
- Fix noqa annotations for some @when decorators

19 files changed, 23 insertions, 32 deletions

diff --git a/var/spack/repos/builtin/packages/autogen/package.py b/var/spack/repos/builtin/packages/autogen/package.py
index 0bfe6159c9..e79af636b5 100644
--- a/var/spack/repos/builtin/packages/autogen/package.py
+++ b/var/spack/repos/builtin/packages/autogen/package.py
@@ -34,7 +34,7 @@ class Autogen(AutotoolsPackage):
     homepage = "https://www.gnu.org/software/autogen/index.html"
     url      = "https://ftp.gnu.org/gnu/autogen/rel5.18.12/autogen-5.18.12.tar.gz"
     list_url = "https://ftp.gnu.org/gnu/autogen"
-    list_depth = 2
+    list_depth = 1
 
     version('5.18.12', '551d15ccbf5b5fc5658da375d5003389')
 
diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py
index 6f891de664..7570987ad8 100644
--- a/var/spack/repos/builtin/packages/boost/package.py
+++ b/var/spack/repos/builtin/packages/boost/package.py
@@ -40,7 +40,7 @@ class Boost(Package):
     homepage = "http://www.boost.org"
     url      = "http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2"
     list_url = "http://sourceforge.net/projects/boost/files/boost/"
-    list_depth = 2
+    list_depth = 1
 
     version('1.63.0', '1c837ecd990bb022d07e7aab32b09847')
     version('1.62.0', '5fb94629535c19e48703bdb2b2e9490f')
diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py
index 4ff9615016..1dd480d675 100644
--- a/var/spack/repos/builtin/packages/cmake/package.py
+++ b/var/spack/repos/builtin/packages/cmake/package.py
@@ -31,7 +31,7 @@ class Cmake(Package):
     homepage = 'https://www.cmake.org'
     url      = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz'
     list_url = 'https://cmake.org/files/'
-    list_depth = 2
+    list_depth = 1
 
     version('3.7.2',    '79bd7e65cd81ea3aa2619484ad6ff25a')
     version('3.7.1',    'd031d5a06e9f1c5367cdfc56fbd2a1c8')
diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py
index 960b9985ad..2594d73c37 100644
--- a/var/spack/repos/builtin/packages/elfutils/package.py
+++ b/var/spack/repos/builtin/packages/elfutils/package.py
@@ -37,10 +37,10 @@ class Elfutils(AutotoolsPackage):
 
     url      = "https://sourceware.org/elfutils/ftp/0.168/elfutils-0.168.tar.bz2"
     list_url = "https://sourceware.org/elfutils/ftp"
-    list_depth = 2
+    list_depth = 1
 
-    version('0.168','52adfa40758d0d39e5d5c57689bf38d6')
-    version('0.163','77ce87f259987d2e54e4d87b86cbee41', preferred=True)
+    version('0.168', '52adfa40758d0d39e5d5c57689bf38d6')
+    version('0.163', '77ce87f259987d2e54e4d87b86cbee41', preferred=True)
 
     provides('elf@1')
 
diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py
index 1fddb77299..ffb85ec15e 100644
--- a/var/spack/repos/builtin/packages/gcc/package.py
+++ b/var/spack/repos/builtin/packages/gcc/package.py
@@ -37,7 +37,7 @@ class Gcc(AutotoolsPackage):
 
     url = "http://ftp.gnu.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2"
     list_url = 'http://ftp.gnu.org/gnu/gcc/'
-    list_depth = 2
+    list_depth = 1
 
     version('6.3.0', '677a7623c7ef6ab99881bc4e048debb6')
     version('6.2.0', '9768625159663b300ae4de2f4745fcc4')
diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py
index b52b1f1038..dc58cc8993 100644
--- a/var/spack/repos/builtin/packages/gdal/package.py
+++ b/var/spack/repos/builtin/packages/gdal/package.py
@@ -38,7 +38,7 @@ class Gdal(Package):
     homepage   = "http://www.gdal.org/"
     url        = "http://download.osgeo.org/gdal/2.1.2/gdal-2.1.2.tar.xz"
     list_url   = "http://download.osgeo.org/gdal/"
-    list_depth = 2
+    list_depth = 1
 
     version('2.1.2', 'ae85b78888514c75e813d658cac9478e')
     version('2.0.2', '940208e737c87d31a90eaae43d0efd65')
diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py
index 0030d5ffde..6efa2dfbd9 100644
--- a/var/spack/repos/builtin/packages/hwloc/package.py
+++ b/var/spack/repos/builtin/packages/hwloc/package.py
@@ -42,7 +42,7 @@ class Hwloc(AutotoolsPackage):
     homepage = "http://www.open-mpi.org/projects/hwloc/"
     url      = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz"
     list_url = "http://www.open-mpi.org/software/hwloc/"
-    list_depth = 3
+    list_depth = 2
 
     version('1.11.6', 'b4e95eadd2fbdb6d40bbd96be6f03c84')
     version('1.11.5', '8f5fe6a9be2eb478409ad5e640b2d3ba')
diff --git a/var/spack/repos/builtin/packages/hydra/package.py b/var/spack/repos/builtin/packages/hydra/package.py
index 4461adae2e..fd894b68ee 100644
--- a/var/spack/repos/builtin/packages/hydra/package.py
+++ b/var/spack/repos/builtin/packages/hydra/package.py
@@ -34,6 +34,6 @@ class Hydra(AutotoolsPackage):
     homepage = "http://www.mpich.org"
     url      = "http://www.mpich.org/static/downloads/3.2/hydra-3.2.tar.gz"
     list_url = "http://www.mpich.org/static/downloads/"
-    list_depth = 2
+    list_depth = 1
 
     version('3.2', '4d670916695bf7e3a869cc336a881b39')
diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py
index 09fc683874..819fc95d5b 100644
--- a/var/spack/repos/builtin/packages/mpich/package.py
+++ b/var/spack/repos/builtin/packages/mpich/package.py
@@ -32,7 +32,7 @@ class Mpich(AutotoolsPackage):
     homepage = "http://www.mpich.org"
     url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz"
     list_url = "http://www.mpich.org/static/downloads/"
-    list_depth = 2
+    list_depth = 1
 
     version('3.2',   'f414cfa77099cd1fa1a5ae4e22db508a')
     version('3.1.4', '2ab544607986486562e076b83937bba2')
diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py
index 3480311aed..7c07c515fa 100644
--- a/var/spack/repos/builtin/packages/openmpi/package.py
+++ b/var/spack/repos/builtin/packages/openmpi/package.py
@@ -61,7 +61,7 @@ class Openmpi(AutotoolsPackage):
     homepage = "http://www.open-mpi.org"
     url = "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2"
     list_url = "http://www.open-mpi.org/software/ompi/"
-    list_depth = 3
+    list_depth = 2
 
     version('2.1.0', '4838a5973115c44e14442c01d3f21d52')
     version('2.0.2', 'ecd99aa436a1ca69ce936a96d6a3fa48')
diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py
index 1432ac76fc..53e94ed445 100644
--- a/var/spack/repos/builtin/packages/openssl/package.py
+++ b/var/spack/repos/builtin/packages/openssl/package.py
@@ -37,7 +37,7 @@ class Openssl(Package):
     # URL must remain http:// so Spack can bootstrap curl
     url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz"
     list_url = "https://www.openssl.org/source/old/"
-    list_depth = 2
+    list_depth = 1
 
     version('1.1.0e', '51c42d152122e474754aea96f66928c6')
     version('1.1.0d', '711ce3cd5f53a99c0e12a7d5804f0f63')
diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py
index 0f082527b9..cc6565dae3 100644
--- a/var/spack/repos/builtin/packages/pango/package.py
+++ b/var/spack/repos/builtin/packages/pango/package.py
@@ -33,7 +33,7 @@ class Pango(AutotoolsPackage):
     homepage = "http://www.pango.org"
     url      = "http://ftp.gnome.org/pub/GNOME/sources/pango/1.40/pango-1.40.3.tar.xz"
     list_url = "http://ftp.gnome.org/pub/gnome/sources/pango/"
-    list_depth = 2
+    list_depth = 1
 
     version('1.40.3', 'abba8b5ce728520c3a0f1535eab19eac3c14aeef7faa5aded90017ceac2711d3')
     version('1.40.1', 'e27af54172c72b3ac6be53c9a4c67053e16c905e02addcf3a603ceb2005c1a40')
diff --git a/var/spack/repos/builtin/packages/patchelf/package.py b/var/spack/repos/builtin/packages/patchelf/package.py
index 1d429361b6..d36366f557 100644
--- a/var/spack/repos/builtin/packages/patchelf/package.py
+++ b/var/spack/repos/builtin/packages/patchelf/package.py
@@ -33,7 +33,7 @@ class Patchelf(AutotoolsPackage):
     url = "http://nixos.org/releases/patchelf/patchelf-0.8/patchelf-0.8.tar.gz"
 
     list_url = "http://nixos.org/releases/patchelf/"
-    list_depth = 2
+    list_depth = 1
 
     version('0.9', '3c265508526760f233620f35d79c79fc')
     version('0.8', '407b229e6a681ffb0e2cdd5915cb2d01')
diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py
index 2b27ed4f1d..f3f08029a4 100644
--- a/var/spack/repos/builtin/packages/py-nose/package.py
+++ b/var/spack/repos/builtin/packages/py-nose/package.py
@@ -31,8 +31,6 @@ class PyNose(PythonPackage):
 
     homepage = "https://pypi.python.org/pypi/nose"
     url      = "https://pypi.io/packages/source/n/nose/nose-1.3.4.tar.gz"
-    list_url = "https://pypi.python.org/pypi/nose/"
-    list_depth = 2
 
     import_modules = [
         'nose', 'nose.ext', 'nose.plugins', 'nose.sphinx', 'nose.tools'
diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
index e61a90f352..844453944a 100644
--- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
@@ -30,8 +30,6 @@ class PyScikitLearn(PythonPackage):
 
     homepage = "https://pypi.python.org/pypi/scikit-learn"
     url      = "https://pypi.io/packages/source/s/scikit-learn/scikit-learn-0.18.1.tar.gz"
-    list_url = "https://pypi.python.org/pypi/scikit-learn"
-    list_depth = 2
 
     version('0.18.1', '6b0ff1eaa5010043895dd63d1e3c60c9')
     version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d')
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index d4d6f713ba..6e0b5b8dc8 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -42,7 +42,7 @@ class Python(Package):
     homepage = "http://www.python.org"
     url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
     list_url = "https://www.python.org/downloads/"
-    list_depth = 2
+    list_depth = 1
 
     version('3.6.0', '3f7062ccf8be76491884d0e47ac8b251')
     version('3.5.2', '3fe8434643a78630c61c6464fe2e7e72')
@@ -99,11 +99,6 @@ class Python(Package):
             r'\1setup.py\2 --no-user-cfg \3\6'
         )
 
-    @when('@:2.6,3.0:3.3')
-    def patch(self):
-        # See https://github.com/LLNL/spack/issues/1490
-        pass
-
     def install(self, spec, prefix):
         # TODO: The '--no-user-cfg' option for Python installation is only in
         # Python v2.7 and v3.4+ (see https://bugs.python.org/issue1180) and
diff --git a/var/spack/repos/builtin/packages/qt-creator/package.py b/var/spack/repos/builtin/packages/qt-creator/package.py
index 347cf4d6ee..abd619530f 100644
--- a/var/spack/repos/builtin/packages/qt-creator/package.py
+++ b/var/spack/repos/builtin/packages/qt-creator/package.py
@@ -32,7 +32,7 @@ class QtCreator(Package):
     url      = 'http://download.qt.io/official_releases/qtcreator/4.1/4.1.0/qt-creator-opensource-src-4.1.0.tar.gz'
 
     list_url = 'http://download.qt.io/official_releases/qtcreator/'
-    list_depth = 3
+    list_depth = 2
 
     version('4.1.0',  '657727e4209befa4bf5889dff62d9e0a')
 
diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py
index 60c5f15ece..b27bc3fe07 100644
--- a/var/spack/repos/builtin/packages/qt/package.py
+++ b/var/spack/repos/builtin/packages/qt/package.py
@@ -33,7 +33,7 @@ class Qt(Package):
     homepage = 'http://qt.io'
     url      = 'http://download.qt.io/archive/qt/5.7/5.7.0/single/qt-everywhere-opensource-src-5.7.0.tar.gz'
     list_url = 'http://download.qt.io/archive/qt/'
-    list_depth = 4
+    list_depth = 3
 
     version('5.7.1',  '031fb3fd0c3cc0f1082644492683f18d')
     version('5.7.0',  '9a46cce61fc64c20c3ac0a0e0fa41b42')
@@ -251,7 +251,7 @@ class Qt(Package):
     # Don't disable all the database drivers, but should
     # really get them into spack at some point.
 
-    @when('@3')
+    @when('@3')  # noqa: F811
     def configure(self):
         # A user reported that this was necessary to link Qt3 on ubuntu.
         # However, if LD_LIBRARY_PATH is not set the qt build fails, check
@@ -268,7 +268,7 @@ class Qt(Package):
                   '-release',
                   '-fast')
 
-    @when('@4')
+    @when('@4')  # noqa: F811
     def configure(self):
         configure('-fast',
                   '-{0}gtkstyle'.format('' if '+gtk' in self.spec else 'no-'),
@@ -276,7 +276,7 @@ class Qt(Package):
                   '-arch', str(self.spec.architecture.target),
                   *self.common_config_args)
 
-    @when('@5.0:5.6')
+    @when('@5.0:5.6')  # noqa: F811
     def configure(self):
         webkit_args = [] if '+webkit' in self.spec else ['-skip', 'qtwebkit']
         configure('-no-eglfs',
@@ -284,7 +284,7 @@ class Qt(Package):
                   '-{0}gtkstyle'.format('' if '+gtk' in self.spec else 'no-'),
                   *(webkit_args + self.common_config_args))
 
-    @when('@5.7:')
+    @when('@5.7:')  # noqa: F811
     def configure(self):
         config_args = self.common_config_args
 
diff --git a/var/spack/repos/builtin/packages/util-linux/package.py b/var/spack/repos/builtin/packages/util-linux/package.py
index 5bbaa995f8..2fc413a98e 100644
--- a/var/spack/repos/builtin/packages/util-linux/package.py
+++ b/var/spack/repos/builtin/packages/util-linux/package.py
@@ -31,7 +31,7 @@ class UtilLinux(AutotoolsPackage):
     homepage = "http://freecode.com/projects/util-linux"
     url      = "https://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.1.tar.gz"
     list_url = "https://www.kernel.org/pub/linux/utils/util-linux"
-    list_depth = 2
+    list_depth = 1
 
     version('2.29.1', 'c7d5c111ef6bc5df65659e0b523ac9d9')
     version('2.25',   'f6d7fc6952ec69c4dc62c8d7c59c1d57')