package: Add fetch_options variable (#15317)

PR #15212 added a new connect_timeout option that can be overridden
using fetch_options but had to specified per-version. This adds a new
per-package variable that can be used to override fetch_options for
all versions in the package. This includes connect_timeout as well
as 'cookie' (e.g. for the jdk package).

Packages can combine package-level fetch_options with per-version
fetch_options, in which case the version fetch_options completely
override the package-level fetch_options.

This commit includes tests for the added behavior.

10 files changed, 73 insertions, 35 deletions

diff --git a/var/spack/repos/builtin.mock/packages/fetch-options/package.py b/var/spack/repos/builtin.mock/packages/fetch-options/package.py
new file mode 100644
index 0000000000..1a31a535e1
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/fetch-options/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class FetchOptions(Package):
+    """Mock package with fetch_options."""
+
+    homepage = "http://www.fetch-options-example.com"
+
+    url = 'https://example.com/some/tarball-1.0.tar.gz'
+
+    fetch_options = {'timeout': 42, 'cookie': 'foobar'}
+    timeout = {'timeout': 65}
+    cookie = {'cookie': 'baz'}
+
+    version('1.2', 'abc12', fetch_options=cookie)
+    version('1.1', 'abc11', fetch_options=timeout)
+    version('1.0', 'abc10')
diff --git a/var/spack/repos/builtin/packages/bzip2/package.py b/var/spack/repos/builtin/packages/bzip2/package.py
index 87e616d76c..08d5978a63 100644
--- a/var/spack/repos/builtin/packages/bzip2/package.py
+++ b/var/spack/repos/builtin/packages/bzip2/package.py
@@ -19,9 +19,9 @@ class Bzip2(Package):
     # The server is sometimes a bit slow to respond
     fetch_options = {'timeout': 60}
 
-    version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269', fetch_options=fetch_options)
-    version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b', fetch_options=fetch_options)
-    version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd', fetch_options=fetch_options)
+    version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269')
+    version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b')
+    version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd')
 
     variant('shared', default=True, description='Enables the build of shared libraries.')
 
diff --git a/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py b/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py
index 11054b409c..13100d89a8 100644
--- a/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py
+++ b/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py
@@ -19,6 +19,9 @@ class EclipseGcjParser(Package):
 
     maintainers = ['citibeth']
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('4.8', sha256='98fd128f1d374d9e42fd9d4836bdd249c6d511ebc6c0df17fbc1b9df96c3d781', expand=False)
 
     phases = ('build', 'install')
diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py
index 8dd2980c3d..e1ffdc0e3b 100644
--- a/var/spack/repos/builtin/packages/elfutils/package.py
+++ b/var/spack/repos/builtin/packages/elfutils/package.py
@@ -22,17 +22,17 @@ class Elfutils(AutotoolsPackage):
     list_depth = 1
 
     # Sourceware is often slow to respond.
-    timeout = {'timeout': 60}
-
-    version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2', fetch_options=timeout)
-    version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e', fetch_options=timeout)
-    version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023', fetch_options=timeout)
-    version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b', fetch_options=timeout)
-    version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a', fetch_options=timeout)
-    version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff', fetch_options=timeout)
-    version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066', fetch_options=timeout)
-    version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276', fetch_options=timeout)
-    version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23', fetch_options=timeout)
+    fetch_options = {'timeout': 60}
+
+    version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2')
+    version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e')
+    version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023')
+    version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b')
+    version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a')
+    version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff')
+    version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066')
+    version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276')
+    version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23')
 
     # Libraries for reading compressed DWARF sections.
     variant('bzip2', default=False,
diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py
index c5b98286b2..d167bc78cb 100644
--- a/var/spack/repos/builtin/packages/gcc/package.py
+++ b/var/spack/repos/builtin/packages/gcc/package.py
@@ -109,12 +109,16 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
     depends_on('zip', type='build', when='languages=java')
     depends_on('cuda', when='+nvptx')
 
+    # The server is sometimes a bit slow to respond
+    timeout = {'timeout': 60}
+
     resource(
              name='newlib',
              url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz',
              sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b',
              destination='newlibsource',
-             when='+nvptx'
+             when='+nvptx',
+             fetch_options=timeout
             )
 
     # nvptx-tools does not seem to work as a dependency,
diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py
index 6e9c0329dc..025d12be65 100644
--- a/var/spack/repos/builtin/packages/jdk/package.py
+++ b/var/spack/repos/builtin/packages/jdk/package.py
@@ -24,9 +24,7 @@ class Jdk(Package):
     # automate this process, we need to utilize these additional curl
     # command-line options. See:
     # http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux
-    fetch_options = {
-        'cookie': 'oraclelicense=accept-securebackup-cookie'
-    }
+    fetch_options = {'cookie': 'oraclelicense=accept-securebackup-cookie'}
 
     # To add the latest version, go to the homepage listed above,
     # click "JDK Download", click "Accept License Agreement", right-click the
@@ -34,29 +32,29 @@ class Jdk(Package):
     # found in a link above. The build number can be deciphered from the URL.
     # Alternatively, run `bin/java -version` after extracting. Replace '+'
     # symbol in version with '_', otherwise it will be interpreted as a variant
-    version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51', fetch_options=fetch_options,
+    version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51',
             url='https://download.oracle.com/otn-pub/java/jdk/12.0.2+10/e482c34c86bd4bf8b56c0b35558996b9/jdk-12.0.2_linux-x64_bin.tar.gz')
-    version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75', fetch_options=fetch_options,
+    version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75',
             url='https://download.oracle.com/otn-pub/java/jdk/12.0.1+12/69cfe15208a647278a19ef0990eea691/jdk-12.0.1_linux-x64_bin.tar.gz')
-    version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8', fetch_options=fetch_options,
+    version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8',
             url='https://download.oracle.com/otn-pub/java/jdk/11.0.2+9/f51449fcd52f4d52b93a989c5c56ed3c/jdk-11.0.2_linux-x64_bin.tar.gz')
-    version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885', fetch_options=fetch_options,
+    version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885',
             url='https://download.oracle.com/otn-pub/java/jdk/11.0.1+13/90cf5d8f270a4347a95050320eef3fb7/jdk-11.0.1_linux-x64_bin.tar.gz')
-    version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b', fetch_options=fetch_options,
+    version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b',
             url='https://download.oracle.com/otn-pub/java/jdk/10.0.2+13/19aef61b38124481863b1413dce1855f/jdk-10.0.2_linux-x64_bin.tar.gz')
-    version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', fetch_options=fetch_options,
+    version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4',
             url='https://download.oracle.com/otn-pub/java/jdk/10.0.1+10/fb4372174a714e6b8c52526dc134031e/jdk-10.0.1_linux-x64_bin.tar.gz')
-    version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb', fetch_options=fetch_options,
+    version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb',
             url='https://download.oracle.com/otn-pub/java/jdk/8u241-b07/1f5b5a70bf22433b84d0e960903adac8/jdk-8u241-linux-x64.tar.gz')
-    version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf', fetch_options=fetch_options,
+    version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf',
             url='https://download.oracle.com/otn-pub/java/jdk/8u231-b11/5b13a193868b4bf28bcb45c792fce896/jdk-8u231-linux-x64.tar.gz')
-    version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb', fetch_options=fetch_options,
+    version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb',
             url='https://download.oracle.com/otn-pub/java/jdk/8u212-b10/59066701cf1a433da9770636fbc4c9aa/jdk-8u212-linux-x64.tar.gz')
-    version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0', fetch_options=fetch_options,
+    version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0',
             url='https://download.oracle.com/otn-pub/java/jdk/8u202-b08/1961070e4c9b4e26a04e7f5a083f551e/jdk-8u202-linux-x64.tar.gz')
-    version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f', fetch_options=fetch_options,
+    version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f',
             url='https://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz')
-    version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236', fetch_options=fetch_options,
+    version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236',
             url='https://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz')
 
     provides('java@12', when='@12.0:12.999')
diff --git a/var/spack/repos/builtin/packages/libffi/package.py b/var/spack/repos/builtin/packages/libffi/package.py
index 028c591355..07ec319fb8 100644
--- a/var/spack/repos/builtin/packages/libffi/package.py
+++ b/var/spack/repos/builtin/packages/libffi/package.py
@@ -17,8 +17,7 @@ class Libffi(AutotoolsPackage):
     fetch_options = {'timeout': 60}
 
     version('3.2.1', sha256='d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37',
-            url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz",
-            fetch_options=fetch_options)
+            url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz")
 
     @property
     def headers(self):
diff --git a/var/spack/repos/builtin/packages/lvm2/package.py b/var/spack/repos/builtin/packages/lvm2/package.py
index c552fe3ef5..55fa1f1d39 100644
--- a/var/spack/repos/builtin/packages/lvm2/package.py
+++ b/var/spack/repos/builtin/packages/lvm2/package.py
@@ -20,6 +20,9 @@ class Lvm2(AutotoolsPackage):
     homepage = "https://www.sourceware.org/lvm2"
     url = "https://sourceware.org/pub/lvm2/releases/LVM2.2.03.05.tgz"
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('2.03.05', sha256='ca52815c999b20c6d25e3192f142f081b93d01f07b9d787e99664b169dba2700')
     version('2.03.04', sha256='f151f36fc0039997d2d9369b607b9262568b1a268afe19fd1535807355402142')
     version('2.03.03', sha256='cedefa63ec5ae1b62fedbfddfc30706c095be0fc7c6aaed6fd1c50bc8c840dde')
diff --git a/var/spack/repos/builtin/packages/valgrind/package.py b/var/spack/repos/builtin/packages/valgrind/package.py
index 684a5bdfa8..3f759efeac 100644
--- a/var/spack/repos/builtin/packages/valgrind/package.py
+++ b/var/spack/repos/builtin/packages/valgrind/package.py
@@ -22,6 +22,9 @@ class Valgrind(AutotoolsPackage):
     url      = "https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2"
     git      = "git://sourceware.org/git/valgrind.git"
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('develop', branch='master')
     version('3.15.0', sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1')
     version('3.14.0', sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5')
diff --git a/var/spack/repos/builtin/packages/verrou/package.py b/var/spack/repos/builtin/packages/verrou/package.py
index 265ee2b096..f92443d144 100644
--- a/var/spack/repos/builtin/packages/verrou/package.py
+++ b/var/spack/repos/builtin/packages/verrou/package.py
@@ -32,18 +32,24 @@ class Verrou(AutotoolsPackage):
     version('2.0.0', sha256='798df6e426ec57646a2a626d756b72f0171647ae5b07c982952dae2d71e26045')
     version('1.1.0', sha256='b5105f61c65680f31551199cd143b2e15f412c34c821537998a7165e315dde2d')
 
+    # The server is sometimes a bit slow to respond
+    timeout = {'timeout': 60}
+
     resource(name='valgrind-3.15.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2',
              sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1',
-             when='@2.2.0:')
+             when='@2.2.0:',
+             fetch_options=timeout)
     resource(name='valgrind-3.14.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.14.0.tar.bz2',
              sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5',
-             when='@2.1.0:2.1.99')
+             when='@2.1.0:2.1.99',
+             fetch_options=timeout)
     resource(name='valgrind-3.13.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2',
              sha256='d76680ef03f00cd5e970bbdcd4e57fb1f6df7d2e2c071635ef2be74790190c3b',
-             when='@1.1.0:2.0.99')
+             when='@1.1.0:2.0.99',
+             fetch_options=timeout)
 
     variant('fma', default=True,
             description='Activates fused multiply-add support for Verrou')