From 14441e00dc1c04df40d361ca46fd63c7446ac2ae Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 13 Mar 2020 19:41:19 +0100 Subject: package: Add fetch_options variable (#15317) PR #15212 added a new connect_timeout option that can be overridden using fetch_options but had to specified per-version. This adds a new per-package variable that can be used to override fetch_options for all versions in the package. This includes connect_timeout as well as 'cookie' (e.g. for the jdk package). Packages can combine package-level fetch_options with per-version fetch_options, in which case the version fetch_options completely override the package-level fetch_options. This commit includes tests for the added behavior. --- .../builtin.mock/packages/fetch-options/package.py | 22 +++++++++++++++++ var/spack/repos/builtin/packages/bzip2/package.py | 6 ++--- .../builtin/packages/eclipse-gcj-parser/package.py | 3 +++ .../repos/builtin/packages/elfutils/package.py | 22 ++++++++--------- var/spack/repos/builtin/packages/gcc/package.py | 6 ++++- var/spack/repos/builtin/packages/jdk/package.py | 28 ++++++++++------------ var/spack/repos/builtin/packages/libffi/package.py | 3 +-- var/spack/repos/builtin/packages/lvm2/package.py | 3 +++ .../repos/builtin/packages/valgrind/package.py | 3 +++ var/spack/repos/builtin/packages/verrou/package.py | 12 +++++++--- 10 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 var/spack/repos/builtin.mock/packages/fetch-options/package.py (limited to 'var') diff --git a/var/spack/repos/builtin.mock/packages/fetch-options/package.py b/var/spack/repos/builtin.mock/packages/fetch-options/package.py new file mode 100644 index 0000000000..1a31a535e1 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/fetch-options/package.py @@ -0,0 +1,22 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class FetchOptions(Package): + """Mock package with fetch_options.""" + + homepage = "http://www.fetch-options-example.com" + + url = 'https://example.com/some/tarball-1.0.tar.gz' + + fetch_options = {'timeout': 42, 'cookie': 'foobar'} + timeout = {'timeout': 65} + cookie = {'cookie': 'baz'} + + version('1.2', 'abc12', fetch_options=cookie) + version('1.1', 'abc11', fetch_options=timeout) + version('1.0', 'abc10') diff --git a/var/spack/repos/builtin/packages/bzip2/package.py b/var/spack/repos/builtin/packages/bzip2/package.py index 87e616d76c..08d5978a63 100644 --- a/var/spack/repos/builtin/packages/bzip2/package.py +++ b/var/spack/repos/builtin/packages/bzip2/package.py @@ -19,9 +19,9 @@ class Bzip2(Package): # The server is sometimes a bit slow to respond fetch_options = {'timeout': 60} - version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269', fetch_options=fetch_options) - version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b', fetch_options=fetch_options) - version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd', fetch_options=fetch_options) + version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269') + version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b') + version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd') variant('shared', default=True, description='Enables the build of shared libraries.') diff --git a/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py b/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py index 11054b409c..13100d89a8 100644 --- a/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py +++ b/var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py @@ -19,6 +19,9 @@ class EclipseGcjParser(Package): maintainers = ['citibeth'] + # The server is sometimes a bit slow to respond + fetch_options = {'timeout': 60} + version('4.8', sha256='98fd128f1d374d9e42fd9d4836bdd249c6d511ebc6c0df17fbc1b9df96c3d781', expand=False) phases = ('build', 'install') diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py index 8dd2980c3d..e1ffdc0e3b 100644 --- a/var/spack/repos/builtin/packages/elfutils/package.py +++ b/var/spack/repos/builtin/packages/elfutils/package.py @@ -22,17 +22,17 @@ class Elfutils(AutotoolsPackage): list_depth = 1 # Sourceware is often slow to respond. - timeout = {'timeout': 60} - - version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2', fetch_options=timeout) - version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e', fetch_options=timeout) - version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023', fetch_options=timeout) - version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b', fetch_options=timeout) - version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a', fetch_options=timeout) - version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff', fetch_options=timeout) - version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066', fetch_options=timeout) - version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276', fetch_options=timeout) - version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23', fetch_options=timeout) + fetch_options = {'timeout': 60} + + version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2') + version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e') + version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023') + version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b') + version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a') + version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff') + version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066') + version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276') + version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23') # Libraries for reading compressed DWARF sections. variant('bzip2', default=False, diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index c5b98286b2..d167bc78cb 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -109,12 +109,16 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage): depends_on('zip', type='build', when='languages=java') depends_on('cuda', when='+nvptx') + # The server is sometimes a bit slow to respond + timeout = {'timeout': 60} + resource( name='newlib', url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz', sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b', destination='newlibsource', - when='+nvptx' + when='+nvptx', + fetch_options=timeout ) # nvptx-tools does not seem to work as a dependency, diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index 6e9c0329dc..025d12be65 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -24,9 +24,7 @@ class Jdk(Package): # automate this process, we need to utilize these additional curl # command-line options. See: # http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux - fetch_options = { - 'cookie': 'oraclelicense=accept-securebackup-cookie' - } + fetch_options = {'cookie': 'oraclelicense=accept-securebackup-cookie'} # To add the latest version, go to the homepage listed above, # click "JDK Download", click "Accept License Agreement", right-click the @@ -34,29 +32,29 @@ class Jdk(Package): # found in a link above. The build number can be deciphered from the URL. # Alternatively, run `bin/java -version` after extracting. Replace '+' # symbol in version with '_', otherwise it will be interpreted as a variant - version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51', fetch_options=fetch_options, + version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51', url='https://download.oracle.com/otn-pub/java/jdk/12.0.2+10/e482c34c86bd4bf8b56c0b35558996b9/jdk-12.0.2_linux-x64_bin.tar.gz') - version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75', fetch_options=fetch_options, + version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75', url='https://download.oracle.com/otn-pub/java/jdk/12.0.1+12/69cfe15208a647278a19ef0990eea691/jdk-12.0.1_linux-x64_bin.tar.gz') - version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8', fetch_options=fetch_options, + version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8', url='https://download.oracle.com/otn-pub/java/jdk/11.0.2+9/f51449fcd52f4d52b93a989c5c56ed3c/jdk-11.0.2_linux-x64_bin.tar.gz') - version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885', fetch_options=fetch_options, + version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885', url='https://download.oracle.com/otn-pub/java/jdk/11.0.1+13/90cf5d8f270a4347a95050320eef3fb7/jdk-11.0.1_linux-x64_bin.tar.gz') - version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b', fetch_options=fetch_options, + version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b', url='https://download.oracle.com/otn-pub/java/jdk/10.0.2+13/19aef61b38124481863b1413dce1855f/jdk-10.0.2_linux-x64_bin.tar.gz') - version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', fetch_options=fetch_options, + version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', url='https://download.oracle.com/otn-pub/java/jdk/10.0.1+10/fb4372174a714e6b8c52526dc134031e/jdk-10.0.1_linux-x64_bin.tar.gz') - version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb', fetch_options=fetch_options, + version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb', url='https://download.oracle.com/otn-pub/java/jdk/8u241-b07/1f5b5a70bf22433b84d0e960903adac8/jdk-8u241-linux-x64.tar.gz') - version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf', fetch_options=fetch_options, + version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf', url='https://download.oracle.com/otn-pub/java/jdk/8u231-b11/5b13a193868b4bf28bcb45c792fce896/jdk-8u231-linux-x64.tar.gz') - version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb', fetch_options=fetch_options, + version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb', url='https://download.oracle.com/otn-pub/java/jdk/8u212-b10/59066701cf1a433da9770636fbc4c9aa/jdk-8u212-linux-x64.tar.gz') - version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0', fetch_options=fetch_options, + version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0', url='https://download.oracle.com/otn-pub/java/jdk/8u202-b08/1961070e4c9b4e26a04e7f5a083f551e/jdk-8u202-linux-x64.tar.gz') - version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f', fetch_options=fetch_options, + version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f', url='https://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz') - version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236', fetch_options=fetch_options, + version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236', url='https://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz') provides('java@12', when='@12.0:12.999') diff --git a/var/spack/repos/builtin/packages/libffi/package.py b/var/spack/repos/builtin/packages/libffi/package.py index 028c591355..07ec319fb8 100644 --- a/var/spack/repos/builtin/packages/libffi/package.py +++ b/var/spack/repos/builtin/packages/libffi/package.py @@ -17,8 +17,7 @@ class Libffi(AutotoolsPackage): fetch_options = {'timeout': 60} version('3.2.1', sha256='d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37', - url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz", - fetch_options=fetch_options) + url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") @property def headers(self): diff --git a/var/spack/repos/builtin/packages/lvm2/package.py b/var/spack/repos/builtin/packages/lvm2/package.py index c552fe3ef5..55fa1f1d39 100644 --- a/var/spack/repos/builtin/packages/lvm2/package.py +++ b/var/spack/repos/builtin/packages/lvm2/package.py @@ -20,6 +20,9 @@ class Lvm2(AutotoolsPackage): homepage = "https://www.sourceware.org/lvm2" url = "https://sourceware.org/pub/lvm2/releases/LVM2.2.03.05.tgz" + # The server is sometimes a bit slow to respond + fetch_options = {'timeout': 60} + version('2.03.05', sha256='ca52815c999b20c6d25e3192f142f081b93d01f07b9d787e99664b169dba2700') version('2.03.04', sha256='f151f36fc0039997d2d9369b607b9262568b1a268afe19fd1535807355402142') version('2.03.03', sha256='cedefa63ec5ae1b62fedbfddfc30706c095be0fc7c6aaed6fd1c50bc8c840dde') diff --git a/var/spack/repos/builtin/packages/valgrind/package.py b/var/spack/repos/builtin/packages/valgrind/package.py index 684a5bdfa8..3f759efeac 100644 --- a/var/spack/repos/builtin/packages/valgrind/package.py +++ b/var/spack/repos/builtin/packages/valgrind/package.py @@ -22,6 +22,9 @@ class Valgrind(AutotoolsPackage): url = "https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2" git = "git://sourceware.org/git/valgrind.git" + # The server is sometimes a bit slow to respond + fetch_options = {'timeout': 60} + version('develop', branch='master') version('3.15.0', sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1') version('3.14.0', sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5') diff --git a/var/spack/repos/builtin/packages/verrou/package.py b/var/spack/repos/builtin/packages/verrou/package.py index 265ee2b096..f92443d144 100644 --- a/var/spack/repos/builtin/packages/verrou/package.py +++ b/var/spack/repos/builtin/packages/verrou/package.py @@ -32,18 +32,24 @@ class Verrou(AutotoolsPackage): version('2.0.0', sha256='798df6e426ec57646a2a626d756b72f0171647ae5b07c982952dae2d71e26045') version('1.1.0', sha256='b5105f61c65680f31551199cd143b2e15f412c34c821537998a7165e315dde2d') + # The server is sometimes a bit slow to respond + timeout = {'timeout': 60} + resource(name='valgrind-3.15.0', url='https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2', sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1', - when='@2.2.0:') + when='@2.2.0:', + fetch_options=timeout) resource(name='valgrind-3.14.0', url='https://sourceware.org/pub/valgrind/valgrind-3.14.0.tar.bz2', sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5', - when='@2.1.0:2.1.99') + when='@2.1.0:2.1.99', + fetch_options=timeout) resource(name='valgrind-3.13.0', url='https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2', sha256='d76680ef03f00cd5e970bbdcd4e57fb1f6df7d2e2c071635ef2be74790190c3b', - when='@1.1.0:2.0.99') + when='@1.1.0:2.0.99', + fetch_options=timeout) variant('fma', default=True, description='Activates fused multiply-add support for Verrou') -- cgit v1.2.3-70-g09d2