diff options
author | Sergey Kosukhin <skosukhin@gmail.com> | 2018-04-26 23:40:11 +0200 |
---|---|---|
committer | Adam J. Stewart <ajstewart426@gmail.com> | 2018-04-26 16:40:11 -0500 |
commit | 0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8 (patch) | |
tree | 4f41782185383cf05714aa754b8f935059472531 | |
parent | 8afdaa58056744a556c74dd938a6f66cf2ac28c5 (diff) | |
download | spack-0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8.tar.gz spack-0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8.tar.bz2 spack-0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8.tar.xz spack-0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8.zip |
Use GitLab's API endpoint for fetching a git snapshot. (#7881)
* Use GitLab's API endpoint for fetching a git snapshot.
* More GitLab packages use the API.
* find_list_url for GitLab's API URLs.
* Flake8
* Url for 'hacckernels'.
* Check GitLab API regexps before the non-API ones.
-rw-r--r-- | lib/spack/spack/test/url_parse.py | 9 | ||||
-rw-r--r-- | lib/spack/spack/url.py | 20 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/busco/package.py | 12 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/hacckernels/package.py | 2 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/icet/package.py | 2 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/libaec/package.py | 2 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/parsplice/package.py | 2 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/swfft/package.py | 3 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/swiftsim/package.py | 5 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/vecgeom/package.py | 4 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/vtkm/package.py | 2 |
11 files changed, 44 insertions, 19 deletions
diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index d030e769a4..d9b4352686 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -167,6 +167,7 @@ def test_url_strip_name_suffixes(url, version, expected): ('libxc', 58, '2.2.2', 64, 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz'), # Version in suffix ('swiftsim', 36, '0.3.0', 76, 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + ('swiftsim', 55, '0.3.0', 95, 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), ('sionlib', 30, '1.7.1', 59, 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'), # Regex in name ('voro++', 40, '0.4.6', 47, 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz'), @@ -207,14 +208,21 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): ('git', '2.7.1', 'https://github.com/git/git/tarball/v2.7.1'), # name/zipball/vver.ver ('git', '2.7.1', 'https://github.com/git/git/zipball/v2.7.1'), + # Common Repositories - gitlab downloads # name/repository/archive.ext?ref=vver.ver ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=vver.ver + ('swiftsim', '0.3.0', + 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), # name/repository/archive.ext?ref=name-ver.ver ('icet', '1.2.3', 'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3'), + # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=name-ver.ver + ('icet', '1.2.3', + 'https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-1.2.3'), # Common Repositories - bitbucket downloads @@ -370,6 +378,7 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): ('nextflow', '0.20.1', 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow'), # suffix queries ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + ('swiftsim', '0.3.0', 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), ('sionlib', '1.7.1', 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'), # stem queries ('slepc', '3.6.2', 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz'), diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index efc1a325e2..329772e17d 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -90,9 +90,14 @@ def find_list_url(url): (r'(.*github\.com/[^/]+/[^/]+)', lambda m: m.group(1) + '/releases'), - # GitLab + # GitLab API endpoint + # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2 + (r'(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)', + lambda m: m.group(1) + '/' + m.group(2) + '/' + m.group(3) + '/tags'), + + # GitLab non-API endpoint # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz - (r'(.*gitlab[^/]+/[^/]+/[^/]+)', + (r'(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)', lambda m: m.group(1) + '/tags'), # BitBucket @@ -529,6 +534,9 @@ def parse_version_offset(path): # 9th Pass: Query strings + # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 + (r'\?sha=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), + # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), @@ -640,9 +648,13 @@ def parse_name_offset(path, v=None): # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz (r'github\.com/[^/]+/([^/]+)', path), - # GitLab: gitlab.*/repo/name/ + # GitLab API endpoint: gitlab.*/api/v4/projects/NAMESPACE%2Fname/ + # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 + (r'gitlab[^/]+/api/v4/projects/[^/]+%2F([^/]+)', path), + + # GitLab non-API endpoint: gitlab.*/repo/name/ # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 - (r'gitlab[^/]+/[^/]+/([^/]+)', path), + (r'gitlab[^/]+/(?!api/v4/projects)[^/]+/([^/]+)', path), # Bitbucket: bitbucket.org/repo/name/ # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2 diff --git a/var/spack/repos/builtin/packages/busco/package.py b/var/spack/repos/builtin/packages/busco/package.py index ca4220f29d..48c4c50719 100644 --- a/var/spack/repos/builtin/packages/busco/package.py +++ b/var/spack/repos/builtin/packages/busco/package.py @@ -30,10 +30,16 @@ class Busco(PythonPackage): Universal Single-Copy Orthologs""" homepage = "http://busco.ezlab.org/" - url = "https://gitlab.com/ezlab/busco" + url = "https://gitlab.com/api/v4/projects/ezlab%2Fbusco/repository/archive.tar.gz?sha=2.0.1" - version('3.0.1', git='https://gitlab.com/ezlab/busco.git', commit='078252e00399550d7b0e8941cd4d986c8e868a83') - version('2.0.1', git='https://gitlab.com/ezlab/busco.git', commit='89aa1ab2527f03a87a214ca90a504ad236582a11') + # TODO: check the installation procedure for version 3.0.2 + # and uncomment the following line + # version('3.0.2', '31d80042bb7e96422843fa43d0acbd21') + + # There is no tag for version 3.0.1 + version('3.0.1', git='https://gitlab.com/ezlab/busco.git', + commit='078252e00399550d7b0e8941cd4d986c8e868a83') + version('2.0.1', '4dbcc8a0c18fa8f8312c103eb2fbb4e2') depends_on('python', type=('build', 'run')) depends_on('blast-plus') diff --git a/var/spack/repos/builtin/packages/hacckernels/package.py b/var/spack/repos/builtin/packages/hacckernels/package.py index 39a11c7011..e3a5c3cc12 100644 --- a/var/spack/repos/builtin/packages/hacckernels/package.py +++ b/var/spack/repos/builtin/packages/hacckernels/package.py @@ -33,7 +33,7 @@ class Hacckernels(CMakePackage): cores and beyond.""" homepage = "https://xgitlab.cels.anl.gov/hacc/HACCKernels" - url = "https://xgitlab.cels.anl.gov/hacc/HACCKernels.git" + url = "https://xgitlab.cels.anl.gov/api/v4/projects/hacc%2FHACCKernels/repository/archive.tar.gz" tags = ['proxy-app'] diff --git a/var/spack/repos/builtin/packages/icet/package.py b/var/spack/repos/builtin/packages/icet/package.py index ff934c553e..1a2181c24f 100644 --- a/var/spack/repos/builtin/packages/icet/package.py +++ b/var/spack/repos/builtin/packages/icet/package.py @@ -30,7 +30,7 @@ class Icet(CMakePackage): sort-last parallel rendering library.""" homepage = "http://icet.sandia.gov" - url = "https://gitlab.kitware.com/icet/icet/repository/archive.tar.bz2?ref=IceT-2.1.1" + url = "https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1" version('develop', branch='master', git='https://gitlab.kitware.com/icet/icet.git') diff --git a/var/spack/repos/builtin/packages/libaec/package.py b/var/spack/repos/builtin/packages/libaec/package.py index f6b349942b..b89723131f 100644 --- a/var/spack/repos/builtin/packages/libaec/package.py +++ b/var/spack/repos/builtin/packages/libaec/package.py @@ -33,7 +33,7 @@ class Libaec(CMakePackage): """ homepage = 'https://gitlab.dkrz.de/k202009/libaec' - url = 'https://gitlab.dkrz.de/k202009/libaec/repository/archive.tar.gz?ref=v1.0.2' + url = 'https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2' list_url = 'https://gitlab.dkrz.de/k202009/libaec/tags' provides('szip') diff --git a/var/spack/repos/builtin/packages/parsplice/package.py b/var/spack/repos/builtin/packages/parsplice/package.py index 24b950548f..10b81171de 100644 --- a/var/spack/repos/builtin/packages/parsplice/package.py +++ b/var/spack/repos/builtin/packages/parsplice/package.py @@ -30,7 +30,7 @@ class Parsplice(CMakePackage): """ParSplice code implements the Parallel Trajectory Splicing algorithm""" homepage = "https://gitlab.com/exaalt/parsplice" - url = "https://gitlab.com/exaalt/parsplice/repository/archive.tar.gz?ref=v1.1" + url = "https://gitlab.com/api/v4/projects/exaalt%2Fparsplice/repository/archive.tar.gz?sha=v1.1" version('1.1', '3a72340d49d731a076e8942f2ae2f4e9') version('develop', git='https://gitlab.com/exaalt/parsplice', branch='master') diff --git a/var/spack/repos/builtin/packages/swfft/package.py b/var/spack/repos/builtin/packages/swfft/package.py index ce0b027c9e..ba77704770 100644 --- a/var/spack/repos/builtin/packages/swfft/package.py +++ b/var/spack/repos/builtin/packages/swfft/package.py @@ -31,8 +31,7 @@ class Swfft(MakefilePackage): parallel 3D FFT.""" homepage = 'https://xgitlab.cels.anl.gov/hacc/SWFFT' - - url = "https://xgitlab.cels.anl.gov/hacc/SWFFT/repository/v1.0/archive.tar.gz" + url = "https://xgitlab.cels.anl.gov/api/v4/projects/hacc%2FSWFFT/repository/archive.tar.gz?sha=v1.0" version('1.0', '0fbc34544b97ba9c3fb19ef2d7a0f076') version('develop', git='https://xgitlab.cels.anl.gov/hacc/SWFFT', diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index 6241634ab2..602a03fd73 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -33,10 +33,9 @@ class Swiftsim(AutotoolsPackage): """ homepage = 'http://icc.dur.ac.uk/swift/' - url = 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0' + url = 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0' - version('0.3.0', git='https://gitlab.cosma.dur.ac.uk/swift/swiftsim.git', - commit='254cc1b563b2f88ddcf437b1f71da123bb9db733') + version('0.3.0', '162ec2bdfdf44a31a08b3fcee23a886a') variant('mpi', default=True, description='Enable distributed memory parallelism') diff --git a/var/spack/repos/builtin/packages/vecgeom/package.py b/var/spack/repos/builtin/packages/vecgeom/package.py index 1e5a1542ff..8945f07ea5 100644 --- a/var/spack/repos/builtin/packages/vecgeom/package.py +++ b/var/spack/repos/builtin/packages/vecgeom/package.py @@ -32,9 +32,9 @@ class Vecgeom(CMakePackage): (toolkits).""" homepage = "https://gitlab.cern.ch/VecGeom/VecGeom" + url = "https://gitlab.cern.ch/api/v4/projects/VecGeom%2FVecGeom/repository/archive.tar.gz?sha=v0.3.rc" - version('0.3.rc', git='https://gitlab.cern.ch/VecGeom/VecGeom.git', - tag='v0.3.rc') + version('0.3.rc', 'c1f5d620f655f3c0610a44e7735203b5') depends_on('cmake@3.5:', type='build') diff --git a/var/spack/repos/builtin/packages/vtkm/package.py b/var/spack/repos/builtin/packages/vtkm/package.py index a461c70b72..f9075b33da 100644 --- a/var/spack/repos/builtin/packages/vtkm/package.py +++ b/var/spack/repos/builtin/packages/vtkm/package.py @@ -36,7 +36,7 @@ class Vtkm(Package): architectures.""" homepage = "https://m.vtk.org/" - url = "https://gitlab.kitware.com/vtk/vtk-m/repository/v1.1.0/archive.tar.gz" + url = "https://gitlab.kitware.com/api/v4/projects/vtk%2Fvtk-m/repository/archive.tar.gz?sha=v1.1.0" version('1.1.0', "6aab1c0885f6ffaaffcf07930873d0df") |