From 0f35f00b6e2e5e8d9f58ca8d899c84ca9da71fb8 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Thu, 26 Apr 2018 23:40:11 +0200 Subject: Use GitLab's API endpoint for fetching a git snapshot. (#7881) * Use GitLab's API endpoint for fetching a git snapshot. * More GitLab packages use the API. * find_list_url for GitLab's API URLs. * Flake8 * Url for 'hacckernels'. * Check GitLab API regexps before the non-API ones. --- lib/spack/spack/test/url_parse.py | 9 +++++++++ lib/spack/spack/url.py | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index d030e769a4..d9b4352686 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -167,6 +167,7 @@ def test_url_strip_name_suffixes(url, version, expected): ('libxc', 58, '2.2.2', 64, 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz'), # Version in suffix ('swiftsim', 36, '0.3.0', 76, 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + ('swiftsim', 55, '0.3.0', 95, 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), ('sionlib', 30, '1.7.1', 59, 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'), # Regex in name ('voro++', 40, '0.4.6', 47, 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz'), @@ -207,14 +208,21 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): ('git', '2.7.1', 'https://github.com/git/git/tarball/v2.7.1'), # name/zipball/vver.ver ('git', '2.7.1', 'https://github.com/git/git/zipball/v2.7.1'), + # Common Repositories - gitlab downloads # name/repository/archive.ext?ref=vver.ver ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=vver.ver + ('swiftsim', '0.3.0', + 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), # name/repository/archive.ext?ref=name-ver.ver ('icet', '1.2.3', 'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3'), + # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=name-ver.ver + ('icet', '1.2.3', + 'https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-1.2.3'), # Common Repositories - bitbucket downloads @@ -370,6 +378,7 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): ('nextflow', '0.20.1', 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow'), # suffix queries ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), + ('swiftsim', '0.3.0', 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), ('sionlib', '1.7.1', 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'), # stem queries ('slepc', '3.6.2', 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz'), diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index efc1a325e2..329772e17d 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -90,9 +90,14 @@ def find_list_url(url): (r'(.*github\.com/[^/]+/[^/]+)', lambda m: m.group(1) + '/releases'), - # GitLab + # GitLab API endpoint + # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2 + (r'(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)', + lambda m: m.group(1) + '/' + m.group(2) + '/' + m.group(3) + '/tags'), + + # GitLab non-API endpoint # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz - (r'(.*gitlab[^/]+/[^/]+/[^/]+)', + (r'(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)', lambda m: m.group(1) + '/tags'), # BitBucket @@ -529,6 +534,9 @@ def parse_version_offset(path): # 9th Pass: Query strings + # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 + (r'\?sha=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), + # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), @@ -640,9 +648,13 @@ def parse_name_offset(path, v=None): # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz (r'github\.com/[^/]+/([^/]+)', path), - # GitLab: gitlab.*/repo/name/ + # GitLab API endpoint: gitlab.*/api/v4/projects/NAMESPACE%2Fname/ + # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 + (r'gitlab[^/]+/api/v4/projects/[^/]+%2F([^/]+)', path), + + # GitLab non-API endpoint: gitlab.*/repo/name/ # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 - (r'gitlab[^/]+/[^/]+/([^/]+)', path), + (r'gitlab[^/]+/(?!api/v4/projects)[^/]+/([^/]+)', path), # Bitbucket: bitbucket.org/repo/name/ # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2 -- cgit v1.2.3-70-g09d2