Use GitLab's API endpoint for fetching a git snapshot. (#7881)

* Use GitLab's API endpoint for fetching a git snapshot.

* More GitLab packages use the API.

* find_list_url for GitLab's API URLs.

* Flake8

* Url for 'hacckernels'.

* Check GitLab API regexps before the non-API ones.

2 files changed, 25 insertions, 4 deletions

diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py
index d030e769a4..d9b4352686 100644
--- a/lib/spack/spack/test/url_parse.py
+++ b/lib/spack/spack/test/url_parse.py
@@ -167,6 +167,7 @@ def test_url_strip_name_suffixes(url, version, expected):
     ('libxc', 58, '2.2.2', 64, 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz'),
     # Version in suffix
     ('swiftsim', 36, '0.3.0', 76, 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'),
+    ('swiftsim', 55, '0.3.0', 95, 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'),
     ('sionlib', 30, '1.7.1', 59, 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'),
     # Regex in name
     ('voro++', 40, '0.4.6', 47, 'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz'),
@@ -207,14 +208,21 @@ def test_url_parse_offset(name, noffset, ver, voffset, path):
     ('git', '2.7.1', 'https://github.com/git/git/tarball/v2.7.1'),
     # name/zipball/vver.ver
     ('git', '2.7.1', 'https://github.com/git/git/zipball/v2.7.1'),
+
     # Common Repositories - gitlab downloads
 
     # name/repository/archive.ext?ref=vver.ver
     ('swiftsim', '0.3.0',
      'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'),
+    # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=vver.ver
+    ('swiftsim', '0.3.0',
+     'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'),
     # name/repository/archive.ext?ref=name-ver.ver
     ('icet', '1.2.3',
      'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3'),
+    # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=name-ver.ver
+    ('icet', '1.2.3',
+     'https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-1.2.3'),
 
     # Common Repositories - bitbucket downloads
 
@@ -370,6 +378,7 @@ def test_url_parse_offset(name, noffset, ver, voffset, path):
     ('nextflow', '0.20.1', 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow'),
     # suffix queries
     ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'),
+    ('swiftsim', '0.3.0', 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'),
     ('sionlib', '1.7.1', 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'),
     # stem queries
     ('slepc', '3.6.2', 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz'),
diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py
index efc1a325e2..329772e17d 100644
--- a/lib/spack/spack/url.py
+++ b/lib/spack/spack/url.py
@@ -90,9 +90,14 @@ def find_list_url(url):
         (r'(.*github\.com/[^/]+/[^/]+)',
          lambda m: m.group(1) + '/releases'),
 
-        # GitLab
+        # GitLab API endpoint
+        # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2
+        (r'(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)',
+         lambda m: m.group(1) + '/' + m.group(2) + '/' + m.group(3) + '/tags'),
+
+        # GitLab non-API endpoint
         # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz
-        (r'(.*gitlab[^/]+/[^/]+/[^/]+)',
+        (r'(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)',
          lambda m: m.group(1) + '/tags'),
 
         # BitBucket
@@ -529,6 +534,9 @@ def parse_version_offset(path):
 
         # 9th Pass: Query strings
 
+        # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0
+        (r'\?sha=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix),
+
         # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
         (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix),
 
@@ -640,9 +648,13 @@ def parse_name_offset(path, v=None):
         # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz
         (r'github\.com/[^/]+/([^/]+)', path),
 
-        # GitLab: gitlab.*/repo/name/
+        # GitLab API endpoint: gitlab.*/api/v4/projects/NAMESPACE%2Fname/
+        # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0
+        (r'gitlab[^/]+/api/v4/projects/[^/]+%2F([^/]+)', path),
+
+        # GitLab non-API endpoint: gitlab.*/repo/name/
         # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
-        (r'gitlab[^/]+/[^/]+/([^/]+)', path),
+        (r'gitlab[^/]+/(?!api/v4/projects)[^/]+/([^/]+)', path),
 
         # Bitbucket: bitbucket.org/repo/name/
         # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2