PythonPackage: add pypi attribute to infer homepage/url/list_url (#17587)

9 files changed, 122 insertions, 66 deletions

diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst
index de92595efa..34cbf4ac3f 100644
--- a/lib/spack/docs/build_systems/pythonpackage.rst
+++ b/lib/spack/docs/build_systems/pythonpackage.rst
@@ -134,9 +134,9 @@ The zip file will not contain a ``setup.py``, but it will contain a
 ``METADATA`` file which contains all the information you need to
 write a ``package.py`` build recipe.
 
-^^^^^^^^^^^^^^^^^^^^^^^
-Finding Python packages
-^^^^^^^^^^^^^^^^^^^^^^^
+^^^^
+PyPI
+^^^^
 
 The vast majority of Python packages are hosted on PyPI - The Python
 Package Index. ``pip`` only supports packages hosted on PyPI, making
@@ -148,6 +148,26 @@ if a newer version is available. The download page is usually at::
 
    https://pypi.org/project/<package-name>
 
+
+Since PyPI is so common, the ``PythonPackage`` base class has a
+``pypi`` attribute that can be set. Once set, ``pypi`` will be used
+to define the ``homepage``, ``url``, and ``list_url``. For example,
+the following:
+
+.. code-block:: python
+
+   homepage = 'https://pypi.org/project/setuptools/'
+   url      = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip'
+   list_url = 'https://pypi.org/simple/setuptools/'
+
+
+is equivalent to:
+
+.. code-block:: python
+
+   pypi = 'setuptools/setuptools-49.2.0.zip'
+
+
 ^^^^^^^^^^^
 Description
 ^^^^^^^^^^^
@@ -184,50 +204,11 @@ also get the homepage on the command-line by running:
 URL
 ^^^
 
-You may have noticed that Spack allows you to add multiple versions of
-the same package without adding multiple versions of the download URL.
-It does this by guessing what the version string in the URL is and
-replacing this with the requested version. Obviously, if Spack cannot
-guess the version correctly, or if non-version-related things change
-in the URL, Spack cannot substitute the version properly.
-
-Once upon a time, PyPI offered nice, simple download URLs like::
-
-   https://pypi.python.org/packages/source/n/numpy/numpy-1.13.1.zip
-
-
-As you can see, the version is 1.13.1. It probably isn't hard to guess
-what URL to use to download version 1.12.0, and Spack was perfectly
-capable of performing this calculation.
-
-However, PyPI switched to a new download URL format::
-
-   https://pypi.python.org/packages/c0/3a/40967d9f5675fbb097ffec170f59c2ba19fc96373e73ad47c2cae9a30aed/numpy-1.13.1.zip#md5=2c3c0f4edf720c3a7b525dacc825b9ae
-
-
-and more recently::
-
-   https://files.pythonhosted.org/packages/b0/2b/497c2bb7c660b2606d4a96e2035e92554429e139c6c71cdff67af66b58d2/numpy-1.14.3.zip
-
-
-As you can imagine, it is impossible for Spack to guess what URL to
-use to download version 1.12.0 given this URL. There is a solution,
-however. PyPI offers a new hidden interface for downloading
-Python packages that does not include a hash in the URL::
-
-   https://pypi.io/packages/source/n/numpy/numpy-1.13.1.zip
-
-
-This URL redirects to the https://files.pythonhosted.org URL. The general
-syntax for this https://pypi.io URL is::
-
-   https://pypi.io/packages/<type>/<first-letter-of-name>/<name>/<name>-<version>.<extension>
-
-
-Please use the https://pypi.io URL instead of the https://pypi.python.org
-URL. If both ``.tar.gz`` and ``.zip`` versions are available, ``.tar.gz``
-is preferred. If some releases offer both ``.tar.gz`` and ``.zip`` versions,
-but some only offer ``.zip`` versions, use ``.zip``.
+If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will
+be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions
+are available, ``.tar.gz`` is preferred. If some releases offer both
+``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions,
+use ``.zip``.
 
 Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip``
 files on either PyPI or GitHub. If this is the case, you can still download
@@ -237,10 +218,9 @@ and can be downloaded from::
    https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl
 
 
-Note that instead of ``<type>`` being ``source``, it is now ``py3`` since this
-wheel will work for any generic version of Python 3. You may see Python-specific
-or OS-specific URLs. Note that when you add a ``.whl`` URL, you should add
-``expand=False`` to ensure that Spack doesn't try to extract the wheel:
+You may see Python-specific or OS-specific URLs. Note that when you add a
+``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't
+try to extract the wheel:
 
 .. code-block:: python
 
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 87bc7e5135..d0d3402d70 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -72,6 +72,9 @@ class PythonPackage(PackageBase):
        def configure(self, spec, prefix):
            self.setup_py('configure')
     """
+    #: Package name, version, and extension on PyPI
+    pypi = None
+
     # Default phases
     phases = ['build', 'install']
 
@@ -89,6 +92,26 @@ class PythonPackage(PackageBase):
     py_namespace = None
 
     @property
+    def homepage(self):
+        if self.pypi:
+            name = self.pypi.split('/')[0]
+            return 'https://pypi.org/project/' + name + '/'
+
+    @property
+    def url(self):
+        if self.pypi:
+            return (
+                'https://files.pythonhosted.org/packages/source/'
+                + self.pypi[0] + '/' + self.pypi
+            )
+
+    @property
+    def list_url(self):
+        if self.pypi:
+            name = self.pypi.split('/')[0]
+            return 'https://pypi.org/simple/' + name + '/'
+
+    @property
     def import_modules(self):
         """Names of modules that the Python package provides.
 
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index a31b9537b6..6f35eefd81 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -117,7 +117,7 @@ class PackageTemplate(BundlePackageTemplate):
         make()
         make('install')"""
 
-    url_line = """    url      = \"{url}\""""
+    url_line = '    url      = "{url}"'
 
     def __init__(self, name, url, versions):
         super(PackageTemplate, self).__init__(name, versions)
@@ -270,14 +270,47 @@ class PythonPackageTemplate(PackageTemplate):
         args = []
         return args"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, *args, **kwargs):
         # If the user provided `--name py-numpy`, don't rename it py-py-numpy
         if not name.startswith('py-'):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to py-{0}".format(name))
             name = 'py-{0}'.format(name)
 
-        super(PythonPackageTemplate, self).__init__(name, *args, **kwargs)
+        # Simple PyPI URLs:
+        # https://<hostname>/packages/<type>/<first character of project>/<project>/<download file>
+        # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip
+
+        # PyPI URLs containing hash:
+        # https://<hostname>/packages/<two character hash>/<two character hash>/<longer hash>/<download file>
+        # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip
+        # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip
+        # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512
+
+        # PyPI URLs for wheels are too complicated, ignore them for now
+
+        match = re.search(
+            r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4,
+            url
+        )
+        if match:
+            if len(match.group(2)) == 1:
+                # Simple PyPI URL
+                url = '/'.join(match.group(3, 4))
+            else:
+                # PyPI URL containing hash
+                # Project name doesn't necessarily match download name, but it
+                # usually does, so this is the best we can do
+                project = parse_name(url)
+                url = '/'.join([project, match.group(4)])
+
+            self.url_line = '    pypi     = "{url}"'
+
+        super(PythonPackageTemplate, self).__init__(name, url, *args, **kwargs)
 
 
 class RPackageTemplate(PackageTemplate):
@@ -545,7 +578,8 @@ class BuildSystemGuesser:
         ]
 
         # Peek inside the compressed file.
-        if stage.archive_file.endswith('.zip'):
+        if (stage.archive_file.endswith('.zip') or
+                '.zip#' in stage.archive_file):
             try:
                 unzip  = which('unzip')
                 output = unzip('-lq', stage.archive_file, output=str)
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 1046b2d5eb..ea831add32 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -646,6 +646,15 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
     #: index of patches by sha256 sum, built lazily
     _patches_by_hash = None
 
+    #: Package homepage where users can find more information about the package
+    homepage = None
+
+    #: Default list URL (place to find available versions)
+    list_url = None
+
+    #: Link depth to which list_url should be searched for new versions
+    list_depth = 0
+
     #: List of strings which contains GitHub usernames of package maintainers.
     #: Do not include @ here in order not to unnecessarily ping the users.
     maintainers = []  # type: List[str]
@@ -683,13 +692,6 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
             msg += " [package '{0.name}' defines both]"
             raise ValueError(msg.format(self))
 
-        # Set a default list URL (place to find available versions)
-        if not hasattr(self, 'list_url'):
-            self.list_url = None
-
-        if not hasattr(self, 'list_depth'):
-            self.list_depth = 0
-
         # init internal variables
         self._stage = None
         self._fetcher = None
diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py
index aa40524ae8..399507c296 100644
--- a/lib/spack/spack/url.py
+++ b/lib/spack/spack/url.py
@@ -56,8 +56,12 @@ def find_list_urls(url):
     GitLab     https://gitlab.\*/<repo>/<name>/tags
     BitBucket  https://bitbucket.org/<repo>/<name>/downloads/?tab=tags
     CRAN       https://\*.r-project.org/src/contrib/Archive/<name>
+    PyPI       https://pypi.org/simple/<name>/
     =========  =======================================================
 
+    Note: this function is called by `spack versions`, `spack checksum`,
+    and `spack create`, but not by `spack fetch` or `spack install`.
+
     Parameters:
         url (str): The download URL for the package
 
@@ -91,6 +95,16 @@ def find_list_urls(url):
         # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz
         (r'(.*\.r-project\.org/src/contrib)/([^_]+)',
          lambda m: m.group(1) + '/Archive/' + m.group(2)),
+
+        # PyPI
+        # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip
+        # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl
+        (r'(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)',
+         lambda m: 'https://pypi.org/simple/' + m.group(1) + '/'),
     ]
 
     list_urls = set([os.path.dirname(url)])
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index 47010fdbfd..0d4cf8abbb 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -554,7 +554,10 @@ def find_versions_of_archive(
         #   .sha256
         #   .sig
         # However, SourceForge downloads still need to end in '/download'.
-        url_regex += r'(\/download)?$'
+        url_regex += r'(\/download)?'
+        # PyPI adds #sha256=... to the end of the URL
+        url_regex += '(#sha256=.*)?'
+        url_regex += '$'
 
         regexes.append(url_regex)
 
diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py
index b8589ffc59..435938d9b5 100644
--- a/var/spack/repos/builtin/packages/py-matplotlib/package.py
+++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py
@@ -11,7 +11,7 @@ class PyMatplotlib(PythonPackage):
     and interactive visualizations in Python."""
 
     homepage = "https://matplotlib.org/"
-    url      = "https://pypi.io/packages/source/m/matplotlib/matplotlib-3.3.2.tar.gz"
+    pypi     = "matplotlib/matplotlib-3.3.2.tar.gz"
 
     maintainers = ['adamjstewart']
     import_modules = [
diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py
index 106f40e76c..4778c64916 100644
--- a/var/spack/repos/builtin/packages/py-numpy/package.py
+++ b/var/spack/repos/builtin/packages/py-numpy/package.py
@@ -16,7 +16,7 @@ class PyNumpy(PythonPackage):
     number capabilities"""
 
     homepage = "https://numpy.org/"
-    url      = "https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip"
+    pypi     = "numpy/numpy-1.19.4.zip"
     git      = "https://github.com/numpy/numpy.git"
 
     maintainers = ['adamjstewart']
diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py
index c01d47e55e..900b9a8f0d 100644
--- a/var/spack/repos/builtin/packages/py-scipy/package.py
+++ b/var/spack/repos/builtin/packages/py-scipy/package.py
@@ -12,7 +12,7 @@ class PyScipy(PythonPackage):
     as routines for numerical integration and optimization."""
 
     homepage = "https://www.scipy.org/"
-    url      = "https://pypi.io/packages/source/s/scipy/scipy-1.5.4.tar.gz"
+    pypi     = "scipy/scipy-1.5.4.tar.gz"
     git      = "https://github.com/scipy/scipy.git"
 
     maintainers = ['adamjstewart']