From 05f8e080676f192d1530e952b57e2655d606ee40 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 29 Dec 2020 02:03:08 -0600 Subject: PythonPackage: add pypi attribute to infer homepage/url/list_url (#17587) --- lib/spack/docs/build_systems/pythonpackage.rst | 82 ++++++++-------------- lib/spack/spack/build_systems/python.py | 23 ++++++ lib/spack/spack/cmd/create.py | 42 +++++++++-- lib/spack/spack/package.py | 16 +++-- lib/spack/spack/url.py | 14 ++++ lib/spack/spack/util/web.py | 5 +- .../builtin/packages/py-matplotlib/package.py | 2 +- .../repos/builtin/packages/py-numpy/package.py | 2 +- .../repos/builtin/packages/py-scipy/package.py | 2 +- 9 files changed, 122 insertions(+), 66 deletions(-) diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst index de92595efa..34cbf4ac3f 100644 --- a/lib/spack/docs/build_systems/pythonpackage.rst +++ b/lib/spack/docs/build_systems/pythonpackage.rst @@ -134,9 +134,9 @@ The zip file will not contain a ``setup.py``, but it will contain a ``METADATA`` file which contains all the information you need to write a ``package.py`` build recipe. -^^^^^^^^^^^^^^^^^^^^^^^ -Finding Python packages -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^ +PyPI +^^^^ The vast majority of Python packages are hosted on PyPI - The Python Package Index. ``pip`` only supports packages hosted on PyPI, making @@ -148,6 +148,26 @@ if a newer version is available. The download page is usually at:: https://pypi.org/project/ + +Since PyPI is so common, the ``PythonPackage`` base class has a +``pypi`` attribute that can be set. Once set, ``pypi`` will be used +to define the ``homepage``, ``url``, and ``list_url``. For example, +the following: + +.. code-block:: python + + homepage = 'https://pypi.org/project/setuptools/' + url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip' + list_url = 'https://pypi.org/simple/setuptools/' + + +is equivalent to: + +.. code-block:: python + + pypi = 'setuptools/setuptools-49.2.0.zip' + + ^^^^^^^^^^^ Description ^^^^^^^^^^^ @@ -184,50 +204,11 @@ also get the homepage on the command-line by running: URL ^^^ -You may have noticed that Spack allows you to add multiple versions of -the same package without adding multiple versions of the download URL. -It does this by guessing what the version string in the URL is and -replacing this with the requested version. Obviously, if Spack cannot -guess the version correctly, or if non-version-related things change -in the URL, Spack cannot substitute the version properly. - -Once upon a time, PyPI offered nice, simple download URLs like:: - - https://pypi.python.org/packages/source/n/numpy/numpy-1.13.1.zip - - -As you can see, the version is 1.13.1. It probably isn't hard to guess -what URL to use to download version 1.12.0, and Spack was perfectly -capable of performing this calculation. - -However, PyPI switched to a new download URL format:: - - https://pypi.python.org/packages/c0/3a/40967d9f5675fbb097ffec170f59c2ba19fc96373e73ad47c2cae9a30aed/numpy-1.13.1.zip#md5=2c3c0f4edf720c3a7b525dacc825b9ae - - -and more recently:: - - https://files.pythonhosted.org/packages/b0/2b/497c2bb7c660b2606d4a96e2035e92554429e139c6c71cdff67af66b58d2/numpy-1.14.3.zip - - -As you can imagine, it is impossible for Spack to guess what URL to -use to download version 1.12.0 given this URL. There is a solution, -however. PyPI offers a new hidden interface for downloading -Python packages that does not include a hash in the URL:: - - https://pypi.io/packages/source/n/numpy/numpy-1.13.1.zip - - -This URL redirects to the https://files.pythonhosted.org URL. The general -syntax for this https://pypi.io URL is:: - - https://pypi.io/packages////-. - - -Please use the https://pypi.io URL instead of the https://pypi.python.org -URL. If both ``.tar.gz`` and ``.zip`` versions are available, ``.tar.gz`` -is preferred. If some releases offer both ``.tar.gz`` and ``.zip`` versions, -but some only offer ``.zip`` versions, use ``.zip``. +If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will +be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions +are available, ``.tar.gz`` is preferred. If some releases offer both +``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions, +use ``.zip``. Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip`` files on either PyPI or GitHub. If this is the case, you can still download @@ -237,10 +218,9 @@ and can be downloaded from:: https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl -Note that instead of ```` being ``source``, it is now ``py3`` since this -wheel will work for any generic version of Python 3. You may see Python-specific -or OS-specific URLs. Note that when you add a ``.whl`` URL, you should add -``expand=False`` to ensure that Spack doesn't try to extract the wheel: +You may see Python-specific or OS-specific URLs. Note that when you add a +``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't +try to extract the wheel: .. code-block:: python diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 87bc7e5135..d0d3402d70 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -72,6 +72,9 @@ class PythonPackage(PackageBase): def configure(self, spec, prefix): self.setup_py('configure') """ + #: Package name, version, and extension on PyPI + pypi = None + # Default phases phases = ['build', 'install'] @@ -88,6 +91,26 @@ class PythonPackage(PackageBase): py_namespace = None + @property + def homepage(self): + if self.pypi: + name = self.pypi.split('/')[0] + return 'https://pypi.org/project/' + name + '/' + + @property + def url(self): + if self.pypi: + return ( + 'https://files.pythonhosted.org/packages/source/' + + self.pypi[0] + '/' + self.pypi + ) + + @property + def list_url(self): + if self.pypi: + name = self.pypi.split('/')[0] + return 'https://pypi.org/simple/' + name + '/' + @property def import_modules(self): """Names of modules that the Python package provides. diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index a31b9537b6..6f35eefd81 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -117,7 +117,7 @@ class PackageTemplate(BundlePackageTemplate): make() make('install')""" - url_line = """ url = \"{url}\"""" + url_line = ' url = "{url}"' def __init__(self, name, url, versions): super(PackageTemplate, self).__init__(name, versions) @@ -270,14 +270,47 @@ class PythonPackageTemplate(PackageTemplate): args = [] return args""" - def __init__(self, name, *args, **kwargs): + def __init__(self, name, url, *args, **kwargs): # If the user provided `--name py-numpy`, don't rename it py-py-numpy if not name.startswith('py-'): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to py-{0}".format(name)) name = 'py-{0}'.format(name) - super(PythonPackageTemplate, self).__init__(name, *args, **kwargs) + # Simple PyPI URLs: + # https:///packages//// + # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip + + # PyPI URLs containing hash: + # https:///packages//// + # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512 + + # PyPI URLs for wheels are too complicated, ignore them for now + + match = re.search( + r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4, + url + ) + if match: + if len(match.group(2)) == 1: + # Simple PyPI URL + url = '/'.join(match.group(3, 4)) + else: + # PyPI URL containing hash + # Project name doesn't necessarily match download name, but it + # usually does, so this is the best we can do + project = parse_name(url) + url = '/'.join([project, match.group(4)]) + + self.url_line = ' pypi = "{url}"' + + super(PythonPackageTemplate, self).__init__(name, url, *args, **kwargs) class RPackageTemplate(PackageTemplate): @@ -545,7 +578,8 @@ class BuildSystemGuesser: ] # Peek inside the compressed file. - if stage.archive_file.endswith('.zip'): + if (stage.archive_file.endswith('.zip') or + '.zip#' in stage.archive_file): try: unzip = which('unzip') output = unzip('-lq', stage.archive_file, output=str) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1046b2d5eb..ea831add32 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -646,6 +646,15 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: index of patches by sha256 sum, built lazily _patches_by_hash = None + #: Package homepage where users can find more information about the package + homepage = None + + #: Default list URL (place to find available versions) + list_url = None + + #: Link depth to which list_url should be searched for new versions + list_depth = 0 + #: List of strings which contains GitHub usernames of package maintainers. #: Do not include @ here in order not to unnecessarily ping the users. maintainers = [] # type: List[str] @@ -683,13 +692,6 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): msg += " [package '{0.name}' defines both]" raise ValueError(msg.format(self)) - # Set a default list URL (place to find available versions) - if not hasattr(self, 'list_url'): - self.list_url = None - - if not hasattr(self, 'list_depth'): - self.list_depth = 0 - # init internal variables self._stage = None self._fetcher = None diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index aa40524ae8..399507c296 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -56,8 +56,12 @@ def find_list_urls(url): GitLab https://gitlab.\*///tags BitBucket https://bitbucket.org///downloads/?tab=tags CRAN https://\*.r-project.org/src/contrib/Archive/ + PyPI https://pypi.org/simple// ========= ======================================================= + Note: this function is called by `spack versions`, `spack checksum`, + and `spack create`, but not by `spack fetch` or `spack install`. + Parameters: url (str): The download URL for the package @@ -91,6 +95,16 @@ def find_list_urls(url): # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz (r'(.*\.r-project\.org/src/contrib)/([^_]+)', lambda m: m.group(1) + '/Archive/' + m.group(2)), + + # PyPI + # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl + (r'(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)', + lambda m: 'https://pypi.org/simple/' + m.group(1) + '/'), ] list_urls = set([os.path.dirname(url)]) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 47010fdbfd..0d4cf8abbb 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -554,7 +554,10 @@ def find_versions_of_archive( # .sha256 # .sig # However, SourceForge downloads still need to end in '/download'. - url_regex += r'(\/download)?$' + url_regex += r'(\/download)?' + # PyPI adds #sha256=... to the end of the URL + url_regex += '(#sha256=.*)?' + url_regex += '$' regexes.append(url_regex) diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py index b8589ffc59..435938d9b5 100644 --- a/var/spack/repos/builtin/packages/py-matplotlib/package.py +++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py @@ -11,7 +11,7 @@ class PyMatplotlib(PythonPackage): and interactive visualizations in Python.""" homepage = "https://matplotlib.org/" - url = "https://pypi.io/packages/source/m/matplotlib/matplotlib-3.3.2.tar.gz" + pypi = "matplotlib/matplotlib-3.3.2.tar.gz" maintainers = ['adamjstewart'] import_modules = [ diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index 106f40e76c..4778c64916 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -16,7 +16,7 @@ class PyNumpy(PythonPackage): number capabilities""" homepage = "https://numpy.org/" - url = "https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip" + pypi = "numpy/numpy-1.19.4.zip" git = "https://github.com/numpy/numpy.git" maintainers = ['adamjstewart'] diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py index c01d47e55e..900b9a8f0d 100644 --- a/var/spack/repos/builtin/packages/py-scipy/package.py +++ b/var/spack/repos/builtin/packages/py-scipy/package.py @@ -12,7 +12,7 @@ class PyScipy(PythonPackage): as routines for numerical integration and optimization.""" homepage = "https://www.scipy.org/" - url = "https://pypi.io/packages/source/s/scipy/scipy-1.5.4.tar.gz" + pypi = "scipy/scipy-1.5.4.tar.gz" git = "https://github.com/scipy/scipy.git" maintainers = ['adamjstewart'] -- cgit v1.2.3-60-g2f50