From 3540f8200a536434946492d60b5e56b302112305 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 14 Jan 2022 12:37:57 -0600 Subject: PythonPackage: install packages with pip (#27798) * Use pip to bootstrap pip * Bootstrap wheel from source * Update PythonPackage to install using pip * Update several packages * Add wheel as base class dep * Build phase no longer exists * Add py-poetry package, fix py-flit-core bootstrapping * Fix isort build * Clean up many more packages * Remove unused import * Fix unit tests * Don't directly run setup.py * Typo fix * Remove unused imports * Fix issues caught by CI * Remove custom setup.py file handling * Use PythonPackage for installing wheels * Remove custom phases in PythonPackages * Remove _args methods * Remove unused import * Fix various packages * Try to test Python packages directly in CI * Actually run the pipeline * Fix more packages * Fix mappings, fix packages * Fix dep version * Work around bug in concretizer * Various concretization fixes * Fix gitlab yaml, packages * Fix typo in gitlab yaml * Skip more packages that fail to concretize * Fix? jupyter ecosystem concretization issues * Solve Jupyter concretization issues * Prevent duplicate entries in PYTHONPATH * Skip fenics-dolfinx * Build fewer Python packages * Fix missing npm dep * Specify image * More package fixes * Add backends for every from-source package * Fix version arg * Remove GitLab CI stuff, add py-installer package * Remove test deps, re-add install_options * Function declaration syntax fix * More build fixes * Update spack create template * Update PythonPackage documentation * Fix documentation build * Fix unit tests * Remove pip flag added only in newer pip * flux: add explicit dependency on jsonschema * Update packages that have been added since this was branched off of develop * Move Python 2 deprecation to a separate PR * py-neurolab: add build dep on py-setuptools * Use wheels for pip/wheel * Allow use of pre-installed pip for external Python * pip -> python -m pip * Use python -m pip for all packages * Fix py-wrapt * Add both platlib and purelib to PYTHONPATH * py-pyyaml: setuptools is needed for all versions * py-pyyaml: link flags aren't needed * Appease spack audit packages * Some build backend is required for all versions, distutils -> setuptools * Correctly handle different setup.py filename * Use wheels for py-tomli to avoid circular dep on py-flit-core * Fix busco installation procedure * Clarify things in spack create template * Test other Python build backends * Undo changes to busco * Various fixes * Don't test other backends --- lib/spack/docs/build_systems/pythonpackage.rst | 777 +++++++++---------------- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/build_systems/python.py | 248 ++------ lib/spack/spack/cmd/create.py | 71 ++- lib/spack/spack/stage.py | 8 +- lib/spack/spack/test/cmd/create.py | 2 +- 6 files changed, 391 insertions(+), 719 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst index 365c5d7bce..6fdd3ee9b0 100644 --- a/lib/spack/docs/build_systems/pythonpackage.rst +++ b/lib/spack/docs/build_systems/pythonpackage.rst @@ -9,216 +9,80 @@ PythonPackage ------------- -Python packages and modules have their own special build system. +Python packages and modules have their own special build system. This +documentation covers everything you'll need to know in order to write +a Spack build recipe for a Python library. -^^^^^^ -Phases -^^^^^^ - -The ``PythonPackage`` base class provides the following phases that -can be overridden: - -* ``build`` -* ``build_py`` -* ``build_ext`` -* ``build_clib`` -* ``build_scripts`` -* ``install`` -* ``install_lib`` -* ``install_headers`` -* ``install_scripts`` -* ``install_data`` - -These are all standard ``setup.py`` commands and can be found by running: - -.. code-block:: console - - $ python setup.py --help-commands - - -By default, only the ``build`` and ``install`` phases are run: - -#. ``build`` - build everything needed to install -#. ``install`` - install everything from build directory - -If for whatever reason you need to run more phases, simply modify your -``phases`` list like so: - -.. code-block:: python - - phases = ['build_ext', 'install'] - - -Each phase provides a function ```` that runs: - -.. code-block:: console - - $ python -s setup.py --no-user-cfg - - -Each phase also has a ```` function that can pass arguments to -this call. All of these functions are empty except for the ``install_args`` -function, which passes ``--prefix=/path/to/installation/prefix``. There is -also some additional logic specific to setuptools and eggs. - -If you need to run a phase that is not a standard ``setup.py`` command, -you'll need to define a function for it like so: - -.. code-block:: python - - phases = ['configure', 'build', 'install'] - - def configure(self, spec, prefix): - self.setup_py('configure') - - -^^^^^^ -Wheels -^^^^^^ - -Some Python packages are closed-source and distributed as wheels. -Instead of using the ``PythonPackage`` base class, you should extend -the ``Package`` base class and implement the following custom installation -procedure: - -.. code-block:: python - - def install(self, spec, prefix): - pip = which('pip') - pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) - - -This will require a dependency on pip, as mentioned below. - -^^^^^^^^^^^^^^^ -Important files -^^^^^^^^^^^^^^^ - -Python packages can be identified by the presence of a ``setup.py`` file. -This file is used by package managers like ``pip`` to determine a -package's dependencies and the version of dependencies required, so if -the ``setup.py`` file is not accurate, the package will not build properly. -For this reason, the ``setup.py`` file should be fairly reliable. If the -documentation and ``setup.py`` disagree on something, the ``setup.py`` -file should be considered to be the truth. As dependencies are added or -removed, the documentation is much more likely to become outdated than -the ``setup.py``. - -The Python ecosystem has evolved significantly over the years. Before -setuptools became popular, most packages listed their dependencies in a -``requirements.txt`` file. Once setuptools took over, these dependencies -were listed directly in the ``setup.py``. Newer PEPs introduced additional -files, like ``setup.cfg`` and ``pyproject.toml``. You should look out for -all of these files, as they may all contain important information about -package dependencies. - -Some Python packages are closed-source and are distributed as Python -wheels. For example, ``py-azureml-sdk`` downloads a ``.whl`` file. This -file is simply a zip file, and can be extracted using: - -.. code-block:: console - - $ unzip *.whl - - -The zip file will not contain a ``setup.py``, but it will contain a -``METADATA`` file which contains all the information you need to -write a ``package.py`` build recipe. - -.. _pypi: - -^^^^ -PyPI -^^^^ +^^^^^^^^^^^ +Terminology +^^^^^^^^^^^ -The vast majority of Python packages are hosted on PyPI (The Python -Package Index), which is :ref:`preferred over GitHub ` -for downloading packages. ``pip`` only supports packages hosted on PyPI, making -it the only option for developers who want a simple installation. -Search for "PyPI " to find the download page. Note that -some pages are versioned, and the first result may not be the newest -version. Click on the "Latest Version" button to the top right to see -if a newer version is available. The download page is usually at:: +In the Python ecosystem, there are a number of terms that are +important to understand. - https://pypi.org/project/ +**PyPI** + The `Python Package Index `_, where most Python + libraries are hosted. +**sdist** + Source distributions, distributed as tarballs (.tar.gz) and zip + files (.zip). Contain the source code of the package. -Since PyPI is so common, the ``PythonPackage`` base class has a -``pypi`` attribute that can be set. Once set, ``pypi`` will be used -to define the ``homepage``, ``url``, and ``list_url``. For example, -the following: +**bdist** + Built distributions, distributed as wheels (.whl). Contain the + pre-built library. -.. code-block:: python - - homepage = 'https://pypi.org/project/setuptools/' - url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip' - list_url = 'https://pypi.org/simple/setuptools/' - - -is equivalent to: - -.. code-block:: python +**wheel** + A binary distribution format common in the Python ecosystem. This + file is actually just a zip file containing specific metadata and + code. See the + `documentation `_ + for more details. - pypi = 'setuptools/setuptools-49.2.0.zip' +**build frontend** + Command-line tools used to build and install wheels. Examples + include `pip `_, + `build `_, and + `installer `_. +**build backend** + Libraries used to define how to build a wheel. Examples + include `setuptools `__, + `flit `_, and + `poetry `_. ^^^^^^^^^^^ -Description +Downloading ^^^^^^^^^^^ -The top of the PyPI downloads page contains a description of the -package. The first line is usually a short description, while there -may be a several line "Project Description" that follows. Choose whichever -is more useful. You can also get these descriptions on the command-line -using: - -.. code-block:: console +The first step in packaging a Python library is to figure out where +to download it from. The vast majority of Python packages are hosted +on `PyPI `_, which is +:ref:`preferred over GitHub ` for downloading +packages. Search for the package name on PyPI to find the project +page. The project page is usually located at:: - $ python setup.py --description - $ python setup.py --long-description + https://pypi.org/project/ +On the project page, there is a "Download files" tab containing +download URLs. Whenever possible, we prefer to build Spack packages +from source. If PyPI only has wheels, check to see if the project is +hosted on GitHub and see if GitHub has source distributions. The +project page usually has a "Homepage" and/or "Source code" link for +this. If the project is closed-source, it may only have wheels +available. For example, ``py-azureml-sdk`` is closed-source and can +be downloaded from:: -^^^^^^^^ -Homepage -^^^^^^^^ + https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl -Package developers use ``setup.py`` to upload new versions to PyPI. -The ``setup`` method often passes metadata like ``homepage`` to PyPI. -This metadata is displayed on the left side of the download page. -Search for the text "Homepage" under "Project links" to find it. You -should use this page instead of the PyPI page if they differ. You can -also get the homepage on the command-line by running: +Once you've found a URL to download the package from, run: .. code-block:: console - $ python setup.py --url - - -^^^ -URL -^^^ - -If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will -be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions -are available, ``.tar.gz`` is preferred. If some releases offer both -``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions, -use ``.zip``. - -Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip`` -files on either PyPI or GitHub. If this is the case, you can still download -and install a Python wheel. For example, ``py-azureml-sdk`` is closed source -and can be downloaded from:: - - https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl - + $ spack create -You may see Python-specific or OS-specific URLs. Note that when you add a -``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't -try to extract the wheel: -.. code-block:: python - - version('1.11.0', sha256='d8c9d24ea90457214d798b0d922489863dad518adde3638e08ef62de28fb183a', expand=False) +to create a new package template. .. _pypi-vs-github: @@ -226,11 +90,13 @@ try to extract the wheel: PyPI vs. GitHub """"""""""""""" -Many packages are hosted on PyPI, but are developed on GitHub or another -version control systems. The tarball can be downloaded from either -location, but PyPI is preferred for the following reasons: +Many packages are hosted on PyPI, but are developed on GitHub or +another version control system hosting service. The source code can +be downloaded from either location, but PyPI is preferred for the +following reasons: -#. PyPI contains the bare minimum number of files needed to install the package. +#. PyPI contains the bare minimum number of files needed to install + the package. You may notice that the tarball you download from PyPI does not have the same checksum as the tarball you download from GitHub. @@ -267,300 +133,217 @@ location, but PyPI is preferred for the following reasons: PyPI is nice because it makes it physically impossible to re-release the same version of a package with a different checksum. -Use the :ref:`pypi attribute ` to facilitate construction of PyPI package -references. - -^^^^^^^^^^^^^^^^^^^^^^^^^ -Build system dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are a few dependencies common to the ``PythonPackage`` build system. +The only reason to use GitHub instead of PyPI is if PyPI only has +wheels or if the PyPI sdist is missing a file needed to build the +package. If this is the case, please add a comment above the ``url`` +explaining this. -"""""" -Python -"""""" +^^^^ +PyPI +^^^^ -Obviously, every ``PythonPackage`` needs Python at build-time to run -``python setup.py build && python setup.py install``. Python is also -needed at run-time if you want to import the module. Due to backwards -incompatible changes between Python 2 and 3, it is very important to -specify which versions of Python are supported. If the documentation -mentions that Python 3 is required, this can be specified as: +Since PyPI is so commonly used to host Python libraries, the +``PythonPackage`` base class has a ``pypi`` attribute that can be +set. Once set, ``pypi`` will be used to define the ``homepage``, +``url``, and ``list_url``. For example, the following: .. code-block:: python - depends_on('python@3:', type=('build', 'run')) + homepage = 'https://pypi.org/project/setuptools/' + url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip' + list_url = 'https://pypi.org/simple/setuptools/' -If Python 2 is required, this would look like: +is equivalent to: .. code-block:: python - depends_on('python@:2', type=('build', 'run')) - - -If Python 2.7 is the only version that works, you can use: - -.. code-block:: python + pypi = 'setuptools/setuptools-49.2.0.zip' - depends_on('python@2.7:2.8', type=('build', 'run')) +If a package has a different homepage listed on PyPI, you can +override it by setting your own ``homepage``. -The documentation may not always specify supported Python versions. -Another place to check is in the ``setup.py`` or ``setup.cfg`` file. -Look for a line containing ``python_requires``. An example from -`py-numpy `_ -looks like: +^^^^^^^^^^^ +Description +^^^^^^^^^^^ -.. code-block:: python +The top of the PyPI project page contains a short description of the +package. The "Project description" tab may also contain a longer +description of the package. Either of these can be used to populate +the package docstring. - python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*' +^^^^^^^^^^^^^ +Build backend +^^^^^^^^^^^^^ +Once you've determined the basic metadata for a package, the next +step is to determine the build backend. ``PythonPackage`` uses +`pip `_ to install the package, but pip +requires a backend to actually build the package. -You may also find a version check at the top of the ``setup.py``: +To determine the build backend, look for a ``pyproject.toml`` file. +If there is no ``pyproject.toml`` file and only a ``setup.py`` or +``setup.cfg`` file, you can assume that the project uses +:ref:`setuptools`. If there is a ``pyproject.toml`` file, see if it +contains a ``[build-system]`` section. For example: -.. code-block:: python +.. code-block:: toml - if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 4): - raise RuntimeError("Python version 2.7 or >= 3.4 required.") + [build-system] + requires = [ + "setuptools>=42", + "wheel", + ] + build-backend = "setuptools.build_meta" -This can be converted to Spack's spec notation like so: +This section does two things: the ``requires`` key lists build +dependencies of the project, and the ``build-backend`` key defines +the build backend. All of these build dependencies should be added as +dependencies to your package: .. code-block:: python - depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) + depends_on('py-setuptools@42:', type='build') -If you are writing a recipe for a package that only distributes -wheels, look for a section in the ``METADATA`` file that looks like:: - - Requires-Python: >=3.5,<4 +Note that ``py-wheel`` is already listed as a build dependency in the +``PythonPackage`` base class, so you don't need to add it unless you +need to specify a specific version requirement or change the +dependency type. +See `PEP 517 `_ and +`PEP 518 `_ for more +information on the design of ``pyproject.toml``. -This would be translated to: +Depending on which build backend a project uses, there are various +places that run-time dependencies can be listed. -.. code-block:: python +""""""""" +distutils +""""""""" - extends('python') - depends_on('python@3.5:3', type=('build', 'run')) +Before the introduction of setuptools and other build backends, +Python packages had to rely on the built-in distutils library. +Distutils is missing many of the features that setuptools and other +build backends offer, and users are encouraged to use setuptools +instead. In fact, distutils was deprecated in Python 3.10 and will be +removed in Python 3.12. Because of this, pip actually replaces all +imports of distutils with setuptools. If a package uses distutils, +you should instead add a build dependency on setuptools. Check for a +``requirements.txt`` file that may list dependencies of the project. - -Many ``setup.py`` or ``setup.cfg`` files also contain information like:: - - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 3.4 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - - -This is a list of versions of Python that the developer likely tests. -However, you should not use this to restrict the versions of Python -the package uses unless one of the two former methods (``python_requires`` -or ``sys.version_info``) is used. There is no logic in setuptools -that prevents the package from building for Python versions not in -this list, and often new releases like Python 3.7 or 3.8 work just fine. +.. _setuptools: """""""""" setuptools """""""""" -Originally, the Python language had a single build system called -distutils, which is built into Python. Distutils provided a common -framework for package authors to describe their project and how it -should be built. However, distutils was not without limitations. -Most notably, there was no way to list a project's dependencies -with distutils. Along came setuptools, a non-builtin build system -designed to overcome the limitations of distutils. Both projects -use a similar API, making the transition easy while adding much -needed functionality. Today, setuptools is used in around 90% of -the Python packages in Spack. - -Since setuptools isn't built-in to Python, you need to add it as a -dependency. To determine whether or not a package uses setuptools, -search the file for an import statement like: +If the ``pyproject.toml`` lists ``setuptools.build_meta`` as a +``build-backend``, or if the package has a ``setup.py`` that imports +``setuptools``, or if the package has a ``setup.cfg`` file, then it +uses setuptools to build. Setuptools is a replacement for the +distutils library, and has almost the exact same API. Dependencies +can be listed in the ``setup.py`` or ``setup.cfg`` file. Look for the +following arguments: -.. code-block:: python - - import setuptools +* ``python_requires`` + This specifies the version of Python that is required. -or: +* ``setup_requires`` -.. code-block:: python + These packages are usually only needed at build-time, so you can + add them with ``type='build'``. - from setuptools import setup +* ``install_requires`` + These packages are required for building and installation. You can + add them with ``type=('build', 'run')``. -Some packages are designed to work with both setuptools and distutils, -so you may find something like: +* ``extras_require`` -.. code-block:: python + These packages are optional dependencies that enable additional + functionality. You should add a variant that optionally adds these + dependencies. This variant should be False by default. - try: - from setuptools import setup - except ImportError: - from distutils.core import setup +* ``tests_require`` + These are packages that are required to run the unit tests for the + package. These dependencies can be specified using the + ``type='test'`` dependency type. However, the PyPI tarballs rarely + contain unit tests, so there is usually no reason to add these. -This uses setuptools if available, and falls back to distutils if not. -In this case, you would still want to add a setuptools dependency, as -it offers us more control over the installation. +See https://setuptools.pypa.io/en/latest/userguide/dependency_management.html +for more information on how setuptools handles dependency management. +See `PEP 440 `_ +for documentation on version specifiers in setuptools. -Unless specified otherwise, setuptools is usually a build-only dependency. -That is, it is needed to install the software, but is not needed at -run-time. This can be specified as: +"""" +flit +"""" -.. code-block:: python +There are actually two possible ``build-backend`` for flit, ``flit`` +and ``flit_core``. If you see these in the ``pyproject.toml``, add a +build dependency to your package. With flit, all dependencies are +listed directly in the ``pyproject.toml`` file. Older versions of +flit used to store this info in a ``flit.ini`` file, so check for +this too. - depends_on('py-setuptools', type='build') +Either of these files may contain keys like: +* ``requires-python`` -""" -pip -""" + This specifies the version of Python that is required -Packages distributed as Python wheels will require an extra dependency -on pip: +* ``dependencies`` or ``requires`` -.. code-block:: python + These packages are required for building and installation. You can + add them with ``type=('build', 'run')``. - depends_on('py-pip', type='build') +* ``project.optional-dependencies`` or ``requires-extra`` + This section includes keys with lists of optional dependencies + needed to enable those features. You should add a variant that + optionally adds these dependencies. This variant should be False + by default. -We will use pip to install the actual wheel. +See https://flit.readthedocs.io/en/latest/pyproject_toml.html for +more information. """""" -cython +poetry """""" -Compared to compiled languages, interpreted languages like Python can -be quite a bit slower. To work around this, some Python developers -rewrite computationally demanding sections of code in C, a process -referred to as "cythonizing". In order to build these package, you -need to add a build dependency on cython: - -.. code-block:: python - - depends_on('py-cython', type='build') - - -Look for references to "cython" in the ``setup.py`` to determine -whether or not this is necessary. Cython may be optional, but -even then you should list it as a required dependency. Spack is -designed to compile software, and is meant for HPC facilities -where speed is crucial. There is no reason why someone would not -want an optimized version of a library instead of the pure-Python -version. - -Note that some release tarballs come pre-cythonized, and cython is -not needed as a dependency. However, this is becoming less common -as Python continues to evolve and developers discover that cythonized -sources are no longer compatible with newer versions of Python and -need to be re-cythonized. - -^^^^^^^^^^^^^^^^^^^ -Python dependencies -^^^^^^^^^^^^^^^^^^^ - -When you install a package with ``pip``, it reads the ``setup.py`` -file in order to determine the dependencies of the package. -If the dependencies are not yet installed, ``pip`` downloads them -and installs them for you. This may sound convenient, but Spack -cannot rely on this behavior for two reasons: - -#. Spack needs to be able to install packages on air-gapped networks. - - If there is no internet connection, ``pip`` can't download the - package dependencies. By explicitly listing every dependency in - the ``package.py``, Spack knows what to download ahead of time. - -#. Duplicate installations of the same dependency may occur. - - Spack supports *activation* of Python extensions, which involves - symlinking the package installation prefix to the Python installation - prefix. If your package is missing a dependency, that dependency - will be installed to the installation directory of the same package. - If you try to activate the package + dependency, it may cause a - problem if that package has already been activated. - -For these reasons, you must always explicitly list all dependencies. -Although the documentation may list the package's dependencies, -often the developers assume people will use ``pip`` and won't have to -worry about it. Always check the ``setup.py`` to find the true -dependencies. - -If the package relies on ``distutils``, it may not explicitly list its -dependencies. Check for statements like: - -.. code-block:: python - - try: - import numpy - except ImportError: - raise ImportError("numpy must be installed prior to installation") - - -Obviously, this means that ``py-numpy`` is a dependency. - -If the package uses ``setuptools``, check for the following clues: - -* ``python_requires`` - - As mentioned above, this specifies which versions of Python are - required. - -* ``setup_requires`` - - These packages are usually only needed at build-time, so you can - add them with ``type='build'``. - -* ``install_requires`` - - These packages are required for building and installation. You can - add them with ``type=('build', 'run')``. - -* ``extra_requires`` - - These packages are optional dependencies that enable additional - functionality. You should add a variant that optionally adds these - dependencies. This variant should be False by default. - -* ``test_requires`` +Like flit, poetry also has two possible ``build-backend``, ``poetry`` +and ``poetry_core``. If you see these in the ``pyproject.toml``, add +a build dependency to your package. With poetry, all dependencies are +listed directly in the ``pyproject.toml`` file. Dependencies are +listed in a ``[tool.poetry.dependencies]`` section, and use a +`custom syntax `_ +for specifying the version requirements. Note that ``~=`` works +differently in poetry than in setuptools and flit for versions that +start with a zero. - These are packages that are required to run the unit tests for the - package. These dependencies can be specified using the - ``type='test'`` dependency type. However, the PyPI tarballs rarely - contain unit tests, so there is usually no reason to add these. +"""""" +wheels +"""""" -In the root directory of the package, you may notice a -``requirements.txt`` file. It may look like this file contains a list -of all of the package's dependencies. Don't be fooled. This file is -used by tools like Travis to install the pre-requisites for the -package... and a whole bunch of other things. It often contains -dependencies only needed for unit tests, like: +Some Python packages are closed-source and are distributed as Python +wheels. For example, ``py-azureml-sdk`` downloads a ``.whl`` file. This +file is simply a zip file, and can be extracted using: -* mock -* nose -* pytest +.. code-block:: console -It can also contain dependencies for building the documentation, like -sphinx. If you can't find any information about the package's -dependencies, you can take a look in ``requirements.txt``, but be sure -not to add test or documentation dependencies. + $ unzip *.whl -Newer PEPs have added alternative ways to specify a package's dependencies. -If you don't see any dependencies listed in the ``setup.py``, look for a -``setup.cfg`` or ``pyproject.toml``. These files can be used to store the -same ``install_requires`` information that ``setup.py`` used to use. -If you are write a recipe for a package that only distributes wheels, -check the ``METADATA`` file for lines like:: +The zip file will not contain a ``setup.py``, but it will contain a +``METADATA`` file which contains all the information you need to +write a ``package.py`` build recipe. Check for lines like:: + Requires-Python: >=3.5,<4 Requires-Dist: azureml-core (~=1.11.0) Requires-Dist: azureml-dataset-runtime[fuse] (~=1.11.0) Requires-Dist: azureml-train (~=1.11.0) @@ -572,62 +355,58 @@ check the ``METADATA`` file for lines like:: Requires-Dist: azureml-train-automl (~=1.11.0); extra == 'automl' -Lines that use ``Requires-Dist`` are similar to ``install_requires``. -Lines that use ``Provides-Extra`` are similar to ``extra_requires``, -and you can add a variant for those dependencies. The ``~=1.11.0`` -syntax is equivalent to ``1.11.0:1.11``. - -"""""""""" -setuptools -"""""""""" - -Setuptools is a bit of a special case. If a package requires setuptools -at run-time, how do they express this? They could add it to -``install_requires``, but setuptools is imported long before this and is -needed to read this line. And since you can't install the package -without setuptools, the developers assume that setuptools will already -be there, so they never mention when it is required. We don't want to -add run-time dependencies if they aren't needed, so you need to -determine whether or not setuptools is needed. Grep the installation -directory for any files containing a reference to ``setuptools`` or -``pkg_resources``. Both modules come from ``py-setuptools``. -``pkg_resources`` is particularly common in scripts found in -``prefix/bin``. +``Requires-Python`` is equivalent to ``python_requires`` and +``Requires-Dist`` is equivalent to ``install_requires``. +``Provides-Extra`` is used to name optional features (variants) and +a ``Requires-Dist`` with ``extra == 'foo'`` will list any +dependencies needed for that feature. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Passing arguments to setup.py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The default build and install phases should be sufficient to install -most packages. However, you may want to pass additional flags to -either phase. +The default install phase should be sufficient to install most +packages. However, the installation instructions for a package may +suggest passing certain flags to the ``setup.py`` call. The +``PythonPackage`` class has two techniques for doing this. -You can view the available options for a particular phase with: +"""""""""""""" +Global options +"""""""""""""" -.. code-block:: console +These flags are added directly after ``setup.py`` when pip runs +``python setup.py install``. For example, the ``py-pyyaml`` package +has an optional dependency on ``libyaml`` that can be enabled like so: - $ python setup.py --help +.. code-block:: python + def global_options(self, spec, prefix): + options = [] + if '+libyaml' in spec: + options.append('--with-libyaml') + else: + options.append('--without-libyaml') + return options -Each phase provides a ```` function that can be used to -pass arguments to that phase. For example, -`py-numpy `_ -adds: -.. code-block:: python +""""""""""""""" +Install options +""""""""""""""" - def build_args(self, spec, prefix): - args = [] +These flags are added directly after ``install`` when pip runs +``python setup.py install``. For example, the ``py-pyyaml`` package +allows you to specify the directories to search for ``libyaml``: - # From NumPy 1.10.0 on it's possible to do a parallel build. - if self.version >= Version('1.10.0'): - # But Parallel build in Python 3.5+ is broken. See: - # https://github.com/spack/spack/issues/7927 - # https://github.com/scipy/scipy/issues/7112 - if spec['python'].version < Version('3.5'): - args = ['-j', str(make_jobs)] +.. code-block:: python - return args + def install_options(self, spec, prefix): + options = [] + if '+libyaml' in spec: + options.extend([ + spec['libyaml'].libs.search_flags, + spec['libyaml'].headers.include_flags, + ]) + return options ^^^^^^^ @@ -669,9 +448,9 @@ a "package" is a directory containing files like: whereas a "module" is a single Python file. -The ``PythonPackage`` base class automatically detects these module -names for you. If, for whatever reason, the module names detected -are wrong, you can provide the names yourself by overriding +The ``PythonPackage`` base class automatically detects these package +and module names for you. If, for whatever reason, the module names +detected are wrong, you can provide the names yourself by overriding ``import_modules`` like so: .. code-block:: python @@ -692,10 +471,8 @@ This can be expressed like so: @property def import_modules(self): modules = ['yaml'] - if '+libyaml' in self.spec: modules.append('yaml.cyaml') - return modules @@ -713,8 +490,8 @@ Unit tests """""""""" The package may have its own unit or regression tests. Spack can -run these tests during the installation by adding phase-appropriate -test methods. +run these tests during the installation by adding test methods after +installation. For example, ``py-numpy`` adds the following as a check to run after the ``install`` phase: @@ -740,34 +517,14 @@ when testing is enabled during the installation (i.e., ``spack install Setup file in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In order to be compatible with package managers like ``pip``, the package -is required to place its ``setup.py`` in the root of the tarball. However, -not every Python package cares about ``pip`` or PyPI. If you are installing -a package that is not hosted on PyPI, you may find that it places its -``setup.py`` in a sub-directory. To handle this, add the directory containing -``setup.py`` to the package like so: - -.. code-block:: python - - build_directory = 'source' - - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Alternate names for setup.py -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As previously mentioned, packages need to call their setup script ``setup.py`` -in order to be compatible with package managers like ``pip``. However, some -packages like -`py-meep `_ and -`py-adios `_ -come with multiple setup scripts, one for a serial build and another for a -parallel build. You can override the default name to use like so: +Many C/C++ libraries provide optional Python bindings in a +subdirectory. To tell pip which directory to build from, you can +override the ``build_directory`` attribute. For example, if a package +provides Python bindings in a ``python`` directory, you can use: .. code-block:: python - def setup_file(self): - return 'setup-mpi.py' if '+mpi' in self.spec else 'setup.py' + build_directory = 'python' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -781,10 +538,14 @@ on Python are not necessarily ``PythonPackage``'s. Choosing a build system """"""""""""""""""""""" -First of all, you need to select a build system. ``spack create`` usually -does this for you, but if for whatever reason you need to do this manually, -choose ``PythonPackage`` if and only if the package contains a ``setup.py`` -file. +First of all, you need to select a build system. ``spack create`` +usually does this for you, but if for whatever reason you need to do +this manually, choose ``PythonPackage`` if and only if the package +contains one of the following files: + +* ``pyproject.toml`` +* ``setup.py`` +* ``setup.cfg`` """"""""""""""""""""""" Choosing a package name @@ -857,10 +618,9 @@ having to add that module to ``PYTHONPATH``. When deciding between ``extends`` and ``depends_on``, the best rule of thumb is to check the installation prefix. If Python libraries are -installed to ``prefix/lib/python2.7/site-packages`` (where 2.7 is the -MAJOR.MINOR version of Python you used to install the package), then -you should use ``extends``. If Python libraries are installed elsewhere -or the only files that get installed reside in ``prefix/bin``, then +installed to ``/lib/pythonX.Y/site-packages``, then you +should use ``extends``. If Python libraries are installed elsewhere +or the only files that get installed reside in ``/bin``, then don't use ``extends``, as symlinking the package wouldn't be useful. ^^^^^^^^^^^^^^^^^^^^^ @@ -893,4 +653,17 @@ External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on Python packaging, see: -https://packaging.python.org/ + +* https://packaging.python.org/ + +For more information on build and installation frontend tools, see: + +* pip: https://pip.pypa.io/ +* build: https://pypa-build.readthedocs.io/ +* installer: https://installer.readthedocs.io/ + +For more information on build backend tools, see: + +* setuptools: https://setuptools.pypa.io/ +* flit: https://flit.readthedocs.io/ +* poetry: https://python-poetry.org/ diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index ac98b972a7..326e140290 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -177,6 +177,7 @@ def clean_environment(): env.unset('OBJC_INCLUDE_PATH') env.unset('CMAKE_PREFIX_PATH') + env.unset('PYTHONPATH') # Affects GNU make, can e.g. indirectly inhibit enabling parallel build env.unset('MAKEFLAGS') @@ -525,9 +526,10 @@ def _set_variables_for_single_module(pkg, module): m.cmake = Executable('cmake') m.ctest = MakeExecutable('ctest', jobs) - # Standard CMake arguments + # Standard build system arguments m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg) m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg) + m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg) # Put spack compiler paths in module scope. link_dir = spack.paths.build_env_path diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 59fd057c6a..036ef52e4e 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -18,65 +18,19 @@ from llnl.util.filesystem import ( ) from llnl.util.lang import match_predicate -from spack.directives import extends +from spack.directives import depends_on, extends from spack.package import PackageBase, run_after class PythonPackage(PackageBase): - """Specialized class for packages that are built using Python - setup.py files - - This class provides the following phases that can be overridden: - - * build - * build_py - * build_ext - * build_clib - * build_scripts - * install - * install_lib - * install_headers - * install_scripts - * install_data - - These are all standard setup.py commands and can be found by running: - - .. code-block:: console - - $ python setup.py --help-commands - - By default, only the 'build' and 'install' phases are run, but if you - need to run more phases, simply modify your ``phases`` list like so: - - .. code-block:: python - - phases = ['build_ext', 'install', 'bdist'] - - Each phase provides a function that runs: - - .. code-block:: console - - $ python -s setup.py --no-user-cfg - - Each phase also has a function that can pass arguments to - this call. All of these functions are empty except for the ``install_args`` - function, which passes ``--prefix=/path/to/installation/directory``. - - If you need to run a phase which is not a standard setup.py command, - you'll need to define a function for it like so: - - .. code-block:: python - - def configure(self, spec, prefix): - self.setup_py('configure') - """ + """Specialized class for packages that are built using pip.""" #: Package name, version, and extension on PyPI pypi = None maintainers = ['adamjstewart'] # Default phases - phases = ['build', 'install'] + phases = ['install'] # To be used in UI queries that require to know which # build-system class we are using @@ -86,9 +40,39 @@ class PythonPackage(PackageBase): install_time_test_callbacks = ['test'] extends('python') + depends_on('py-pip', type='build') + # FIXME: technically wheel is only needed when building from source, not when + # installing a downloaded wheel, but I don't want to add wheel as a dep to every + # package manually + depends_on('py-wheel', type='build') py_namespace = None + @staticmethod + def _std_args(cls): + return [ + # Verbose + '-vvv', + # Disable prompting for input + '--no-input', + # Disable the cache + '--no-cache-dir', + # Don't check to see if pip is up-to-date + '--disable-pip-version-check', + # Install packages + 'install', + # Don't install package dependencies + '--no-deps', + # Overwrite existing packages + '--ignore-installed', + # Use env vars like PYTHONPATH + '--no-build-isolation', + # Don't warn that prefix.bin is not in PATH + '--no-warn-script-location', + # Ignore the PyPI package index + '--no-index', + ] + @property def homepage(self): if self.pypi: @@ -153,163 +137,45 @@ class PythonPackage(PackageBase): return modules - def setup_file(self): - """Returns the name of the setup file to use.""" - return 'setup.py' - @property def build_directory(self): - """The directory containing the ``setup.py`` file.""" - return self.stage.source_path - - def python(self, *args, **kwargs): - inspect.getmodule(self).python(*args, **kwargs) - - def setup_py(self, *args, **kwargs): - setup = self.setup_file() - - with working_dir(self.build_directory): - self.python('-s', setup, '--no-user-cfg', *args, **kwargs) - - # The following phases and their descriptions come from: - # $ python setup.py --help-commands - - # Standard commands - - def build(self, spec, prefix): - """Build everything needed to install.""" - args = self.build_args(spec, prefix) + """The root directory of the Python package. - self.setup_py('build', *args) + This is usually the directory containing one of the following files: - def build_args(self, spec, prefix): - """Arguments to pass to build.""" - return [] - - def build_py(self, spec, prefix): - '''"Build" pure Python modules (copy to build directory).''' - args = self.build_py_args(spec, prefix) - - self.setup_py('build_py', *args) - - def build_py_args(self, spec, prefix): - """Arguments to pass to build_py.""" - return [] - - def build_ext(self, spec, prefix): - """Build C/C++ extensions (compile/link to build directory).""" - args = self.build_ext_args(spec, prefix) - - self.setup_py('build_ext', *args) + * ``pyproject.toml`` + * ``setup.cfg`` + * ``setup.py`` + """ + return self.stage.source_path - def build_ext_args(self, spec, prefix): - """Arguments to pass to build_ext.""" + def install_options(self, spec, prefix): + """Extra arguments to be supplied to the setup.py install command.""" return [] - def build_clib(self, spec, prefix): - """Build C/C++ libraries used by Python extensions.""" - args = self.build_clib_args(spec, prefix) - - self.setup_py('build_clib', *args) - - def build_clib_args(self, spec, prefix): - """Arguments to pass to build_clib.""" - return [] - - def build_scripts(self, spec, prefix): - '''"Build" scripts (copy and fixup #! line).''' - args = self.build_scripts_args(spec, prefix) - - self.setup_py('build_scripts', *args) - - def build_scripts_args(self, spec, prefix): - """Arguments to pass to build_scripts.""" + def global_options(self, spec, prefix): + """Extra global options to be supplied to the setup.py call before the install + or bdist_wheel command.""" return [] def install(self, spec, prefix): """Install everything from build directory.""" - args = self.install_args(spec, prefix) - - self.setup_py('install', *args) - - def install_args(self, spec, prefix): - """Arguments to pass to install.""" - args = ['--prefix={0}'.format(prefix)] - - # This option causes python packages (including setuptools) NOT - # to create eggs or easy-install.pth files. Instead, they - # install naturally into $prefix/pythonX.Y/site-packages. - # - # Eggs add an extra level of indirection to sys.path, slowing - # down large HPC runs. They are also deprecated in favor of - # wheels, which use a normal layout when installed. - # - # Spack manages the package directory on its own by symlinking - # extensions into the site-packages directory, so we don't really - # need the .pth files or egg directories, anyway. - # - # We need to make sure this is only for build dependencies. A package - # such as py-basemap will not build properly with this flag since - # it does not use setuptools to build and those does not recognize - # the --single-version-externally-managed flag - if ('py-setuptools' == spec.name or # this is setuptools, or - 'py-setuptools' in spec._dependencies and # it's an immediate dep - 'build' in spec._dependencies['py-setuptools'].deptypes): - args += ['--single-version-externally-managed'] - - # Get all relative paths since we set the root to `prefix` - # We query the python with which these will be used for the lib and inc - # directories. This ensures we use `lib`/`lib64` as expected by python. - pkg = spec['python'].package - args += ['--root=%s' % prefix, - '--install-purelib=%s' % pkg.purelib, - '--install-platlib=%s' % pkg.platlib, - '--install-scripts=bin', - '--install-data=', - '--install-headers=%s' % pkg.include, - ] - - return args - - def install_lib(self, spec, prefix): - """Install all Python modules (extensions and pure Python).""" - args = self.install_lib_args(spec, prefix) - - self.setup_py('install_lib', *args) - - def install_lib_args(self, spec, prefix): - """Arguments to pass to install_lib.""" - return [] - - def install_headers(self, spec, prefix): - """Install C/C++ header files.""" - args = self.install_headers_args(spec, prefix) - - self.setup_py('install_headers', *args) - - def install_headers_args(self, spec, prefix): - """Arguments to pass to install_headers.""" - return [] - - def install_scripts(self, spec, prefix): - """Install scripts (Python or otherwise).""" - args = self.install_scripts_args(spec, prefix) - - self.setup_py('install_scripts', *args) - def install_scripts_args(self, spec, prefix): - """Arguments to pass to install_scripts.""" - return [] + args = PythonPackage._std_args(self) + ['--prefix=' + prefix] - def install_data(self, spec, prefix): - """Install data files.""" - args = self.install_data_args(spec, prefix) + for option in self.install_options(spec, prefix): + args.append('--install-option=' + option) + for option in self.global_options(spec, prefix): + args.append('--global-option=' + option) - self.setup_py('install_data', *args) + if self.stage.archive_file and self.stage.archive_file.endswith('.whl'): + args.append(self.stage.archive_file) + else: + args.append('.') - def install_data_args(self, spec, prefix): - """Arguments to pass to install_data.""" - return [] + pip = inspect.getmodule(self).pip + with working_dir(self.build_directory): + pip(*args) # Testing diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index baeccc513e..cc9fffa0c2 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -263,19 +263,34 @@ class PythonPackageTemplate(PackageTemplate): base_class_name = 'PythonPackage' dependencies = """\ - # FIXME: Add dependencies if required. Only add the python dependency - # if you need specific versions. A generic python dependency is - # added implicity by the PythonPackage class. + # FIXME: Only add the python/pip/wheel dependencies if you need specific versions + # or need to change the dependency type. Generic python/pip/wheel dependencies are + # added implicity by the PythonPackage base class. # depends_on('python@2.X:2.Y,3.Z:', type=('build', 'run')) + # depends_on('py-pip@X.Y:', type='build') + # depends_on('py-wheel@X.Y:', type='build') + + # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file + # exists, use setuptools. # depends_on('py-setuptools', type='build') - # depends_on('py-foo', type=('build', 'run'))""" + # depends_on('py-flit-core', type='build') + # depends_on('py-poetry-core', type='build') + + # FIXME: Add additional dependencies if required. + # depends_on('py-foo', type=('build', 'run'))""" body_def = """\ - def build_args(self, spec, prefix): - # FIXME: Add arguments other than --prefix - # FIXME: If not needed delete this function - args = [] - return args""" + def global_options(self, spec, prefix): + # FIXME: Add options to pass to setup.py + # FIXME: If not needed, delete this function + options = [] + return options + + def install_options(self, spec, prefix): + # FIXME: Add options to pass to setup.py install + # FIXME: If not needed, delete this function + options = [] + return options""" def __init__(self, name, url, *args, **kwargs): # If the user provided `--name py-numpy`, don't rename it py-py-numpy @@ -298,24 +313,32 @@ class PythonPackageTemplate(PackageTemplate): # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512 - # PyPI URLs for wheels are too complicated, ignore them for now + # PyPI URLs for wheels: + # https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.11.0-py3-none-any.whl + # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-macosx_10_9_x86_64.whl + # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl + # https://files.pythonhosted.org/packages/cp35.cp36.cp37.cp38.cp39/s/shiboken2/shiboken2-5.15.2-5.15.2-cp35.cp36.cp37.cp38.cp39-abi3-manylinux1_x86_64.whl + # https://files.pythonhosted.org/packages/f4/99/ad2ef1aeeb395ee2319bb981ea08dbbae878d30dd28ebf27e401430ae77a/azureml_core-1.36.0.post2-py3-none-any.whl#sha256=60bcad10b4380d78a8280deb7365de2c2cd66527aacdcb4a173f613876cbe739 match = re.search( r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4, url ) if match: - if len(match.group(2)) == 1: - # Simple PyPI URL - url = '/'.join(match.group(3, 4)) - else: - # PyPI URL containing hash - # Project name doesn't necessarily match download name, but it - # usually does, so this is the best we can do - project = parse_name(url) - url = '/'.join([project, match.group(4)]) - - self.url_line = ' pypi = "{url}"' + # PyPI URLs for wheels are too complicated, ignore them for now + # https://www.python.org/dev/peps/pep-0427/#file-name-convention + if not match.group(4).endswith('.whl'): + if len(match.group(2)) == 1: + # Simple PyPI URL + url = '/'.join(match.group(3, 4)) + else: + # PyPI URL containing hash + # Project name doesn't necessarily match download name, but it + # usually does, so this is the best we can do + project = parse_name(url) + url = '/'.join([project, match.group(4)]) + + self.url_line = ' pypi = "{url}"' else: # Add a reminder about spack preferring PyPI URLs self.url_line = ''' @@ -581,6 +604,9 @@ class BuildSystemGuesser: if url.endswith('.gem'): self.build_system = 'ruby' return + if url.endswith('.whl') or '.whl#' in url: + self.build_system = 'python' + return # A list of clues that give us an idea of the build system a package # uses. If the regular expression matches a file contained in the @@ -596,7 +622,8 @@ class BuildSystemGuesser: (r'/pom\.xml$', 'maven'), (r'/SConstruct$', 'scons'), (r'/waf$', 'waf'), - (r'/setup\.py$', 'python'), + (r'/pyproject.toml', 'python'), + (r'/setup\.(py|cfg)$', 'python'), (r'/WORKSPACE$', 'bazel'), (r'/Build\.PL$', 'perlbuild'), (r'/Makefile\.PL$', 'perlmake'), diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b447867727..3e01c56374 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -897,6 +897,10 @@ def get_checksums_for_versions(url_dict, name, **kwargs): i = 0 errors = [] for url, version in zip(urls, versions): + # Wheels should not be expanded during staging + expand_arg = '' + if url.endswith('.whl') or '.whl#' in url: + expand_arg = ', expand=False' try: if fetch_options: url_or_fs = fs.URLFetchStrategy( @@ -931,8 +935,8 @@ def get_checksums_for_versions(url_dict, name, **kwargs): # Generate the version directives to put in a package.py version_lines = "\n".join([ - " version('{0}', {1}sha256='{2}')".format( - v, ' ' * (max_len - len(str(v))), h) for v, h in version_hashes + " version('{0}', {1}sha256='{2}'{3})".format( + v, ' ' * (max_len - len(str(v))), h, expand_arg) for v, h in version_hashes ]) num_hash = len(version_hashes) diff --git a/lib/spack/spack/test/cmd/create.py b/lib/spack/spack/test/cmd/create.py index 896d8e1fe1..08854bda3b 100644 --- a/lib/spack/spack/test/cmd/create.py +++ b/lib/spack/spack/test/cmd/create.py @@ -63,7 +63,7 @@ def parser(): r'def configure_args(self']), (['-t', 'python', 'test-python'], 'py-test-python', [r'PyTestPython(PythonPackage)', r"depends_on('py-", - r'def build_args(self']), + r'def global_options(self', r'def install_options(self']), (['-t', 'qmake', 'test-qmake'], 'test-qmake', [r'TestQmake(QMakePackage)', r'def qmake_args(self']), (['-t', 'r', 'test-r'], 'r-test-r', -- cgit v1.2.3-70-g09d2