diff options
Diffstat (limited to 'lib')
500 files changed, 4671 insertions, 2857 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index f7def9d439..738b4c579b 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -133,8 +133,7 @@ dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. You'll see a number of messages from Spack, a lot of build output, and a message -that the package is installed. Add one or more debug options (``-d``) -to get increasingly detailed output. +that the package is installed. .. code-block:: console @@ -151,8 +150,8 @@ to get increasingly detailed output. The last line, with the ``[+]``, indicates where the package is installed. -Add the debug option -- ``spack install -d mpileaks`` -- to get additional -output. +Add the Spack debug option (one or more times) -- ``spack -d install +mpileaks`` -- to get additional (and even more verbose) output. ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Building a specific version diff --git a/lib/spack/docs/binary_caches.rst b/lib/spack/docs/binary_caches.rst index 6e8c91c943..17617b1491 100644 --- a/lib/spack/docs/binary_caches.rst +++ b/lib/spack/docs/binary_caches.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst index 3e7a21c3e8..a0d4445d3f 100644 --- a/lib/spack/docs/build_settings.rst +++ b/lib/spack/docs/build_settings.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems.rst b/lib/spack/docs/build_systems.rst index 84c608077f..784730ce40 100644 --- a/lib/spack/docs/build_systems.rst +++ b/lib/spack/docs/build_systems.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -60,6 +60,7 @@ on these ideas for each distinct build system that Spack supports: build_systems/bundlepackage build_systems/cudapackage build_systems/intelpackage + build_systems/rocmpackage build_systems/custompackage For reference, the :py:mod:`Build System API docs <spack.build_systems>` diff --git a/lib/spack/docs/build_systems/autotoolspackage.rst b/lib/spack/docs/build_systems/autotoolspackage.rst index a4c15e5155..be9833b70e 100644 --- a/lib/spack/docs/build_systems/autotoolspackage.rst +++ b/lib/spack/docs/build_systems/autotoolspackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/bundlepackage.rst b/lib/spack/docs/build_systems/bundlepackage.rst index e22e4b26be..d6429c2519 100644 --- a/lib/spack/docs/build_systems/bundlepackage.rst +++ b/lib/spack/docs/build_systems/bundlepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/cmakepackage.rst b/lib/spack/docs/build_systems/cmakepackage.rst index 76e89c80b1..a3282514e8 100644 --- a/lib/spack/docs/build_systems/cmakepackage.rst +++ b/lib/spack/docs/build_systems/cmakepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/cudapackage.rst b/lib/spack/docs/build_systems/cudapackage.rst index 15150599d3..2d148d0483 100644 --- a/lib/spack/docs/build_systems/cudapackage.rst +++ b/lib/spack/docs/build_systems/cudapackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,35 +9,120 @@ CudaPackage ----------- -Different from other packages, ``CudaPackage`` does not represent a build -system. Instead its goal is to simplify and unify usage of ``CUDA`` in other -packages. +Different from other packages, ``CudaPackage`` does not represent a build system. +Instead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a ` mixin-class <https://en.wikipedia.org/wiki/Mixin>`__. -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Provided variants and dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +You can find source for the package at +`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cuda.py>`__. -``CudaPackage`` provides ``cuda`` variant (default to ``off``) to enable/disable -``CUDA``, and ``cuda_arch`` variant to optionally specify the architecture. -It also declares dependencies on the ``CUDA`` package ``depends_on('cuda@...')`` -based on the architecture as well as specifies conflicts for certain compiler versions. +^^^^^^^^ +Variants +^^^^^^^^ + +This package provides the following variants: + +* **cuda** + + This variant is used to enable/disable building with ``CUDA``. The default + is disabled (or ``False``). + +* **cuda_arch** + + This variant supports the optional specification of the architecture. + Valid values are maintained in the ``cuda_arch_values`` property and + are the numeric character equivalent of the compute capability version + (e.g., '10' for version 1.0). Each provided value affects associated + ``CUDA`` dependencies and compiler conflicts. + + GPUs and their compute capability versions are listed at + https://developer.nvidia.com/cuda-gpus . + +^^^^^^^^^ +Conflicts +^^^^^^^^^ + +Conflicts are used to prevent builds with known bugs or issues. While +base ``CUDA`` conflicts have been included with this package, you may +want to add more for your software. + +For example, if your package requires ``cuda_arch`` to be specified when +``cuda`` is enabled, you can add the following conflict to your package +to terminate such build attempts with a suitable message: + +.. code-block:: python + + conflicts('cuda_arch=none', when='+cuda', + msg='CUDA architecture is required') + +Similarly, if your software does not support all versions of the property, +you could add ``conflicts`` to your package for those versions. For example, +suppose your software does not work with CUDA compute capability versions +prior to SM 5.0 (``50``). You can add the following code to display a +custom message should a user attempt such a build: + +.. code-block:: python + + unsupported_cuda_archs = [ + '10', '11', '12', '13', + '20', '21', + '30', '32', '35', '37' + ] + for value in unsupported_cuda_archs: + conflicts('cuda_arch={0}'.format(value), when='+cuda', + msg='CUDA architecture {0} is not supported'.format(value)) + +^^^^^^^ +Methods +^^^^^^^ + +This package provides one custom helper method, which is used to build +standard CUDA compiler flags. + +**cuda_flags** + + This built-in static method returns a list of command line flags + for the chosen ``cuda_arch`` value(s). The flags are intended to + be passed to the CUDA compiler driver (i.e., ``nvcc``). + + This method must be explicitly called when you are creating the + arguments for your build in order to use the values. ^^^^^ Usage ^^^^^ -In order to use it, just add another base class to your package, for example: +This helper package can be added to your package by adding it as a base +class of your package. For example, you can add it to your +:ref:`CMakePackage <cmakepackage>`-based package as follows: .. code-block:: python + :emphasize-lines: 1,7-16 - class MyPackage(CMakePackage, CudaPackage): + class MyCudaPackage(CMakePackage, CudaPackage): ... def cmake_args(self): spec = self.spec + args = [] + ... if '+cuda' in spec: - options.append('-DWITH_CUDA=ON') - cuda_arch = spec.variants['cuda_arch'].value + # Set up the cuda macros needed by the build + args.append('-DWITH_CUDA=ON') + cuda_arch_list = spec.variants['cuda_arch'].value + cuda_arch = cuda_arch_list[0] if cuda_arch != 'none': - options.append('-DCUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch[0])) + args.append('-DCUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch)) else: - options.append('-DWITH_CUDA=OFF') + # Ensure build with cuda is disabled + args.append('-DWITH_CUDA=OFF') + ... + return args + +assuming only the ``WITH_CUDA`` and ``CUDA_FLAGS`` flags are required. +You will need to customize options as needed for your build. + +This example also illustrates how to check for the ``cuda`` variant using +``self.spec`` and how to retrieve the ``cuda_arch`` variant's value, which +is a list, using ``self.spec.variants['cuda_arch'].value``. + +With over 70 packages using ``CudaPackage`` as of January 2021 there are +lots of examples to choose from to get more ideas for using this package. diff --git a/lib/spack/docs/build_systems/custompackage.rst b/lib/spack/docs/build_systems/custompackage.rst index 8a20dada92..b513e3963a 100644 --- a/lib/spack/docs/build_systems/custompackage.rst +++ b/lib/spack/docs/build_systems/custompackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/intelpackage.rst b/lib/spack/docs/build_systems/intelpackage.rst index 8594c8d425..2c1723bbbc 100644 --- a/lib/spack/docs/build_systems/intelpackage.rst +++ b/lib/spack/docs/build_systems/intelpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/makefilepackage.rst b/lib/spack/docs/build_systems/makefilepackage.rst index 406ec8ce89..f425855b69 100644 --- a/lib/spack/docs/build_systems/makefilepackage.rst +++ b/lib/spack/docs/build_systems/makefilepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/mavenpackage.rst b/lib/spack/docs/build_systems/mavenpackage.rst index 6f833e8bec..7c8c0e3741 100644 --- a/lib/spack/docs/build_systems/mavenpackage.rst +++ b/lib/spack/docs/build_systems/mavenpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/mesonpackage.rst b/lib/spack/docs/build_systems/mesonpackage.rst index 75b0e1caae..652a10db92 100644 --- a/lib/spack/docs/build_systems/mesonpackage.rst +++ b/lib/spack/docs/build_systems/mesonpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/octavepackage.rst b/lib/spack/docs/build_systems/octavepackage.rst index 90e582e36c..baccd0c0e7 100644 --- a/lib/spack/docs/build_systems/octavepackage.rst +++ b/lib/spack/docs/build_systems/octavepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/perlpackage.rst b/lib/spack/docs/build_systems/perlpackage.rst index 11968e1560..9fc57f6337 100644 --- a/lib/spack/docs/build_systems/perlpackage.rst +++ b/lib/spack/docs/build_systems/perlpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -120,8 +120,6 @@ so ``PerlPackage`` contains: extends('perl') - depends_on('perl', type=('build', 'run')) - If your package requires a specific version of Perl, you should specify this. diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst index ef06725e01..f1c85d92cb 100644 --- a/lib/spack/docs/build_systems/pythonpackage.rst +++ b/lib/spack/docs/build_systems/pythonpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,20 +23,11 @@ can be overridden: * ``build_ext`` * ``build_clib`` * ``build_scripts`` -* ``clean`` * ``install`` * ``install_lib`` * ``install_headers`` * ``install_scripts`` * ``install_data`` -* ``sdist`` -* ``register`` -* ``bdist`` -* ``bdist_dumb`` -* ``bdist_rpm`` -* ``bdist_wininst`` -* ``upload`` -* ``check`` These are all standard ``setup.py`` commands and can be found by running: @@ -55,7 +46,7 @@ If for whatever reason you need to run more phases, simply modify your .. code-block:: python - phases = ['build_ext', 'install', 'bdist'] + phases = ['build_ext', 'install'] Each phase provides a function ``<phase>`` that runs: @@ -90,7 +81,7 @@ Instead of using the ``PythonPackage`` base class, you should extend the ``Package`` base class and implement the following custom installation procedure: -.. code-block:: +.. code-block:: python def install(self, spec, prefix): pip = which('pip') @@ -134,9 +125,9 @@ The zip file will not contain a ``setup.py``, but it will contain a ``METADATA`` file which contains all the information you need to write a ``package.py`` build recipe. -^^^^^^^^^^^^^^^^^^^^^^^ -Finding Python packages -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^ +PyPI +^^^^ The vast majority of Python packages are hosted on PyPI - The Python Package Index. ``pip`` only supports packages hosted on PyPI, making @@ -148,6 +139,26 @@ if a newer version is available. The download page is usually at:: https://pypi.org/project/<package-name> + +Since PyPI is so common, the ``PythonPackage`` base class has a +``pypi`` attribute that can be set. Once set, ``pypi`` will be used +to define the ``homepage``, ``url``, and ``list_url``. For example, +the following: + +.. code-block:: python + + homepage = 'https://pypi.org/project/setuptools/' + url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip' + list_url = 'https://pypi.org/simple/setuptools/' + + +is equivalent to: + +.. code-block:: python + + pypi = 'setuptools/setuptools-49.2.0.zip' + + ^^^^^^^^^^^ Description ^^^^^^^^^^^ @@ -184,50 +195,11 @@ also get the homepage on the command-line by running: URL ^^^ -You may have noticed that Spack allows you to add multiple versions of -the same package without adding multiple versions of the download URL. -It does this by guessing what the version string in the URL is and -replacing this with the requested version. Obviously, if Spack cannot -guess the version correctly, or if non-version-related things change -in the URL, Spack cannot substitute the version properly. - -Once upon a time, PyPI offered nice, simple download URLs like:: - - https://pypi.python.org/packages/source/n/numpy/numpy-1.13.1.zip - - -As you can see, the version is 1.13.1. It probably isn't hard to guess -what URL to use to download version 1.12.0, and Spack was perfectly -capable of performing this calculation. - -However, PyPI switched to a new download URL format:: - - https://pypi.python.org/packages/c0/3a/40967d9f5675fbb097ffec170f59c2ba19fc96373e73ad47c2cae9a30aed/numpy-1.13.1.zip#md5=2c3c0f4edf720c3a7b525dacc825b9ae - - -and more recently:: - - https://files.pythonhosted.org/packages/b0/2b/497c2bb7c660b2606d4a96e2035e92554429e139c6c71cdff67af66b58d2/numpy-1.14.3.zip - - -As you can imagine, it is impossible for Spack to guess what URL to -use to download version 1.12.0 given this URL. There is a solution, -however. PyPI offers a new hidden interface for downloading -Python packages that does not include a hash in the URL:: - - https://pypi.io/packages/source/n/numpy/numpy-1.13.1.zip - - -This URL redirects to the https://files.pythonhosted.org URL. The general -syntax for this https://pypi.io URL is:: - - https://pypi.io/packages/<type>/<first-letter-of-name>/<name>/<name>-<version>.<extension> - - -Please use the https://pypi.io URL instead of the https://pypi.python.org -URL. If both ``.tar.gz`` and ``.zip`` versions are available, ``.tar.gz`` -is preferred. If some releases offer both ``.tar.gz`` and ``.zip`` versions, -but some only offer ``.zip`` versions, use ``.zip``. +If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will +be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions +are available, ``.tar.gz`` is preferred. If some releases offer both +``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions, +use ``.zip``. Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip`` files on either PyPI or GitHub. If this is the case, you can still download @@ -237,10 +209,9 @@ and can be downloaded from:: https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl -Note that instead of ``<type>`` being ``source``, it is now ``py3`` since this -wheel will work for any generic version of Python 3. You may see Python-specific -or OS-specific URLs. Note that when you add a ``.whl`` URL, you should add -``expand=False`` to ensure that Spack doesn't try to extract the wheel: +You may see Python-specific or OS-specific URLs. Note that when you add a +``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't +try to extract the wheel: .. code-block:: python @@ -255,7 +226,7 @@ Many packages are hosted on PyPI, but are developed on GitHub or another version control systems. The tarball can be downloaded from either location, but PyPI is preferred for the following reasons: -#. PyPI contains the bare minimum of files to install the package. +#. PyPI contains the bare minimum number of files needed to install the package. You may notice that the tarball you download from PyPI does not have the same checksum as the tarball you download from GitHub. @@ -292,19 +263,6 @@ location, but PyPI is preferred for the following reasons: PyPI is nice because it makes it physically impossible to re-release the same version of a package with a different checksum. -There are some reasons to prefer downloading from GitHub: - -#. The GitHub tarball may contain unit tests. - - As previously mentioned, the PyPI tarball contains the bare minimum - of files to install the package. Unless explicitly specified by the - developers, it will not contain development files like unit tests. - If you desire to run the unit tests during installation, you should - use the GitHub tarball instead. - -If you really want to run these unit tests, no one will stop you from -submitting a PR for a new package that downloads from GitHub. - ^^^^^^^^^^^^^^^^^^^^^^^^^ Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -569,7 +527,8 @@ If the package uses ``setuptools``, check for the following clues: These are packages that are required to run the unit tests for the package. These dependencies can be specified using the - ``type='test'`` dependency type. + ``type='test'`` dependency type. However, the PyPI tarballs rarely + contain unit tests, so there is usually no reason to add these. In the root directory of the package, you may notice a ``requirements.txt`` file. It may look like this file contains a list @@ -625,7 +584,8 @@ add run-time dependencies if they aren't needed, so you need to determine whether or not setuptools is needed. Grep the installation directory for any files containing a reference to ``setuptools`` or ``pkg_resources``. Both modules come from ``py-setuptools``. -``pkg_resources`` is particularly common in scripts in ``prefix/bin``. +``pkg_resources`` is particularly common in scripts found in +``prefix/bin``. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Passing arguments to setup.py @@ -699,48 +659,64 @@ a "package" is a directory containing files like: foo/baz.py -whereas a "module" is a single Python file. Since ``find_packages`` -only returns packages, you'll have to determine the correct module -names yourself. You can now add these packages and modules to the -package like so: +whereas a "module" is a single Python file. + +The ``PythonPackage`` base class automatically detects these module +names for you. If, for whatever reason, the module names detected +are wrong, you can provide the names yourself by overriding +``import_modules`` like so: .. code-block:: python import_modules = ['six'] -When you run ``spack install --test=root py-six``, Spack will attempt -to import the ``six`` module after installation. +Sometimes the list of module names to import depends on how the +package was built. For example, the ``py-pyyaml`` package has a +``+libyaml`` variant that enables the build of a faster optimized +version of the library. If the user chooses ``~libyaml``, only the +``yaml`` library will be importable. If the user chooses ``+libyaml``, +both the ``yaml`` and ``yaml.cyaml`` libraries will be available. +This can be expressed like so: + +.. code-block:: python + + @property + def import_modules(self): + modules = ['yaml'] + + if '+libyaml' in self.spec: + modules.append('yaml.cyaml') + + return modules -These tests most often catch missing dependencies and non-RPATHed + +These tests often catch missing dependencies and non-RPATHed libraries. Make sure not to add modules/packages containing the word -"test", as these likely won't end up in installation directory. +"test", as these likely won't end up in the installation directory, +or may require test dependencies like pytest to be installed. + +These tests can be triggered by running ``spack install --test=root`` +or by running ``spack test run`` after the installation has finished. """""""""" Unit tests """""""""" The package you want to install may come with additional unit tests. -By default, Spack runs: - -.. code-block:: console - - $ python setup.py test - - -if it detects that the ``setup.py`` file supports a ``test`` phase. -You can add additional build-time or install-time tests by overriding -``test`` or adding a custom install-time test function. For example, -``py-numpy`` adds: +You can add additional build-time or install-time tests by adding +additional testing functions. For example, ``py-numpy`` adds: .. code-block:: python - install_time_test_callbacks = ['install_test', 'import_module_test'] - + @run_after('install') + @on_package_attributes(run_tests=True) def install_test(self): - with working_dir('..'): - python('-c', 'import numpy; numpy.test("full", verbose=2)') + with working_dir('spack-test', create=True): + python('-c', 'import numpy; numpy.test("full", verbose=2)') + +These tests can be triggered by running ``spack install --test=root``. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setup file in a sub-directory @@ -781,7 +757,7 @@ PythonPackage vs. packages that use Python ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are many packages that make use of Python, but packages that depend -on Python are not necessarily ``PythonPackages``. +on Python are not necessarily ``PythonPackage``'s. """"""""""""""""""""""" Choosing a build system @@ -878,8 +854,8 @@ and ``pip`` may be a perfectly valid alternative to using Spack. The main advantage of Spack over ``pip`` is its ability to compile non-Python dependencies. It can also build cythonized versions of a package or link to an optimized BLAS/LAPACK library like MKL, -resulting in calculations that run orders of magnitude faster. -Spack does not offer a significant advantage to other python-management +resulting in calculations that run orders of magnitudes faster. +Spack does not offer a significant advantage over other python-management systems for installing and using tools like flake8 and sphinx. But if you need packages with non-Python dependencies like numpy and scipy, Spack will be very valuable to you. diff --git a/lib/spack/docs/build_systems/qmakepackage.rst b/lib/spack/docs/build_systems/qmakepackage.rst index 81d0f7f798..6e85d2bbcd 100644 --- a/lib/spack/docs/build_systems/qmakepackage.rst +++ b/lib/spack/docs/build_systems/qmakepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -108,6 +108,19 @@ override the ``qmake_args`` method like so: This method can be used to pass flags as well as variables. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``*.pro`` file in a sub-directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the ``*.pro`` file used to tell QMake how to build the package is +found in a sub-directory, you can tell Spack to run all phases in this +sub-directory by adding the following to the package: + +.. code-block:: python + + build_directory = 'src' + + ^^^^^^^^^^^^^^^^^^^^^^ External documentation ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/build_systems/rocmpackage.rst b/lib/spack/docs/build_systems/rocmpackage.rst new file mode 100644 index 0000000000..e322194adf --- /dev/null +++ b/lib/spack/docs/build_systems/rocmpackage.rst @@ -0,0 +1,122 @@ +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _rocmpackage: + +----------- +ROCmPackage +----------- + +The ``ROCmPackage`` is not a build system but a helper package. Like ``CudaPackage``, +it provides standard variants, dependencies, and conflicts to facilitate building +packages using GPUs though for AMD in this case. + +You can find the source for this package (and suggestions for setting up your +``compilers.yaml`` and ``packages.yaml`` files) at +`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/rocm.py>`__. + +^^^^^^^^ +Variants +^^^^^^^^ + +This package provides the following variants: + +* **rocm** + + This variant is used to enable/disable building with ``rocm``. + The default is disabled (or ``False``). + +* **amdgpu_target** + + This variant supports the optional specification of the AMD GPU architecture. + Valid values are the names of the GPUs (e.g., ``gfx701``), which are maintained + in the ``amdgpu_targets`` property. + +^^^^^^^^^^^^ +Dependencies +^^^^^^^^^^^^ + +This package defines basic ``rocm`` dependencies, including ``llvm`` and ``hip``. + +^^^^^^^^^ +Conflicts +^^^^^^^^^ + +Conflicts are used to prevent builds with known bugs or issues. This package +already requires that the ``amdgpu_target`` always be specified for ``rocm`` +builds. It also defines a conflict that prevents builds with an ``amdgpu_target`` +when ``rocm`` is disabled. + +Refer to `Conflicts <https://spack.readthedocs.io/en/latest/packaging_guide.html?highlight=conflicts#conflicts>`__ +for more information on package conflicts. + +^^^^^^^ +Methods +^^^^^^^ + +This package provides one custom helper method, which is used to build +standard AMD hip compiler flags. + +**hip_flags** + + This built-in static method returns the appropriately formatted + ``--amdgpu-target`` build option for ``hipcc``. + + This method must be explicitly called when you are creating the + arguments for your build in order to use the values. + +^^^^^ +Usage +^^^^^ + +This helper package can be added to your package by adding it as a base +class of your package. For example, you can add it to your +:ref:`CMakePackage <cmakepackage>`-based package as follows: + +.. code-block:: python + :emphasize-lines: 1,3-7,14-25 + + class MyRocmPackage(CMakePackage, ROCmPackage): + ... + # Ensure +rocm and amdgpu_targets are passed to dependencies + depends_on('mydeppackage', when='+rocm') + for val in ROCmPackage.amdgpu_targets: + depends_on('mydeppackage amdgpu_target={0}'.format(val), + when='amdgpu_target={0}'.format(val)) + ... + + def cmake_args(self): + spec = self.spec + args = [] + ... + if '+rocm' in spec: + # Set up the hip macros needed by the build + args.extend([ + '-DENABLE_HIP=ON', + '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix]) + rocm_archs = spec.variants['amdgpu_target'].value + if 'none' not in rocm_archs: + args.append('-DHIP_HIPCC_FLAGS=--amdgpu-target={0}' + .format(",".join(rocm_archs))) + else: + # Ensure build with hip is disabled + args.append('-DENABLE_HIP=OFF') + ... + return args + ... + +assuming only on the ``ENABLE_HIP``, ``HIP_ROOT_DIR``, and ``HIP_HIPCC_FLAGS`` +macros are required to be set and the only dependency needing rocm options +is ``mydeppackage``. You will need to customize the flags as needed for your +build. + +This example also illustrates how to check for the ``rocm`` variant using +``self.spec`` and how to retrieve the ``amdgpu_target`` variant's value +using ``self.spec.variants['amdgpu_target'].value``. + +All five packages using ``ROCmPackage`` as of January 2021 also use the +:ref:`CudaPackage <cudapackage>`. So it is worth looking at those packages +to get ideas for creating a package that can support both ``cuda`` and +``rocm``. diff --git a/lib/spack/docs/build_systems/rpackage.rst b/lib/spack/docs/build_systems/rpackage.rst index 60c76816c5..486fd95b69 100644 --- a/lib/spack/docs/build_systems/rpackage.rst +++ b/lib/spack/docs/build_systems/rpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -138,7 +138,6 @@ every R package needs this, the ``RPackage`` base class contains: .. code-block:: python extends('r') - depends_on('r', type=('build', 'run')) Take a close look at the homepage for ``caret``. If you look at the @@ -298,8 +297,8 @@ like so: .. code-block:: python - def configure_args(self, spec, prefix): - mpi_name = spec['mpi'].name + def configure_args(self): + mpi_name = self.spec['mpi'].name # The type of MPI. Supported values are: # OPENMPI, LAM, MPICH, MPICH2, or CRAY diff --git a/lib/spack/docs/build_systems/rubypackage.rst b/lib/spack/docs/build_systems/rubypackage.rst index d1deaf6fa7..226d0bc4a2 100644 --- a/lib/spack/docs/build_systems/rubypackage.rst +++ b/lib/spack/docs/build_systems/rubypackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -113,7 +113,6 @@ the base class contains: .. code-block:: python extends('ruby') - depends_on('ruby', type=('build', 'run')) The ``*.gemspec`` file may contain something like: diff --git a/lib/spack/docs/build_systems/sconspackage.rst b/lib/spack/docs/build_systems/sconspackage.rst index f29d6c5f45..f1f67b76b4 100644 --- a/lib/spack/docs/build_systems/sconspackage.rst +++ b/lib/spack/docs/build_systems/sconspackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/sippackage.rst b/lib/spack/docs/build_systems/sippackage.rst index ddf9a26ab9..f72d83ca30 100644 --- a/lib/spack/docs/build_systems/sippackage.rst +++ b/lib/spack/docs/build_systems/sippackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -93,10 +93,17 @@ in the site-packages directory: $ python >>> import setuptools >>> setuptools.find_packages() - ['QtPy5'] + [ + 'PyQt5', 'PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtHelp', + 'PyQt5.QtMultimedia', 'PyQt5.QtMultimediaWidgets', 'PyQt5.QtNetwork', + 'PyQt5.QtOpenGL', 'PyQt5.QtPrintSupport', 'PyQt5.QtQml', + 'PyQt5.QtQuick', 'PyQt5.QtSvg', 'PyQt5.QtTest', 'PyQt5.QtWebChannel', + 'PyQt5.QtWebSockets', 'PyQt5.QtWidgets', 'PyQt5.QtXml', + 'PyQt5.QtXmlPatterns' + ] -Large, complex packages like ``QtPy5`` will return a long list of +Large, complex packages like ``py-pyqt5`` will return a long list of packages, while other packages may return an empty list. These packages only install a single ``foo.py`` file. In Python packaging lingo, a "package" is a directory containing files like: @@ -108,21 +115,25 @@ a "package" is a directory containing files like: foo/baz.py -whereas a "module" is a single Python file. Since ``find_packages`` -only returns packages, you'll have to determine the correct module -names yourself. You can now add these packages and modules to the -package like so: +whereas a "module" is a single Python file. + +The ``SIPPackage`` base class automatically detects these module +names for you. If, for whatever reason, the module names detected +are wrong, you can provide the names yourself by overriding +``import_modules`` like so: .. code-block:: python import_modules = ['PyQt5'] -When you run ``spack install --test=root py-pyqt5``, Spack will attempt -to import the ``PyQt5`` module after installation. +These tests often catch missing dependencies and non-RPATHed +libraries. Make sure not to add modules/packages containing the word +"test", as these likely won't end up in the installation directory, +or may require test dependencies like pytest to be installed. -These tests most often catch missing dependencies and non-RPATHed -libraries. +These tests can be triggered by running ``spack install --test=root`` +or by running ``spack test run`` after the installation has finished. ^^^^^^^^^^^^^^^^^^^^^^ External documentation diff --git a/lib/spack/docs/build_systems/wafpackage.rst b/lib/spack/docs/build_systems/wafpackage.rst index 36fc21a772..1bb879e4f2 100644 --- a/lib/spack/docs/build_systems/wafpackage.rst +++ b/lib/spack/docs/build_systems/wafpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index d84db87ff3..ddebbbeec3 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index e19dd05511..2de12a5a99 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -128,7 +128,7 @@ master_doc = 'index' # General information about the project. project = u'Spack' -copyright = u'2013-2019, Lawrence Livermore National Laboratory.' +copyright = u'2013-2021, Lawrence Livermore National Laboratory.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index bcb41527a9..66df916193 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 41c9de79f9..0671beb080 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst index f5bd5602bc..d06f538e99 100644 --- a/lib/spack/docs/containers.rst +++ b/lib/spack/docs/containers.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst index 8df8ad65ba..f4527355c4 100644 --- a/lib/spack/docs/contribution_guide.rst +++ b/lib/spack/docs/contribution_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -179,24 +179,26 @@ how to write tests! run the unit tests yourself, we suggest you use ``spack unit-test``. ^^^^^^^^^^^^ -Flake8 Tests +Style Tests ^^^^^^^^^^^^ Spack uses `Flake8 <http://flake8.pycqa.org/en/latest/>`_ to test for -`PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance. PEP 8 is +`PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance and +`mypy <https://mypy.readthedocs.io/en/stable/>` for type checking. PEP 8 is a series of style guides for Python that provide suggestions for everything from variable naming to indentation. In order to limit the number of PRs that were mostly style changes, we decided to enforce PEP 8 conformance. Your PR -needs to comply with PEP 8 in order to be accepted. +needs to comply with PEP 8 in order to be accepted, and if it modifies the +spack library it needs to successfully type-check with mypy as well. -Testing for PEP 8 compliance is easy. Simply run the ``spack flake8`` +Testing for compliance with spack's style is easy. Simply run the ``spack style`` command: .. code-block:: console - $ spack flake8 + $ spack style -``spack flake8`` has a couple advantages over running ``flake8`` by hand: +``spack style`` has a couple advantages over running the tools by hand: #. It only tests files that you have modified since branching off of ``develop``. @@ -207,7 +209,9 @@ command: checks. For example, URLs are often longer than 80 characters, so we exempt them from line length checks. We also exempt lines that start with "homepage", "url", "version", "variant", "depends_on", and - "extends" in ``package.py`` files. + "extends" in ``package.py`` files. This is now also possible when directly + running flake8 if you can use the ``spack`` formatter plugin included with + spack. More approved flake8 exemptions can be found `here <https://github.com/spack/spack/blob/develop/.flake8>`_. @@ -240,13 +244,13 @@ However, if you aren't compliant with PEP 8, flake8 will complain: Most of the error messages are straightforward, but if you don't understand what they mean, just ask questions about them when you submit your PR. The line numbers -will change if you add or delete lines, so simply run ``spack flake8`` again +will change if you add or delete lines, so simply run ``spack style`` again to update them. .. tip:: Try fixing flake8 errors in reverse order. This eliminates the need for - multiple runs of ``spack flake8`` just to re-compute line numbers and + multiple runs of ``spack style`` just to re-compute line numbers and makes it much easier to fix errors directly off of the CI output. .. warning:: diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index b4b8b5d321..9964bb6e51 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -396,20 +396,42 @@ other Spack modules: True >>> -You can also run a single command: +If you prefer using an IPython interpreter, given that IPython is installed +you can specify the interpreter with ``-i``: + +.. code-block:: console + + $ spack python -i ipython + Python 3.8.3 (default, May 19 2020, 18:47:26) + Type 'copyright', 'credits' or 'license' for more information + IPython 7.17.0 -- An enhanced Interactive Python. Type '?' for help. + + + Spack version 0.16.0 + Python 3.8.3, Linux x86_64 + + In [1]: + + +With either interpreter you can run a single command: .. code-block:: console $ spack python -c 'import distro; distro.linux_distribution()' - ('Fedora', '25', 'Workstation Edition') + ('Ubuntu', '18.04', 'Bionic Beaver') + + $ spack python -i ipython -c 'import distro; distro.linux_distribution()' + Out[1]: ('Ubuntu', '18.04', 'Bionic Beaver') or a file: .. code-block:: console $ spack python ~/test_fetching.py + $ spack python -i ipython ~/test_fetching.py + +just like you would with the normal ``python`` command. -just like you would with the normal ``python`` command. .. _cmd-spack-url: diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 1710ec6b5a..064cfad336 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -441,8 +441,8 @@ Environments can include files with either relative or absolute paths. Inline configurations take precedence over included configurations, so you don't have to change shared configuration files to make small changes to an individual Environment. Included configs -listed later will have higher precedence, as the included configs are -applied in order. +listed earlier will have higher precedence, as the included configs are +applied in reverse order. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Manually Editing the Specs List diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst index 15c59c76ef..b0cb38d00b 100644 --- a/lib/spack/docs/extensions.rst +++ b/lib/spack/docs/extensions.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index df212c11fc..7edfba01cc 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index f1df0343bd..6f3340b18b 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -75,6 +75,14 @@ shell integration for :ref:`certain commands <packaging-shell-support>`, If you do not want to use Spack's shell support, you can always just run the ``spack`` command directly from ``spack/bin/spack``. +When the ``spack`` command is executed it searches for an appropriate +Python interpreter to use, which can be explicitly overridden by setting +the ``SPACK_PYTHON`` environment variable. When sourcing the appropriate shell +setup script, ``SPACK_PYTHON`` will be set to the interpreter found at +sourcing time, ensuring future invocations of the ``spack`` command will +continue to use the same consistent python version regardless of changes in +the environment. + ^^^^^^^^^^^^^^^^^^ Check Installation diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 39fee81387..7a9ac09ec2 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/known_issues.rst b/lib/spack/docs/known_issues.rst index accec2422b..b9c4dd088f 100644 --- a/lib/spack/docs/known_issues.rst +++ b/lib/spack/docs/known_issues.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 5268c87067..8ed8c375a9 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index 721b18d5fc..773f347ed6 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/package_list.rst b/lib/spack/docs/package_list.rst index 67f78e202d..b0607edb86 100644 --- a/lib/spack/docs/package_list.rst +++ b/lib/spack/docs/package_list.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index ec83679a47..fa85097021 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -612,6 +612,62 @@ it executable, then runs it with some arguments. installer = Executable(self.stage.archive_file) installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') + +^^^^^^^^^^^^^^^^^^^^^^^^ +Deprecating old versions +^^^^^^^^^^^^^^^^^^^^^^^^ + +There are many reasons to remove old versions of software: + +#. Security vulnerabilities (most serious reason) +#. Changing build systems that increase package complexity +#. Changing dependencies/patches/resources/flags that increase package complexity +#. Maintainer/developer inability/unwillingness to support old versions +#. No longer available for download (right to be forgotten) +#. Package or version rename + +At the same time, there are many reasons to keep old versions of software: + +#. Reproducibility +#. Requirements for older packages (e.g. some packages still rely on Qt 3) + +In general, you should not remove old versions from a ``package.py``. Instead, +you should first deprecate them using the following syntax: + +.. code-block:: python + + version('1.2.3', sha256='...', deprecated=True) + + +This has two effects. First, ``spack info`` will no longer advertise that +version. Second, commands like ``spack install`` that fetch the package will +require user approval: + +.. code-block:: console + + $ spack install openssl@1.0.1e + ==> Warning: openssl@1.0.1e is deprecated and may be removed in a future Spack release. + ==> Fetch anyway? [y/N] + + +If you use ``spack install --deprecated``, this check can be skipped. + +This also applies to package recipes that are renamed or removed. You should +first deprecate all versions before removing a package. If you need to rename +it, you can deprecate the old package and create a new package at the same +time. + +Version deprecations should always last at least one Spack minor release cycle +before the version is completely removed. For example, if a version is +deprecated in Spack 0.16.0, it should not be removed until Spack 0.17.0. No +version should be removed without such a deprecation process. This gives users +a chance to complain about the deprecation in case the old version is needed +for some application. If you require a deprecated version of a package, simply +submit a PR to remove ``deprecated=True`` from the package. However, you may be +asked to help maintain this version of the package if the current maintainers +are unwilling to support this older version. + + ^^^^^^^^^^^^^^^^ Download caching ^^^^^^^^^^^^^^^^ @@ -1864,14 +1920,21 @@ of ``mpich`` using the following command: $ srun -N 2 -n 8 spack install -j 4 mpich@3.3.2 -This will create eight concurrent four-job installation on two different +This will create eight concurrent, four-job installs on two different nodes. +Alternatively, you could run the same installs on one node by entering +the following at the command line of a bash shell: + +.. code-block:: console + + $ for i in {1..12}; do nohup spack install -j 4 mpich@3.3.2 >> mpich_install.txt 2>&1 &; done + .. note:: - The effective parallelism will be based on the maximum number of - packages that can be installed at the same time, which will limited - by the number of packages with no (remaining) uninstalled dependencies. + The effective parallelism is based on the maximum number of packages + that can be installed at the same time, which is limited by the + number of packages with no (remaining) uninstalled dependencies. .. _dependencies: @@ -2737,12 +2800,14 @@ The classes that are currently provided by Spack are: | | built using CMake | +-------------------------------+----------------------------------+ | :py:class:`.CudaPackage` | A helper class for packages that | - | | use CUDA. It is intended to be | - | | used in combination with others | + | | use CUDA | +-------------------------------+----------------------------------+ | :py:class:`.QMakePackage` | Specialized class for packages | | | build using QMake | +-------------------------------+----------------------------------+ + | :py:class:`.ROCmPackage` | A helper class for packages that | + | | use ROCm | + +-------------------------------+----------------------------------+ | :py:class:`.SConsPackage` | Specialized class for packages | | | built using SCons | +-------------------------------+----------------------------------+ @@ -3946,7 +4011,7 @@ using the ``run_before`` decorator. The API for adding tests is not yet considered stable and may change drastically in future releases. -.. _file-manipulation: +.. _cmd-spack-test: ^^^^^^^^^^^^^ Install Tests @@ -3956,109 +4021,205 @@ Install Tests The API for adding and running install tests is not yet considered stable and may change drastically in future releases. Packages with - upstreamed tests will be refactored to match changes to the API. + install tests will be refactored to match changes to the API. -While build-tests are integrated with the build system, install tests -may be added to Spack packages to be run independently of the install -method. +While build tests are integrated with the build system, install tests +are for testing package installations independent of the build process. -Install tests may be added by defining a ``test`` method with the following signature: +Install tests are executed in a test stage directory that defaults to +``~/.spack/test``. You can change the location in the high-level ``config`` +by adding the ``test_stage`` path as follows: + +.. code-block:: yaml + + config: + test_stage: /path/to/stage + +"""""""""""""""""""" +Adding install tests +"""""""""""""""""""" + +Install tests are added to a package by defining a ``test`` method +with the following signature: .. code-block:: python def test(self): -These tests will be run in an environment set up to provide access to -this package and all of its dependencies, including ``test``-type -dependencies. Inside the ``test`` method, standard python ``assert`` -statements and other error reporting mechanisms can be used. Spack -will report any errors as a test failure. +These tests run in an environment that provides access to the +package and all of its dependencies, including ``test``-type +dependencies. + +Standard python ``assert`` statements and other error reporting +mechanisms can be used in the ``test`` method. Spack will report +such errors as test failures. -Inside the test method, individual tests can be run separately (and -continue transparently after a test failure) using the ``run_test`` -method. The signature for the ``run_test`` method is: +You can implement multiple tests (or test parts) within the ``test`` +method, where each is run separately and testing continues after +failures by using the ``run_test`` method. The signature for the +method is: .. code-block:: python def run_test(self, exe, options=[], expected=[], status=0, installed=False, purpose='', skip_missing=False, work_dir=None): -This method will operate in ``work_dir`` if one is specified. It will -search for an executable in the ``PATH`` variable named ``exe``, and -if ``installed=True`` it will fail if that executable does not come -from the prefix of the package being tested. If the executable is not -found, it will fail the test unless ``skip_missing`` is set to -``True``. The executable will be run with the options specified, and -the return code will be checked against the ``status`` argument, which -can be an integer or list of integers. Spack will also check that -every string in ``expected`` is a regex matching part of the output of -the executable. The ``purpose`` argument is recorded in the test log -for debugging purposes. - -"""""""""""""""""""""""""""""""""""""" -Install tests that require compilation -"""""""""""""""""""""""""""""""""""""" +The test fails if there is no executable named ``exe`` found in the +paths of the ``PATH`` variable **unless** ``skip_missing`` is ``True``. +The test also fails if the resulting path is not within the prefix of +the package being tested when ``installed`` is ``True``. + +The executable runs in ``work_dir``, when specified, using the provided +``options``. The return code is checked against the ``status`` argument, +which can be an integer or list of integers representing status codes +corresponding to successful execution (e.g. ``status=[0,3,7]``). +Spack also checks that every string in ``expected`` is a regex matching +part of the output from the test run (e.g. +``expected=['completed successfully', 'converged in']``). Default behavior +is to behave as though ``status=[0]`` and ``expected=[]`` are specified. + +Output from the test is written to its log file. The ``purpose`` argument +serves as the heading in text logs to highlight the start of each test part. + +""""""""""""""""""""""""" +Enabling test compilation +""""""""""""""""""""""""" Some tests may require access to the compiler with which the package -was built, especially to test library-only packages. To ensure the -compiler is configured as part of the test environment, set the -attribute ``tests_require_compiler = True`` on the package. The -compiler will be available through the canonical environment variables -(``CC``, ``CXX``, ``FC``, ``F77``) in the test environment. +was built, especially to test library-only packages. You must enable +loading the package's compiler configuration by setting the attribute +``test_requires_compiler`` to ``True``. Doing so makes the compiler +available in the test environment through the canonical environment +variables (``CC``, ``CXX``, ``FC``, ``F77``). + +""""""""""""""""""""""" +Adding build-time files +""""""""""""""""""""""" + +.. note:: + + We highly recommend the re-use of build-time tests and input files + for testing installed software. These files are easier to keep + synchronized with the software than creating custom install tests. -"""""""""""""""""""""""""""""""""""""""""""""""" -Install tests that require build-time components -"""""""""""""""""""""""""""""""""""""""""""""""" +You can use the ``cache_extra_test_sources`` method copy directories +and or files from the build stage directory to the package's installation +directory. -Some packages cannot be easily tested without components from the -build-time test suite. For those packages, the -``cache_extra_test_sources`` method can be used. +For example, a package method for copying everything in the ``tests`` +subdirectory plus the ``foo.c`` and ``bar.c`` files from ``examples`` +can be implemented as follows: .. code-block:: python @run_after('install') def cache_test_sources(self): - srcs = ['./tests/foo.c', './tests/bar.c'] + srcs = ['tests', 'examples/foo.c', 'examples/bar.c'] self.cache_extra_test_sources(srcs) -This method will copy the listed methods into the metadata directory -of the package at the end of the install phase of the build. They will -be available to the test method in the directory -``self._extra_tests_path``. +The use of the ``run_after`` directive ensures the associated files +are copied **after** the package is installed during the build process. -While source files are generally recommended, for many packages -binaries may also technically be cached in this way for later testing. +The method copies files to the package's metadata directory under +the ``self.install_test_root``. All files in the package source's +``tests`` directory for the example above will be copied to the +``join_path(self.install_test_root, 'tests')`` directory. The two +files listed under the staged ``examples`` directory will be copied +to the ``join_path(self.install_test_root, 'examples')`` directory. -""""""""""""""""""""" -Running install tests -""""""""""""""""""""" +.. note:: -Install tests can be run using the ``spack test run`` command. The -``spack test run`` command will create a ``test suite`` out of the -specs provided to it, or if no specs are provided it will test all -specs in the active environment, or all specs installed in Spack if no -environment is active. Test suites can be named using the ``--alias`` -option; test suites not aliased will use the content hash of their -specs as their name. + While source and input files are generally recommended, binaries + may also be cached by the build process for install testing. -Packages to install test can be queried using the ``spack test list`` -command, which outputs all installed packages with defined ``test`` -methods. +""""""""""""""""""" +Adding custom files +""""""""""""""""""" -Test suites can be found using the ``spack test find`` command. It -will list all test suites that have been run and have not been removed -using the ``spack test remove`` command. The ``spack test remove`` -command will remove tests to declutter the test stage. The ``spack -test results`` command will show results for completed test suites. +Some tests may require additional files not available from the build. +Examples include: -The test stage is the working directory for all install tests run with -Spack. By default, Spack uses ``~/.spack/test`` as the test stage. The -test stage can be set in the high-level config: +- test source files +- test input files +- test build scripts +- expected test output -.. code-block:: yaml +These extra files should be added to the ``test`` subdirectory of the +package in the repository. Spack will automaticaly copy any files in +that directory to the test staging directory during install testing. +The ``test`` method can access those files from the +``self.test_suite.current_test_data_dir`` directory. - config: - test_stage: /path/to/stage +.. _cmd-spack-test-list: + +""""""""""""""""""" +``spack test list`` +""""""""""""""""""" + +Packages available for install testing can be found using the +``spack test list`` command. The command outputs all installed +packages that have defined ``test`` methods. + +.. _cmd-spack-test-run: + +"""""""""""""""""" +``spack test run`` +"""""""""""""""""" + +Install tests can be run for one or more installed packages using +the ``spack test run`` command. A ``test suite`` is created from +the provided specs. If no specs are provided it will test all specs +in the active environment or all specs installed in Spack if no +environment is active. + +Test suites can be named using the ``--alias`` option. Unaliased +Test suites will use the content hash of their specs as their name. + +Some of the more commonly used debugging options are: + +- ``--fail-fast`` stops testing each package after the first failure +- ``--fail-first`` stops testing packages after the first failure + +Test output is written to a text log file by default but ``junit`` +and ``cdash`` are outputs are available through the ``--log-format`` +option. + +.. _cmd-spack-test-results: + +"""""""""""""""""""""" +``spack test results`` +"""""""""""""""""""""" + +The ``spack test results`` command shows results for all completed +test suites. Providing the alias or content hash limits reporting +to the corresponding test suite. + +The ``--logs`` option includes the output generated by the associated +test(s) to facilitate debugging. + +The ``--failed`` option limits results shown to that of the failed +tests, if any, of matching packages. + +.. _cmd-spack-test-find: + +""""""""""""""""""" +``spack test find`` +""""""""""""""""""" + +The ``spack test find`` command lists the aliases or content hashes +of all test suites whose results are available. + +.. _cmd-spack-test-remove: + +""""""""""""""""""""" +``spack test remove`` +""""""""""""""""""""" + +The ``spack test remove`` command removes test suites to declutter +the test results directory. You are prompted to confirm the removal +of each test suite **unless** you use the ``--yes-to-all`` option. + +.. _file-manipulation: --------------------------- File manipulation functions diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst index d0cb6fa670..c3c86e75c9 100644 --- a/lib/spack/docs/pipelines.rst +++ b/lib/spack/docs/pipelines.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -122,9 +122,26 @@ pipeline jobs. Concretizes the specs in the active environment, stages them (as described in :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. -This sub-command takes two arguments, but the most useful is ``--output-file``, -which should be an absolute path (including file name) to the generated -pipeline, if the default (``./.gitlab-ci.yml``) is not desired. +Using ``--prune-dag`` or ``--no-prune-dag`` configures whether or not jobs are +generated for specs that are already up to date on the mirror. If enabling +DAG pruning using ``--prune-dag``, more information may be required in your +``spack.yaml`` file, see the :ref:`noop_jobs` section below regarding +``service-job-attributes``. + +The ``--optimize`` argument is experimental and runs the generated pipeline +document through a series of optimization passes designed to reduce the size +of the generated file. + +The ``--dependencies`` is also experimental and disables what in Gitlab is +referred to as DAG scheduling, internally using the ``dependencies`` keyword +rather than ``needs`` to list dependency jobs. The drawback of using this option +is that before any job can begin, all jobs in previous stages must first +complete. The benefit is that Gitlab allows more dependencies to be listed +when using ``dependencies`` instead of ``needs``. + +The optional ``--output-file`` argument should be an absolute path (including +file name) to the generated pipeline, and if not given, the default is +``./.gitlab-ci.yml``. .. _cmd-spack-ci-rebuild: @@ -223,21 +240,6 @@ takes a boolean and determines whether the pipeline uses artifacts to store and pass along the buildcaches from one stage to the next (the default if you don't provide this option is ``False``). -The -``final-stage-rebuild-index`` section controls whether an extra job is added to the -end of your pipeline (in a stage by itself) which will regenerate the mirror's -buildcache index. Under normal operation, each pipeline job that rebuilds a package -will re-generate the mirror's buildcache index after the buildcache entry for that -job has been created and pushed to the mirror. Since jobs in the same stage can run in -parallel, there is the possibility that at the end of some stage, the index may not -reflect all the binaries in the buildcache. Adding the ``final-stage-rebuild-index`` -section ensures that at the end of the pipeline, the index will be in sync with the -binaries on the mirror. If the mirror lives in an S3 bucket, this job will need to -run on a machine with the Python ``boto3`` module installed, and consequently the -``final-stage-rebuild-index`` needs to specify a list of ``tags`` to pick a runner -satisfying that condition. It can also take an ``image`` key so Docker executor type -runners can pick the right image for the index regeneration job. - The optional ``cdash`` section provides information that will be used by the ``spack ci generate`` command (invoked by ``spack ci start``) for reporting to CDash. All the jobs generated from this environment will belong to a @@ -251,6 +253,76 @@ Take a look at the for the gitlab-ci section of the spack environment file, to see precisely what syntax is allowed there. +.. _rebuild_index: + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Note about rebuilding buildcache index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, while a pipeline job may rebuild a package, create a buildcache +entry, and push it to the mirror, it does not automatically re-generate the +mirror's buildcache index afterward. Because the index is not needed by the +default rebuild jobs in the pipeline, not updating the index at the end of +each job avoids possible race conditions between simultaneous jobs, and it +avoids the computational expense of regenerating the index. This potentially +saves minutes per job, depending on the number of binary packages in the +mirror. As a result, the default is that the mirror's buildcache index may +not correctly reflect the mirror's contents at the end of a pipeline. + +To make sure the buildcache index is up to date at the end of your pipeline, +spack generates a job to update the buildcache index of the target mirror +at the end of each pipeline by default. You can disable this behavior by +adding ``rebuild-index: False`` inside the ``gitlab-ci`` section of your +spack environment. Spack will assign the job any runner attributes found +on the ``service-job-attributes``, if you have provided that in your +``spack.yaml``. + +.. _noop_jobs: + +^^^^^^^^^^^^^^^^^^^^^^^ +Note about "no-op" jobs +^^^^^^^^^^^^^^^^^^^^^^^ + +If no specs in an environment need to be rebuilt during a given pipeline run +(meaning all are already up to date on the mirror), a single succesful job +(a NO-OP) is still generated to avoid an empty pipeline (which GitLab +considers to be an error). An optional ``service-job-attributes`` section +can be added to your ``spack.yaml`` where you can provide ``tags`` and +``image`` or ``variables`` for the generated NO-OP job. This section also +supports providing ``before_script``, ``script``, and ``after_script``, in +case you want to take some custom actions in the case of any empty pipeline. + +Following is an example of this section added to a ``spack.yaml``: + +.. code-block:: yaml + + spack: + specs: + - openmpi + mirrors: + cloud_gitlab: https://mirror.spack.io + gitlab-ci: + mappings: + - match: + - os=centos8 + runner-attributes: + tags: + - custom + - tag + image: spack/centos7 + service-job-attributes: + tags: ['custom', 'tag'] + image: + name: 'some.image.registry/custom-image:latest' + entrypoint: ['/bin/bash'] + script: + - echo "Custom message in a custom script" + +The example above illustrates how you can provide the attributes used to run +the NO-OP job in the case of an empty pipeline. The only field for the NO-OP +job that might be generated for you is ``script``, but that will only happen +if you do not provide one yourself. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Assignment of specs to runners ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 4a2c163886..3368c26a28 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,7 +9,7 @@ Package Repositories ============================= -Spack comes with over 1,000 built-in package recipes in +Spack comes with thousands of built-in package recipes in ``var/spack/repos/builtin/``. This is a **package repository** -- a directory that Spack searches when it needs to find a package by name. You may need to maintain packages for restricted, proprietary or diff --git a/lib/spack/docs/spack.yaml b/lib/spack/docs/spack.yaml index 3c8cd14d10..69d0e301fd 100644 --- a/lib/spack/docs/spack.yaml +++ b/lib/spack/docs/spack.yaml @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst index 4d3a97012c..c327c137ac 100644 --- a/lib/spack/docs/workflows.rst +++ b/lib/spack/docs/workflows.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 5efe015c9e..99e164dbc7 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -125,7 +125,7 @@ case "$command" in comp="FC" lang_flags=F ;; - f77|xlf|xlf_r|pgf77|frt|flang) + f77|xlf|xlf_r|pgf77) command="$SPACK_F77" language="Fortran 77" comp="F77" @@ -320,9 +320,13 @@ while [ -n "$1" ]; do if [ -z "$arg" ]; then shift; arg="$1"; fi if [[ "$arg" = -rpath=* ]]; then rp="${arg#-rpath=}" + elif [[ "$arg" = --rpath=* ]]; then + rp="${arg#--rpath=}" elif [[ "$arg" = -rpath,* ]]; then rp="${arg#-rpath,}" - elif [[ "$arg" = -rpath ]]; then + elif [[ "$arg" = --rpath,* ]]; then + rp="${arg#--rpath,}" + elif [[ "$arg" =~ ^-?-rpath$ ]]; then shift; arg="$1" if [[ "$arg" != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -339,7 +343,9 @@ while [ -n "$1" ]; do if [ -z "$arg" ]; then shift; arg="$1"; fi if [[ "$arg" = -rpath=* ]]; then rp="${arg#-rpath=}" - elif [[ "$arg" = -rpath ]]; then + elif [[ "$arg" = --rpath=* ]]; then + rp="${arg#--rpath=}" + elif [[ "$arg" = -rpath ]] || [[ "$arg" = --rpath ]]; then shift; arg="$1" if [[ "$arg" != -Xlinker,* ]]; then die "-Xlinker,-rpath was not followed by -Xlinker,*" @@ -434,7 +440,7 @@ then ld) flags=("${flags[@]}" -headerpad_max_install_names) ;; ccld) - flags=("${flags[@]}" -Wl,-headerpad_max_install_names) ;; + flags=("${flags[@]}" "-Wl,-headerpad_max_install_names") ;; esac fi @@ -491,19 +497,19 @@ args+=("${flags[@]}") # Insert include directories just prior to any system include directories for dir in "${includes[@]}"; do args+=("-I$dir"); done -for dir in "${isystem_includes[@]}"; do args+=("-isystem$dir"); done +for dir in "${isystem_includes[@]}"; do args+=("-isystem" "$dir"); done IFS=':' read -ra spack_include_dirs <<< "$SPACK_INCLUDE_DIRS" if [[ $mode == cpp || $mode == cc || $mode == as || $mode == ccld ]]; then if [[ "$isystem_was_used" == "true" ]] ; then - for dir in "${spack_include_dirs[@]}"; do args+=("-isystem$dir"); done + for dir in "${spack_include_dirs[@]}"; do args+=("-isystem" "$dir"); done else for dir in "${spack_include_dirs[@]}"; do args+=("-I$dir"); done fi fi for dir in "${system_includes[@]}"; do args+=("-I$dir"); done -for dir in "${isystem_system_includes[@]}"; do args+=("-isystem$dir"); done +for dir in "${isystem_system_includes[@]}"; do args+=("-isystem" "$dir"); done # Library search paths for dir in "${libdirs[@]}"; do args+=("-L$dir"); done @@ -512,12 +518,12 @@ for dir in "${system_libdirs[@]}"; do args+=("-L$dir"); done # RPATHs arguments case "$mode" in ccld) - if [ ! -z "$dtags_to_add" ] ; then args+=("$linker_arg$dtags_to_add") ; fi + if [ -n "$dtags_to_add" ] ; then args+=("$linker_arg$dtags_to_add") ; fi for dir in "${rpaths[@]}"; do args+=("$rpath$dir"); done for dir in "${system_rpaths[@]}"; do args+=("$rpath$dir"); done ;; ld) - if [ ! -z "$dtags_to_add" ] ; then args+=("$dtags_to_add") ; fi + if [ -n "$dtags_to_add" ] ; then args+=("$dtags_to_add") ; fi for dir in "${rpaths[@]}"; do args+=("-rpath" "$dir"); done for dir in "${system_rpaths[@]}"; do args+=("-rpath" "$dir"); done ;; diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 1c98852eb6..82c425846b 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,7 +11,7 @@ archspec * Homepage: https://pypi.python.org/pypi/archspec * Usage: Labeling, comparison and detection of microarchitectures -* Version: 0.1.2 (commit 2846749dc5b12ae2b30ff1d3f0270a4a5954710d) +* Version: 0.1.2 (commit 068b0ebd641211971acf10f39aa876703a34bae4) argparse -------- diff --git a/lib/spack/external/_pytest/assertion/util.py b/lib/spack/external/_pytest/assertion/util.py index 9f00929073..c09eff06b0 100644 --- a/lib/spack/external/_pytest/assertion/util.py +++ b/lib/spack/external/_pytest/assertion/util.py @@ -5,9 +5,12 @@ import pprint import _pytest._code import py try: - from collections import Sequence + from collections.abc import Sequence except ImportError: - Sequence = list + try: + from collections import Sequence + except ImportError: + Sequence = list u = py.builtin._totext diff --git a/lib/spack/external/_pytest/main.py b/lib/spack/external/_pytest/main.py index eacae8dab1..98aa28eb34 100644 --- a/lib/spack/external/_pytest/main.py +++ b/lib/spack/external/_pytest/main.py @@ -10,9 +10,12 @@ from _pytest import nodes import _pytest._code import py try: - from collections import MutableMapping as MappingMixin + from collections.abc import MutableMapping as MappingMixin except ImportError: - from UserDict import DictMixin as MappingMixin + try: + from collections import MutableMapping as MappingMixin + except ImportError: + from UserDict import DictMixin as MappingMixin from _pytest.config import directory_arg, UsageError, hookimpl from _pytest.outcomes import exit diff --git a/lib/spack/external/_pytest/python_api.py b/lib/spack/external/_pytest/python_api.py index cfc01193b0..a931b4d2c7 100644 --- a/lib/spack/external/_pytest/python_api.py +++ b/lib/spack/external/_pytest/python_api.py @@ -398,7 +398,10 @@ def approx(expected, rel=None, abs=None, nan_ok=False): __ https://docs.python.org/3/reference/datamodel.html#object.__ge__ """ - from collections import Mapping, Sequence + if sys.version_info >= (3, 3): + from collections.abc import Mapping, Sequence + else: + from collections import Mapping, Sequence from _pytest.compat import STRING_TYPES as String # Delegate the comparison to a class that knows how to deal with the type diff --git a/lib/spack/external/archspec/__init__.py b/lib/spack/external/archspec/__init__.py index 1349425634..1188c6cecc 100644 --- a/lib/spack/external/archspec/__init__.py +++ b/lib/spack/external/archspec/__init__.py @@ -1,2 +1,2 @@ """Init file to avoid namespace packages""" -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/lib/spack/external/archspec/json/cpu/microarchitectures.json b/lib/spack/external/archspec/json/cpu/microarchitectures.json index 889848c8ec..7fed4d7a52 100644 --- a/lib/spack/external/archspec/json/cpu/microarchitectures.json +++ b/lib/spack/external/archspec/json/cpu/microarchitectures.json @@ -75,6 +75,13 @@ "flags": "-march={name} -mtune=generic" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "x86-64", + "flags": "-march={name} -mtune=generic" + } + ], "intel": [ { "versions": ":", @@ -106,6 +113,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -143,6 +156,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -152,7 +171,7 @@ "intel": [ { "versions": "16.0:", - "flags": "-march={name} -mtune={name}}" + "flags": "-march={name} -mtune={name}" } ] } @@ -187,6 +206,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -229,6 +254,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -277,6 +308,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -331,6 +368,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -390,6 +433,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -446,6 +495,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -500,6 +555,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -558,6 +619,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "knl", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -621,6 +689,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "skylake-avx512", + "flags": "-march={name} -mtune=generic" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -685,6 +760,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "8.0:", @@ -746,6 +827,12 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "11.0:", @@ -827,6 +914,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "icelake-client", + "flags": "-march={name} -mtune={name}" + } + ], "apple-clang": [ { "versions": "10.0.1:", @@ -875,6 +969,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "amdfam10", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -918,6 +1019,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "bdver1", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -965,6 +1073,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "bdver2", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -1013,6 +1128,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "bdver3", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -1064,6 +1186,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "bdver4", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -1119,6 +1248,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "znver1", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", @@ -1175,6 +1311,13 @@ "flags": "-march={name} -mtune={name}" } ], + "aocc": [ + { + "versions": "2.2:", + "name": "znver2", + "flags": "-march={name} -mtune={name}" + } + ], "intel": [ { "versions": "16.0:", diff --git a/lib/spack/external/jinja2/runtime.py b/lib/spack/external/jinja2/runtime.py index f9d7a6806c..52dfeaebd6 100644 --- a/lib/spack/external/jinja2/runtime.py +++ b/lib/spack/external/jinja2/runtime.py @@ -315,10 +315,14 @@ class Context(with_metaclass(ContextMeta)): # register the context as mapping if possible try: - from collections import Mapping + from collections.abc import Mapping Mapping.register(Context) except ImportError: - pass + try: + from collections import Mapping + Mapping.register(Context) + except ImportError: + pass class BlockReference(object): diff --git a/lib/spack/external/jinja2/sandbox.py b/lib/spack/external/jinja2/sandbox.py index 93fb9d45f3..b9e5ec495a 100644 --- a/lib/spack/external/jinja2/sandbox.py +++ b/lib/spack/external/jinja2/sandbox.py @@ -14,7 +14,7 @@ """ import types import operator -from collections import Mapping +import sys from jinja2.environment import Environment from jinja2.exceptions import SecurityError from jinja2._compat import string_types, PY2 @@ -23,6 +23,11 @@ from jinja2.utils import Markup from markupsafe import EscapeFormatter from string import Formatter +if sys.version_info >= (3, 3): + from collections.abc import Mapping +else: + from collections import Mapping + #: maximum number of items a range may produce MAX_RANGE = 100000 @@ -79,7 +84,10 @@ except ImportError: pass #: register Python 2.6 abstract base classes -from collections import MutableSet, MutableMapping, MutableSequence +if sys.version_info >= (3, 3): + from collections.abc import MutableSet, MutableMapping, MutableSequence +else: + from collections import MutableSet, MutableMapping, MutableSequence _mutable_set_types += (MutableSet,) _mutable_mapping_types += (MutableMapping,) _mutable_sequence_types += (MutableSequence,) diff --git a/lib/spack/external/jinja2/tests.py b/lib/spack/external/jinja2/tests.py index 0adc3d4dbc..d5d6b5b33f 100644 --- a/lib/spack/external/jinja2/tests.py +++ b/lib/spack/external/jinja2/tests.py @@ -10,11 +10,16 @@ """ import operator import re -from collections import Mapping +import sys from jinja2.runtime import Undefined from jinja2._compat import text_type, string_types, integer_types import decimal +if sys.version_info >= (3, 3): + from collections.abc import Mapping +else: + from collections import Mapping + number_re = re.compile(r'^-?\d+(\.\d+)?$') regex_type = type(number_re) diff --git a/lib/spack/external/jinja2/utils.py b/lib/spack/external/jinja2/utils.py index 502a311c08..cff4e783a8 100644 --- a/lib/spack/external/jinja2/utils.py +++ b/lib/spack/external/jinja2/utils.py @@ -482,10 +482,14 @@ class LRUCache(object): # register the LRU cache as mutable mapping if possible try: - from collections import MutableMapping + from collections.abc import MutableMapping MutableMapping.register(LRUCache) except ImportError: - pass + try: + from collections import MutableMapping + MutableMapping.register(LRUCache) + except ImportError: + pass def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), diff --git a/lib/spack/external/markupsafe/__init__.py b/lib/spack/external/markupsafe/__init__.py index 68dc85f612..506326f450 100644 --- a/lib/spack/external/markupsafe/__init__.py +++ b/lib/spack/external/markupsafe/__init__.py @@ -10,10 +10,15 @@ """ import re import string -from collections import Mapping +import sys from markupsafe._compat import text_type, string_types, int_types, \ unichr, iteritems, PY2 +if sys.version_info >= (3, 3): + from collections.abc import Mapping +else: + from collections import Mapping + __version__ = "1.0" __all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] diff --git a/lib/spack/external/ordereddict_backport.py b/lib/spack/external/ordereddict_backport.py index 5ec8493cc9..3c7f012e9e 100644 --- a/lib/spack/external/ordereddict_backport.py +++ b/lib/spack/external/ordereddict_backport.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/external/py2/typing.py b/lib/spack/external/py2/typing.py new file mode 100644 index 0000000000..2b06797660 --- /dev/null +++ b/lib/spack/external/py2/typing.py @@ -0,0 +1,84 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +""" +This is a fake set of symbols to allow spack to import typing in python +versions where we do not support type checking (<3) +""" +Annotated = None +Any = None +Callable = None +ForwardRef = None +Generic = None +Literal = None +Optional = None +Tuple = None +TypeVar = None +Union = None +AbstractSet = None +ByteString = None +Container = None +Hashable = None +ItemsView = None +Iterable = None +Iterator = None +KeysView = None +Mapping = None +MappingView = None +MutableMapping = None +MutableSequence = None +MutableSet = None +Sequence = None +Sized = None +ValuesView = None +Awaitable = None +AsyncIterator = None +AsyncIterable = None +Coroutine = None +Collection = None +AsyncGenerator = None +AsyncContextManager = None +Reversible = None +SupportsAbs = None +SupportsBytes = None +SupportsComplex = None +SupportsFloat = None +SupportsInt = None +SupportsRound = None +ChainMap = None +Dict = None +List = None +OrderedDict = None +Set = None +FrozenSet = None +NamedTuple = None +Generator = None +AnyStr = None +cast = None +get_args = None +get_origin = None +get_type_hints = None +no_type_check = None +no_type_check_decorator = None +NoReturn = None + +# these are the typing extension symbols +ClassVar = None +Final = None +Protocol = None +Type = None +TypedDict = None +ContextManager = None +Counter = None +Deque = None +DefaultDict = None +SupportsIndex = None +final = None +IntVar = None +Literal = None +NewType = None +overload = None +runtime_checkable = None +Text = None +TYPE_CHECKING = None diff --git a/lib/spack/external/ruamel/yaml/comments.py b/lib/spack/external/ruamel/yaml/comments.py index b8a5010ad8..a517072087 100644 --- a/lib/spack/external/ruamel/yaml/comments.py +++ b/lib/spack/external/ruamel/yaml/comments.py @@ -9,7 +9,12 @@ these are not really related, formatting could be factored out as a separate base """ -from collections import MutableSet +import sys + +if sys.version_info >= (3, 3): + from collections.abc import MutableSet +else: + from collections import MutableSet __all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap", "CommentedSet", 'comment_attrib', 'merge_attrib'] diff --git a/lib/spack/external/ruamel/yaml/compat.py b/lib/spack/external/ruamel/yaml/compat.py index a1778bef28..28a981dc43 100644 --- a/lib/spack/external/ruamel/yaml/compat.py +++ b/lib/spack/external/ruamel/yaml/compat.py @@ -12,9 +12,12 @@ try: from ruamel.ordereddict import ordereddict except: try: - from collections import OrderedDict + from collections.abc import OrderedDict except ImportError: - from ordereddict import OrderedDict + try: + from collections import OrderedDict + except ImportError: + from ordereddict import OrderedDict # to get the right name import ... as ordereddict doesn't do that class ordereddict(OrderedDict): diff --git a/lib/spack/external/ruamel/yaml/constructor.py b/lib/spack/external/ruamel/yaml/constructor.py index f809df4bf9..69ad0a74ac 100644 --- a/lib/spack/external/ruamel/yaml/constructor.py +++ b/lib/spack/external/ruamel/yaml/constructor.py @@ -3,7 +3,6 @@ from __future__ import absolute_import from __future__ import print_function -import collections import datetime import base64 import binascii @@ -26,6 +25,12 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.scalarstring import * # NOQA +if sys.version_info >= (3, 3): + from collections.abc import Hashable +else: + from collections import Hashable + + __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor', 'ConstructorError', 'RoundTripConstructor'] @@ -163,7 +168,7 @@ class BaseConstructor(object): # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): if isinstance(key, list): key = tuple(key) if PY2: @@ -175,7 +180,7 @@ class BaseConstructor(object): "found unacceptable key (%s)" % exc, key_node.start_mark) else: - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) @@ -959,7 +964,7 @@ class RoundTripConstructor(SafeConstructor): # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): if isinstance(key, list): key = tuple(key) if PY2: @@ -971,7 +976,7 @@ class RoundTripConstructor(SafeConstructor): "found unacceptable key (%s)" % exc, key_node.start_mark) else: - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) @@ -1003,7 +1008,7 @@ class RoundTripConstructor(SafeConstructor): # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): if isinstance(key, list): key = tuple(key) if PY2: @@ -1015,7 +1020,7 @@ class RoundTripConstructor(SafeConstructor): "found unacceptable key (%s)" % exc, key_node.start_mark) else: - if not isinstance(key, collections.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) diff --git a/lib/spack/llnl/__init__.py b/lib/spack/llnl/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/llnl/__init__.py +++ b/lib/spack/llnl/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/__init__.py b/lib/spack/llnl/util/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/llnl/util/__init__.py +++ b/lib/spack/llnl/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py index 8ecf6acc88..9b90096460 100644 --- a/lib/spack/llnl/util/argparsewriter.py +++ b/lib/spack/llnl/util/argparsewriter.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index d6579555ad..c7ecf09af2 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,6 +23,12 @@ from llnl.util import tty from llnl.util.lang import dedupe, memoized from spack.util.executable import Executable + +if sys.version_info >= (3, 3): + from collections.abc import Sequence # novm +else: + from collections import Sequence + __all__ = [ 'FileFilter', 'FileList', @@ -63,7 +69,8 @@ __all__ = [ 'touchp', 'traverse_tree', 'unset_executable_mode', - 'working_dir' + 'working_dir', + 'keep_modification_time' ] @@ -1105,7 +1112,7 @@ def find(root, files, recursive=True): Parameters: root (str): The root directory to start searching from - files (str or collections.Sequence): Library name(s) to search for + files (str or Sequence): Library name(s) to search for recurse (bool, optional): if False search only root folder, if True descends top-down from the root. Defaults to True. @@ -1168,7 +1175,7 @@ def _find_non_recursive(root, search_files): # Utilities for libraries and headers -class FileList(collections.Sequence): +class FileList(Sequence): """Sequence of absolute paths to files. Provides a few convenience methods to manipulate file paths. @@ -1411,7 +1418,7 @@ def find_headers(headers, root, recursive=False): """ if isinstance(headers, six.string_types): headers = [headers] - elif not isinstance(headers, collections.Sequence): + elif not isinstance(headers, Sequence): message = '{0} expects a string or sequence of strings as the ' message += 'first argument [got {1} instead]' message = message.format(find_headers.__name__, type(headers)) @@ -1566,7 +1573,7 @@ def find_system_libraries(libraries, shared=True): """ if isinstance(libraries, six.string_types): libraries = [libraries] - elif not isinstance(libraries, collections.Sequence): + elif not isinstance(libraries, Sequence): message = '{0} expects a string or sequence of strings as the ' message += 'first argument [got {1} instead]' message = message.format(find_system_libraries.__name__, @@ -1620,7 +1627,7 @@ def find_libraries(libraries, root, shared=True, recursive=False): """ if isinstance(libraries, six.string_types): libraries = [libraries] - elif not isinstance(libraries, collections.Sequence): + elif not isinstance(libraries, Sequence): message = '{0} expects a string or sequence of strings as the ' message += 'first argument [got {1} instead]' message = message.format(find_libraries.__name__, type(libraries)) @@ -1819,3 +1826,24 @@ def remove_directory_contents(dir): os.unlink(entry) else: shutil.rmtree(entry) + + +@contextmanager +def keep_modification_time(*filenames): + """ + Context manager to keep the modification timestamps of the input files. + Tolerates and has no effect on non-existent files and files that are + deleted by the nested code. + + Parameters: + *filenames: one or more files that must have their modification + timestamps unchanged + """ + mtimes = {} + for f in filenames: + if os.path.exists(f): + mtimes[f] = os.path.getmtime(f) + yield + for f, mtime in mtimes.items(): + if os.path.exists(f): + os.utime(f, (os.path.getatime(f), mtime)) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 8d7d5e0767..17c1fe8427 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,13 +9,18 @@ import multiprocessing import os import re import functools -import collections import inspect from datetime import datetime, timedelta from six import string_types import sys +if sys.version_info >= (3, 3): + from collections.abc import Hashable, MutableMapping # novm +else: + from collections import Hashable, MutableMapping + + # Ignore emacs backups when listing modules ignore_modules = [r'^\.#', '~$'] @@ -189,7 +194,7 @@ def memoized(func): @functools.wraps(func) def _memoized_function(*args): - if not isinstance(args, collections.Hashable): + if not isinstance(args, Hashable): # Not hashable, so just call the function. return func(*args) @@ -264,7 +269,7 @@ def key_ordering(cls): @key_ordering -class HashableMap(collections.MutableMapping): +class HashableMap(MutableMapping): """This is a hashable, comparable dictionary. Hash is performed on a tuple of the values in the dictionary.""" diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index d29b1289e3..c286671401 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 5fd7163e2e..968e07dbb6 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py index f87b32c307..c73ebaed20 100644 --- a/lib/spack/llnl/util/multiproc.py +++ b/lib/spack/llnl/util/multiproc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 79ca5a9929..cf70834e8d 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index e0b4a5fa05..56baeb5bfb 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index b6f5ec782f..ffd1b8c497 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 658de5c596..ef14baeacf 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -20,12 +20,17 @@ from contextlib import contextmanager from six import string_types from six import StringIO +from typing import Optional # novm +from types import ModuleType # novm + import llnl.util.tty as tty +termios = None # type: Optional[ModuleType] try: - import termios + import termios as term_mod + termios = term_mod except ImportError: - termios = None + pass # Use this to strip escape sequences diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py index 84c272a6e2..2a5fb7afa4 100644 --- a/lib/spack/llnl/util/tty/pty.py +++ b/lib/spack/llnl/util/tty/pty.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 986583d270..da796e2100 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c0930fe2d0..6341df857d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -56,6 +56,7 @@ set. The user can set the front-end and back-end operating setting by the class attributes front_os and back_os. The operating system as described earlier, will be responsible for compiler detection. """ +import contextlib import functools import inspect import warnings @@ -67,6 +68,8 @@ import llnl.util.tty as tty from llnl.util.lang import memoized, list_modules, key_ordering import spack.compiler +import spack.compilers +import spack.config import spack.paths import spack.error as serr import spack.util.executable @@ -235,18 +238,19 @@ class Platform(object): Will return a instance of it once it is returned. """ - priority = None # Subclass sets number. Controls detection order + # Subclass sets number. Controls detection order + priority = None # type: int #: binary formats used on this platform; used by relocation logic binary_formats = ['elf'] - front_end = None - back_end = None - default = None # The default back end target. + front_end = None # type: str + back_end = None # type: str + default = None # type: str # The default back end target. - front_os = None - back_os = None - default_os = None + front_os = None # type: str + back_os = None # type: str + default_os = None # type: str reserved_targets = ['default_target', 'frontend', 'fe', 'backend', 'be'] reserved_oss = ['default_os', 'frontend', 'fe', 'backend', 'be'] @@ -491,7 +495,7 @@ def arch_for_spec(arch_spec): @memoized -def all_platforms(): +def _all_platforms(): classes = [] mod_path = spack.paths.platform_path parent_module = "spack.platforms" @@ -512,7 +516,7 @@ def all_platforms(): @memoized -def platform(): +def _platform(): """Detects the platform for this machine. Gather a list of all available subclasses of platforms. @@ -521,7 +525,7 @@ def platform(): a file path (/opt/cray...) """ # Try to create a Platform object using the config file FIRST - platform_list = all_platforms() + platform_list = _all_platforms() platform_list.sort(key=lambda a: a.priority) for platform_cls in platform_list: @@ -529,6 +533,19 @@ def platform(): return platform_cls() +#: The "real" platform of the host running Spack. This should not be changed +#: by any method and is here as a convenient way to refer to the host platform. +real_platform = _platform + +#: The current platform used by Spack. May be swapped by the use_platform +#: context manager. +platform = _platform + +#: The list of all platform classes. May be swapped by the use_platform +#: context manager. +all_platforms = _all_platforms + + @memoized def default_arch(): """Default ``Arch`` object for this machine. @@ -563,3 +580,39 @@ def compatible_sys_types(): arch = Arch(platform(), 'default_os', target) compatible_archs.append(str(arch)) return compatible_archs + + +class _PickleableCallable(object): + """Class used to pickle a callable that may substitute either + _platform or _all_platforms. Lambda or nested functions are + not pickleable. + """ + def __init__(self, return_value): + self.return_value = return_value + + def __call__(self): + return self.return_value + + +@contextlib.contextmanager +def use_platform(new_platform): + global platform, all_platforms + + msg = '"{0}" must be an instance of Platform' + assert isinstance(new_platform, Platform), msg.format(new_platform) + + original_platform_fn, original_all_platforms_fn = platform, all_platforms + platform = _PickleableCallable(new_platform) + all_platforms = _PickleableCallable([type(new_platform)]) + + # Clear configuration and compiler caches + spack.config.config.clear_caches() + spack.compilers._cache_config_files = [] + + yield new_platform + + platform, all_platforms = original_platform_fn, original_all_platforms_fn + + # Clear configuration and compiler caches + spack.config.config.clear_caches() + spack.compilers._cache_config_files = [] diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 00fffdfe51..99dfcd9292 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -763,6 +763,10 @@ def generate_package_index(cache_prefix): url_util.join(cache_prefix, 'index.json.hash'), keep_original=False, extra_args={'ContentType': 'text/plain'}) + except Exception as err: + msg = 'Encountered problem pushing package index to {0}: {1}'.format( + cache_prefix, err) + tty.warn(msg) finally: shutil.rmtree(tmpdir) @@ -823,6 +827,10 @@ def generate_key_index(key_prefix, tmpdir=None): url_util.join(key_prefix, 'index.json'), keep_original=False, extra_args={'ContentType': 'application/json'}) + except Exception as err: + msg = 'Encountered problem pushing key index to {0}: {1}'.format( + key_prefix, err) + tty.warn(msg) finally: if remove_tmpdir: shutil.rmtree(tmpdir) @@ -1081,6 +1089,8 @@ def relocate_package(spec, allow_root): """ Relocate the given package """ + import spack.hooks.sbang as sbang + workdir = str(spec.prefix) buildinfo = read_buildinfo_file(workdir) new_layout_root = str(spack.store.layout.root) @@ -1119,9 +1129,7 @@ def relocate_package(spec, allow_root): prefix_to_prefix_bin = OrderedDict({}) if old_sbang_install_path: - import spack.hooks.sbang as sbang - prefix_to_prefix_text[old_sbang_install_path] = \ - sbang.sbang_install_path() + prefix_to_prefix_text[old_sbang_install_path] = sbang.sbang_install_path() prefix_to_prefix_text[old_prefix] = new_prefix prefix_to_prefix_bin[old_prefix] = new_prefix @@ -1134,7 +1142,6 @@ def relocate_package(spec, allow_root): # sbang was a bash script, and it lived in the spack prefix. It is # now a POSIX script that lives in the install prefix. Old packages # will have the old sbang location in their shebangs. - import spack.hooks.sbang as sbang orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(old_spack_prefix) new_sbang = sbang.sbang_shebang_line() prefix_to_prefix_text[orig_sbang] = new_sbang @@ -1153,7 +1160,7 @@ def relocate_package(spec, allow_root): if not is_backup_file(text_name): text_names.append(text_name) -# If we are not installing back to the same install tree do the relocation + # If we are not installing back to the same install tree do the relocation if old_layout_root != new_layout_root: files_to_relocate = [os.path.join(workdir, filename) for filename in buildinfo.get('relocate_binaries') @@ -1323,7 +1330,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, os.remove(filename) -def try_direct_fetch(spec, force=False, full_hash_match=False, mirrors=None): +def try_direct_fetch(spec, full_hash_match=False, mirrors=None): """ Try to find the spec directly on the configured mirrors """ @@ -1361,11 +1368,26 @@ def try_direct_fetch(spec, force=False, full_hash_match=False, mirrors=None): return found_specs -def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False, - mirrors_to_check=None): +def get_mirrors_for_spec(spec=None, full_hash_match=False, + mirrors_to_check=None, index_only=False): """ Check if concrete spec exists on mirrors and return a list indicating the mirrors on which it can be found + + Args: + spec (Spec): The spec to look for in binary mirrors + full_hash_match (bool): If True, only includes mirrors where the spec + full hash matches the locally computed full hash of the ``spec`` + argument. If False, any mirror which has a matching DAG hash + is included in the results. + mirrors_to_check (dict): Optionally override the configured mirrors + with the mirrors in this dictionary. + index_only (bool): Do not attempt direct fetching of ``spec.yaml`` + files from remote mirrors, only consider the indices. + + Return: + A list of objects, each containing a ``mirror_url`` and ``spec`` key + indicating all mirrors where the spec can be found. """ if spec is None: return [] @@ -1391,10 +1413,9 @@ def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False, results = filter_candidates(candidates) # Maybe we just didn't have the latest information from the mirror, so - # try to fetch directly. - if not results: + # try to fetch directly, unless we are only considering the indices. + if not results and not index_only: results = try_direct_fetch(spec, - force=force, full_hash_match=full_hash_match, mirrors=mirrors_to_check) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 48ce594a4b..f7fd76d82a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -149,12 +149,17 @@ def clean_environment(): # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') - env.unset('LIBRARY_PATH') - env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') env.unset('DYLD_FALLBACK_LIBRARY_PATH') + # These vars affect how the compiler finds libraries and include dirs. + env.unset('LIBRARY_PATH') + env.unset('CPATH') + env.unset('C_INCLUDE_PATH') + env.unset('CPLUS_INCLUDE_PATH') + env.unset('OBJC_INCLUDE_PATH') + # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid # interference with Spack dependencies. # CNL requires these variables to be set (or at least some of them, @@ -412,8 +417,7 @@ def set_build_environment_variables(pkg, env, dirty): # directory. Add that to the path too. env_paths = [] compiler_specific = os.path.join( - spack.paths.build_env_path, - os.path.dirname(pkg.compiler.link_paths['cc'])) + spack.paths.build_env_path, os.path.dirname(pkg.compiler.link_paths['cc'])) for item in [spack.paths.build_env_path, compiler_specific]: env_paths.append(item) ci = os.path.join(item, 'case-insensitive') diff --git a/lib/spack/spack/build_systems/__init__.py b/lib/spack/spack/build_systems/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/build_systems/__init__.py +++ b/lib/spack/spack/build_systems/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/aspell_dict.py b/lib/spack/spack/build_systems/aspell_dict.py index 7da4d2f10c..55ea78145a 100644 --- a/lib/spack/spack/build_systems/aspell_dict.py +++ b/lib/spack/spack/build_systems/aspell_dict.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index ee9fb6884e..6b47e3edea 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import os import os.path from subprocess import PIPE from subprocess import check_call +from typing import List # novm import llnl.util.tty as tty import llnl.util.filesystem as fs @@ -61,7 +62,7 @@ class AutotoolsPackage(PackageBase): #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.build` #: phase - build_targets = [] + build_targets = [] # type: List[str] #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.install` #: phase install_targets = ['install'] @@ -75,7 +76,7 @@ class AutotoolsPackage(PackageBase): #: Set to true to force the autoreconf step even if configure is present force_autoreconf = False #: Options to be passed to autoreconf when using the default implementation - autoreconf_extra_args = [] + autoreconf_extra_args = [] # type: List[str] #: If False deletes all the .la files in the prefix folder #: after the installation. If True instead it installs them. diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py index 1336069846..61b3a3cb57 100644 --- a/lib/spack/spack/build_systems/cmake.py +++ b/lib/spack/spack/build_systems/cmake.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import inspect import os import platform import re +from typing import List # novm import spack.build_environment from llnl.util.filesystem import working_dir @@ -74,7 +75,7 @@ class CMakePackage(PackageBase): #: system base class build_system_class = 'CMakePackage' - build_targets = [] + build_targets = [] # type: List[str] install_targets = ['install'] build_time_test_callbacks = ['check'] diff --git a/lib/spack/spack/build_systems/cuda.py b/lib/spack/spack/build_systems/cuda.py index 61007431a4..25b5381900 100644 --- a/lib/spack/spack/build_systems/cuda.py +++ b/lib/spack/spack/build_systems/cuda.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/gnu.py b/lib/spack/spack/build_systems/gnu.py index c5b5e74bb9..6f5771214e 100644 --- a/lib/spack/spack/build_systems/gnu.py +++ b/lib/spack/spack/build_systems/gnu.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/intel.py b/lib/spack/spack/build_systems/intel.py index 0e0bb9378b..18628b8910 100644 --- a/lib/spack/spack/build_systems/intel.py +++ b/lib/spack/spack/build_systems/intel.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -815,6 +815,7 @@ class IntelPackage(PackageBase): # Was supported only up to 2015. blacs_lib = 'libmkl_blacs' elif ('^mpich@2:' in spec_root or + '^cray-mpich' in spec_root or '^mvapich2' in spec_root or '^intel-mpi' in spec_root or '^intel-parallel-studio' in spec_root): diff --git a/lib/spack/spack/build_systems/makefile.py b/lib/spack/spack/build_systems/makefile.py index 6cd05c7ad9..fbc415e0ed 100644 --- a/lib/spack/spack/build_systems/makefile.py +++ b/lib/spack/spack/build_systems/makefile.py @@ -1,10 +1,11 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import inspect +from typing import List # novm import llnl.util.tty as tty from llnl.util.filesystem import working_dir @@ -48,7 +49,7 @@ class MakefilePackage(PackageBase): #: Targets for ``make`` during the :py:meth:`~.MakefilePackage.build` #: phase - build_targets = [] + build_targets = [] # type: List[str] #: Targets for ``make`` during the :py:meth:`~.MakefilePackage.install` #: phase install_targets = ['install'] diff --git a/lib/spack/spack/build_systems/maven.py b/lib/spack/spack/build_systems/maven.py index e9df34e791..7a5caabd29 100644 --- a/lib/spack/spack/build_systems/maven.py +++ b/lib/spack/spack/build_systems/maven.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/meson.py b/lib/spack/spack/build_systems/meson.py index 825b4c98c3..33ed756786 100644 --- a/lib/spack/spack/build_systems/meson.py +++ b/lib/spack/spack/build_systems/meson.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ import inspect import os +from typing import List # novm from llnl.util.filesystem import working_dir from spack.directives import depends_on, variant @@ -46,7 +47,7 @@ class MesonPackage(PackageBase): #: system base class build_system_class = 'MesonPackage' - build_targets = [] + build_targets = [] # type: List[str] install_targets = ['install'] build_time_test_callbacks = ['check'] diff --git a/lib/spack/spack/build_systems/octave.py b/lib/spack/spack/build_systems/octave.py index 16d6a0a1b8..bedf717811 100644 --- a/lib/spack/spack/build_systems/octave.py +++ b/lib/spack/spack/build_systems/octave.py @@ -1,11 +1,11 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import inspect -from spack.directives import depends_on, extends +from spack.directives import extends from spack.package import PackageBase, run_after @@ -27,7 +27,6 @@ class OctavePackage(PackageBase): build_system_class = 'OctavePackage' extends('octave') - depends_on('octave', type=('build', 'run')) def setup_build_environment(self, env): # octave does not like those environment variables to be set: diff --git a/lib/spack/spack/build_systems/oneapi.py b/lib/spack/spack/build_systems/oneapi.py index ec8732bbd6..5babdc5a75 100644 --- a/lib/spack/spack/build_systems/oneapi.py +++ b/lib/spack/spack/build_systems/oneapi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/perl.py b/lib/spack/spack/build_systems/perl.py index d134dc06f2..a805c51176 100644 --- a/lib/spack/spack/build_systems/perl.py +++ b/lib/spack/spack/build_systems/perl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,7 +7,7 @@ import inspect import os -from spack.directives import depends_on, extends +from spack.directives import extends from spack.package import PackageBase, run_after from spack.util.executable import Executable from llnl.util.filesystem import filter_file @@ -45,8 +45,6 @@ class PerlPackage(PackageBase): extends('perl') - depends_on('perl', type=('build', 'run')) - def configure_args(self): """Produces a list containing the arguments that must be passed to :py:meth:`~.PerlPackage.configure`. Arguments should not include diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 76159d88a1..c3ad02bb2f 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,12 +6,13 @@ import inspect import os import shutil -from spack.directives import depends_on, extends +from spack.directives import extends from spack.package import PackageBase, run_after from llnl.util.filesystem import (working_dir, get_filetype, filter_file, - path_contains_subdirectory, same_path) + path_contains_subdirectory, same_path, find) from llnl.util.lang import match_predicate +import llnl.util.tty as tty class PythonPackage(PackageBase): @@ -25,20 +26,11 @@ class PythonPackage(PackageBase): * build_ext * build_clib * build_scripts - * clean * install * install_lib * install_headers * install_scripts * install_data - * sdist - * register - * bdist - * bdist_dumb - * bdist_rpm - * bdist_wininst - * upload - * check These are all standard setup.py commands and can be found by running: @@ -71,35 +63,84 @@ class PythonPackage(PackageBase): def configure(self, spec, prefix): self.setup_py('configure') """ + #: Package name, version, and extension on PyPI + pypi = None + + maintainers = ['adamjstewart'] + # Default phases phases = ['build', 'install'] - # Name of modules that the Python package provides - # This is used to test whether or not the installation succeeded - # These names generally come from running: - # - # >>> import setuptools - # >>> setuptools.find_packages() - # - # in the source tarball directory - import_modules = [] - # To be used in UI queries that require to know which # build-system class we are using build_system_class = 'PythonPackage' - #: Callback names for build-time test - build_time_test_callbacks = ['build_test'] - #: Callback names for install-time test - install_time_test_callbacks = ['import_module_test'] + install_time_test_callbacks = ['test'] extends('python') - depends_on('python', type=('build', 'run')) - py_namespace = None + @property + def homepage(self): + if self.pypi: + name = self.pypi.split('/')[0] + return 'https://pypi.org/project/' + name + '/' + + @property + def url(self): + if self.pypi: + return ( + 'https://files.pythonhosted.org/packages/source/' + + self.pypi[0] + '/' + self.pypi + ) + + @property + def list_url(self): + if self.pypi: + name = self.pypi.split('/')[0] + return 'https://pypi.org/simple/' + name + '/' + + @property + def import_modules(self): + """Names of modules that the Python package provides. + + These are used to test whether or not the installation succeeded. + These names generally come from running: + + .. code-block:: python + + >> import setuptools + >> setuptools.find_packages() + + in the source tarball directory. If the module names are incorrectly + detected, this property can be overridden by the package. + + Returns: + list: list of strings of module names + """ + modules = [] + + # Python libraries may be installed in lib or lib64 + # See issues #18520 and #17126 + for lib in ['lib', 'lib64']: + root = os.path.join(self.prefix, lib, 'python{0}'.format( + self.spec['python'].version.up_to(2)), 'site-packages') + # Some Python libraries are packages: collections of modules + # distributed in directories containing __init__.py files + for path in find(root, '__init__.py', recursive=True): + modules.append(path.replace(root + os.sep, '', 1).replace( + os.sep + '__init__.py', '').replace('/', '.')) + # Some Python libraries are modules: individual *.py files + # found in the site-packages directory + for path in find(root, '*.py', recursive=False): + modules.append(path.replace(root + os.sep, '', 1).replace( + '.py', '').replace('/', '.')) + + tty.debug('Detected the following modules: {0}'.format(modules)) + return modules + def setup_file(self): """Returns the name of the setup file to use.""" return 'setup.py' @@ -118,27 +159,6 @@ class PythonPackage(PackageBase): with working_dir(self.build_directory): self.python('-s', setup, '--no-user-cfg', *args, **kwargs) - def _setup_command_available(self, command): - """Determines whether or not a setup.py command exists. - - Args: - command (str): The command to look for - - Returns: - bool: True if the command is found, else False - """ - kwargs = { - 'output': os.devnull, - 'error': os.devnull, - 'fail_on_error': False - } - - python = inspect.getmodule(self).python - setup = self.setup_file() - - python('-s', setup, '--no-user-cfg', command, '--help', **kwargs) - return python.returncode == 0 - # The following phases and their descriptions come from: # $ python setup.py --help-commands @@ -194,16 +214,6 @@ class PythonPackage(PackageBase): """Arguments to pass to build_scripts.""" return [] - def clean(self, spec, prefix): - """Clean up temporary files from 'build' command.""" - args = self.clean_args(spec, prefix) - - self.setup_py('clean', *args) - - def clean_args(self, spec, prefix): - """Arguments to pass to clean.""" - return [] - def install(self, spec, prefix): """Install everything from build directory.""" args = self.install_args(spec, prefix) @@ -277,115 +287,18 @@ class PythonPackage(PackageBase): """Arguments to pass to install_data.""" return [] - def sdist(self, spec, prefix): - """Create a source distribution (tarball, zip file, etc.).""" - args = self.sdist_args(spec, prefix) - - self.setup_py('sdist', *args) - - def sdist_args(self, spec, prefix): - """Arguments to pass to sdist.""" - return [] - - def register(self, spec, prefix): - """Register the distribution with the Python package index.""" - args = self.register_args(spec, prefix) - - self.setup_py('register', *args) - - def register_args(self, spec, prefix): - """Arguments to pass to register.""" - return [] - - def bdist(self, spec, prefix): - """Create a built (binary) distribution.""" - args = self.bdist_args(spec, prefix) - - self.setup_py('bdist', *args) - - def bdist_args(self, spec, prefix): - """Arguments to pass to bdist.""" - return [] - - def bdist_dumb(self, spec, prefix): - '''Create a "dumb" built distribution.''' - args = self.bdist_dumb_args(spec, prefix) - - self.setup_py('bdist_dumb', *args) - - def bdist_dumb_args(self, spec, prefix): - """Arguments to pass to bdist_dumb.""" - return [] - - def bdist_rpm(self, spec, prefix): - """Create an RPM distribution.""" - args = self.bdist_rpm(spec, prefix) - - self.setup_py('bdist_rpm', *args) - - def bdist_rpm_args(self, spec, prefix): - """Arguments to pass to bdist_rpm.""" - return [] - - def bdist_wininst(self, spec, prefix): - """Create an executable installer for MS Windows.""" - args = self.bdist_wininst_args(spec, prefix) - - self.setup_py('bdist_wininst', *args) - - def bdist_wininst_args(self, spec, prefix): - """Arguments to pass to bdist_wininst.""" - return [] - - def upload(self, spec, prefix): - """Upload binary package to PyPI.""" - args = self.upload_args(spec, prefix) - - self.setup_py('upload', *args) - - def upload_args(self, spec, prefix): - """Arguments to pass to upload.""" - return [] - - def check(self, spec, prefix): - """Perform some checks on the package.""" - args = self.check_args(spec, prefix) - - self.setup_py('check', *args) - - def check_args(self, spec, prefix): - """Arguments to pass to check.""" - return [] - # Testing - def build_test(self): - """Run unit tests after in-place build. - - These tests are only run if the package actually has a 'test' command. - """ - if self._setup_command_available('test'): - args = self.test_args(self.spec, self.prefix) - - self.setup_py('test', *args) - - def test_args(self, spec, prefix): - """Arguments to pass to test.""" - return [] - - run_after('build')(PackageBase._run_default_build_time_test_callbacks) - - def import_module_test(self): - """Attempts to import the module that was just installed. - - This test is only run if the package overrides - :py:attr:`import_modules` with a list of module names.""" + def test(self): + """Attempts to import modules of the installed package.""" # Make sure we are importing the installed modules, - # not the ones in the current directory - with working_dir('spack-test', create=True): - for module in self.import_modules: - self.python('-c', 'import {0}'.format(module)) + # not the ones in the source directory + for module in self.import_modules: + self.run_test(inspect.getmodule(self).python.path, + ['-c', 'import {0}'.format(module)], + purpose='checking import of {0}'.format(module), + work_dir='spack-test') run_after('install')(PackageBase._run_default_install_time_test_callbacks) diff --git a/lib/spack/spack/build_systems/qmake.py b/lib/spack/spack/build_systems/qmake.py index 22914d4d3a..5ddb8d7fa3 100644 --- a/lib/spack/spack/build_systems/qmake.py +++ b/lib/spack/spack/build_systems/qmake.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ import inspect +from llnl.util.filesystem import working_dir from spack.directives import depends_on from spack.package import PackageBase, run_after @@ -37,6 +38,11 @@ class QMakePackage(PackageBase): depends_on('qt', type='build') + @property + def build_directory(self): + """The directory containing the ``*.pro`` file.""" + return self.stage.source_path + def qmake_args(self): """Produces a list containing all the arguments that must be passed to qmake @@ -45,22 +51,30 @@ class QMakePackage(PackageBase): def qmake(self, spec, prefix): """Run ``qmake`` to configure the project and generate a Makefile.""" - inspect.getmodule(self).qmake(*self.qmake_args()) + + with working_dir(self.build_directory): + inspect.getmodule(self).qmake(*self.qmake_args()) def build(self, spec, prefix): """Make the build targets""" - inspect.getmodule(self).make() + + with working_dir(self.build_directory): + inspect.getmodule(self).make() def install(self, spec, prefix): """Make the install targets""" - inspect.getmodule(self).make('install') + + with working_dir(self.build_directory): + inspect.getmodule(self).make('install') # Tests def check(self): """Searches the Makefile for a ``check:`` target and runs it if found. """ - self._if_make_target_execute('check') + + with working_dir(self.build_directory): + self._if_make_target_execute('check') run_after('build')(PackageBase._run_default_build_time_test_callbacks) diff --git a/lib/spack/spack/build_systems/r.py b/lib/spack/spack/build_systems/r.py index 6ee2668833..f713706882 100644 --- a/lib/spack/spack/build_systems/r.py +++ b/lib/spack/spack/build_systems/r.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,7 +6,7 @@ import inspect -from spack.directives import depends_on, extends +from spack.directives import extends from spack.package import PackageBase, run_after @@ -25,14 +25,14 @@ class RPackage(PackageBase): """ phases = ['install'] + maintainers = ['glennpj'] + #: This attribute is used in UI queries that need to know the build #: system base class build_system_class = 'RPackage' extends('r') - depends_on('r', type=('build', 'run')) - def configure_args(self): """Arguments to pass to install via ``--configure-args``.""" return [] diff --git a/lib/spack/spack/build_systems/rocm.py b/lib/spack/spack/build_systems/rocm.py index 0107c6376b..21a200e159 100644 --- a/lib/spack/spack/build_systems/rocm.py +++ b/lib/spack/spack/build_systems/rocm.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,6 +30,8 @@ # environment: {} # extra_rpaths: [] # +# It is advisable to replace /rocm/ in the paths above with /rocm-version/ +# and introduce spec version numbers to ensure reproducible results. # # 2. hip and its dependencies are currently NOT picked up by spack # automatically, and should therefore be added to packages.yaml by hand: @@ -37,7 +39,7 @@ # in packages.yaml: # hip: # externals: -# - spec: hip@3.8.20371-d1886b0b +# - spec: hip # prefix: /opt/rocm/hip # extra_attributes: # compilers: @@ -64,6 +66,9 @@ # cxx: /opt/rocm/llvm/bin/clang++ # buildable: false # +# It is advisable to replace /rocm/ in the paths above with /rocm-version/ +# and introduce spec version numbers to ensure reproducible results. +# # 3. In part 2, DO NOT list the path to hsa as /opt/rocm/hsa ! You want spack # to find hsa in /opt/rocm/include/hsa/hsa.h . The directory of # /opt/rocm/hsa also has an hsa.h file, but it won't be found because spack @@ -102,6 +107,8 @@ class ROCmPackage(PackageBase): depends_on('hsa-rocr-dev', when='+rocm') depends_on('hip', when='+rocm') + conflicts('^blt@:0.3.6', when='+rocm') + # need amd gpu type for rocm builds conflicts('amdgpu_target=none', when='+rocm') diff --git a/lib/spack/spack/build_systems/ruby.py b/lib/spack/spack/build_systems/ruby.py index 32d6633164..5736d0913a 100644 --- a/lib/spack/spack/build_systems/ruby.py +++ b/lib/spack/spack/build_systems/ruby.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,7 +6,7 @@ import glob import inspect -from spack.directives import depends_on, extends +from spack.directives import extends from spack.package import PackageBase, run_after @@ -27,8 +27,6 @@ class RubyPackage(PackageBase): extends('ruby') - depends_on('ruby', type=('build', 'run')) - def build(self, spec, prefix): """Build a Ruby gem.""" diff --git a/lib/spack/spack/build_systems/scons.py b/lib/spack/spack/build_systems/scons.py index 5e17666b71..56ee08a113 100644 --- a/lib/spack/spack/build_systems/scons.py +++ b/lib/spack/spack/build_systems/scons.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/sip.py b/lib/spack/spack/build_systems/sip.py index f814ef1837..47713a100a 100644 --- a/lib/spack/spack/build_systems/sip.py +++ b/lib/spack/spack/build_systems/sip.py @@ -1,14 +1,15 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import inspect +import os -from llnl.util.filesystem import working_dir, join_path +from llnl.util.filesystem import find, working_dir, join_path from spack.directives import depends_on, extends from spack.package import PackageBase, run_after -import os +import llnl.util.tty as tty class SIPPackage(PackageBase): @@ -36,13 +37,52 @@ class SIPPackage(PackageBase): sip_module = 'sip' #: Callback names for install-time test - install_time_test_callbacks = ['import_module_test'] + install_time_test_callbacks = ['test'] extends('python') depends_on('qt') depends_on('py-sip') + @property + def import_modules(self): + """Names of modules that the Python package provides. + + These are used to test whether or not the installation succeeded. + These names generally come from running: + + .. code-block:: python + + >> import setuptools + >> setuptools.find_packages() + + in the source tarball directory. If the module names are incorrectly + detected, this property can be overridden by the package. + + Returns: + list: list of strings of module names + """ + modules = [] + + # Python libraries may be installed in lib or lib64 + # See issues #18520 and #17126 + for lib in ['lib', 'lib64']: + root = os.path.join(self.prefix, lib, 'python{0}'.format( + self.spec['python'].version.up_to(2)), 'site-packages') + # Some Python libraries are packages: collections of modules + # distributed in directories containing __init__.py files + for path in find(root, '__init__.py', recursive=True): + modules.append(path.replace(root + os.sep, '', 1).replace( + os.sep + '__init__.py', '').replace('/', '.')) + # Some Python libraries are modules: individual *.py files + # found in the site-packages directory + for path in find(root, '*.py', recursive=False): + modules.append(path.replace(root + os.sep, '', 1).replace( + '.py', '').replace('/', '.')) + + tty.debug('Detected the following modules: {0}'.format(modules)) + return modules + def python(self, *args, **kwargs): """The python ``Executable``.""" inspect.getmodule(self).python(*args, **kwargs) @@ -98,17 +138,16 @@ class SIPPackage(PackageBase): # Testing - def import_module_test(self): - """Attempts to import the module that was just installed. - - This test is only run if the package overrides - :py:attr:`import_modules` with a list of module names.""" + def test(self): + """Attempts to import modules of the installed package.""" # Make sure we are importing the installed modules, - # not the ones in the current directory - with working_dir('spack-test', create=True): - for module in self.import_modules: - self.python('-c', 'import {0}'.format(module)) + # not the ones in the source directory + for module in self.import_modules: + self.run_test(inspect.getmodule(self).python.path, + ['-c', 'import {0}'.format(module)], + purpose='checking import of {0}'.format(module), + work_dir='spack-test') run_after('install')(PackageBase._run_default_install_time_test_callbacks) diff --git a/lib/spack/spack/build_systems/sourceforge.py b/lib/spack/spack/build_systems/sourceforge.py index 602a099c17..cfea664f51 100644 --- a/lib/spack/spack/build_systems/sourceforge.py +++ b/lib/spack/spack/build_systems/sourceforge.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/sourceware.py b/lib/spack/spack/build_systems/sourceware.py index b779b530dc..89a157c22c 100644 --- a/lib/spack/spack/build_systems/sourceware.py +++ b/lib/spack/spack/build_systems/sourceware.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/waf.py b/lib/spack/spack/build_systems/waf.py index a6dbbbdb35..99679ba64a 100644 --- a/lib/spack/spack/build_systems/waf.py +++ b/lib/spack/spack/build_systems/waf.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/xorg.py b/lib/spack/spack/build_systems/xorg.py index ae28b30929..801ad37eeb 100644 --- a/lib/spack/spack/build_systems/xorg.py +++ b/lib/spack/spack/build_systems/xorg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 49624c06b2..d06ce05525 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index f38ab95d96..e853dc9f98 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -22,7 +22,7 @@ import llnl.util.tty as tty import spack import spack.binary_distribution as bindist -import spack.cmd.buildcache as buildcache +import spack.cmd import spack.compilers as compilers import spack.config as cfg import spack.environment as ev @@ -41,7 +41,8 @@ JOB_RETRY_CONDITIONS = [ 'always', ] -SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-pr-mirrors' +SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs' +TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror' spack_gpg = spack.main.SpackCommand('gpg') spack_compiler = spack.main.SpackCommand('compiler') @@ -202,8 +203,8 @@ def format_root_spec(spec, main_phase, strip_compiler): # spec.name, spec.version, spec.compiler, spec.architecture) -def spec_deps_key_label(s): - return s.dag_hash(), "%s/%s" % (s.name, s.dag_hash(7)) +def spec_deps_key(s): + return '{0}/{1}'.format(s.name, s.dag_hash(7)) def _add_dependency(spec_label, dep_label, deps): @@ -214,8 +215,8 @@ def _add_dependency(spec_label, dep_label, deps): deps[spec_label].add(dep_label) -def get_spec_dependencies(specs, deps, spec_labels): - spec_deps_obj = compute_spec_deps(specs) +def get_spec_dependencies(specs, deps, spec_labels, check_index_only=False): + spec_deps_obj = compute_spec_deps(specs, check_index_only=check_index_only) if spec_deps_obj: dependencies = spec_deps_obj['dependencies'] @@ -225,19 +226,25 @@ def get_spec_dependencies(specs, deps, spec_labels): spec_labels[entry['label']] = { 'spec': Spec(entry['spec']), 'rootSpec': entry['root_spec'], + 'needs_rebuild': entry['needs_rebuild'], } for entry in dependencies: _add_dependency(entry['spec'], entry['depends'], deps) -def stage_spec_jobs(specs): +def stage_spec_jobs(specs, check_index_only=False): """Take a set of release specs and generate a list of "stages", where the jobs in any stage are dependent only on jobs in previous stages. This allows us to maximize build parallelism within the gitlab-ci framework. Arguments: specs (Iterable): Specs to build + check_index_only (bool): Regardless of whether DAG pruning is enabled, + all configured mirrors are searched to see if binaries for specs + are up to date on those mirrors. This flag limits that search to + the binary cache indices on those mirrors to speed the process up, + even though there is no garantee the index is up to date. Returns: A tuple of information objects describing the specs, dependencies and stages: @@ -274,7 +281,8 @@ def stage_spec_jobs(specs): deps = {} spec_labels = {} - get_spec_dependencies(specs, deps, spec_labels) + get_spec_dependencies( + specs, deps, spec_labels, check_index_only=check_index_only) # Save the original deps, as we need to return them at the end of the # function. In the while loop below, the "dependencies" variable is @@ -311,12 +319,15 @@ def print_staging_summary(spec_labels, dependencies, stages): for job in sorted(stage): s = spec_labels[job]['spec'] - tty.msg(' {0} -> {1}'.format(job, get_spec_string(s))) + tty.msg(' [{1}] {0} -> {2}'.format( + job, + 'x' if spec_labels[job]['needs_rebuild'] else ' ', + get_spec_string(s))) stage_index += 1 -def compute_spec_deps(spec_list): +def compute_spec_deps(spec_list, check_index_only=False): """ Computes all the dependencies for the spec(s) and generates a JSON object which provides both a list of unique spec names as well as a @@ -386,33 +397,35 @@ def compute_spec_deps(spec_list): # root_spec = get_spec_string(spec) root_spec = spec - rkey, rlabel = spec_deps_key_label(spec) - for s in spec.traverse(deptype=all): if s.external: tty.msg('Will not stage external pkg: {0}'.format(s)) continue - skey, slabel = spec_deps_key_label(s) - spec_labels[slabel] = { + up_to_date_mirrors = bindist.get_mirrors_for_spec( + spec=s, full_hash_match=True, index_only=check_index_only) + + skey = spec_deps_key(s) + spec_labels[skey] = { 'spec': get_spec_string(s), 'root': root_spec, + 'needs_rebuild': not up_to_date_mirrors, } - append_dep(rlabel, slabel) for d in s.dependencies(deptype=all): - dkey, dlabel = spec_deps_key_label(d) + dkey = spec_deps_key(d) if d.external: tty.msg('Will not stage external dep: {0}'.format(d)) continue - append_dep(slabel, dlabel) + append_dep(skey, dkey) for spec_label, spec_holder in spec_labels.items(): specs.append({ 'label': spec_label, 'spec': spec_holder['spec'], 'root_spec': spec_holder['root'], + 'needs_rebuild': spec_holder['needs_rebuild'], }) deps_json_obj = { @@ -481,22 +494,43 @@ def pkg_name_from_spec_label(spec_label): def format_job_needs(phase_name, strip_compilers, dep_jobs, - osname, build_group, enable_artifacts_buildcache): + osname, build_group, prune_dag, stage_spec_dict, + enable_artifacts_buildcache): needs_list = [] for dep_job in dep_jobs: - needs_list.append({ - 'job': get_job_name(phase_name, - strip_compilers, - dep_job, - osname, - build_group), - 'artifacts': enable_artifacts_buildcache, - }) + dep_spec_key = spec_deps_key(dep_job) + dep_spec_info = stage_spec_dict[dep_spec_key] + + if not prune_dag or dep_spec_info['needs_rebuild']: + needs_list.append({ + 'job': get_job_name(phase_name, + strip_compilers, + dep_job, + osname, + build_group), + 'artifacts': enable_artifacts_buildcache, + }) return needs_list -def generate_gitlab_ci_yaml(env, print_summary, output_file, - run_optimizer=False, use_dependencies=False): +def add_pr_mirror(url): + cfg_scope = cfg.default_modify_scope() + mirrors = cfg.get('mirrors', scope=cfg_scope) + items = [(n, u) for n, u in mirrors.items()] + items.insert(0, ('ci_pr_mirror', url)) + cfg.set('mirrors', syaml.syaml_dict(items), scope=cfg_scope) + + +def remove_pr_mirror(): + cfg_scope = cfg.default_modify_scope() + mirrors = cfg.get('mirrors', scope=cfg_scope) + mirrors.pop('ci_pr_mirror') + cfg.set('mirrors', mirrors, scope=cfg_scope) + + +def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False, + check_index_only=False, run_optimizer=False, + use_dependencies=False): # FIXME: What's the difference between one that opens with 'spack' # and one that opens with 'env'? This will only handle the former. with spack.concretize.disable_compiler_existence_check(): @@ -509,10 +543,6 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, gitlab_ci = yaml_root['gitlab-ci'] - final_job_config = None - if 'final-stage-rebuild-index' in gitlab_ci: - final_job_config = gitlab_ci['final-stage-rebuild-index'] - build_group = None enable_cdash_reporting = False cdash_auth_token = None @@ -539,6 +569,9 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, pr_mirror_url = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, spack_pr_branch) + if 'mirrors' not in yaml_root or len(yaml_root['mirrors'].values()) < 1: + tty.die('spack ci generate requires an env containing a mirror') + ci_mirrors = yaml_root['mirrors'] mirror_urls = [url for url in ci_mirrors.values()] @@ -546,6 +579,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, if 'enable-artifacts-buildcache' in gitlab_ci: enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache'] + rebuild_index_enabled = True + if 'rebuild-index' in gitlab_ci and gitlab_ci['rebuild-index'] is False: + rebuild_index_enabled = False + + temp_storage_url_prefix = None + if 'temporary-storage-url-prefix' in gitlab_ci: + temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix'] + bootstrap_specs = [] phases = [] if 'bootstrap' in gitlab_ci: @@ -573,19 +614,27 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'strip-compilers': False, }) - staged_phases = {} - for phase in phases: - phase_name = phase['name'] - with spack.concretize.disable_compiler_existence_check(): - staged_phases[phase_name] = stage_spec_jobs( - env.spec_lists[phase_name]) + # Add this mirror if it's enabled, as some specs might be up to date + # here and thus not need to be rebuilt. + if pr_mirror_url: + add_pr_mirror(pr_mirror_url) - if print_summary: + # Speed up staging by first fetching binary indices from all mirrors + # (including the per-PR mirror we may have just added above). + bindist.binary_index.update() + + staged_phases = {} + try: for phase in phases: phase_name = phase['name'] - tty.msg('Stages for phase "{0}"'.format(phase_name)) - phase_stages = staged_phases[phase_name] - print_staging_summary(*phase_stages) + with spack.concretize.disable_compiler_existence_check(): + staged_phases[phase_name] = stage_spec_jobs( + env.spec_lists[phase_name], + check_index_only=check_index_only) + finally: + # Clean up PR mirror if enabled + if pr_mirror_url: + remove_pr_mirror() all_job_names = [] output_object = {} @@ -611,7 +660,8 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, stage_id += 1 for spec_label in stage_jobs: - root_spec = spec_labels[spec_label]['rootSpec'] + spec_record = spec_labels[spec_label] + root_spec = spec_record['rootSpec'] pkg_name = pkg_name_from_spec_label(spec_label) release_spec = root_spec[pkg_name] @@ -678,11 +728,15 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, job_dependencies = [] if spec_label in dependencies: if enable_artifacts_buildcache: + # Get dependencies transitively, so they're all + # available in the artifacts buildcache. dep_jobs = [ d for d in release_spec.traverse(deptype=all, root=False) ] else: + # In this case, "needs" is only used for scheduling + # purposes, so we only get the direct dependencies. dep_jobs = [] for dep_label in dependencies[spec_label]: dep_pkg = pkg_name_from_spec_label(dep_label) @@ -690,10 +744,13 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, dep_jobs.append(dep_root[dep_pkg]) job_dependencies.extend( - format_job_needs(phase_name, strip_compilers, dep_jobs, - osname, build_group, + format_job_needs(phase_name, strip_compilers, + dep_jobs, osname, build_group, + prune_dag, spec_labels, enable_artifacts_buildcache)) + rebuild_spec = spec_record['needs_rebuild'] + # This next section helps gitlab make sure the right # bootstrapped compiler exists in the artifacts buildcache by # creating an artificial dependency between this spec and its @@ -709,11 +766,12 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, compiler_pkg_spec = compilers.pkg_spec_for_compiler( release_spec.compiler) for bs in bootstrap_specs: - bs_arch = bs['spec'].architecture + c_spec = bs['spec'] + bs_arch = c_spec.architecture bs_arch_family = (bs_arch.target .microarchitecture .family) - if (bs['spec'].satisfies(compiler_pkg_spec) and + if (c_spec.satisfies(compiler_pkg_spec) and bs_arch_family == spec_arch_family): # We found the bootstrap compiler this release spec # should be built with, so for DAG scheduling @@ -721,10 +779,24 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, # to the jobs "needs". But if artifact buildcache # is enabled, we'll have to add all transtive deps # of the compiler as well. - dep_jobs = [bs['spec']] + + # Here we check whether the bootstrapped compiler + # needs to be rebuilt. Until compilers are proper + # dependencies, we artificially force the spec to + # be rebuilt if the compiler targeted to build it + # needs to be rebuilt. + bs_specs, _, _ = staged_phases[bs['phase-name']] + c_spec_key = spec_deps_key(c_spec) + rbld_comp = bs_specs[c_spec_key]['needs_rebuild'] + rebuild_spec = rebuild_spec or rbld_comp + # Also update record so dependents do not fail to + # add this spec to their "needs" + spec_record['needs_rebuild'] = rebuild_spec + + dep_jobs = [c_spec] if enable_artifacts_buildcache: dep_jobs = [ - d for d in bs['spec'].traverse(deptype=all) + d for d in c_spec.traverse(deptype=all) ] job_dependencies.extend( @@ -733,6 +805,8 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, dep_jobs, str(bs_arch), build_group, + prune_dag, + bs_specs, enable_artifacts_buildcache)) else: debug_msg = ''.join([ @@ -741,9 +815,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'not the compiler required by the spec, or ', 'because the target arch families of the ', 'spec and the compiler did not match' - ]).format(bs['spec'], release_spec) + ]).format(c_spec, release_spec) tty.debug(debug_msg) + if prune_dag and not rebuild_spec: + continue + + job_vars['SPACK_SPEC_NEEDS_REBUILD'] = str(rebuild_spec) + if enable_cdash_reporting: cdash_build_name = get_cdash_build_name( release_spec, build_group) @@ -812,11 +891,19 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, output_object[job_name] = job_object job_id += 1 + if print_summary: + for phase in phases: + phase_name = phase['name'] + tty.msg('Stages for phase "{0}"'.format(phase_name)) + phase_stages = staged_phases[phase_name] + print_staging_summary(*phase_stages) + tty.debug('{0} build jobs generated in {1} stages'.format( job_id, stage_id)) - tty.debug('The max_needs_job is {0}, with {1} needs'.format( - max_needs_job, max_length_needs)) + if job_id > 0: + tty.debug('The max_needs_job is {0}, with {1} needs'.format( + max_needs_job, max_length_needs)) # Use "all_job_names" to populate the build group for this set if enable_cdash_reporting and cdash_auth_token: @@ -828,63 +915,118 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, else: tty.warn('Unable to populate buildgroup without CDash credentials') - if final_job_config and not is_pr_pipeline: - # Add an extra, final job to regenerate the index - final_stage = 'stage-rebuild-index' - final_job = { - 'stage': final_stage, - 'script': 'spack buildcache update-index --keys -d {0}'.format( - mirror_urls[0]), - 'tags': final_job_config['tags'], - 'when': 'always' - } - if 'image' in final_job_config: - final_job['image'] = final_job_config['image'] - if before_script: - final_job['before_script'] = before_script - if after_script: - final_job['after_script'] = after_script - output_object['rebuild-index'] = final_job - stage_names.append(final_stage) - - output_object['stages'] = stage_names - - # Capture the version of spack used to generate the pipeline, transform it - # into a value that can be passed to "git checkout", and save it in a - # global yaml variable - spack_version = spack.main.get_version() - version_to_clone = None - v_match = re.match(r"^\d+\.\d+\.\d+$", spack_version) - if v_match: - version_to_clone = 'v{0}'.format(v_match.group(0)) - else: - v_match = re.match(r"^[^-]+-[^-]+-([a-f\d]+)$", spack_version) + service_job_config = None + if 'service-job-attributes' in gitlab_ci: + service_job_config = gitlab_ci['service-job-attributes'] + + default_attrs = [ + 'image', + 'tags', + 'variables', + 'before_script', + # 'script', + 'after_script', + ] + + if job_id > 0: + if temp_storage_url_prefix: + # There were some rebuild jobs scheduled, so we will need to + # schedule a job to clean up the temporary storage location + # associated with this pipeline. + stage_names.append('cleanup-temp-storage') + cleanup_job = {} + + if service_job_config: + copy_attributes(default_attrs, + service_job_config, + cleanup_job) + + cleanup_job['stage'] = 'cleanup-temp-storage' + cleanup_job['script'] = [ + 'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format( + temp_storage_url_prefix) + ] + cleanup_job['when'] = 'always' + + output_object['cleanup'] = cleanup_job + + if rebuild_index_enabled: + # Add a final job to regenerate the index + stage_names.append('stage-rebuild-index') + final_job = {} + + if service_job_config: + copy_attributes(default_attrs, + service_job_config, + final_job) + + index_target_mirror = mirror_urls[0] + if is_pr_pipeline: + index_target_mirror = pr_mirror_url + + final_job['stage'] = 'stage-rebuild-index' + final_job['script'] = [ + 'spack buildcache update-index --keys -d {0}'.format( + index_target_mirror) + ] + final_job['when'] = 'always' + + output_object['rebuild-index'] = final_job + + output_object['stages'] = stage_names + + # Capture the version of spack used to generate the pipeline, transform it + # into a value that can be passed to "git checkout", and save it in a + # global yaml variable + spack_version = spack.main.get_version() + version_to_clone = None + v_match = re.match(r"^\d+\.\d+\.\d+$", spack_version) if v_match: - version_to_clone = v_match.group(1) + version_to_clone = 'v{0}'.format(v_match.group(0)) else: - version_to_clone = spack_version + v_match = re.match(r"^[^-]+-[^-]+-([a-f\d]+)$", spack_version) + if v_match: + version_to_clone = v_match.group(1) + else: + version_to_clone = spack_version - output_object['variables'] = { - 'SPACK_VERSION': spack_version, - 'SPACK_CHECKOUT_VERSION': version_to_clone, - } + output_object['variables'] = { + 'SPACK_VERSION': spack_version, + 'SPACK_CHECKOUT_VERSION': version_to_clone, + } - if pr_mirror_url: - output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url + if pr_mirror_url: + output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url + + sorted_output = {} + for output_key, output_value in sorted(output_object.items()): + sorted_output[output_key] = output_value + + # TODO(opadron): remove this or refactor + if run_optimizer: + import spack.ci_optimization as ci_opt + sorted_output = ci_opt.optimizer(sorted_output) - sorted_output = {} - for output_key, output_value in sorted(output_object.items()): - sorted_output[output_key] = output_value + # TODO(opadron): remove this or refactor + if use_dependencies: + import spack.ci_needs_workaround as cinw + sorted_output = cinw.needs_to_dependencies(sorted_output) + else: + # No jobs were generated + tty.debug('No specs to rebuild, generating no-op job') + noop_job = {} + + if service_job_config: + copy_attributes(default_attrs, + service_job_config, + noop_job) - # TODO(opadron): remove this or refactor - if run_optimizer: - import spack.ci_optimization as ci_opt - sorted_output = ci_opt.optimizer(sorted_output) + if 'script' not in noop_job: + noop_job['script'] = [ + 'echo "All specs already up to date, nothing to rebuild."', + ] - # TODO(opadron): remove this or refactor - if use_dependencies: - import spack.ci_needs_workaround as cinw - sorted_output = cinw.needs_to_dependencies(sorted_output) + sorted_output = {'no-specs-to-rebuild': noop_job} with open(output_file, 'w') as outf: outf.write(syaml.dump_config(sorted_output, default_flow_style=True)) @@ -1120,16 +1262,35 @@ def read_cdashid_from_mirror(spec, mirror_url): def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id, sign_binaries): if mirror_url: - unsigned = not sign_binaries - tty.debug('Creating buildcache ({0})'.format( - 'unsigned' if unsigned else 'signed')) - buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False, - output_location=mirror_url, force=True, - allow_root=True, unsigned=unsigned) - if build_id: - tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( - build_id, mirror_url)) - write_cdashid_to_mirror(build_id, spec, mirror_url) + try: + unsigned = not sign_binaries + tty.debug('Creating buildcache ({0})'.format( + 'unsigned' if unsigned else 'signed')) + spack.cmd.buildcache._createtarball( + env, spec_yaml=yaml_path, add_deps=False, + output_location=mirror_url, force=True, allow_root=True, + unsigned=unsigned) + if build_id: + tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( + build_id, mirror_url)) + write_cdashid_to_mirror(build_id, spec, mirror_url) + except Exception as inst: + # If the mirror we're pushing to is on S3 and there's some + # permissions problem, for example, we can't just target + # that exception type here, since users of the + # `spack ci rebuild' may not need or want any dependency + # on boto3. So we use the first non-boto exception type + # in the heirarchy: + # boto3.exceptions.S3UploadFailedError + # boto3.exceptions.Boto3Error + # Exception + # BaseException + # object + err_msg = 'Error msg: {0}'.format(inst) + if 'Access Denied' in err_msg: + tty.msg('Permission problem writing to {0}'.format( + mirror_url)) + tty.msg(err_msg) def copy_stage_logs_to_artifacts(job_spec, job_log_dir): diff --git a/lib/spack/spack/ci_needs_workaround.py b/lib/spack/spack/ci_needs_workaround.py index ba9f7c62af..ff3c621f97 100644 --- a/lib/spack/spack/ci_needs_workaround.py +++ b/lib/spack/spack/ci_needs_workaround.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/ci_optimization.py b/lib/spack/spack/ci_optimization.py index c5e69c649c..94528c856a 100644 --- a/lib/spack/spack/ci_optimization.py +++ b/lib/spack/spack/ci_optimization.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 130abea4cc..4295ffe6a2 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index bfca9f5604..a6c27b2f1d 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/add.py b/lib/spack/spack/cmd/add.py index 94c8620dbb..ae1a9c9b7e 100644 --- a/lib/spack/spack/cmd/add.py +++ b/lib/spack/spack/cmd/add.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index 9751599b10..7435fb1246 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/blame.py b/lib/spack/spack/cmd/blame.py index b806058aec..f61ed03b4a 100644 --- a/lib/spack/spack/cmd/blame.py +++ b/lib/spack/spack/cmd/blame.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/build_env.py b/lib/spack/spack/cmd/build_env.py index ef8b1f6e6b..fc9f95f5ba 100644 --- a/lib/spack/spack/cmd/build_env.py +++ b/lib/spack/spack/cmd/build_env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 025de653c5..f5cb94e7c7 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -769,19 +769,14 @@ def buildcache_copy(args): shutil.copyfile(cdashid_src_path, cdashid_dest_path) -def buildcache_update_index(args): - """Update a buildcache index.""" - outdir = '.' - if args.mirror_url: - outdir = args.mirror_url - - mirror = spack.mirror.MirrorCollection().lookup(outdir) +def update_index(mirror_url, update_keys=False): + mirror = spack.mirror.MirrorCollection().lookup(mirror_url) outdir = url_util.format(mirror.push_url) bindist.generate_package_index( url_util.join(outdir, bindist.build_cache_relative_path())) - if args.keys: + if update_keys: keys_url = url_util.join(outdir, bindist.build_cache_relative_path(), bindist.build_cache_keys_relative_path()) @@ -789,6 +784,15 @@ def buildcache_update_index(args): bindist.generate_key_index(keys_url) +def buildcache_update_index(args): + """Update a buildcache index.""" + outdir = '.' + if args.mirror_url: + outdir = args.mirror_url + + update_index(outdir, update_keys=args.keys) + + def buildcache(parser, args): if args.func: args.func(args) diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py index cbbe9db04d..f0f3ba9eb3 100644 --- a/lib/spack/spack/cmd/cd.py +++ b/lib/spack/spack/cmd/cd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 188deb1149..937b054c27 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 3dd97e53b0..bd42e10238 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -17,6 +17,7 @@ import spack.cmd.buildcache as buildcache import spack.environment as ev import spack.hash_types as ht import spack.util.executable as exe +import spack.util.url as url_util description = "manage continuous integration pipelines" @@ -54,6 +55,26 @@ def setup_parser(subparser): '--dependencies', action='store_true', default=False, help="(Experimental) disable DAG scheduling; use " ' "plain" dependencies.') + prune_group = generate.add_mutually_exclusive_group() + prune_group.add_argument( + '--prune-dag', action='store_true', dest='prune_dag', + default=True, help="""Do not generate jobs for specs already up to +date on the mirror""") + prune_group.add_argument( + '--no-prune-dag', action='store_false', dest='prune_dag', + default=True, help="""Generate jobs for specs already up to date +on the mirror""") + generate.add_argument( + '--check-index-only', action='store_true', dest='index_only', + default=False, help="""Spack always check specs against configured +binary mirrors when generating the pipeline, regardless of whether or not +DAG pruning is enabled. This flag controls whether it might attempt to +fetch remote spec.yaml files directly (ensuring no spec is rebuilt if it is +present on the mirror), or whether it should reduce pipeline generation time +by assuming all remote buildcache indices are up to date and only use those +to determine whether a given spec is up to date on mirrors. In the latter +case, specs might be needlessly rebuilt if remote buildcache indices are out +of date.""") generate.set_defaults(func=ci_generate) # Check a spec against mirror. Rebuild, create buildcache and push to @@ -61,6 +82,11 @@ def setup_parser(subparser): rebuild = subparsers.add_parser('rebuild', help=ci_rebuild.__doc__) rebuild.set_defaults(func=ci_rebuild) + # Rebuild the buildcache index associated with the mirror in the + # active, gitlab-enabled environment. + index = subparsers.add_parser('rebuild-index', help=ci_reindex.__doc__) + index.set_defaults(func=ci_reindex) + def ci_generate(args): """Generate jobs file from a spack environment file containing CI info. @@ -75,6 +101,8 @@ def ci_generate(args): copy_yaml_to = args.copy_to run_optimizer = args.optimize use_dependencies = args.dependencies + prune_dag = args.prune_dag + index_only = args.index_only if not output_file: output_file = os.path.abspath(".gitlab-ci.yml") @@ -86,7 +114,8 @@ def ci_generate(args): # Generate the jobs spack_ci.generate_gitlab_ci_yaml( - env, True, output_file, run_optimizer=run_optimizer, + env, True, output_file, prune_dag=prune_dag, + check_index_only=index_only, run_optimizer=run_optimizer, use_dependencies=use_dependencies) if copy_yaml_to: @@ -131,6 +160,7 @@ def ci_rebuild(args): # SPACK_SIGNING_KEY ci_artifact_dir = get_env_var('CI_PROJECT_DIR') + ci_pipeline_id = get_env_var('CI_PIPELINE_ID') signing_key = get_env_var('SPACK_SIGNING_KEY') root_spec = get_env_var('SPACK_ROOT_SPEC') job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME') @@ -191,19 +221,28 @@ def ci_rebuild(args): spack_is_pr_pipeline = True if pr_env_var == 'True' else False + pipeline_mirror_url = None + temp_storage_url_prefix = None + if 'temporary-storage-url-prefix' in gitlab_ci: + temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix'] + pipeline_mirror_url = url_util.join( + temp_storage_url_prefix, ci_pipeline_id) + enable_artifacts_mirror = False - artifact_mirror_url = None if 'enable-artifacts-buildcache' in gitlab_ci: enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache'] - if enable_artifacts_mirror or spack_is_pr_pipeline: - # If this is a PR pipeline, we will override the setting to - # make sure that artifacts buildcache is enabled. Otherwise - # jobs will not have binary deps available since we do not - # allow pushing binaries to remote mirror during PR pipelines + if (enable_artifacts_mirror or (spack_is_pr_pipeline and + not enable_artifacts_mirror and not temp_storage_url_prefix)): + # If you explicitly enabled the artifacts buildcache feature, or + # if this is a PR pipeline but you did not enable either of the + # per-pipeline temporary storage features, we force the use of + # artifacts buildcache. Otherwise jobs will not have binary + # dependencies from previous stages available since we do not + # allow pushing binaries to the remote mirror during PR pipelines. enable_artifacts_mirror = True - artifact_mirror_url = 'file://' + local_mirror_dir + pipeline_mirror_url = 'file://' + local_mirror_dir mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format( - artifact_mirror_url) + pipeline_mirror_url) tty.debug(mirror_msg) # Clean out scratch directory from last stage @@ -297,8 +336,8 @@ def ci_rebuild(args): if pr_mirror_url: add_mirror('ci_pr_mirror', pr_mirror_url) - if enable_artifacts_mirror: - add_mirror('ci_artifact_mirror', artifact_mirror_url) + if pipeline_mirror_url: + add_mirror(spack_ci.TEMP_STORAGE_MIRROR_NAME, pipeline_mirror_url) tty.debug('listing spack mirrors:') spack_cmd('mirror', 'list') @@ -306,8 +345,8 @@ def ci_rebuild(args): # Checks all mirrors for a built spec with a matching full hash matches = bindist.get_mirrors_for_spec( - job_spec, force=False, full_hash_match=True, - mirrors_to_check=mirrors_to_check) + job_spec, full_hash_match=True, mirrors_to_check=mirrors_to_check, + index_only=False) if matches: # Got at full hash match on at least one configured mirror. All @@ -374,38 +413,40 @@ def ci_rebuild(args): else: buildcache_mirror_url = remote_mirror_url - try: - spack_ci.push_mirror_contents( - env, job_spec, job_spec_yaml_path, buildcache_mirror_url, - cdash_build_id, sign_binaries) - except Exception as inst: - # If the mirror we're pushing to is on S3 and there's some - # permissions problem, for example, we can't just target - # that exception type here, since users of the - # `spack ci rebuild' may not need or want any dependency - # on boto3. So we use the first non-boto exception type - # in the heirarchy: - # boto3.exceptions.S3UploadFailedError - # boto3.exceptions.Boto3Error - # Exception - # BaseException - # object - err_msg = 'Error msg: {0}'.format(inst) - if 'Access Denied' in err_msg: - tty.msg('Permission problem writing to mirror') - tty.msg(err_msg) - - # Create another copy of that buildcache on "local artifact - # mirror" (only done if artifacts buildcache is enabled) - spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, - artifact_mirror_url, cdash_build_id, - sign_binaries) + # Create buildcache in either the main remote mirror, or in the + # per-PR mirror, if this is a PR pipeline + spack_ci.push_mirror_contents( + env, job_spec, job_spec_yaml_path, buildcache_mirror_url, + cdash_build_id, sign_binaries) + + # Create another copy of that buildcache in the per-pipeline + # temporary storage mirror (this is only done if either artifacts + # buildcache is enabled or a temporary storage url prefix is set) + spack_ci.push_mirror_contents( + env, job_spec, job_spec_yaml_path, pipeline_mirror_url, + cdash_build_id, sign_binaries) # Relate this build to its dependencies on CDash (if enabled) if enable_cdash: spack_ci.relate_cdash_builds( spec_map, cdash_base_url, cdash_build_id, cdash_project, - artifact_mirror_url or pr_mirror_url or remote_mirror_url) + pipeline_mirror_url or pr_mirror_url or remote_mirror_url) + + +def ci_reindex(args): + """Rebuild the buildcache index associated with the mirror in the + active, gitlab-enabled environment. """ + env = ev.get_env(args, 'ci rebuild-index', required=True) + yaml_root = ev.config_dict(env.yaml) + + if 'mirrors' not in yaml_root or len(yaml_root['mirrors'].values()) < 1: + tty.die('spack ci rebuild-index requires an env containing a mirror') + + ci_mirrors = yaml_root['mirrors'] + mirror_urls = [url for url in ci_mirrors.values()] + remote_mirror_url = mirror_urls[0] + + buildcache.update_index(remote_mirror_url, update_keys=True) def ci(parser, args): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index f69b959293..5c7e02c685 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/clone.py b/lib/spack/spack/cmd/clone.py index ecf5855dab..0ea9cf7ef1 100644 --- a/lib/spack/spack/cmd/clone.py +++ b/lib/spack/spack/cmd/clone.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index be934e0048..e63e162c3f 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -262,20 +262,7 @@ def _commands(parser, args): if args.header and not os.path.exists(args.header): tty.die("No such file: '%s'" % args.header) - # if we're updating an existing file, only write output if a command - # or the header is newer than the file. if args.update: - if os.path.exists(args.update): - files = [ - spack.cmd.get_module(command).__file__.rstrip('c') # pyc -> py - for command in spack.cmd.all_commands()] - if args.header: - files.append(args.header) - last_update = os.path.getmtime(args.update) - if not any(os.path.getmtime(f) > last_update for f in files): - tty.msg('File is up to date: %s' % args.update) - return - tty.msg('Updating file: %s' % args.update) with open(args.update, 'w') as f: prepend_header(args, f) diff --git a/lib/spack/spack/cmd/common/__init__.py b/lib/spack/spack/cmd/common/__init__.py index 35b802db48..7cd8c313e0 100644 --- a/lib/spack/spack/cmd/common/__init__.py +++ b/lib/spack/spack/cmd/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index e5c4c0dde8..e6a0edd599 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -250,8 +250,8 @@ def very_long(): @arg def tags(): return Args( - '-t', '--tags', action='append', - help='filter a package query by tags') + '-t', '--tag', action='append', dest='tags', metavar='TAG', + help='filter a package query by tag (multiple use allowed)') @arg @@ -277,6 +277,13 @@ def no_checksum(): help="do not use checksums to verify downloaded files (unsafe)") +@arg +def deprecated(): + return Args( + '--deprecated', action='store_true', default=False, + help='fetch deprecated versions without warning') + + def add_cdash_args(subparser, add_help): cdash_help = {} if add_help: diff --git a/lib/spack/spack/cmd/common/env_utility.py b/lib/spack/spack/cmd/common/env_utility.py index e3f32737b4..6a706b1b87 100644 --- a/lib/spack/spack/cmd/common/env_utility.py +++ b/lib/spack/spack/cmd/common/env_utility.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index fb9e24ea76..08232022da 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index c29e2257b7..b30b27233f 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/concretize.py b/lib/spack/spack/cmd/concretize.py index d28f7b4a5d..b94511569b 100644 --- a/lib/spack/spack/cmd/concretize.py +++ b/lib/spack/spack/cmd/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 2cd914ef89..81136d7859 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py index cc2c001560..27ef988f69 100644 --- a/lib/spack/spack/cmd/containerize.py +++ b/lib/spack/spack/cmd/containerize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index a31b9537b6..71d34a7ebb 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,7 +28,7 @@ level = "short" package_template = '''\ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -117,7 +117,7 @@ class PackageTemplate(BundlePackageTemplate): make() make('install')""" - url_line = """ url = \"{url}\"""" + url_line = ' url = "{url}"' def __init__(self, name, url, versions): super(PackageTemplate, self).__init__(name, versions) @@ -270,14 +270,47 @@ class PythonPackageTemplate(PackageTemplate): args = [] return args""" - def __init__(self, name, *args, **kwargs): + def __init__(self, name, url, *args, **kwargs): # If the user provided `--name py-numpy`, don't rename it py-py-numpy if not name.startswith('py-'): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to py-{0}".format(name)) name = 'py-{0}'.format(name) - super(PythonPackageTemplate, self).__init__(name, *args, **kwargs) + # Simple PyPI URLs: + # https://<hostname>/packages/<type>/<first character of project>/<project>/<download file> + # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip + + # PyPI URLs containing hash: + # https://<hostname>/packages/<two character hash>/<two character hash>/<longer hash>/<download file> + # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512 + + # PyPI URLs for wheels are too complicated, ignore them for now + + match = re.search( + r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4, + url + ) + if match: + if len(match.group(2)) == 1: + # Simple PyPI URL + url = '/'.join(match.group(3, 4)) + else: + # PyPI URL containing hash + # Project name doesn't necessarily match download name, but it + # usually does, so this is the best we can do + project = parse_name(url) + url = '/'.join([project, match.group(4)]) + + self.url_line = ' pypi = "{url}"' + + super(PythonPackageTemplate, self).__init__(name, url, *args, **kwargs) class RPackageTemplate(PackageTemplate): @@ -289,7 +322,7 @@ class RPackageTemplate(PackageTemplate): # depends_on('r-foo', type=('build', 'run'))""" body_def = """\ - def configure_args(self, spec, prefix): + def configure_args(self): # FIXME: Add arguments to pass to install via --configure-args # FIXME: If not needed delete this function args = [] @@ -545,7 +578,8 @@ class BuildSystemGuesser: ] # Peek inside the compressed file. - if stage.archive_file.endswith('.zip'): + if (stage.archive_file.endswith('.zip') or + '.zip#' in stage.archive_file): try: unzip = which('unzip') output = unzip('-lq', stage.archive_file, output=str) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 3c72531a9c..0afa4af82b 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py index 074e95209a..8580416cba 100644 --- a/lib/spack/spack/cmd/debug.py +++ b/lib/spack/spack/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -15,6 +15,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import working_dir import spack.architecture as architecture +import spack.config import spack.paths from spack.main import get_version from spack.util.executable import which @@ -89,6 +90,7 @@ def report(args): print('* **Python:**', platform.python_version()) print('* **Platform:**', architecture.Arch( architecture.platform(), 'frontend', 'frontend')) + print('* **Concretizer:**', spack.config.get('config:concretizer')) def debug(parser, args): diff --git a/lib/spack/spack/cmd/dependencies.py b/lib/spack/spack/cmd/dependencies.py index bbccbe23ee..2f923fece0 100644 --- a/lib/spack/spack/cmd/dependencies.py +++ b/lib/spack/spack/cmd/dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 5563701801..242a07da36 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/deprecate.py b/lib/spack/spack/cmd/deprecate.py index 6172033fa7..2928a5b264 100644 --- a/lib/spack/spack/cmd/deprecate.py +++ b/lib/spack/spack/cmd/deprecate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/dev_build.py b/lib/spack/spack/cmd/dev_build.py index 429ba07246..2e7c51da42 100644 --- a/lib/spack/spack/cmd/dev_build.py +++ b/lib/spack/spack/cmd/dev_build.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -26,7 +26,7 @@ def setup_parser(subparser): subparser.add_argument( '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="don't try to install dependencies of requested packages") - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'deprecated']) subparser.add_argument( '--keep-prefix', action='store_true', help="do not remove the install prefix if installation fails") @@ -98,6 +98,9 @@ def dev_build(self, args): if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + tests = False if args.test == 'all': tests = True diff --git a/lib/spack/spack/cmd/develop.py b/lib/spack/spack/cmd/develop.py index 957060536d..d2dd8d0049 100644 --- a/lib/spack/spack/cmd/develop.py +++ b/lib/spack/spack/cmd/develop.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/docs.py b/lib/spack/spack/cmd/docs.py index a7f2a641ee..ac3a91721e 100644 --- a/lib/spack/spack/cmd/docs.py +++ b/lib/spack/spack/cmd/docs.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index e01d50dd03..a277ed519e 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 4699efd26b..224b74bdeb 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index b4fb7faa1f..a57aa031ff 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index 52af72ed8f..c7d045e66d 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,6 +16,7 @@ import llnl.util.tty.colify as colify import six import spack import spack.cmd +import spack.cmd.common.arguments import spack.error import spack.util.environment import spack.util.spack_yaml as syaml @@ -42,6 +43,7 @@ def setup_parser(subparser): '--scope', choices=scopes, metavar=scopes_metavar, default=spack.config.default_modify_scope('packages'), help="configuration scope to modify") + spack.cmd.common.arguments.add_common_arguments(find_parser, ['tags']) find_parser.add_argument('packages', nargs=argparse.REMAINDER) sp.add_parser( @@ -148,9 +150,28 @@ def _spec_is_valid(spec): def external_find(args): + # Construct the list of possible packages to be detected + packages_to_check = [] + + # Add the packages that have been required explicitly if args.packages: packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages) - else: + if args.tags: + allowed = set(spack.repo.path.packages_with_tags(*args.tags)) + packages_to_check = [x for x in packages_to_check if x in allowed] + + if args.tags and not packages_to_check: + # If we arrived here we didn't have any explicit package passed + # as argument, which means to search all packages. + # Since tags are cached it's much faster to construct what we need + # to search directly, rather than filtering after the fact + packages_to_check = [ + spack.repo.get(pkg) for pkg in + spack.repo.path.packages_with_tags(*args.tags) + ] + + # If the list of packages is empty, search for every possible package + if not args.tags and not packages_to_check: packages_to_check = spack.repo.path.all_packages() pkg_to_entries = _get_external_packages(packages_to_check) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index b91eb52ab8..962bcefce6 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,7 +16,7 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'deprecated']) subparser.add_argument( '-m', '--missing', action='store_true', help="fetch only missing (not yet installed) dependencies") @@ -33,6 +33,9 @@ def fetch(parser, args): if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: if args.missing or args.dependencies: diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index d9d51b853c..9ed889f928 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/flake8.py b/lib/spack/spack/cmd/flake8.py index cdfce2cab2..7da059ba69 100644 --- a/lib/spack/spack/cmd/flake8.py +++ b/lib/spack/spack/cmd/flake8.py @@ -1,321 +1,26 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function -import re -import os -import sys -import shutil -import tempfile -import argparse +import llnl.util.tty as tty -from llnl.util.filesystem import working_dir, mkdirp +import spack.cmd.style -import spack.paths -from spack.util.executable import which - -description = "runs source code style checks on Spack. requires flake8" -section = "developer" -level = "long" - - -def is_package(f): - """Whether flake8 should consider a file as a core file or a package. - - We run flake8 with different exceptions for the core and for - packages, since we allow `from spack import *` and poking globals - into packages. - """ - return f.startswith('var/spack/repos/') or 'docs/tutorial/examples' in f - - -#: List of directories to exclude from checks. -exclude_directories = [spack.paths.external_path] - -#: max line length we're enforcing (note: this duplicates what's in .flake8) -max_line_length = 79 - -#: This is a dict that maps: -#: filename pattern -> -#: flake8 exemption code -> -#: list of patterns, for which matching lines should have codes applied. -#: -#: For each file, if the filename pattern matches, we'll add per-line -#: exemptions if any patterns in the sub-dict match. -pattern_exemptions = { - # exemptions applied only to package.py files. - r'package.py$': { - # Allow 'from spack import *' in packages, but no other wildcards - 'F403': [ - r'^from spack import \*$' - ], - # Exempt lines with urls and descriptions from overlong line errors. - 'E501': [ - r'^\s*homepage\s*=', - r'^\s*url\s*=', - r'^\s*git\s*=', - r'^\s*svn\s*=', - r'^\s*hg\s*=', - r'^\s*list_url\s*=', - r'^\s*version\(', - r'^\s*variant\(', - r'^\s*provides\(', - r'^\s*extends\(', - r'^\s*depends_on\(', - r'^\s*conflicts\(', - r'^\s*resource\(', - r'^\s*patch\(', - ], - # Exempt '@when' decorated functions from redefinition errors. - 'F811': [ - r'^\s*@when\(.*\)', - ], - }, - - # exemptions applied to all files. - r'.py$': { - 'E501': [ - r'(https?|ftp|file)\:', # URLs - r'([\'"])[0-9a-fA-F]{32,}\1', # long hex checksums - ] - }, -} - -# compile all regular expressions. -pattern_exemptions = dict( - (re.compile(file_pattern), - dict((code, [re.compile(p) for p in patterns]) - for code, patterns in error_dict.items())) - for file_pattern, error_dict in pattern_exemptions.items()) - - -def changed_files(base=None, untracked=True, all_files=False): - """Get list of changed files in the Spack repository.""" - - git = which('git', required=True) - - if base is None: - base = os.environ.get('TRAVIS_BRANCH', 'develop') - - range = "{0}...".format(base) - - git_args = [ - # Add changed files committed since branching off of develop - ['diff', '--name-only', '--diff-filter=ACMR', range], - # Add changed files that have been staged but not yet committed - ['diff', '--name-only', '--diff-filter=ACMR', '--cached'], - # Add changed files that are unstaged - ['diff', '--name-only', '--diff-filter=ACMR'], - ] - - # Add new files that are untracked - if untracked: - git_args.append(['ls-files', '--exclude-standard', '--other']) - - # add everything if the user asked for it - if all_files: - git_args.append(['ls-files', '--exclude-standard']) - - excludes = [os.path.realpath(f) for f in exclude_directories] - changed = set() - - for arg_list in git_args: - files = git(*arg_list, output=str).split('\n') - - for f in files: - # Ignore non-Python files - if not (f.endswith('.py') or f == 'bin/spack'): - continue - - # Ignore files in the exclude locations - if any(os.path.realpath(f).startswith(e) for e in excludes): - continue - - changed.add(f) - - return sorted(changed) - - -def add_pattern_exemptions(line, codes): - """Add a flake8 exemption to a line.""" - if line.startswith('#'): - return line - - line = line.rstrip('\n') - - # Line is already ignored - if line.endswith('# noqa'): - return line + '\n' - - orig_len = len(line) - codes = set(codes) - - # don't add E501 unless the line is actually too long, as it can mask - # other errors like trailing whitespace - if orig_len <= max_line_length and "E501" in codes: - codes.remove("E501") - if not codes: - return line + "\n" - - exemptions = ','.join(sorted(codes)) - - # append exemption to line - if '# noqa: ' in line: - line += ',{0}'.format(exemptions) - elif line: # ignore noqa on empty lines - line += ' # noqa: {0}'.format(exemptions) - - # if THIS made the line too long, add an exemption for that - if len(line) > max_line_length and orig_len <= max_line_length: - line += ',E501' - - return line + '\n' - - -def filter_file(source, dest, output=False): - """Filter a single file through all the patterns in pattern_exemptions.""" - - # Prior to Python 3.8, `noqa: F811` needed to be placed on the `@when` line - # Starting with Python 3.8, it must be placed on the `def` line - # https://gitlab.com/pycqa/flake8/issues/583 - ignore_f811_on_previous_line = False - - with open(source) as infile: - parent = os.path.dirname(dest) - mkdirp(parent) - - with open(dest, 'w') as outfile: - for line in infile: - line_errors = [] - - # pattern exemptions - for file_pattern, errors in pattern_exemptions.items(): - if not file_pattern.search(source): - continue - - for code, patterns in errors.items(): - for pattern in patterns: - if pattern.search(line): - line_errors.append(code) - break - - if 'F811' in line_errors: - ignore_f811_on_previous_line = True - elif ignore_f811_on_previous_line: - line_errors.append('F811') - ignore_f811_on_previous_line = False - - if line_errors: - line = add_pattern_exemptions(line, line_errors) - - outfile.write(line) - if output: - sys.stdout.write(line) +description = "alias for spack style (deprecated)" +section = spack.cmd.style.section +level = spack.cmd.style.level def setup_parser(subparser): - subparser.add_argument( - '-b', '--base', action='store', default=None, - help="select base branch for collecting list of modified files") - subparser.add_argument( - '-k', '--keep-temp', action='store_true', - help="do not delete temporary directory where flake8 runs. " - "use for debugging, to see filtered files") - subparser.add_argument( - '-a', '--all', action='store_true', - help="check all files, not just changed files") - subparser.add_argument( - '-o', '--output', action='store_true', - help="send filtered files to stdout as well as temp files") - subparser.add_argument( - '-r', '--root-relative', action='store_true', default=False, - help="print root-relative paths (default: cwd-relative)") - subparser.add_argument( - '-U', '--no-untracked', dest='untracked', action='store_false', - default=True, help="exclude untracked files from checks") - subparser.add_argument( - 'files', nargs=argparse.REMAINDER, help="specific files to check") + spack.cmd.style.setup_parser(subparser) def flake8(parser, args): - flake8 = which('flake8', required=True) - - temp = tempfile.mkdtemp() - try: - file_list = args.files - if file_list: - def prefix_relative(path): - return os.path.relpath( - os.path.abspath(os.path.realpath(path)), - spack.paths.prefix) - - file_list = [prefix_relative(p) for p in file_list] - - with working_dir(spack.paths.prefix): - if not file_list: - file_list = changed_files(args.base, args.untracked, args.all) - - print('=======================================================') - print('flake8: running flake8 code checks on spack.') - print() - print('Modified files:') - for filename in file_list: - print(' {0}'.format(filename.strip())) - print('=======================================================') - - # filter files into a temporary directory with exemptions added. - for filename in file_list: - src_path = os.path.join(spack.paths.prefix, filename) - dest_path = os.path.join(temp, filename) - filter_file(src_path, dest_path, args.output) - - # run flake8 on the temporary tree, once for core, once for pkgs - package_file_list = [f for f in file_list if is_package(f)] - file_list = [f for f in file_list if not is_package(f)] - - returncode = 0 - with working_dir(temp): - output = '' - if file_list: - output += flake8( - '--format', 'pylint', - '--config=%s' % os.path.join(spack.paths.prefix, - '.flake8'), - *file_list, fail_on_error=False, output=str) - returncode |= flake8.returncode - if package_file_list: - output += flake8( - '--format', 'pylint', - '--config=%s' % os.path.join(spack.paths.prefix, - '.flake8_packages'), - *package_file_list, fail_on_error=False, output=str) - returncode |= flake8.returncode - - if args.root_relative: - # print results relative to repo root. - print(output) - else: - # print results relative to current working directory - def cwd_relative(path): - return '{0}: ['.format(os.path.relpath( - os.path.join( - spack.paths.prefix, path.group(1)), os.getcwd())) - - for line in output.split('\n'): - print(re.sub(r'^(.*): \[', cwd_relative, line)) - - if returncode != 0: - print('Flake8 found errors.') - sys.exit(1) - else: - print('Flake8 checks were clean.') - - finally: - if args.keep_temp: - print('Temporary files are in: ', temp) - else: - shutil.rmtree(temp, ignore_errors=True) + tty.warn( + "spack flake8 is deprecated", "please use `spack style` to run style checks" + ) + return spack.cmd.style.style(parser, args) diff --git a/lib/spack/spack/cmd/gc.py b/lib/spack/spack/cmd/gc.py index ca9d88169a..f85f27bfca 100644 --- a/lib/spack/spack/cmd/gc.py +++ b/lib/spack/spack/cmd/gc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 59fba4a4df..e34c2c4c84 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index d0fbf8e6c6..3d962a52f8 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py index d730d4f0fe..bc289c0363 100644 --- a/lib/spack/spack/cmd/help.py +++ b/lib/spack/spack/cmd/help.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index fa674317ab..6b316f3cc4 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -111,7 +111,7 @@ class VariantFormatter(object): yield ' None' else: yield ' ' + self.fmt % self.headers - underline = tuple([l * "=" for l in self.column_widths]) + underline = tuple([w * "=" for w in self.column_widths]) yield ' ' + self.fmt % underline yield '' for k, v in sorted(self.variants.items()): @@ -189,10 +189,11 @@ def print_text_info(pkg): color.cprint(section_title('Safe versions: ')) for v in reversed(sorted(pkg.versions)): - if pkg.has_code: - url = fs.for_package_version(pkg, v) - line = version(' {0}'.format(pad(v))) + color.cescape(url) - color.cprint(line) + if not pkg.versions[v].get('deprecated', False): + if pkg.has_code: + url = fs.for_package_version(pkg, v) + line = version(' {0}'.format(pad(v))) + color.cescape(url) + color.cprint(line) color.cprint('') color.cprint(section_title('Variants:')) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 8f3902d71c..5e572f82f6 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -43,6 +43,7 @@ def update_kwargs_from_args(args, kwargs): 'dirty': args.dirty, 'use_cache': args.use_cache, 'cache_only': args.cache_only, + 'include_build_deps': args.include_build_deps, 'explicit': True, # Always true for install command 'stop_at': args.until, 'unsigned': args.unsigned, @@ -106,6 +107,11 @@ the dependencies""" help="only install package from binary mirrors") subparser.add_argument( + '--include-build-deps', action='store_true', dest='include_build_deps', + default=False, help="""include build deps when installing from cache, +which is useful for CI pipeline troubleshooting""") + + subparser.add_argument( '--no-check-signature', action='store_true', dest='unsigned', default=False, help="do not check signatures of binary packages") @@ -120,7 +126,7 @@ remote spec matches that of the local spec""") subparser.add_argument( '--source', action='store_true', dest='install_source', help="install source files in prefix") - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'deprecated']) subparser.add_argument( '-v', '--verbose', action='store_true', help="display verbose build output while installing") @@ -282,6 +288,9 @@ environment variables: if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + # Parse cli arguments and construct a dictionary # that will be passed to the package installer update_kwargs_from_args(args, kwargs) diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index 84a4a02282..8af0f4fa26 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,6 +9,7 @@ import os import re from collections import defaultdict +import llnl.util.filesystem as fs import llnl.util.tty as tty import spack.paths @@ -36,12 +37,14 @@ licensed_files = [ # all of spack core r'^lib/spack/spack/.*\.py$', r'^lib/spack/spack/.*\.sh$', + r'^lib/spack/spack/.*\.lp$', r'^lib/spack/llnl/.*\.py$', r'^lib/spack/env/cc$', # rst files in documentation r'^lib/spack/docs/(?!command_index|spack|llnl).*\.rst$', r'^lib/spack/docs/.*\.py$', + r'^lib/spack/docs/spack.yaml$', # 2 files in external r'^lib/spack/external/__init__.py$', @@ -51,7 +54,13 @@ licensed_files = [ r'^share/spack/.*\.sh$', r'^share/spack/.*\.bash$', r'^share/spack/.*\.csh$', + r'^share/spack/.*\.fish$', r'^share/spack/qa/run-[^/]*$', + r'^share/spack/bash/spack-completion.in$', + r'^share/spack/templates/misc/coconcretization.pyt$', + + # action workflows + r'^.github/actions/.*\.py$', # all packages r'^var/spack/repos/.*/package.py$' @@ -77,15 +86,15 @@ def _all_spack_files(root=spack.paths.prefix): visited.add(path) -def _licensed_files(root=spack.paths.prefix): - for relpath in _all_spack_files(root): +def _licensed_files(args): + for relpath in _all_spack_files(args.root): if any(regex.match(relpath) for regex in licensed_files): yield relpath def list_files(args): """list files in spack that should have license headers""" - for relpath in sorted(_licensed_files()): + for relpath in sorted(_licensed_files(args)): print(os.path.join(spack.paths.spack_root, relpath)) @@ -93,6 +102,8 @@ def list_files(args): # bool(value) evaluates to True OLD_LICENSE, SPDX_MISMATCH, GENERAL_MISMATCH = range(1, 4) +strict_date = r'Copyright 2013-2021' + class LicenseError(object): def __init__(self): @@ -118,17 +129,15 @@ class LicenseError(object): def _check_license(lines, path): license_lines = [ - r'Copyright 2013-(?:201[789]|202\d) Lawrence Livermore National Security, LLC and other', # noqa: E501 + r'Copyright 2013-(?:202[01]) Lawrence Livermore National Security, LLC and other', # noqa: E501 r'Spack Project Developers\. See the top-level COPYRIGHT file for details.', # noqa: E501 r'SPDX-License-Identifier: \(Apache-2\.0 OR MIT\)' ] - strict_date = r'Copyright 2013-2020' - found = [] for line in lines: - line = re.sub(r'^[\s#\.]*', '', line) + line = re.sub(r'^[\s#\%\.]*', '', line) line = line.rstrip() for i, license_line in enumerate(license_lines): if re.match(license_line, line): @@ -175,7 +184,7 @@ def verify(args): license_errors = LicenseError() - for relpath in _licensed_files(args.root): + for relpath in _licensed_files(args): path = os.path.join(args.root, relpath) with open(path) as f: lines = [line for line in f][:license_lines] @@ -190,15 +199,28 @@ def verify(args): tty.msg('No license issues found.') -def setup_parser(subparser): - sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='license_command') - sp.add_parser('list-files', help=list_files.__doc__) +def update_copyright_year(args): + """update copyright for the current year in all licensed files""" - verify_parser = sp.add_parser('verify', help=verify.__doc__) - verify_parser.add_argument( + llns_and_other = ' Lawrence Livermore National Security, LLC and other' + for filename in _licensed_files(args): + fs.filter_file( + r'Copyright \d{4}-\d{4}' + llns_and_other, + strict_date + llns_and_other, + os.path.join(args.root, filename) + ) + + +def setup_parser(subparser): + subparser.add_argument( '--root', action='store', default=spack.paths.prefix, help='scan a different prefix for license issues') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='license_command') + sp.add_parser('list-files', help=list_files.__doc__) + sp.add_parser('verify', help=verify.__doc__) + sp.add_parser('update-copyright-year', help=update_copyright_year.__doc__) + def license(parser, args): if not git: @@ -209,5 +231,6 @@ def license(parser, args): commands = { 'list-files': list_files, 'verify': verify, + 'update-copyright-year': update_copyright_year, } return commands[args.license_command](args) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index a7d4cb0ac3..c00e7a903d 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index a8f59fef2f..df0d56c444 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -63,7 +63,7 @@ def load(parser, args): specs_str = ' '.join(args.specs) or "SPECS" spack.cmd.common.shell_init_instructions( "spack load", - " eval `spack load {sh_arg}` %s" % specs_str, + " eval `spack load {sh_arg} %s`" % specs_str, ) return 1 diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 60978fe404..a1f35423ff 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/log_parse.py b/lib/spack/spack/cmd/log_parse.py index faa44a4ef0..4f7fd119c0 100644 --- a/lib/spack/spack/cmd/log_parse.py +++ b/lib/spack/spack/cmd/log_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/maintainers.py b/lib/spack/spack/cmd/maintainers.py index a1cf477146..9813103704 100644 --- a/lib/spack/spack/cmd/maintainers.py +++ b/lib/spack/spack/cmd/maintainers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/mark.py b/lib/spack/spack/cmd/mark.py index 85a22b9741..96c041ba72 100644 --- a/lib/spack/spack/cmd/mark.py +++ b/lib/spack/spack/cmd/mark.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2d338204d3..45aba2441a 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,7 +28,7 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'deprecated']) sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='mirror_command') @@ -67,6 +67,19 @@ def setup_parser(subparser): " retrieve all versions of each package") arguments.add_common_arguments(create_parser, ['specs']) + # Destroy + destroy_parser = sp.add_parser('destroy', help=mirror_destroy.__doc__) + + destroy_target = destroy_parser.add_mutually_exclusive_group(required=True) + destroy_target.add_argument('-m', '--mirror-name', + metavar='mirror_name', + type=str, + help="find mirror to destroy by name") + destroy_target.add_argument('--mirror-url', + metavar='mirror_url', + type=str, + help="find mirror to destroy by url") + # used to construct scope arguments below scopes = spack.config.scopes() scopes_metavar = spack.config.scopes_metavar @@ -360,8 +373,22 @@ def mirror_create(args): sys.exit(1) +def mirror_destroy(args): + """Given a url, recursively delete everything under it.""" + mirror_url = None + + if args.mirror_name: + result = spack.mirror.MirrorCollection().lookup(args.mirror_name) + mirror_url = result.push_url + elif args.mirror_url: + mirror_url = args.mirror_url + + web_util.remove_url(mirror_url, recursive=True) + + def mirror(parser, args): action = {'create': mirror_create, + 'destroy': mirror_destroy, 'add': mirror_add, 'remove': mirror_remove, 'rm': mirror_remove, @@ -371,4 +398,7 @@ def mirror(parser, args): if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + action[args.mirror_command](args) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index f86f3e5f25..c05d31ac40 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -1,9 +1,10 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse +from typing import Dict, Callable # novm import llnl.util.tty as tty @@ -15,7 +16,7 @@ section = "user environment" level = "short" -_subcommands = {} +_subcommands = {} # type: Dict[str, Callable] _deprecated_commands = ('refresh', 'find', 'rm', 'loads') diff --git a/lib/spack/spack/cmd/modules/__init__.py b/lib/spack/spack/cmd/modules/__init__.py index 7f3dd29ef1..7fbce3bb9d 100644 --- a/lib/spack/spack/cmd/modules/__init__.py +++ b/lib/spack/spack/cmd/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/modules/lmod.py b/lib/spack/spack/cmd/modules/lmod.py index d3610bf333..61f2fc28d8 100644 --- a/lib/spack/spack/cmd/modules/lmod.py +++ b/lib/spack/spack/cmd/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/modules/tcl.py b/lib/spack/spack/cmd/modules/tcl.py index cdde21e821..a9486b9de4 100644 --- a/lib/spack/spack/cmd/modules/tcl.py +++ b/lib/spack/spack/cmd/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 8f91edb8f1..14739b8d7d 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,7 +16,8 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum', 'specs']) + arguments.add_common_arguments( + subparser, ['no_checksum', 'deprecated', 'specs']) def patch(parser, args): @@ -26,6 +27,9 @@ def patch(parser, args): if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index b988d6a848..f7e3523ff4 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 8ca34542eb..5effdacf8c 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/pydoc.py b/lib/spack/spack/cmd/pydoc.py index 1f81718e38..6e834932d0 100644 --- a/lib/spack/spack/cmd/pydoc.py +++ b/lib/spack/spack/cmd/pydoc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 019e0cf27e..588ddce853 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,6 +28,9 @@ def setup_parser(subparser): subparser.add_argument( '-c', dest='python_command', help='command to execute') subparser.add_argument( + '-i', dest='python_interpreter', help='python interpreter', + choices=['python', 'ipython'], default='python') + subparser.add_argument( '-m', dest='module', action='store', help='run library module as a script') subparser.add_argument( @@ -48,24 +51,63 @@ def python(parser, args, unknown_args): if unknown_args: tty.die("Unknown arguments:", " ".join(unknown_args)) + # Unexpected behavior from supplying both + if args.python_command and args.python_args: + tty.die("You can only specify a command OR script, but not both.") + + # Run user choice of interpreter + if args.python_interpreter == "ipython": + return spack.cmd.python.ipython_interpreter(args) + return spack.cmd.python.python_interpreter(args) + + +def ipython_interpreter(args): + """An ipython interpreter is intended to be interactive, so it doesn't + support running a script or arguments + """ + try: + import IPython + except ImportError: + tty.die("ipython is not installed, install and try again.") + + if "PYTHONSTARTUP" in os.environ: + startup_file = os.environ["PYTHONSTARTUP"] + if os.path.isfile(startup_file): + with open(startup_file) as startup: + exec(startup.read()) + + # IPython can also support running a script OR command, not both + if args.python_args: + IPython.start_ipython(argv=args.python_args) + elif args.python_command: + IPython.start_ipython(argv=['-c', args.python_command]) + else: + header = ("Spack version %s\nPython %s, %s %s" + % (spack.spack_version, platform.python_version(), + platform.system(), platform.machine())) + + __name__ = "__main__" # noqa + IPython.embed(module="__main__", header=header) + + +def python_interpreter(args): + """A python interpreter is the default interpreter + """ # Fake a main python shell by setting __name__ to __main__. console = code.InteractiveConsole({'__name__': '__main__', 'spack': spack}) - if "PYTHONSTARTUP" in os.environ: startup_file = os.environ["PYTHONSTARTUP"] if os.path.isfile(startup_file): with open(startup_file) as startup: console.runsource(startup.read(), startup_file, 'exec') - python_args = args.python_args - python_command = args.python_command - if python_command: - console.runsource(python_command) - elif python_args: - sys.argv = python_args - with open(python_args[0]) as file: - console.runsource(file.read(), python_args[0], 'exec') + if args.python_command: + console.runsource(args.python_command) + elif args.python_args: + sys.argv = args.python_args + with open(args.python_args[0]) as file: + console.runsource(file.read(), args.python_args[0], 'exec') else: # Provides readline support, allowing user to use arrow keys console.push('import readline') diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index b9f3a2ac3b..1ff5488f5b 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py index ef01052c29..376d73b687 100644 --- a/lib/spack/spack/cmd/remove.py +++ b/lib/spack/spack/cmd/remove.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index f271790994..3930a03d0f 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/resource.py b/lib/spack/spack/cmd/resource.py index 0d26c39ab8..ed9e7982fa 100644 --- a/lib/spack/spack/cmd/resource.py +++ b/lib/spack/spack/cmd/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 0f55884bfe..b0582498d6 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py deleted file mode 100644 index 45919893cc..0000000000 --- a/lib/spack/spack/cmd/setup.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import argparse -import copy -import os -import sys - -import llnl.util.tty as tty -from llnl.util.filesystem import set_executable - -import spack.repo -import spack.store -import spack.build_systems.cmake -import spack.cmd -import spack.cmd.install as install -import spack.cmd.common.arguments as arguments -from spack.util.executable import which - -from spack.stage import DIYStage - -description = "create a configuration script and module, but don't build" -section = "build" -level = "long" - - -def setup_parser(subparser): - subparser.add_argument( - '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - help="do not try to install dependencies of requested packages") - arguments.add_common_arguments(subparser, ['no_checksum', 'spec']) - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="display verbose build output while installing") - - cd_group = subparser.add_mutually_exclusive_group() - arguments.add_common_arguments(cd_group, ['clean', 'dirty']) - subparser.epilog = 'DEPRECATED: use `spack dev-build` instead' - - -def write_spconfig(package, dirty): - # Set-up the environment - spack.build_environment.setup_package(package, dirty) - - cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args() - - env = dict() - - paths = os.environ['PATH'].split(':') - paths = [item for item in paths if 'spack/env' not in item] - env['PATH'] = ':'.join(paths) - env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH'] - env['SPACK_INCLUDE_DIRS'] = os.environ['SPACK_INCLUDE_DIRS'] - env['CC'] = os.environ['SPACK_CC'] - env['CXX'] = os.environ['SPACK_CXX'] - env['FC'] = os.environ['SPACK_FC'] - - setup_fname = 'spconfig.py' - with open(setup_fname, 'w') as fout: - fout.write( - r"""#!%s -# - -import sys -import os -import subprocess - -def cmdlist(str): - return list(x.strip().replace("'",'') for x in str.split('\n') if x) -env = dict(os.environ) -""" % sys.executable) - - env_vars = sorted(list(env.keys())) - for name in env_vars: - val = env[name] - if name.find('PATH') < 0: - fout.write('env[%s] = %s\n' % (repr(name), repr(val))) - else: - if name == 'SPACK_INCLUDE_DIRS': - sep = ';' - else: - sep = ':' - - fout.write( - 'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep)) - for part in val.split(sep): - fout.write(' %s\n' % part) - fout.write('"""))\n') - - fout.write('\ncmd = cmdlist("""\n') - fout.write('%s\n' % cmd[0]) - for arg in cmd[1:]: - fout.write(' %s\n' % arg) - fout.write('""") + sys.argv[1:]\n') - fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n') - set_executable(setup_fname) - - -def setup(self, args): - tty.warn('DEPRECATED: use `spack dev-build` instead') - - if not args.spec: - tty.die("spack setup requires a package spec argument.") - - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack setup only takes one spec.") - - # Take a write lock before checking for existence. - with spack.store.db.write_transaction(): - spec = specs[0] - if not spack.repo.path.exists(spec.name): - tty.die("No package for '{0}' was found.".format(spec.name), - " Use `spack create` to create a new package") - if not spec.versions.concrete: - tty.die( - "spack setup spec must have a single, concrete version. " - "Did you forget a package version number?") - - spec.concretize() - package = spack.repo.get(spec) - if not isinstance(package, spack.build_systems.cmake.CMakePackage): - tty.die( - 'Support for {0} derived packages not yet implemented'.format( - package.build_system_class)) - - # It's OK if the package is already installed. - - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) - - # disable checksumming if requested - if args.no_checksum: - spack.config.set('config:checksum', False, scope='command_line') - - # Install dependencies if requested to do so - if not args.ignore_deps: - parser = argparse.ArgumentParser() - install.setup_parser(parser) - inst_args = copy.deepcopy(args) - inst_args = parser.parse_args( - ['--only=dependencies'] + args.spec, - namespace=inst_args - ) - install.install(parser, inst_args) - - # Generate spconfig.py - tty.msg( - 'Generating spconfig.py [{0}]'.format(package.spec.cshort_spec) - ) - dirty = args.dirty - write_spconfig(package, dirty) - - # Install this package to register it in the DB and permit - # module file regeneration - inst_args = copy.deepcopy(args) - inst_args = parser.parse_args( - ['--only=package', '--fake'] + args.spec, - namespace=inst_args - ) - install.install(parser, inst_args) diff --git a/lib/spack/spack/cmd/solve.py b/lib/spack/spack/cmd/solve.py index ff6ab4e676..64219b300f 100644 --- a/lib/spack/spack/cmd/solve.py +++ b/lib/spack/spack/cmd/solve.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index cbfd58ec1a..8b4e9e5276 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 1acefb723c..62073f10bd 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,7 +16,8 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum', 'specs']) + arguments.add_common_arguments( + subparser, ['no_checksum', 'deprecated', 'specs']) subparser.add_argument( '-p', '--path', dest='path', help="path to stage package, does not add to spack tree") @@ -37,6 +38,9 @@ def stage(parser, args): if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') + if args.deprecated: + spack.config.set('config:deprecated', True, scope='command_line') + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py new file mode 100644 index 0000000000..c35c07a8f2 --- /dev/null +++ b/lib/spack/spack/cmd/style.py @@ -0,0 +1,320 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from __future__ import print_function + +import re +import os +import sys +import argparse + +from llnl.util.filesystem import working_dir +import llnl.util.tty as tty + +import spack.paths +from spack.util.executable import which + +if sys.version_info < (3, 0): + from itertools import izip_longest # novm + + zip_longest = izip_longest +else: + from itertools import zip_longest # novm + + +description = ( + "runs source code style checks on Spack. Requires flake8, mypy, black for " + + "their respective checks" +) +section = "developer" +level = "long" + + +def grouper(iterable, n, fillvalue=None): + "Collect data into fixed-length chunks or blocks" + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) + + +#: List of directories to exclude from checks. +exclude_directories = [spack.paths.external_path] + +#: max line length we're enforcing (note: this duplicates what's in .flake8) +max_line_length = 79 + + +def is_package(f): + """Whether flake8 should consider a file as a core file or a package. + + We run flake8 with different exceptions for the core and for + packages, since we allow `from spack import *` and poking globals + into packages. + """ + return f.startswith("var/spack/repos/") or "docs/tutorial/examples" in f + + +def changed_files(base=None, untracked=True, all_files=False): + """Get list of changed files in the Spack repository.""" + + git = which("git", required=True) + + if base is None: + base = os.environ.get("TRAVIS_BRANCH", "develop") + + range = "{0}...".format(base) + + git_args = [ + # Add changed files committed since branching off of develop + ["diff", "--name-only", "--diff-filter=ACMR", range], + # Add changed files that have been staged but not yet committed + ["diff", "--name-only", "--diff-filter=ACMR", "--cached"], + # Add changed files that are unstaged + ["diff", "--name-only", "--diff-filter=ACMR"], + ] + + # Add new files that are untracked + if untracked: + git_args.append(["ls-files", "--exclude-standard", "--other"]) + + # add everything if the user asked for it + if all_files: + git_args.append(["ls-files", "--exclude-standard"]) + + excludes = [os.path.realpath(f) for f in exclude_directories] + changed = set() + + for arg_list in git_args: + files = git(*arg_list, output=str).split("\n") + + for f in files: + # Ignore non-Python files + if not (f.endswith(".py") or f == "bin/spack"): + continue + + # Ignore files in the exclude locations + if any(os.path.realpath(f).startswith(e) for e in excludes): + continue + + changed.add(f) + + return sorted(changed) + + +def setup_parser(subparser): + subparser.add_argument( + "-b", + "--base", + action="store", + default=None, + help="select base branch for collecting list of modified files", + ) + subparser.add_argument( + "-a", + "--all", + action="store_true", + help="check all files, not just changed files", + ) + subparser.add_argument( + "-o", + "--output", + action="store_true", + help="send filtered files to stdout as well as temp files", + ) + subparser.add_argument( + "-r", + "--root-relative", + action="store_true", + default=False, + help="print root-relative paths (default: cwd-relative)", + ) + subparser.add_argument( + "-U", + "--no-untracked", + dest="untracked", + action="store_false", + default=True, + help="exclude untracked files from checks", + ) + subparser.add_argument( + "--no-flake8", + dest="flake8", + action="store_false", + help="Do not run flake8, default is run flake8", + ) + subparser.add_argument( + "--no-mypy", + dest="mypy", + action="store_false", + help="Do not run mypy, default is run mypy if available", + ) + subparser.add_argument( + "--black", + dest="black", + action="store_true", + help="Run black checks, default is skip", + ) + subparser.add_argument( + "files", nargs=argparse.REMAINDER, help="specific files to check" + ) + + +def rewrite_and_print_output( + output, args, re_obj=re.compile(r"^(.+):([0-9]+):"), replacement=r"{0}:{1}:" +): + """rewrite ouput with <file>:<line>: format to respect path args""" + if args.root_relative or re_obj is None: + # print results relative to repo root. + print(output) + else: + # print results relative to current working directory + def cwd_relative(path): + return replacement.format( + os.path.relpath( + os.path.join(spack.paths.prefix, path.group(1)), os.getcwd() + ), + *list(path.groups()[1:]) + ) + + for line in output.split("\n"): + if not line: + continue + print(re_obj.sub(cwd_relative, line)) + + +def print_style_header(file_list, args): + tty.msg("style: running code checks on spack.") + tools = [] + if args.flake8: + tools.append("flake8") + if args.mypy: + tools.append("mypy") + if args.black: + tools.append("black") + tty.msg("style: tools selected: " + ", ".join(tools)) + tty.msg("Modified files:", *[filename.strip() for filename in file_list]) + sys.stdout.flush() + + +def print_tool_header(tool): + sys.stdout.flush() + tty.msg("style: running %s checks on spack." % tool) + sys.stdout.flush() + + +def run_flake8(file_list, args): + returncode = 0 + print_tool_header("flake8") + flake8_cmd = which("flake8", required=True) + + output = "" + # run in chunks of 100 at a time to avoid line length limit + # filename parameter in config *does not work* for this reliably + for chunk in grouper(file_list, 100): + chunk = filter(lambda e: e is not None, chunk) + + output = flake8_cmd( + # use .flake8 implicitly to work around bug in flake8 upstream + # append-config is ignored if `--config` is explicitly listed + # see: https://gitlab.com/pycqa/flake8/-/issues/455 + # "--config=.flake8", + *chunk, + fail_on_error=False, + output=str + ) + returncode |= flake8_cmd.returncode + + rewrite_and_print_output(output, args) + + if returncode == 0: + tty.msg("Flake8 style checks were clean") + else: + tty.error("Flake8 style checks found errors") + return returncode + + +def run_mypy(file_list, args): + mypy_cmd = which("mypy") + if mypy_cmd is None: + tty.error("style: mypy is not available in path, skipping") + return 1 + + print_tool_header("mypy") + mpy_args = ["--package", "spack", "--package", "llnl"] + # not yet, need other updates to enable this + # if any([is_package(f) for f in file_list]): + # mpy_args.extend(["--package", "packages"]) + + output = mypy_cmd(*mpy_args, fail_on_error=False, output=str) + returncode = mypy_cmd.returncode + + rewrite_and_print_output(output, args) + + if returncode == 0: + tty.msg("mypy checks were clean") + else: + tty.error("mypy checks found errors") + return returncode + + +def run_black(file_list, args): + black_cmd = which("black") + if black_cmd is None: + tty.error("style: black is not available in path, skipping") + return 1 + + print_tool_header("black") + + pat = re.compile("would reformat +(.*)") + replacement = "would reformat {0}" + returncode = 0 + output = "" + # run in chunks of 100 at a time to avoid line length limit + # filename parameter in config *does not work* for this reliably + for chunk in grouper(file_list, 100): + chunk = filter(lambda e: e is not None, chunk) + + output = black_cmd( + "--check", "--diff", *chunk, fail_on_error=False, output=str, error=str + ) + returncode |= black_cmd.returncode + + rewrite_and_print_output(output, args, pat, replacement) + + if returncode == 0: + tty.msg("black style checks were clean") + else: + tty.error("black checks found errors") + return returncode + + +def style(parser, args): + file_list = args.files + if file_list: + + def prefix_relative(path): + return os.path.relpath( + os.path.abspath(os.path.realpath(path)), spack.paths.prefix + ) + + file_list = [prefix_relative(p) for p in file_list] + + returncode = 0 + with working_dir(spack.paths.prefix): + if not file_list: + file_list = changed_files(args.base, args.untracked, args.all) + print_style_header(file_list, args) + if args.flake8: + returncode = run_flake8(file_list, args) + if args.mypy: + returncode |= run_mypy(file_list, args) + if args.black: + returncode |= run_black(file_list, args) + + if returncode != 0: + print("spack style found errors.") + sys.exit(1) + else: + print("spack style checks were clean.") diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index d164e26334..e1aedabb81 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/test_env.py b/lib/spack/spack/cmd/test_env.py index a0f54d482f..3d3827a7da 100644 --- a/lib/spack/spack/cmd/test_env.py +++ b/lib/spack/spack/cmd/test_env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/tutorial.py b/lib/spack/spack/cmd/tutorial.py index 2a32242c2d..fb51e27036 100644 --- a/lib/spack/spack/cmd/tutorial.py +++ b/lib/spack/spack/cmd/tutorial.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/undevelop.py b/lib/spack/spack/cmd/undevelop.py index 6319b88e06..09fe558d56 100644 --- a/lib/spack/spack/cmd/undevelop.py +++ b/lib/spack/spack/cmd/undevelop.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e541eaf91b..2a9e6b00b7 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 509211de04..c290bc8572 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -146,7 +146,7 @@ def add_back_pytest_args(args, unknown_args): def unit_test(parser, args, unknown_args): if args.pytest_help: # make the pytest.main help output more accurate - sys.argv[0] = 'spack test' + sys.argv[0] = 'spack unit-test' return pytest.main(['-h']) # add back any parsed pytest args we need to pass to pytest diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 8494453489..c854466e05 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index db8f358887..e5dde5e9c7 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/verify.py b/lib/spack/spack/cmd/verify.py index b20d795ce5..6e7ff0f3bd 100644 --- a/lib/spack/spack/cmd/verify.py +++ b/lib/spack/spack/cmd/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index 366307f0b2..c124ed37bc 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -12,6 +12,7 @@ import llnl.util.tty as tty import spack.cmd.common.arguments as arguments import spack.repo +from spack.version import VersionList, ver description = "list available versions of a package" section = "packaging" @@ -19,8 +20,17 @@ level = "long" def setup_parser(subparser): - subparser.add_argument('-s', '--safe-only', action='store_true', - help='only list safe versions of the package') + output = subparser.add_mutually_exclusive_group() + output.add_argument('-s', '--safe', action='store_true', + help='only list safe versions of the package') + output.add_argument('--safe-only', action='store_true', + help='[deprecated] only list safe versions ' + 'of the package') + output.add_argument('-r', '--remote', action='store_true', + help='only list remote versions of the package') + output.add_argument('-n', '--new', action='store_true', + help='only list remote versions newer than ' + 'the latest checksummed version') subparser.add_argument( '-c', '--concurrency', default=32, type=int, help='number of concurrent requests' @@ -31,26 +41,38 @@ def setup_parser(subparser): def versions(parser, args): pkg = spack.repo.get(args.package) - if sys.stdout.isatty(): - tty.msg('Safe versions (already checksummed):') - safe_versions = pkg.versions - if not safe_versions: + if args.safe_only: + tty.warn('"--safe-only" is deprecated. Use "--safe" instead.') + args.safe = args.safe_only + + if not (args.remote or args.new): if sys.stdout.isatty(): - tty.warn('Found no versions for {0}'.format(pkg.name)) - tty.debug('Manually add versions to the package.') - else: - colify(sorted(safe_versions, reverse=True), indent=2) + tty.msg('Safe versions (already checksummed):') - if args.safe_only: - return + if not safe_versions: + if sys.stdout.isatty(): + tty.warn('Found no versions for {0}'.format(pkg.name)) + tty.debug('Manually add versions to the package.') + else: + colify(sorted(safe_versions, reverse=True), indent=2) - if sys.stdout.isatty(): - tty.msg('Remote versions (not yet checksummed):') + if args.safe: + return fetched_versions = pkg.fetch_remote_versions(args.concurrency) - remote_versions = set(fetched_versions).difference(safe_versions) + + if args.new: + if sys.stdout.isatty(): + tty.msg('New remote versions (not yet checksummed):') + highest_safe_version = VersionList(safe_versions).highest_numeric() + remote_versions = set([ver(v) for v in set(fetched_versions) + if v > highest_safe_version]) + else: + if sys.stdout.isatty(): + tty.msg('Remote versions (not yet checksummed):') + remote_versions = set(fetched_versions).difference(safe_versions) if not remote_versions: if sys.stdout.isatty(): diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 151f6c1564..ec8003bbf7 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index d498803633..f6c7e98ff0 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,6 +10,7 @@ import re import itertools import shutil import tempfile +from typing import Sequence, List # novm import llnl.util.lang from llnl.util.filesystem import ( @@ -190,20 +191,20 @@ class Compiler(object): and how to identify the particular type of compiler.""" # Subclasses use possible names of C compiler - cc_names = [] + cc_names = [] # type: List[str] # Subclasses use possible names of C++ compiler - cxx_names = [] + cxx_names = [] # type: List[str] # Subclasses use possible names of Fortran 77 compiler - f77_names = [] + f77_names = [] # type: List[str] # Subclasses use possible names of Fortran 90 compiler - fc_names = [] + fc_names = [] # type: List[str] # Optional prefix regexes for searching for this type of compiler. # Prefixes are sometimes used for toolchains - prefixes = [] + prefixes = [] # type: List[str] # Optional suffix regexes for searching for this type of compiler. # Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y' @@ -214,7 +215,7 @@ class Compiler(object): version_argument = '-dumpversion' #: Return values to ignore when invoking the compiler to get its version - ignore_version_errors = () + ignore_version_errors = () # type: Sequence[int] #: Regex used to extract version from compiler's output version_regex = '(.*)' @@ -266,9 +267,9 @@ class Compiler(object): return ['-O', '-O0', '-O1', '-O2', '-O3'] # Cray PrgEnv name that can be used to load this compiler - PrgEnv = None + PrgEnv = None # type: str # Name of module used to switch versions of this compiler - PrgEnv_compiler = None + PrgEnv_compiler = None # type: str def __init__(self, cspec, operating_system, target, paths, modules=None, alias=None, environment=None, diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3c1f9ffa44..83503f09ed 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,6 +11,7 @@ import itertools import multiprocessing.pool import os import six +from typing import Dict # novm import llnl.util.lang import llnl.util.filesystem as fs @@ -21,6 +22,7 @@ import spack.paths import spack.error import spack.spec import spack.config +import spack.compiler import spack.architecture import spack.util.imp as simp from spack.util.environment import get_path @@ -36,7 +38,7 @@ _cache_config_file = [] # TODO: Caches at module level make it difficult to mock configurations in # TODO: unit tests. It might be worth reworking their implementation. #: cache of compilers constructed from config data, keyed by config entry id. -_compiler_cache = {} +_compiler_cache = {} # type: Dict[str, spack.compiler.Compiler] _compiler_to_pkg = { 'clang': 'llvm+clang', diff --git a/lib/spack/spack/compilers/aocc.py b/lib/spack/spack/compilers/aocc.py index 5c9e885490..725767b212 100644 --- a/lib/spack/spack/compilers/aocc.py +++ b/lib/spack/spack/compilers/aocc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -114,3 +114,7 @@ class Aocc(Compiler): @classmethod def f77_version(cls, f77): return cls.fc_version(f77) + + @property + def stdcxx_libs(self): + return ('-lstdc++', ) diff --git a/lib/spack/spack/compilers/apple_clang.py b/lib/spack/spack/compilers/apple_clang.py index e03117ae05..372a677a47 100644 --- a/lib/spack/spack/compilers/apple_clang.py +++ b/lib/spack/spack/compilers/apple_clang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/arm.py b/lib/spack/spack/compilers/arm.py index 59eb1714b8..4c59d89210 100644 --- a/lib/spack/spack/compilers/arm.py +++ b/lib/spack/spack/compilers/arm.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py index a325bf3a45..9c90690808 100644 --- a/lib/spack/spack/compilers/cce.py +++ b/lib/spack/spack/compilers/cce.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 5eb08cbf0a..d2dc3db192 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -159,11 +159,11 @@ class Clang(Compiler): match = re.search( # Normal clang compiler versions are left as-is - r'clang version ([^ )]+)-svn[~.\w\d-]*|' + r'clang version ([^ )\n]+)-svn[~.\w\d-]*|' # Don't include hyphenated patch numbers in the version # (see https://github.com/spack/spack/pull/14365 for details) - r'clang version ([^ )]+?)-[~.\w\d-]*|' - r'clang version ([^ )]+)', + r'clang version ([^ )\n]+?)-[~.\w\d-]*|' + r'clang version ([^ )\n]+)', output ) if match: diff --git a/lib/spack/spack/compilers/fj.py b/lib/spack/spack/compilers/fj.py index c2cc66e110..d53fa02fba 100644 --- a/lib/spack/spack/compilers/fj.py +++ b/lib/spack/spack/compilers/fj.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -35,8 +35,12 @@ class Fj(spack.compiler.Compiler): return "-v" @property + def debug_flags(self): + return "-g" + + @property def opt_flags(self): - return ['-O', '-O0', '-O1', '-O2', '-O3', '-O4'] + return ['-O0', '-O1', '-O2', '-O3', '-Ofast'] @property def openmp_flag(self): @@ -55,6 +59,10 @@ class Fj(spack.compiler.Compiler): return "-std=c++14" @property + def cxx17_flag(self): + return "-std=c++17" + + @property def c99_flag(self): return "-std=c99" diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 02ee3e5db9..ff8675324e 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 5e64a1439e..a05b992a10 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 503a31e404..c72a980327 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,17 +1,19 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from typing import List # novm + import spack.compiler class Nag(spack.compiler.Compiler): # Subclasses use possible names of C compiler - cc_names = [] + cc_names = [] # type: List[str] # Subclasses use possible names of C++ compiler - cxx_names = [] + cxx_names = [] # type: List[str] # Subclasses use possible names of Fortran 77 compiler f77_names = ['nagfor'] @@ -100,3 +102,13 @@ class Nag(spack.compiler.Compiler): @property def linker_arg(self): return '-Wl,-Wl,,' + + @property + def disable_new_dtags(self): + # Disable RPATH/RUNPATH forcing for NAG/GCC mixed toolchains: + return '' + + @property + def enable_new_dtags(self): + # Disable RPATH/RUNPATH forcing for NAG/GCC mixed toolchains: + return '' diff --git a/lib/spack/spack/compilers/nvhpc.py b/lib/spack/spack/compilers/nvhpc.py index c102320132..1929d50613 100644 --- a/lib/spack/spack/compilers/nvhpc.py +++ b/lib/spack/spack/compilers/nvhpc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/oneapi.py b/lib/spack/spack/compilers/oneapi.py index 1b029699b5..5b3a81b454 100644 --- a/lib/spack/spack/compilers/oneapi.py +++ b/lib/spack/spack/compilers/oneapi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -36,7 +36,8 @@ class Oneapi(Compiler): return "-v" required_libs = ['libirc', 'libifcore', 'libifcoremt', 'libirng', - 'libsvml', 'libintlc', 'libimf'] + 'libsvml', 'libintlc', 'libimf', 'libsycl', + 'libOpenCL'] @property def debug_flags(self): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 383281a9f4..c1cd8e3a7f 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 66009646e3..7e39d1c530 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/xl_r.py b/lib/spack/spack/compilers/xl_r.py index c9a785d5d8..81100e782a 100644 --- a/lib/spack/spack/compilers/xl_r.py +++ b/lib/spack/spack/compilers/xl_r.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3c3801bf5b..f5c33e5974 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -578,10 +578,14 @@ class Concretizer(object): True if spec was modified, False otherwise """ # To minimize the impact on performance this function will attempt - # to adjust the target only at the very first call. It will just - # return False on subsequent calls. The way this is achieved is by - # initializing a generator and making this function return the next - # answer. + # to adjust the target only at the very first call once necessary + # information is set. It will just return False on subsequent calls. + # The way this is achieved is by initializing a generator and making + # this function return the next answer. + if not (spec.architecture and spec.architecture.concrete): + # Not ready, but keep going because we have work to do later + return True + def _make_only_one_call(spec): yield self._adjust_target(spec) while True: @@ -619,9 +623,10 @@ class Concretizer(object): if PackagePrefs.has_preferred_targets(spec.name): default_target = self.target_from_package_preferences(spec) - if current_target != default_target or \ - (self.abstract_spec.architecture is not None and - self.abstract_spec.architecture.target is not None): + if current_target != default_target or ( + self.abstract_spec and + self.abstract_spec.architecture and + self.abstract_spec.architecture.concrete): return False try: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 84d8e8ca3f..73d5d06e1c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -29,6 +29,7 @@ schemas are in submodules of :py:mod:`spack.schema`. """ import collections +import contextlib import copy import functools import os @@ -38,6 +39,7 @@ import multiprocessing from contextlib import contextmanager from six import iteritems from ordereddict_backport import OrderedDict +from typing import List # novm import ruamel.yaml as yaml from ruamel.yaml.error import MarkedYAMLError @@ -48,6 +50,7 @@ from llnl.util.filesystem import mkdirp import spack.paths import spack.architecture +import spack.compilers import spack.schema import spack.schema.compilers import spack.schema.mirrors @@ -735,7 +738,7 @@ def override(path_or_scope, value=None): #: configuration scopes added on the command line #: set by ``spack.main.main()``. -command_line_scopes = [] +command_line_scopes = [] # type: List[str] def _add_platform_scope(cfg, scope_type, name, path): @@ -803,22 +806,6 @@ def _config(): config = llnl.util.lang.Singleton(_config) -def replace_config(configuration): - """Replace the current global configuration with the instance passed as - argument. - - Args: - configuration (Configuration): the new configuration to be used. - - Returns: - The old configuration that has been removed - """ - global config - config.clear_caches(), configuration.clear_caches() - old_config, config = config, configuration - return old_config - - def get(path, default=None, scope=None): """Module-level wrapper for ``Configuration.get()``.""" return config.get(path, default, scope) @@ -1133,6 +1120,55 @@ def ensure_latest_format_fn(section): return update_fn +@contextlib.contextmanager +def use_configuration(*scopes_or_paths): + """Use the configuration scopes passed as arguments within the + context manager. + + Args: + *scopes_or_paths: scope objects or paths to be used + + Returns: + Configuration object associated with the scopes passed as arguments + """ + global config + + # Normalize input and construct a Configuration object + configuration = _config_from(scopes_or_paths) + config.clear_caches(), configuration.clear_caches() + + # Save and clear the current compiler cache + saved_compiler_cache = spack.compilers._cache_config_file + spack.compilers._cache_config_file = [] + + saved_config, config = config, configuration + + yield configuration + + # Restore previous config files + spack.compilers._cache_config_file = saved_compiler_cache + config = saved_config + + +@llnl.util.lang.memoized +def _config_from(scopes_or_paths): + scopes = [] + for scope_or_path in scopes_or_paths: + # If we have a config scope we are already done + if isinstance(scope_or_path, ConfigScope): + scopes.append(scope_or_path) + continue + + # Otherwise we need to construct it + path = os.path.normpath(scope_or_path) + assert os.path.isdir(path), '"{0}" must be a directory'.format(path) + name = os.path.basename(path) + scopes.append(ConfigScope(name, path)) + + configuration = Configuration(*scopes) + return configuration + + class ConfigError(SpackError): """Superclass for all Spack config related errors.""" diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py index 8206efef01..7211f94ff1 100644 --- a/lib/spack/spack/container/__init__.py +++ b/lib/spack/spack/container/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py index 32fcff2770..9d2e15f195 100644 --- a/lib/spack/spack/container/images.py +++ b/lib/spack/spack/container/images.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py index 176dd7a50c..b1c82a7bdf 100644 --- a/lib/spack/spack/container/writers/__init__.py +++ b/lib/spack/spack/container/writers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/container/writers/docker.py b/lib/spack/spack/container/writers/docker.py index 557d22c803..7c9372fb86 100644 --- a/lib/spack/spack/container/writers/docker.py +++ b/lib/spack/spack/container/writers/docker.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/container/writers/singularity.py b/lib/spack/spack/container/writers/singularity.py index 32f29eb83d..e3eaaea008 100644 --- a/lib/spack/spack/container/writers/singularity.py +++ b/lib/spack/spack/container/writers/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index db1e6a636b..650ab48492 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -27,6 +27,8 @@ import six import socket import sys import time +from typing import Dict # novm + try: import uuid _use_uuid = True @@ -289,10 +291,10 @@ _query_docstring = """ class Database(object): """Per-process lock objects for each install prefix.""" - _prefix_locks = {} + _prefix_locks = {} # type: Dict[str, lk.Lock] """Per-process failure (lock) objects for each install prefix.""" - _prefix_failures = {} + _prefix_failures = {} # type: Dict[str, lk.Lock] def __init__(self, root, db_dir=None, upstream_dbs=None, is_upstream=False, enable_transaction_locking=True, @@ -793,7 +795,7 @@ class Database(object): # do it *while* we're constructing specs,it causes hashes to be # cached prematurely. for hash_key, rec in data.items(): - rec.spec._mark_concrete() + rec.spec._mark_root_concrete() self._data = data @@ -1452,11 +1454,12 @@ class Database(object): rec.spec.name) != known: continue - inst_date = datetime.datetime.fromtimestamp( - rec.installation_time - ) - if not (start_date < inst_date < end_date): - continue + if start_date or end_date: + inst_date = datetime.datetime.fromtimestamp( + rec.installation_time + ) + if not (start_date < inst_date < end_date): + continue if (query_spec is any or rec.spec.satisfies(query_spec, strict=True)): @@ -1464,6 +1467,8 @@ class Database(object): return results + if _query.__doc__ is None: + _query.__doc__ = "" _query.__doc__ += _query_docstring def query_local(self, *args, **kwargs): @@ -1471,6 +1476,8 @@ class Database(object): with self.read_transaction(): return sorted(self._query(*args, **kwargs)) + if query_local.__doc__ is None: + query_local.__doc__ = "" query_local.__doc__ += _query_docstring def query(self, *args, **kwargs): @@ -1489,6 +1496,8 @@ class Database(object): return sorted(results) + if query.__doc__ is None: + query.__doc__ = "" query.__doc__ += _query_docstring def query_one(self, query_spec, known=any, installed=True): diff --git a/lib/spack/spack/dependency.py b/lib/spack/spack/dependency.py index fe7d6b5983..0773ce0763 100644 --- a/lib/spack/spack/dependency.py +++ b/lib/spack/spack/dependency.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 41276b5b48..30a45f1b06 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,11 +28,13 @@ The available directives are: """ -import collections import functools import os.path import re +import sys + from six import string_types +from typing import Set, List # novm import llnl.util.lang import llnl.util.tty.color @@ -47,6 +49,13 @@ from spack.fetch_strategy import from_kwargs from spack.resource import Resource from spack.version import Version, VersionChecksumError + +if sys.version_info >= (3, 3): + from collections.abc import Sequence # novm +else: + from collections import Sequence + + __all__ = [] #: These are variant names used by Spack internally; packages can't use them @@ -103,8 +112,8 @@ class DirectiveMeta(type): """ # Set of all known directives - _directive_names = set() - _directives_to_be_executed = [] + _directive_names = set() # type: Set[str] + _directives_to_be_executed = [] # type: List[str] def __new__(cls, name, bases, attr_dict): # Initialize the attribute containing the list of directives @@ -202,7 +211,7 @@ class DirectiveMeta(type): if isinstance(dicts, string_types): dicts = (dicts, ) - if not isinstance(dicts, collections.Sequence): + if not isinstance(dicts, Sequence): message = "dicts arg must be list, tuple, or string. Found {0}" raise TypeError(message.format(type(dicts))) # Add the dictionary names if not already there @@ -243,7 +252,7 @@ class DirectiveMeta(type): # ...so if it is not a sequence make it so values = result - if not isinstance(values, collections.Sequence): + if not isinstance(values, Sequence): values = (values, ) DirectiveMeta._directives_to_be_executed.extend(values) @@ -269,6 +278,9 @@ def version(ver, checksum=None, **kwargs): The ``dict`` of arguments is turned into a valid fetch strategy for code packages later. See ``spack.fetch_strategy.for_package_version()``. + + Keyword Arguments: + deprecated (bool): whether or not this version is deprecated """ def _execute_version(pkg): if checksum is not None: @@ -392,7 +404,7 @@ def depends_on(spec, when=None, type=default_deptype, patches=None): @directive(('extendees', 'dependencies')) -def extends(spec, **kwargs): +def extends(spec, type=('build', 'run'), **kwargs): """Same as depends_on, but allows symlinking into dependency's prefix tree. @@ -413,7 +425,7 @@ def extends(spec, **kwargs): if not when_spec: return - _depends_on(pkg, spec, when=when) + _depends_on(pkg, spec, when=when, type=type) pkg.extendees[spec] = (spack.spec.Spec(spec), kwargs) return _execute_extends diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 06a652f450..cbaa9b47fd 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,6 +7,7 @@ import os import shutil import glob import tempfile +import errno from contextlib import contextmanager import ruamel.yaml as yaml @@ -14,6 +15,7 @@ import ruamel.yaml as yaml from llnl.util.filesystem import mkdirp import spack.config +import spack.hash_types as ht import spack.spec from spack.error import SpackError @@ -119,9 +121,17 @@ class DirectoryLayout(object): path = os.path.dirname(path) while path != self.root: if os.path.isdir(path): - if os.listdir(path): - return - os.rmdir(path) + try: + os.rmdir(path) + except OSError as e: + if e.errno == errno.ENOENT: + # already deleted, continue with parent + pass + elif e.errno == errno.ENOTEMPTY: + # directory wasn't empty, done + return + else: + raise e path = os.path.dirname(path) @@ -233,7 +243,9 @@ class YamlDirectoryLayout(DirectoryLayout): """Write a spec out to a file.""" _check_concrete(spec) with open(path, 'w') as f: - spec.to_yaml(f) + # The hash the the projection is the DAG hash but we write out the + # full provenance by full hash so it's availabe if we want it later + spec.to_yaml(f, hash=ht.full_hash) def read_spec(self, path): """Read the contents of a file and parse them as a spec""" diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 38c2c4cdb7..4518cb25a9 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -40,6 +40,7 @@ from spack.variant import UnknownVariantError import spack.util.lock as lk from spack.util.path import substitute_path_variables from spack.installer import PackageInstaller +import spack.util.path #: environment variable used to indicate the active environment spack_env_var = 'SPACK_ENV' @@ -468,7 +469,7 @@ class ViewDescriptor(object): def __init__(self, base_path, root, projections={}, select=[], exclude=[], link=default_view_link): self.base = base_path - self.root = root + self.root = spack.util.path.canonicalize_path(root) self.projections = projections self.select = select self.select_fn = lambda x: any(x.satisfies(s) for s in self.select) @@ -555,11 +556,20 @@ class ViewDescriptor(object): # that cannot be resolved or have repos that have been removed # we always regenerate the view from scratch. We must first make # sure the root directory exists for the very first time though. - root = self.root - if not os.path.isabs(root): - root = os.path.normpath(os.path.join(self.base, self.root)) + root = os.path.normpath( + self.root if os.path.isabs(self.root) else os.path.join( + self.base, self.root) + ) fs.mkdirp(root) - with fs.replace_directory_transaction(root): + + # The tempdir for the directory transaction must be in the same + # filesystem mount as the view for symlinks to work. Provide + # dirname(root) as the tempdir for the + # replace_directory_transaction because it must be on the same + # filesystem mount as the view itself. Otherwise it may be + # impossible to construct the view in the tempdir even when it can + # be constructed in-place. + with fs.replace_directory_transaction(root, os.path.dirname(root)): view = self.view() view.clean() diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index f79744f6a5..fa66a35b0e 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index 4358dcd52f..9ea7373b8c 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index f075b893ae..ff41c759ea 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -29,6 +29,7 @@ import os.path import re import shutil import sys +from typing import Optional, List # novm import llnl.util.tty as tty import six @@ -92,11 +93,12 @@ class FetchStrategy(object): #: The URL attribute must be specified either at the package class #: level, or as a keyword argument to ``version()``. It is used to #: distinguish fetchers for different versions in the package DSL. - url_attr = None + url_attr = None # type: Optional[str] #: Optional attributes can be used to distinguish fetchers when : #: classes have multiple ``url_attrs`` at the top-level. - optional_attrs = [] # optional attributes in version() args. + # optional attributes in version() args. + optional_attrs = [] # type: List[str] def __init__(self, **kwargs): # The stage is initialized late, so that fetch strategies can be @@ -324,6 +326,8 @@ class URLFetchStrategy(FetchStrategy): # Telling curl to fetch the first byte (-r 0-0) is supposed to be # portable. curl_args = ['--stderr', '-', '-s', '-f', '-r', '0-0', url] + if not spack.config.get('config:verify_ssl'): + curl_args.append('-k') _ = curl(*curl_args, fail_on_error=False, output=os.devnull) return curl.returncode == 0 @@ -420,7 +424,7 @@ class URLFetchStrategy(FetchStrategy): warn_content_type_mismatch(self.archive_file or "the archive") return partial_file, save_file - @property + @property # type: ignore # decorated properties unsupported in mypy @_needs_stage def archive_file(self): """Path to the source archive within this stage directory.""" @@ -774,6 +778,12 @@ class GitFetchStrategy(VCSFetchStrategy): if not self._git: self._git = which('git', required=True) + # Disable advice for a quieter fetch + # https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt + if self.git_version >= Version('1.7.2'): + self._git.add_default_arg('-c') + self._git.add_default_arg('advice.detachedHead=false') + # If the user asked for insecure fetching, make that work # with git as well. if not spack.config.get('config:verify_ssl'): diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index b6501064b9..232223b69f 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index b41c923a16..2ddc10ac7a 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hash_types.py b/lib/spack/spack/hash_types.py index 9404076e81..0ad321dec6 100644 --- a/lib/spack/spack/hash_types.py +++ b/lib/spack/spack/hash_types.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -35,5 +35,5 @@ build_hash = SpecHashDescriptor( #: Full hash used in build pipelines to determine when to rebuild packages. -full_hash = SpecHashDescriptor(deptype=('link', 'run'), package_hash=True, - attr='_full_hash') +full_hash = SpecHashDescriptor( + deptype=('build', 'link', 'run'), package_hash=True, attr='_full_hash') diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 5a39f242fd..681fec0feb 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 05fce302ac..a639df6181 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index d4e5376cda..8e789d9635 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/module_file_generation.py b/lib/spack/spack/hooks/module_file_generation.py index ba30561d86..363654efc4 100644 --- a/lib/spack/spack/hooks/module_file_generation.py +++ b/lib/spack/spack/hooks/module_file_generation.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/permissions_setters.py b/lib/spack/spack/hooks/permissions_setters.py index 0a3c2acf21..d640018515 100644 --- a/lib/spack/spack/hooks/permissions_setters.py +++ b/lib/spack/spack/hooks/permissions_setters.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 687cf651f4..7dc00165d5 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/write_install_manifest.py b/lib/spack/spack/hooks/write_install_manifest.py index 5b678fed88..0fb6acdf35 100644 --- a/lib/spack/spack/hooks/write_install_manifest.py +++ b/lib/spack/spack/hooks/write_install_manifest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 6c2c095a2e..ae50a49215 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 15bc23738b..a9c0d7e85c 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -392,7 +392,7 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False, pkg_id = package_id(pkg) tty.debug('Searching for binary cache of {0}'.format(pkg_id)) matches = binary_distribution.get_mirrors_for_spec( - pkg.spec, force=False, full_hash_match=full_hash_match) + pkg.spec, full_hash_match=full_hash_match) if not matches: return False @@ -1405,7 +1405,6 @@ class PackageInstaller(object): pkg, pkg_id, spec = task.pkg, task.pkg_id, task.pkg.spec tty.verbose('Processing {0}: task={1}'.format(pkg_id, task)) - # Ensure that the current spec has NO uninstalled dependencies, # which is assumed to be reflected directly in its priority. # @@ -1543,7 +1542,7 @@ class PackageInstaller(object): (stop_before_phase is None and last_phase is None) except spack.directory_layout.InstallDirectoryAlreadyExistsError \ - as err: + as exc: tty.debug('Install prefix for {0} exists, keeping {1} in ' 'place.'.format(pkg.name, pkg.prefix)) self._update_installed(task) @@ -1554,7 +1553,7 @@ class PackageInstaller(object): raise if task.explicit: - exists_errors.append((pkg_id, str(err))) + exists_errors.append((pkg_id, str(exc))) except KeyboardInterrupt as exc: # The build has been terminated with a Ctrl-C so terminate @@ -1721,11 +1720,10 @@ def build_process(pkg, kwargs): pkg._total_time = time.time() - start_time build_time = pkg._total_time - pkg._fetch_time - tty.debug('{0} Successfully installed {1}' - .format(pre, pkg_id), - 'Fetch: {0}. Build: {1}. Total: {2}.' - .format(_hms(pkg._fetch_time), _hms(build_time), - _hms(pkg._total_time))) + tty.msg('{0} Successfully installed {1}'.format(pre, pkg_id), + 'Fetch: {0}. Build: {1}. Total: {2}.' + .format(_hms(pkg._fetch_time), _hms(build_time), + _hms(pkg._total_time))) _print_installed_pkg(pkg.prefix) # preserve verbosity across runs @@ -2013,7 +2011,8 @@ class BuildRequest(object): (tuple) required dependency type(s) for the package """ deptypes = ['link', 'run'] - if not self.install_args.get('cache_only'): + include_build_deps = self.install_args.get('include_build_deps') + if not self.install_args.get('cache_only') or include_build_deps: deptypes.append('build') if self.run_tests(pkg): deptypes.append('test') diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index f4265a5d68..6b323f8eb3 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 045ca5ffec..cdebc7fd4e 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,9 +23,9 @@ import ruamel.yaml.error as yaml_error from ordereddict_backport import OrderedDict -try: +if sys.version_info >= (3, 5): from collections.abc import Mapping # novm -except ImportError: +else: from collections import Mapping import llnl.util.tty as tty diff --git a/lib/spack/spack/mixins.py b/lib/spack/spack/mixins.py index 6da12e4dce..be13e3ea3a 100644 --- a/lib/spack/spack/mixins.py +++ b/lib/spack/spack/mixins.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,12 @@ package. """ import collections import os +import sys +from typing import Callable, DefaultDict, Dict, List # novm +if sys.version_info >= (3, 5): + CallbackDict = DefaultDict[str, List[Callable]] +else: + CallbackDict = None import llnl.util.filesystem @@ -26,12 +32,12 @@ class PackageMixinsMeta(type): gets implicitly attached to the package class by calling the mixin. """ - _methods_to_be_added = {} - _add_method_before = collections.defaultdict(list) - _add_method_after = collections.defaultdict(list) + _methods_to_be_added = {} # type: Dict[str, Callable] + _add_method_before = collections.defaultdict(list) # type: CallbackDict + _add_method_after = collections.defaultdict(list) # type: CallbackDict @staticmethod - def register_method_before(fn, phase): + def register_method_before(fn, phase): # type: (Callable, str) -> None """Registers a method to be run before a certain phase. Args: @@ -42,7 +48,7 @@ class PackageMixinsMeta(type): PackageMixinsMeta._add_method_before[phase].append(fn) @staticmethod - def register_method_after(fn, phase): + def register_method_after(fn, phase): # type: (Callable, str) -> None """Registers a method to be run after a certain phase. Args: diff --git a/lib/spack/spack/modules/__init__.py b/lib/spack/spack/modules/__init__.py index 2fb4d94b05..b65260277f 100644 --- a/lib/spack/spack/modules/__init__.py +++ b/lib/spack/spack/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index cbc064520c..ed843e03af 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -34,6 +34,7 @@ import datetime import inspect import os.path import re +from typing import Optional # novm import llnl.util.filesystem from llnl.util.lang import dedupe @@ -540,7 +541,7 @@ class BaseFileLayout(object): """ #: This needs to be redefined - extension = None + extension = None # type: Optional[str] def __init__(self, configuration): self.conf = configuration diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index 80f6933063..bb4476a7b0 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import os.path import llnl.util.lang as lang import itertools import collections +from typing import Dict, Any # novm import spack.config import spack.compilers @@ -26,7 +27,7 @@ def configuration(): #: Caches the configuration {spec_hash: configuration} -configuration_registry = {} +configuration_registry = {} # type: Dict[str, Any] def make_configuration(spec): diff --git a/lib/spack/spack/modules/tcl.py b/lib/spack/spack/modules/tcl.py index 0efc6332fe..e1d2ac7fe3 100644 --- a/lib/spack/spack/modules/tcl.py +++ b/lib/spack/spack/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ non-hierarchical modules. """ import os.path import string +from typing import Dict, Any # novm import llnl.util.tty as tty @@ -24,7 +25,7 @@ def configuration(): #: Caches the configuration {spec_hash: configuration} -configuration_registry = {} +configuration_registry = {} # type: Dict[str, Any] def make_configuration(spec): diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 73cf342b5b..d9fe6882b9 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/operating_systems/__init__.py +++ b/lib/spack/spack/operating_systems/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/cray_backend.py b/lib/spack/spack/operating_systems/cray_backend.py index 5f113eba0b..0e59361d70 100644 --- a/lib/spack/spack/operating_systems/cray_backend.py +++ b/lib/spack/spack/operating_systems/cray_backend.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/cray_frontend.py b/lib/spack/spack/operating_systems/cray_frontend.py index d06920af43..820db506cb 100644 --- a/lib/spack/spack/operating_systems/cray_frontend.py +++ b/lib/spack/spack/operating_systems/cray_frontend.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index a6608e6d3b..b1901fc41c 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/mac_os.py b/lib/spack/spack/operating_systems/mac_os.py index 0efc298338..de9bcf659d 100644 --- a/lib/spack/spack/operating_systems/mac_os.py +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index de394e2d45..9dfd8bf01f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -26,6 +26,7 @@ import time import traceback import six import types +from typing import Optional, List, Dict, Any, Callable # novm import llnl.util.filesystem as fsys import llnl.util.tty as tty @@ -253,8 +254,9 @@ class PackageMeta( """ phase_fmt = '_InstallPhase_{0}' - _InstallPhase_run_before = {} - _InstallPhase_run_after = {} + # These are accessed only through getattr, by name + _InstallPhase_run_before = {} # type: Dict[str, List[Callable]] + _InstallPhase_run_after = {} # type: Dict[str, List[Callable]] def __new__(cls, name, bases, attr_dict): """ @@ -555,7 +557,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: A list or set of build time test functions to be called when tests #: are executed or 'None' if there are no such test functions. - build_time_test_callbacks = None + build_time_test_callbacks = None # type: Optional[List[str]] #: By default, packages are not virtual #: Virtual packages override this attribute @@ -567,7 +569,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: A list or set of install time test functions to be called when tests #: are executed or 'None' if there are no such test functions. - install_time_test_callbacks = None + install_time_test_callbacks = None # type: Optional[List[str]] #: By default we build in parallel. Subclasses can override this. parallel = True @@ -589,19 +591,19 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: List of prefix-relative file paths (or a single path). If these do #: not exist after install, or if they exist but are not files, #: sanity checks fail. - sanity_check_is_file = [] + sanity_check_is_file = [] # type: List[str] #: List of prefix-relative directory paths (or a single path). If #: these do not exist after install, or if they exist but are not #: directories, sanity checks will fail. - sanity_check_is_dir = [] + sanity_check_is_dir = [] # type: List[str] #: List of glob expressions. Each expression must either be #: absolute or relative to the package source path. #: Matching artifacts found at the end of the build process will be #: copied in the same directory tree as _spack_build_logfile and #: _spack_build_envfile. - archive_files = [] + archive_files = [] # type: List[str] #: Boolean. Set to ``True`` for packages that require a manual download. #: This is currently used by package sanity tests and generation of a @@ -609,7 +611,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): manual_download = False #: Set of additional options used when fetching package versions. - fetch_options = {} + fetch_options = {} # type: Dict[str, Any] # # Set default licensing information @@ -627,12 +629,12 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: looking for a license. All file paths must be relative to the #: installation directory. More complex packages like Intel may require #: multiple licenses for individual components. Defaults to the empty list. - license_files = [] + license_files = [] # type: List[str] #: List of strings. Environment variables that can be set to tell the #: software where to look for a license if it is not in the usual location. #: Defaults to the empty list. - license_vars = [] + license_vars = [] # type: List[str] #: String. A URL pointing to license setup instructions for the software. #: Defaults to the empty string. @@ -644,9 +646,18 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): #: index of patches by sha256 sum, built lazily _patches_by_hash = None + #: Package homepage where users can find more information about the package + homepage = None # type: str + + #: Default list URL (place to find available versions) + list_url = None # type: str + + #: Link depth to which list_url should be searched for new versions + list_depth = 0 + #: List of strings which contains GitHub usernames of package maintainers. #: Do not include @ here in order not to unnecessarily ping the users. - maintainers = [] + maintainers = [] # type: List[str] #: List of attributes to be excluded from a package's hash. metadata_attrs = ['homepage', 'url', 'urls', 'list_url', 'extendable', @@ -681,13 +692,6 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): msg += " [package '{0.name}' defines both]" raise ValueError(msg.format(self)) - # Set a default list URL (place to find available versions) - if not hasattr(self, 'list_url'): - self.list_url = None - - if not hasattr(self, 'list_depth'): - self.list_depth = 0 - # init internal variables self._stage = None self._fetcher = None @@ -1305,7 +1309,6 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): tty.debug('No fetch required for {0}: package has no code.' .format(self.name)) - start_time = time.time() checksum = spack.config.get('config:checksum') fetch = self.stage.managed_by_spack if checksum and fetch and self.version not in self.versions: @@ -1327,8 +1330,34 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): raise FetchError("Will not fetch %s" % self.spec.format('{name}{@version}'), ck_msg) + deprecated = spack.config.get('config:deprecated') + if not deprecated and self.versions.get( + self.version, {}).get('deprecated', False): + tty.warn("{0} is deprecated and may be removed in a future Spack " + "release.".format( + self.spec.format('{name}{@version}'))) + + # Ask the user whether to install deprecated version if we're + # interactive, but just fail if non-interactive. + dp_msg = ("If you are willing to be a maintainer for this version " + "of the package, submit a PR to remove `deprecated=False" + "`, or use `--deprecated` to skip this check.") + ignore_deprecation = False + if sys.stdout.isatty(): + ignore_deprecation = tty.get_yes_or_no(" Fetch anyway?", + default=False) + + if ignore_deprecation: + tty.debug("Fetching deprecated version. {0}".format( + dp_msg)) + + if not ignore_deprecation: + raise FetchError("Will not fetch {0}".format( + self.spec.format('{name}{@version}')), dp_msg) + self.stage.create() err_msg = None if not self.manual_download else self.download_instr + start_time = time.time() self.stage.fetch(mirror_only, err_msg=err_msg) self._fetch_time = time.time() - start_time @@ -1375,7 +1404,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): # If there are no patches, note it. if not patches and not has_patch_fun: - tty.debug('No patches needed for {0}'.format(self.name)) + tty.msg('No patches needed for {0}'.format(self.name)) return # Construct paths to special files in the archive dir used to @@ -1393,10 +1422,10 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): # If this file exists, then we already applied all the patches. if os.path.isfile(good_file): - tty.debug('Already patched {0}'.format(self.name)) + tty.msg('Already patched {0}'.format(self.name)) return elif os.path.isfile(no_patches_file): - tty.debug('No patches needed for {0}'.format(self.name)) + tty.msg('No patches needed for {0}'.format(self.name)) return # Apply all the patches for specs that match this one @@ -1405,7 +1434,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): try: with fsys.working_dir(self.stage.source_path): patch.apply(self.stage) - tty.debug('Applied patch {0}'.format(patch.path_or_url)) + tty.msg('Applied patch {0}'.format(patch.path_or_url)) patched = True except spack.error.SpackError as e: tty.debug(e) @@ -1419,7 +1448,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): try: with fsys.working_dir(self.stage.source_path): self.patch() - tty.debug('Ran patch() for {0}'.format(self.name)) + tty.msg('Ran patch() for {0}'.format(self.name)) patched = True except spack.multimethod.NoSuchMethodError: # We are running a multimethod without a default case. @@ -1429,7 +1458,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): # directive, AND the patch function didn't apply, say # no patches are needed. Otherwise, we already # printed a message for each patch. - tty.debug('No patches needed for {0}'.format(self.name)) + tty.msg('No patches needed for {0}'.format(self.name)) except spack.error.SpackError as e: tty.debug(e) @@ -1497,10 +1526,16 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): hash_content.extend(':'.join((p.sha256, str(p.level))).encode('utf-8') for p in self.spec.patches) hash_content.append(package_hash(self.spec, content)) - return base64.b32encode( + b32_hash = base64.b32encode( hashlib.sha256(bytes().join( sorted(hash_content))).digest()).lower() + # convert from bytes if running python 3 + if sys.version_info[0] >= 3: + b32_hash = b32_hash.decode('utf-8') + + return b32_hash + def _has_make_target(self, target): """Checks to see if 'target' is a valid target in a Makefile. @@ -1761,7 +1796,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): work_dir (str or None): path to the smoke test directory """ wdir = '.' if work_dir is None else work_dir - with fsys.working_dir(wdir): + with fsys.working_dir(wdir, create=True): try: runner = which(exe) if runner is None and skip_missing: @@ -2591,7 +2626,7 @@ class BundlePackage(PackageBase): """General purpose bundle, or no-code, package class.""" #: There are no phases by default but the property is required to support #: post-install hooks (e.g., for module generation). - phases = [] + phases = [] # type: List[str] #: This attribute is used in UI queries that require to know which #: build-system class we are using build_system_class = 'BundlePackage' diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index 149e10ea26..cf74c7ee52 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/package_test.py b/lib/spack/spack/package_test.py index 6e9da9a985..6a4df096fc 100644 --- a/lib/spack/spack/package_test.py +++ b/lib/spack/spack/package_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py index 12bbcee588..1fc853bee4 100644 --- a/lib/spack/spack/parse.py +++ b/lib/spack/spack/parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index 3a839c5b0f..d38b2ef767 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9c803cba7e..47574196eb 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/pkgkit.py b/lib/spack/spack/pkgkit.py index 423c0fb05f..133a581e20 100644 --- a/lib/spack/spack/pkgkit.py +++ b/lib/spack/spack/pkgkit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/platforms/__init__.py +++ b/lib/spack/spack/platforms/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 091f8d3ffb..a949042133 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index e36722723b..3969cc9ec8 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index c7e1def2d9..46906229ba 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index f938eb749d..80d9431879 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/projections.py b/lib/spack/spack/projections.py index b91d321436..9f78e24409 100644 --- a/lib/spack/spack/projections.py +++ b/lib/spack/spack/projections.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py index 326f6aa8f1..6a84d8f40a 100644 --- a/lib/spack/spack/provider_index.py +++ b/lib/spack/spack/provider_index.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index e1726b060e..e167472b75 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 4af7b382f0..b103ac1caf 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -13,18 +13,18 @@ import itertools import os import re import shutil +import six import stat import sys import traceback import types +from typing import Dict # novm -try: +if sys.version_info >= (3, 5): from collections.abc import Mapping # novm -except ImportError: +else: from collections import Mapping -import six - import ruamel.yaml as yaml import llnl.util.lang @@ -118,7 +118,11 @@ class SpackNamespace(types.ModuleType): def __getattr__(self, name): """Getattr lazily loads modules if they're not already loaded.""" submodule = self.__package__ + '.' + name - setattr(self, name, __import__(submodule)) + try: + setattr(self, name, __import__(submodule)) + except ImportError: + msg = "'{0}' object has no attribute {1}" + raise AttributeError(msg.format(type(self), name)) return getattr(self, name) @@ -131,7 +135,7 @@ class FastPackageChecker(Mapping): during instance initialization. """ #: Global cache, reused by every instance - _paths_cache = {} + _paths_cache = {} # type: Dict[str, Dict[str, os.stat_result]] def __init__(self, packages_path): # The path of the repository managed by this instance @@ -149,7 +153,7 @@ class FastPackageChecker(Mapping): self._paths_cache[self.packages_path] = self._create_new_cache() self._packages_to_stats = self._paths_cache[self.packages_path] - def _create_new_cache(self): + def _create_new_cache(self): # type: () -> Dict[str, os.stat_result] """Create a new cache for packages in a repo. The implementation here should try to minimize filesystem @@ -159,7 +163,7 @@ class FastPackageChecker(Mapping): """ # Create a dictionary that will store the mapping between a # package name and its stat info - cache = {} + cache = {} # type: Dict[str, os.stat_result] for pkg_name in os.listdir(self.packages_path): # Skip non-directories in the package root. pkg_dir = os.path.join(self.packages_path, pkg_name) @@ -341,7 +345,7 @@ class PatchIndexer(Indexer): def _create(self): return spack.patch.PatchCache() - def needs_update(): + def needs_update(self): # TODO: patches can change under a package and we should handle # TODO: it, but we currently punt. This should be refactored to # TODO: check whether patches changed each time a package loads, @@ -1256,23 +1260,6 @@ def set_path(repo): @contextlib.contextmanager -def swap(repo_path): - """Temporarily use another RepoPath.""" - global path - - # swap out _path for repo_path - saved = path - remove_from_meta = set_path(repo_path) - - yield - - # restore _path and sys.meta_path - if remove_from_meta: - sys.meta_path.remove(repo_path) - path = saved - - -@contextlib.contextmanager def additional_repository(repository): """Adds temporarily a repository to the default one. @@ -1284,6 +1271,34 @@ def additional_repository(repository): path.remove(repository) +@contextlib.contextmanager +def use_repositories(*paths_and_repos): + """Use the repositories passed as arguments within the context manager. + + Args: + *paths_and_repos: paths to the repositories to be used, or + already constructed Repo objects + + Returns: + Corresponding RepoPath object + """ + global path + + # Construct a temporary RepoPath object from + temporary_repositories = RepoPath(*paths_and_repos) + + # Swap the current repository out + saved = path + remove_from_meta = set_path(temporary_repositories) + + yield temporary_repositories + + # Restore _path and sys.meta_path + if remove_from_meta: + sys.meta_path.remove(temporary_repositories) + path = saved + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index ebae2d0adc..8e97d036c0 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporter.py b/lib/spack/spack/reporter.py index 6314054139..629051e223 100644 --- a/lib/spack/spack/reporter.py +++ b/lib/spack/spack/reporter.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporters/__init__.py b/lib/spack/spack/reporters/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/reporters/__init__.py +++ b/lib/spack/spack/reporters/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index c1a220963e..44d3f7b78d 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py index 598b308934..9120850c4c 100644 --- a/lib/spack/spack/reporters/junit.py +++ b/lib/spack/spack/reporters/junit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 5cec54f6de..488c6a846d 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/s3_handler.py b/lib/spack/spack/s3_handler.py index 24f0a46221..d906577bed 100644 --- a/lib/spack/spack/s3_handler.py +++ b/lib/spack/spack/s3_handler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/__init__.py b/lib/spack/spack/schema/__init__.py index 1ddca90cc6..804b7c989d 100644 --- a/lib/spack/spack/schema/__init__.py +++ b/lib/spack/spack/schema/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py index 5ba07a27f1..f1c515852c 100644 --- a/lib/spack/spack/schema/buildcache_spec.py +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/cdash.py b/lib/spack/spack/schema/cdash.py index 2df2ea3f32..41c0146b1d 100644 --- a/lib/spack/spack/schema/cdash.py +++ b/lib/spack/spack/schema/cdash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index c994bef819..91ace05f4e 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 0f83eb86f4..f0669600e1 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -78,6 +78,7 @@ properties = { 'install_missing_compilers': {'type': 'boolean'}, 'debug': {'type': 'boolean'}, 'checksum': {'type': 'boolean'}, + 'deprecated': {'type': 'boolean'}, 'locks': {'type': 'boolean'}, 'dirty': {'type': 'boolean'}, 'build_language': {'type': 'string'}, diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index 7e14efd75c..5223efa041 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/database_index.py b/lib/spack/spack/schema/database_index.py index eec3d4ce02..105f6ad6a3 100644 --- a/lib/spack/spack/schema/database_index.py +++ b/lib/spack/spack/schema/database_index.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index a7e3dc608f..82971505e4 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py index d251aeba96..6431916c56 100644 --- a/lib/spack/spack/schema/environment.py +++ b/lib/spack/spack/schema/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -38,11 +38,12 @@ def parse(config_obj): config_obj: a configuration dictionary conforming to the schema definition for environment modifications """ + import sys import spack.util.environment as ev - try: - from collections import Sequence # novm - except ImportError: + if sys.version_info >= (3, 5): from collections.abc import Sequence # novm + else: + from collections import Sequence # novm env = ev.EnvironmentModifications() for command, variable in config_obj.items(): diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index 79ffee2074..bfc8ca2647 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,6 +9,8 @@ :lines: 13- """ +from llnl.util.lang import union_dicts + image_schema = { 'oneOf': [ { @@ -28,130 +30,125 @@ image_schema = { ], } -#: Properties for inclusion in other schemas -properties = { - 'gitlab-ci': { +runner_attributes_schema_items = { + 'image': image_schema, + 'tags': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'variables': { 'type': 'object', - 'additionalProperties': False, - 'required': ['mappings'], 'patternProperties': { - 'bootstrap': { - 'type': 'array', - 'items': { - 'anyOf': [ - { - 'type': 'string', - }, { - 'type': 'object', - 'additionalProperties': False, - 'required': ['name'], - 'properties': { - 'name': { - 'type': 'string', - }, - 'compiler-agnostic': { - 'type': 'boolean', - 'default': False, - }, - }, - }, - ], - }, + r'[\w\d\-_\.]+': { + 'type': 'string', }, - 'mappings': { - 'type': 'array', - 'items': { - 'type': 'object', - 'additionalProperties': False, - 'required': ['match'], - 'properties': { - 'match': { - 'type': 'array', - 'items': { + }, + }, + 'before_script': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'script': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'after_script': { + 'type': 'array', + 'items': {'type': 'string'} + }, +} + +runner_selector_schema = { + 'type': 'object', + 'additionalProperties': False, + 'required': ['tags'], + 'properties': runner_attributes_schema_items, +} + + +core_shared_properties = union_dicts( + runner_attributes_schema_items, + { + 'bootstrap': { + 'type': 'array', + 'items': { + 'anyOf': [ + { + 'type': 'string', + }, { + 'type': 'object', + 'additionalProperties': False, + 'required': ['name'], + 'properties': { + 'name': { 'type': 'string', }, - }, - 'runner-attributes': { - 'type': 'object', - 'additionalProperties': True, - 'required': ['tags'], - 'properties': { - 'image': image_schema, - 'tags': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'variables': { - 'type': 'object', - 'patternProperties': { - r'[\w\d\-_\.]+': { - 'type': 'string', - }, - }, - }, - 'before_script': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'script': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'after_script': { - 'type': 'array', - 'items': {'type': 'string'} - }, + 'compiler-agnostic': { + 'type': 'boolean', + 'default': False, }, }, }, - }, + ], }, - 'image': image_schema, - 'tags': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'variables': { - 'type': 'object', - 'patternProperties': { - r'[\w\d\-_\.]+': { - 'type': 'string', - }, - }, - }, - 'before_script': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'script': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'after_script': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'enable-artifacts-buildcache': { - 'type': 'boolean', - 'default': False, - }, - 'final-stage-rebuild-index': { + }, + 'mappings': { + 'type': 'array', + 'items': { 'type': 'object', 'additionalProperties': False, - 'required': ['tags'], + 'required': ['match'], 'properties': { - 'image': image_schema, - 'tags': { + 'match': { 'type': 'array', - 'default': [], - 'items': {'type': 'string'} + 'items': { + 'type': 'string', + }, }, + 'runner-attributes': runner_selector_schema, }, }, }, + 'service-job-attributes': runner_selector_schema, + 'rebuild-index': {'type': 'boolean'}, }, +) + +gitlab_ci_properties = { + 'anyOf': [ + { + 'type': 'object', + 'additionalProperties': False, + 'required': ['mappings'], + 'properties': union_dicts( + core_shared_properties, + { + 'enable-artifacts-buildcache': { + 'type': 'boolean', + }, + }, + ), + }, + { + 'type': 'object', + 'additionalProperties': False, + 'required': ['mappings'], + 'properties': union_dicts( + core_shared_properties, + { + 'temporary-storage-url-prefix': { + 'type': 'string', + }, + }, + ), + }, + ] } +#: Properties for inclusion in other schemas +properties = { + 'gitlab-ci': gitlab_ci_properties, +} #: Full schema with metadata schema = { diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index e118acf286..a58f47cb16 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 5bb641df05..6dec5aac97 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index f44ba7d97d..39db0bf9a7 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index f2626c83be..e24f610c74 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/projections.py b/lib/spack/spack/schema/projections.py index bd0693f30e..cab512fe3b 100644 --- a/lib/spack/spack/schema/projections.py +++ b/lib/spack/spack/schema/projections.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/repos.py b/lib/spack/spack/schema/repos.py index 6aa9bcf57e..44cf688a48 100644 --- a/lib/spack/spack/schema/repos.py +++ b/lib/spack/spack/schema/repos.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py index e2c8efad4c..fecfcb2af4 100644 --- a/lib/spack/spack/schema/spec.py +++ b/lib/spack/spack/schema/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/upstreams.py b/lib/spack/spack/schema/upstreams.py index f5192aa096..7d721332eb 100644 --- a/lib/spack/spack/schema/upstreams.py +++ b/lib/spack/spack/schema/upstreams.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/solver/__init__.py b/lib/spack/spack/solver/__init__.py index 94f8ac4d9e..103eae6134 100644 --- a/lib/spack/spack/solver/__init__.py +++ b/lib/spack/spack/solver/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 18e08b3199..4d7b019ea6 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -19,7 +19,7 @@ import archspec.cpu try: import clingo except ImportError: - clingo = None + clingo = None # type: ignore import llnl.util.lang import llnl.util.tty as tty @@ -40,6 +40,12 @@ import spack.variant import spack.version +if sys.version_info >= (3, 3): + from collections.abc import Sequence # novm +else: + from collections import Sequence + + class Timer(object): """Simple timer for timing phases of a solve""" def __init__(self): @@ -64,7 +70,7 @@ class Timer(object): def issequence(obj): if isinstance(obj, string_types): return False - return isinstance(obj, (collections.Sequence, types.GeneratorType)) + return isinstance(obj, (Sequence, types.GeneratorType)) def listify(args): @@ -204,19 +210,6 @@ class Result(object): *sorted(str(symbol) for symbol in core)) -def _normalize(body): - """Accept an AspAnd object or a single Symbol and return a list of - symbols. - """ - if isinstance(body, clingo.Symbol): - args = [body] - elif hasattr(body, 'symbol'): - args = [body.symbol()] - else: - raise TypeError("Invalid typee: ", type(body)) - return args - - def _normalize_packages_yaml(packages_yaml): normalized_yaml = copy.copy(packages_yaml) for pkg_name in packages_yaml: @@ -274,19 +267,14 @@ class PyclingoDriver(object): def fact(self, head): """ASP fact (a rule without a body).""" - symbols = _normalize(head) - self.out.write("%s.\n" % ','.join(str(a) for a in symbols)) + symbol = head.symbol() if hasattr(head, 'symbol') else head - atoms = {} - for s in symbols: - atoms[s] = self.backend.add_atom(s) + self.out.write("%s.\n" % str(symbol)) - self.backend.add_rule( - [atoms[s] for s in symbols], [], choice=self.cores - ) + atom = self.backend.add_atom(symbol) + self.backend.add_rule([atom], [], choice=self.cores) if self.cores: - for s in symbols: - self.assumptions.append(atoms[s]) + self.assumptions.append(atom) def solve( self, solver_setup, specs, dump=None, nmodels=0, @@ -1091,7 +1079,6 @@ class SpackSolverSetup(object): if not spec.architecture or not spec.architecture.target: continue - print("TTYPE:", type(platform.target(spec.target.name))) target = archspec.cpu.TARGETS.get(spec.target.name) if not target: self.target_ranges(spec, None) diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index d0174ca2e0..22ba77142a 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -1,3 +1,8 @@ +% Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + %============================================================================= % This logic program implements Spack's concretizer %============================================================================= @@ -222,6 +227,7 @@ provider_weight(Package, 100) provider(Package, Virtual), not default_provider_preference(Virtual, Package, _). +#defined possible_provider/2. #defined provider_condition/3. #defined required_provider_condition/3. #defined required_provider_condition/4. @@ -686,26 +692,6 @@ no_flags(Package, FlagType) %----------------------------------------------------------------------------- % How to optimize the spec (high to low priority) %----------------------------------------------------------------------------- -% weight root preferences higher -% -% TODO: how best to deal with this issue? It's not clear how best to -% weight all the constraints. Without this root preference, `spack solve -% hdf5` will pick mpich instead of openmpi, even if openmpi is the -% preferred provider, because openmpi has a version constraint on hwloc. -% It ends up choosing between settling for an old version of hwloc, or -% picking the second-best provider. This workaround weights root -% preferences higher so that hdf5's prefs are more important, but it's -% not clear this is a general solution. It would be nice to weight by -% distance to root, but that seems to slow down the solve a lot. -% -% One option is to make preferences hard constraints. Or maybe we need -% to look more closely at where a constraint came from and factor that -% into our weights. e.g., a non-default variant resulting from a version -% constraint counts like a version constraint. Needs more thought later. -% - -root(Package, 2) :- root(Package), node(Package). -root(Dependency, 1) :- not root(Dependency), node(Dependency). % The highest priority is to minimize the: % 1. Version weight diff --git a/lib/spack/spack/solver/display.lp b/lib/spack/spack/solver/display.lp index 4b862a26e2..ad4cfdd2ec 100644 --- a/lib/spack/spack/solver/display.lp +++ b/lib/spack/spack/solver/display.lp @@ -1,3 +1,8 @@ +% Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + %=============================================================================- % Display Results % diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f0ae6a1431..5986cf8e18 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -116,6 +116,13 @@ import spack.util.string import spack.variant as vt import spack.version as vn + +if sys.version_info >= (3, 3): + from collections.abc import Mapping # novm +else: + from collections import Mapping + + __all__ = [ 'Spec', 'parse', @@ -1779,10 +1786,12 @@ class Spec(object): name = next(iter(node)) node = node[name] - spec = Spec(name, full_hash=node.get('full_hash', None)) + spec = Spec() + spec.name = name spec.namespace = node.get('namespace', None) spec._hash = node.get('hash', None) spec._build_hash = node.get('build_hash', None) + spec._full_hash = node.get('full_hash', None) if 'version' in node or 'versions' in node: spec.versions = vn.VersionList.from_dict(node) @@ -2120,7 +2129,7 @@ class Spec(object): # which likely means the spec was created with Spec.from_detection msg = ('cannot validate "{0}" since it was not created ' 'using Spec.from_detection'.format(self)) - assert isinstance(self.extra_attributes, collections.Mapping), msg + assert isinstance(self.extra_attributes, Mapping), msg # Validate the spec calling a package specific method validate_fn = getattr( @@ -2529,6 +2538,13 @@ class Spec(object): else: self._old_concretize(tests) + def _mark_root_concrete(self, value=True): + """Mark just this spec (not dependencies) concrete.""" + if (not value) and self.concrete and self.package.installed: + return + self._normal = value + self._concrete = value + def _mark_concrete(self, value=True): """Mark this spec and its dependencies as concrete. @@ -2536,10 +2552,7 @@ class Spec(object): unless there is a need to force a spec to be concrete. """ for s in self.traverse(): - if (not value) and s.concrete and s.package.installed: - continue - s._normal = value - s._concrete = value + s._mark_root_concrete(value) def concretized(self, tests=False): """This is a non-destructive version of concretize(). @@ -3269,7 +3282,7 @@ class Spec(object): """Return list of any virtual deps in this spec.""" return [spec for spec in self.traverse() if spec.virtual] - @property + @property # type: ignore[misc] # decorated prop not supported in mypy @lang.memoized def patches(self): """Return patch objects for any patch sha256 sums on this Spec. diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py index 1aa0ab870e..4f7ae79ed9 100644 --- a/lib/spack/spack/spec_list.py +++ b/lib/spack/spack/spec_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 719d408d84..c39b4d2f33 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,6 +16,7 @@ import sys import tempfile from six import string_types from six import iteritems +from typing import Dict # novm import llnl.util.tty as tty from llnl.util.filesystem import mkdirp, can_access, install, install_tree @@ -221,7 +222,7 @@ class Stage(object): """ """Shared dict of all stage locks.""" - stage_locks = {} + stage_locks = {} # type: Dict[str, spack.util.lock.Lock] """Most staging is managed by Spack. DIYStage is one exception.""" managed_by_spack = True @@ -602,12 +603,12 @@ class Stage(object): """ Ensures the top-level (config:build_stage) directory exists. """ - # Emulate file permissions for tempfile.mkdtemp. + # User has full permissions and group has only read permissions if not os.path.exists(self.path): - mkdirp(self.path, mode=stat.S_IRWXU) + mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) elif not os.path.isdir(self.path): os.remove(self.path) - mkdirp(self.path, mode=stat.S_IRWXU) + mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) # Make sure we can actually do something with the stage we made. ensure_access(self.path) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 779a5531a4..80a7ae0831 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,6 +23,7 @@ install trees to define their own layouts with some per-tree configuration. """ +import contextlib import os import re import six @@ -157,6 +158,8 @@ class Store(object): ): self.root = root self.unpadded_root = unpadded_root or root + self.projections = projections + self.hash_length = hash_length self.db = spack.database.Database( root, upstream_dbs=retrieve_upstream_dbs()) self.layout = spack.directory_layout.YamlDirectoryLayout( @@ -166,6 +169,27 @@ class Store(object): """Convenience function to reindex the store DB with its own layout.""" return self.db.reindex(self.layout) + def serialize(self): + """Return a pickle-able object that can be used to reconstruct + a store. + """ + return ( + self.root, self.unpadded_root, self.projections, self.hash_length + ) + + @staticmethod + def deserialize(token): + """Return a store reconstructed from a token created by + the serialize method. + + Args: + token: return value of the serialize method + + Returns: + Store object reconstructed from the token + """ + return Store(*token) + def _store(): """Get the singleton store instance.""" @@ -227,3 +251,34 @@ def _construct_upstream_dbs_from_install_roots( accumulated_upstream_dbs.insert(0, next_db) return accumulated_upstream_dbs + + +@contextlib.contextmanager +def use_store(store_or_path): + """Use the store passed as argument within the context manager. + + Args: + store_or_path: either a Store object ot a path to where the store resides + + Returns: + Store object associated with the context manager's store + """ + global store, db, layout, root, unpadded_root + + # Normalize input arguments + temporary_store = store_or_path + if not isinstance(store_or_path, Store): + temporary_store = Store(store_or_path) + + # Swap the store with the one just constructed and return it + _ = store.db + original_store, store = store, temporary_store + db, layout = store.db, store.layout + root, unpadded_root = store.root, store.unpadded_root + + yield temporary_store + + # Restore the original store + store = original_store + db, layout = original_store.db, original_store.layout + root, unpadded_root = original_store.root, original_store.unpadded_root diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index b8e157740f..3eee2125d2 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -69,10 +69,12 @@ class PackageInstallContext(object): self.serialized_pkg = serialize(pkg) else: self.pkg = pkg + self.spack_working_dir = spack.main.spack_working_dir self.test_state = TestState() def restore(self): self.test_state.restore() + spack.main.spack_working_dir = self.spack_working_dir if _serialize: return pickle.load(self.serialized_pkg) else: @@ -91,18 +93,21 @@ class TestState(object): self.config = spack.config.config self.platform = spack.architecture.platform self.test_patches = store_patches() - - # TODO: transfer spack.store.store? note that you should not - # transfer spack.store.store and spack.store.db: 'db' is a - # shortcut that accesses the store (so transferring both can - # create an inconsistency). Some tests set 'db' directly, and - # others set 'store' + self.store_token = spack.store.store.serialize() def restore(self): if _serialize: spack.repo.path = spack.repo._path(self.repo_dirs) spack.config.config = self.config spack.architecture.platform = self.platform + + new_store = spack.store.Store.deserialize(self.store_token) + spack.store.store = new_store + spack.store.root = new_store.root + spack.store.unpadded_root = new_store.unpadded_root + spack.store.db = new_store.db + spack.store.layout = new_store.layout + self.test_patches.restore() diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py index 15268e682d..fc2f19f57f 100644 --- a/lib/spack/spack/tengine.py +++ b/lib/spack/spack/tengine.py @@ -1,9 +1,10 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools import textwrap +from typing import List # novm import llnl.util.lang import six @@ -18,7 +19,7 @@ class ContextMeta(type): """ #: Keeps track of the context properties that have been added #: by the class that is being defined - _new_context_properties = [] + _new_context_properties = [] # type: List[str] def __new__(cls, name, bases, attr_dict): # Merge all the context properties that are coming from base classes diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/abi.py b/lib/spack/spack/test/abi.py index dd41228941..ee2aa1f3e6 100644 --- a/lib/spack/spack/test/abi.py +++ b/lib/spack/spack/test/abi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 66c87240a1..e37818f10c 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ """ Test checks if the architecture class is created correctly and also that the functions are looking for the correct architecture name """ +import itertools import os import platform as py_platform @@ -116,20 +117,26 @@ def test_user_defaults(config): assert default_target == default_spec.architecture.target -@pytest.mark.parametrize('operating_system', [ - x for x in spack.architecture.platform().operating_sys -] + ["fe", "be", "frontend", "backend"]) -@pytest.mark.parametrize('target', [ - x for x in spack.architecture.platform().targets -] + ["fe", "be", "frontend", "backend"]) -def test_user_input_combination(config, operating_system, target): - platform = spack.architecture.platform() - spec = Spec("libelf os=%s target=%s" % (operating_system, target)) - spec.concretize() - assert spec.architecture.os == str( - platform.operating_system(operating_system) - ) - assert spec.architecture.target == platform.target(target) +def test_user_input_combination(config): + valid_keywords = ["fe", "be", "frontend", "backend"] + + possible_targets = ([x for x in spack.architecture.platform().targets] + + valid_keywords) + + possible_os = ([x for x in spack.architecture.platform().operating_sys] + + valid_keywords) + + for target, operating_system in itertools.product( + possible_targets, possible_os + ): + platform = spack.architecture.platform() + spec_str = "libelf os={0} target={1}".format(operating_system, target) + spec = Spec(spec_str) + spec.concretize() + assert spec.architecture.os == str( + platform.operating_system(operating_system) + ) + assert spec.architecture.target == platform.target(target) def test_operating_system_conversion_to_dict(): diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index 05cfda5019..1868d74ccc 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -1,96 +1,62 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -""" -This test checks creating and install buildcaches -""" import os import sys +import platform + import py import pytest -import argparse -import platform -import spack.repo -import spack.store + import spack.binary_distribution as bindist -import spack.cmd.buildcache as buildcache -import spack.cmd.install as install -import spack.cmd.uninstall as uninstall -import spack.cmd.mirror as mirror +import spack.config import spack.hooks.sbang as sbang -from spack.main import SpackCommand +import spack.main import spack.mirror +import spack.repo +import spack.store import spack.util.gpg import spack.util.web as web_util + from spack.directory_layout import YamlDirectoryLayout from spack.spec import Spec - -def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}' # noqa: E501 -ndef_install_path_scheme = '${PACKAGE}/${VERSION}/${ARCHITECTURE}-${COMPILERNAME}-${COMPILERVER}-${HASH}' # noqa: E501 - -mirror_path_def = None -mirror_path_rel = None - -mirror_cmd = SpackCommand('mirror') -install_cmd = SpackCommand('install') -uninstall_cmd = SpackCommand('uninstall') -buildcache_cmd = SpackCommand('buildcache') +mirror_cmd = spack.main.SpackCommand('mirror') +install_cmd = spack.main.SpackCommand('install') +uninstall_cmd = spack.main.SpackCommand('uninstall') +buildcache_cmd = spack.main.SpackCommand('buildcache') @pytest.fixture(scope='function') def cache_directory(tmpdir): - old_cache_path = spack.caches.fetch_cache - tmpdir.ensure('fetch_cache', dir=True) - fsc = spack.fetch_strategy.FsCache(str(tmpdir.join('fetch_cache'))) - spack.config.caches = fsc - yield spack.config.caches - tmpdir.join('fetch_cache').remove() - spack.config.caches = old_cache_path - - -@pytest.fixture(scope='session') -def session_mirror_def(tmpdir_factory): - dir = tmpdir_factory.mktemp('mirror') - global mirror_path_rel - mirror_path_rel = dir - dir.ensure('build_cache', dir=True) - yield dir - dir.join('build_cache').remove() + fetch_cache_dir = tmpdir.ensure('fetch_cache', dir=True) + fsc = spack.fetch_strategy.FsCache(str(fetch_cache_dir)) + spack.config.caches, old_cache_path = fsc, spack.caches.fetch_cache + yield spack.config.caches -@pytest.fixture(scope='function') -def mirror_directory_def(session_mirror_def): - yield str(session_mirror_def) + fetch_cache_dir.remove() + spack.config.caches = old_cache_path -@pytest.fixture(scope='session') -def session_mirror_rel(tmpdir_factory): +@pytest.fixture(scope='module') +def mirror_dir(tmpdir_factory): dir = tmpdir_factory.mktemp('mirror') - global mirror_path_rel - mirror_path_rel = dir dir.ensure('build_cache', dir=True) - yield dir + yield str(dir) dir.join('build_cache').remove() @pytest.fixture(scope='function') -def mirror_directory_rel(session_mirror_rel): - yield(session_mirror_rel) - - -@pytest.fixture(scope='function') -def function_mirror(tmpdir): - mirror_dir = str(tmpdir.join('mirror')) - mirror_cmd('add', '--scope', 'site', 'test-mirror-func', - 'file://%s' % mirror_dir) +def test_mirror(mirror_dir): + mirror_url = 'file://%s' % mirror_dir + mirror_cmd('add', '--scope', 'site', 'test-mirror-func', mirror_url) yield mirror_dir mirror_cmd('rm', '--scope=site', 'test-mirror-func') -@pytest.fixture(scope='session') +@pytest.fixture(scope='module') def config_directory(tmpdir_factory): tmpdir = tmpdir_factory.mktemp('test_configs') # restore some sane defaults for packages and config @@ -116,8 +82,14 @@ def config_directory(tmpdir_factory): @pytest.fixture(scope='function') -def default_config(tmpdir_factory, config_directory, monkeypatch): - +def default_config( + tmpdir_factory, config_directory, monkeypatch, + install_mockery_mutable_config +): + # This fixture depends on install_mockery_mutable_config to ensure + # there is a clear order of initialization. The substitution of the + # config scopes here is done on top of the substitution that comes with + # install_mockery_mutable_config mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp') config_directory.copy(mutable_dir) @@ -126,7 +98,7 @@ def default_config(tmpdir_factory, config_directory, monkeypatch): for name in ['site/%s' % platform.system().lower(), 'site', 'user']]) - monkeypatch.setattr(spack.config, 'config', cfg) + spack.config.config, old_config = cfg, spack.config.config # This is essential, otherwise the cache will create weird side effects # that will compromise subsequent tests if compilers.yaml is modified @@ -148,18 +120,25 @@ def default_config(tmpdir_factory, config_directory, monkeypatch): timeout = spack.config.get('config:connect_timeout') if not timeout: spack.config.set('config:connect_timeout', 10, scope='user') + yield spack.config.config + + spack.config.config = old_config mutable_dir.remove() @pytest.fixture(scope='function') def install_dir_default_layout(tmpdir): """Hooks a fake install directory with a default layout""" - real_store = spack.store.store - real_layout = spack.store.layout - spack.store.store = spack.store.Store(str(tmpdir.join('opt'))) - spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')), - path_scheme=def_install_path_scheme) # noqa: E501 + scheme = os.path.join( + '${architecture}', + '${compiler.name}-${compiler.version}', + '${name}-${version}-${hash}' + ) + real_store, real_layout = spack.store.store, spack.store.layout + opt_dir = tmpdir.join('opt') + spack.store.store = spack.store.Store(str(opt_dir)) + spack.store.layout = YamlDirectoryLayout(str(opt_dir), path_scheme=scheme) try: yield spack.store finally: @@ -170,11 +149,14 @@ def install_dir_default_layout(tmpdir): @pytest.fixture(scope='function') def install_dir_non_default_layout(tmpdir): """Hooks a fake install directory with a non-default layout""" - real_store = spack.store.store - real_layout = spack.store.layout - spack.store.store = spack.store.Store(str(tmpdir.join('opt'))) - spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')), - path_scheme=ndef_install_path_scheme) # noqa: E501 + scheme = os.path.join( + '${name}', '${version}', + '${architecture}-${compiler.name}-${compiler.version}-${hash}' + ) + real_store, real_layout = spack.store.store, spack.store.layout + opt_dir = tmpdir.join('opt') + spack.store.store = spack.store.Store(str(opt_dir)) + spack.store.layout = YamlDirectoryLayout(str(opt_dir), path_scheme=scheme) try: yield spack.store finally: @@ -190,308 +172,149 @@ else: @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_default_layout') -def test_default_rpaths_create_install_default_layout(tmpdir, - mirror_directory_def, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_default_layout', + 'test_mirror' +) +def test_default_rpaths_create_install_default_layout(mirror_dir): """ Test the creation and installation of buildcaches with default rpaths into the default directory layout scheme. """ + gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized() - gspec = Spec('garply') - gspec.concretize() - cspec = Spec('corge') - cspec.concretize() - - iparser = argparse.ArgumentParser() - install.setup_parser(iparser) - # Install some packages with dependent packages - iargs = iparser.parse_args(['--no-cache', cspec.name]) - install.install(iparser, iargs) - - global mirror_path_def - mirror_path_def = mirror_directory_def - mparser = argparse.ArgumentParser() - mirror.setup_parser(mparser) - margs = mparser.parse_args( - ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def]) - mirror.mirror(mparser, margs) - margs = mparser.parse_args(['list']) - mirror.mirror(mparser, margs) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - # Set default buildcache args - create_args = ['create', '-a', '-u', '-d', str(mirror_path_def), - cspec.name] - install_args = ['install', '-a', '-u', cspec.name] + # Install 'corge' without using a cache + install_cmd('--no-cache', cspec.name) # Create a buildache - args = parser.parse_args(create_args) - buildcache.buildcache(parser, args) - # Test force overwrite create buildcache - create_args.insert(create_args.index('-a'), '-f') - args = parser.parse_args(create_args) - buildcache.buildcache(parser, args) - # create mirror index - args = parser.parse_args(['update-index', '-d', 'file://%s' % str(mirror_path_def)]) - buildcache.buildcache(parser, args) - # list the buildcaches in the mirror - args = parser.parse_args(['list', '-a', '-l', '-v']) - buildcache.buildcache(parser, args) + buildcache_cmd('create', '-au', '-d', mirror_dir, cspec.name) + # Test force overwrite create buildcache (-f option) + buildcache_cmd('create', '-auf', '-d', mirror_dir, cspec.name) + + # Create mirror index + mirror_url = 'file://{0}'.format(mirror_dir) + buildcache_cmd('update-index', '-d', mirror_url) + # List the buildcaches in the mirror + buildcache_cmd('list', '-alv') # Uninstall the package and deps - uparser = argparse.ArgumentParser() - uninstall.setup_parser(uparser) - uargs = uparser.parse_args(['-y', '--dependents', gspec.name]) - uninstall.uninstall(uparser, uargs) + uninstall_cmd('-y', '--dependents', gspec.name) - # test install - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) + # Test installing from build caches + buildcache_cmd('install', '-au', cspec.name) # This gives warning that spec is already installed - buildcache.buildcache(parser, args) - - # test overwrite install - install_args.insert(install_args.index('-a'), '-f') - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) + buildcache_cmd('install', '-au', cspec.name) - args = parser.parse_args(['keys', '-f']) - buildcache.buildcache(parser, args) + # Test overwrite install + buildcache_cmd('install', '-afu', cspec.name) - args = parser.parse_args(['list']) - buildcache.buildcache(parser, args) + buildcache_cmd('keys', '-f') + buildcache_cmd('list') - args = parser.parse_args(['list', '-a']) - buildcache.buildcache(parser, args) - - args = parser.parse_args(['list', '-l', '-v']) - buildcache.buildcache(parser, args) - bindist.clear_spec_cache() - spack.stage.purge() - margs = mparser.parse_args( - ['rm', '--scope', 'site', 'test-mirror-def']) - mirror.mirror(mparser, margs) + buildcache_cmd('list', '-a') + buildcache_cmd('list', '-l', '-v') @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow @pytest.mark.nomockstage -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_non_default_layout') -def test_default_rpaths_install_nondefault_layout(tmpdir, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_non_default_layout', + 'test_mirror' +) +def test_default_rpaths_install_nondefault_layout(mirror_dir): """ Test the creation and installation of buildcaches with default rpaths into the non-default directory layout scheme. """ - - gspec = Spec('garply') - gspec.concretize() - cspec = Spec('corge') - cspec.concretize() - - global mirror_path_def - mparser = argparse.ArgumentParser() - mirror.setup_parser(mparser) - margs = mparser.parse_args( - ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def]) - mirror.mirror(mparser, margs) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - # Set default buildcache args - install_args = ['install', '-a', '-u', '%s' % cspec.name] + cspec = Spec('corge').concretized() # Install some packages with dependent packages # test install in non-default install path scheme - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) - # test force install in non-default install path scheme - install_args.insert(install_args.index('-a'), '-f') - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) + buildcache_cmd('install', '-au', cspec.name) - bindist.clear_spec_cache() - spack.stage.purge() - margs = mparser.parse_args( - ['rm', '--scope', 'site', 'test-mirror-def']) - mirror.mirror(mparser, margs) + # Test force install in non-default install path scheme + buildcache_cmd('install', '-auf', cspec.name) @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow @pytest.mark.nomockstage -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_default_layout') -def test_relative_rpaths_create_default_layout(tmpdir, - mirror_directory_rel, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_default_layout' +) +def test_relative_rpaths_create_default_layout(mirror_dir): """ Test the creation and installation of buildcaches with relative rpaths into the default directory layout scheme. """ - gspec = Spec('garply') - gspec.concretize() - cspec = Spec('corge') - cspec.concretize() + gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized() - global mirror_path_rel - mirror_path_rel = mirror_directory_rel - # Install patchelf needed for relocate in linux test environment - iparser = argparse.ArgumentParser() - install.setup_parser(iparser) - # Install some packages with dependent packages - iargs = iparser.parse_args(['--no-cache', cspec.name]) - install.install(iparser, iargs) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - # set default buildcache args - create_args = ['create', '-a', '-u', '-r', '-d', - str(mirror_path_rel), - cspec.name] - - # create build cache with relatived rpaths - args = parser.parse_args(create_args) - buildcache.buildcache(parser, args) - # create mirror index - args = parser.parse_args(['update-index', '-d', 'file://%s' % str(mirror_path_rel)]) - buildcache.buildcache(parser, args) - # Uninstall the package and deps - uparser = argparse.ArgumentParser() - uninstall.setup_parser(uparser) - uargs = uparser.parse_args(['-y', '--dependents', gspec.name]) - uninstall.uninstall(uparser, uargs) + # Install 'corge' without using a cache + install_cmd('--no-cache', cspec.name) - bindist.clear_spec_cache() - spack.stage.purge() + # Create build cache with relative rpaths + buildcache_cmd( + 'create', '-aur', '-d', mirror_dir, cspec.name + ) + + # Create mirror index + mirror_url = 'file://%s' % mirror_dir + buildcache_cmd('update-index', '-d', mirror_url) + + # Uninstall the package and deps + uninstall_cmd('-y', '--dependents', gspec.name) @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow @pytest.mark.nomockstage -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_default_layout') -def test_relative_rpaths_install_default_layout(tmpdir, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_default_layout', + 'test_mirror' +) +def test_relative_rpaths_install_default_layout(mirror_dir): """ Test the creation and installation of buildcaches with relative rpaths into the default directory layout scheme. """ + gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized() - gspec = Spec('garply') - gspec.concretize() - cspec = Spec('corge') - cspec.concretize() - - global mirror_path_rel - mparser = argparse.ArgumentParser() - mirror.setup_parser(mparser) - margs = mparser.parse_args( - ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel]) - mirror.mirror(mparser, margs) - - iparser = argparse.ArgumentParser() - install.setup_parser(iparser) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - # set default buildcache args - install_args = ['install', '-a', '-u', '-f', - cspec.name] - - # install buildcache created with relativized rpaths - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) + # Install buildcache created with relativized rpaths + buildcache_cmd('install', '-auf', cspec.name) # This gives warning that spec is already installed - buildcache.buildcache(parser, args) + buildcache_cmd('install', '-auf', cspec.name) # Uninstall the package and deps - uparser = argparse.ArgumentParser() - uninstall.setup_parser(uparser) - uargs = uparser.parse_args(['-y', '--dependents', gspec.name]) - uninstall.uninstall(uparser, uargs) - - # install build cache - buildcache.buildcache(parser, args) + uninstall_cmd('-y', '--dependents', gspec.name) - # test overwrite install - install_args.insert(install_args.index('-a'), '-f') - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) + # Install build cache + buildcache_cmd('install', '-auf', cspec.name) - bindist.clear_spec_cache() - spack.stage.purge() - margs = mparser.parse_args( - ['rm', '--scope', 'site', 'test-mirror-rel']) - mirror.mirror(mparser, margs) + # Test overwrite install + buildcache_cmd('install', '-auf', cspec.name) @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow @pytest.mark.nomockstage -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_non_default_layout') -def test_relative_rpaths_install_nondefault(tmpdir, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_non_default_layout', + 'test_mirror' +) +def test_relative_rpaths_install_nondefault(mirror_dir): """ Test the installation of buildcaches with relativized rpaths into the non-default directory layout scheme. """ + cspec = Spec('corge').concretized() - gspec = Spec('garply') - gspec.concretize() - cspec = Spec('corge') - cspec.concretize() - - global mirror_path_rel - - mparser = argparse.ArgumentParser() - mirror.setup_parser(mparser) - margs = mparser.parse_args( - ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel]) - mirror.mirror(mparser, margs) - - iparser = argparse.ArgumentParser() - install.setup_parser(iparser) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - # Set default buildcache args - install_args = ['install', '-a', '-u', '-f', '%s' % cspec.name] - - # test install in non-default install path scheme and relative path - args = parser.parse_args(install_args) - buildcache.buildcache(parser, args) - - bindist.clear_spec_cache() - spack.stage.purge() - margs = mparser.parse_args( - ['rm', '--scope', 'site', 'test-mirror-rel']) - mirror.mirror(mparser, margs) + # Test install in non-default install path scheme and relative path + buildcache_cmd('install', '-auf', cspec.name) @pytest.mark.skipif(not spack.util.gpg.has_gpg(), @@ -534,38 +357,20 @@ def test_push_and_fetch_keys(mock_gnupghome): @pytest.mark.requires_executables(*args) -@pytest.mark.disable_clean_stage_check @pytest.mark.maybeslow @pytest.mark.nomockstage -@pytest.mark.usefixtures('default_config', 'cache_directory', - 'install_dir_non_default_layout') -def test_built_spec_cache(tmpdir, - install_mockery): +@pytest.mark.usefixtures( + 'default_config', 'cache_directory', 'install_dir_non_default_layout', + 'test_mirror' +) +def test_built_spec_cache(mirror_dir): """ Because the buildcache list command fetches the buildcache index and uses it to populate the binary_distribution built spec cache, when this test calls get_mirrors_for_spec, it is testing the popluation of that cache from a buildcache index. """ - global mirror_path_rel - - mparser = argparse.ArgumentParser() - mirror.setup_parser(mparser) - margs = mparser.parse_args( - ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel]) - mirror.mirror(mparser, margs) - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - list_args = ['list', '-a', '-l'] - args = parser.parse_args(list_args) - buildcache.buildcache(parser, args) - - gspec = Spec('garply') - gspec.concretize() + buildcache_cmd('list', '-a', '-l') - cspec = Spec('corge') - cspec.concretize() + gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized() full_hash_map = { 'garply': gspec.full_hash(), @@ -590,12 +395,6 @@ def test_built_spec_cache(tmpdir, assert(result['mirror_url'] not in cspec_mirrors) cspec_mirrors[result['mirror_url']] = True - bindist.clear_spec_cache() - - margs = mparser.parse_args( - ['rm', '--scope', 'site', 'test-mirror-rel']) - mirror.mirror(mparser, margs) - def fake_full_hash(spec): # Generate an arbitrary hash that is intended to be different than @@ -604,8 +403,11 @@ def fake_full_hash(spec): return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6' -def test_spec_needs_rebuild(install_mockery_mutable_config, mock_packages, - mock_fetch, monkeypatch, tmpdir): +@pytest.mark.usefixtures( + 'install_mockery_mutable_config', 'mock_packages', 'mock_fetch', + 'test_mirror' +) +def test_spec_needs_rebuild(monkeypatch, tmpdir): """Make sure needs_rebuild properly compares remote full_hash against locally computed one, avoiding unnecessary rebuilds""" @@ -613,8 +415,6 @@ def test_spec_needs_rebuild(install_mockery_mutable_config, mock_packages, mirror_dir = tmpdir.join('mirror_dir') mirror_url = 'file://{0}'.format(mirror_dir.strpath) - mirror_cmd('add', 'test-mirror', mirror_url) - s = Spec('libdwarf').concretized() # Install a package @@ -683,47 +483,49 @@ def test_generate_indices_exception(monkeypatch, capfd): assert expect in err -@pytest.mark.usefixtures('mock_fetch') -def test_update_sbang(tmpdir, install_mockery, function_mirror): - """ - Test the creation and installation of buildcaches with default rpaths +@pytest.mark.usefixtures('mock_fetch', 'install_mockery') +def test_update_sbang(tmpdir, test_mirror): + """Test the creation and installation of buildcaches with default rpaths into the non-default directory layout scheme, triggering an update of the sbang. """ - - # Save the original store and layout before we touch ANYTHING. - real_store = spack.store.store - real_layout = spack.store.layout - + scheme = os.path.join( + '${name}', '${version}', + '${architecture}-${compiler.name}-${compiler.version}-${hash}' + ) + spec_str = 'old-sbang' # Concretize a package with some old-fashioned sbang lines. - sspec = Spec('old-sbang') - sspec.concretize() + old_spec = Spec(spec_str).concretized() + old_spec_hash_str = '/{0}'.format(old_spec.dag_hash()) # Need a fake mirror with *function* scope. - mirror_dir = function_mirror + mirror_dir = test_mirror + mirror_url = 'file://{0}'.format(mirror_dir) - # Assumes all commands will concretize sspec the same way. - install_cmd('--no-cache', sspec.name) + # Assume all commands will concretize old_spec the same way. + install_cmd('--no-cache', old_spec.name) # Create a buildcache with the installed spec. - buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, - '/%s' % sspec.dag_hash()) + buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, old_spec_hash_str) # Need to force an update of the buildcache index - buildcache_cmd('update-index', '-d', 'file://%s' % mirror_dir) + buildcache_cmd('update-index', '-d', mirror_url) # Uninstall the original package. - uninstall_cmd('-y', '/%s' % sspec.dag_hash()) + uninstall_cmd('-y', old_spec_hash_str) - try: - # New install tree locations... - # Too fine-grained to do be done in a fixture - spack.store.store = spack.store.Store(str(tmpdir.join('newtree'))) - spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('newtree')), - path_scheme=ndef_install_path_scheme) # noqa: E501 + # Switch the store to the new install tree locations + newtree_dir = tmpdir.join('newtree') + s = spack.store.Store(str(newtree_dir)) + s.layout = YamlDirectoryLayout(str(newtree_dir), path_scheme=scheme) + + with spack.store.use_store(s): + new_spec = Spec('old-sbang') + new_spec.concretize() + assert new_spec.dag_hash() == old_spec.dag_hash() # Install package from buildcache - buildcache_cmd('install', '-a', '-u', '-f', sspec.name) + buildcache_cmd('install', '-a', '-u', '-f', new_spec.name) # Continue blowing away caches bindist.clear_spec_cache() @@ -734,25 +536,19 @@ def test_update_sbang(tmpdir, install_mockery, function_mirror): #!/usr/bin/env python {1} - '''.format(sbang.sbang_shebang_line(), sspec.prefix.bin) +'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin) sbang_style_2_expected = '''{0} #!/usr/bin/env python {1} - '''.format(sbang.sbang_shebang_line(), sspec.prefix.bin) +'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin) - installed_script_style_1_path = \ - sspec.prefix.bin.join('sbang-style-1.sh') + installed_script_style_1_path = new_spec.prefix.bin.join('sbang-style-1.sh') assert sbang_style_1_expected == \ open(str(installed_script_style_1_path)).read() - installed_script_style_2_path = \ - sspec.prefix.bin.join('sbang-style-2.sh') + installed_script_style_2_path = new_spec.prefix.bin.join('sbang-style-2.sh') assert sbang_style_2_expected == \ open(str(installed_script_style_2_path)).read() - uninstall_cmd('-y', '/%s' % sspec.dag_hash()) - - finally: - spack.store.store = real_store - spack.store.layout = real_layout + uninstall_cmd('-y', '/%s' % new_spec.dag_hash()) diff --git a/lib/spack/spack/test/build_distribution.py b/lib/spack/spack/test/build_distribution.py index 2d567aee14..873a57a0e9 100644 --- a/lib/spack/spack/test/build_distribution.py +++ b/lib/spack/spack/test/build_distribution.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index a7a55af0ca..28f6aefcb8 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index 6c2e3c6be5..ab9735fd48 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/build_systems.py b/lib/spack/spack/test/build_systems.py index 1606ba8ebd..b02da380a8 100644 --- a/lib/spack/spack/test/build_systems.py +++ b/lib/spack/spack/test/build_systems.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/buildrequest.py b/lib/spack/spack/test/buildrequest.py index 5918838edc..22d7b5fff1 100644 --- a/lib/spack/spack/test/buildrequest.py +++ b/lib/spack/spack/test/buildrequest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/buildtask.py b/lib/spack/spack/test/buildtask.py index f82ff6861a..44faf6244d 100644 --- a/lib/spack/spack/test/buildtask.py +++ b/lib/spack/spack/test/buildtask.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cache_fetch.py b/lib/spack/spack/test/cache_fetch.py index 3b4c3cb887..8771769307 100644 --- a/lib/spack/spack/test/cache_fetch.py +++ b/lib/spack/spack/test/cache_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 7b8d34fbde..59b2f8e4cb 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,6 +28,7 @@ test_args = [ '-Wl,--end-group', '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath', + '-Wl,--rpath,/fifth/rpath', '-Wl,--rpath', '-Wl,/sixth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] @@ -45,11 +46,13 @@ test_library_paths = [ test_wl_rpaths = [ '-Wl,-rpath,/first/rpath', '-Wl,-rpath,/second/rpath', - '-Wl,-rpath,/third/rpath', '-Wl,-rpath,/fourth/rpath'] + '-Wl,-rpath,/third/rpath', '-Wl,-rpath,/fourth/rpath', + '-Wl,-rpath,/fifth/rpath', '-Wl,-rpath,/sixth/rpath'] test_rpaths = [ '-rpath', '/first/rpath', '-rpath', '/second/rpath', - '-rpath', '/third/rpath', '-rpath', '/fourth/rpath'] + '-rpath', '/third/rpath', '-rpath', '/fourth/rpath', + '-rpath', '/fifth/rpath', '-rpath', '/sixth/rpath'] test_args_without_paths = [ 'arg1', @@ -347,15 +350,15 @@ def test_ccld_deps_isystem(): with set_env(SPACK_INCLUDE_DIRS='xinc:yinc:zinc', SPACK_RPATH_DIRS='xlib:ylib:zlib', SPACK_LINK_DIRS='xlib:ylib:zlib'): - mytest_args = test_args + ['-isystemfooinc'] + mytest_args = test_args + ['-isystem', 'fooinc'] check_args( cc, mytest_args, [real_cc] + test_include_paths + - ['-isystemfooinc', - '-isystemxinc', - '-isystemyinc', - '-isystemzinc'] + + ['-isystem', 'fooinc', + '-isystem', 'xinc', + '-isystem', 'yinc', + '-isystem', 'zinc'] + test_library_paths + ['-Lxlib', '-Lylib', @@ -429,20 +432,20 @@ def test_ccld_with_system_dirs_isystem(): SPACK_RPATH_DIRS='xlib:ylib:zlib', SPACK_LINK_DIRS='xlib:ylib:zlib'): - sys_path_args = ['-isystem/usr/include', + sys_path_args = ['-isystem', '/usr/include', '-L/usr/local/lib', '-Wl,-rpath,/usr/lib64', - '-isystem/usr/local/include', + '-isystem', '/usr/local/include', '-L/lib64/'] check_args( cc, sys_path_args + test_args, [real_cc] + test_include_paths + - ['-isystemxinc', - '-isystemyinc', - '-isystemzinc'] + - ['-isystem/usr/include', - '-isystem/usr/local/include'] + + ['-isystem', 'xinc', + '-isystem', 'yinc', + '-isystem', 'zinc'] + + ['-isystem', '/usr/include', + '-isystem', '/usr/local/include'] + test_library_paths + ['-Lxlib', '-Lylib', diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index f1986b34d0..33488a2d8a 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/__init__.py b/lib/spack/spack/test/cmd/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/cmd/__init__.py +++ b/lib/spack/spack/test/cmd/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/activate.py b/lib/spack/spack/test/cmd/activate.py index a3546230f1..48806c5833 100644 --- a/lib/spack/spack/test/cmd/activate.py +++ b/lib/spack/spack/test/cmd/activate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/arch.py b/lib/spack/spack/test/cmd/arch.py index ec3c92a3d0..cf4969ce28 100644 --- a/lib/spack/spack/test/cmd/arch.py +++ b/lib/spack/spack/test/cmd/arch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index 61bdf41084..5ca69bb7b4 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/build_env.py b/lib/spack/spack/test/cmd/build_env.py index b7de759a2e..4c0f337aac 100644 --- a/lib/spack/spack/test/cmd/build_env.py +++ b/lib/spack/spack/test/cmd/build_env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index dcd9b822c0..27ef4fca8f 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/cd.py b/lib/spack/spack/test/cmd/cd.py index eda6994aec..c8766503b4 100644 --- a/lib/spack/spack/test/cmd/cd.py +++ b/lib/spack/spack/test/cmd/cd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 4d8468c66c..24992d78c2 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,10 +7,11 @@ import filecmp import json import os import pytest -from jsonschema import validate +from jsonschema import validate, ValidationError import spack import spack.ci as ci +import spack.compilers as compilers import spack.config import spack.environment as ev import spack.hash_types as ht @@ -19,7 +20,8 @@ import spack.paths as spack_paths import spack.repo as repo from spack.schema.buildcache_spec import schema as spec_yaml_schema from spack.schema.database_index import schema as db_idx_schema -from spack.spec import Spec +from spack.schema.gitlab_ci import schema as gitlab_ci_schema +from spack.spec import Spec, CompilerSpec from spack.util.mock_package import MockPackageMultiRepo import spack.util.executable as exe import spack.util.spack_yaml as syaml @@ -31,6 +33,7 @@ env_cmd = spack.main.SpackCommand('env') mirror_cmd = spack.main.SpackCommand('mirror') gpg_cmd = spack.main.SpackCommand('gpg') install_cmd = spack.main.SpackCommand('install') +uninstall_cmd = spack.main.SpackCommand('uninstall') buildcache_cmd = spack.main.SpackCommand('buildcache') git = exe.which('git', required=True) @@ -73,17 +76,17 @@ and then 'd', 'b', and 'a' to be put in the next three stages, respectively. b = mock_repo.add_package('b', [d, e], [default, default]) mock_repo.add_package('a', [b, c], [default, default]) - with repo.swap(mock_repo): + with repo.use_repositories(mock_repo): spec_a = Spec('a') spec_a.concretize() - spec_a_label = ci.spec_deps_key_label(spec_a)[1] - spec_b_label = ci.spec_deps_key_label(spec_a['b'])[1] - spec_c_label = ci.spec_deps_key_label(spec_a['c'])[1] - spec_d_label = ci.spec_deps_key_label(spec_a['d'])[1] - spec_e_label = ci.spec_deps_key_label(spec_a['e'])[1] - spec_f_label = ci.spec_deps_key_label(spec_a['f'])[1] - spec_g_label = ci.spec_deps_key_label(spec_a['g'])[1] + spec_a_label = ci.spec_deps_key(spec_a) + spec_b_label = ci.spec_deps_key(spec_a['b']) + spec_c_label = ci.spec_deps_key(spec_a['c']) + spec_d_label = ci.spec_deps_key(spec_a['d']) + spec_e_label = ci.spec_deps_key(spec_a['e']) + spec_f_label = ci.spec_deps_key(spec_a['f']) + spec_g_label = ci.spec_deps_key(spec_a['g']) spec_labels, dependencies, stages = ci.stage_spec_jobs([spec_a]) @@ -109,6 +112,7 @@ def test_ci_generate_with_env(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, mock_packages): """Make sure we can get a .gitlab-ci.yml from an environment file which has the gitlab-ci, cdash, and mirrors sections.""" + mirror_url = 'https://my.fake.mirror' filename = str(tmpdir.join('spack.yaml')) with open(filename, 'w') as f: f.write("""\ @@ -128,7 +132,7 @@ spack: - matrix: - [$old-gcc-pkgs] mirrors: - some-mirror: https://my.fake.mirror + some-mirror: {0} gitlab-ci: bootstrap: - name: bootstrap @@ -140,7 +144,7 @@ spack: tags: - donotcare image: donotcare - final-stage-rebuild-index: + service-job-attributes: image: donotcare tags: [donotcare] cdash: @@ -148,7 +152,7 @@ spack: url: https://my.fake.cdash project: Not used site: Nothing -""") +""".format(mirror_url)) with tmpdir.as_cwd(): env_cmd('create', 'test', './spack.yaml') outputfile = str(tmpdir.join('.gitlab-ci.yml')) @@ -170,16 +174,30 @@ spack: assert(yaml_contents['stages'][0] == 'stage-0') assert(yaml_contents['stages'][5] == 'stage-rebuild-index') + assert('rebuild-index' in yaml_contents) + rebuild_job = yaml_contents['rebuild-index'] + expected = 'spack buildcache update-index --keys -d {0}'.format( + mirror_url) + assert(rebuild_job['script'][0] == expected) + def _validate_needs_graph(yaml_contents, needs_graph, artifacts): for job_name, job_def in yaml_contents.items(): for needs_def_name, needs_list in needs_graph.items(): if job_name.startswith(needs_def_name): # check job needs against the expected needs definition + j_needs = job_def['needs'] + print('job {0} needs:'.format(needs_def_name)) + print([j['job'] for j in j_needs]) + print('expected:') + print([nl for nl in needs_list]) assert all([job_needs['job'][:job_needs['job'].index('/')] - in needs_list for job_needs in job_def['needs']]) + in needs_list for job_needs in j_needs]) + assert(all([nl in + [n['job'][:n['job'].index('/')] for n in j_needs] + for nl in needs_list])) assert all([job_needs['artifacts'] == artifacts - for job_needs in job_def['needs']]) + for job_needs in j_needs]) break @@ -369,7 +387,10 @@ spack: fake_token = 'notreallyatokenbutshouldnotmatter' os.environ['SPACK_CDASH_AUTH_TOKEN'] = fake_token copy_to_file = str(tmpdir.join('backup-ci.yml')) - output = ci_cmd('generate', '--copy-to', copy_to_file, output=str) + try: + output = ci_cmd('generate', '--copy-to', copy_to_file, output=str) + finally: + del os.environ['SPACK_CDASH_AUTH_TOKEN'] # That fake token should still have resulted in being unable to # register build group with cdash, but the workload should # still have been generated. @@ -533,7 +554,7 @@ spack: def test_ci_generate_for_pr_pipeline(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, - mock_packages): + mock_packages, monkeypatch): """Test that PR pipelines do not include a final stage job for rebuilding the mirror index, even if that job is specifically configured""" @@ -558,9 +579,10 @@ spack: runner-attributes: tags: - donotcare - final-stage-rebuild-index: + service-job-attributes: image: donotcare tags: [donotcare] + rebuild-index: False """) with tmpdir.as_cwd(): @@ -569,7 +591,14 @@ spack: with ev.read('test'): os.environ['SPACK_IS_PR_PIPELINE'] = 'True' - ci_cmd('generate', '--output-file', outputfile) + os.environ['SPACK_PR_BRANCH'] = 'fake-test-branch' + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + try: + ci_cmd('generate', '--output-file', outputfile) + finally: + del os.environ['SPACK_IS_PR_PIPELINE'] + del os.environ['SPACK_PR_BRANCH'] with open(outputfile) as f: contents = f.read() @@ -579,10 +608,17 @@ spack: assert('rebuild-index' not in yaml_contents) + for ci_key in yaml_contents.keys(): + if ci_key.startswith('(specs) '): + job_object = yaml_contents[ci_key] + job_vars = job_object['variables'] + assert('SPACK_IS_PR_PIPELINE' in job_vars) + assert(job_vars['SPACK_IS_PR_PIPELINE'] == 'True') + def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, - mock_packages): + mock_packages, monkeypatch): """Make sure we do not generate jobs for external pkgs""" filename = str(tmpdir.join('spack.yaml')) with open(filename, 'w') as f: @@ -609,6 +645,8 @@ spack: outputfile = str(tmpdir.join('.gitlab-ci.yml')) with ev.read('test'): + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -712,6 +750,19 @@ spack: - $packages mirrors: test-mirror: {0} + gitlab-ci: + enable-artifacts-buildcache: True + mappings: + - match: + - patchelf + runner-attributes: + tags: + - donotcare + image: donotcare + service-job-attributes: + tags: + - nonbuildtag + image: basicimage """.format(mirror_url) print('spack.yaml:\n{0}\n'.format(spack_yaml_contents)) @@ -739,6 +790,48 @@ spack: buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache') + # Now test the --prune-dag (default) option of spack ci generate + mirror_cmd('add', 'test-ci', mirror_url) + + outputfile_pruned = str(tmpdir.join('pruned_pipeline.yml')) + ci_cmd('generate', '--output-file', outputfile_pruned) + + with open(outputfile_pruned) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + assert('no-specs-to-rebuild' in yaml_contents) + # Make sure there are no other spec jobs or rebuild-index + assert(len(yaml_contents.keys()) == 1) + the_elt = yaml_contents['no-specs-to-rebuild'] + assert('tags' in the_elt) + assert('nonbuildtag' in the_elt['tags']) + assert('image' in the_elt) + assert(the_elt['image'] == 'basicimage') + + outputfile_not_pruned = str(tmpdir.join('unpruned_pipeline.yml')) + ci_cmd('generate', '--no-prune-dag', '--output-file', + outputfile_not_pruned) + + # Test the --no-prune-dag option of spack ci generate + with open(outputfile_not_pruned) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + + found_spec_job = False + + for ci_key in yaml_contents.keys(): + if '(specs) patchelf' in ci_key: + the_elt = yaml_contents[ci_key] + assert('variables' in the_elt) + job_vars = the_elt['variables'] + assert('SPACK_SPEC_NEEDS_REBUILD' in job_vars) + assert(job_vars['SPACK_SPEC_NEEDS_REBUILD'] == 'False') + found_spec_job = True + + assert(found_spec_job) + + mirror_cmd('rm', 'test-ci') + # Test generating buildcache index while we have bin mirror buildcache_cmd('update-index', '--mirror-url', mirror_url) index_path = os.path.join(buildcache_path, 'index.json') @@ -785,6 +878,26 @@ spack: assert(len(dl_dir_list) == 3) +def test_push_mirror_contents_exceptions(monkeypatch, capsys): + def faked(env, spec_yaml=None, packages=None, add_spec=True, + add_deps=True, output_location=os.getcwd(), + signing_key=None, force=False, make_relative=False, + unsigned=False, allow_root=False, rebuild_index=False): + raise Exception('Error: Access Denied') + + import spack.cmd.buildcache as buildcache + monkeypatch.setattr(buildcache, '_createtarball', faked) + + url = 'fakejunk' + ci.push_mirror_contents(None, None, None, url, None, None) + + captured = capsys.readouterr() + std_out = captured[0] + expect_msg = 'Permission problem writing to {0}'.format(url) + + assert(expect_msg in std_out) + + def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path, env_deactivate, install_mockery, mock_packages, monkeypatch): @@ -839,7 +952,7 @@ spack: - custom main step after_script: - custom post step one - final-stage-rebuild-index: + service-job-attributes: image: donotcare tags: [donotcare] """) @@ -851,6 +964,8 @@ spack: with ev.read('test'): monkeypatch.setattr( spack.main, 'get_version', lambda: '0.15.3-416-12ad69eb1') + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -925,3 +1040,364 @@ spack: assert(the_elt['script'][0] == 'main step') assert(len(the_elt['after_script']) == 1) assert(the_elt['after_script'][0] == 'post step one') + + +def test_ci_generate_with_workarounds(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages, monkeypatch): + """Make sure the post-processing cli workarounds do what they should""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - callpath%gcc@3.0 + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + mappings: + - match: ['%gcc@3.0'] + runner-attributes: + tags: + - donotcare + image: donotcare + enable-artifacts-buildcache: true +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + ci_cmd('generate', '--output-file', outputfile, '--dependencies') + + with open(outputfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + + found_one = False + + for ci_key in yaml_contents.keys(): + if ci_key.startswith('(specs) '): + found_one = True + job_obj = yaml_contents[ci_key] + assert('needs' not in job_obj) + assert('dependencies' in job_obj) + + assert(found_one is True) + + +@pytest.mark.disable_clean_stage_check +def test_ci_rebuild_index(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages, mock_fetch, + mock_stage): + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + spack_yaml_contents = """ +spack: + specs: + - callpath + mirrors: + test-mirror: {0} + gitlab-ci: + mappings: + - match: + - patchelf + runner-attributes: + tags: + - donotcare + image: donotcare +""".format(mirror_url) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + with ev.read('test'): + spec_map = ci.get_concrete_specs( + 'callpath', 'callpath', '', 'FIND_ANY') + concrete_spec = spec_map['callpath'] + spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash) + yaml_path = str(tmpdir.join('spec.yaml')) + with open(yaml_path, 'w') as ypfd: + ypfd.write(spec_yaml) + + install_cmd('--keep-stage', '-f', yaml_path) + buildcache_cmd('create', '-u', '-a', '-f', '--mirror-url', + mirror_url, 'callpath') + ci_cmd('rebuild-index') + + buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache') + index_path = os.path.join(buildcache_path, 'index.json') + with open(index_path) as idx_fd: + index_object = json.load(idx_fd) + validate(index_object, db_idx_schema) + + +def test_ci_generate_bootstrap_prune_dag( + install_mockery_mutable_config, mock_packages, mock_fetch, + mock_archive, mutable_config, monkeypatch, tmpdir, + mutable_mock_env_path, env_deactivate): + """Test compiler bootstrapping with DAG pruning. Specifically, make + sure that if we detect the bootstrapped compiler needs to be rebuilt, + we ensure the spec we want to build with that compiler is scheduled + for rebuild as well.""" + + # Create a temp mirror directory for buildcache usage + mirror_dir = tmpdir.join('mirror_dir') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + # Install a compiler, because we want to put it in a buildcache + install_cmd('gcc@10.1.0%gcc@4.5.0') + + # Put installed compiler in the buildcache + buildcache_cmd('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, + 'gcc@10.1.0%gcc@4.5.0') + + # Now uninstall the compiler + uninstall_cmd('-y', 'gcc@10.1.0%gcc@4.5.0') + + monkeypatch.setattr(spack.concretize.Concretizer, + 'check_for_compiler_existence', False) + spack.config.set('config:install_missing_compilers', True) + assert CompilerSpec('gcc@10.1.0') not in compilers.all_compiler_specs() + + # Configure the mirror where we put that buildcache w/ the compiler + mirror_cmd('add', 'test-mirror', mirror_url) + + install_cmd('--no-check-signature', 'a%gcc@10.1.0') + + # Put spec built with installed compiler in the buildcache + buildcache_cmd('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, + 'a%gcc@10.1.0') + + # Now uninstall the spec + uninstall_cmd('-y', 'a%gcc@10.1.0') + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + definitions: + - bootstrap: + - gcc@10.1.0%gcc@4.5.0 + specs: + - a%gcc@10.1.0 + mirrors: + atestm: {0} + gitlab-ci: + bootstrap: + - name: bootstrap + compiler-agnostic: true + mappings: + - match: + - arch=test-debian6-x86_64 + runner-attributes: + tags: + - donotcare + - match: + - arch=test-debian6-core2 + runner-attributes: + tags: + - meh +""".format(mirror_url)) + + # Without this monkeypatch, pipeline generation process would think that + # nothing in the environment needs rebuilding. With the monkeypatch, the + # process sees the compiler as needing a rebuild, which should then result + # in the specs built with that compiler needing a rebuild too. + def fake_get_mirrors_for_spec(spec=None, full_hash_match=False, + mirrors_to_check=None, index_only=False): + if spec.name == 'gcc': + return [] + else: + return [{ + 'spec': spec, + 'mirror_url': mirror_url, + }] + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as of: + yaml_contents = of.read() + original_yaml_contents = syaml.load(yaml_contents) + + # without the monkeypatch, everything appears up to date and no + # rebuild jobs are generated. + assert(original_yaml_contents) + assert('no-specs-to-rebuild' in original_yaml_contents) + + monkeypatch.setattr(spack.binary_distribution, + 'get_mirrors_for_spec', + fake_get_mirrors_for_spec) + + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as of: + yaml_contents = of.read() + new_yaml_contents = syaml.load(yaml_contents) + + assert(new_yaml_contents) + + # This 'needs' graph reflects that even though specs 'a' and 'b' do + # not otherwise need to be rebuilt (thanks to DAG pruning), they + # both end up in the generated pipeline because the compiler they + # depend on is bootstrapped, and *does* need to be rebuilt. + needs_graph = { + '(bootstrap) gcc': [], + '(specs) b': [ + '(bootstrap) gcc', + ], + '(specs) a': [ + '(bootstrap) gcc', + '(specs) b', + ], + } + + _validate_needs_graph(new_yaml_contents, needs_graph, False) + + +def test_ci_subcommands_without_mirror(tmpdir, mutable_mock_env_path, + env_deactivate, mock_packages, + install_mockery): + """Make sure we catch if there is not a mirror and report an error""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + gitlab-ci: + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + # Check the 'generate' subcommand + output = ci_cmd('generate', '--output-file', outputfile, + output=str, fail_on_error=False) + ex = 'spack ci generate requires an env containing a mirror' + assert(ex in output) + + # Also check the 'rebuild-index' subcommand + output = ci_cmd('rebuild-index', output=str, fail_on_error=False) + ex = 'spack ci rebuild-index requires an env containing a mirror' + assert(ex in output) + + +def test_ensure_only_one_temporary_storage(): + """Make sure 'gitlab-ci' section of env does not allow specification of + both 'enable-artifacts-buildcache' and 'temporary-storage-url-prefix'.""" + gitlab_ci_template = """ + gitlab-ci: + {0} + mappings: + - match: + - notcheckedhere + runner-attributes: + tags: + - donotcare +""" + + enable_artifacts = 'enable-artifacts-buildcache: True' + temp_storage = 'temporary-storage-url-prefix: file:///temp/mirror' + specify_both = """{0} + {1} +""".format(enable_artifacts, temp_storage) + specify_neither = '' + + # User can specify "enable-artifacts-buildcache" (boolean) + yaml_obj = syaml.load(gitlab_ci_template.format(enable_artifacts)) + validate(yaml_obj, gitlab_ci_schema) + + # User can also specify "temporary-storage-url-prefix" (string) + yaml_obj = syaml.load(gitlab_ci_template.format(temp_storage)) + validate(yaml_obj, gitlab_ci_schema) + + # However, specifying both should fail to validate + yaml_obj = syaml.load(gitlab_ci_template.format(specify_both)) + with pytest.raises(ValidationError): + validate(yaml_obj, gitlab_ci_schema) + + # Specifying neither should be fine too, as neither of these properties + # should be required + yaml_obj = syaml.load(gitlab_ci_template.format(specify_neither)) + validate(yaml_obj, gitlab_ci_schema) + + +def test_ci_generate_temp_storage_url(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages, monkeypatch): + """Verify correct behavior when using temporary-storage-url-prefix""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + temporary-storage-url-prefix: file:///work/temp/mirror + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + monkeypatch.setattr( + ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as of: + pipeline_doc = syaml.load(of.read()) + + print(pipeline_doc) + + assert('cleanup' in pipeline_doc) + cleanup_job = pipeline_doc['cleanup'] + + assert('script' in cleanup_job) + cleanup_task = cleanup_job['script'][0] + + assert(cleanup_task.startswith('spack -d mirror destroy')) + + assert('stages' in pipeline_doc) + stages = pipeline_doc['stages'] + + # Cleanup job should be 2nd to last, just before rebuild-index + assert('stage' in cleanup_job) + assert(cleanup_job['stage'] == stages[-2]) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index dcaf0c916c..daf561eea4 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 409dea7c51..2427511538 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -118,48 +118,21 @@ def test_rst_with_header(tmpdir): def test_rst_update(tmpdir): update_file = tmpdir.join('output') - # not yet created when commands is run commands('--update', str(update_file)) assert update_file.exists() - with update_file.open() as f: - assert f.read() - - # created but older than commands - with update_file.open('w') as f: - f.write('empty\n') - update_file.setmtime(0) - commands('--update', str(update_file)) - assert update_file.exists() - with update_file.open() as f: - assert f.read() != 'empty\n' - - # newer than commands - with update_file.open('w') as f: - f.write('empty\n') - commands('--update', str(update_file)) - assert update_file.exists() - with update_file.open() as f: - assert f.read() == 'empty\n' def test_update_with_header(tmpdir): update_file = tmpdir.join('output') - # not yet created when commands is run commands('--update', str(update_file)) assert update_file.exists() - with update_file.open() as f: - assert f.read() fake_header = 'this is a header!\n\n' filename = tmpdir.join('header.txt') with filename.open('w') as f: f.write(fake_header) - # created, newer than commands, but older than header - commands('--update', str(update_file), '--header', str(filename)) - - # newer than commands and header commands('--update', str(update_file), '--header', str(filename)) @@ -229,7 +202,6 @@ def test_update_completion_arg(tmpdir, monkeypatch): old_file = old.read() with open(mock_args['bash']['update'], 'w') as mock: mock.write(old_file.replace("--update-completion", "")) - mock_bashfile.setmtime(0) # ensure mtime triggers update monkeypatch.setattr( spack.cmd.commands, 'update_completion_args', mock_args) diff --git a/lib/spack/spack/test/cmd/common/__init__.py b/lib/spack/spack/test/cmd/common/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/cmd/common/__init__.py +++ b/lib/spack/spack/test/cmd/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/common/arguments.py b/lib/spack/spack/test/cmd/common/arguments.py index 76e39bcad0..dd92ec92ea 100644 --- a/lib/spack/spack/test/cmd/common/arguments.py +++ b/lib/spack/spack/test/cmd/common/arguments.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index c5c354221e..109695edd8 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 7830f1c4e2..d80932a0f7 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/create.py b/lib/spack/spack/test/cmd/create.py index 4262744317..39b076df08 100644 --- a/lib/spack/spack/test/cmd/create.py +++ b/lib/spack/spack/test/cmd/create.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/debug.py b/lib/spack/spack/test/cmd/debug.py index 2898ad670e..72190431de 100644 --- a/lib/spack/spack/test/cmd/debug.py +++ b/lib/spack/spack/test/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,6 +11,7 @@ import os import os.path import spack.architecture as architecture +import spack.config from spack.main import SpackCommand, get_version from spack.util.executable import which @@ -53,3 +54,4 @@ def test_report(): assert get_version() in out assert platform.python_version() in out assert str(arch) in out + assert spack.config.get('config:concretizer') in out diff --git a/lib/spack/spack/test/cmd/dependencies.py b/lib/spack/spack/test/cmd/dependencies.py index 05d0556936..4c0de355ef 100644 --- a/lib/spack/spack/test/cmd/dependencies.py +++ b/lib/spack/spack/test/cmd/dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -51,8 +51,11 @@ def test_direct_installed_dependencies(mock_packages, database): with color_when(False): out = dependencies('--installed', 'mpileaks^mpich') - lines = [l for l in out.strip().split('\n') if not l.startswith('--')] - hashes = set([re.split(r'\s+', l)[0] for l in lines]) + lines = [ + line for line in out.strip().split('\n') + if not line.startswith('--') + ] + hashes = set([re.split(r'\s+', line)[0] for line in lines]) expected = set([spack.store.db.query_one(s).dag_hash(7) for s in ['mpich', 'callpath^mpich']]) @@ -65,8 +68,11 @@ def test_transitive_installed_dependencies(mock_packages, database): with color_when(False): out = dependencies('--installed', '--transitive', 'mpileaks^zmpi') - lines = [l for l in out.strip().split('\n') if not l.startswith('--')] - hashes = set([re.split(r'\s+', l)[0] for l in lines]) + lines = [ + line for line in out.strip().split('\n') + if not line.startswith('--') + ] + hashes = set([re.split(r'\s+', line)[0] for line in lines]) expected = set([spack.store.db.query_one(s).dag_hash(7) for s in ['zmpi', 'callpath^zmpi', 'fake', diff --git a/lib/spack/spack/test/cmd/dependents.py b/lib/spack/spack/test/cmd/dependents.py index bd87f92a65..376332cf88 100644 --- a/lib/spack/spack/test/cmd/dependents.py +++ b/lib/spack/spack/test/cmd/dependents.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/deprecate.py b/lib/spack/spack/test/cmd/deprecate.py index 7b38c177f5..370d267753 100644 --- a/lib/spack/spack/test/cmd/deprecate.py +++ b/lib/spack/spack/test/cmd/deprecate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index 3067f5a0dd..273b311e9c 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/develop.py b/lib/spack/spack/test/cmd/develop.py index c539cd91c0..b6d8b11105 100644 --- a/lib/spack/spack/test/cmd/develop.py +++ b/lib/spack/spack/test/cmd/develop.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index c2d75d9d1f..f83fcc12dd 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -330,7 +330,7 @@ def test_env_status_broken_view( # switch to a new repo that doesn't include the installed package # test that Spack detects the missing package and warns the user new_repo = MockPackageMultiRepo() - with spack.repo.swap(new_repo): + with spack.repo.use_repositories(new_repo): output = env('status') assert 'In environment test' in output assert 'Environment test includes out of date' in output @@ -351,7 +351,7 @@ def test_env_activate_broken_view( # switch to a new repo that doesn't include the installed package # test that Spack detects the missing package and fails gracefully new_repo = MockPackageMultiRepo() - with spack.repo.swap(new_repo): + with spack.repo.use_repositories(new_repo): with pytest.raises(SpackCommandError): env('activate', '--sh', 'test') @@ -929,7 +929,7 @@ def test_read_old_lock_and_write_new(tmpdir): y = mock_repo.add_package('y', [], []) mock_repo.add_package('x', [y], [build_only]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): x = Spec('x') x.concretize() @@ -960,7 +960,7 @@ def test_read_old_lock_creates_backup(tmpdir): mock_repo = MockPackageMultiRepo() y = mock_repo.add_package('y', [], []) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): y = Spec('y') y.concretize() @@ -997,7 +997,7 @@ def test_indirect_build_dep(): pass setattr(mock_repo, 'dump_provenance', noop) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): x_spec = Spec('x') x_concretized = x_spec.concretized() @@ -1038,7 +1038,7 @@ def test_store_different_build_deps(): pass setattr(mock_repo, 'dump_provenance', noop) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): y_spec = Spec('y ^z@3') y_concretized = y_spec.concretized() diff --git a/lib/spack/spack/test/cmd/extensions.py b/lib/spack/spack/test/cmd/extensions.py index 7fc56593eb..d2093fa887 100644 --- a/lib/spack/spack/test/cmd/extensions.py +++ b/lib/spack/spack/test/cmd/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -27,7 +27,7 @@ def python_database(mock_packages, mutable_database): @pytest.mark.db -def test_extensions(mock_packages, python_database, capsys): +def test_extensions(mock_packages, python_database, config, capsys): ext2 = Spec("py-extension2").concretized() def check_output(ni, na): diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py index 53e1f9eaf2..3249a92894 100644 --- a/lib/spack/spack/test/cmd/external.py +++ b/lib/spack/spack/test/cmd/external.py @@ -1,7 +1,9 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + import os import os.path @@ -242,3 +244,26 @@ def test_new_entries_are_reported_correctly( # has been found output = external('find', 'gcc') assert 'No new external packages detected' in output + + +@pytest.mark.parametrize('command_args', [ + ('-t', 'build-tools'), + ('-t', 'build-tools', 'cmake'), +]) +def test_use_tags_for_detection( + command_args, mock_executable, mutable_config, monkeypatch +): + # Prepare an environment to detect a fake cmake + cmake_exe = mock_executable('cmake', output="echo cmake version 3.19.1") + prefix = os.path.dirname(cmake_exe) + monkeypatch.setenv('PATH', prefix) + + openssl_exe = mock_executable('openssl', output="OpenSSL 2.8.3") + prefix = os.path.dirname(openssl_exe) + monkeypatch.setenv('PATH', prefix) + + # Test that we detect specs + output = external('find', *command_args) + assert 'The following specs have been' in output + assert 'cmake' in output + assert 'openssl' not in output diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 9d2203aaaf..762578a028 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -89,7 +89,7 @@ def test_query_arguments(): @pytest.mark.usefixtures('database', 'mock_display') def test_tag1(parser, specs): - args = parser.parse_args(['--tags', 'tag1']) + args = parser.parse_args(['--tag', 'tag1']) spack.cmd.find.find(parser, args) assert len(specs) == 2 @@ -100,7 +100,7 @@ def test_tag1(parser, specs): @pytest.mark.db @pytest.mark.usefixtures('database', 'mock_display') def test_tag2(parser, specs): - args = parser.parse_args(['--tags', 'tag2']) + args = parser.parse_args(['--tag', 'tag2']) spack.cmd.find.find(parser, args) assert len(specs) == 1 @@ -110,7 +110,7 @@ def test_tag2(parser, specs): @pytest.mark.db @pytest.mark.usefixtures('database', 'mock_display') def test_tag2_tag3(parser, specs): - args = parser.parse_args(['--tags', 'tag2', '--tags', 'tag3']) + args = parser.parse_args(['--tag', 'tag2', '--tag', 'tag3']) spack.cmd.find.find(parser, args) assert len(specs) == 0 diff --git a/lib/spack/spack/test/cmd/flake8.py b/lib/spack/spack/test/cmd/flake8.py index d4bfc5618d..c1de26a278 100644 --- a/lib/spack/spack/test/cmd/flake8.py +++ b/lib/spack/spack/test/cmd/flake8.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,12 +11,12 @@ import sys from llnl.util.filesystem import FileFilter import spack.paths -from spack.cmd.flake8 import flake8, setup_parser, changed_files +from spack.cmd.style import style, setup_parser, changed_files from spack.repo import Repo from spack.util.executable import which -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def parser(): """Returns the parser for the ``flake8`` command""" parser = argparse.ArgumentParser() @@ -24,7 +24,7 @@ def parser(): return parser -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def flake8_package(): """Flake8 only checks files that have been modified. This fixture makes a small change to the ``flake8`` @@ -32,7 +32,7 @@ def flake8_package(): change on cleanup. """ repo = Repo(spack.paths.mock_packages_path) - filename = repo.filename_for_package_name('flake8') + filename = repo.filename_for_package_name("flake8") package = FileFilter(filename) # Make the change @@ -60,16 +60,18 @@ def test_changed_files(parser, flake8_package): # As of flake8 3.0.0, Python 2.6 and 3.3 are no longer supported # http://flake8.pycqa.org/en/latest/release-notes/3.0.0.html @pytest.mark.skipif( - sys.version_info[:2] <= (2, 6) or - (3, 0) <= sys.version_info[:2] <= (3, 3), - reason='flake8 no longer supports Python 2.6 or 3.3 and older') -@pytest.mark.skipif(not which('flake8'), reason='flake8 is not installed.') + sys.version_info[:2] <= (2, 6) or (3, 0) <= sys.version_info[:2] <= (3, 3), + reason="flake8 no longer supports Python 2.6 or 3.3 and older", +) +@pytest.mark.skipif(not which("flake8"), reason="flake8 is not installed.") def test_flake8(parser, flake8_package): # Only test the flake8_package that we modified # Otherwise, the unit tests would fail every time # the flake8 tests fail - args = parser.parse_args([flake8_package]) - flake8(parser, args) + args = parser.parse_args(["--no-mypy", flake8_package]) + style(parser, args) # Get even more coverage - args = parser.parse_args(['--output', '--root-relative', flake8_package]) - flake8(parser, args) + args = parser.parse_args( + ["--no-mypy", "--output", "--root-relative", flake8_package] + ) + style(parser, args) diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py index 22c85a1d78..70df8af687 100644 --- a/lib/spack/spack/test/cmd/gc.py +++ b/lib/spack/spack/test/cmd/gc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 751790182b..099d53e039 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/graph.py b/lib/spack/spack/test/cmd/graph.py index c402f35a9f..8d6da1049f 100644 --- a/lib/spack/spack/test/cmd/graph.py +++ b/lib/spack/spack/test/cmd/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/help.py b/lib/spack/spack/test/cmd/help.py index 00edfa187a..8e0f6d02c9 100644 --- a/lib/spack/spack/test/cmd/help.py +++ b/lib/spack/spack/test/cmd/help.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/info.py b/lib/spack/spack/test/cmd/info.py index b57ef3257b..0ff6857617 100644 --- a/lib/spack/spack/test/cmd/info.py +++ b/lib/spack/spack/test/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/init_py_functions.py b/lib/spack/spack/test/cmd/init_py_functions.py index 2e63106283..56b0ef7171 100644 --- a/lib/spack/spack/test/cmd/init_py_functions.py +++ b/lib/spack/spack/test/cmd/init_py_functions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 8e87f97c98..dfc48ece07 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/is_git_repo.py b/lib/spack/spack/test/cmd/is_git_repo.py index 724925e5e4..109f5030c7 100644 --- a/lib/spack/spack/test/cmd/is_git_repo.py +++ b/lib/spack/spack/test/cmd/is_git_repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index 614d093c5f..6063a8d9a8 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import re from llnl.util.filesystem import touch, mkdirp +import spack.cmd.license import spack.paths from spack.main import SpackCommand @@ -31,7 +32,7 @@ def test_verify(tmpdir): lgpl_header = source_dir.join('lgpl_header.py') with lgpl_header.open('w') as f: f.write("""\ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: LGPL-2.1-only @@ -48,13 +49,13 @@ def test_verify(tmpdir): correct_header = source_dir.join('correct_header.py') with correct_header.open('w') as f: f.write("""\ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """) - out = license('verify', '--root', str(tmpdir), fail_on_error=False) + out = license('--root', str(tmpdir), 'verify', fail_on_error=False) assert str(no_header) in out assert str(lgpl_header) in out @@ -66,3 +67,28 @@ def test_verify(tmpdir): assert re.search(r'files with old license header:\s*1', out) assert license.returncode == 1 + + +def test_update_copyright_year(tmpdir): + source_dir = tmpdir.join('lib', 'spack', 'spack') + mkdirp(str(source_dir)) + + years = list(range(2018, 2021)) + + for year in years: + outdated = source_dir.join('header_%d.py' % year) + with outdated.open('w') as f: + f.write("""\ +# Copyright 2013-%d Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +""" % year) + + license('--root', str(tmpdir), 'update-copyright-year') + + for year in years: + outdated = source_dir.join('header_%d.py' % year) + first_line = outdated.open().read().split("\n")[0] + assert str(year) not in first_line + assert spack.cmd.license.strict_date in first_line diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index 17f5a1b493..f99b7420a4 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -33,15 +33,15 @@ def test_list_search_description(): def test_list_tags(): - output = list('--tags', 'proxy-app') + output = list('--tag', 'proxy-app') assert 'cloverleaf3d' in output assert 'hdf5' not in output - output = list('--tags', 'hpc') + output = list('--tag', 'hpc') assert 'nek5000' in output assert 'mfem' in output - output = list('--tags', 'HPC') + output = list('--tag', 'HPC') assert 'nek5000' in output assert 'mfem' in output diff --git a/lib/spack/spack/test/cmd/load.py b/lib/spack/spack/test/cmd/load.py index b1697bbfc4..620280e016 100644 --- a/lib/spack/spack/test/cmd/load.py +++ b/lib/spack/spack/test/cmd/load.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index 3977cba3e5..4141c4fbbe 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/maintainers.py b/lib/spack/spack/test/cmd/maintainers.py index 4c566c7513..f6520b4a1b 100644 --- a/lib/spack/spack/test/cmd/maintainers.py +++ b/lib/spack/spack/test/cmd/maintainers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/mark.py b/lib/spack/spack/test/cmd/mark.py index 5b488bbf3e..450bc1f30e 100644 --- a/lib/spack/spack/test/cmd/mark.py +++ b/lib/spack/spack/test/cmd/mark.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py index 0957624cba..b72487e5ab 100644 --- a/lib/spack/spack/test/cmd/mirror.py +++ b/lib/spack/spack/test/cmd/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -14,6 +14,9 @@ mirror = SpackCommand('mirror') env = SpackCommand('env') add = SpackCommand('add') concretize = SpackCommand('concretize') +install = SpackCommand('install') +buildcache = SpackCommand('buildcache') +uninstall = SpackCommand('uninstall') @pytest.fixture @@ -183,3 +186,39 @@ def test_mirror_name_collision(tmp_scope): with pytest.raises(SpackCommandError): mirror('add', '--scope', tmp_scope, 'first', '1') + + +def test_mirror_destroy(install_mockery_mutable_config, + mock_packages, mock_fetch, mock_archive, + mutable_config, monkeypatch, tmpdir): + # Create a temp mirror directory for buildcache usage + mirror_dir = tmpdir.join('mirror_dir') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + mirror('add', 'atest', mirror_url) + + spec_name = 'libdwarf' + + # Put a binary package in a buildcache + install('--no-cache', spec_name) + buildcache('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, spec_name) + + contents = os.listdir(mirror_dir.strpath) + assert('build_cache' in contents) + + # Destroy mirror by name + mirror('destroy', '-m', 'atest') + + assert(not os.path.exists(mirror_dir.strpath)) + + buildcache('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, spec_name) + + contents = os.listdir(mirror_dir.strpath) + assert('build_cache' in contents) + + # Destroy mirror by url + mirror('destroy', '--mirror-url', mirror_url) + + assert(not os.path.exists(mirror_dir.strpath)) + + uninstall('-y', spec_name) + mirror('remove', 'atest') diff --git a/lib/spack/spack/test/cmd/module.py b/lib/spack/spack/test/cmd/module.py index c3fe279306..9acb21fdef 100644 --- a/lib/spack/spack/test/cmd/module.py +++ b/lib/spack/spack/test/cmd/module.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,9 +8,10 @@ import re import pytest +import spack.config import spack.main import spack.modules -from spack.test.conftest import use_store, use_configuration, use_repo +import spack.store module = spack.main.SpackCommand('module') @@ -18,12 +19,13 @@ module = spack.main.SpackCommand('module') #: make sure module files are generated for all the tests here @pytest.fixture(scope='module', autouse=True) def ensure_module_files_are_there( - mock_repo_path, mock_store, mock_configuration): + mock_repo_path, mock_store, mock_configuration_scopes +): """Generate module files for module tests.""" module = spack.main.SpackCommand('module') - with use_store(mock_store): - with use_configuration(mock_configuration): - with use_repo(mock_repo_path): + with spack.store.use_store(str(mock_store)): + with spack.config.use_configuration(*mock_configuration_scopes): + with spack.repo.use_repositories(mock_repo_path): module('tcl', 'refresh', '-y') diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index ca9e3a1a3e..4f4293d004 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -82,7 +82,7 @@ def mock_pkg_git_repo(tmpdir_factory): git('-c', 'commit.gpgsign=false', 'commit', '-m', 'change pkg-b, remove pkg-c, add pkg-d') - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): yield mock_repo_packages diff --git a/lib/spack/spack/test/cmd/print_shell_vars.py b/lib/spack/spack/test/cmd/print_shell_vars.py index 5f4f8164eb..3c56ee6e49 100644 --- a/lib/spack/spack/test/cmd/print_shell_vars.py +++ b/lib/spack/spack/test/cmd/print_shell_vars.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/providers.py b/lib/spack/spack/test/cmd/providers.py index dd0c8a9eb7..daab694032 100644 --- a/lib/spack/spack/test/cmd/providers.py +++ b/lib/spack/spack/test/cmd/providers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/python.py b/lib/spack/spack/test/cmd/python.py index b1c9d3db00..360d947a07 100644 --- a/lib/spack/spack/test/cmd/python.py +++ b/lib/spack/spack/test/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/reindex.py b/lib/spack/spack/test/cmd/reindex.py index feb5545ae6..77c672251e 100644 --- a/lib/spack/spack/test/cmd/reindex.py +++ b/lib/spack/spack/test/cmd/reindex.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/repo.py b/lib/spack/spack/test/cmd/repo.py index 82fe872710..541ffeddf0 100644 --- a/lib/spack/spack/test/cmd/repo.py +++ b/lib/spack/spack/test/cmd/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/resource.py b/lib/spack/spack/test/cmd/resource.py index 4fc5fd5c50..ba1bf48daf 100644 --- a/lib/spack/spack/test/cmd/resource.py +++ b/lib/spack/spack/test/cmd/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/spec.py b/lib/spack/spack/test/cmd/spec.py index 3c50a66ce7..7d5cf12e75 100644 --- a/lib/spack/spack/test/cmd/spec.py +++ b/lib/spack/spack/test/cmd/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 4163853274..8b1a2d053f 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/undevelop.py b/lib/spack/spack/test/cmd/undevelop.py index d44330fe76..0c8f2df978 100644 --- a/lib/spack/spack/test/cmd/undevelop.py +++ b/lib/spack/spack/test/cmd/undevelop.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -12,7 +12,7 @@ env = SpackCommand('env') concretize = SpackCommand('concretize') -def test_undevelop(tmpdir, mock_packages, mutable_mock_env_path): +def test_undevelop(tmpdir, config, mock_packages, mutable_mock_env_path): # setup environment envdir = tmpdir.mkdir('env') with envdir.as_cwd(): @@ -39,7 +39,7 @@ env: assert not after.satisfies('dev_path=*') -def test_undevelop_nonexistent(tmpdir, mock_packages, mutable_mock_env_path): +def test_undevelop_nonexistent(tmpdir, config, mock_packages, mutable_mock_env_path): # setup environment envdir = tmpdir.mkdir('env') with envdir.as_cwd(): diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index a5adcdc74b..9a5d1587f0 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/unit_test.py b/lib/spack/spack/test/cmd/unit_test.py index c5b8eb765e..1a273ff244 100644 --- a/lib/spack/spack/test/cmd/unit_test.py +++ b/lib/spack/spack/test/cmd/unit_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index 2350fc5fdb..f2466dcdf2 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/verify.py b/lib/spack/spack/test/cmd/verify.py index dda992fe49..de7544d9f1 100644 --- a/lib/spack/spack/test/cmd/verify.py +++ b/lib/spack/spack/test/cmd/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/versions.py b/lib/spack/spack/test/cmd/versions.py index 1e2bcbedb3..e4bf711a8a 100644 --- a/lib/spack/spack/test/cmd/versions.py +++ b/lib/spack/spack/test/cmd/versions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,10 +10,18 @@ from spack.main import SpackCommand versions = SpackCommand('versions') +def test_safe_only_versions(): + """Only test the safe versions of a package. + (Using the deprecated command line argument) + """ + + versions('--safe-only', 'zlib') + + def test_safe_versions(): """Only test the safe versions of a package.""" - versions('--safe-only', 'zlib') + versions('--safe', 'zlib') @pytest.mark.network @@ -24,6 +32,21 @@ def test_remote_versions(): @pytest.mark.network +def test_remote_versions_only(): + """Test a package for which remote versions should be available.""" + + versions('--remote', 'zlib') + + +@pytest.mark.network +@pytest.mark.usefixtures('mock_packages') +def test_new_versions_only(): + """Test a package for which new versions should be available.""" + + versions('--new', 'brillig') + + +@pytest.mark.network def test_no_versions(): """Test a package for which no remote versions are available.""" diff --git a/lib/spack/spack/test/cmd/view.py b/lib/spack/spack/test/cmd/view.py index 4ff1592035..b5a503d278 100644 --- a/lib/spack/spack/test/cmd/view.py +++ b/lib/spack/spack/test/cmd/view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd_extensions.py b/lib/spack/spack/test/cmd_extensions.py index cb16609278..f611bd5c8f 100644 --- a/lib/spack/spack/test/cmd_extensions.py +++ b/lib/spack/spack/test/cmd_extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/compilers/__init__.py b/lib/spack/spack/test/compilers/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/compilers/__init__.py +++ b/lib/spack/spack/test/compilers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 78d8484b3b..005000fdcb 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -453,6 +453,8 @@ def test_aocc_flags(): '-Os', '-Oz', '-Og', '-O', '-O4'], 'aocc@2.2.0') + + supported_flag_test("stdcxx_libs", ("-lstdc++",), "aocc@2.2.0") supported_flag_test("openmp_flag", "-fopenmp", "aocc@2.2.0") supported_flag_test("cxx11_flag", "-std=c++11", "aocc@2.2.0") supported_flag_test("cxx14_flag", "-std=c++14", "aocc@2.2.0") @@ -471,14 +473,16 @@ def test_fj_flags(): supported_flag_test("cxx98_flag", "-std=c++98", "fj@4.0.0") supported_flag_test("cxx11_flag", "-std=c++11", "fj@4.0.0") supported_flag_test("cxx14_flag", "-std=c++14", "fj@4.0.0") + supported_flag_test("cxx17_flag", "-std=c++17", "fj@4.0.0") supported_flag_test("c99_flag", "-std=c99", "fj@4.0.0") supported_flag_test("c11_flag", "-std=c11", "fj@4.0.0") supported_flag_test("cc_pic_flag", "-KPIC", "fj@4.0.0") supported_flag_test("cxx_pic_flag", "-KPIC", "fj@4.0.0") supported_flag_test("f77_pic_flag", "-KPIC", "fj@4.0.0") supported_flag_test("fc_pic_flag", "-KPIC", "fj@4.0.0") - supported_flag_test("opt_flags", ['-O', '-O0', '-O1', '-O2', '-O3', '-O4'], + supported_flag_test("opt_flags", ['-O0', '-O1', '-O2', '-O3', '-Ofast'], 'fj@4.0.0') + supported_flag_test("debug_flags", "-g", "fj@4.0.0") def test_gcc_flags(): diff --git a/lib/spack/spack/test/compilers/detection.py b/lib/spack/spack/test/compilers/detection.py index 47e078f242..e68c54c6ff 100644 --- a/lib/spack/spack/test/compilers/detection.py +++ b/lib/spack/spack/test/compilers/detection.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -96,7 +96,11 @@ def test_apple_clang_version_detection( ('clang version 8.0.0-3 (tags/RELEASE_800/final)\n' 'Target: aarch64-unknown-linux-gnu\n' 'Thread model: posix\n' - 'InstalledDir: /usr/bin\n', '8.0.0') + 'InstalledDir: /usr/bin\n', '8.0.0'), + ('clang version 11.0.0\n' + 'Target: aarch64-unknown-linux-gnu\n' + 'Thread model: posix\n' + 'InstalledDir: /usr/bin\n', '11.0.0') ]) def test_clang_version_detection(version_str, expected_version): version = spack.compilers.clang.Clang.extract_version_from_output( @@ -317,6 +321,11 @@ def test_cray_frontend_compiler_detection( @pytest.mark.parametrize('version_str,expected_version', [ # This applies to C,C++ and FORTRAN compiler + ('AMD clang version 11.0.0 (CLANG: AOCC_2.3.0-Build#85 2020_11_10)' + '(based on LLVM Mirror.Version.11.0.0)\n' + 'Target: x86_64-unknown-linux-gnu\n' + 'Thread model: posix\n', '2.3.0' + ), ('AMD clang version 10.0.0 (CLANG: AOCC_2.2.0-Build#93 2020_06_25)' '(based on LLVM Mirror.Version.10.0.0)\n' 'Target: x86_64-unknown-linux-gnu\n' diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index dae162a12f..283b9052cc 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -304,7 +304,7 @@ class TestConcretize(object): barpkg = mock_repo.add_package('barpkg', [bazpkg], [default_dep]) mock_repo.add_package('foopkg', [barpkg], [default_dep]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = Spec('foopkg %gcc@4.5.0 os=CNL target=nocona' + ' ^barpkg os=SuSE11 ^bazpkg os=be') spec.concretize() @@ -688,13 +688,14 @@ class TestConcretize(object): with pytest.raises(spack.error.SpackError): Spec(spec).concretized() + # Include targets to prevent regression on 20537 @pytest.mark.parametrize('spec, best_achievable', [ - ('mpileaks%gcc@4.4.7', 'core2'), - ('mpileaks%gcc@4.8', 'haswell'), - ('mpileaks%gcc@5.3.0', 'broadwell'), - ('mpileaks%apple-clang@5.1.0', 'x86_64') + ('mpileaks%gcc@4.4.7 target=x86_64:', 'core2'), + ('mpileaks%gcc@4.8 target=x86_64:', 'haswell'), + ('mpileaks%gcc@5.3.0 target=x86_64:', 'broadwell'), + ('mpileaks%apple-clang@5.1.0 target=x86_64:', 'x86_64') ]) - @pytest.mark.regression('13361') + @pytest.mark.regression('13361', '20537') def test_adjusting_default_target_based_on_compiler( self, spec, best_achievable, current_host, mock_targets ): diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py index 9abac7221d..7abbaa6e92 100644 --- a/lib/spack/spack/test/concretize_preferences.py +++ b/lib/spack/spack/test/concretize_preferences.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f1081be737..b5c3dd657b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/config_values.py b/lib/spack/spack/test/config_values.py index a0bf7e52f8..367138f098 100644 --- a/lib/spack/spack/test/config_values.py +++ b/lib/spack/spack/test/config_values.py @@ -1,37 +1,31 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest import spack.spec +import spack.store -def test_set_install_hash_length(mock_packages, mutable_config, monkeypatch, - tmpdir): - # spack.store.layout caches initial config values, so we monkeypatch - mutable_config.set('config:install_hash_length', 5) +@pytest.mark.parametrize('hash_length', [1, 2, 3, 4, 5, 9]) +@pytest.mark.use_fixtures('mock_packages') +def test_set_install_hash_length(hash_length, mutable_config, tmpdir): + mutable_config.set('config:install_hash_length', hash_length) mutable_config.set('config:install_tree', {'root': str(tmpdir)}) - monkeypatch.setattr(spack.store, 'store', spack.store._store()) - - spec = spack.spec.Spec('libelf').concretized() - prefix = spec.prefix - hash = prefix.rsplit('-')[-1] - - assert len(hash) == 5 - - mutable_config.set('config:install_hash_length', 9) - monkeypatch.setattr(spack.store, 'store', spack.store._store()) - - spec = spack.spec.Spec('libelf').concretized() - prefix = spec.prefix - hash = prefix.rsplit('-')[-1] - - assert len(hash) == 9 - - -def test_set_install_hash_length_upper_case(mock_packages, mutable_config, - monkeypatch, tmpdir): - # spack.store.layout caches initial config values, so we monkeypatch + # The call below is to reinitialize the directory layout associated + # with the store according to the configuration changes above (i.e. + # with the shortened hash) + store = spack.store._store() + with spack.store.use_store(store): + spec = spack.spec.Spec('libelf').concretized() + prefix = spec.prefix + hash_str = prefix.rsplit('-')[-1] + assert len(hash_str) == hash_length + + +@pytest.mark.use_fixtures('mock_packages') +def test_set_install_hash_length_upper_case(mutable_config, tmpdir): mutable_config.set('config:install_hash_length', 5) mutable_config.set( 'config:install_tree', @@ -42,10 +36,12 @@ def test_set_install_hash_length_upper_case(mock_packages, mutable_config, } } ) - monkeypatch.setattr(spack.store, 'store', spack.store._store()) - - spec = spack.spec.Spec('libelf').concretized() - prefix = spec.prefix - hash = prefix.rsplit('-')[-1] - - assert len(hash) == 5 + # The call below is to reinitialize the directory layout associated + # with the store according to the configuration changes above (i.e. + # with the shortened hash and projection) + store = spack.store._store() + with spack.store.use_store(store): + spec = spack.spec.Spec('libelf').concretized() + prefix = spec.prefix + hash_str = prefix.rsplit('-')[-1] + assert len(hash_str) == 5 diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index d4f38f2c1b..6cc3ad16db 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1,10 +1,9 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections -import contextlib import errno import inspect import itertools @@ -35,6 +34,7 @@ import spack.paths import spack.platforms.test import spack.repo import spack.stage +import spack.store import spack.util.executable import spack.util.gpg import spack.subprocess_context @@ -315,27 +315,21 @@ def _skip_if_missing_executables(request): pytest.skip(msg.format(', '.join(missing_execs))) -# FIXME: The lines below should better be added to a fixture with -# FIXME: session-scope. Anyhow doing it is not easy, as it seems -# FIXME: there's some weird interaction with compilers during concretization. -spack.architecture.real_platform = spack.architecture.platform - - +@pytest.fixture(scope='session') def test_platform(): return spack.platforms.test.Test() -spack.architecture.platform = test_platform - - -# FIXME: Since we change the architecture above, we have to (re)initialize -# FIXME: the config singleton. If it gets initialized too early with the -# FIXME: actual architecture, tests will fail. -spack.config.config = spack.config._config() - +@pytest.fixture(autouse=True, scope='session') +def _use_test_platform(test_platform): + # This is the only context manager used at session scope (see note + # below for more insight) since we want to use the test platform as + # a default during tests. + with spack.architecture.use_platform(test_platform): + yield # -# Context managers used by fixtures +# Note on context managers used by fixtures # # Because these context managers modify global state, they should really # ONLY be used persistently (i.e., around yield statements) in @@ -356,45 +350,13 @@ spack.config.config = spack.config._config() # *USE*, or things can get really confusing. # -@contextlib.contextmanager -def use_configuration(config): - """Context manager to swap out the global Spack configuration.""" - saved = spack.config.replace_config(config) - - # Avoid using real spack configuration that has been cached by other - # tests, and avoid polluting the cache with spack test configuration - # (including modified configuration) - saved_compiler_cache = spack.compilers._cache_config_file - spack.compilers._cache_config_file = [] - - yield - - spack.config.replace_config(saved) - spack.compilers._cache_config_file = saved_compiler_cache - - -@contextlib.contextmanager -def use_store(store): - """Context manager to swap out the global Spack store.""" - saved = spack.store.store - spack.store.store = store - yield - spack.store.store = saved - - -@contextlib.contextmanager -def use_repo(repo): - """Context manager to swap out the global Spack repo path.""" - with spack.repo.swap(repo): - yield - # # Test-specific fixtures # @pytest.fixture(scope='session') def mock_repo_path(): - yield spack.repo.RepoPath(spack.paths.mock_packages_path) + yield spack.repo.Repo(spack.paths.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): @@ -411,15 +373,15 @@ def mock_pkg_install(monkeypatch): @pytest.fixture(scope='function') def mock_packages(mock_repo_path, mock_pkg_install): """Use the 'builtin.mock' repository instead of 'builtin'""" - with use_repo(mock_repo_path): - yield mock_repo_path + with spack.repo.use_repositories(mock_repo_path) as mock_repo: + yield mock_repo @pytest.fixture(scope='function') def mutable_mock_repo(mock_repo_path): """Function-scoped mock packages, for tests that need to modify them.""" - mock_repo_path = spack.repo.RepoPath(spack.paths.mock_packages_path) - with use_repo(mock_repo_path): + mock_repo = spack.repo.Repo(spack.paths.mock_packages_path) + with spack.repo.use_repositories(mock_repo) as mock_repo_path: yield mock_repo_path @@ -445,9 +407,7 @@ def default_config(): This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" defaults_path = os.path.join(spack.paths.etc_path, 'spack', 'defaults') - defaults_scope = spack.config.ConfigScope('defaults', defaults_path) - defaults_config = spack.config.Configuration(defaults_scope) - with use_configuration(defaults_config): + with spack.config.use_configuration(defaults_path) as defaults_config: yield defaults_config @@ -469,9 +429,8 @@ def mock_uarch_configuration(mock_uarch_json): with open(mock_uarch_json) as f: return json.load(f) - targets_json = archspec.cpu.schema.LazyDictionary(load_json) - targets = archspec.cpu.microarchitecture.LazyDictionary( - archspec.cpu.microarchitecture._known_microarchitectures) + targets_json = load_json() + targets = archspec.cpu.microarchitecture._known_microarchitectures() yield targets_json, targets @@ -523,7 +482,7 @@ def configuration_dir(tmpdir_factory, linux_os): @pytest.fixture(scope='session') -def mock_configuration(configuration_dir): +def mock_configuration_scopes(configuration_dir): """Create a persistent Configuration object from the configuration_dir.""" defaults = spack.config.InternalConfigScope( '_builtin', spack.config.config_defaults @@ -534,14 +493,14 @@ def mock_configuration(configuration_dir): for name in ['site', 'system', 'user']] test_scopes.append(spack.config.InternalConfigScope('command_line')) - yield spack.config.Configuration(*test_scopes) + yield test_scopes @pytest.fixture(scope='function') -def config(mock_configuration): +def config(mock_configuration_scopes): """This fixture activates/deactivates the mock configuration.""" - with use_configuration(mock_configuration): - yield mock_configuration + with spack.config.use_configuration(*mock_configuration_scopes) as config: + yield config @pytest.fixture(scope='function') @@ -550,11 +509,10 @@ def mutable_config(tmpdir_factory, configuration_dir): mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp') configuration_dir.copy(mutable_dir) - cfg = spack.config.Configuration( - *[spack.config.ConfigScope(name, str(mutable_dir.join(name))) - for name in ['site', 'system', 'user']]) + scopes = [spack.config.ConfigScope(name, str(mutable_dir.join(name))) + for name in ['site', 'system', 'user']] - with use_configuration(cfg): + with spack.config.use_configuration(*scopes) as cfg: yield cfg @@ -562,23 +520,20 @@ def mutable_config(tmpdir_factory, configuration_dir): def mutable_empty_config(tmpdir_factory, configuration_dir): """Empty configuration that can be modified by the tests.""" mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp') + scopes = [spack.config.ConfigScope(name, str(mutable_dir.join(name))) + for name in ['site', 'system', 'user']] - cfg = spack.config.Configuration( - *[spack.config.ConfigScope(name, str(mutable_dir.join(name))) - for name in ['site', 'system', 'user']]) - - with use_configuration(cfg): + with spack.config.use_configuration(*scopes) as cfg: yield cfg @pytest.fixture() def mock_low_high_config(tmpdir): """Mocks two configuration scopes: 'low' and 'high'.""" - config = spack.config.Configuration( - *[spack.config.ConfigScope(name, str(tmpdir.join(name))) - for name in ['low', 'high']]) + scopes = [spack.config.ConfigScope(name, str(tmpdir.join(name))) + for name in ['low', 'high']] - with use_configuration(config): + with spack.config.use_configuration(*scopes) as config: yield config @@ -608,12 +563,10 @@ def _populate(mock_db): pkg = spack.repo.get(s) pkg.do_install(fake=True, explicit=True) - # Transaction used to avoid repeated writes. - with mock_db.write_transaction(): - _install('mpileaks ^mpich') - _install('mpileaks ^mpich2') - _install('mpileaks ^zmpi') - _install('externaltest') + _install('mpileaks ^mpich') + _install('mpileaks ^mpich2') + _install('mpileaks ^zmpi') + _install('externaltest') @pytest.fixture(scope='session') @@ -627,7 +580,7 @@ def _store_dir_and_cache(tmpdir_factory): @pytest.fixture(scope='session') -def mock_store(tmpdir_factory, mock_repo_path, mock_configuration, +def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes, _store_dir_and_cache): """Creates a read-only mock database with some packages installed note that the ref count for dyninst here will be 3, as it's recycled @@ -638,50 +591,19 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration, """ store_path, store_cache = _store_dir_and_cache - store = spack.store.Store(str(store_path)) # If the cache does not exist populate the store and create it if not os.path.exists(str(store_cache.join('.spack-db'))): - with use_configuration(mock_configuration): - with use_store(store): - with use_repo(mock_repo_path): + with spack.config.use_configuration(*mock_configuration_scopes): + with spack.store.use_store(str(store_path)) as store: + with spack.repo.use_repositories(mock_repo_path): _populate(store.db) store_path.copy(store_cache, mode=True, stat=True) # Make the DB filesystem read-only to ensure we can't modify entries store_path.join('.spack-db').chmod(mode=0o555, rec=1) - yield store - - store_path.join('.spack-db').chmod(mode=0o755, rec=1) - - -@pytest.fixture(scope='function') -def mutable_mock_store(tmpdir_factory, mock_repo_path, mock_configuration, - _store_dir_and_cache): - """Creates a read-only mock database with some packages installed note - that the ref count for dyninst here will be 3, as it's recycled - across each install. - - This does not actually activate the store for use by Spack -- see the - ``database`` fixture for that. - - """ - store_path, store_cache = _store_dir_and_cache - store = spack.store.Store(str(store_path)) - - # If the cache does not exist populate the store and create it - if not os.path.exists(str(store_cache.join('.spack-db'))): - with use_configuration(mock_configuration): - with use_store(store): - with use_repo(mock_repo_path): - _populate(store.db) - store_path.copy(store_cache, mode=True, stat=True) - - # Make the DB filesystem read-only to ensure we can't modify entries - store_path.join('.spack-db').chmod(mode=0o555, rec=1) - - yield store + yield store_path store_path.join('.spack-db').chmod(mode=0o755, rec=1) @@ -689,10 +611,9 @@ def mutable_mock_store(tmpdir_factory, mock_repo_path, mock_configuration, @pytest.fixture(scope='function') def database(mock_store, mock_packages, config, monkeypatch): """This activates the mock store, packages, AND config.""" - monkeypatch.setattr(spack.store, 'store', mock_store) - yield mock_store.db - # Force reading the database again between tests - mock_store.db.last_seen_verifier = '' + with spack.store.use_store(str(mock_store)) as store: + yield store.db + store.db.last_seen_verifier = '' @pytest.fixture(scope='function') @@ -763,20 +684,15 @@ def disable_compiler_execution(monkeypatch, request): @pytest.fixture(scope='function') -def install_mockery(tmpdir, config, mock_packages, monkeypatch): +def install_mockery(temporary_store, config, mock_packages): """Hooks a fake install directory, DB, and stage directory into Spack.""" - monkeypatch.setattr( - spack.store, 'store', spack.store.Store(str(tmpdir.join('opt')))) - # We use a fake package, so temporarily disable checksumming with spack.config.override('config:checksum', False): yield - tmpdir.join('opt').remove() - # Also wipe out any cached prefix failure locks (associated with # the session-scoped mock archive). - for pkg_id in list(spack.store.db._prefix_failures.keys()): + for pkg_id in list(temporary_store.db._prefix_failures.keys()): lock = spack.store.db._prefix_failures.pop(pkg_id, None) if lock: try: @@ -786,23 +702,28 @@ def install_mockery(tmpdir, config, mock_packages, monkeypatch): @pytest.fixture(scope='function') +def temporary_store(tmpdir): + """Hooks a temporary empty store for the test function.""" + temporary_store_path = tmpdir.join('opt') + with spack.store.use_store(str(temporary_store_path)) as s: + yield s + temporary_store_path.remove() + + +@pytest.fixture(scope='function') def install_mockery_mutable_config( - tmpdir, mutable_config, mock_packages, monkeypatch): + temporary_store, mutable_config, mock_packages +): """Hooks a fake install directory, DB, and stage directory into Spack. This is specifically for tests which want to use 'install_mockery' but also need to modify configuration (and hence would want to use 'mutable config'): 'install_mockery' does not support this. """ - monkeypatch.setattr( - spack.store, 'store', spack.store.Store(str(tmpdir.join('opt')))) - # We use a fake package, so temporarily disable checksumming with spack.config.override('config:checksum', False): yield - tmpdir.join('opt').remove() - @pytest.fixture() def mock_fetch(mock_archive, monkeypatch): @@ -1250,8 +1171,7 @@ repo: namespace: mock_test_repo """) - repo = spack.repo.RepoPath(str(repodir)) - with spack.repo.swap(repo): + with spack.repo.use_repositories(str(repodir)) as repo: yield repo, repodir shutil.rmtree(str(repodir)) @@ -1268,7 +1188,7 @@ repo: class MockBundle(object): has_code = False name = 'mock-bundle' - versions = {} + versions = {} # type: ignore @pytest.fixture diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py index f9b3b43f83..26ae9f0c68 100644 --- a/lib/spack/spack/test/container/cli.py +++ b/lib/spack/spack/test/container/cli.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py index 70a1ae2547..43e16cd180 100644 --- a/lib/spack/spack/test/container/conftest.py +++ b/lib/spack/spack/test/container/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py index 6392f97db2..43a32b0720 100644 --- a/lib/spack/spack/test/container/docker.py +++ b/lib/spack/spack/test/container/docker.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/images.py b/lib/spack/spack/test/container/images.py index 6cec1cc592..d04bf60b08 100644 --- a/lib/spack/spack/test/container/images.py +++ b/lib/spack/spack/test/container/images.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/schema.py b/lib/spack/spack/test/container/schema.py index e86f9d3fe8..4bb0d574a9 100644 --- a/lib/spack/spack/test/container/schema.py +++ b/lib/spack/spack/test/container/schema.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py index ab342cacec..12245e935e 100644 --- a/lib/spack/spack/test/container/singularity.py +++ b/lib/spack/spack/test/container/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh index 8fa1774b17..6d473ea6d3 100644 --- a/lib/spack/spack/test/data/sourceme_first.sh +++ b/lib/spack/spack/test/data/sourceme_first.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_lmod.sh b/lib/spack/spack/test/data/sourceme_lmod.sh index b71e338ec9..0ce370d09e 100644 --- a/lib/spack/spack/test/data/sourceme_lmod.sh +++ b/lib/spack/spack/test/data/sourceme_lmod.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_parameters.sh b/lib/spack/spack/test/data/sourceme_parameters.sh index d11590d224..205a7006df 100644 --- a/lib/spack/spack/test/data/sourceme_parameters.sh +++ b/lib/spack/spack/test/data/sourceme_parameters.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh index 3f16153627..79af1aea36 100644 --- a/lib/spack/spack/test/data/sourceme_second.sh +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_unicode.sh b/lib/spack/spack/test/data/sourceme_unicode.sh index d819126bb6..b396dc43d7 100644 --- a/lib/spack/spack/test/data/sourceme_unicode.sh +++ b/lib/spack/spack/test/data/sourceme_unicode.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_unset.sh b/lib/spack/spack/test/data/sourceme_unset.sh index 0366833838..2f1658529a 100644 --- a/lib/spack/spack/test/data/sourceme_unset.sh +++ b/lib/spack/spack/test/data/sourceme_unset.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 4682c9850d..8d1a204ba9 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -38,16 +38,6 @@ pytestmark = pytest.mark.db @pytest.fixture() -def test_store(tmpdir): - real_store = spack.store.store - spack.store.store = spack.store.Store(str(tmpdir.join('test_store'))) - - yield - - spack.store.store = real_store - - -@pytest.fixture() def upstream_and_downstream_db(tmpdir_factory, gen_mock_layout): mock_db_root = str(tmpdir_factory.mktemp('mock_db_root')) upstream_write_db = spack.database.Database(mock_db_root) @@ -81,7 +71,7 @@ def test_installed_upstream(upstream_and_downstream_db): y = mock_repo.add_package('y', [z], [default]) mock_repo.add_package('w', [x, y], [default, default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('w') spec.concretize() @@ -122,7 +112,7 @@ def test_removed_upstream_dep(upstream_and_downstream_db): z = mock_repo.add_package('z', [], []) mock_repo.add_package('y', [z], [default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('y') spec.concretize() @@ -155,7 +145,7 @@ def test_add_to_upstream_after_downstream(upstream_and_downstream_db): mock_repo = MockPackageMultiRepo() mock_repo.add_package('x', [], []) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('x') spec.concretize() @@ -180,8 +170,8 @@ def test_add_to_upstream_after_downstream(upstream_and_downstream_db): spack.store.db = orig_db -@pytest.mark.usefixtures('config') -def test_cannot_write_upstream(tmpdir_factory, test_store, gen_mock_layout): +@pytest.mark.usefixtures('config', 'temporary_store') +def test_cannot_write_upstream(tmpdir_factory, gen_mock_layout): roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b']] layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/']] @@ -197,7 +187,7 @@ def test_cannot_write_upstream(tmpdir_factory, test_store, gen_mock_layout): upstream_dbs = spack.store._construct_upstream_dbs_from_install_roots( [roots[1]], _test=True) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('x') spec.concretize() @@ -205,8 +195,8 @@ def test_cannot_write_upstream(tmpdir_factory, test_store, gen_mock_layout): upstream_dbs[0].add(spec, layouts[1]) -@pytest.mark.usefixtures('config') -def test_recursive_upstream_dbs(tmpdir_factory, test_store, gen_mock_layout): +@pytest.mark.usefixtures('config', 'temporary_store') +def test_recursive_upstream_dbs(tmpdir_factory, gen_mock_layout): roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b', 'c']] layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/', '/rc/']] @@ -216,7 +206,7 @@ def test_recursive_upstream_dbs(tmpdir_factory, test_store, gen_mock_layout): y = mock_repo.add_package('y', [z], [default]) mock_repo.add_package('x', [y], [default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('x') spec.concretize() db_c = spack.database.Database(roots[2]) @@ -694,7 +684,7 @@ def test_115_reindex_with_packages_not_in_repo(mutable_database): # Dont add any package definitions to this repository, the idea is that # packages should not have to be defined in the repository once they # are installed - with spack.repo.swap(MockPackageMultiRepo()): + with spack.repo.use_repositories(MockPackageMultiRepo()): spack.store.store.reindex() _check_db_sanity(mutable_database) diff --git a/lib/spack/spack/test/directives.py b/lib/spack/spack/test/directives.py index ae1b6b7e2a..55ec46d01d 100644 --- a/lib/spack/spack/test/directives.py +++ b/lib/spack/spack/test/directives.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 3a144621eb..04e42549d6 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,6 +7,7 @@ This test verifies that the Spack directory layout works properly. """ import os +import os.path import pytest import spack.paths @@ -19,16 +20,6 @@ from spack.spec import Spec max_packages = 10 -@pytest.fixture() -def layout_and_dir(tmpdir): - """Returns a directory layout and the corresponding directory.""" - layout = YamlDirectoryLayout(str(tmpdir)) - old_layout = spack.store.layout - spack.store.layout = layout - yield layout, str(tmpdir) - spack.store.layout = old_layout - - def test_yaml_directory_layout_parameters(tmpdir, config): """This tests the various parameters that can be used to configure the install location """ @@ -84,14 +75,14 @@ def test_yaml_directory_layout_parameters(tmpdir, config): projections=projections_package7) -def test_read_and_write_spec(layout_and_dir, config, mock_packages): +def test_read_and_write_spec(temporary_store, config, mock_packages): """This goes through each package in spack and creates a directory for it. It then ensures that the spec for the directory's installed package can be read back in consistently, and finally that the directory can be removed by the directory layout. """ - layout, tmpdir = layout_and_dir + layout = temporary_store.layout packages = list(spack.repo.path.all_packages())[:max_packages] for pkg in packages: @@ -114,7 +105,7 @@ def test_read_and_write_spec(layout_and_dir, config, mock_packages): # Ensure directory has been created in right place. assert os.path.isdir(install_dir) - assert install_dir.startswith(str(tmpdir)) + assert install_dir.startswith(temporary_store.root) # Ensure spec file exists when directory is created assert os.path.isfile(spec_path) @@ -160,7 +151,7 @@ def test_read_and_write_spec(layout_and_dir, config, mock_packages): assert not os.path.exists(install_dir) -def test_handle_unknown_package(layout_and_dir, config, mock_packages): +def test_handle_unknown_package(temporary_store, config, mock_packages): """This test ensures that spack can at least do *some* operations with packages that are installed but that it does not know about. This is actually not such an uncommon @@ -171,7 +162,7 @@ def test_handle_unknown_package(layout_and_dir, config, mock_packages): information about installed packages' specs to uninstall or query them again if the package goes away. """ - layout, _ = layout_and_dir + layout = temporary_store.layout mock_db = spack.repo.RepoPath(spack.paths.mock_packages_path) not_in_mock = set.difference( @@ -194,7 +185,7 @@ def test_handle_unknown_package(layout_and_dir, config, mock_packages): layout.create_install_directory(spec) installed_specs[spec] = layout.path_for_spec(spec) - with spack.repo.swap(mock_db): + with spack.repo.use_repositories(mock_db): # Now check that even without the package files, we know # enough to read a spec from the spec file. for spec, path in installed_specs.items(): @@ -209,9 +200,9 @@ def test_handle_unknown_package(layout_and_dir, config, mock_packages): assert spec.dag_hash() == spec_from_file.dag_hash() -def test_find(layout_and_dir, config, mock_packages): +def test_find(temporary_store, config, mock_packages): """Test that finding specs within an install layout works.""" - layout, _ = layout_and_dir + layout = temporary_store.layout packages = list(spack.repo.path.all_packages())[:max_packages] # Create install prefixes for all packages in the list diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index d1f5a4b791..014bf340bf 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/fetch_strategy.py b/lib/spack/spack/test/fetch_strategy.py index 5470d88c13..d7f770eac3 100644 --- a/lib/spack/spack/test/fetch_strategy.py +++ b/lib/spack/spack/test/fetch_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/flag_handlers.py b/lib/spack/spack/test/flag_handlers.py index 6c7651ad19..8fac5a898c 100644 --- a/lib/spack/spack/test/flag_handlers.py +++ b/lib/spack/spack/test/flag_handlers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 8dc57da793..ad010e3ca5 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/graph.py b/lib/spack/spack/test/graph.py index 7ad1a75a90..9242cd2717 100644 --- a/lib/spack/spack/test/graph.py +++ b/lib/spack/spack/test/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 2cdeace3e4..d0f7629cf8 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index c2a8d0b100..77810ff0e3 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -183,70 +183,70 @@ def test_flatten_deps( assert os.path.isdir(dependency_dir) -def test_installed_upstream_external( - tmpdir_factory, install_mockery, mock_fetch, gen_mock_layout, - monkeypatch): - """Check that when a dependency package is recorded as installed in - an upstream database that it is not reinstalled. +@pytest.fixture() +def install_upstream(tmpdir_factory, gen_mock_layout, install_mockery): + """Provides a function that installs a specified set of specs to an + upstream database. The function returns a store which points to the + upstream, as well as the upstream layout (for verifying that dependent + installs are using the upstream installs). """ mock_db_root = str(tmpdir_factory.mktemp('mock_db_root')) prepared_db = spack.database.Database(mock_db_root) - upstream_layout = gen_mock_layout('/a/') - dependency = spack.spec.Spec('externaltool') - dependency.concretize() - prepared_db.add(dependency, upstream_layout) - - downstream_db_root = str( - tmpdir_factory.mktemp('mock_downstream_db_root')) - db_for_test = spack.database.Database( - downstream_db_root, upstream_dbs=[prepared_db]) - monkeypatch.setattr(spack.store, 'db', db_for_test) - dependent = spack.spec.Spec('externaltest') - dependent.concretize() - - new_dependency = dependent['externaltool'] - assert new_dependency.external - assert new_dependency.prefix == '/path/to/external_tool' + def _install_upstream(*specs): + for spec_str in specs: + s = spack.spec.Spec(spec_str).concretized() + prepared_db.add(s, upstream_layout) - dependent.package.do_install() + downstream_root = str(tmpdir_factory.mktemp('mock_downstream_db_root')) + db_for_test = spack.database.Database( + downstream_root, upstream_dbs=[prepared_db] + ) + store = spack.store.Store(downstream_root) + store.db = db_for_test + return store, upstream_layout - assert not os.path.exists(new_dependency.prefix) - assert os.path.exists(dependent.prefix) + return _install_upstream -def test_installed_upstream(tmpdir_factory, install_mockery, mock_fetch, - gen_mock_layout, monkeypatch): +def test_installed_upstream_external(install_upstream, mock_fetch): """Check that when a dependency package is recorded as installed in - an upstream database that it is not reinstalled. + an upstream database that it is not reinstalled. """ - mock_db_root = str(tmpdir_factory.mktemp('mock_db_root')) - prepared_db = spack.database.Database(mock_db_root) + s, _ = install_upstream('externaltool') + with spack.store.use_store(s): + dependent = spack.spec.Spec('externaltest') + dependent.concretize() - upstream_layout = gen_mock_layout('/a/') + new_dependency = dependent['externaltool'] + assert new_dependency.external + assert new_dependency.prefix == '/path/to/external_tool' - dependency = spack.spec.Spec('dependency-install') - dependency.concretize() - prepared_db.add(dependency, upstream_layout) - - downstream_db_root = str( - tmpdir_factory.mktemp('mock_downstream_db_root')) - db_for_test = spack.database.Database( - downstream_db_root, upstream_dbs=[prepared_db]) - monkeypatch.setattr(spack.store, 'db', db_for_test) - dependent = spack.spec.Spec('dependent-install') - dependent.concretize() + dependent.package.do_install() + + assert not os.path.exists(new_dependency.prefix) + assert os.path.exists(dependent.prefix) + + +def test_installed_upstream(install_upstream, mock_fetch): + """Check that when a dependency package is recorded as installed in + an upstream database that it is not reinstalled. + """ + s, upstream_layout = install_upstream('dependency-install') + with spack.store.use_store(s): + dependency = spack.spec.Spec('dependency-install').concretized() + dependent = spack.spec.Spec('dependent-install').concretized() - new_dependency = dependent['dependency-install'] - assert new_dependency.package.installed_upstream - assert (new_dependency.prefix == - upstream_layout.path_for_spec(dependency)) + new_dependency = dependent['dependency-install'] + assert new_dependency.package.installed_upstream + assert (new_dependency.prefix == + upstream_layout.path_for_spec(dependency)) - dependent.package.do_install() + dependent.package.do_install() - assert not os.path.exists(new_dependency.prefix) - assert os.path.exists(dependent.prefix) + assert not os.path.exists(new_dependency.prefix) + assert os.path.exists(dependent.prefix) @pytest.mark.disable_clean_stage_check diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 3bf818544e..fb00081931 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -231,7 +231,7 @@ def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): def test_try_install_from_binary_cache(install_mockery, mock_packages, monkeypatch, capsys): """Tests SystemExit path for_try_install_from_binary_cache.""" - def _mirrors_for_spec(spec, force, full_hash_match=False): + def _mirrors_for_spec(spec, full_hash_match=False): spec = spack.spec.Spec('mpi').concretized() return [{ 'mirror_url': 'notused', @@ -474,14 +474,14 @@ def test_packages_needed_to_bootstrap_compiler_packages(install_mockery, assert packages -def test_dump_packages_deps_ok(install_mockery, tmpdir, mock_repo_path): +def test_dump_packages_deps_ok(install_mockery, tmpdir, mock_packages): """Test happy path for dump_packages with dependencies.""" spec_name = 'simple-inheritance' spec = spack.spec.Spec(spec_name).concretized() inst.dump_packages(spec, str(tmpdir)) - repo = mock_repo_path.repos[0] + repo = mock_packages.repos[0] dest_pkg = repo.filename_for_package_name(spec_name) assert os.path.isfile(dest_pkg) diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 79a69942aa..4485bf3ae8 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/__init__.py b/lib/spack/spack/test/llnl/util/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/llnl/util/__init__.py +++ b/lib/spack/spack/test/llnl/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/argparsewriter.py b/lib/spack/spack/test/llnl/util/argparsewriter.py index 127149bbaa..708110558e 100644 --- a/lib/spack/spack/test/llnl/util/argparsewriter.py +++ b/lib/spack/spack/test/llnl/util/argparsewriter.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index e4036fe492..4efec18067 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 501c07a1f6..5733be6b59 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -588,3 +588,24 @@ def test_content_of_files_with_same_name(tmpdir): # and have not been mixed assert file1.read().strip() == 'file1' assert file2.read().strip() == 'file2' + + +def test_keep_modification_time(tmpdir): + file1 = tmpdir.ensure('file1') + file2 = tmpdir.ensure('file2') + + # Shift the modification time of the file 10 seconds back: + mtime1 = file1.mtime() - 10 + file1.setmtime(mtime1) + + with fs.keep_modification_time(file1.strpath, + file2.strpath, + 'non-existing-file'): + file1.write('file1') + file2.remove() + + # Assert that the modifications took place the modification time has not + # changed; + assert file1.read().strip() == 'file1' + assert not file2.exists() + assert int(mtime1) == int(file1.mtime()) diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py index 83342dd6ed..6555a81825 100644 --- a/lib/spack/spack/test/llnl/util/lang.py +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py index bebb3da805..af7012692f 100644 --- a/lib/spack/spack/test/llnl/util/link_tree.py +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index d4b61929c2..ff30cedf6d 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/tty/__init__.py b/lib/spack/spack/test/llnl/util/tty/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/llnl/util/tty/__init__.py +++ b/lib/spack/spack/test/llnl/util/tty/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/tty/log.py b/lib/spack/spack/test/llnl/util/tty/log.py index 811b6c6642..080a6e1339 100644 --- a/lib/spack/spack/test/llnl/util/tty/log.py +++ b/lib/spack/spack/test/llnl/util/tty/log.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,10 +11,8 @@ import signal import sys import time -try: - import termios -except ImportError: - termios = None +from typing import Optional # novm +from types import ModuleType # novm import pytest @@ -25,6 +23,13 @@ from llnl.util.tty.pty import PseudoShell from spack.util.executable import which +termios = None # type: Optional[ModuleType] +try: + import termios as term_mod + termios = term_mod +except ImportError: + pass + @contextlib.contextmanager def nullcontext(): @@ -432,20 +437,14 @@ def test_foreground_background_output( with open(log_path) as log: log = log.read().strip().split("\n") - # Controller and minion process coordinate with locks such that the minion - # writes "off" when echo is off, and "on" when echo is on. The - # output should contain mostly "on" lines, but may contain an "off" - # or two. This is because the controller toggles echo by sending "v" on - # stdin to the minion, but this is not synchronized with our locks. - # It's good enough for a test, though. We allow at most 4 "off"'s in - # the output to account for the race. - # - # Originally we only allowed 2, but GitHub's macOS runners seem to be - # very slow, and frequently we get 3 "off"'s. Increased limit to 4 to - # account for this. Real errors should still be caught with this limit. + # Controller and minion process coordinate with locks such that the + # minion writes "off" when echo is off, and "on" when echo is on. The + # output should contain mostly "on" lines, but may contain "off" + # lines if the controller is slow. The important thing to observe + # here is that we started seeing 'on' in the end. assert ( ['forced output', 'on'] == uniq(output) or - output.count("off") <= 4 # if controller_fd is a bit slow + ['forced output', 'off', 'on'] == uniq(output) ) # log should be off for a while, then on, then off diff --git a/lib/spack/spack/test/llnl/util/tty/tty.py b/lib/spack/spack/test/llnl/util/tty/tty.py index b8366a9738..bc68543ec3 100644 --- a/lib/spack/spack/test/llnl/util/tty/tty.py +++ b/lib/spack/spack/test/llnl/util/tty/tty.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index c35a6e195b..068dce0bef 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index f554fd038c..510487002a 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 8d3ec2d07b..2eef86e4f4 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/module_parsing.py b/lib/spack/spack/test/module_parsing.py index 8dc06b058b..8779deb0ac 100644 --- a/lib/spack/spack/test/module_parsing.py +++ b/lib/spack/spack/test/module_parsing.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/__init__.py b/lib/spack/spack/test/modules/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/modules/__init__.py +++ b/lib/spack/spack/test/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/common.py b/lib/spack/spack/test/modules/common.py index f2cb60e1db..0918cf2dfd 100644 --- a/lib/spack/spack/test/modules/common.py +++ b/lib/spack/spack/test/modules/common.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/conftest.py b/lib/spack/spack/test/modules/conftest.py index 4ab129a5dd..dbfac6b0bc 100644 --- a/lib/spack/spack/test/modules/conftest.py +++ b/lib/spack/spack/test/modules/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 474277f239..7239c487aa 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/tcl.py b/lib/spack/spack/test/modules/tcl.py index 70e3fc6c56..e5f2797e39 100644 --- a/lib/spack/spack/test/modules/tcl.py +++ b/lib/spack/spack/test/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 0e5bd5dad5..beb92bbb6d 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 2bf83c59f2..afe3af7386 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 97def3feda..c2fa52fa3e 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index d9268e7e14..fe73a0ae8a 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/package_class.py b/lib/spack/spack/test/package_class.py index 33e5eb1c0a..686c956186 100644 --- a/lib/spack/spack/test/package_class.py +++ b/lib/spack/spack/test/package_class.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/package_hash.py b/lib/spack/spack/test/package_hash.py index a8a160d96e..bcb1f6b804 100644 --- a/lib/spack/spack/test/package_hash.py +++ b/lib/spack/spack/test/package_hash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index d50169a1a4..308706a5a7 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,7 +16,7 @@ import spack.util.executable as executable import spack.variant # A few functions from this module are used to # do sanity checks only on packagess modified by a PR -import spack.cmd.flake8 as flake8 +import spack.cmd.style as style import spack.util.crypto as crypto import pickle @@ -73,7 +73,7 @@ def test_repo_getpkg_names_and_classes(): def test_get_all_mock_packages(): """Get the mock packages once each too.""" db = spack.repo.RepoPath(spack.paths.mock_packages_path) - with spack.repo.swap(db): + with spack.repo.use_repositories(db): check_repo() @@ -207,7 +207,7 @@ def test_prs_update_old_api(): deprecated calls to any method. """ changed_package_files = [ - x for x in flake8.changed_files() if flake8.is_package(x) + x for x in style.changed_files() if style.is_package(x) ] failing = [] for file in changed_package_files: diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index ffaad396c1..c821d5b5c7 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 31ad3f1a7a..57446b6e2d 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index 2dc781538a..84954874d9 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py index 582ece3d51..94f4e6d5f0 100644 --- a/lib/spack/spack/test/pattern.py +++ b/lib/spack/spack/test/pattern.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/permissions.py b/lib/spack/spack/test/permissions.py index 26974c8096..320ff2fdba 100644 --- a/lib/spack/spack/test/permissions.py +++ b/lib/spack/spack/test/permissions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 0855ac50ec..1f9f0a74e0 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/relocate.py b/lib/spack/spack/test/relocate.py index afeb2162db..fb1ed4eb60 100644 --- a/lib/spack/spack/test/relocate.py +++ b/lib/spack/spack/test/relocate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index d10349e6ec..df3015500d 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -67,3 +67,15 @@ def test_repo_invisibles(mutable_mock_repo, extra_repo): with open(os.path.join(extra_repo.root, 'packages', '.invisible'), 'w'): pass extra_repo.all_package_names() + + +@pytest.mark.parametrize('attr_name,exists', [ + ('cmake', True), + ('__sphinx_mock__', False) +]) +@pytest.mark.regression('20661') +def test_namespace_hasattr(attr_name, exists, mutable_mock_repo): + # Check that we don't fail on 'hasattr' checks because + # of a custom __getattr__ implementation + nms = spack.repo.SpackNamespace('spack.pkg.builtin.mock') + assert hasattr(nms, attr_name) == exists diff --git a/lib/spack/spack/test/s3_fetch.py b/lib/spack/spack/test/s3_fetch.py index 70efad19ce..e339a1c173 100644 --- a/lib/spack/spack/test/s3_fetch.py +++ b/lib/spack/spack/test/s3_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index 9307d2bc4b..422ec3cdc0 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -44,7 +44,7 @@ sbang_line = '#!/bin/sh %s/bin/sbang\n' % spack.store.store.unpadded_root last_line = "last!\n" -@pytest.fixture +@pytest.fixture # type: ignore[no-redef] def sbang_line(): yield '#!/bin/sh %s/bin/sbang\n' % spack.store.layout.root diff --git a/lib/spack/spack/test/schema.py b/lib/spack/spack/test/schema.py index 14c9014d61..1b9694a8e8 100644 --- a/lib/spack/spack/test/schema.py +++ b/lib/spack/spack/test/schema.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spack_yaml.py b/lib/spack/spack/test/spack_yaml.py index fb4abbf1d8..8e4fe3c9f4 100644 --- a/lib/spack/spack/test/spack_yaml.py +++ b/lib/spack/spack/test/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 2ac5bec4d9..efee120ae5 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -75,7 +75,7 @@ w->y deptypes are (link, build), w->x and y->z deptypes are (test) y = mock_repo.add_package('y', [z], [test_only]) w = mock_repo.add_package('w', [x, y], [test_only, default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = Spec('w') spec.concretize(tests=(w.name,)) @@ -114,7 +114,7 @@ def test_installed_deps(): b = mock_repo.add_package('b', [d, e], [default, default]) mock_repo.add_package('a', [b, c], [default, default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): c_spec = Spec('c') c_spec.concretize() assert c_spec['d'].version == spack.version.Version('2') @@ -143,7 +143,7 @@ def test_specify_preinstalled_dep(): b = mock_repo.add_package('b', [c], [default]) mock_repo.add_package('a', [b], [default]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): b_spec = Spec('b') b_spec.concretize() for spec in b_spec.traverse(): @@ -186,7 +186,7 @@ def test_conditional_dep_with_user_constraints(spec_str, expr_str, expected): } mock_repo.add_package('x', [y], [default], conditions=x_on_y_conditions) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): spec = Spec(spec_str) spec.concretize() diff --git a/lib/spack/spack/test/spec_list.py b/lib/spack/spack/test/spec_list.py index 27567b4080..48796d0ad9 100644 --- a/lib/spack/spack/test/spec_list.py +++ b/lib/spack/spack/test/spec_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b2161d4a0e..6aa3068da0 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index b59828206f..a523dffca8 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 98fb1e68fe..74ebf0a541 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,8 +11,7 @@ YAML format preserves DAG information in the spec. import ast import inspect import os - -from collections import Iterable, Mapping +import sys import pytest @@ -29,6 +28,12 @@ from spack.util.spack_yaml import syaml_dict from spack.util.mock_package import MockPackageMultiRepo +if sys.version_info >= (3, 3): + from collections.abc import Iterable, Mapping # novm +else: + from collections import Iterable, Mapping + + def check_yaml_round_trip(spec): yaml_text = spec.to_yaml() spec_from_yaml = Spec.from_yaml(yaml_text) @@ -125,8 +130,13 @@ def test_to_record_dict(mock_packages, config): assert record[key] == value +@pytest.mark.parametrize("hash_type", [ + ht.dag_hash, + ht.build_hash, + ht.full_hash +]) def test_ordered_read_not_required_for_consistent_dag_hash( - config, mock_packages + hash_type, config, mock_packages ): """Make sure ordered serialization isn't required to preserve hashes. @@ -143,15 +153,15 @@ def test_ordered_read_not_required_for_consistent_dag_hash( # # Dict & corresponding YAML & JSON from the original spec. # - spec_dict = spec.to_dict() - spec_yaml = spec.to_yaml() - spec_json = spec.to_json() + spec_dict = spec.to_dict(hash=hash_type) + spec_yaml = spec.to_yaml(hash=hash_type) + spec_json = spec.to_json(hash=hash_type) # # Make a spec with reversed OrderedDicts for every # OrderedDict in the original. # - reversed_spec_dict = reverse_all_dicts(spec.to_dict()) + reversed_spec_dict = reverse_all_dicts(spec.to_dict(hash=hash_type)) # # Dump to YAML and JSON @@ -185,11 +195,13 @@ def test_ordered_read_not_required_for_consistent_dag_hash( reversed_json_string ) - # TODO: remove this when build deps are in provenance. - spec = spec.copy(deps=('link', 'run')) + # Strip spec if we stripped the yaml + spec = spec.copy(deps=hash_type.deptype) + # specs are equal to the original assert spec == round_trip_yaml_spec assert spec == round_trip_json_spec + assert spec == round_trip_reversed_yaml_spec assert spec == round_trip_reversed_json_spec assert round_trip_yaml_spec == round_trip_reversed_yaml_spec @@ -199,16 +211,18 @@ def test_ordered_read_not_required_for_consistent_dag_hash( assert spec.dag_hash() == round_trip_json_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash() - # full_hashes are equal - spec.concretize() - round_trip_yaml_spec.concretize() - round_trip_json_spec.concretize() - round_trip_reversed_yaml_spec.concretize() - round_trip_reversed_json_spec.concretize() - assert spec.full_hash() == round_trip_yaml_spec.full_hash() - assert spec.full_hash() == round_trip_json_spec.full_hash() - assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash() - assert spec.full_hash() == round_trip_reversed_json_spec.full_hash() + + # full_hashes are equal if we round-tripped by build_hash or full_hash + if hash_type in (ht.build_hash, ht.full_hash): + spec.concretize() + round_trip_yaml_spec.concretize() + round_trip_json_spec.concretize() + round_trip_reversed_yaml_spec.concretize() + round_trip_reversed_json_spec.concretize() + assert spec.full_hash() == round_trip_yaml_spec.full_hash() + assert spec.full_hash() == round_trip_json_spec.full_hash() + assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash() + assert spec.full_hash() == round_trip_reversed_json_spec.full_hash() @pytest.mark.parametrize("module", [ @@ -310,7 +324,7 @@ def test_save_dependency_spec_yamls_subset(tmpdir, config): b = mock_repo.add_package('b', [d, e], [default, default]) mock_repo.add_package('a', [b, c], [default, default]) - with repo.swap(mock_repo): + with repo.use_repositories(mock_repo): spec_a = Spec('a') spec_a.concretize() b_spec = spec_a['b'] diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 9dea091730..3e3ceeaaa0 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index f435b07db2..87e9779ca9 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/tengine.py b/lib/spack/spack/test/tengine.py index 01f2d65780..00216a7fdc 100644 --- a/lib/spack/spack/test/tengine.py +++ b/lib/spack/spack/test/tengine.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py index d1780b8963..a05032d5a2 100644 --- a/lib/spack/spack/test/test_activations.py +++ b/lib/spack/spack/test/test_activations.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -54,7 +54,7 @@ def builtin_and_mock_packages(): repo_dirs = [spack.paths.packages_path, spack.paths.mock_packages_path] path = RepoPath(*repo_dirs) - with spack.repo.swap(path): + with spack.repo.use_repositories(path): yield diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py index 1ec5106182..50c5989a3d 100644 --- a/lib/spack/spack/test/test_suite.py +++ b/lib/spack/spack/test/test_suite.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py index 28b6c87385..9804e4eb72 100644 --- a/lib/spack/spack/test/url_fetch.py +++ b/lib/spack/spack/test/url_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 846e24d242..332c988fca 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index 08a3b99a4a..8fbfd48714 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/__init__.py b/lib/spack/spack/test/util/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/test/util/__init__.py +++ b/lib/spack/spack/test/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/editor.py b/lib/spack/spack/test/util/editor.py index e1495a70cc..aa85323212 100644 --- a/lib/spack/spack/test/util/editor.py +++ b/lib/spack/spack/test/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/environment.py b/lib/spack/spack/test/util/environment.py index a555c35c9c..b7b6a0edea 100644 --- a/lib/spack/spack/test/util/environment.py +++ b/lib/spack/spack/test/util/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/executable.py b/lib/spack/spack/test/util/executable.py index ae2859ea4b..9bfa9a408a 100644 --- a/lib/spack/spack/test/util/executable.py +++ b/lib/spack/spack/test/util/executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/file_cache.py b/lib/spack/spack/test/util/file_cache.py index 0fec1a5dc9..c033ca49d9 100644 --- a/lib/spack/spack/test/util/file_cache.py +++ b/lib/spack/spack/test/util/file_cache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/log_parser.py b/lib/spack/spack/test/util/log_parser.py index 0f829082b6..911226f381 100644 --- a/lib/spack/spack/test/util/log_parser.py +++ b/lib/spack/spack/test/util/log_parser.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/mock_package.py b/lib/spack/spack/test/util/mock_package.py index 376ac581bd..2609d849db 100644 --- a/lib/spack/spack/test/util/mock_package.py +++ b/lib/spack/spack/test/util/mock_package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -15,7 +15,7 @@ def test_mock_package_possible_dependencies(): b = mock_repo.add_package('b', [d]) a = mock_repo.add_package('a', [b, c]) - with spack.repo.swap(mock_repo): + with spack.repo.use_repositories(mock_repo): assert set(a.possible_dependencies()) == set(['a', 'b', 'c', 'd', 'e']) assert set(b.possible_dependencies()) == set(['b', 'd', 'e']) assert set(c.possible_dependencies()) == set(['c', 'd', 'e']) diff --git a/lib/spack/spack/test/util/prefix.py b/lib/spack/spack/test/util/prefix.py index cee3ad028b..d7724f616e 100644 --- a/lib/spack/spack/test/util/prefix.py +++ b/lib/spack/spack/test/util/prefix.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/spack_lock_wrapper.py b/lib/spack/spack/test/util/spack_lock_wrapper.py index d7819c84cb..73a2e41eb0 100644 --- a/lib/spack/spack/test/util/spack_lock_wrapper.py +++ b/lib/spack/spack/test/util/spack_lock_wrapper.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/spack_yaml.py b/lib/spack/spack/test/util/spack_yaml.py index 23e02d7be6..530226ad8d 100644 --- a/lib/spack/spack/test/util/spack_yaml.py +++ b/lib/spack/spack/test/util/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_gpg.py b/lib/spack/spack/test/util/util_gpg.py index 83fad508f9..b8adbc537d 100644 --- a/lib/spack/spack/test/util/util_gpg.py +++ b/lib/spack/spack/test/util/util_gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_string.py b/lib/spack/spack/test/util/util_string.py index bcaa7d6211..7c68895be2 100644 --- a/lib/spack/spack/test/util/util_string.py +++ b/lib/spack/spack/test/util/util_string.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_url.py b/lib/spack/spack/test/util/util_url.py index 5d0d9fa4eb..07a055c572 100644 --- a/lib/spack/spack/test/util/util_url.py +++ b/lib/spack/spack/test/util/util_url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/variant.py b/lib/spack/spack/test/variant.py index 10e8ea7e7f..501d897321 100644 --- a/lib/spack/spack/test/variant.py +++ b/lib/spack/spack/test/variant.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/verification.py b/lib/spack/spack/test/verification.py index 009f407186..cbeba8f635 100644 --- a/lib/spack/spack/test/verification.py +++ b/lib/spack/spack/test/verification.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 315788f7f3..ce222c5ea2 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py index e1dcec4c5c..18fd88b093 100644 --- a/lib/spack/spack/test/views.py +++ b/lib/spack/spack/test/views.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 472ab0dab7..4d80507c52 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,10 +6,14 @@ import os import ordereddict_backport import pytest +import spack.config import spack.paths import spack.util.web +import spack.util.s3 from spack.version import ver +import llnl.util.tty as tty + def _create_url(relative_url): web_data_path = os.path.join(spack.paths.test_path, 'data', 'web') @@ -195,3 +199,58 @@ def test_list_url(tmpdir): 'file-0.txt', 'file-1.txt', 'file-2.txt'] + + +class MockPages(object): + def search(self, *args, **kwargs): + return [ + {'Key': 'keyone'}, + {'Key': 'keytwo'}, + {'Key': 'keythree'}, + ] + + +class MockPaginator(object): + def paginate(self, *args, **kwargs): + return MockPages() + + +class MockS3Client(object): + def get_paginator(self, *args, **kwargs): + return MockPaginator() + + def delete_objects(self, *args, **kwargs): + return { + 'Errors': [ + {'Key': 'keyone', 'Message': 'Access Denied'} + ], + 'Deleted': [ + {'Key': 'keytwo'}, + {'Key': 'keythree'} + ], + } + + def delete_object(self, *args, **kwargs): + pass + + +def test_remove_s3_url(monkeypatch, capfd): + fake_s3_url = 's3://my-bucket/subdirectory/mirror' + + def mock_create_s3_session(url): + return MockS3Client() + + monkeypatch.setattr( + spack.util.s3, 'create_s3_session', mock_create_s3_session) + + current_debug_level = tty.debug_level() + tty.set_debug(1) + + spack.util.web.remove_url(fake_s3_url, recursive=True) + err = capfd.readouterr()[1] + + tty.set_debug(current_debug_level) + + assert('Failed to delete keyone (Access Denied)' in err) + assert('Deleted keythree' in err) + assert('Deleted keytwo' in err) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index aa40524ae8..1fc0f2f522 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -56,8 +56,12 @@ def find_list_urls(url): GitLab https://gitlab.\*/<repo>/<name>/tags BitBucket https://bitbucket.org/<repo>/<name>/downloads/?tab=tags CRAN https://\*.r-project.org/src/contrib/Archive/<name> + PyPI https://pypi.org/simple/<name>/ ========= ======================================================= + Note: this function is called by `spack versions`, `spack checksum`, + and `spack create`, but not by `spack fetch` or `spack install`. + Parameters: url (str): The download URL for the package @@ -91,6 +95,16 @@ def find_list_urls(url): # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz (r'(.*\.r-project\.org/src/contrib)/([^_]+)', lambda m: m.group(1) + '/Archive/' + m.group(2)), + + # PyPI + # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl + (r'(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)', + lambda m: 'https://pypi.org/simple/' + m.group(1) + '/'), ] list_urls = set([os.path.dirname(url)]) diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py index f08f21e0ec..f111aaedc3 100644 --- a/lib/spack/spack/user_environment.py +++ b/lib/spack/spack/user_environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 9f87532b85..103eae6134 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index ebbe0519d0..2f190e94e1 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 7e264ff0c7..809b36ffea 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -1,14 +1,14 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys import hashlib +from typing import Dict, Callable, Any # novm import llnl.util.tty as tty - #: Set of hash algorithms that Spack can use, mapped to digest size in bytes hashes = { 'md5': 16, @@ -30,7 +30,7 @@ _deprecated_hash_algorithms = ['md5'] #: cache of hash functions generated -_hash_functions = {} +_hash_functions = {} # type: Dict[str, Callable[[], Any]] class DeprecatedHash(object): diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py index 1ff74f24cf..290ad832f8 100644 --- a/lib/spack/spack/util/debug.py +++ b/lib/spack/spack/util/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/editor.py b/lib/spack/spack/util/editor.py index 00a095252e..9b0081b19a 100644 --- a/lib/spack/spack/util/editor.py +++ b/lib/spack/spack/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index b370053c3c..a9b3529869 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 614bc1725a..d536f91612 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/file_cache.py b/lib/spack/spack/util/file_cache.py index 9342f9772b..670b2ed698 100644 --- a/lib/spack/spack/util/file_cache.py +++ b/lib/spack/spack/util/file_cache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/file_permissions.py b/lib/spack/spack/util/file_permissions.py index b133d2569e..7ed974816e 100644 --- a/lib/spack/spack/util/file_permissions.py +++ b/lib/spack/spack/util/file_permissions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index eaa199e417..aa376bc041 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -115,7 +115,7 @@ if not cached_property: return result -class GpgConstants(object): +class _GpgConstants(object): @cached_property def target_version(self): return spack.version.Version('2') @@ -244,7 +244,7 @@ class GpgConstants(object): pass -GpgConstants = GpgConstants() +GpgConstants = _GpgConstants() def ensure_gpg(reevaluate=False): @@ -382,7 +382,7 @@ def gpg(*args, **kwargs): return get_global_gpg_instance()(*args, **kwargs) -gpg.name = 'gpg' +gpg.name = 'gpg' # type: ignore[attr-defined] @functools.wraps(Gpg.create) diff --git a/lib/spack/spack/util/imp/__init__.py b/lib/spack/spack/util/imp/__init__.py index 27e2b00f9f..975b9a49a7 100644 --- a/lib/spack/spack/util/imp/__init__.py +++ b/lib/spack/spack/util/imp/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/imp/imp_importer.py b/lib/spack/spack/util/imp/imp_importer.py index 810e1d01a3..a9b5cd641a 100644 --- a/lib/spack/spack/util/imp/imp_importer.py +++ b/lib/spack/spack/util/imp/imp_importer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/imp/importlib_importer.py b/lib/spack/spack/util/imp/importlib_importer.py index c230f4c50a..4c8a2673e5 100644 --- a/lib/spack/spack/util/imp/importlib_importer.py +++ b/lib/spack/spack/util/imp/importlib_importer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index b748aa056a..2f465f590a 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -15,7 +15,7 @@ import spack.error import spack.paths -class Lock(llnl.util.lock.Lock): +class Lock(llnl.util.lock.Lock): # type: ignore[no-redef] """Lock that can be disabled. This overrides the ``_lock()`` and ``_unlock()`` methods from diff --git a/lib/spack/spack/util/log_parse.py b/lib/spack/spack/util/log_parse.py index 4b01c18c91..7973bfd6b0 100644 --- a/lib/spack/spack/util/log_parse.py +++ b/lib/spack/spack/util/log_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -43,7 +43,7 @@ def parse_log_events(stream, context=6, jobs=None, profile=False): #: lazily constructed CTest log parser -parse_log_events.ctest_parser = None +parse_log_events.ctest_parser = None # type: ignore[attr-defined] def _wrap(text, width): @@ -107,7 +107,7 @@ def make_log_context(log_events, width=None): for i in range(start, event.end): # wrap to width lines = _wrap(event[i], wrap_width) - lines[1:] = [indent + l for l in lines[1:]] + lines[1:] = [indent + ln for ln in lines[1:]] wrapped_line = line_fmt % (i, '\n'.join(lines)) if i in error_lines: diff --git a/lib/spack/spack/util/mock_package.py b/lib/spack/spack/util/mock_package.py index 5286b50464..0b5e681f26 100644 --- a/lib/spack/spack/util/mock_package.py +++ b/lib/spack/spack/util/mock_package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import ordereddict_backport import spack.util.naming +import spack.provider_index from spack.dependency import Dependency from spack.spec import Spec from spack.version import Version @@ -80,6 +81,8 @@ class MockPackageMultiRepo(object): def __init__(self): self.spec_to_pkg = {} + self.namespace = '' + self.full_namespace = 'spack.pkg.mock' def get(self, spec): if not isinstance(spec, spack.spec.Spec): @@ -171,3 +174,7 @@ class MockPackageMultiRepo(object): self.spec_to_pkg["mockrepo." + name] = mock_package return mock_package + + @property + def provider_index(self): + return spack.provider_index.ProviderIndex() diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index bc994fd4b4..babac6543a 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 1cc6d6109f..db5d1d4907 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index adea1a498b..462f43d223 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 182f93c585..5e717cbb93 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index fcbb4e6baa..fc4b324b17 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -1,11 +1,17 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import inspect -import collections import functools +import sys + + +if sys.version_info >= (3, 3): + from collections.abc import MutableSequence # novm +else: + from collections import MutableSequence class Delegate(object): @@ -52,7 +58,7 @@ def composite(interface=None, method_list=None, container=list): # exception if it doesn't. The patched class returned by the decorator will # inherit from the container class to expose the interface needed to manage # objects composition - if not issubclass(container, collections.MutableSequence): + if not issubclass(container, MutableSequence): raise TypeError("Container must fulfill the MutableSequence contract") # Check if at least one of the 'interface' or the 'method_list' arguments diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index 10ec50d30c..b4542c7eb0 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index c9ed22f72c..a82aca10c1 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py index 01f773058b..62fd9ca824 100644 --- a/lib/spack/spack/util/spack_json.py +++ b/lib/spack/spack/util/spack_json.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -38,10 +38,14 @@ def dump(data, stream=None): def _strify(data, ignore_dicts=False): + """Converts python 2 unicodes to str in JSON data.""" + # this is a no-op in python 3 + if sys.version_info[0] >= 3: + return data + # if this is a unicode string in python 2, return its string representation - if sys.version_info[0] < 3: - if isinstance(data, string_types): - return data.encode('utf-8') + if isinstance(data, string_types): + return data.encode('utf-8') # if this is a list of values, return list of byteified values if isinstance(data, list): diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index 40e66ee62c..e27c48860b 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -13,7 +13,8 @@ """ import ctypes -import collections +import sys +from typing import List # novm from ordereddict_backport import OrderedDict from six import string_types, StringIO @@ -25,6 +26,13 @@ from llnl.util.tty.color import colorize, clen, cextra import spack.error + +if sys.version_info >= (3, 3): + from collections.abc import Mapping # novm +else: + from collections import Mapping + + # Only export load and dump __all__ = ['load', 'dump', 'SpackYAMLError'] @@ -219,7 +227,7 @@ def file_line(mark): #: This is nasty but YAML doesn't give us many ways to pass arguments -- #: yaml.dump() takes a class (not an instance) and instantiates the dumper #: itself, so we can't just pass an instance -_annotations = [] +_annotations = [] # type: List[str] class LineAnnotationDumper(OrderedLineDumper): @@ -343,7 +351,7 @@ def sorted_dict(dict_like): """ result = syaml_dict(sorted(dict_like.items())) for key, value in result.items(): - if isinstance(value, collections.Mapping): + if isinstance(value, Mapping): result[key] = sorted_dict(value) return result diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index 7284cef2d3..6137c32d5a 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/url.py b/lib/spack/spack/util/url.py index ab5503229f..f8abf4fa96 100644 --- a/lib/spack/spack/util/url.py +++ b/lib/spack/spack/util/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 1b415f5de2..1dac12faca 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -20,10 +20,10 @@ import six from six.moves.urllib.error import URLError from six.moves.urllib.request import urlopen, Request -try: +if sys.version_info < (3, 0): # Python 2 had these in the HTMLParser package. from HTMLParser import HTMLParser, HTMLParseError # novm -except ImportError: +else: # In Python 3, things moved to html.parser from html.parser import HTMLParser @@ -229,17 +229,57 @@ def url_exists(url): return False -def remove_url(url): +def _debug_print_delete_results(result): + if 'Deleted' in result: + for d in result['Deleted']: + tty.debug('Deleted {0}'.format(d['Key'])) + if 'Errors' in result: + for e in result['Errors']: + tty.debug('Failed to delete {0} ({1})'.format( + e['Key'], e['Message'])) + + +def remove_url(url, recursive=False): url = url_util.parse(url) local_path = url_util.local_file_path(url) if local_path: - os.remove(local_path) + if recursive: + shutil.rmtree(local_path) + else: + os.remove(local_path) return if url.scheme == 's3': s3 = s3_util.create_s3_session(url) - s3.delete_object(Bucket=url.netloc, Key=url.path) + bucket = url.netloc + if recursive: + # Because list_objects_v2 can only return up to 1000 items + # at a time, we have to paginate to make sure we get it all + prefix = url.path.strip('/') + paginator = s3.get_paginator('list_objects_v2') + pages = paginator.paginate(Bucket=bucket, Prefix=prefix) + + delete_request = {'Objects': []} + for item in pages.search('Contents'): + if not item: + continue + + delete_request['Objects'].append({'Key': item['Key']}) + + # Make sure we do not try to hit S3 with a list of more + # than 1000 items + if len(delete_request['Objects']) >= 1000: + r = s3.delete_objects(Bucket=bucket, Delete=delete_request) + _debug_print_delete_results(r) + delete_request = {'Objects': []} + + # Delete any items that remain + if len(delete_request['Objects']): + r = s3.delete_objects(Bucket=bucket, Delete=delete_request) + _debug_print_delete_results(r) + else: + s3.delete_object(Bucket=bucket, Key=url.path) return # Don't even try for other URL schemes. @@ -549,7 +589,10 @@ def find_versions_of_archive( # .sha256 # .sig # However, SourceForge downloads still need to end in '/download'. - url_regex += r'(\/download)?$' + url_regex += r'(\/download)?' + # PyPI adds #sha256=... to the end of the URL + url_regex += '(#sha256=.*)?' + url_regex += '$' regexes.append(url_regex) diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 8fa52f9738..51918af737 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -12,6 +12,12 @@ import inspect import itertools import re from six import StringIO +import sys + +if sys.version_info >= (3, 5): + from collections.abc import Sequence # novm +else: + from collections import Sequence import llnl.util.tty.color import llnl.util.lang as lang @@ -20,11 +26,6 @@ from spack.util.string import comma_or import spack.directives import spack.error as error -try: - from collections.abc import Sequence # novm -except ImportError: - from collections import Sequence - special_variant_values = [None, 'none', '*'] diff --git a/lib/spack/spack/verify.py b/lib/spack/spack/verify.py index 528fa7903b..b8d76e03e2 100644 --- a/lib/spack/spack/verify.py +++ b/lib/spack/spack/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 44e03edb5c..321a03efc7 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -37,7 +37,10 @@ from spack.util.spack_yaml import syaml_dict __all__ = ['Version', 'VersionRange', 'VersionList', 'ver'] # Valid version characters -VALID_VERSION = r'[A-Za-z0-9_.-]' +VALID_VERSION = re.compile(r'[A-Za-z0-9_.-]') + +# regex for version segments +SEGMENT_REGEX = re.compile(r'[a-zA-Z]+|[0-9]+') # Infinity-like versions. The order in the list implies the comparison rules infinity_versions = ['develop', 'main', 'master', 'head', 'trunk'] @@ -97,9 +100,10 @@ class Version(object): """Class to represent versions""" def __init__(self, string): - string = str(string) + if not isinstance(string, str): + string = str(string) - if not re.match(VALID_VERSION, string): + if not VALID_VERSION.match(string): raise ValueError("Bad characters in version string: %s" % string) # preserve the original string, but trimmed. @@ -107,12 +111,11 @@ class Version(object): self.string = string # Split version into alphabetical and numeric segments - segment_regex = r'[a-zA-Z]+|[0-9]+' - segments = re.findall(segment_regex, string) + segments = SEGMENT_REGEX.findall(string) self.version = tuple(int_if_int(seg) for seg in segments) # Store the separators from the original version string as well. - self.separators = tuple(re.split(segment_regex, string)[1:]) + self.separators = tuple(SEGMENT_REGEX.split(string)[1:]) @property def dotted(self): |